package org.simantics.db.layer0.util; import java.io.Closeable; import java.io.DataInput; import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.TreeMap; import org.simantics.databoard.Bindings; import org.simantics.databoard.Databoard; import org.simantics.databoard.binding.Binding; import org.simantics.databoard.binding.mutable.Variant; import org.simantics.databoard.container.DataContainer; import org.simantics.databoard.serialization.Serializer; import org.simantics.databoard.type.Datatype; import org.simantics.databoard.util.binary.ByteBufferReadable; import org.simantics.db.ReadGraph; import org.simantics.db.Resource; import org.simantics.db.common.utils.NameUtils; import org.simantics.db.exception.DatabaseException; import org.simantics.db.exception.RuntimeDatabaseException; import org.simantics.db.exception.ValidationException; import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus; import org.simantics.db.layer0.util.ConsistsOfProcess.ConsistsOfProcessEntry; import org.simantics.db.layer0.util.TransferableGraphConfiguration2.SeedSpec; import org.simantics.db.layer0.util.TransferableGraphConfiguration2.SeedSpec.SeedSpecType; import org.simantics.db.service.SerialisationSupport; import org.simantics.graph.db.TransferableGraphSource; import org.simantics.graph.representation.External; import org.simantics.graph.representation.Identity; import org.simantics.graph.representation.Internal; import org.simantics.graph.representation.Root; import org.simantics.graph.representation.Value; import org.simantics.layer0.Layer0; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import gnu.trove.list.array.TIntArrayList; import gnu.trove.map.TIntObjectMap; import gnu.trove.map.hash.TIntObjectHashMap; import gnu.trove.procedure.TIntIntProcedure; public class ModelTransferableGraphSource implements TransferableGraphSource { private static final Logger LOGGER = LoggerFactory .getLogger(ModelTransferableGraphSource.class); final private TransferableGraphConfiguration2 configuration; final private DomainProcessorState state; final private int externalBase; final private int resourceCount; final private File[] files; final private TGValueModifier valueModifier; private volatile boolean closed = false; TIntArrayList externalParents = new TIntArrayList(); ArrayList externalNames = new ArrayList<>(); TreeMap downloads = new TreeMap(); public ModelTransferableGraphSource(final ReadGraph graph, TransferableGraphConfiguration2 configuration, final DomainProcessorState state, File... fs) throws DatabaseException { this.configuration = configuration; this.state = state; this.files = fs; this.valueModifier = state.valueModifier; SerialisationSupport ss = graph.getService(SerialisationSupport.class); // At this point ids contains all internal resources. Now add roots and // externals. // Root Library state.ids.put(ss.getTransientId(graph.getRootLibrary()), state.id++); // External roots - internal roots were already processed as internal // resources by domain processor for (SeedSpec spec : configuration.seeds) { if (SeedSpecType.SPECIAL_ROOT.equals(spec.specType)) { int resourceId = ss.getTransientId(spec.resource); state.ids.put(resourceId, state.id++); // The fixed roots have been seen as externals by domain // processor. Now remove them from external set. state.externals.remove(resourceId); } } this.externalBase = state.id; final Collection errors = new HashSet<>(); // All resource considered as not internal by domain processor. Can also // contain roots. int[] externals = state.externals.toArray(); // Build up the state.externals, externalNames and externalParents for (int i = 0; i < externals.length; i++) { getId(graph, externals[i], errors); } state.inverses.forEachEntry(new TIntIntProcedure() { @Override public boolean execute(int predicate, int inverse) { try { getId(graph, predicate, errors); if (inverse != 0) getId(graph, inverse, errors); } catch (DatabaseException e) { throw new RuntimeDatabaseException(e); } return true; } }); if (!errors.isEmpty()) { ArrayList sorted = new ArrayList<>(errors); Collections.sort(sorted); StringBuilder message = new StringBuilder(); message.append("Errors in exported model:\n"); for (String error : sorted) { message.append(error); message.append("\n"); } throw new DatabaseException(message.toString()); } this.resourceCount = state.id; state.extensions.put(ExternalDownloadBean.EXTENSION_KEY, new Variant(ExternalDownloadBean.BINDING, new ExternalDownloadBean(downloads))); } int indent = 0; public boolean validateExternal(Resource ext) { if (configuration.validate) { ExtentStatus status = configuration.preStatus.get(ext); if (status != null) { if (ExtentStatus.INTERNAL.equals(status)) return false; else if (ExtentStatus.EXCLUDED.equals(status)) return false; } } return true; } private Resource getResource(ReadGraph graph, int r) throws DatabaseException { SerialisationSupport ss = graph.getService(SerialisationSupport.class); return ss.getResource(r); } final public int getExistingId(ReadGraph graph, int r) throws DatabaseException { int ret = state.ids.get(r); if (ret != -1) { return ret; } else { SerialisationSupport ss = graph.getService(SerialisationSupport.class); throw new DatabaseException( "Id has not been created for " + NameUtils.getSafeName(graph, ss.getResource(r))); } } /* * * @return -2 if r is not really external and the statement should be * excluded * */ public int getId(ReadGraph graph, int r, Collection errors) throws DatabaseException { // // First external is root library // if(r == rootId) return internalCount; SerialisationSupport ss = graph.getService(SerialisationSupport.class); Layer0 L0 = Layer0.getInstance(graph); if (state.ids.containsKey(r)) { int ret = state.ids.get(r); if (ret == -1) { for (int i = 0; i <= indent; ++i) System.out.print(" "); System.out.println("Cycle!!!"); // with " + // GraphUtils.getReadableName(g, // r)); } return ret; } else { Resource res = getResource(graph, r); if (!validateExternal(res)) { errors.add("Illegal reference to " + graph.getPossibleURI(getResource(graph, r))); return -2; } Collection parents = graph.getObjects(res, L0.PartOf); if (parents.size() != 1) { throw new ValidationException( "Reference to external resource " + NameUtils.getSafeName(graph, getResource(graph, r), true) + " without unique uri (" + parents.size() + " parents)."); } int pid = 0; for (Resource p : parents) { ++indent; pid = getId(graph, ss.getTransientId(p), errors); if (pid == -2) { errors.add("Illegal reference to " + graph.getPossibleURI(getResource(graph, r))); return -2; } } --indent; String name = graph.getRelatedValue(res, L0.HasName); // Record the external entry externalParents.add(pid); externalNames.add(name); state.ids.put(r, state.id); // Ensure that this resource is included into the set of externals // to maintain the total number of externals state.externals.add(r); String download = graph.getPossibleRelatedValue(res, L0.Ontology_download, Bindings.STRING); if (download != null) { String uri = graph.getURI(res); downloads.put(uri, download); } return state.id++; } } @Override public DataContainer getHeader() throws Exception { return null; } @Override public int getResourceCount() { return resourceCount; } private int countRootSeeds() { int result = 0; for (SeedSpec spec : configuration.seeds) { if (SeedSpecType.INTERNAL.equals(spec.specType)) continue; result++; } return result; } @Override public int getIdentityCount() { return countRootSeeds() + state.externals.size() + state.internalEntries.size() + 1; } @Override public int getStatementCount() { return state.statementCount; } @Override public int getValueCount() { return state.valueCount; } @Override public void forStatements(ReadGraph graph, TransferableGraphSourceProcedure procedure) throws Exception { int[] value = new int[4]; long length = state.otherStatementsInput.length(); state.otherStatementsInput.position(0); while (state.otherStatementsInput.position() < length && !state.monitor.isCanceled()) { int s = state.otherStatementsInput.readInt(); int subjectId = state.ids.get(s); boolean exclude = subjectId == -1; int size = state.otherStatementsInput.readInt(); for (int i = 0; i < size; i++) { int p = state.otherStatementsInput.readInt(); int o = state.otherStatementsInput.readInt(); if (!exclude) { if (state.pending.contains(o)) { System.err.println("excluding garbage statement " + s + " " + p + " " + o + ", object resource is garbage"); } else if (state.excludedShared.contains(o)) { System.err.println("excluding shared " + s + " " + p + " " + o); } else { int objectId = getExistingId(graph, o); // The statement can be denied still if (objectId != -2) { value[0] = subjectId; value[1] = getExistingId(graph, p); int inverse = state.inverses.get(p); if (inverse != 0) { value[2] = getExistingId(graph, inverse); } else { value[2] = -1; } value[3] = objectId; try { procedure.execute(value); } catch (Exception e) { LOGGER.error("Error while processing a statement in transferable graph source", e); } } else { System.err.println("Denied (" + NameUtils.getSafeName(graph, getResource(graph, s)) + ", " + NameUtils.getSafeName(graph, getResource(graph, p)) + "," + NameUtils.getSafeName(graph, getResource(graph, o)) + ")"); } } } else { System.err.println("excluding shared " + s); } } } } @Override public void forValues(ReadGraph graph, TransferableGraphSourceProcedure procedure) throws Exception { Serializer variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT); List idContext = new ArrayList<>(); long length = state.valueInput.length(); state.valueInput.position(0); while (state.valueInput.position() < length && !state.monitor.isCanceled()) { // Ignore value type tag int s = state.valueInput.readInt(); byte valueType = state.valueInput.readByte(); switch (valueType) { case TAG_RAW_COPY_VARIANT_VALUE: state.valueInput.readInt(); // Intentional fallthrough. case TAG_POTENTIALLY_MODIFIED_VARIANT_VALUE: { idContext.clear(); Variant variant = (Variant) variantSerializer.deserialize((DataInput) state.valueInput, idContext); if (valueModifier.mayNeedModification(variant.type())) { Object currentObject = variant.getValue(); Object newObject = valueModifier.modify(state, variant.getBinding(), currentObject); if (newObject != currentObject) variant = new Variant(variant.getBinding(), newObject); } try { procedure.execute(new Value(state.ids.get(s), variant)); } catch (Exception e) { LOGGER.error("Error while processing a value in transferable graph source", e); } break; } default: throw new IllegalArgumentException("Unrecognized variant value type encountered: " + valueType); } } } @Override public void forValues2(ReadGraph graph, TransferableGraphSourceValueProcedure procedure) throws Exception { Serializer datatypeSerializer = graph.getService(Databoard.class) .getSerializerUnchecked(Bindings.getBindingUnchecked(Datatype.class)); List idContext = new ArrayList<>(); long length = state.valueInput.length(); state.valueInput.position(0); while (state.valueInput.position() < length && !state.monitor.isCanceled()) { int s = state.valueInput.readInt(); byte valueType = state.valueInput.readByte(); switch (valueType) { case TAG_RAW_COPY_VARIANT_VALUE: { // Variant data could be copied raw, no need for // modifications // Note that the variant data is a concatenation of the // variant datatype serialization and the value // serialization. // variantLength contains both datatype and value data. int variantLength = state.valueInput.readInt(); try { procedure.rawCopy(state.ids.get(s), variantLength, state.valueInput); } catch (Exception e) { LOGGER.error("Error while processing a raw value in transferable graph source", e); } break; } case TAG_POTENTIALLY_MODIFIED_VARIANT_VALUE: { // Variant data may need to be modified. // Cannot optimize this case with raw copying. idContext.clear(); Datatype type = (Datatype) datatypeSerializer.deserialize((DataInput) state.valueInput, idContext); if (valueModifier.mayNeedModification(type)) { Binding binding = Bindings.getBinding(type); Serializer serializer = Bindings.getSerializerUnchecked(binding); Object value = serializer.deserialize((DataInput) state.valueInput); value = valueModifier.modify(state, binding, value); byte[] bytes = serializer.serialize(value); try { procedure.execute(state.ids.get(s), type, new ByteBufferReadable(bytes)); } catch (Exception e) { LOGGER.error("Error while processing a data type in transferable graph source", e); } } else { try { procedure.execute(state.ids.get(s), type, state.valueInput); } catch (Exception e) { LOGGER.error("Error while processing a raw value in transferable graph source", e); } } break; } default: throw new IllegalArgumentException("Unrecognized variant value type encountered: " + valueType); } } } protected Identity getRootIdentity(DomainProcessorState state, SerialisationSupport support, Resource rootLibrary) throws DatabaseException { return new Identity(state.ids.get(support.getTransientId(rootLibrary)), new External(-1, "")); } @Override public void forIdentities(ReadGraph graph, TransferableGraphSourceProcedure procedure) throws Exception { SerialisationSupport support = graph.getService(SerialisationSupport.class); Layer0 L0 = Layer0.getInstance(graph); // TODO: this should be Root with name "" try { procedure.execute(getRootIdentity(state, support, graph.getRootLibrary())); } catch (Exception e) { LOGGER.error("Error while processing a root identity in transferable graph source", e); } TIntObjectMap internalMap = new TIntObjectHashMap<>(100, 0.5f, Integer.MIN_VALUE); // Declare internal and external roots for (SeedSpec r : configuration.seeds) { if (SeedSpecType.INTERNAL.equals(r.specType)) continue; String typeId = r.type; if (typeId == null) { Resource type = graph.getPossibleType(r.resource, L0.Entity); typeId = type != null ? graph.getURI(type) : Layer0.URIs.Entity; } int id = state.ids.get(support.getTransientId(r.resource)); Root root = new Root(r.name, typeId); Identity rootId = new Identity(id, root); internalMap.put(id, rootId); try { procedure.execute(rootId); } catch (Exception e) { LOGGER.error("Error while processing a root in transferable graph source", e); } } for (int i = 0; i < state.externals.size(); i++) { int parent = externalParents.get(i); String name = externalNames.get(i); try { procedure.execute(new Identity(externalBase + i, new External(parent, name))); } catch (Exception e) { LOGGER.error("Error while processing a an external identity in transferable graph source", e); } } if (state.internalEntries != null) { for (ConsistsOfProcessEntry ie : state.internalEntries) { if (ie.parent != null) { if (ie.name != null) { try { procedure.execute(resolveInternal(graph, support, ie, internalMap)); } catch (Exception e) { LOGGER.error("Error while processing an internal identity in transferable graph source", e); } } else { // In this case there is a child that has no HasName => // this should be treated as a blank } } else { try { procedure.execute(resolveInternal(graph, support, ie, internalMap)); } catch (Exception e) { LOGGER.error("Error while processing an internal identity in transferable graph source", e); } } } } } private Identity resolveInternal(ReadGraph graph, SerialisationSupport ss, ConsistsOfProcessEntry entry, TIntObjectMap internalMap) throws DatabaseException { int id = state.ids.get(ss.getTransientId(entry.resource)); Identity existing = internalMap.get(id); if (existing != null) return existing; if (entry.parent == null) { Layer0 L0 = Layer0.getInstance(graph); Resource possibleParent = graph.getPossibleObject(entry.resource, L0.PartOf); if (possibleParent == null) throw new DatabaseException("Invalid root or internal parent path: " + entry.resource); int externalId = state.ids.get(ss.getTransientId(possibleParent)); Identity result = new Identity(id, new Internal(externalId, entry.name)); internalMap.put(id, result); return result; } else { Identity parent = resolveInternal(graph, ss, entry.parent, internalMap); Identity result = new Identity(id, new Internal(parent.resource, entry.name)); internalMap.put(id, result); return result; } } @Override public TreeMap getExtensions() { return state.extensions; } public File[] getFiles() { return files; } private static T tryClose(T c) throws IOException { if (c != null && c instanceof Closeable) ((Closeable) c).close(); return null; } public void closeStreams() throws IOException { state.valueInput = tryClose(state.valueInput); state.otherStatementsInput = tryClose(state.otherStatementsInput); state.statementsOutput = tryClose(state.statementsOutput); state.valueOutput = tryClose(state.valueOutput); } @Override public void reset() throws Exception { throw new UnsupportedOperationException(); } public long[] getResourceArray(ReadGraph graph) throws DatabaseException { final SerialisationSupport ss = graph.getService(SerialisationSupport.class); final long[] result = new long[state.ids.size()]; state.ids.forEachEntry(new TIntIntProcedure() { @Override public boolean execute(int a, int b) { try { Resource r = ss.getResource(a); result[b] = r.getResourceId(); } catch (DatabaseException e) { e.printStackTrace(); } return true; } }); return result; } public DomainProcessorState getState() { return state; } public void forResourceStatements(ReadGraph graph, TransferableGraphSourceProcedure procedure) throws Exception { int[] value = new int[4]; long length = state.otherStatementsInput.length(); state.otherStatementsInput.position(0); while (state.otherStatementsInput.position() < length) { int s = state.otherStatementsInput.readInt(); int subjectId = state.ids.get(s); boolean exclude = subjectId == -1; int size = state.otherStatementsInput.readInt(); for (int i = 0; i < size; i++) { int p = state.otherStatementsInput.readInt(); int o = state.otherStatementsInput.readInt(); if (!exclude) { if (state.excludedShared.contains(o)) { System.err.println("excluding shared " + s + " " + p + " " + o); } else { int objectId = getExistingId(graph, o); // The statement can be denied still if (objectId != -2) { value[0] = s; value[1] = p; int inverse = state.inverses.get(p); if (inverse != 0) { value[2] = inverse; } else { value[2] = -1; } value[3] = o; try { procedure.execute(value); } catch (Exception e) { LOGGER.error("Error while processing a statement in transferable graph source", e); } } else { System.err.println("Denied (" + NameUtils.getSafeName(graph, getResource(graph, s)) + ", " + NameUtils.getSafeName(graph, getResource(graph, p)) + "," + NameUtils.getSafeName(graph, getResource(graph, o)) + ")"); } } } else { System.err.println("excluding shared " + s); } } } } public void forValueResources(ReadGraph graph, TransferableGraphSourceProcedure procedure) throws Exception { int[] value = { 0 }; long length = state.valueInput.length(); while (state.valueInput.position() < length) { value[0] = state.valueInput.readInt(); try { procedure.execute(value); } catch (Exception e) { LOGGER.error("Error while processing a value in transferable graph source", e); } } } public TransferableGraphConfiguration2 getConfiguration() { return configuration; } @Override public void close() throws IOException { synchronized (this) { if (closed) return; closed = true; } closeStreams(); if (files != null) { for (File f : files) { Files.deleteIfExists(f.toPath()); } } } }