X-Git-Url: https://gerrit.simantics.org/r/gitweb?p=simantics%2Fplatform.git;a=blobdiff_plain;f=bundles%2Forg.simantics.graph.db%2Fsrc%2Forg%2Fsimantics%2Fgraph%2Fdb%2FStreamingTransferableGraphImportProcess.java;h=e61937fc9e90aaf92d2dc57767d35c4545d6b91e;hp=3599eefc0ecc81c9cb6d012bf31415f48bcfc34f;hb=55feb8ea3a74bf5ce73619d62d4c73e576ae89fc;hpb=c26409b1caf2f1e560d37c5befd11b442399c3fe diff --git a/bundles/org.simantics.graph.db/src/org/simantics/graph/db/StreamingTransferableGraphImportProcess.java b/bundles/org.simantics.graph.db/src/org/simantics/graph/db/StreamingTransferableGraphImportProcess.java index 3599eefc0..e61937fc9 100644 --- a/bundles/org.simantics.graph.db/src/org/simantics/graph/db/StreamingTransferableGraphImportProcess.java +++ b/bundles/org.simantics.graph.db/src/org/simantics/graph/db/StreamingTransferableGraphImportProcess.java @@ -72,6 +72,7 @@ public class StreamingTransferableGraphImportProcess implements TransferableGrap TransferableGraphSource tg; VirtualGraph vg; IImportAdvisor2 advisor; + TGStatusMonitor monitor; ClusterBuilder2 builder; final TGResourceUtil resourceUtil = new TGResourceUtil(); @@ -99,9 +100,25 @@ public class StreamingTransferableGraphImportProcess implements TransferableGrap Resource NameOf; public StreamingTransferableGraphImportProcess(Session session, VirtualGraph vg, TransferableGraphSource tg, IImportAdvisor2 advisor) { + this(session, vg, tg, advisor, null); + } + + public StreamingTransferableGraphImportProcess(Session session, VirtualGraph vg, TransferableGraphSource tg, IImportAdvisor2 advisor, TGStatusMonitor monitor) { this.tg = tg; this.vg = vg; this.advisor = advisor; + this.monitor = monitor; + } + + private int updatePercentage(int percentage, int done, int total) { + if (monitor != null && (done & 63) == 0) { + int current = 100*done / total; + if (current > percentage) { + percentage = current; + monitor.status(percentage); + } + } + return percentage; } public void readIdentities(ReadGraph g) throws Exception { @@ -146,8 +163,8 @@ public class StreamingTransferableGraphImportProcess implements TransferableGrap void addMissing(int handleIndex, String external) { allMissingExternals.put(external, handleIndex); Set removals = new HashSet<>(); - for(String ext : missingExternals) if(ext.startsWith(external)) return; - for(String ext : missingExternals) if(external.startsWith(ext)) removals.add(ext); + for(String ext : missingExternals) if(ext.startsWith(external + "/")) return; + for(String ext : missingExternals) if(external.startsWith(ext + "/")) removals.add(ext); missingExternals.removeAll(removals); missingExternals.add(external); } @@ -171,6 +188,21 @@ public class StreamingTransferableGraphImportProcess implements TransferableGrap this.handles = new int[resourceCount]; TIntObjectMap identityMap = TransferableGraphUtils.mapIdentities(identities); + // We must process roots first, because internal identifiers depend on them. + for(Identity identity : identities) { + IdentityDefinition definition = identity.definition; + if (definition instanceof Root) { + Root root = (Root) definition; + if (root.name.equals("")) + handles[identity.resource] = builder.handle(RootLibrary); + else { + Resource existing = advisor.analyzeRoot(graph, root); + if (existing != null) + handles[identity.resource] = builder.handle(existing); + } + } + } + for(Identity identity : identities) { IdentityDefinition definition = identity.definition; if(definition instanceof External) { @@ -193,12 +225,12 @@ public class StreamingTransferableGraphImportProcess implements TransferableGrap TransientCacheAsyncListener.instance()); Resource child = childMap.get(def.name); if(child == null) { - addMissing(identity.resource, graph.getURI(parent) + "/" + def.name); + addMissing(identity.resource, graph.getURI(parent) + "/" + URIStringUtils.escape(def.name)); } else { handles[identity.resource] = builder.handle(child); } } else { - addMissing(identity.resource, TransferableGraphUtils.getURI(resourceCount, identityMap, def.parent) + "/" + def.name); + addMissing(identity.resource, TransferableGraphUtils.getURI(resourceCount, identityMap, def.parent) + "/" + URIStringUtils.escape(def.name)); } } } @@ -210,16 +242,6 @@ public class StreamingTransferableGraphImportProcess implements TransferableGrap existingInternalMap.put(identity.resource, existing); } } - else if(definition instanceof Root) { - Root root = (Root)definition; - if(root.name.equals("")) - handles[identity.resource] = builder.handle(RootLibrary); - else { - Resource existing = advisor.analyzeRoot(graph, root); - if(existing != null) - handles[identity.resource] = builder.handle(existing); - } - } else if(definition instanceof Optional) { External def = (External)definition; Resource parent = builder.resource(handles[def.parent]); @@ -227,22 +249,33 @@ public class StreamingTransferableGraphImportProcess implements TransferableGrap handles[identity.resource] = builder.handle(graph.syncRequest(new UnescapedChildMapOfResource(parent)).get(def.name)); } } - - //if(!missingExternals.isEmpty()) throw new MissingDependencyException(this); - + + if (!missingExternals.isEmpty() && failOnMissingEntities()) + throw new MissingDependencyException(this); + } + + private boolean failOnMissingEntities() { + return "true".equalsIgnoreCase( + System.getProperty( + "org.simantics.tg.import.failOnMissingEntities", + "false") ); } @Override public Resource createChild(WriteOnlyGraph graph, Resource parent, Resource child, String name) throws DatabaseException { + //System.err.println("child " + parent + " - " + child + " = " + name); if(child == null) child = graph.newResource(); Resource nameResource = graph.newResource(); graph.claim(nameResource, InstanceOf, null, String); graph.claimValue(nameResource, name, WriteBindings.STRING); graph.claim(child, HasName, NameOf, nameResource); + graph.claim(parent, ConsistsOf, PartOf, child); return child; } int[] getClustering() { + if (vg != null) + return null; Variant v = extensions.get(Extensions.CLUSTERING); if(v == null) return null; try { @@ -254,6 +287,8 @@ public class StreamingTransferableGraphImportProcess implements TransferableGrap } int[] getClusterSets() { + if (vg != null) + return null; Variant v = extensions.get(Extensions.CLUSTER_SETS); if(v == null) return null; try { @@ -271,6 +306,14 @@ public class StreamingTransferableGraphImportProcess implements TransferableGrap void findClusterSet(WriteOnlyGraph graph, Resource rootLibrary, int[] clustering, int[] clusterSets, long[] clusters, int id) throws DatabaseException { ClusteringSupport support = graph.getService(ClusteringSupport.class); if(id == Extensions.ROOT_LIBRARY_CLUSTER_SET || id == Extensions.INDEX_ROOT_CLUSTER_SET) return; + Resource indexRootClusterSetResource = rootLibrary; + if(indexRoot != null && support.isClusterSet(indexRoot)) { + indexRootClusterSetResource = indexRoot; + } else { + graph.setClusterSet4NewResource(rootLibrary); + graph.flushCluster(); + } + int indexRootCsHandle = builder.handle(indexRootClusterSetResource); for(int pos=0,index=0;index() { @Override @@ -480,13 +531,17 @@ public class StreamingTransferableGraphImportProcess implements TransferableGrap int inverse = handles[inv]; builder.addStatement(graph, object, inverse, subject); } - + + // Count from 0% -> 50% => total = statementCount*2 + percentage[0] = updatePercentage(percentage[0], done[0]++, statementCount*2); + } }); - tg.getValueCount(); - + int valueCount = tg.getValueCount(); + done[0] = 0; + class ValueProcedure extends InputStream implements TransferableGraphSourceValueProcedure { private TGResourceUtil util = new TGResourceUtil(); @@ -526,7 +581,8 @@ public class StreamingTransferableGraphImportProcess implements TransferableGrap s.skip(this); } builder.endValue(); - + work(); + } @Override @@ -546,14 +602,23 @@ public class StreamingTransferableGraphImportProcess implements TransferableGrap for (int i = 0; i < length; ++i) builder.appendValue(input.readUnsignedByte()); builder.endValue(); + work(); } + private void work() { + // Count from 50% -> 100% => [valueCount, valueCount*2) + percentage[0] = updatePercentage(percentage[0], valueCount + done[0]++, valueCount*2); + } }; tg.forValues2(null, new ValueProcedure()); for(Resource r : existingInternalMap.valueCollection()) { - graph.deny(r, InstanceOf, null, ExternalEntity, null); + try { + graph.deny(r, InstanceOf, null, ExternalEntity, null); + } catch (DatabaseException e) { + graph.deny(r, InstanceOf, null, ExternalEntity, vg); + } } }