From: Antti Villberg Date: Mon, 29 Jan 2018 07:27:55 +0000 (+0200) Subject: Clean up and support internal seed resources in tg export X-Git-Tag: v1.43.0~136^2~617 X-Git-Url: https://gerrit.simantics.org/r/gitweb?a=commitdiff_plain;h=b9b77f4296158fcb87261ff359842099b5de1d6a;p=simantics%2Fplatform.git Clean up and support internal seed resources in tg export refs #7728 Change-Id: I95abd44fe4dea1ec639c4b8a328bdbfb0de3d4ae --- diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/adapter/impl/DefaultPasteHandler.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/adapter/impl/DefaultPasteHandler.java index 6b30e4ddb..ccef12ebf 100644 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/adapter/impl/DefaultPasteHandler.java +++ b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/adapter/impl/DefaultPasteHandler.java @@ -31,7 +31,8 @@ import org.simantics.db.layer0.util.RemoverUtil; import org.simantics.db.layer0.util.SimanticsClipboard; import org.simantics.db.layer0.util.SimanticsClipboard.Representation; import org.simantics.db.layer0.util.SimanticsKeys; -import org.simantics.db.layer0.util.TransferableGraphConfiguration2.RootSpec; +import org.simantics.db.layer0.util.TransferableGraphConfiguration2.SeedSpec; +import org.simantics.db.layer0.util.TransferableGraphConfiguration2.SeedSpec.SeedSpecType; import org.simantics.graph.db.IImportAdvisor; import org.simantics.graph.db.IImportAdvisor2; import org.simantics.graph.db.IImportAdvisor2.RootInfo; @@ -119,8 +120,8 @@ public class DefaultPasteHandler extends PasteHandlerAdapter { ModelTransferableGraphSource mtgs = (ModelTransferableGraphSource)tgs; - loop: for(RootSpec spec : mtgs.getConfiguration().roots) { - if(!spec.internal) continue; + loop: for(SeedSpec spec : mtgs.getConfiguration().seeds) { + if(SeedSpecType.SPECIAL_ROOT.equals(spec.specType)) continue; for(RootInfo info : roots) { if(spec.name.equals(info.root.name)) { result.add(info.resource); diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/adapter/impl/SharedOntologyCopyHandler.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/adapter/impl/SharedOntologyCopyHandler.java index 16ec6e7cb..3c7dfb40a 100644 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/adapter/impl/SharedOntologyCopyHandler.java +++ b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/adapter/impl/SharedOntologyCopyHandler.java @@ -19,7 +19,8 @@ import org.simantics.db.Resource; import org.simantics.db.exception.DatabaseException; import org.simantics.db.layer0.util.TGConfigurer; import org.simantics.db.layer0.util.TransferableGraphConfiguration2; -import org.simantics.db.layer0.util.TransferableGraphConfiguration2.RootSpec; +import org.simantics.db.layer0.util.TransferableGraphConfiguration2.SeedSpec; +import org.simantics.db.layer0.util.TransferableGraphConfiguration2.SeedSpec.SeedSpecType; public class SharedOntologyCopyHandler extends DefaultCopyHandler { @@ -38,7 +39,7 @@ public class SharedOntologyCopyHandler extends DefaultCopyHandler { Resource root = resources.iterator().next(); String uri = graph.getURI(root); - Collection roots = Collections.singletonList(new RootSpec(root, uri, true)); + Collection roots = Collections.singletonList(new SeedSpec(root, uri, SeedSpecType.ROOT)); return new TGConfigurer(graph, true, false).roots(roots).create(); } diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/ConsistsOfProcess.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/ConsistsOfProcess.java index a2e64868f..4124add42 100644 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/ConsistsOfProcess.java +++ b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/ConsistsOfProcess.java @@ -14,6 +14,8 @@ import org.simantics.db.common.request.ReadRequest; import org.simantics.db.common.utils.Logger; import org.simantics.db.exception.DatabaseException; import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus; +import org.simantics.db.layer0.util.TransferableGraphConfiguration2.SeedSpec; +import org.simantics.db.layer0.util.TransferableGraphConfiguration2.SeedSpec.SeedSpecType; import org.simantics.db.procedure.AsyncContextMultiProcedure; import org.simantics.db.procedure.Procedure; import org.simantics.db.service.DirectQuerySupport; @@ -22,39 +24,53 @@ import org.simantics.utils.datastructures.Pair; class ConsistsOfProcess { - final List result; + final List result; final Set childrenWithNoName; - final AsyncContextMultiProcedure structure; - final AsyncContextMultiProcedure names; + final AsyncContextMultiProcedure structure; + final AsyncContextMultiProcedure names; - public static Pair,Set> walk(ReadGraph graph, ResourceMap status, Collection resources, Set exclusions, boolean ignoreVirtual) throws DatabaseException { - ConsistsOfProcess process = new ConsistsOfProcess(graph, status, resources, exclusions, ignoreVirtual); + public static Pair,Set> walk(ReadGraph graph, Collection specs, boolean ignoreVirtual) throws DatabaseException { + return walk(graph, null, specs, ignoreVirtual); + } + + public static Pair,Set> walk(ReadGraph graph, ResourceMap status, Collection specs, boolean ignoreVirtual) throws DatabaseException { + + Collection entries = new ArrayList<>(); + for(SeedSpec r : specs) { + if(SeedSpecType.INTERNAL.equals(r.specType)) + entries.add(new ConsistsOfProcessEntry(null, r.resource, true)); + else + entries.add(new ConsistsOfProcessEntry(null, r.resource, false)); + } + ConsistsOfProcess process = new ConsistsOfProcess(graph, status, entries, ignoreVirtual); return Pair.make(process.result, process.childrenWithNoName); + } - static class InternalEntry { - public InternalEntry parent; + static class ConsistsOfProcessEntry { + public ConsistsOfProcessEntry parent; public Resource resource; - public String name; + public boolean internal; public boolean valid = true; - InternalEntry(InternalEntry parent, Resource resource, String name) { + public String name = null; + ConsistsOfProcessEntry(ConsistsOfProcessEntry parent, Resource resource, boolean internal) { this.parent = parent; this.resource = resource; - this.name = name; + this.internal = internal; } } - private ConsistsOfProcess(ReadGraph graph, ResourceMap status, final Collection resources, final Set exclusions, final boolean ignoreVirtual) throws DatabaseException { + private ConsistsOfProcess(ReadGraph graph, ResourceMap status, final Collection entries, final boolean ignoreVirtual) throws DatabaseException { final Layer0 L0 = Layer0.getInstance(graph); final DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class); - result = new ArrayList(); + result = new ArrayList<>(); childrenWithNoName = new HashSet<>(); - names = dqs.compileForEachObject(graph, L0.HasName, new AsyncContextMultiProcedure() { + names = dqs.compileForEachObject(graph, L0.HasName, new AsyncContextMultiProcedure() { @Override - public void execute(AsyncReadGraph graph, InternalEntry entry, Resource nameResource) { + public void execute(AsyncReadGraph graph, ConsistsOfProcessEntry entry, Resource nameResource) { if(status != null) status.put(nameResource, ExtentStatus.EXCLUDED); @@ -88,7 +104,7 @@ class ConsistsOfProcess { } @Override - public void finished(AsyncReadGraph graph, InternalEntry entry) { + public void finished(AsyncReadGraph graph, ConsistsOfProcessEntry entry) { if(entry.valid) { if(entry.name != null) { result.add(entry); @@ -103,15 +119,16 @@ class ConsistsOfProcess { } }); - structure = dqs.compileForEachObject(graph, L0.ConsistsOf, new AsyncContextMultiProcedure() { + structure = dqs.compileForEachObject(graph, L0.ConsistsOf, new AsyncContextMultiProcedure() { @Override - public void execute(AsyncReadGraph graph, InternalEntry parent, Resource child) { - - if(exclusions.contains(child)) return; + public void execute(AsyncReadGraph graph, ConsistsOfProcessEntry parent, Resource child) { + + if(status != null) + if(ExtentStatus.EXCLUDED.equals(status.get(child))) return; if(!ignoreVirtual || child.isPersistent()) { - InternalEntry entry = new InternalEntry(parent, child, null); + ConsistsOfProcessEntry entry = new ConsistsOfProcessEntry(parent, child, false); dqs.forEachObjectCompiled(graph, child, entry, structure); dqs.forEachObjectCompiled(graph, child, entry, names); } @@ -119,7 +136,7 @@ class ConsistsOfProcess { } @Override - public void finished(AsyncReadGraph graph, InternalEntry parent) { + public void finished(AsyncReadGraph graph, ConsistsOfProcessEntry parent) { } @Override @@ -133,9 +150,12 @@ class ConsistsOfProcess { @Override public void run(ReadGraph graph) throws DatabaseException { - for(Resource r : resources) { - InternalEntry root = new InternalEntry(null, r, null); - dqs.forEachObjectCompiled(graph, r, root, structure); + for(ConsistsOfProcessEntry entry : entries) { + dqs.forEachObjectCompiled(graph, entry.resource, entry, structure); + if(entry.internal) { + // For roots names are not processed + dqs.forEachObjectCompiled(graph, entry.resource, entry, names); + } } } diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/DomainProcessor3.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/DomainProcessor3.java index 20856d4ce..c7bc380e5 100644 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/DomainProcessor3.java +++ b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/DomainProcessor3.java @@ -28,7 +28,7 @@ import org.simantics.db.common.utils.NameUtils; import org.simantics.db.exception.CancelTransactionException; import org.simantics.db.exception.DatabaseException; import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus; -import org.simantics.db.layer0.util.ConsistsOfProcess.InternalEntry; +import org.simantics.db.layer0.util.ConsistsOfProcess.ConsistsOfProcessEntry; import org.simantics.db.layer0.util.ModelTransferableGraphSourceRequest.Expansion3; import org.simantics.db.service.CollectionSupport; import org.simantics.db.service.SerialisationSupport; @@ -57,7 +57,6 @@ public class DomainProcessor3 { int id = 0; Set fringe = null; - Set exclusions = null; Function1 exclusionFunction = null; Set predicates = null; Map isRelatedToPredicates = null; @@ -108,7 +107,6 @@ public class DomainProcessor3 { ids = state.ids; status = cs.createMap(ExtentStatus.class); predicates = cs.createSet(); - exclusions = cs.createSet(); exclusionFunction = conf.exclusionFunction; fringe = new TreeSet(); isRelatedToPredicates = cs.createMap(Boolean.class); @@ -117,17 +115,10 @@ public class DomainProcessor3 { for(Map.Entry entry : conf.preStatus.entrySet()) { status.put(entry.getKey(), entry.getValue()); - if(ExtentStatus.EXCLUDED.equals(entry.getValue())) exclusions.add(entry.getKey()); - if(ExtentStatus.INTERNAL.equals(entry.getValue())) fringe.add(entry.getKey()); } if(PROFILE) startupTime += System.nanoTime(); - -// for(RootSpec p : conf.roots) { -// if(p.internal) -// fringe.add(p.resource); -// } } @@ -185,10 +176,10 @@ public class DomainProcessor3 { if(graph.isSubrelationOf(predicate, L0.IsRelatedTo)) { isRelatedTo = Boolean.TRUE; - if(ModelTransferableGraphSourceRequest.LOG) ModelTransferableGraphSourceRequest.log("isRelatedToPredicates4 => " + NameUtils.getSafeName(graph, predicate)); + if(ModelTransferableGraphSourceRequest.DEBUG) ModelTransferableGraphSourceRequest.log("isRelatedToPredicates4 => " + NameUtils.getSafeName(graph, predicate)); } else { if (!graph.hasStatement(predicate)) { - if(ModelTransferableGraphSourceRequest.LOG) ModelTransferableGraphSourceRequest.log("FOUND DEAD PREDICATE (no statements): " + predicate); + if(ModelTransferableGraphSourceRequest.DEBUG) ModelTransferableGraphSourceRequest.log("FOUND DEAD PREDICATE (no statements): " + predicate); deadPredicates.add(predicate); // Prevents ModelTransferableGraphSource from // trying to export these statements. @@ -219,14 +210,13 @@ public class DomainProcessor3 { Resource predicate = stm.getPredicate(); Resource object = stm.getObject(); - if (exclusions.contains(object) || exclusions.contains(predicate)) - continue; + if(ExtentStatus.EXCLUDED.equals(status.get(predicate))) continue; + if(ExtentStatus.EXCLUDED.equals(status.get(object))) continue; if (exclusionFunction != null) { ExclusionDecision decision = exclusionFunction.apply(stm); if(ExclusionDecision.EXCLUDE_OBJECT.equals(decision)) { status.put(object, ExtentStatus.EXCLUDED); - exclusions.add(object); continue; } } @@ -239,10 +229,10 @@ public class DomainProcessor3 { if(predicates.add(inverse)) schedule.add(inverse); state.inverses.put(support.getTransientId(predicate), support.getTransientId(inverse)); state.inverses.put(support.getTransientId(inverse), support.getTransientId(predicate)); - if(ModelTransferableGraphSourceRequest.LOG) ModelTransferableGraphSourceRequest.log("INVERSE FOR " + graph.getPossibleURI(predicate) + " => " + graph.getPossibleURI(inverse)); + if(ModelTransferableGraphSourceRequest.DEBUG) ModelTransferableGraphSourceRequest.log("INVERSE FOR " + graph.getPossibleURI(predicate) + " => " + graph.getPossibleURI(inverse)); } else { state.inverses.put(support.getTransientId(predicate), 0); - if(ModelTransferableGraphSourceRequest.LOG) ModelTransferableGraphSourceRequest.log("NO INVERSE FOR " + graph.getPossibleURI(predicate)); + if(ModelTransferableGraphSourceRequest.DEBUG) ModelTransferableGraphSourceRequest.log("NO INVERSE FOR " + graph.getPossibleURI(predicate)); } } @@ -277,7 +267,7 @@ public class DomainProcessor3 { public void processValue(ReadGraph graph, Resource subject, int sId, final DomainProcessorState state) throws DatabaseException, IOException { final InputStream valueStream = tgs.getValueStream(graph, subject); if (valueStream != null) { - if(ModelTransferableGraphSourceRequest.LOG) ModelTransferableGraphSourceRequest.log("[VALUE] " + NameUtils.getSafeName(graph, subject, true)); + if(ModelTransferableGraphSourceRequest.DEBUG) ModelTransferableGraphSourceRequest.log("[VALUE] " + NameUtils.getSafeName(graph, subject, true)); state.valueOutput.writeInt(sId); if (conf.values) { @@ -343,12 +333,12 @@ public class DomainProcessor3 { Boolean isRelatedTo = isRelatedToPredicates.get(predicate); if ((objectStatus != ExtentStatus.EXCLUDED) && isRelatedTo) { - if(ModelTransferableGraphSourceRequest.LOG) logStatementWithExtent(graph, "related", objectStatus, subject, predicate, object); + if(ModelTransferableGraphSourceRequest.DEBUG) logStatementWithExtent(graph, "related", objectStatus, subject, predicate, object); addToStream(predicate, object); if(objectStatus == null || objectStatus == ExtentStatus.PENDING) { - if(ModelTransferableGraphSourceRequest.LOG) ModelTransferableGraphSourceRequest.log("[ADDED TO FRINGE] " + NameUtils.getSafeName(graph, object)); + if(ModelTransferableGraphSourceRequest.DEBUG) ModelTransferableGraphSourceRequest.log("[ADDED TO FRINGE] " + NameUtils.getSafeName(graph, object)); fringe.add(object); } @@ -356,14 +346,14 @@ public class DomainProcessor3 { // Dead predicate if (deadPredicates.contains(predicate)) { - if(ModelTransferableGraphSourceRequest.LOG) logStatementWithExtent(graph, "excluding statement with dead predicate ", objectStatus, subject, predicate, object); + if(ModelTransferableGraphSourceRequest.DEBUG) logStatementWithExtent(graph, "excluding statement with dead predicate ", objectStatus, subject, predicate, object); return; } // Weak predicate if(objectStatus == ExtentStatus.EXCLUDED) { - if(ModelTransferableGraphSourceRequest.LOG) logStatementWithExtent(graph, "weak reference to excluded object ", objectStatus, subject, predicate, object); + if(ModelTransferableGraphSourceRequest.DEBUG) logStatementWithExtent(graph, "weak reference to excluded object ", objectStatus, subject, predicate, object); } else { @@ -377,11 +367,11 @@ public class DomainProcessor3 { status.put(object, ExtentStatus.PENDING); } - if(ModelTransferableGraphSourceRequest.LOG) logStatementWithExtent(graph, "fully weak internal", objectStatus, subject, predicate, object); + if(ModelTransferableGraphSourceRequest.DEBUG) logStatementWithExtent(graph, "fully weak internal", objectStatus, subject, predicate, object); } else { - if(ModelTransferableGraphSourceRequest.LOG) logStatementWithExtent(graph, "strong inverse internal ", objectStatus, subject, predicate, object); + if(ModelTransferableGraphSourceRequest.DEBUG) logStatementWithExtent(graph, "strong inverse internal ", objectStatus, subject, predicate, object); } @@ -428,7 +418,7 @@ public class DomainProcessor3 { } status.put(subject, ExtentStatus.INTERNAL); - if(ModelTransferableGraphSourceRequest.LOG) ModelTransferableGraphSourceRequest.log("[INTERNAL] " + NameUtils.getSafeName(graph, subject, true)); + if(ModelTransferableGraphSourceRequest.DEBUG) ModelTransferableGraphSourceRequest.log("[INTERNAL] " + NameUtils.getSafeName(graph, subject, true)); int sId = support.getTransientId(subject); @@ -476,12 +466,12 @@ public class DomainProcessor3 { } ExtentStatus subjectStatus = status.get(subject); - if(ModelTransferableGraphSourceRequest.LOG && subjectStatus != null) ModelTransferableGraphSourceRequest.log("EXISTING STATUS FOR " + graph.getPossibleURI(subject) + " - " + subjectStatus); + if(ModelTransferableGraphSourceRequest.DEBUG && subjectStatus != null) ModelTransferableGraphSourceRequest.log("EXISTING STATUS FOR " + graph.getPossibleURI(subject) + " - " + subjectStatus); if(subjectStatus == ExtentStatus.EXTERNAL || subjectStatus == ExtentStatus.EXCLUDED) continue; if(partOf && (subjectStatus == null || ExtentStatus.PENDING == subjectStatus) && graph.getPossibleURI(subject) != null) { status.put(subject, ExtentStatus.EXTERNAL); - if(ModelTransferableGraphSourceRequest.LOG) { + if(ModelTransferableGraphSourceRequest.DEBUG) { String uri = graph.getPossibleURI(subject); if(uri == null) ModelTransferableGraphSourceRequest.log("[EXTERNAL]: No URI for " + subject); else ModelTransferableGraphSourceRequest.log("[EXTERNAL] " + uri); @@ -508,13 +498,13 @@ public class DomainProcessor3 { this.datatypeBinding = Bindings.getBindingUnchecked(Datatype.class); this.datatypeSerializer = graph.getService(Databoard.class).getSerializerUnchecked(this.datatypeBinding); - Pair,Set> pair = ConsistsOfProcess.walk(graph, status, fringe, exclusions, ignoreVirtual); + Pair,Set> pair = ConsistsOfProcess.walk(graph, status, conf.seeds, ignoreVirtual); state.internalEntries = pair.first; - for(InternalEntry entry : state.internalEntries) { + for(ConsistsOfProcessEntry entry : state.internalEntries) { Resource r = entry.resource; if (status.put(r, ExtentStatus.INTERNAL) == null) { - if(ModelTransferableGraphSourceRequest.LOG) { + if(ModelTransferableGraphSourceRequest.DEBUG) { String URI = graph.getPossibleURI(r); if(URI != null) ModelTransferableGraphSourceRequest.log("URI INTERNAL " + URI); else ModelTransferableGraphSourceRequest.log("URI has no URI for " + r); @@ -555,7 +545,7 @@ public class DomainProcessor3 { } void logStatementWithExtent(ReadGraph graph, String header, ExtentStatus status, int sId, int pId, int oId) throws DatabaseException { - if(ModelTransferableGraphSourceRequest.LOG) { + if(ModelTransferableGraphSourceRequest.DEBUG) { SerialisationSupport support = graph.getService(SerialisationSupport.class); String s = NameUtils.getURIOrSafeNameInternal(graph, support.getResource(sId)); String p = NameUtils.getURIOrSafeNameInternal(graph, support.getResource(pId)); @@ -565,7 +555,7 @@ public class DomainProcessor3 { } void logStatementWithExtent(ReadGraph graph, String header, ExtentStatus status, Resource sId, Resource pId, Resource oId) throws DatabaseException { - if(ModelTransferableGraphSourceRequest.LOG) { + if(ModelTransferableGraphSourceRequest.DEBUG) { String s = NameUtils.getURIOrSafeNameInternal(graph, sId); String p = NameUtils.getURIOrSafeNameInternal(graph, pId); String o = NameUtils.getURIOrSafeNameInternal(graph, oId); diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/DomainProcessorState.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/DomainProcessorState.java index 3f2541018..e8d97255b 100644 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/DomainProcessorState.java +++ b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/DomainProcessorState.java @@ -32,6 +32,6 @@ public class DomainProcessorState { public SubMonitor monitor; public TGValueModifier valueModifier; - public List internalEntries; + public List internalEntries; } \ No newline at end of file diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/ModelTransferableGraphSource.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/ModelTransferableGraphSource.java index 1d3cf61f5..95923c799 100644 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/ModelTransferableGraphSource.java +++ b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/ModelTransferableGraphSource.java @@ -27,8 +27,9 @@ import org.simantics.db.exception.DatabaseException; import org.simantics.db.exception.RuntimeDatabaseException; import org.simantics.db.exception.ValidationException; import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus; -import org.simantics.db.layer0.util.ConsistsOfProcess.InternalEntry; -import org.simantics.db.layer0.util.TransferableGraphConfiguration2.RootSpec; +import org.simantics.db.layer0.util.ConsistsOfProcess.ConsistsOfProcessEntry; +import org.simantics.db.layer0.util.TransferableGraphConfiguration2.SeedSpec; +import org.simantics.db.layer0.util.TransferableGraphConfiguration2.SeedSpec.SeedSpecType; import org.simantics.db.service.SerialisationSupport; import org.simantics.graph.db.TransferableGraphSource; import org.simantics.graph.representation.External; @@ -68,34 +69,34 @@ public class ModelTransferableGraphSource implements TransferableGraphSource { SerialisationSupport ss = graph.getService(SerialisationSupport.class); // At this point ids contains all internal resources. Now add roots and externals. - + // Root Library state.ids.put(ss.getTransientId(graph.getRootLibrary()), state.id++); - + // External roots - internal roots were already processed as internal resources by domain processor - for(RootSpec rs : configuration.roots) { - if(!rs.internal) { - int resourceId = ss.getTransientId(rs.resource); + for(SeedSpec spec : configuration.seeds) { + if(SeedSpecType.SPECIAL_ROOT.equals(spec.specType)) { + int resourceId = ss.getTransientId(spec.resource); state.ids.put(resourceId, state.id++); // The fixed roots have been seen as externals by domain processor. Now remove them from external set. state.externals.remove(resourceId); } } - + this.externalBase = state.id; final Collection errors = new HashSet<>(); // All resource considered as not internal by domain processor. Can also contain roots. int[] externals = state.externals.toArray(); - + // Build up the state.externals, externalNames and externalParents for(int i=0;i sorted = new ArrayList<>(errors); Collections.sort(sorted); @@ -120,15 +121,15 @@ public class ModelTransferableGraphSource implements TransferableGraphSource { } throw new DatabaseException(message.toString()); } - + this.resourceCount = state.id; - + state.extensions.put(ExternalDownloadBean.EXTENSION_KEY, new Variant(ExternalDownloadBean.BINDING, new ExternalDownloadBean(downloads))); - + } int indent = 0; - + public boolean validateExternal(Resource ext) { if(configuration.validate) { ExtentStatus status = configuration.preStatus.get(ext); @@ -139,14 +140,14 @@ public class ModelTransferableGraphSource implements TransferableGraphSource { } return true; } - + private Resource getResource(ReadGraph graph, int r) throws DatabaseException { SerialisationSupport ss = graph.getService(SerialisationSupport.class); return ss.getResource(r); } - + final public int getExistingId(ReadGraph graph, int r) throws DatabaseException { - + int ret = state.ids.get(r); if(ret != -1) { return ret; @@ -156,7 +157,7 @@ public class ModelTransferableGraphSource implements TransferableGraphSource { } } - + /* * * @return -2 if r is not really external and the statement should be excluded @@ -166,10 +167,10 @@ public class ModelTransferableGraphSource implements TransferableGraphSource { // // First external is root library // if(r == rootId) return internalCount; - + SerialisationSupport ss = graph.getService(SerialisationSupport.class); Layer0 L0 = Layer0.getInstance(graph); - + if(state.ids.containsKey(r)) { int ret = state.ids.get(r); if(ret == -1) { @@ -185,7 +186,7 @@ public class ModelTransferableGraphSource implements TransferableGraphSource { errors.add("Illegal reference to " + graph.getPossibleURI(getResource(graph, r))); return -2; } - Collection parents = graph.getObjects(res, L0.PartOf); + Collection parents = graph.getObjects(res, L0.PartOf); if(parents.size() != 1) { throw new ValidationException("Reference to external resource " + NameUtils.getSafeName(graph, getResource(graph, r), true) + " without unique uri (" + parents.size() + " parents)."); @@ -215,27 +216,36 @@ public class ModelTransferableGraphSource implements TransferableGraphSource { return state.id++; } } - + @Override public DataContainer getHeader() throws Exception { return null; } - + @Override public int getResourceCount() { return resourceCount; } - + + private int countRootSeeds() { + int result = 0; + for(SeedSpec spec : configuration.seeds) { + if(SeedSpecType.INTERNAL.equals(spec.specType)) continue; + result++; + } + return result; + } + @Override public int getIdentityCount() { - return configuration.roots.size() + state.externals.size() + state.internalEntries.size() + 1; + return countRootSeeds() + state.externals.size() + state.internalEntries.size() + 1; } - + @Override public int getStatementCount() { return state.statementCount; } - + @Override public int getValueCount() { return state.valueCount; @@ -243,18 +253,18 @@ public class ModelTransferableGraphSource implements TransferableGraphSource { @Override public void forStatements(ReadGraph graph, TransferableGraphSourceProcedure procedure) throws Exception { - + int[] value = new int[4]; long length = state.otherStatementsInput.length(); state.otherStatementsInput.position(0); - + while(state.otherStatementsInput.position() < length && !state.monitor.isCanceled()) { - + int s = state.otherStatementsInput.readInt(); int subjectId = state.ids.get(s); - + boolean exclude = subjectId == -1; - + int size = state.otherStatementsInput.readInt(); for(int i=0;i procedure) throws Exception { - + SerialisationSupport support = graph.getService(SerialisationSupport.class); Layer0 L0 = Layer0.getInstance(graph); @@ -395,7 +405,8 @@ public class ModelTransferableGraphSource implements TransferableGraphSource { TIntObjectMap internalMap = new TIntObjectHashMap<>(100, 0.5f, Integer.MIN_VALUE); // Declare internal and external roots - for(RootSpec r : configuration.roots) { + for(SeedSpec r : configuration.seeds) { + if(SeedSpecType.INTERNAL.equals(r.specType)) continue; String typeId = r.type; if (typeId == null) { Resource type = graph.getPossibleType(r.resource, L0.Entity); @@ -413,32 +424,44 @@ public class ModelTransferableGraphSource implements TransferableGraphSource { String name = externalNames.get(i); procedure.execute(new Identity(externalBase + i, new External(parent,name))); } - + if(state.internalEntries != null) { - for(InternalEntry ie : state.internalEntries) { - if(ie.parent != null) { - if(ie.name != null) { - procedure.execute(resolveInternal(graph, support, ie, internalMap)); - } else { - // In this case there is a child that has no HasName => this should be treated as a blank - } + for(ConsistsOfProcessEntry ie : state.internalEntries) { + if(ie.parent != null) { + if(ie.name != null) { + procedure.execute(resolveInternal(graph, support, ie, internalMap)); + } else { + // In this case there is a child that has no HasName => this should be treated as a blank + } } else { - throw new DatabaseException("Invalid internal entry " + ie); + procedure.execute(resolveInternal(graph, support, ie, internalMap)); } } } - + } - - private Identity resolveInternal(ReadGraph graph, SerialisationSupport ss, InternalEntry entry, TIntObjectMap internalMap) throws DatabaseException { + + private Identity resolveInternal(ReadGraph graph, SerialisationSupport ss, ConsistsOfProcessEntry entry, TIntObjectMap internalMap) throws DatabaseException { int id = state.ids.get(ss.getTransientId(entry.resource)); Identity existing = internalMap.get(id); if(existing != null) return existing; - Identity parent = resolveInternal(graph, ss, entry.parent, internalMap); - Identity result = new Identity(id, - new Internal(parent.resource, entry.name)); - internalMap.put(id, result); - return result; + + if(entry.parent == null) { + Layer0 L0 = Layer0.getInstance(graph); + Resource possibleParent = graph.getPossibleObject(entry.resource, L0.PartOf); + if(possibleParent == null) throw new DatabaseException("Invalid root or internal parent path: " + entry.resource); + int externalId = state.ids.get(ss.getTransientId(possibleParent)); + Identity result = new Identity(id, + new Internal(externalId, entry.name)); + internalMap.put(id, result); + return result; + } else { + Identity parent = resolveInternal(graph, ss, entry.parent, internalMap); + Identity result = new Identity(id, + new Internal(parent.resource, entry.name)); + internalMap.put(id, result); + return result; + } } @Override @@ -467,7 +490,7 @@ public class ModelTransferableGraphSource implements TransferableGraphSource { public void reset() throws Exception { throw new UnsupportedOperationException(); } - + public long[] getResourceArray(ReadGraph graph) throws DatabaseException { final SerialisationSupport ss = graph.getService(SerialisationSupport.class); final long[] result = new long[state.ids.size()]; @@ -475,38 +498,38 @@ public class ModelTransferableGraphSource implements TransferableGraphSource { @Override public boolean execute(int a, int b) { - + try { Resource r = ss.getResource(a); result[b] = r.getResourceId(); } catch (DatabaseException e) { e.printStackTrace(); } - + return true; - + } }); return result; } - + public DomainProcessorState getState() { return state; } - + public void forResourceStatements(ReadGraph graph, TransferableGraphSourceProcedure procedure) throws Exception { - + int[] value = new int[4]; long length = state.otherStatementsInput.length(); state.otherStatementsInput.position(0); - + while(state.otherStatementsInput.position() < length) { - + int s = state.otherStatementsInput.readInt(); int subjectId = state.ids.get(s); - + boolean exclude = subjectId == -1; - + int size = state.otherStatementsInput.readInt(); for(int i=0;i 500) line = line.substring(0, 500); log.writeUTF(line + "\n"); } catch (IOException e) { e.printStackTrace(); diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/Subgraphs.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/Subgraphs.java deleted file mode 100644 index 518efc882..000000000 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/Subgraphs.java +++ /dev/null @@ -1,2130 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2007, 2010 Association for Decentralized Information Management - * in Industry THTH ry. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Eclipse Public License v1.0 - * which accompanies this distribution, and is available at - * http://www.eclipse.org/legal/epl-v10.html - * - * Contributors: - * VTT Technical Research Centre of Finland - initial API and implementation - *******************************************************************************/ -package org.simantics.db.layer0.util; - -import java.io.DataOutput; -import java.io.DataOutputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.ObjectOutputStream; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.ConcurrentSkipListSet; - -import org.simantics.databoard.Bindings; -import org.simantics.databoard.Databoard; -import org.simantics.databoard.binding.Binding; -import org.simantics.databoard.binding.mutable.Variant; -import org.simantics.databoard.serialization.Serializer; -import org.simantics.databoard.type.Datatype; -import org.simantics.db.AsyncReadGraph; -import org.simantics.db.DirectStatements; -import org.simantics.db.ReadGraph; -import org.simantics.db.Resource; -import org.simantics.db.Statement; -import org.simantics.db.common.request.AsyncReadRequest; -import org.simantics.db.common.request.ReadRequest; -import org.simantics.db.common.request.ResourceAsyncRead; -import org.simantics.db.common.utils.NameUtils; -import org.simantics.db.exception.DatabaseException; -import org.simantics.db.layer0.adapter.SubgraphAdvisor; -import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus; -import org.simantics.db.layer0.util.ConsistsOfProcess.InternalEntry; -import org.simantics.db.procedure.AsyncProcedure; -import org.simantics.db.request.AsyncRead; -import org.simantics.db.service.ClusteringSupport; -import org.simantics.db.service.CollectionSupport; -import org.simantics.db.service.DirectQuerySupport; -import org.simantics.db.service.QueryControl; -import org.simantics.db.service.QueryControl.ControlProcedure; -import org.simantics.db.service.SerialisationSupport; -import org.simantics.db.service.StatementSupport; -import org.simantics.db.service.TransferableGraphSupport; -import org.simantics.graph.representation.Extensions; -import org.simantics.layer0.Layer0; -import org.simantics.utils.datastructures.Pair; -import org.simantics.utils.threads.logger.ITask; -import org.simantics.utils.threads.logger.ThreadLogger; - -import gnu.trove.list.array.TIntArrayList; -import gnu.trove.map.hash.TIntIntHashMap; -import gnu.trove.map.hash.TLongObjectHashMap; -import gnu.trove.procedure.TIntProcedure; -import gnu.trove.procedure.TLongObjectProcedure; -import gnu.trove.set.TIntSet; -import gnu.trove.set.hash.THashSet; -import gnu.trove.set.hash.TIntHashSet; - -public class Subgraphs { - - public static String LOG_FILE = "export.log"; - final static private boolean LOG = false; - final static private boolean DEBUG = false; - final static private boolean PARENT_DEBUG = DEBUG | false; - final static private boolean EXTERNAL_DEBUG = DEBUG | false; - final static private boolean ADVISOR_LOG = LOG & false; - final static private boolean EXPANSION_LOG = LOG & false; - final static private boolean INTERNAL_LOG = LOG & false; - final static private boolean COMPOSED_LOG = LOG & false; - final static private boolean RESOLVE_LOG = LOG & false; - final static private boolean CLASSIFY_LOG = LOG & false; - final static private boolean EXTERNAL_LOG = LOG & false; - final static private boolean PROFILE = false; - - static enum WeakStatus { - STRONG, WEAK - } - - - static DataOutput log; - - static { - - if (LOG) { - try { - FileOutputStream stream = new FileOutputStream(LOG_FILE, false); - log = new DataOutputStream(stream); - } catch (FileNotFoundException e) { - e.printStackTrace(); - } - } - - } - - private static void log(String line) { - if (LOG) { - try { - log.write((line + "\n").getBytes()); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - - public static Collection getParents(ReadGraph g, Resource r) - throws DatabaseException { - return getParents(g, r, false); - } - - static class FastInternalRequest extends ResourceAsyncRead { - - final DirectQuerySupport dqs; - final ConcurrentLinkedQueue queue; - final Map weakInverses; - final Map status; - - public FastInternalRequest(DirectQuerySupport dqs, Resource resource, - Map status, - Map weakInverses, - ConcurrentLinkedQueue queue) { - super(resource); - this.dqs = dqs; - this.status = status; - this.weakInverses = weakInverses; - this.queue = queue; - } - - @Override - public int getFlags() { - return 0; - } - - @Override - public void perform(AsyncReadGraph graph, final AsyncProcedure procedure) { - - dqs.forEachDirectStatement(graph, resource, new AsyncProcedure() { - - @Override - public void execute(AsyncReadGraph graph, DirectStatements ss) { - boolean ok = true; - for(Statement statement : ss) { - if (status.get(statement.getObject()) == ExtentStatus.INTERNAL) continue; - WeakStatus status = weakInverses.get(statement.getPredicate()); - if(status == WeakStatus.WEAK) continue; - else if (status == null) { - queue.add(statement.getPredicate()); - } - ok = false; - } - procedure.execute(graph, ok); - } - - @Override - public void exception(AsyncReadGraph graph, Throwable throwable) { - throwable.printStackTrace(); - } - - }); - - } - - } - - static class ClassifyStatementsRequest implements AsyncRead { - - final Set schedule; - final Map weakMap; - - public ClassifyStatementsRequest(Set schedule, Map weakMap) { - this.weakMap = weakMap; - this.schedule = schedule; - } - - @Override - public int threadHash() { - return hashCode(); - } - - @Override - public int getFlags() { - return 0; - } - - @Override - public void perform(AsyncReadGraph graph, final AsyncProcedure procedure) { - - for (final Resource p : schedule) { - - graph.forPossibleInverse(p, new AsyncProcedure() { - - private void register(AsyncReadGraph graph, Resource predicate, Resource superRelation, WeakStatus status) { - synchronized (weakMap) { - weakMap.put(predicate, status); - if(superRelation != null) weakMap.put(superRelation, status); - } - } - - @Override - public void exception(AsyncReadGraph graph, Throwable throwable) { - throwable.printStackTrace(); - } - - @Override - public void execute(AsyncReadGraph graph, final Resource inverse) { - - if (inverse == null) { - - register(graph, p, null, WeakStatus.WEAK); - - } else { - - graph.forPossibleSuperrelation(inverse, new AsyncProcedure() { - - @Override - public void exception(AsyncReadGraph graph, Throwable throwable) { - throwable.printStackTrace(); - } - - @Override - public void execute(AsyncReadGraph graph, final Resource superRelation) { - - if(superRelation != null && weakMap.containsKey(superRelation)) { - register(graph, p, null, weakMap.get(superRelation)); - return; - } - - graph.forIsSubrelationOf(inverse, graph.getService(Layer0.class).IsRelatedTo, new AsyncProcedure() { - - @Override - public void exception(AsyncReadGraph graph,Throwable throwable) { - throwable.printStackTrace(); - } - - @Override - public void execute(AsyncReadGraph graph,Boolean strong) { - register(graph, p, superRelation, strong ? WeakStatus.STRONG : WeakStatus.WEAK); - } - - }); - - } - - }); - - } - - } - - }); - - } - - procedure.execute(graph, false); - - } - - } - - private static Collection getParents(ReadGraph g, Resource r, boolean isStrong) throws DatabaseException { - - System.out.println("getParents " + NameUtils.getSafeName(g, r)); - - Layer0 l0 = Layer0.getInstance(g); - - Collection predicates = g.getPredicates(r); - - // --- Consists Of ---------------------------------------------------- - - if (predicates.contains(l0.PartOf)) { - Collection parents = g.getObjects(r, l0.PartOf); - if (parents.size() == 1) - return parents; - ArrayList libraryParents = new ArrayList(1); - for (Resource p : parents) - if (g.isInstanceOf(p, l0.Library)) - libraryParents.add(p); - if (!libraryParents.isEmpty()) - return libraryParents; - else - return parents; - } - - // --- Ordered sets --------------------------------------------------- - - { - Collection parents = null; - for (Resource p : predicates) - if (g.isInstanceOf(p, l0.OrderedSet) && !p.equals(r)) { - if (parents == null) - parents = new ArrayList(1); - parents.add(p); - } - if (parents != null) { - if (DEBUG) - System.out.println("ORDERED SET"); - return parents; - } - } - - - if (isStrong) - return Collections.emptyList(); - else { - - if (predicates.contains(l0.InverseOf)) { - - Resource inv = g.getInverse(r); - return getParents(g, inv, true); - - } else { - - /* - * Depends On - * - * If there are DependsOn parents, then IsRelatedTo parents are discarded - * - */ - HashSet result = new HashSet(); - for(Resource predicate : predicates) { - if(g.isSubrelationOf(predicate, l0.IsDependencyOf)) result.addAll(g.getObjects(r, predicate)); - } - if(!result.isEmpty()) return result; - - /* - * Is Related To - * - * At this point all Is Related To are parents. - * - */ - for(Resource predicate : predicates) { - Resource inv = g.getPossibleInverse(predicate); - if(inv != null) { - if(g.isSubrelationOf(inv, l0.IsRelatedTo)) result.addAll(g.getObjects(r, predicate)); - } - } - - return result; - - } - - /* - Collection invR = g.getObjects(r, b.IsRelatedTo_Inverse); - if (predicates.contains(b.InverseOf)) { - if (invR.size() > 1) { - if (DEBUG) - System.out - .println("###########################################"); - Resource inv = g.getInverse(r); - Collection ret = new ArrayList(); - for (Statement pp : g.getStatements(r, - b.IsRelatedTo_Inverse)) - if (!pp.getPredicate().equals(inv)) { - if (DEBUG) { - System.out.println("<" - + NameUtils.getSafeName(g, pp - .getSubject()) - + "," - + NameUtils.getSafeName(g, pp - .getPredicate()) - + "," - + NameUtils.getSafeName(g, pp - .getObject()) + ">"); - } - ret.add(pp.getObject()); - } - return ret; - } - // System.out.println("?????????????????"); - Collection invParents = getParents(g, - g.getInverse(r), true); - if (!invParents.isEmpty()) - return invParents; - } - if (DEBUG) { - System.out.print("invR"); - for (Resource res : invR) - System.out.print(" " + NameUtils.getSafeName(g, res)); - System.out.println(); - } - return invR; - */ - } - - } - -// public static String getIdentifier(ReadGraph g, Resource r) -// throws DatabaseException { -// Layer0 L0 = Layer0.getInstance(g); -// if (r.equals(g.getRootLibrary())) -// return ""; -// String name = g.getPossibleRelatedValue(r, L0.HasName); -// if (name == null) -// return null; -// Collection parents = getParents(g, r, true); -// if (parents.size() != 1) -// return null; -// for (Resource p : parents) { -// String parentIdentifier = getIdentifier(g, p); -// if (parentIdentifier == null) -// return null; -// return parentIdentifier + "/" + name; -// } -// return null; -// } - - static int kess = 0; - - static class Expansion extends AsyncReadRequest { - - final private Collection roots; - final Collection[] results; - final Collection[] listElements; - - public Expansion(Collection roots, Collection[] results, Collection[] listElements) { - this.roots = roots; - this.results = results; - this.listElements = listElements; - } - - @Override - public void run(AsyncReadGraph graph) { - - QueryControl control = graph.getService(QueryControl.class); - final DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class); - - final DomainStatementProcedure proc = new DomainStatementProcedure(dqs, graph.getService(StatementSupport.class), graph.getService(Layer0.class), results, listElements); - - int slice = (int) (roots.size() / control.getAmountOfQueryThreads()) + 1; - - final Resource[] rootArray = roots.toArray(Resource.NONE); - for (int i = 0; i < control.getAmountOfQueryThreads(); i++) { - - final int start = i * slice; - final int end = Math.min(start + slice, rootArray.length); - - control.schedule(graph, i, new ControlProcedure() { - - @Override - public void execute(AsyncReadGraph graph) { - for (int index = start; index < end; index++) { - dqs.forEachDirectStatement(graph, rootArray[index], proc); - } - - } - - }); - - } - - } - - @Override - public int getFlags() { - return 0; - } - - } - - static class Expansion2 extends AsyncReadRequest { - - final private Collection roots; - final Collection[] results; - final boolean ignoreVirtual; - - public Expansion2(Collection roots, Collection[] results) { - this(roots, results, true); - } - - public Expansion2(Collection roots, Collection[] results, boolean ignoreVirtual) { - this.roots = roots; - this.results = results; - this.ignoreVirtual = ignoreVirtual; - } - - @Override - public void run(AsyncReadGraph graph) { - - QueryControl control = graph.getService(QueryControl.class); - final DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class); - - final DomainStatementProcedure2 proc = - new DomainStatementProcedure2(results); - - int slice = (int) (roots.size() / control.getAmountOfQueryThreads()) + 1; - - final Resource[] rootArray = roots.toArray(Resource.NONE); - for (int i = 0; i < control.getAmountOfQueryThreads(); i++) { - - final int start = i * slice; - final int end = Math.min(start + slice, rootArray.length); - - control.schedule(graph, i, new ControlProcedure() { - @Override - public void execute(AsyncReadGraph graph) { - if (ignoreVirtual) { - for (int index = start; index < end; index++) { - dqs.forEachDirectPersistentStatement(graph, rootArray[index], proc); - } - } else { - for (int index = start; index < end; index++) { - dqs.forEachDirectStatement(graph, rootArray[index], proc); - } - } - } - }); - - } - - } - - @Override - public int getFlags() { - return 0; - } - - } - - static class DomainProcessor2 { - - Serializer variantSerializer; - - int id = 0; - - Set fringe = null; - Set exclusions = new HashSet(); - Set internalDomain = new HashSet(); - Set sharedExternalReferences = null; - TIntSet sharedExternalIds = null; - Set sharedExternalFringe = null; - Set predicates = null; - Set isRelatedToPredicates = null; - Set sharedPredicates = null; - TIntIntHashMap ids = null; - Map specials = null; - Map status = null; - Map weakInverses = null; - -// final ArrayList priorityList = new ArrayList(); - - private long composedObjectCounter = 0; - private long fastInternalCounter = 0; - private long parentExternalCounter = 0; - private long fullInternalCounter = 0; - private long fullExternalCounter = 0; - - private long startupTime = 0; - private long expandTime = 0; - private long fullResolveTime = 0; - private long fastResolveTime = 0; - private long otherStatementTime = 0; - private long parentResolveTime = 0; - private long extentSeedTime = 0; - private long composedPredicateTime = 0; - private long composedObjectTime = 0; - - public void expand(ReadGraph graph, Set fringe, Collection[] expansion) throws DatabaseException { - - long start = System.nanoTime(); - - Collection[]> fullExpansion = new ArrayList[]>(); - QueryControl control = graph.getService(QueryControl.class); - for (int i = 0; i < control.getAmountOfQueryThreads(); i++) { - expansion[i] = new ArrayList(); - } - - graph.syncRequest(new Expansion2(fringe, expansion)); - - fringe.clear(); - - expandTime += (System.nanoTime() - start); - - } - - public void classifyPredicates(ReadGraph graph, final Set schedule) throws DatabaseException { - - CollectionSupport cs = graph.getService(CollectionSupport.class); - - final Layer0 L0 = Layer0.getInstance(graph); - - long start = System.nanoTime(); - - final ConcurrentLinkedQueue composedResult = new ConcurrentLinkedQueue(); - final ConcurrentLinkedQueue singleResult = new ConcurrentLinkedQueue(); - final ConcurrentLinkedQueue sharedResult = new ConcurrentLinkedQueue(); - final ConcurrentLinkedQueue> singles = new ConcurrentLinkedQueue>(); - - // Discover singles - graph.syncRequest(new AsyncReadRequest() { - - @Override - public void run(AsyncReadGraph graph) { - - for (final Resource predicate : schedule) { - - graph.forPossibleSuperrelation(predicate, new AsyncProcedure() { - - @Override - public void exception(AsyncReadGraph graph, Throwable throwable) { - throwable.printStackTrace(); - } - - @Override - public void execute(AsyncReadGraph graph, final Resource single) { - singles.add(Pair.make(predicate, single)); - } - - }); - - graph.forHasStatement(predicate, L0.SharedRange, new AsyncProcedure() { - - @Override - public void exception(AsyncReadGraph graph, Throwable throwable) { - throwable.printStackTrace(); - } - - @Override - public void execute(AsyncReadGraph graph, final Boolean shared) { - if(shared) sharedResult.add(predicate); - } - - }); - - } - - } - - }); - - // Determine singles - final Set singleSchedule = cs.createSet(); - for(Pair pair : singles) { - - Resource single = pair.second; - if(single != null && predicates.add(single)) singleSchedule.add(single); - - } - - graph.syncRequest(new AsyncReadRequest() { - - @Override - public void run(AsyncReadGraph graph) { - - for (final Resource predicate : singleSchedule) { - - graph.forIsSubrelationOf(predicate, L0.IsRelatedTo, new AsyncProcedure() { - - @Override - public void exception(AsyncReadGraph graph, Throwable throwable) { - throwable.printStackTrace(); - } - - @Override - public void execute(AsyncReadGraph graph, Boolean strong) { - if (strong) singleResult.add(predicate); - } - - }); - - } - - } - - }); - - isRelatedToPredicates.addAll(singleResult); - sharedPredicates.addAll(sharedResult); - - final Set specialSchedule = cs.createSet(); - - // Classify - for(Pair pair : singles) { - - Resource single = pair.second; - if(single != null) { - if(isRelatedToPredicates.contains(single)) { - isRelatedToPredicates.add(pair.first); - } - } else { - specialSchedule.add(pair.first); - } - - } - - graph.syncRequest(new AsyncReadRequest() { - - @Override - public void run(AsyncReadGraph graph) { - - for (final Resource predicate : specialSchedule) { - - graph.forIsSubrelationOf(predicate, L0.IsRelatedTo, new AsyncProcedure() { - - @Override - public void exception(AsyncReadGraph graph, Throwable throwable) { - throwable.printStackTrace(); - } - - @Override - public void execute(AsyncReadGraph graph, Boolean composed) { - if (composed) composedResult.add(predicate); - } - - }); - - } - - } - - }); - - isRelatedToPredicates.addAll(composedResult); - - composedPredicateTime += (System.nanoTime() - start); - - } - - private Set strongInverseSet = new HashSet(); - - public void classifyPredicates(ReadGraph graph, final Collection[] expansion) throws DatabaseException { - - CollectionSupport cs = graph.getService(CollectionSupport.class); - final Set schedule = cs.createSet(); - final Map newPredicates = cs.createMap(Resource.class); - - for (Collection coll : expansion) - for (DirectStatements stms : coll) - for(Statement stm : stms) { - - Resource predicate = stm.getPredicate(); - - if(predicates.add(predicate)) { - Resource inverse = graph.getPossibleInverse(predicate); - schedule.add(predicate); - if(inverse != null) { - newPredicates.put(predicate, inverse); - if(predicates.add(inverse)) schedule.add(inverse); - } - - } - - } - - classifyPredicates(graph, schedule); - - for(Map.Entry entry : newPredicates.entrySet()) { - // Inverse is strong => this has strong inverse - if(isRelatedToPredicates.contains(entry.getValue())) { - strongInverseSet.add(entry.getKey()); - } - // This is strong => inverse has strong inverse - if(isRelatedToPredicates.contains(entry.getKey())) { - strongInverseSet.add(entry.getValue()); - } - } - - } - - /* - * Composed objects are internal. Mark them for expansion. - */ - - public void processFringe(ReadGraph graph, Collection[] expansion, - ObjectOutputStream otherStatementsOutput, ObjectOutputStream valueOutput) throws DatabaseException, IOException { - - SerialisationSupport support = graph.getService(SerialisationSupport.class); - TransferableGraphSupport tgs = graph.getService(TransferableGraphSupport.class); - - Layer0 L0 = Layer0.getInstance(graph); - - long start = System.nanoTime(); - - for (Collection coll : expansion) - for (DirectStatements stms : coll) { - - Resource subject = stms.getSubject(); - - boolean partOf = false; - for(Statement stm : stms) { - Resource predicate = stm.getPredicate(); - if(L0.PartOf.equals(predicate)) { - partOf = true; - break; - } - } - - ExtentStatus subjectStatus = status.get(subject); - if(LOG && subjectStatus != null) log("EXISTING STATUS FOR " + graph.getPossibleURI(subject) + " - " + subjectStatus); - if(subjectStatus == ExtentStatus.EXTERNAL) continue; - if(partOf && (subjectStatus == null) && graph.getPossibleURI(subject) != null) { - - status.put(subject, ExtentStatus.EXTERNAL); - if(LOG) { - String uri = graph.getPossibleURI(subject); - if(uri == null) log("[EXTERNAL]: No URI for " + subject); - else log("[EXTERNAL] " + uri); - } - - // Check for SharedRange statements - for(Statement stm : stms) { - Resource predicate = stm.getPredicate(); - if(sharedPredicates.contains(predicate)) { - sharedExternalFringe.add(stm.getObject()); - if(LOG) { - log("[SHARED EXTERNAL FRINGE]: " + NameUtils.getSafeName(graph, stm.getObject())); - } - } - } - - } else { - - boolean special = specials.containsKey(subject); - if(LOG) { - if(special) { - log("[SPECIAL] " + NameUtils.getSafeName(graph, subject)); - } - } - - status.put(subject, ExtentStatus.INTERNAL); - if(LOG) log("[INTERNAL] " + NameUtils.getSafeName(graph, subject)); - - int sId = support.getTransientId(subject); - - if(graph.hasValue(subject)) { - Datatype dt = graph.getRelatedValue(subject, L0.HasDataType, Bindings.getBindingUnchecked(Datatype.class)); - Binding b = Bindings.getBinding(dt); - Object _value = graph.getValue(subject, b); - Variant variant = new Variant(b, _value); - byte[] value = variantSerializer.serialize(variant); - if(LOG) log("[VALUE] " + NameUtils.getSafeName(graph, subject)); - valueOutput.writeInt(sId); - valueOutput.writeInt(value.length); - assert (value.length > 0); - valueOutput.write(value); - } - - TIntArrayList stream = new TIntArrayList(); - - for(Statement stm : stms) { - - if(special) { - -// System.err.println("stm=" + stm + " special=" + specials.get(subject)); - - } - - Resource predicate = stm.getPredicate(); - Resource object = stm.getObject(); - - ExtentStatus objectStatus = status.get(object); - - // Strong predicate - if (isRelatedToPredicates.contains(predicate) && (objectStatus != ExtentStatus.EXCLUDED)) { - - int pId = support.getTransientId(predicate); - int oId = support.getTransientId(object); - - if(LOG) { - String s = NameUtils.getSafeName(graph, subject); - String p = NameUtils.getSafeName(graph, predicate); - String o = NameUtils.getSafeName(graph, object); - log("related=" + s + " - " + p + " - " + o); - } - - stream.add(pId); - stream.add(oId); - - if(objectStatus == null) - fringe.add(object); - - } else { - - // Weak predicate - if(objectStatus == ExtentStatus.INTERNAL) { - - // The inverse is also weak (or there is no inverse) - if(!strongInverseSet.contains(predicate)) { - - int pId = support.getTransientId(predicate); - int oId = support.getTransientId(object); - - stream.add(pId); - stream.add(oId); - - if(LOG) { - String s = NameUtils.getSafeName(graph, subject); - String p = NameUtils.getSafeName(graph, predicate); - String o = NameUtils.getSafeName(graph, object); - log("fully weak internal=" + s + " - " + p + " - " + o + " - " + objectStatus); - } - - } else { - - if(LOG) { - String s = NameUtils.getSafeName(graph, subject); - String p = NameUtils.getSafeName(graph, predicate); - String o = NameUtils.getSafeName(graph, object); - log("strong inverse internals=" + s + " - " + p + " - " + o + " - " + objectStatus); - } - - } - - } else { - - if(special) { - -// System.err.println("stm=" + stm + " special=" + specials.get(subject)); - - Statement spec = specials.get(subject); - - // This statement can be specially treated - if(stm.getPredicate().equals(spec.getPredicate()) && stm.getObject().equals(spec.getObject())) { - - int pId = support.getTransientId(predicate); - int oId = support.getTransientId(object); - - if(LOG) { - String s = NameUtils.getSafeName(graph, subject); - String p = NameUtils.getSafeName(graph, predicate); - String o = NameUtils.getSafeName(graph, object); - log("special=" + s + " - " + p + " - " + o); - } - - stream.add(pId); - stream.add(oId); - - } - - } else { - - if(LOG) { - String s = NameUtils.getSafeName(graph, subject); - String p = NameUtils.getSafeName(graph, predicate); - String o = NameUtils.getSafeName(graph, object); - log("weak with unknown object=" + s + " - " + p + " - " + o + " - " + objectStatus); - } - - } - - } - - } - - } - - if(!stream.isEmpty()) { - otherStatementsOutput.writeInt(sId); - otherStatementsOutput.writeInt(stream.size() / 2); - for (int i = 0; i < stream.size(); i++) - otherStatementsOutput.writeInt(stream.getQuick(i)); - } - - } - - } - - composedObjectTime += System.nanoTime() - start; - - } - - public void process(ReadGraph graph, - ObjectOutputStream otherStatementsOutput, - ObjectOutputStream valueOutput) - throws DatabaseException, IOException { - - this.variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT); - - QueryControl control = graph.getService(QueryControl.class); - -// System.err.println("Begin ConsistsOfProcess"); - - /* - * Browse all stm = (s, ConsistsOf, o) - * � All internal URIs are found => from now on, if unidentified resource has PartOf it is external. - * � All s are internal - * � All o are internal - * � All stm are included - */ - Pair,Set> pair = ConsistsOfProcess.walk(graph, null, fringe, exclusions, true); - List entries = pair.first; - for(InternalEntry entry : entries) { - Resource r = entry.resource; - if (status.put(r, ExtentStatus.INTERNAL) == null) { - String URI = graph.getPossibleURI(r); - if(URI != null) log("URI INTERNAL " + URI); - else log("URI has no URI for " + r); - fringe.add(r); - internalDomain.add(r); - } - } - - for(Resource unnamedChild : pair.second) { - if (status.put(unnamedChild, ExtentStatus.INTERNAL) == null) { - fringe.add(unnamedChild); - } - } - - /* - * This loop resolves the transitive closure of all p < IsRelatedTo such that p does not contain the SharedRange tag. - * Such resources are guaranteed to be internal. - */ - while(!fringe.isEmpty()) { - -// System.err.println("Process Fringe with " + fringe.size() + "."); - - Collection[] expansion = new ArrayList[control.getAmountOfQueryThreads()]; - -// System.err.println("-expand"); - - // Expand fringe - expand(graph, fringe, expansion); - - /* - * classify all p - * -IsRelatedTo - * -SharedRange - * -Internal / External - */ - -// System.err.println("-classify"); - - classifyPredicates(graph, expansion); - - /* - * for stms in [stms] - * if stms contains predicate PartOf => s is External - * else s is Internal - * for all stm=(s,p,o) in stms - * if p stm is included - * Fringe <- o - */ - -// System.err.println("-process"); - - processFringe(graph, expansion, otherStatementsOutput, valueOutput); - - } - - while(!sharedExternalFringe.isEmpty()) { - - Collection[] expansion = new ArrayList[control.getAmountOfQueryThreads()]; - expand(graph, sharedExternalFringe, expansion); - - for (Collection coll : expansion) - for (DirectStatements stms : coll) { - - Resource subject = stms.getSubject(); - ExtentStatus subjectStatus = status.get(subject); - - if(ExtentStatus.INTERNAL == subjectStatus) { - - if(internalDomain.contains(subject)) continue; - - status.put(subject, ExtentStatus.EXTERNAL); - sharedExternalReferences.add(subject); - - if(LOG) { - log("[SHARED EXTERNAL REFERENCE]: " + NameUtils.getSafeName(graph, subject)); - } - - for(Statement stm : stms) { - Resource predicate = stm.getPredicate(); - if (isRelatedToPredicates.contains(predicate)) { - sharedExternalFringe.add(stm.getObject()); - } - } - - } - } - - } - - if (PROFILE) { - System.out.println(composedObjectCounter + " " + fastInternalCounter - + " " + parentExternalCounter + " " - + fullExternalCounter + " " + fullInternalCounter); - } - - } - - } - - static class DomainProcessor { - - Serializer variantSerializer; - - int id = 0; - - Set predicates = null; - Set composedPredicates = null; - Set expansionSeeds = null; - Map ids = null; - Map status = null; - Map weakInverses = null; - - final Set advisors; - final ArrayList priorityList = new ArrayList(); - - private long composedObjectCounter = 0; - private long fastInternalCounter = 0; - private long parentExternalCounter = 0; - private long fullInternalCounter = 0; - private long fullExternalCounter = 0; - - private long startupTime = 0; - private long expandTime = 0; - private long fullResolveTime = 0; - private long fastResolveTime = 0; - private long otherStatementTime = 0; - private long parentResolveTime = 0; - private long extentSeedTime = 0; - private long composedPredicateTime = 0; - private long composedObjectTime = 0; - - public DomainProcessor(Set advisors) { - this.advisors = advisors; - HashSet prioritySet = new HashSet(); - for (SubgraphAdvisor advisor : advisors) - prioritySet.add(advisor.priority()); - priorityList.addAll(prioritySet); - Collections.sort(priorityList); - } - - public void expand(ReadGraph graph, Collection[] expansion, Set schedule) throws DatabaseException { - - long start = System.nanoTime(); - -// if (DEBUG) -// System.out.println("expanding " + expansionSeeds.size() + " resources."); - - QueryControl control = graph.getService(QueryControl.class); -// final Collection[] results = new ArrayList[control.getAmountOfQueryThreads()]; - final ArrayList[] listElements = new ArrayList[control.getAmountOfQueryThreads()]; - for (int i = 0; i < control.getAmountOfQueryThreads(); i++) { -// results[i] = new ArrayList(); - listElements[i] = new ArrayList(); - } - -// if(DEBUG) { -// for(Resource r : expansionSeeds) -// System.out.println("Expanding " + NameUtils.getSafeName(graph, r, true)); -// } - - graph.syncRequest(new Expansion(expansionSeeds, expansion, listElements)); - for (int i = 0; i < control.getAmountOfQueryThreads(); i++) { -// for (DirectStatements s : results[i]) { -// expansion.put(s.getSubject(), s); -// } - for (Resource s : listElements[i]) { - schedule.add(s); -// if(status.put(s, ExtentStatus.INTERNAL) == null) { -// ids.put(s, id++); -// } - } - } - - expandTime += (System.nanoTime() - start); - - } - - public void extractComposedPredicates(ReadGraph graph, final Collection[] expansion) throws DatabaseException { - - long start = System.nanoTime(); - - CollectionSupport cs = graph.getService(CollectionSupport.class); - - final ConcurrentLinkedQueue composedResult = new ConcurrentLinkedQueue(); - final ConcurrentLinkedQueue singleResult = new ConcurrentLinkedQueue(); - final ConcurrentLinkedQueue> singles = new ConcurrentLinkedQueue>(); - - final Set schedule = cs.createSet(); - - for (Collection coll : expansion) - for (DirectStatements stms : coll) - for(Statement stm : stms) { - Resource predicate = stm.getPredicate(); - if(predicates.add(predicate)) schedule.add(predicate); - } - - // Discover singles - graph.syncRequest(new AsyncReadRequest() { - - @Override - public void run(AsyncReadGraph graph) { - - for (final Resource predicate : schedule) { - - graph.forPossibleSuperrelation(predicate, new AsyncProcedure() { - - @Override - public void exception(AsyncReadGraph graph, Throwable throwable) { - throwable.printStackTrace(); - } - - @Override - public void execute(AsyncReadGraph graph, final Resource single) { - singles.add(Pair.make(predicate, single)); - } - - }); - - } - - } - - }); - - // Determine singles - final Set singleSchedule = cs.createSet(); - for(Pair pair : singles) { - - Resource single = pair.second; - if(single != null && predicates.add(single)) singleSchedule.add(single); - - } - - graph.syncRequest(new AsyncReadRequest() { - - @Override - public void run(AsyncReadGraph graph) { - - for (final Resource predicate : singleSchedule) { - - graph.forIsSubrelationOf(predicate, graph.getService(Layer0.class).IsComposedOf, new AsyncProcedure() { - - @Override - public void exception(AsyncReadGraph graph, Throwable throwable) { - throwable.printStackTrace(); - } - - @Override - public void execute(AsyncReadGraph graph, Boolean composed) { - if (composed) singleResult.add(predicate); - } - - }); - - } - - } - - }); - - composedPredicates.addAll(singleResult); - - final Set specialSchedule = cs.createSet(); - - // Classify - for(Pair pair : singles) { - - Resource single = pair.second; - if(single != null) { - if(composedPredicates.contains(single)) { - composedPredicates.add(pair.first); - } - } else { - specialSchedule.add(pair.first); - } - - } - - graph.syncRequest(new AsyncReadRequest() { - - @Override - public void run(AsyncReadGraph graph) { - - for (final Resource predicate : specialSchedule) { - - graph.forIsSubrelationOf(predicate, graph.getService(Layer0.class).IsComposedOf, new AsyncProcedure() { - - @Override - public void exception(AsyncReadGraph graph, Throwable throwable) { - throwable.printStackTrace(); - } - - @Override - public void execute(AsyncReadGraph graph, Boolean composed) { - if (composed) composedResult.add(predicate); - } - - }); - - } - - } - - }); - - composedPredicates.addAll(composedResult); - - composedPredicateTime += (System.nanoTime() - start); - - } - - /* - * Composed objects are internal. Mark them for expansion. - */ - - public void collectComposedObjects(ReadGraph graph, Collection[] expansion, Set typeTodo, Set objectTodo, - Set predicateTodo) throws DatabaseException { - - long start = System.nanoTime(); - - Layer0 l0 = Layer0.getInstance(graph); - - for (Collection coll : expansion) - for (DirectStatements stms : coll) - for(Statement stm : stms) { - - Resource predicate = stm.getPredicate(); - Resource object = stm.getObject(); - - if (composedPredicates.contains(predicate)) { - - ExtentStatus existing = status.put(object, ExtentStatus.INTERNAL); - if(existing == null) { - ids.put(object, id++); - composedObjectCounter++; - expansionSeeds.add(object); -// System.err.println("internal: " + NameUtils.getSafeName(graph, object, true)); - if(LOG) log("[INTERNAL] (composed object) " + NameUtils.getSafeName(graph, object, true)); - } else if (existing == ExtentStatus.EXCLUDED) { - System.err.println("preExcluded: " + NameUtils.getSafeName(graph, object, true)); - status.put(object, ExtentStatus.EXCLUDED); - } else if (existing == ExtentStatus.EXTERNAL) { - System.err.println("preExternal: " + NameUtils.getSafeName(graph, object, true)); - status.put(object, ExtentStatus.EXTERNAL); - } - - } else { - -// System.err.println("internal2: " + NameUtils.getSafeName(graph, object, true)); - - if (!status.containsKey(object)) { - if (l0.InstanceOf.equalsResource(predicate)) { - typeTodo.add(object); - } else { - objectTodo.add(object); - } - } - - if (!status.containsKey(predicate)) { - predicateTodo.add(predicate); - } - - } - } - - composedObjectTime += System.nanoTime() - start; - - } - - public void writeOtherStatements(ReadGraph graph, Collection[]> expansion, ObjectOutputStream composedStatementsOutput, ObjectOutputStream otherStatementsOutput, - ObjectOutputStream valueOutput) throws DatabaseException { - - long start = System.nanoTime(); - - Layer0 l0 = Layer0.getInstance(graph); - SerialisationSupport support = graph.getService(SerialisationSupport.class); - TransferableGraphSupport tgs = graph.getService(TransferableGraphSupport.class); - - TIntArrayList other = new TIntArrayList(); - TIntArrayList composed = new TIntArrayList(); - - try { - - for (Collection[] colls : expansion) - for (Collection coll : colls) - for (DirectStatements stms : coll) { - - Resource subject = stms.getSubject(); - composed.resetQuick(); - - int sId = support.getTransientId(subject); - - composedStatementsOutput.writeInt(sId); - - if(graph.hasValue(subject)) { - Datatype dt = graph.getRelatedValue(subject, l0.HasDataType, Bindings.getBindingUnchecked(Datatype.class)); - Binding b = Bindings.getBinding(dt); - Object _value = graph.getValue(subject, b); - Variant variant = new Variant(b, _value); - byte[] value = variantSerializer.serialize(variant); - if(LOG) log("[VALUE] " + NameUtils.getSafeName(graph, subject)); - valueOutput.writeInt(sId); - valueOutput.writeInt(value.length); - assert (value.length > 0); - valueOutput.write(value); - } - - for (Statement s : stms) { - - Resource object = s.getObject(); - Resource predicate = s.getPredicate(); - - ExtentStatus objectStatus = status.get(object); - - if(objectStatus == ExtentStatus.INTERNAL) { - composed.add(support.getTransientId(predicate)); - composed.add(support.getTransientId(object)); - if(LOG) log("[COMPOSED] (internal object) " + NameUtils.toIdString(graph, s)); - } else if (l0.InstanceOf.equalsResource(predicate)) { - composed.add(support.getTransientId(predicate)); - composed.add(support.getTransientId(object)); - if(LOG) log("[COMPOSED] (instanceOf) " + NameUtils.toIdString(graph, s)); - } else if (l0.SubrelationOf.equalsResource(predicate)) { - composed.add(support.getTransientId(predicate)); - composed.add(support.getTransientId(object)); - if(LOG) log("[COMPOSED] (subrelationOf) " + NameUtils.toIdString(graph, s)); - } else { - if(objectStatus == ExtentStatus.EXTERNAL) { - if(DEBUG) - System.out.println("other " + NameUtils.toIdString(graph, s)); - //System.out.println("other.add " + predicate + " - " + object); - other.add(support.getTransientId(predicate)); - other.add(support.getTransientId(object)); - if(LOG) log("[OTHER] (object is external) " + NameUtils.toIdString(graph, s)); - } - } - - } - - if(!other.isEmpty()) { - otherStatementsOutput.writeInt(sId); - otherStatementsOutput.writeInt(other.size() / 2); - for (int i = 0; i < other.size(); i++) - otherStatementsOutput.writeInt(other.getQuick(i)); - other.resetQuick(); - } - - composedStatementsOutput.writeInt(composed.size() / 2); - for (int i = 0; i < composed.size(); i++) - composedStatementsOutput.writeInt(composed.getQuick(i)); - - } - - } catch (IOException e) { - e.printStackTrace(); - } - - otherStatementTime += (System.nanoTime() - start); - - } - - boolean hasStrictParents(ReadGraph g, Resource r) - throws DatabaseException { - if (g.getPossibleURI(r) != null) - return true; - return false; - } - - public boolean getExpansionSeedsFromExtents(ReadGraph graph, final Collection[] expansion) throws DatabaseException { - - long start = System.nanoTime(); - - final ConcurrentLinkedQueue accepts = new ConcurrentLinkedQueue(); - - /* - * Determine statements which could accept statements with todo - * objects - */ - search: for (Double priority : priorityList) { - - for (final SubgraphAdvisor advisor : advisors) { - - if (advisor.priority() > 0) - continue; - - if (advisor.priority() == priority) { - - graph.syncRequest(new ReadRequest() { - - @Override - public void run(ReadGraph graph) throws DatabaseException { - - for (Collection coll : expansion) - for (DirectStatements stms : coll) - for(final Statement stm : stms) { - - advisor.advise(graph, stm, new AsyncProcedure() { - - @Override - public void exception(AsyncReadGraph graph, Throwable throwable) { - throwable.printStackTrace(); - } - - @Override - public void execute(AsyncReadGraph graph, Boolean accept) { - if (accept) { - accepts.add(stm.getObject()); - } - } - - }); - - } - - } - - }); - } - if (!accepts.isEmpty()) - break search; - } - } - - CollectionSupport cs = graph.getService(CollectionSupport.class); - Set schedule = cs.createSet(); - for (Resource r : accepts) { - if(!status.containsKey(r)) - schedule.add(r); - } - - extentSeedTime += (System.nanoTime() - start); - - if (schedule.isEmpty()) - return false; - - fastResolve(graph, schedule); - uriResolve(graph, schedule); - fullResolve(graph, schedule, "accepts"); - - return true; - - } - - ConcurrentLinkedQueue fastInternals = new ConcurrentLinkedQueue(); - - public void fastResolve(ReadGraph graph, final Set rs) - throws DatabaseException { - // This collects and resolves weaks - if(fastResolveLoop(graph, rs)) - // Weaks are now resolved - fastResolveLoop(graph, rs); - } - - public boolean fastResolveLoop(ReadGraph graph, final Set rs) - throws DatabaseException { - - long start = System.nanoTime(); - - final ConcurrentLinkedQueue weakSchedule = new ConcurrentLinkedQueue(); - - graph.syncRequest(new AsyncRead() { - - @Override - public int threadHash() { - return hashCode(); - } - - @Override - public int getFlags() { - return 0; - } - - @Override - public void perform(AsyncReadGraph graph, - AsyncProcedure procedure) { - - QueryControl control = graph.getService(QueryControl.class); - final DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class); - - int slice = (int) (rs.size() / control - .getAmountOfQueryThreads()) + 1; - - final Resource[] rootArray = rs.toArray(Resource.NONE); - for (int i = 0; i < control.getAmountOfQueryThreads(); i++) { - - final int start = i * slice; - final int end = Math.min(start + slice, - rootArray.length); - - control.schedule(graph, i, new ControlProcedure() { - - @Override - public void execute(AsyncReadGraph graph) { - - for (int index = start; index < end; index++) { - - final Resource r = rootArray[index]; - - //if (status.containsKey(r)) continue; - - graph.asyncRequest(new FastInternalRequest(dqs, r, status, weakInverses, weakSchedule),new AsyncProcedure() { - - @Override - public void exception(AsyncReadGraph graph,Throwable throwable) { - throwable.printStackTrace(); - } - - @Override - public void execute(AsyncReadGraph graph,Boolean isInternal) { - if (isInternal) { - fastInternals.add(r); - } - } - - }); - - } - - } - - }); - - } - - procedure.execute(graph, true); - - } - - }); - - if (!weakSchedule.isEmpty()) { - THashSet weaks = new THashSet(weakSchedule); - if (CLASSIFY_LOG) - for (Resource p : weakSchedule) - System.out.println("classify " - + NameUtils.getSafeName(graph, p)); - graph.syncRequest(new ClassifyStatementsRequest(weaks, weakInverses)); - } - - for (Resource r : fastInternals) { - rs.remove(r); - if (status.put(r, ExtentStatus.INTERNAL) == null) { - if(LOG) log("[INTERNAL] (fast) " + NameUtils.getSafeName(graph, r, true)); - ids.put(r, id++); - fastInternalCounter++; - expansionSeeds.add(r); - } - } - - fastResolveTime += (System.nanoTime() - start); - - return !weakSchedule.isEmpty(); - - } - - private ExtentStatus resolveExtent(ReadGraph graph, Resource r, Map statuses, Set expansionSeeds, THashSet pending, - ArrayList stack) throws DatabaseException { - - ExtentStatus current = statuses.get(r); - if(current != null) return current; - - if (pending.contains(r)) - return ExtentStatus.PENDING; - - // In order to break cyclic dependencies - pending.add(r); - - if (PARENT_DEBUG) - System.out.println("resolveExtent " - + NameUtils.getSafeName(graph, r)); - - ExtentStatus status = ExtentStatus.INTERNAL; - for (Resource p : getParents(graph, r)) { - if (PARENT_DEBUG) { - ExtentStatus ps = statuses.get(p); - System.out.println(" parent " + NameUtils.getSafeName(graph, p) + "(" + ps + ")"); - } - switch (resolveExtent(graph, p, statuses, - expansionSeeds, pending, stack)) { - case EXTERNAL: - return ExtentStatus.EXTERNAL; - case PENDING: - status = ExtentStatus.PENDING; - } - } - if (status == ExtentStatus.INTERNAL) { - pending.remove(r); - stack.add(r); - if (DEBUG) - System.out.println(NameUtils.getSafeName(graph, r, true) - + " is internal."); - } - return status; - } - - public void uriResolve(ReadGraph graph, final Set todo) - throws DatabaseException { - - long start = System.nanoTime(); - - for(Resource r : todo) System.out.println("uriResolve " + - NameUtils.getSafeName(graph, r)); - - final ConcurrentSkipListSet found = new ConcurrentSkipListSet(); - - graph.syncRequest(new AsyncReadRequest() { - - @Override - public void run(AsyncReadGraph graph) { - - for (final Resource r : todo) { - - // System.out.println("uriresolve before " + r); - - if (status.containsKey(r)) continue; - - // System.out.println("uriresolve " + r); - - graph.forURI(r, new AsyncProcedure() { - - @Override - public void exception(AsyncReadGraph graph, Throwable throwable) { - throwable.printStackTrace(); - } - - @Override - public void execute(AsyncReadGraph graph, String uri) { - - if (uri != null) { - - // System.out.println("uriresolve has uri " - // + r); - - if(found.add(r)) { - parentExternalCounter++; - } - - } else { - - // System.out.println("uriresolve ask inverse " - // + r); - - graph.forPossibleInverse(r, new AsyncProcedure() { - - @Override - public void exception(AsyncReadGraph graph, Throwable throwable) { - throwable.printStackTrace(); - } - - @Override - public void execute(AsyncReadGraph graph, Resource inverse) { - - if (inverse != null) { - - graph.forURI(inverse, new AsyncProcedure() { - - @Override - public void exception(AsyncReadGraph graph, Throwable throwable) { - throwable.printStackTrace(); - } - - @Override - public void execute(AsyncReadGraph graph, String uri) { - - if (uri != null) { - - if(found.add(r)) { - parentExternalCounter++; - } - - } - - } - - }); - - } - - } - - }); - - } - } - - }); - - } - - } - - }); - - todo.removeAll(found); - for(Resource r : found) { - status.put(r, ExtentStatus.EXTERNAL); - if(LOG) log("[EXTERNAL] (uriResolve) " + NameUtils.getSafeName(graph, r, true)); - } - - parentResolveTime += System.nanoTime() - start; - - } - - public void fullResolve(ReadGraph graph, Collection rs, - String koss) throws DatabaseException { - - long start = System.nanoTime(); - - for (final Resource r : rs) { - - if(status.containsKey(r)) continue; - - THashSet pending = new THashSet(); - ArrayList stack = new ArrayList(); - - ExtentStatus s = resolveExtent(graph, r, status, expansionSeeds, pending, stack); - if (ExtentStatus.INTERNAL == s || ExtentStatus.PENDING == s) { - if (status.put(r, ExtentStatus.INTERNAL) == null) { - if(LOG) log("[INTERNAL] (resolveExtent) " + NameUtils.getSafeName(graph, r, true)); - ids.put(r, id++); - fullInternalCounter++; - expansionSeeds.add(r); - } - } - if (ExtentStatus.EXTERNAL == s) { - if (status.put(r, ExtentStatus.EXTERNAL) == null) { - if(LOG) log("[EXTERNAL] (resolveExtent) " + NameUtils.getSafeName(graph, r, true)); - fullExternalCounter++; - } - } - - } - - fullResolveTime += (System.nanoTime() - start); - - } - - public void process(ReadGraph graph, - ObjectOutputStream composedStatementsOutput, - ObjectOutputStream otherStatementsOutput, - ObjectOutputStream valueOutput) - throws DatabaseException { - - this.variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT); - - CollectionSupport cs = graph.getService(CollectionSupport.class); - - Set typeTodo = cs.createSet(); - Set objectTodo = cs.createSet(); - Set predicateTodo = cs.createSet(); - - Collection[]> fullExpansion = new ArrayList[]>(); - - do { - - QueryControl control = graph.getService(QueryControl.class); - Collection[] expansion = new ArrayList[control.getAmountOfQueryThreads()]; - for (int i = 0; i < control.getAmountOfQueryThreads(); i++) { - expansion[i] = new ArrayList(); - } - - // Expand expansionSeeds - expand(graph, expansion, objectTodo); - - // Start collecting new seeds - expansionSeeds = cs.createSet(); - - // Collect predicates which are roots, Map preStatus, - Map specials, - ObjectOutputStream otherStatementsOutput, - ObjectOutputStream valueOutput, - TreeMap extensions, - TIntHashSet excludedShared) throws DatabaseException { - - ITask task = ThreadLogger.getInstance().begin("getDomain2"); - - final DomainProcessor2 processor = new DomainProcessor2(); - - processor.startupTime = System.nanoTime(); - - Layer0 l0 = Layer0.getInstance(graph); - - CollectionSupport cs = graph.getService(CollectionSupport.class); - SerialisationSupport support = graph.getService(SerialisationSupport.class); - - processor.ids = ids; - processor.specials = specials; - processor.status = cs.createMap(ExtentStatus.class); - processor.weakInverses = cs.createMap(WeakStatus.class); - processor.predicates = cs.createSet(); - processor.isRelatedToPredicates = cs.createSet(); - processor.sharedPredicates = cs.createSet(); -// processor.expansionSeeds = cs.createSet(); - - for(Map.Entry entry : preStatus.entrySet()) { - processor.status.put(entry.getKey(), entry.getValue()); - if(ExtentStatus.EXCLUDED.equals(entry.getValue())) processor.exclusions.add(entry.getKey()); - } - -// for (Resource r : excluded) { -// processor.status.put(r, ExtentStatus.EXCLUDED); -// } - - Resource rootLibrary = graph.getResource("http:/"); - - if (!roots.contains(rootLibrary)) - processor.status.put(rootLibrary, ExtentStatus.EXTERNAL); - - for (Resource root : roots) { - processor.status.put(root, ExtentStatus.INTERNAL); - //processor.ids.put(support.getTransientId(root), processor.ids.size()); - for (Resource owner : graph.getObjects(root, l0.IsOwnedBy)) { - processor.status.put(owner, ExtentStatus.EXTERNAL); - } - } - - processor.startupTime = System.nanoTime() - processor.startupTime; - - processor.fringe = new HashSet(); - processor.fringe.addAll(roots); - - processor.internalDomain.addAll(roots); - - processor.sharedExternalReferences = new HashSet(); - processor.sharedExternalFringe = new HashSet(); - - try { - - processor.process(graph, otherStatementsOutput, valueOutput); - - } catch (IOException e) { - e.printStackTrace(); - } - - for(Resource r : processor.sharedExternalReferences) excludedShared.add(support.getTransientId(r)); - - ClusteringSupport cls = graph.getService(ClusteringSupport.class); - TLongObjectHashMap clusterMap = new TLongObjectHashMap(); - for(Map.Entry entry : processor.status.entrySet()) { - if(ExtentStatus.INTERNAL == entry.getValue()) { - - long cluster = cls.getCluster(entry.getKey()); - TIntArrayList list = clusterMap.get(cluster); - if(list == null) { - list = new TIntArrayList(); - clusterMap.put(cluster, list); - } - list.add(support.getTransientId(entry.getKey())); - - } - } - final TIntArrayList clustering = new TIntArrayList(); - clusterMap.forEachEntry(new TLongObjectProcedure() { - - @Override - public boolean execute(long cluster, TIntArrayList b) { - clustering.add(b.size()); - b.forEach(new TIntProcedure() { - - @Override - public boolean execute(int rId) { - processor.ids.put(rId, processor.id++); - return true; - } - - }); - return true; - } - - }); - - extensions.put(Extensions.CLUSTERING, new Variant(Bindings.INT_ARRAY, clustering.toArray())); - - long total = processor.startupTime + processor.expandTime - + processor.composedPredicateTime - + processor.composedObjectTime + processor.extentSeedTime - + processor.fullResolveTime + processor.fastResolveTime + - + processor.parentResolveTime + processor.otherStatementTime; - - if (PROFILE) { - System.out.println("startup took " + 1e-9 * processor.startupTime - + "s."); - System.out.println("expand took " + 1e-9 * processor.expandTime - + "s."); - System.out.println("composedPredicates took " + 1e-9 - * processor.composedPredicateTime + "s."); - System.out.println("composedObjects took " + 1e-9 - * processor.composedObjectTime + "s."); - System.out.println("extentSeeding took " + 1e-9 - * processor.extentSeedTime + "s."); - System.out.println("fullResolve took " + 1e-9 - * processor.fullResolveTime + "s."); - System.out.println("fastResolve took " + 1e-9 - * processor.fastResolveTime + "s."); - System.out.println("parentResolve took " + 1e-9 - * processor.parentResolveTime + "s."); - System.out.println("otherStatements took " + 1e-9 - * processor.otherStatementTime + "s."); - System.out.println("total " + 1e-9 * total + "s."); - } - - task.finish(); - - - } - - - public static void getDomain(ReadGraph graph, Map ids, - Collection roots, Map preStatus, Set advisors, - ObjectOutputStream composedStatementsOutput, - ObjectOutputStream otherStatementsOutput, - ObjectOutputStream valueOutput) throws DatabaseException { - - ITask task = ThreadLogger.getInstance().begin("getDomain"); - - DomainProcessor processor = new DomainProcessor(advisors); - - processor.startupTime = System.nanoTime(); - - Layer0 l0 = Layer0.getInstance(graph); - - CollectionSupport cs = graph.getService(CollectionSupport.class); - - processor.ids = ids; - processor.status = cs.createMap(ExtentStatus.class); - processor.weakInverses = cs.createMap(WeakStatus.class); - processor.predicates = cs.createSet(); - processor.composedPredicates = cs.createSet(); - processor.expansionSeeds = cs.createSet(); - - for(Map.Entry entry : preStatus.entrySet()) { - processor.status.put(entry.getKey(), entry.getValue()); - } - -// for (Resource r : excluded) { -// processor.status.put(r, ExtentStatus.EXCLUDED); -// } - - if (!roots.contains(graph.getRootLibrary())) - processor.status.put(graph.getRootLibrary(), ExtentStatus.EXTERNAL); - - for (Resource root : roots) { - processor.status.put(root, ExtentStatus.INTERNAL); - processor.ids.put(root, processor.id++); - for (Resource owner : graph.getObjects(root, l0.IsOwnedBy)) { - processor.status.put(owner, ExtentStatus.EXTERNAL); - } - } - - - processor.expansionSeeds.addAll(roots); - - processor.startupTime = System.nanoTime() - processor.startupTime; - - while (!processor.expansionSeeds.isEmpty()) { - - processor.process(graph, composedStatementsOutput, - otherStatementsOutput, valueOutput); - - } - - long total = processor.startupTime + processor.expandTime - + processor.composedPredicateTime - + processor.composedObjectTime + processor.extentSeedTime - + processor.fullResolveTime + processor.fastResolveTime + - + processor.parentResolveTime + processor.otherStatementTime; - - if (PROFILE) { - System.out.println("startup took " + 1e-9 * processor.startupTime - + "s."); - System.out.println("expand took " + 1e-9 * processor.expandTime - + "s."); - System.out.println("composedPredicates took " + 1e-9 - * processor.composedPredicateTime + "s."); - System.out.println("composedObjects took " + 1e-9 - * processor.composedObjectTime + "s."); - System.out.println("extentSeeding took " + 1e-9 - * processor.extentSeedTime + "s."); - System.out.println("fullResolve took " + 1e-9 - * processor.fullResolveTime + "s."); - System.out.println("fastResolve took " + 1e-9 - * processor.fastResolveTime + "s."); - System.out.println("parentResolve took " + 1e-9 - * processor.parentResolveTime + "s."); - System.out.println("otherStatements took " + 1e-9 - * processor.otherStatementTime + "s."); - System.out.println("total " + 1e-9 * total + "s."); - } - - task.finish(); - - } - -} diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TGConfigurer.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TGConfigurer.java index 350f4ca31..4e23fa575 100644 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TGConfigurer.java +++ b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TGConfigurer.java @@ -13,14 +13,15 @@ import org.simantics.db.common.request.PossibleIndexRoot; import org.simantics.db.common.utils.CommonDBUtils; import org.simantics.db.exception.DatabaseException; import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus; -import org.simantics.db.layer0.util.TransferableGraphConfiguration2.RootSpec; +import org.simantics.db.layer0.util.TransferableGraphConfiguration2.SeedSpec; +import org.simantics.db.layer0.util.TransferableGraphConfiguration2.SeedSpec.SeedSpecType; import org.simantics.layer0.Layer0; public class TGConfigurer { final private ReadGraph graph; - final Collection roots = new ArrayList(); - final Map preStatus = new HashMap(); + final Collection roots = new ArrayList<>(); + final Map preStatus = new HashMap<>(); final boolean ignoreVirtualResources; final boolean validate; @@ -41,13 +42,12 @@ public class TGConfigurer { return this; } - public TGConfigurer roots(Collection rs) throws DatabaseException { + public TGConfigurer roots(Collection rs) throws DatabaseException { Layer0 L0 = Layer0.getInstance(graph); - for(RootSpec root : rs) { + for(SeedSpec root : rs) { Resource nameResource = graph.getPossibleObject(root.resource, L0.HasName); if(nameResource != null) preStatus.put(nameResource, ExtentStatus.EXCLUDED); -// String name = graph.getValue(nameResource, Bindings.STRING); - roots.add(new RootSpec(root.resource, root.name, root.internal, root.type)); + roots.add(new SeedSpec(root.resource, root.name, root.specType, root.type)); preStatus.put(root.resource, ExtentStatus.INTERNAL); } return this; @@ -63,7 +63,7 @@ public class TGConfigurer { if(name == null) name = ""; preStatus.put(nameResource, ExtentStatus.EXCLUDED); } - roots.add(new RootSpec(root, name, true)); + roots.add(new SeedSpec(root, name, SeedSpecType.ROOT)); preStatus.put(root, ExtentStatus.INTERNAL); } return this; @@ -73,8 +73,7 @@ public class TGConfigurer { Layer0 L0 = Layer0.getInstance(graph); for(NamedResource root : rs) { Resource nameResource = graph.getPossibleObject(root.getResource(), L0.HasName); -// String name = graph.getValue(nameResource, Bindings.STRING); - roots.add(new RootSpec(root.getResource(), root.getName(), true)); + roots.add(new SeedSpec(root.getResource(), root.getName(), SeedSpecType.ROOT)); preStatus.put(root.getResource(), ExtentStatus.INTERNAL); preStatus.put(nameResource, ExtentStatus.EXCLUDED); } @@ -91,9 +90,9 @@ public class TGConfigurer { // -Exclude owners if not explicitly defined Layer0 L0 = Layer0.getInstance(graph); - ArrayList rootList = new ArrayList(); - for (RootSpec root : roots) { - if(root.internal) { + ArrayList rootList = new ArrayList<>(); + for (SeedSpec root : roots) { + if(!SeedSpecType.SPECIAL_ROOT.equals(root.specType)) { rootList.add(root.resource); for (Resource owner : graph.getObjects(root.resource, L0.IsOwnedBy)) { ExtentStatus ownerStatus = preStatus.get(owner); diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TGConfigurer2.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TGConfigurer2.java new file mode 100644 index 000000000..f3758aebe --- /dev/null +++ b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TGConfigurer2.java @@ -0,0 +1,78 @@ +package org.simantics.db.layer0.util; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.simantics.db.ReadGraph; +import org.simantics.db.Resource; +import org.simantics.db.common.request.IsParent; +import org.simantics.db.exception.DatabaseException; +import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus; +import org.simantics.db.layer0.util.TransferableGraphConfiguration2.SeedSpec; +import org.simantics.db.layer0.util.TransferableGraphConfiguration2.SeedSpec.SeedSpecType; + +public class TGConfigurer2 { + + final Collection seeds = new ArrayList(); + final Map preStatus = new HashMap(); + final boolean ignoreVirtualResources; + final boolean validate; + + public TGConfigurer2(boolean ignoreVirtualResources, boolean validate) { + this.ignoreVirtualResources = ignoreVirtualResources; + this.validate = validate; + } + + public TGConfigurer2 addInternalSeed(ReadGraph graph, Resource r) throws DatabaseException { + + // If parent is already an internal => skip this + for(SeedSpec spec : seeds) { + if(SeedSpecType.INTERNAL.equals(spec.specType)) { + if(graph.syncRequest(new IsParent(spec.resource, r))) return this; + } + } + // If some existing internal is a child of this one => remove them + List removals = new ArrayList<>(); + for(SeedSpec spec : seeds) { + if(SeedSpecType.INTERNAL.equals(spec.specType)) { + if(graph.syncRequest(new IsParent(r, spec.resource))) removals.add(spec.resource); + } + } + for(Resource removal : removals) { + SeedSpec spec = getSeed(removal); + if(spec != null) { + seeds.remove(spec); + } + } + + seeds.add(new SeedSpec(r, "", SeedSpecType.INTERNAL)); + + return this; + + } + + private SeedSpec getSeed(Resource r) { + for(SeedSpec spec : seeds) { + if(spec.resource.equals(r)) return spec; + } + return null; + } + + public TGConfigurer2 addExternal(Resource r) throws DatabaseException { + preStatus.put(r, ExtentStatus.EXTERNAL); + return this; + } + + public TGConfigurer2 addExcluded(Resource r) throws DatabaseException { + preStatus.put(r, ExtentStatus.EXCLUDED); + return this; + } + + public TransferableGraphConfiguration2 create() throws DatabaseException { + return new TransferableGraphConfiguration2(null, seeds, preStatus, ignoreVirtualResources, validate); + } + +} diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TGRepresentationUtils.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TGRepresentationUtils.java index 0bd71af86..89da10f44 100644 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TGRepresentationUtils.java +++ b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TGRepresentationUtils.java @@ -23,8 +23,9 @@ import org.simantics.db.Statement; import org.simantics.db.common.request.PossibleIndexRoot; import org.simantics.db.exception.DatabaseException; import org.simantics.db.layer0.adapter.impl.EntityInstances.QueryIndex; -import org.simantics.db.layer0.util.ConsistsOfProcess.InternalEntry; +import org.simantics.db.layer0.util.ConsistsOfProcess.ConsistsOfProcessEntry; import org.simantics.db.layer0.util.DomainProcessor3.ExclusionDecision; +import org.simantics.db.layer0.util.TransferableGraphConfiguration2.SeedSpec; import org.simantics.layer0.Layer0; import org.simantics.scl.runtime.function.Function1; import org.simantics.utils.datastructures.Pair; @@ -65,9 +66,9 @@ public class TGRepresentationUtils { return new GUIDExclusionFunction(graph); // The root is OK - check everything beneath - Pair,Set> pair = ConsistsOfProcess.walk(graph, null, Collections.singleton(r), Collections.emptySet(), true); - List entries = pair.first; - for(InternalEntry entry : entries) { + Pair,Set> pair = ConsistsOfProcess.walk(graph, Collections.singleton(SeedSpec.internal(r)), true); + List entries = pair.first; + for(ConsistsOfProcessEntry entry : entries) { if(findByIdentifier(graph, targetRoot, entry.resource)) return new GUIDExclusionFunction(graph); } diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TransferableGraphConfiguration2.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TransferableGraphConfiguration2.java index 2203ecaeb..ec9c8b80c 100644 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TransferableGraphConfiguration2.java +++ b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TransferableGraphConfiguration2.java @@ -16,44 +16,55 @@ import org.simantics.db.common.request.UniqueRead; import org.simantics.db.exception.DatabaseException; import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus; import org.simantics.db.layer0.util.DomainProcessor3.ExclusionDecision; +import org.simantics.db.layer0.util.TransferableGraphConfiguration2.SeedSpec.SeedSpecType; import org.simantics.scl.runtime.function.Function1; public class TransferableGraphConfiguration2 { - - public static class RootSpec { + + public static class SeedSpec { + + public static enum SeedSpecType { + INTERNAL, ROOT, SPECIAL_ROOT + } + public final Resource resource; public final String name; - public final boolean internal; + // Special roots are not normal seeds - e.g. %model + public final SeedSpecType specType; /** * Optional, may be null. */ public final String type; - public RootSpec(Resource resource, String name, boolean internal) { - this(resource, name, internal, null); + public SeedSpec(Resource resource, String name, SeedSpecType specType) { + this(resource, name, specType, null); } - public RootSpec(Resource resource, String name, boolean internal, String type) { + public SeedSpec(Resource resource, String name, SeedSpecType specType, String type) { if (resource == null) throw new NullPointerException("null resource"); if (name == null) throw new NullPointerException("null name"); this.resource = resource; this.name = name; - this.internal = internal; + this.specType = specType; this.type = type; } + public static SeedSpec internal(Resource resource) { + return new SeedSpec(resource, null, SeedSpecType.ROOT); + } + @Override public String toString() { - return "RootSpec[" + name + ", " + resource + ", " + internal + ", " + type + "]"; + return "SeedSpec[" + name + ", " + resource + ", " + specType + ", " + type + "]"; } @Override public int hashCode() { final int prime = 31; int result = 1; - result = prime * result + (internal ? 1231 : 1237); + result = prime * result + specType.hashCode(); result = prime * result + resource.hashCode(); result = prime * result + name.hashCode(); result = prime * result + (type != null ? type.hashCode() : 0); @@ -68,8 +79,8 @@ public class TransferableGraphConfiguration2 { return false; if (getClass() != obj.getClass()) return false; - RootSpec other = (RootSpec) obj; - return internal == other.internal && resource.equals(other.resource) && name.equals(other.name) + SeedSpec other = (SeedSpec) obj; + return specType.equals(other.specType) && resource.equals(other.resource) && name.equals(other.name) && objectEquals(type, other.type); } @@ -81,13 +92,13 @@ public class TransferableGraphConfiguration2 { } } - final public TreeMap baseExtensions = new TreeMap(); + final public TreeMap baseExtensions = new TreeMap<>(); final public Resource indexRoot; - final public Collection roots; + final public Collection seeds; final public Map preStatus; final public boolean ignoreVirtualResources; final public boolean validate; - + /** * true to export values, false to skip the actual * values and only write the resource id. @@ -97,25 +108,25 @@ public class TransferableGraphConfiguration2 { public Collection valueModifiers; public Function1 exclusionFunction; - public TransferableGraphConfiguration2(Resource indexRoot, Collection roots, Map preStatus, boolean ignoreVirtualResources, boolean validate) { + public TransferableGraphConfiguration2(Resource indexRoot, Collection seeds, Map preStatus, boolean ignoreVirtualResources, boolean validate) { this.indexRoot = indexRoot; - this.roots = roots; + this.seeds = seeds; this.preStatus = preStatus; this.ignoreVirtualResources = ignoreVirtualResources; this.validate = validate; this.valueModifiers = null; } - public TransferableGraphConfiguration2(Resource indexRoot, Collection roots, Map preStatus, boolean ignoreVirtualResources) { + public TransferableGraphConfiguration2(Resource indexRoot, Collection roots, Map preStatus, boolean ignoreVirtualResources) { this(indexRoot, roots, preStatus, ignoreVirtualResources, true); } - public TransferableGraphConfiguration2(Resource indexRoot, Collection roots, Map preStatus) { + public TransferableGraphConfiguration2(Resource indexRoot, Collection roots, Map preStatus) { this(indexRoot, roots, preStatus, true); } public TransferableGraphConfiguration2(TransferableGraphConfiguration2 conf) throws DatabaseException { - this(conf.indexRoot, conf.roots, conf.preStatus, conf.ignoreVirtualResources, conf.validate); + this(conf.indexRoot, conf.seeds, conf.preStatus, conf.ignoreVirtualResources, conf.validate); } public TransferableGraphConfiguration2(ReadGraph graph, Resource model, boolean ignoreVirtualResources, boolean validate) throws DatabaseException { @@ -133,12 +144,12 @@ public class TransferableGraphConfiguration2 { public TransferableGraphConfiguration2(ReadGraph graph, Resource model) throws DatabaseException { this(graph, model, true); } - - public TransferableGraphConfiguration2(ReadGraph graph, Collection roots, Collection resourceRoots, Collection exclusions) throws DatabaseException { + + public TransferableGraphConfiguration2(ReadGraph graph, Collection roots, Collection resourceRoots, Collection exclusions) throws DatabaseException { this(new TGConfigurer(graph, true).roots(roots).roots2(resourceRoots).exclusions(exclusions).create()); } - public TransferableGraphConfiguration2(ReadGraph graph, Collection roots, Collection resourceRoots, Collection exclusions, boolean ignoreVirtualResource, boolean validate) throws DatabaseException { + public TransferableGraphConfiguration2(ReadGraph graph, Collection roots, Collection resourceRoots, Collection exclusions, boolean ignoreVirtualResource, boolean validate) throws DatabaseException { this(new TGConfigurer(graph, ignoreVirtualResource, validate).roots(roots).roots2(resourceRoots).exclusions(exclusions).create()); } @@ -149,18 +160,18 @@ public class TransferableGraphConfiguration2 { public TransferableGraphConfiguration2 perform(ReadGraph graph) throws DatabaseException { return new TransferableGraphConfiguration2(graph, translate(roots), Collections.emptyList(), exclusions, ignoreVirtualResource, validate); } - + }); } - public static TransferableGraphConfiguration2 createWithNames2(RequestProcessor processor, final Collection roots, final Collection exclusions, final boolean ignoreVirtualResource, final boolean validate) throws DatabaseException { + public static TransferableGraphConfiguration2 createWithNames2(RequestProcessor processor, final Collection roots, final Collection exclusions, final boolean ignoreVirtualResource, final boolean validate) throws DatabaseException { return processor.sync(new UniqueRead() { @Override public TransferableGraphConfiguration2 perform(ReadGraph graph) throws DatabaseException { return new TransferableGraphConfiguration2(graph, roots, Collections.emptyList(), exclusions, ignoreVirtualResource, validate); } - + }); } @@ -173,9 +184,9 @@ public class TransferableGraphConfiguration2 { @Override public TransferableGraphConfiguration2 perform(ReadGraph graph) throws DatabaseException { - return new TransferableGraphConfiguration2(graph, Collections.emptyList(), roots, exclusions); + return new TransferableGraphConfiguration2(graph, Collections.emptyList(), roots, exclusions); } - + }); } @@ -183,10 +194,10 @@ public class TransferableGraphConfiguration2 { return createWithResources(processor, Collections.singletonList(model), Collections.emptyList()); } - private static Collection translate(Collection roots) { - ArrayList result = new ArrayList(); - for(NamedResource nr : roots) result.add(new RootSpec(nr.getResource(), nr.getName(), true)); + private static Collection translate(Collection roots) { + ArrayList result = new ArrayList<>(); + for(NamedResource nr : roots) result.add(new SeedSpec(nr.getResource(), nr.getName(), SeedSpecType.ROOT)); return result; } - + } diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TransferableGraphRequest.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TransferableGraphRequest.java deleted file mode 100644 index 6b31692a1..000000000 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TransferableGraphRequest.java +++ /dev/null @@ -1,748 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2007, 2010 Association for Decentralized Information Management - * in Industry THTH ry. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Eclipse Public License v1.0 - * which accompanies this distribution, and is available at - * http://www.eclipse.org/legal/epl-v10.html - * - * Contributors: - * VTT Technical Research Centre of Finland - initial API and implementation - *******************************************************************************/ -package org.simantics.db.layer0.util; - -import gnu.trove.list.array.TIntArrayList; - -import java.io.ByteArrayInputStream; -import java.io.DataOutput; -import java.io.DataOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ConcurrentSkipListSet; -import java.util.concurrent.atomic.AtomicInteger; - -import org.apache.commons.io.output.DeferredFileOutputStream; -import org.simantics.databoard.Bindings; -import org.simantics.databoard.Databoard; -import org.simantics.databoard.binding.mutable.Variant; -import org.simantics.databoard.serialization.Serializer; -import org.simantics.db.AsyncReadGraph; -import org.simantics.db.ReadGraph; -import org.simantics.db.Resource; -import org.simantics.db.Statement; -import org.simantics.db.common.StandardStatement; -import org.simantics.db.common.utils.NameUtils; -import org.simantics.db.exception.DatabaseException; -import org.simantics.db.exception.ValidationException; -import org.simantics.db.layer0.adapter.Instances; -import org.simantics.db.layer0.adapter.SubgraphAdvisor; -import org.simantics.db.layer0.adapter.SubgraphExtent; -import org.simantics.db.layer0.adapter.SubgraphExtent.Classifier; -import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus; -import org.simantics.db.procedure.AsyncProcedure; -import org.simantics.db.request.AsyncRead; -import org.simantics.db.request.Read; -import org.simantics.db.service.CollectionSupport; -import org.simantics.db.service.QueryControl; -import org.simantics.db.service.QueryControl.ControlProcedure; -import org.simantics.db.service.SerialisationSupport; -import org.simantics.graph.representation.External; -import org.simantics.graph.representation.Identity; -import org.simantics.graph.representation.Root; -import org.simantics.graph.representation.TransferableGraph1; -import org.simantics.graph.representation.Value; -import org.simantics.layer0.Layer0; -import org.simantics.operation.Layer0X; -import org.simantics.utils.datastructures.Pair; -import org.simantics.utils.datastructures.Triple; - -/** - * IsComposedOf objects are always in domain - * Ordered set elements are always in domain - * r is in domain if for all statements (s, IsRelatedTo, r) s is in domain - * - * IsWeaklyRelatedTo statements where subject is in domain are only accepted if object is in domain or has URI - * - * @deprecated in favor of {@link TransferableGraphRequest2} - */ -public class TransferableGraphRequest implements Read { - - public static String LOG_FILE = "transferableGraph.log"; - final static private boolean LOG = false; - final static private boolean DEBUG = false; - final static private boolean PROFILE = false; - -// final private Collection> roots; -// private Collection preExternals = Collections.emptyList(); -// private Resource model; - - private TransferableGraphConfiguration configuration; - - static DataOutput log; - - static { - - if (LOG) { - try { - FileOutputStream stream = new FileOutputStream(LOG_FILE); - log = new DataOutputStream(stream); - } catch (FileNotFoundException e) { - e.printStackTrace(); - } - } - - } - - private static void log(String line) { - if (LOG) { - try { - log.writeUTF(line + "\n"); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - - - - public TransferableGraphRequest(Collection> roots, Resource model) { - - configuration = new TransferableGraphConfiguration(); - configuration.roots = roots; - configuration.model = model; - -// this.roots = roots; -// this.model = model; - - } - - - public TransferableGraphRequest(Collection> roots) { - this(roots, null); - } - - public TransferableGraphRequest(TransferableGraphConfiguration conf) { - this.configuration = conf; - } - - Layer0 l0; - - TIntArrayList inverses = new TIntArrayList(); - int statements[]; - int statementIndex = 0; - Map ids; - Map values; - TIntArrayList externalParents = new TIntArrayList(); - ArrayList externalNames = new ArrayList(); -// ConcurrentLinkedQueue> values = new ConcurrentLinkedQueue>(); - int id = 0; - int internalCount; - - int indent = 0; - - private Serializer variantSerializer; - - private boolean validateExternal(Resource r) { - if(configuration.disallowedExternals != null) { - System.err.println("validateExternal agains " + configuration.disallowedExternals); - return !configuration.disallowedExternals.contains(r); - } - return true; - } - - public int getId(ReadGraph graph, Resource r, Resource predicate) throws DatabaseException { -// for(int i=0;i parents = graph.getObjects(r, l0.PartOf); - if(parents.size() != 1) { - if(parents.size() == 0) { - Resource inv = graph.getPossibleObject(r, l0.InverseOf); - if(inv != null) { - ++indent; - if(DEBUG) System.out.println("inverse " + NameUtils.getSafeName(graph, inv)); - int invId = getId(graph, inv, null); - externalParents.add(invId); - --indent; - externalNames.add("@inverse"); - ids.put(r, id); - return id++; - } - } -// for(Statement stat : graph.getStatements(r, b.IsWeaklyRelatedTo)) -// System.out.println(NameUtils.getSafeName(graph, stat.getPredicate()) -// + " -> " + NameUtils.getSafeName(graph, stat.getObject())); -// if(predicate != null) { -// if(!graph.isSubrelationOf(predicate, b.IsRelatedTo)) { -// return -2; -// } -// } - throw new ValidationException("Reference to external resource " - + NameUtils.getSafeName(graph, r, true) + " without unique uri (" + parents.size() + " parents)."); - } - for(Resource p : parents) { -// System.out.println("Parent " + NameUtils.getSafeName(graph, p)); - ++indent; - if(!validateExternal(p)) throw new ValidationException("References to '" + graph.getURI(p) + "' are not allowed."); - externalParents.add(getId(graph, p, null)); - --indent; - } -// System.out.println("Request named for " + GraphUtils.getReadableName(g, r)); -// String name = graph.getPossibleRelatedValue(r, b.HasName); - externalNames.add((String)graph.getRelatedValue(r, l0.HasName)); -// if(name != null) externalNames.add(name); -// else externalNames.add("@resource"); - ids.put(r, id); - return id++; - } - } - - public void addId(ReadGraph graph, Resource r, Resource predicate) throws DatabaseException { - statements[statementIndex++] = getId(graph, r, predicate); - } - - private void searchStatementPart(ReadGraph graph, final ConcurrentSkipListSet statementSet, final ArrayList> part, final Set extents) throws DatabaseException { - - final SubgraphExtent.Callback callback = new SubgraphExtent.Callback() { - - @Override - public void statement(Statement statement, boolean accept) { - if(accept) { - statementSet.add(statement); - } - } - - }; - - graph.syncRequest(new AsyncRead() { - - @Override - public int threadHash() { - return hashCode(); - } - - @Override - public void perform(AsyncReadGraph graph, AsyncProcedure procedure) { - - QueryControl control = graph.getService(QueryControl.class); - - int slice = (int)(part.size() / control.getAmountOfQueryThreads()) + 1; - - final Pair[] rootArray = (Pair[])part.toArray(new Pair[part.size()]); - for(int i=0;i r = (Pair)rootArray[index]; - - final AtomicInteger position = new AtomicInteger(0); - final SubgraphExtent.Classifier[] classifiers = new SubgraphExtent.Classifier[extents.size()]; - - for(SubgraphExtent extent : extents) { - - extent.accept(graph, r.first, new AsyncProcedure() { - - public void execute(AsyncReadGraph graph, Classifier c) { - - int current = position.incrementAndGet(); - classifiers[current-1] = c; - - if(current == extents.size()) { - - for(Triple statement : r.second) { - if(!r.first.isPersistent()) continue; - for(Classifier classifier : classifiers) { - classifier.classify(graph, new StandardStatement(r.first, (Resource)statement.first, (Resource)statement.second), (ExtentStatus)statement.third, callback); - } - } - -// graph.forEachDirectStatement(r, new AsyncMultiProcedure() { -// -// @Override -// public void exception(AsyncReadGraph graph, Throwable throwable) { -// throwable.printStackTrace(); -// } -// -// @Override -// public void execute(AsyncReadGraph graph, Statement statement) { -// } -// -// @Override -// public void finished(AsyncReadGraph graph) { -// } -// -// }); - - } - - } - - public void exception(AsyncReadGraph graph, Throwable throwable) { - throwable.printStackTrace(); - } - - }, callback); - - } - - } - - } - - }); - - } - - procedure.execute(graph, false); - - } - - @Override - public int getFlags() { - return 0; - } - - }); - - } - - class StatementSetBuilder { - - private int SLICE = 100000; - - final private CollectionSupport cs; - final private LinkedList> sets = new LinkedList>(); - - private Collection current; - - StatementSetBuilder(CollectionSupport cs) { - this.cs = cs; - current = cs.createStatementList(); - } - - LinkedList> get() { - sets.add(current); - return sets; - } - - void add(Statement stm) { - current.add(stm); - if(current.size() == SLICE) { - sets.add(current); - current = cs.createStatementList(); - } - } - - int size() { - int result = 0; - for(Collection c : sets) result += c.size(); - return result; - } - - - void addAll(Collection ss) { - for(Statement s : ss) add(s); - } - - } - - private void searchStatements(ReadGraph graph, ObjectInputStream composedInput, ObjectInputStream statementInput, ObjectInputStream valueInput, final Set extents) throws DatabaseException, IOException { - - CollectionSupport cs = graph.getService(CollectionSupport.class); - - ConcurrentSkipListSet statementSet = new ConcurrentSkipListSet(); - - StatementSetBuilder builder = new StatementSetBuilder(cs); - - SerialisationSupport support = graph.getService(SerialisationSupport.class); - - while(composedInput.available() > 0) { - - int s = composedInput.readInt(); - Resource subject = support.getResource(s); - - int size = composedInput.readInt(); - for(int i=0;i 0) { - - ArrayList> stms = new ArrayList>(); - - int counter = 0; - - while(statementInput.available() > 0 && counter++ < 1000) { - - int s = statementInput.readInt(); - Resource subject = support.getResource(s); - int size = statementInput.readInt(); - Triple[] list = new Triple[size]; - stms.add(Pair.make(subject, list)); - for(int i=0;i(support.getResource(p), support.getResource(o), ExtentStatus.EXTERNAL); - } - } - - searchStatementPart(graph, statementSet, stms, extents); - - } - - if(DEBUG) { - for(Statement stm : statementSet) - System.out.println("other " + NameUtils.toString(graph, stm)); - } - - builder.addAll(statementSet); - - if(DEBUG) System.out.println("total " + statementSet.size() + " statements found."); - - statementSet = null; - - LinkedList> statementSets = builder.get(); - - Map inverses = cs.createMap(Resource.class); - Set predicateSet = cs.createSet(); - for(Collection set : statementSets) - for(Statement s : set) { - Resource predicate = s.getPredicate(); - if(predicateSet.add(predicate)) { - Resource inverse = graph.getPossibleInverse(predicate); - inverses.put(predicate, inverse); - } - } - - predicateSet = null; - - statements = new int[4 * builder.size()]; - - while(!statementSets.isEmpty()) { - Collection set = statementSets.pop(); - for(Statement s : set) { - Resource subject = s.getSubject(); - Resource predicate = s.getPredicate(); - Resource object = s.getObject(); - int subjectId = ids.get(subject); - int objectId = getId(graph, object, predicate); - // The statement can be denied still - if(objectId != -2) { - statements[statementIndex++] = subjectId; - addId(graph, predicate, null); - Resource inverse = inverses.get(predicate); - if(inverse != null) { - if(LOG) log("[STM] " + subject.getResourceId() + ", " + predicate.getResourceId() + ", " + inverse.getResourceId() + ", " + object.getResourceId()); - addId(graph, inverse, null); - } else { - if(LOG) log("[STM] " + subject.getResourceId() + ", " + predicate.getResourceId() + ", -1, " + object.getResourceId()); - statements[statementIndex++] = -1; - } - statements[statementIndex++] = objectId; - - - - } else { - System.out.println("denied"); - } - } - } - - inverses = null; - - while(valueInput.available() > 0) { - - int s = valueInput.readInt(); - Resource subject = support.getResource(s); - int valueSize = valueInput.readInt(); - byte[] value = new byte[valueSize]; - valueInput.readFully(value); - Variant variant = (Variant) variantSerializer.deserialize(value); - values.put(subject, variant); - - } - - } - - public void setExternals(Collection rs) { - configuration.externals = rs; - } - - @Override - public TransferableGraph1 perform(ReadGraph graph) throws DatabaseException { - - this.variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT); - - this.l0 = Layer0.getInstance(graph); - Layer0X L0X = Layer0X.getInstance(graph); - - long total = System.nanoTime(); - - long startupTime = System.nanoTime(); - - CollectionSupport cs = graph.getService(CollectionSupport.class); - - ids = cs.createMap(Integer.class); - values = cs.createMap(byte[].class); - - ArrayList rootResources = new ArrayList(); - for(Pair p : configuration.roots) rootResources.add(p.first); - -// if(model == null) { -// SimulationResource SIMU = SimulationResource.getInstance(graph); -// for(Resource root : rootResources) { -// if(graph.isInstanceOf(root, SIMU.Model)) { -// model = root; -// break; -// } -// model = graph.syncRequest(new TypedParent(root, SIMU.Model)); -// if(model != null) break; -// } -// } - - Map preStatus = new HashMap(); - - for(Resource root : rootResources) { - Resource name = graph.getPossibleObject(root, l0.HasName); - if(name != null) { - preStatus.put(name, ExtentStatus.EXCLUDED); - } - } - - for(Resource r : configuration.externals) preStatus.put(r, ExtentStatus.EXTERNAL); - - Set extents = configuration.extents; - if(extents == null) { - - Instances extentInstances = graph.adapt(L0X.SubgraphExtent, Instances.class); - Collection extentResources = extentInstances.find(graph, configuration.model); - extents = new HashSet(); - for(Resource r : extentResources) { - extents.add(graph.getPossibleAdapter(r, SubgraphExtent.class)); - } - - if(DEBUG) { - System.out.println("Extents " + extents.size()); - for(Resource extent : extentResources) System.out.println("-" + NameUtils.getSafeName(graph, extent)); - } - - } - - Set advisors = configuration.advisors; - if(advisors == null) { - - Instances advisorInstances = graph.adapt(L0X.SubgraphAdvisor, Instances.class); - Collection advisorResources = advisorInstances.find(graph, configuration.model); - advisors = new HashSet(); - for(Resource r : advisorResources) { - advisors.add(graph.getPossibleAdapter(r, SubgraphAdvisor.class)); - } - - if(DEBUG) { - System.out.println("Advisors " + advisors.size()); - for(Resource advisor : advisorResources) System.out.println("-" + NameUtils.getSafeName(graph, advisor)); - } - - } - - long startupTimeEnd = System.nanoTime(); - - long domainTime = System.nanoTime(); - - String composedStatements = "composed" + UUID.randomUUID().toString(); - String otherStatements = "other" + UUID.randomUUID().toString(); - String valueFileName = "value" + UUID.randomUUID().toString(); - - File composedStatementsFile = new File(composedStatements); - File otherStatementsFile = new File(otherStatements); - File valueFile = new File(valueFileName); - - if(DEBUG) - System.out.println("getDomain writes " + composedStatementsFile.getAbsolutePath()); - - try { - - DeferredFileOutputStream composedStatementsStream = new DeferredFileOutputStream(5 * 1024*1024, composedStatementsFile); - DeferredFileOutputStream otherStatementsStream = new DeferredFileOutputStream(1024*1024, otherStatementsFile); - DeferredFileOutputStream valueStream = new DeferredFileOutputStream(1024*1024, valueFile); - - ObjectOutputStream composedStatementsOutput = new ObjectOutputStream(composedStatementsStream); - ObjectOutputStream otherStatementsOutput = new ObjectOutputStream(otherStatementsStream); - ObjectOutputStream valueOutput = new ObjectOutputStream(valueStream); - - Subgraphs.getDomain(graph, ids, rootResources, preStatus, advisors, composedStatementsOutput, otherStatementsOutput, valueOutput); - id = ids.size(); - - composedStatementsOutput.flush(); - otherStatementsOutput.flush(); - valueOutput.flush(); - composedStatementsStream.close(); - otherStatementsStream.close(); - valueStream.close(); - - long domainTimeEnd = System.nanoTime(); - - long fileTime = System.nanoTime(); - - internalCount = id; - - ids.put(graph.getRootLibrary(), id++); - externalNames.add("http:/"); - externalParents.add(-1); - -// if(model == null) return null; - - InputStream composedStatementsInputStream = null; - InputStream otherStatementsInputStream = null; - InputStream valueInputStream = null; - - if(composedStatementsStream.isInMemory()) { - composedStatementsInputStream = new ByteArrayInputStream(composedStatementsStream.getData()); - } else { - composedStatementsInputStream = new FileInputStream(composedStatementsFile); - } - - if(otherStatementsStream.isInMemory()) { - otherStatementsInputStream = new ByteArrayInputStream(otherStatementsStream.getData()); - } else { - otherStatementsInputStream = new FileInputStream(otherStatementsFile); - } - - if(valueStream.isInMemory()) { - valueInputStream = new ByteArrayInputStream(valueStream.getData()); - } else { - valueInputStream = new FileInputStream(valueFile); - } - - composedStatementsStream = null; - otherStatementsStream = null; - valueStream = null; - - ObjectInputStream composedStatementsInput = new ObjectInputStream(composedStatementsInputStream); - ObjectInputStream otherStatementsInput = new ObjectInputStream(otherStatementsInputStream); - ObjectInputStream valueInput = new ObjectInputStream(valueInputStream); - - long fileTimeEnd = System.nanoTime(); - - long statementTime = System.nanoTime(); - searchStatements(graph, composedStatementsInput, otherStatementsInput, valueInput, extents); - long statementTimeEnd = System.nanoTime(); - - // domainParts = null; - - long buildTime = System.nanoTime(); - - int resourceCount = ids.size(); - - Identity[] identityArray; - { // Identities - ArrayList identities = new ArrayList(); - - for(Pair r : configuration.roots) { - Resource type = graph.getPossibleType(r.first, l0.Entity); - if(type == null) type = l0.Entity; - identities.add(new Identity( - ids.get(r.first), - new Root(r.second, graph.getURI(type)) - )); - } - - int internalsPlusExternals = ids.size(); - for(int i = internalCount; i < internalsPlusExternals ; i++) { - int parent = externalParents.get(i - internalCount); - String name = externalNames.get(i - internalCount); - identities.add(new Identity( - i, - new External(parent, name) - )); - } - identityArray = identities.toArray(new Identity[identities.size()]); - } - - Value[] valueArray = new Value[values.size()]; - { // Values - int index = 0; - for(Map.Entry entry : values.entrySet()) { - if(LOG) log("[VALUE] " + entry.getKey().getResourceId()); - int r = getId(graph, entry.getKey(), null); - valueArray[index++] = new Value(r, entry.getValue()); - } - } - ids = null; - values = null; - - long buildTimeEnd = System.nanoTime(); - - TransferableGraph1 result = - new TransferableGraph1(resourceCount, - identityArray, - statements, - valueArray); - - if(DEBUG) { - System.out.println("transferable graph content: " + result); - } - - long totalEnd = System.nanoTime(); - - if(PROFILE) { - System.out.println("startup in " + 1e-9*(startupTimeEnd - startupTime) + "s."); - System.out.println("domain was found in " + 1e-9*(domainTimeEnd - domainTime) + "s."); - System.out.println("files were written in " + 1e-9*(fileTimeEnd - fileTime) + "s."); - System.out.println("statements were found in " + 1e-9*(statementTimeEnd - statementTime) + "s."); - System.out.println("tg was built in " + 1e-9*(buildTimeEnd - buildTime) + "s."); - System.out.println("total time for building subgraph was " + 1e-9*(totalEnd-total) + "s."); - } - - return result; - } catch (IOException e) { - e.printStackTrace(); - } - - return null; - - } - -} diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TransferableGraphRequest2.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TransferableGraphRequest2.java deleted file mode 100644 index c95a56558..000000000 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TransferableGraphRequest2.java +++ /dev/null @@ -1,509 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2007, 2010 Association for Decentralized Information Management - * in Industry THTH ry. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Eclipse Public License v1.0 - * which accompanies this distribution, and is available at - * http://www.eclipse.org/legal/epl-v10.html - * - * Contributors: - * VTT Technical Research Centre of Finland - initial API and implementation - *******************************************************************************/ -package org.simantics.db.layer0.util; - -import gnu.trove.list.array.TIntArrayList; -import gnu.trove.map.TIntObjectMap; -import gnu.trove.map.hash.TIntIntHashMap; -import gnu.trove.map.hash.TIntObjectHashMap; -import gnu.trove.procedure.TIntObjectProcedure; -import gnu.trove.set.hash.TIntHashSet; - -import java.io.ByteArrayInputStream; -import java.io.DataOutput; -import java.io.DataOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.lang.management.ManagementFactory; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.TreeMap; -import java.util.UUID; - -import org.apache.commons.io.output.DeferredFileOutputStream; -import org.simantics.databoard.Bindings; -import org.simantics.databoard.binding.mutable.Variant; -import org.simantics.db.ReadGraph; -import org.simantics.db.Resource; -import org.simantics.db.common.utils.NameUtils; -import org.simantics.db.exception.DatabaseException; -import org.simantics.db.exception.ValidationException; -import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus; -import org.simantics.db.request.Read; -import org.simantics.db.service.ClusterControl; -import org.simantics.db.service.ClusterControl.ClusterState; -import org.simantics.db.service.SerialisationSupport; -import org.simantics.graph.representation.External; -import org.simantics.graph.representation.Identity; -import org.simantics.graph.representation.Root; -import org.simantics.graph.representation.TransferableGraph1; -import org.simantics.graph.representation.Value; -import org.simantics.layer0.Layer0; -import org.simantics.utils.datastructures.Pair; - -/** - * @deprecated in favor of {@link ModelTransferableGraphSourceRequest} - */ -@Deprecated -public class TransferableGraphRequest2 implements Read { - - public static String LOG_FILE = "transferableGraph.log"; - final static private boolean LOG = false; - final static private boolean DEBUG = false; - final static private boolean PROFILE = false; - - private TransferableGraphConfiguration configuration; - - static DataOutput log; - - static { - - if (LOG) { - try { - FileOutputStream stream = new FileOutputStream(LOG_FILE); - log = new DataOutputStream(stream); - } catch (FileNotFoundException e) { - e.printStackTrace(); - } - } - - } - - private static void log(String line) { - if (LOG) { - try { - log.writeUTF(line + "\n"); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - - - - public TransferableGraphRequest2(Collection> roots, Resource model) { - - configuration = new TransferableGraphConfiguration(); - configuration.roots = roots; - configuration.model = model; - - } - - - public TransferableGraphRequest2(Collection> roots) { - this(roots, null); - } - - public TransferableGraphRequest2(TransferableGraphConfiguration conf) { - this.configuration = conf; - } - - Layer0 L0; - - TIntArrayList inverses = new TIntArrayList(); - int statements[]; - int statementIndex = 0; - TIntIntHashMap ids; - TIntObjectMap values; - TIntArrayList externalParents = new TIntArrayList(); - ArrayList externalNames = new ArrayList(); - int id = 0; - int internalCount; - - int indent = 0; - - private SerialisationSupport support; - - private boolean validateExternal(Resource r) { - if(configuration.disallowedExternals != null) { - System.err.println("validateExternal agains " + configuration.disallowedExternals); - return !configuration.disallowedExternals.contains(r); - } - return true; - } - - private Resource getResource(int r) throws DatabaseException { - return support.getResource(r); - } - - public int getInternalId(int r) { - return ids.get(r); - } - - public int getId(ReadGraph graph, int r, int predicate) throws DatabaseException { - if(ids.containsKey(r)) { - int ret = ids.get(r); - if(ret == -1) { - for(int i=0;i<=indent;++i) - System.out.print(" "); - System.out.println("Cycle!!!"); // with " + GraphUtils.getReadableName(g, r)); - } - return ret; - } - else { - Collection parents = graph.getObjects(getResource(r), L0.PartOf); - if(parents.size() != 1) { - throw new ValidationException("Reference to external resource " - + NameUtils.getSafeName(graph, getResource(r), true) + " without unique uri (" + parents.size() + " parents)."); - } - for(Resource p : parents) { - ++indent; - if(!validateExternal(p)) throw new ValidationException("References to '" + graph.getURI(p) + "' are not allowed."); - externalParents.add(getId(graph, support.getTransientId(p), 0)); - --indent; - } - externalNames.add((String)graph.getRelatedValue(getResource(r), L0.HasName)); - ids.put(r, id); - return id++; - } - } - - public void addId(ReadGraph graph, int r, int predicate) throws DatabaseException { - statements[statementIndex++] = getId(graph, r, predicate); - } - - public void setExternals(Collection rs) { - configuration.externals = rs; - } - - @Override - public TransferableGraph1 perform(ReadGraph graph) throws DatabaseException { - - support = graph.getService(SerialisationSupport.class); - - this.L0 = Layer0.getInstance(graph); - - long total = System.nanoTime(); - - long startupTime = System.nanoTime(); - - ClusterControl cc = graph.getService(ClusterControl.class); - - ids = new TIntIntHashMap(); - values = new TIntObjectHashMap(); - - ArrayList rootResources = new ArrayList(); - for(Pair p : configuration.roots) rootResources.add(p.first); - - Map preStatus = new HashMap(); - - for(Resource root : rootResources) { - Resource name = graph.getPossibleObject(root, L0.HasName); - if(name != null) { - preStatus.put(name, ExtentStatus.EXCLUDED); - } - } - - for(Resource r : configuration.externals) preStatus.put(r, ExtentStatus.EXTERNAL); - - long startupTimeEnd = System.nanoTime(); - - long domainTime = System.nanoTime(); - - String otherStatements = "other" + UUID.randomUUID().toString(); - String valueFileName = "value" + UUID.randomUUID().toString(); - - File otherStatementsFile = new File(otherStatements); - File valueFile = new File(valueFileName); - - try { - - DeferredFileOutputStream otherStatementsStream = new DeferredFileOutputStream(1024*1024, otherStatementsFile); - DeferredFileOutputStream valueStream = new DeferredFileOutputStream(1024*1024, valueFile); - - ObjectOutputStream otherStatementsOutput = new ObjectOutputStream(otherStatementsStream); - ObjectOutputStream valueOutput = new ObjectOutputStream(valueStream); - - ClusterState clusterState = cc.getClusterState(); - - TIntHashSet excludedShared = new TIntHashSet(); - - TreeMap extensions = new TreeMap(); - - Subgraphs.getDomain2(graph, ids, rootResources, preStatus, configuration.specials, otherStatementsOutput, valueOutput, extensions, excludedShared); - - id = ids.size(); - - cc.restoreClusterState(clusterState); - -// dumpHeap("domain.hprof"); - - otherStatementsOutput.flush(); - valueOutput.flush(); - otherStatementsStream.close(); - valueStream.close(); - - long domainDuration = System.nanoTime() - domainTime; - System.err.println("Analysed graph in " + 1e-9*domainDuration + "s."); - - internalCount = id; - - ids.put(support.getTransientId(graph.getResource("http:/")), id++); - externalNames.add("http:/"); - externalParents.add(-1); - - InputStream otherStatementsInputStream = null; - InputStream valueInputStream = null; - - if(otherStatementsStream.isInMemory()) { - otherStatementsInputStream = new ByteArrayInputStream(otherStatementsStream.getData()); - } else { - otherStatementsInputStream = new FileInputStream(otherStatementsFile); - } - - if(valueStream.isInMemory()) { - valueInputStream = new ByteArrayInputStream(valueStream.getData()); - } else { - valueInputStream = new FileInputStream(valueFile); - } - - otherStatementsStream = null; - valueStream = null; - - ObjectInputStream otherStatementsInput = new ObjectInputStream(otherStatementsInputStream); - ObjectInputStream valueInput = new ObjectInputStream(valueInputStream); - - long statementTime = System.nanoTime(); - - TIntArrayList statementSet = new TIntArrayList(); - - while(otherStatementsInput.available() > 0) { - - int s = otherStatementsInput.readInt(); - - boolean exclude = !ids.contains(s); - - int size = otherStatementsInput.readInt(); - for(int i=0;i0;i-=3) { - - if(trim-- == 0) { - statementSet.remove(i, statementSet.size()-i); - statementSet.trimToSize(); - trim = Math.max(65536,statementSet.size()/12); - } - - int s = statementSet.getQuick(i-3); - int p = statementSet.getQuick(i-2); - int o = statementSet.getQuick(i-1); - - int subjectId = ids.get(s); - if(subjectId >= internalCount) System.err.println("Statement for external: " + s + " " + p + " " + o); - - int objectId = getId(graph, o, p); - // The statement can be denied still - if(objectId != -2) { - tgStatements.add(subjectId); - tgStatements.add(getId(graph, p, 0)); - int inverse = inverses.get(p); - if(inverse != 0) { - tgStatements.add(getId(graph, inverse, 0)); - } else { - tgStatements.add(-1); - } - tgStatements.add(objectId); - } else { - System.out.println("denied"); - } - - } - - statements = tgStatements.toArray(); - - long statementDuration = System.nanoTime() - statementTime; - System.err.println("Built transferable statements in " + 1e-9*statementDuration + "s."); - - inverses = null; - - while(valueInput.available() > 0) { - - int s = valueInput.readInt(); -// Resource subject = support.getResource(s); - int valueSize = valueInput.readInt(); - byte[] value = new byte[valueSize]; - valueInput.readFully(value); - Variant variant = (Variant)Bindings.VARIANT.serializer().deserialize(value); - values.put(s, variant); - - } - - int resourceCount = ids.size(); - - Identity[] identityArray; - { // Identities - ArrayList identities = new ArrayList(); - - for(Pair r : configuration.roots) { - Resource type = graph.getPossibleType(r.first, L0.Entity); - if(type == null) type = L0.Entity; - identities.add(new Identity( - ids.get(support.getTransientId(r.first)), - new Root(r.second, graph.getURI(type)) - )); - } - - int internalsPlusExternals = ids.size(); - for(int i = internalCount; i < internalsPlusExternals ; i++) { - int parent = externalParents.get(i - internalCount); - String name = externalNames.get(i - internalCount); - identities.add(new Identity( - i, - new External(parent, name) - )); - } - identityArray = identities.toArray(new Identity[identities.size()]); - } - - final Value[] valueArray = new Value[values.size()]; - { // Values - values.forEachEntry(new TIntObjectProcedure() { - - int index = 0; - - @Override - public boolean execute(int subject, Variant bytes) { - //if(LOG) log("[VALUE] " + entry.getKey().getResourceId()); - int r = getInternalId(subject); - if(r==-1) System.err.println("No id for value resource " + subject); - else valueArray[index++] = new Value(r, bytes); - return true; - } - - }); - } - ids = null; - values = null; - - TransferableGraph1 result = - new TransferableGraph1(resourceCount, - identityArray, - statements, - valueArray, extensions); - - if(DEBUG) { - System.out.println("transferable graph content: " + result); - } - - long totalEnd = System.nanoTime(); - - if(PROFILE) { - System.out.println("startup in " + 1e-9*(startupTimeEnd - startupTime) + "s."); - System.out.println("domain was found in " + 1e-9*(domainDuration) + "s."); - System.out.println("statements were found in " + 1e-9*(statementDuration) + "s."); - System.out.println("total time for building subgraph was " + 1e-9*(totalEnd-total) + "s."); - } - - return result; - - } catch (IOException e) { - e.printStackTrace(); - } catch (Throwable t) { - t.printStackTrace(); - dumpHeap("crash.hprof"); - } - - return null; - - } - - private static void dumpHeap(String path) { - - try { - Object bean = getBean(); - if (bean == null) - return; - - Method m = bean.getClass().getMethod("dumpHeap", String.class, boolean.class); - m.invoke(bean, path, true); - - } catch (IllegalArgumentException e) { - } catch (IllegalAccessException e) { - } catch (SecurityException e) { - } catch (NoSuchMethodException e) { - } catch (InvocationTargetException e) { - } finally { - } - - } - - private static Object getBean() { - Class beanClass = getBeanClass(); - if (beanClass == null) - return null; - try { - Object bean = ManagementFactory.newPlatformMXBeanProxy( - ManagementFactory.getPlatformMBeanServer(), - "com.sun.management:type=HotSpotDiagnostic", - beanClass); - return bean; - } catch (IOException e) { - return null; - } - } - - private static Class getBeanClass() { - try { - Class clazz = Class.forName("com.sun.management.HotSpotDiagnosticMXBean"); - return clazz; - } catch (ClassNotFoundException e) { - return null; - } - } - - -} diff --git a/bundles/org.simantics.graph.db/src/org/simantics/graph/db/StreamingTransferableGraphImportProcess.java b/bundles/org.simantics.graph.db/src/org/simantics/graph/db/StreamingTransferableGraphImportProcess.java index 5472ccfd6..c9b48b325 100644 --- a/bundles/org.simantics.graph.db/src/org/simantics/graph/db/StreamingTransferableGraphImportProcess.java +++ b/bundles/org.simantics.graph.db/src/org/simantics/graph/db/StreamingTransferableGraphImportProcess.java @@ -251,11 +251,13 @@ public class StreamingTransferableGraphImportProcess implements TransferableGrap @Override public Resource createChild(WriteOnlyGraph graph, Resource parent, Resource child, String name) throws DatabaseException { + //System.err.println("child " + parent + " - " + child + " = " + name); if(child == null) child = graph.newResource(); Resource nameResource = graph.newResource(); graph.claim(nameResource, InstanceOf, null, String); graph.claimValue(nameResource, name, WriteBindings.STRING); graph.claim(child, HasName, NameOf, nameResource); + graph.claim(parent, ConsistsOf, PartOf, child); return child; } @@ -457,7 +459,6 @@ public class StreamingTransferableGraphImportProcess implements TransferableGrap } else if(definition instanceof Internal) { Internal def = (Internal)definition; - Resource external = existingInternalMap.get(identity.resource); if(external != null) { handles[identity.resource] = builder.handle(external); diff --git a/bundles/org.simantics.modeling/src/org/simantics/modeling/adapters/CompositeCopyHandler.java b/bundles/org.simantics.modeling/src/org/simantics/modeling/adapters/CompositeCopyHandler.java index 6167334fd..8c91852cd 100644 --- a/bundles/org.simantics.modeling/src/org/simantics/modeling/adapters/CompositeCopyHandler.java +++ b/bundles/org.simantics.modeling/src/org/simantics/modeling/adapters/CompositeCopyHandler.java @@ -26,7 +26,8 @@ import org.simantics.db.function.DbConsumer; import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus; import org.simantics.db.layer0.adapter.impl.DefaultCopyHandler; import org.simantics.db.layer0.util.TransferableGraphConfiguration2; -import org.simantics.db.layer0.util.TransferableGraphConfiguration2.RootSpec; +import org.simantics.db.layer0.util.TransferableGraphConfiguration2.SeedSpec; +import org.simantics.db.layer0.util.TransferableGraphConfiguration2.SeedSpec.SeedSpecType; import org.simantics.diagram.stubs.DiagramResource; import org.simantics.layer0.Layer0; import org.simantics.modeling.ModelingResources; @@ -59,7 +60,7 @@ public class CompositeCopyHandler extends DefaultCopyHandler { ? (Set) resources : new HashSet<>(resources); Set exclusions = new HashSet<>(); Set externals = new HashSet<>(); - List roots = new ArrayList<>(); + List roots = new ArrayList<>(); Resource indexRoot = graph.syncRequest(new PossibleIndexRoot(resources.iterator().next())); if(indexRoot == null) throw new DatabaseException("Composite is not part of any index root"); @@ -143,17 +144,17 @@ public class CompositeCopyHandler extends DefaultCopyHandler { // Include resource as root CompositeInfo info = CompositeInfo.fromResource(graph, resource); - roots.add(new RootSpec(resource, info.getTGName(), true, typeId(graph, L0, indexRootUri, resource))); + roots.add(new SeedSpec(resource, info.getTGName(), SeedSpecType.ROOT, typeId(graph, L0, indexRootUri, resource))); identifierExcluder.accept(resource); // Include components as roots for (Resource child : graph.sync(new ObjectsWithType(resource, L0.ConsistsOf, SR.Component))) { DiagramComponentInfo cinfo = DiagramComponentInfo.fromResource(graph, info, child); identifierExcluder.accept(child); - roots.add(new RootSpec(child, cinfo.getTGName(info), true, typeId(graph, L0, indexRootUri, child))); + roots.add(new SeedSpec(child, cinfo.getTGName(info), SeedSpecType.ROOT, typeId(graph, L0, indexRootUri, child))); } } - roots.add(new RootSpec(indexRoot, "%model", false)); + roots.add(new SeedSpec(indexRoot, "%model", SeedSpecType.SPECIAL_ROOT)); TransferableGraphConfiguration2 config = TransferableGraphConfiguration2.createWithNames2(graph, roots, exclusions, true, false); for (Resource external : externals)