1 /*******************************************************************************
2 * Copyright (c) 2007, 2010 Association for Decentralized Information Management
4 * All rights reserved. This program and the accompanying materials
5 * are made available under the terms of the Eclipse Public License v1.0
6 * which accompanies this distribution, and is available at
7 * http://www.eclipse.org/legal/epl-v10.html
10 * VTT Technical Research Centre of Finland - initial API and implementation
11 *******************************************************************************/
12 package org.simantics.db.layer0.util;
14 import gnu.trove.list.array.TIntArrayList;
15 import gnu.trove.map.hash.TIntIntHashMap;
16 import gnu.trove.map.hash.TLongObjectHashMap;
17 import gnu.trove.procedure.TIntProcedure;
18 import gnu.trove.procedure.TLongObjectProcedure;
19 import gnu.trove.set.TIntSet;
20 import gnu.trove.set.hash.THashSet;
21 import gnu.trove.set.hash.TIntHashSet;
23 import java.io.DataOutput;
24 import java.io.DataOutputStream;
25 import java.io.FileNotFoundException;
26 import java.io.FileOutputStream;
27 import java.io.IOException;
28 import java.io.ObjectOutputStream;
29 import java.util.ArrayList;
30 import java.util.Collection;
31 import java.util.Collections;
32 import java.util.HashSet;
35 import java.util.TreeMap;
36 import java.util.concurrent.ConcurrentLinkedQueue;
37 import java.util.concurrent.ConcurrentSkipListSet;
39 import org.simantics.databoard.Bindings;
40 import org.simantics.databoard.Databoard;
41 import org.simantics.databoard.binding.Binding;
42 import org.simantics.databoard.binding.mutable.Variant;
43 import org.simantics.databoard.serialization.Serializer;
44 import org.simantics.databoard.type.Datatype;
45 import org.simantics.db.AsyncReadGraph;
46 import org.simantics.db.DirectStatements;
47 import org.simantics.db.ReadGraph;
48 import org.simantics.db.Resource;
49 import org.simantics.db.Statement;
50 import org.simantics.db.common.request.AsyncReadRequest;
51 import org.simantics.db.common.request.ReadRequest;
52 import org.simantics.db.common.request.ResourceAsyncRead;
53 import org.simantics.db.common.utils.NameUtils;
54 import org.simantics.db.exception.DatabaseException;
55 import org.simantics.db.layer0.adapter.SubgraphAdvisor;
56 import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus;
57 import org.simantics.db.procedure.AsyncProcedure;
58 import org.simantics.db.request.AsyncRead;
59 import org.simantics.db.service.ClusteringSupport;
60 import org.simantics.db.service.CollectionSupport;
61 import org.simantics.db.service.DirectQuerySupport;
62 import org.simantics.db.service.QueryControl;
63 import org.simantics.db.service.QueryControl.ControlProcedure;
64 import org.simantics.db.service.SerialisationSupport;
65 import org.simantics.db.service.StatementSupport;
66 import org.simantics.db.service.TransferableGraphSupport;
67 import org.simantics.graph.representation.Extensions;
68 import org.simantics.layer0.Layer0;
69 import org.simantics.utils.datastructures.Pair;
70 import org.simantics.utils.threads.logger.ITask;
71 import org.simantics.utils.threads.logger.ThreadLogger;
73 public class Subgraphs {
75 public static String LOG_FILE = "export.log";
76 final static private boolean LOG = false;
77 final static private boolean DEBUG = false;
78 final static private boolean PARENT_DEBUG = DEBUG | false;
79 final static private boolean EXTERNAL_DEBUG = DEBUG | false;
80 final static private boolean ADVISOR_LOG = LOG & false;
81 final static private boolean EXPANSION_LOG = LOG & false;
82 final static private boolean INTERNAL_LOG = LOG & false;
83 final static private boolean COMPOSED_LOG = LOG & false;
84 final static private boolean RESOLVE_LOG = LOG & false;
85 final static private boolean CLASSIFY_LOG = LOG & false;
86 final static private boolean EXTERNAL_LOG = LOG & false;
87 final static private boolean PROFILE = false;
89 static enum WeakStatus {
94 static DataOutput log;
100 FileOutputStream stream = new FileOutputStream(LOG_FILE, false);
101 log = new DataOutputStream(stream);
102 } catch (FileNotFoundException e) {
109 private static void log(String line) {
112 log.write((line + "\n").getBytes());
113 } catch (IOException e) {
119 public static Collection<Resource> getParents(ReadGraph g, Resource r)
120 throws DatabaseException {
121 return getParents(g, r, false);
124 static class FastInternalRequest extends ResourceAsyncRead<Boolean> {
126 final DirectQuerySupport dqs;
127 final ConcurrentLinkedQueue<Resource> queue;
128 final Map<Resource, WeakStatus> weakInverses;
129 final Map<Resource, ExtentStatus> status;
131 public FastInternalRequest(DirectQuerySupport dqs, Resource resource,
132 Map<Resource, ExtentStatus> status,
133 Map<Resource, WeakStatus> weakInverses,
134 ConcurrentLinkedQueue<Resource> queue) {
137 this.status = status;
138 this.weakInverses = weakInverses;
143 public int getFlags() {
148 public void perform(AsyncReadGraph graph, final AsyncProcedure<Boolean> procedure) {
150 dqs.forEachDirectStatement(graph, resource, new AsyncProcedure<DirectStatements>() {
153 public void execute(AsyncReadGraph graph, DirectStatements ss) {
155 for(Statement statement : ss) {
156 if (status.get(statement.getObject()) == ExtentStatus.INTERNAL) continue;
157 WeakStatus status = weakInverses.get(statement.getPredicate());
158 if(status == WeakStatus.WEAK) continue;
159 else if (status == null) {
160 queue.add(statement.getPredicate());
164 procedure.execute(graph, ok);
168 public void exception(AsyncReadGraph graph, Throwable throwable) {
169 throwable.printStackTrace();
178 static class ClassifyStatementsRequest implements AsyncRead<Boolean> {
180 final Set<Resource> schedule;
181 final Map<Resource, WeakStatus> weakMap;
183 public ClassifyStatementsRequest(Set<Resource> schedule, Map<Resource, WeakStatus> weakMap) {
184 this.weakMap = weakMap;
185 this.schedule = schedule;
189 public int threadHash() {
194 public int getFlags() {
199 public void perform(AsyncReadGraph graph, final AsyncProcedure<Boolean> procedure) {
201 for (final Resource p : schedule) {
203 graph.forPossibleInverse(p, new AsyncProcedure<Resource>() {
205 private void register(AsyncReadGraph graph, Resource predicate, Resource superRelation, WeakStatus status) {
206 synchronized (weakMap) {
207 weakMap.put(predicate, status);
208 if(superRelation != null) weakMap.put(superRelation, status);
213 public void exception(AsyncReadGraph graph, Throwable throwable) {
214 throwable.printStackTrace();
218 public void execute(AsyncReadGraph graph, final Resource inverse) {
220 if (inverse == null) {
222 register(graph, p, null, WeakStatus.WEAK);
226 graph.forPossibleSuperrelation(inverse, new AsyncProcedure<Resource>() {
229 public void exception(AsyncReadGraph graph, Throwable throwable) {
230 throwable.printStackTrace();
234 public void execute(AsyncReadGraph graph, final Resource superRelation) {
236 if(superRelation != null && weakMap.containsKey(superRelation)) {
237 register(graph, p, null, weakMap.get(superRelation));
241 graph.forIsSubrelationOf(inverse, graph.getService(Layer0.class).IsRelatedTo, new AsyncProcedure<Boolean>() {
244 public void exception(AsyncReadGraph graph,Throwable throwable) {
245 throwable.printStackTrace();
249 public void execute(AsyncReadGraph graph,Boolean strong) {
250 register(graph, p, superRelation, strong ? WeakStatus.STRONG : WeakStatus.WEAK);
267 procedure.execute(graph, false);
273 private static Collection<Resource> getParents(ReadGraph g, Resource r, boolean isStrong) throws DatabaseException {
275 System.out.println("getParents " + NameUtils.getSafeName(g, r));
277 Layer0 l0 = Layer0.getInstance(g);
279 Collection<Resource> predicates = g.getPredicates(r);
281 // --- Consists Of ----------------------------------------------------
283 if (predicates.contains(l0.PartOf)) {
284 Collection<Resource> parents = g.getObjects(r, l0.PartOf);
285 if (parents.size() == 1)
287 ArrayList<Resource> libraryParents = new ArrayList<Resource>(1);
288 for (Resource p : parents)
289 if (g.isInstanceOf(p, l0.Library))
290 libraryParents.add(p);
291 if (!libraryParents.isEmpty())
292 return libraryParents;
297 // --- Ordered sets ---------------------------------------------------
300 Collection<Resource> parents = null;
301 for (Resource p : predicates)
302 if (g.isInstanceOf(p, l0.OrderedSet) && !p.equals(r)) {
304 parents = new ArrayList<Resource>(1);
307 if (parents != null) {
309 System.out.println("ORDERED SET");
316 return Collections.emptyList();
319 if (predicates.contains(l0.InverseOf)) {
321 Resource inv = g.getInverse(r);
322 return getParents(g, inv, true);
329 * If there are DependsOn parents, then IsRelatedTo parents are discarded
332 HashSet<Resource> result = new HashSet<Resource>();
333 for(Resource predicate : predicates) {
334 if(g.isSubrelationOf(predicate, l0.IsDependencyOf)) result.addAll(g.getObjects(r, predicate));
336 if(!result.isEmpty()) return result;
341 * At this point all Is Related To are parents.
344 for(Resource predicate : predicates) {
345 Resource inv = g.getPossibleInverse(predicate);
347 if(g.isSubrelationOf(inv, l0.IsRelatedTo)) result.addAll(g.getObjects(r, predicate));
356 Collection<Resource> invR = g.getObjects(r, b.IsRelatedTo_Inverse);
357 if (predicates.contains(b.InverseOf)) {
358 if (invR.size() > 1) {
361 .println("###########################################");
362 Resource inv = g.getInverse(r);
363 Collection<Resource> ret = new ArrayList<Resource>();
364 for (Statement pp : g.getStatements(r,
365 b.IsRelatedTo_Inverse))
366 if (!pp.getPredicate().equals(inv)) {
368 System.out.println("<"
369 + NameUtils.getSafeName(g, pp
372 + NameUtils.getSafeName(g, pp
375 + NameUtils.getSafeName(g, pp
376 .getObject()) + ">");
378 ret.add(pp.getObject());
382 // System.out.println("?????????????????");
383 Collection<Resource> invParents = getParents(g,
384 g.getInverse(r), true);
385 if (!invParents.isEmpty())
389 System.out.print("invR");
390 for (Resource res : invR)
391 System.out.print(" " + NameUtils.getSafeName(g, res));
392 System.out.println();
400 // public static String getIdentifier(ReadGraph g, Resource r)
401 // throws DatabaseException {
402 // Layer0 L0 = Layer0.getInstance(g);
403 // if (r.equals(g.getRootLibrary()))
405 // String name = g.getPossibleRelatedValue(r, L0.HasName);
408 // Collection<Resource> parents = getParents(g, r, true);
409 // if (parents.size() != 1)
411 // for (Resource p : parents) {
412 // String parentIdentifier = getIdentifier(g, p);
413 // if (parentIdentifier == null)
415 // return parentIdentifier + "/" + name;
422 static class Expansion extends AsyncReadRequest {
424 final private Collection<Resource> roots;
425 final Collection<DirectStatements>[] results;
426 final Collection<Resource>[] listElements;
428 public Expansion(Collection<Resource> roots, Collection<DirectStatements>[] results, Collection<Resource>[] listElements) {
430 this.results = results;
431 this.listElements = listElements;
435 public void run(AsyncReadGraph graph) {
437 QueryControl control = graph.getService(QueryControl.class);
438 final DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class);
440 final DomainStatementProcedure proc = new DomainStatementProcedure(dqs, graph.getService(StatementSupport.class), graph.getService(Layer0.class), results, listElements);
442 int slice = (int) (roots.size() / control.getAmountOfQueryThreads()) + 1;
444 final Resource[] rootArray = roots.toArray(Resource.NONE);
445 for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
447 final int start = i * slice;
448 final int end = Math.min(start + slice, rootArray.length);
450 control.schedule(graph, i, new ControlProcedure() {
453 public void execute(AsyncReadGraph graph) {
454 for (int index = start; index < end; index++) {
455 dqs.forEachDirectStatement(graph, rootArray[index], proc);
467 public int getFlags() {
473 static class Expansion2 extends AsyncReadRequest {
475 final private Collection<Resource> roots;
476 final Collection<DirectStatements>[] results;
477 final boolean ignoreVirtual;
479 public Expansion2(Collection<Resource> roots, Collection<DirectStatements>[] results) {
480 this(roots, results, true);
483 public Expansion2(Collection<Resource> roots, Collection<DirectStatements>[] results, boolean ignoreVirtual) {
485 this.results = results;
486 this.ignoreVirtual = ignoreVirtual;
490 public void run(AsyncReadGraph graph) {
492 QueryControl control = graph.getService(QueryControl.class);
493 final DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class);
495 final DomainStatementProcedure2 proc =
496 new DomainStatementProcedure2(results);
498 int slice = (int) (roots.size() / control.getAmountOfQueryThreads()) + 1;
500 final Resource[] rootArray = roots.toArray(Resource.NONE);
501 for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
503 final int start = i * slice;
504 final int end = Math.min(start + slice, rootArray.length);
506 control.schedule(graph, i, new ControlProcedure() {
508 public void execute(AsyncReadGraph graph) {
510 for (int index = start; index < end; index++) {
511 dqs.forEachDirectPersistentStatement(graph, rootArray[index], proc);
514 for (int index = start; index < end; index++) {
515 dqs.forEachDirectStatement(graph, rootArray[index], proc);
526 public int getFlags() {
532 static class DomainProcessor2 {
534 Serializer variantSerializer;
538 Set<Resource> fringe = null;
539 Set<Resource> exclusions = new HashSet<Resource>();
540 Set<Resource> internalDomain = new HashSet<Resource>();
541 Set<Resource> sharedExternalReferences = null;
542 TIntSet sharedExternalIds = null;
543 Set<Resource> sharedExternalFringe = null;
544 Set<Resource> predicates = null;
545 Set<Resource> isRelatedToPredicates = null;
546 Set<Resource> sharedPredicates = null;
547 TIntIntHashMap ids = null;
548 Map<Resource, Statement> specials = null;
549 Map<Resource, ExtentStatus> status = null;
550 Map<Resource, WeakStatus> weakInverses = null;
552 // final ArrayList<Double> priorityList = new ArrayList<Double>();
554 private long composedObjectCounter = 0;
555 private long fastInternalCounter = 0;
556 private long parentExternalCounter = 0;
557 private long fullInternalCounter = 0;
558 private long fullExternalCounter = 0;
560 private long startupTime = 0;
561 private long expandTime = 0;
562 private long fullResolveTime = 0;
563 private long fastResolveTime = 0;
564 private long otherStatementTime = 0;
565 private long parentResolveTime = 0;
566 private long extentSeedTime = 0;
567 private long composedPredicateTime = 0;
568 private long composedObjectTime = 0;
570 public void expand(ReadGraph graph, Set<Resource> fringe, Collection<DirectStatements>[] expansion) throws DatabaseException {
572 long start = System.nanoTime();
574 Collection<Collection<DirectStatements>[]> fullExpansion = new ArrayList<Collection<DirectStatements>[]>();
575 QueryControl control = graph.getService(QueryControl.class);
576 for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
577 expansion[i] = new ArrayList<DirectStatements>();
580 graph.syncRequest(new Expansion2(fringe, expansion));
584 expandTime += (System.nanoTime() - start);
588 public void classifyPredicates(ReadGraph graph, final Set<Resource> schedule) throws DatabaseException {
590 CollectionSupport cs = graph.getService(CollectionSupport.class);
592 final Layer0 L0 = Layer0.getInstance(graph);
594 long start = System.nanoTime();
596 final ConcurrentLinkedQueue<Resource> composedResult = new ConcurrentLinkedQueue<Resource>();
597 final ConcurrentLinkedQueue<Resource> singleResult = new ConcurrentLinkedQueue<Resource>();
598 final ConcurrentLinkedQueue<Resource> sharedResult = new ConcurrentLinkedQueue<Resource>();
599 final ConcurrentLinkedQueue<Pair<Resource, Resource>> singles = new ConcurrentLinkedQueue<Pair<Resource, Resource>>();
602 graph.syncRequest(new AsyncReadRequest() {
605 public void run(AsyncReadGraph graph) {
607 for (final Resource predicate : schedule) {
609 graph.forPossibleSuperrelation(predicate, new AsyncProcedure<Resource>() {
612 public void exception(AsyncReadGraph graph, Throwable throwable) {
613 throwable.printStackTrace();
617 public void execute(AsyncReadGraph graph, final Resource single) {
618 singles.add(Pair.make(predicate, single));
623 graph.forHasStatement(predicate, L0.SharedRange, new AsyncProcedure<Boolean>() {
626 public void exception(AsyncReadGraph graph, Throwable throwable) {
627 throwable.printStackTrace();
631 public void execute(AsyncReadGraph graph, final Boolean shared) {
632 if(shared) sharedResult.add(predicate);
644 final Set<Resource> singleSchedule = cs.createSet();
645 for(Pair<Resource, Resource> pair : singles) {
647 Resource single = pair.second;
648 if(single != null && predicates.add(single)) singleSchedule.add(single);
652 graph.syncRequest(new AsyncReadRequest() {
655 public void run(AsyncReadGraph graph) {
657 for (final Resource predicate : singleSchedule) {
659 graph.forIsSubrelationOf(predicate, L0.IsRelatedTo, new AsyncProcedure<Boolean>() {
662 public void exception(AsyncReadGraph graph, Throwable throwable) {
663 throwable.printStackTrace();
667 public void execute(AsyncReadGraph graph, Boolean strong) {
668 if (strong) singleResult.add(predicate);
679 isRelatedToPredicates.addAll(singleResult);
680 sharedPredicates.addAll(sharedResult);
682 final Set<Resource> specialSchedule = cs.createSet();
685 for(Pair<Resource, Resource> pair : singles) {
687 Resource single = pair.second;
689 if(isRelatedToPredicates.contains(single)) {
690 isRelatedToPredicates.add(pair.first);
693 specialSchedule.add(pair.first);
698 graph.syncRequest(new AsyncReadRequest() {
701 public void run(AsyncReadGraph graph) {
703 for (final Resource predicate : specialSchedule) {
705 graph.forIsSubrelationOf(predicate, L0.IsRelatedTo, new AsyncProcedure<Boolean>() {
708 public void exception(AsyncReadGraph graph, Throwable throwable) {
709 throwable.printStackTrace();
713 public void execute(AsyncReadGraph graph, Boolean composed) {
714 if (composed) composedResult.add(predicate);
725 isRelatedToPredicates.addAll(composedResult);
727 composedPredicateTime += (System.nanoTime() - start);
731 private Set<Resource> strongInverseSet = new HashSet<Resource>();
733 public void classifyPredicates(ReadGraph graph, final Collection<DirectStatements>[] expansion) throws DatabaseException {
735 CollectionSupport cs = graph.getService(CollectionSupport.class);
736 final Set<Resource> schedule = cs.createSet();
737 final Map<Resource, Resource> newPredicates = cs.createMap(Resource.class);
739 for (Collection<DirectStatements> coll : expansion)
740 for (DirectStatements stms : coll)
741 for(Statement stm : stms) {
743 Resource predicate = stm.getPredicate();
745 if(predicates.add(predicate)) {
746 Resource inverse = graph.getPossibleInverse(predicate);
747 schedule.add(predicate);
748 if(inverse != null) {
749 newPredicates.put(predicate, inverse);
750 if(predicates.add(inverse)) schedule.add(inverse);
757 classifyPredicates(graph, schedule);
759 for(Map.Entry<Resource, Resource> entry : newPredicates.entrySet()) {
760 // Inverse is strong => this has strong inverse
761 if(isRelatedToPredicates.contains(entry.getValue())) {
762 strongInverseSet.add(entry.getKey());
764 // This is strong => inverse has strong inverse
765 if(isRelatedToPredicates.contains(entry.getKey())) {
766 strongInverseSet.add(entry.getValue());
773 * Composed objects are internal. Mark them for expansion.
776 public void processFringe(ReadGraph graph, Collection<DirectStatements>[] expansion,
777 ObjectOutputStream otherStatementsOutput, ObjectOutputStream valueOutput) throws DatabaseException, IOException {
779 SerialisationSupport support = graph.getService(SerialisationSupport.class);
780 TransferableGraphSupport tgs = graph.getService(TransferableGraphSupport.class);
782 Layer0 L0 = Layer0.getInstance(graph);
784 long start = System.nanoTime();
786 for (Collection<DirectStatements> coll : expansion)
787 for (DirectStatements stms : coll) {
789 Resource subject = stms.getSubject();
791 boolean partOf = false;
792 for(Statement stm : stms) {
793 Resource predicate = stm.getPredicate();
794 if(L0.PartOf.equals(predicate)) {
800 ExtentStatus subjectStatus = status.get(subject);
801 if(LOG && subjectStatus != null) log("EXISTING STATUS FOR " + graph.getPossibleURI(subject) + " - " + subjectStatus);
802 if(subjectStatus == ExtentStatus.EXTERNAL) continue;
803 if(partOf && (subjectStatus == null) && graph.getPossibleURI(subject) != null) {
805 status.put(subject, ExtentStatus.EXTERNAL);
807 String uri = graph.getPossibleURI(subject);
808 if(uri == null) log("[EXTERNAL]: No URI for " + subject);
809 else log("[EXTERNAL] " + uri);
812 // Check for SharedRange statements
813 for(Statement stm : stms) {
814 Resource predicate = stm.getPredicate();
815 if(sharedPredicates.contains(predicate)) {
816 sharedExternalFringe.add(stm.getObject());
818 log("[SHARED EXTERNAL FRINGE]: " + NameUtils.getSafeName(graph, stm.getObject()));
825 boolean special = specials.containsKey(subject);
828 log("[SPECIAL] " + NameUtils.getSafeName(graph, subject));
832 status.put(subject, ExtentStatus.INTERNAL);
833 if(LOG) log("[INTERNAL] " + NameUtils.getSafeName(graph, subject));
835 int sId = support.getTransientId(subject);
837 if(graph.hasValue(subject)) {
838 Datatype dt = graph.getRelatedValue(subject, L0.HasDataType, Bindings.getBindingUnchecked(Datatype.class));
839 Binding b = Bindings.getBinding(dt);
840 Object _value = graph.getValue(subject, b);
841 Variant variant = new Variant(b, _value);
842 byte[] value = variantSerializer.serialize(variant);
843 if(LOG) log("[VALUE] " + NameUtils.getSafeName(graph, subject));
844 valueOutput.writeInt(sId);
845 valueOutput.writeInt(value.length);
846 assert (value.length > 0);
847 valueOutput.write(value);
850 TIntArrayList stream = new TIntArrayList();
852 for(Statement stm : stms) {
856 // System.err.println("stm=" + stm + " special=" + specials.get(subject));
860 Resource predicate = stm.getPredicate();
861 Resource object = stm.getObject();
863 ExtentStatus objectStatus = status.get(object);
866 if (isRelatedToPredicates.contains(predicate) && (objectStatus != ExtentStatus.EXCLUDED)) {
868 int pId = support.getTransientId(predicate);
869 int oId = support.getTransientId(object);
872 String s = NameUtils.getSafeName(graph, subject);
873 String p = NameUtils.getSafeName(graph, predicate);
874 String o = NameUtils.getSafeName(graph, object);
875 log("related=" + s + " - " + p + " - " + o);
881 if(objectStatus == null)
887 if(objectStatus == ExtentStatus.INTERNAL) {
889 // The inverse is also weak (or there is no inverse)
890 if(!strongInverseSet.contains(predicate)) {
892 int pId = support.getTransientId(predicate);
893 int oId = support.getTransientId(object);
899 String s = NameUtils.getSafeName(graph, subject);
900 String p = NameUtils.getSafeName(graph, predicate);
901 String o = NameUtils.getSafeName(graph, object);
902 log("fully weak internal=" + s + " - " + p + " - " + o + " - " + objectStatus);
908 String s = NameUtils.getSafeName(graph, subject);
909 String p = NameUtils.getSafeName(graph, predicate);
910 String o = NameUtils.getSafeName(graph, object);
911 log("strong inverse internals=" + s + " - " + p + " - " + o + " - " + objectStatus);
920 // System.err.println("stm=" + stm + " special=" + specials.get(subject));
922 Statement spec = specials.get(subject);
924 // This statement can be specially treated
925 if(stm.getPredicate().equals(spec.getPredicate()) && stm.getObject().equals(spec.getObject())) {
927 int pId = support.getTransientId(predicate);
928 int oId = support.getTransientId(object);
931 String s = NameUtils.getSafeName(graph, subject);
932 String p = NameUtils.getSafeName(graph, predicate);
933 String o = NameUtils.getSafeName(graph, object);
934 log("special=" + s + " - " + p + " - " + o);
945 String s = NameUtils.getSafeName(graph, subject);
946 String p = NameUtils.getSafeName(graph, predicate);
947 String o = NameUtils.getSafeName(graph, object);
948 log("weak with unknown object=" + s + " - " + p + " - " + o + " - " + objectStatus);
959 if(!stream.isEmpty()) {
960 otherStatementsOutput.writeInt(sId);
961 otherStatementsOutput.writeInt(stream.size() / 2);
962 for (int i = 0; i < stream.size(); i++)
963 otherStatementsOutput.writeInt(stream.getQuick(i));
970 composedObjectTime += System.nanoTime() - start;
974 public void process(ReadGraph graph,
975 ObjectOutputStream otherStatementsOutput,
976 ObjectOutputStream valueOutput)
977 throws DatabaseException, IOException {
979 this.variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT);
981 QueryControl control = graph.getService(QueryControl.class);
983 // System.err.println("Begin ConsistsOfProcess");
986 * Browse all stm = (s, ConsistsOf, o)
987 * � All internal URIs are found => from now on, if unidentified resource has PartOf it is external.
988 * � All s are internal
989 * � All o are internal
990 * � All stm are included
992 for(Resource r : ConsistsOfProcess.walk(graph, fringe, exclusions, true)) {
993 if (status.put(r, ExtentStatus.INTERNAL) == null) {
994 String URI = graph.getPossibleURI(r);
995 if(URI != null) log("URI INTERNAL " + URI);
996 else log("URI has no URI for " + r);
998 internalDomain.add(r);
1003 * This loop resolves the transitive closure of all p < IsRelatedTo such that p does not contain the SharedRange tag.
1004 * Such resources are guaranteed to be internal.
1006 while(!fringe.isEmpty()) {
1008 // System.err.println("Process Fringe with " + fringe.size() + ".");
1010 Collection<DirectStatements>[] expansion = new ArrayList[control.getAmountOfQueryThreads()];
1012 // System.err.println("-expand");
1015 expand(graph, fringe, expansion);
1021 * -Internal / External
1024 // System.err.println("-classify");
1026 classifyPredicates(graph, expansion);
1029 * for stms in [stms]
1030 * if stms contains predicate PartOf => s is External
1031 * else s is Internal
1032 * for all stm=(s,p,o) in stms
1033 * if p <R IsRelatedTo => stm is included
1037 // System.err.println("-process");
1039 processFringe(graph, expansion, otherStatementsOutput, valueOutput);
1043 while(!sharedExternalFringe.isEmpty()) {
1045 Collection<DirectStatements>[] expansion = new ArrayList[control.getAmountOfQueryThreads()];
1046 expand(graph, sharedExternalFringe, expansion);
1048 for (Collection<DirectStatements> coll : expansion)
1049 for (DirectStatements stms : coll) {
1051 Resource subject = stms.getSubject();
1052 ExtentStatus subjectStatus = status.get(subject);
1054 if(ExtentStatus.INTERNAL == subjectStatus) {
1056 if(internalDomain.contains(subject)) continue;
1058 status.put(subject, ExtentStatus.EXTERNAL);
1059 sharedExternalReferences.add(subject);
1062 log("[SHARED EXTERNAL REFERENCE]: " + NameUtils.getSafeName(graph, subject));
1065 for(Statement stm : stms) {
1066 Resource predicate = stm.getPredicate();
1067 if (isRelatedToPredicates.contains(predicate)) {
1068 sharedExternalFringe.add(stm.getObject());
1078 System.out.println(composedObjectCounter + " " + fastInternalCounter
1079 + " " + parentExternalCounter + " "
1080 + fullExternalCounter + " " + fullInternalCounter);
1087 static class DomainProcessor {
1089 Serializer variantSerializer;
1093 Set<Resource> predicates = null;
1094 Set<Resource> composedPredicates = null;
1095 Set<Resource> expansionSeeds = null;
1096 Map<Resource, Integer> ids = null;
1097 Map<Resource, ExtentStatus> status = null;
1098 Map<Resource, WeakStatus> weakInverses = null;
1100 final Set<SubgraphAdvisor> advisors;
1101 final ArrayList<Double> priorityList = new ArrayList<Double>();
1103 private long composedObjectCounter = 0;
1104 private long fastInternalCounter = 0;
1105 private long parentExternalCounter = 0;
1106 private long fullInternalCounter = 0;
1107 private long fullExternalCounter = 0;
1109 private long startupTime = 0;
1110 private long expandTime = 0;
1111 private long fullResolveTime = 0;
1112 private long fastResolveTime = 0;
1113 private long otherStatementTime = 0;
1114 private long parentResolveTime = 0;
1115 private long extentSeedTime = 0;
1116 private long composedPredicateTime = 0;
1117 private long composedObjectTime = 0;
1119 public DomainProcessor(Set<SubgraphAdvisor> advisors) {
1120 this.advisors = advisors;
1121 HashSet<Double> prioritySet = new HashSet<Double>();
1122 for (SubgraphAdvisor advisor : advisors)
1123 prioritySet.add(advisor.priority());
1124 priorityList.addAll(prioritySet);
1125 Collections.sort(priorityList);
1128 public void expand(ReadGraph graph, Collection<DirectStatements>[] expansion, Set<Resource> schedule) throws DatabaseException {
1130 long start = System.nanoTime();
1133 // System.out.println("expanding " + expansionSeeds.size() + " resources.");
1135 QueryControl control = graph.getService(QueryControl.class);
1136 // final Collection<DirectStatements>[] results = new ArrayList[control.getAmountOfQueryThreads()];
1137 final ArrayList<Resource>[] listElements = new ArrayList[control.getAmountOfQueryThreads()];
1138 for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
1139 // results[i] = new ArrayList<DirectStatements>();
1140 listElements[i] = new ArrayList<Resource>();
1144 // for(Resource r : expansionSeeds)
1145 // System.out.println("Expanding " + NameUtils.getSafeName(graph, r, true));
1148 graph.syncRequest(new Expansion(expansionSeeds, expansion, listElements));
1149 for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
1150 // for (DirectStatements s : results[i]) {
1151 // expansion.put(s.getSubject(), s);
1153 for (Resource s : listElements[i]) {
1155 // if(status.put(s, ExtentStatus.INTERNAL) == null) {
1156 // ids.put(s, id++);
1161 expandTime += (System.nanoTime() - start);
1165 public void extractComposedPredicates(ReadGraph graph, final Collection<DirectStatements>[] expansion) throws DatabaseException {
1167 long start = System.nanoTime();
1169 CollectionSupport cs = graph.getService(CollectionSupport.class);
1171 final ConcurrentLinkedQueue<Resource> composedResult = new ConcurrentLinkedQueue<Resource>();
1172 final ConcurrentLinkedQueue<Resource> singleResult = new ConcurrentLinkedQueue<Resource>();
1173 final ConcurrentLinkedQueue<Pair<Resource, Resource>> singles = new ConcurrentLinkedQueue<Pair<Resource, Resource>>();
1175 final Set<Resource> schedule = cs.createSet();
1177 for (Collection<DirectStatements> coll : expansion)
1178 for (DirectStatements stms : coll)
1179 for(Statement stm : stms) {
1180 Resource predicate = stm.getPredicate();
1181 if(predicates.add(predicate)) schedule.add(predicate);
1185 graph.syncRequest(new AsyncReadRequest() {
1188 public void run(AsyncReadGraph graph) {
1190 for (final Resource predicate : schedule) {
1192 graph.forPossibleSuperrelation(predicate, new AsyncProcedure<Resource>() {
1195 public void exception(AsyncReadGraph graph, Throwable throwable) {
1196 throwable.printStackTrace();
1200 public void execute(AsyncReadGraph graph, final Resource single) {
1201 singles.add(Pair.make(predicate, single));
1212 // Determine singles
1213 final Set<Resource> singleSchedule = cs.createSet();
1214 for(Pair<Resource, Resource> pair : singles) {
1216 Resource single = pair.second;
1217 if(single != null && predicates.add(single)) singleSchedule.add(single);
1221 graph.syncRequest(new AsyncReadRequest() {
1224 public void run(AsyncReadGraph graph) {
1226 for (final Resource predicate : singleSchedule) {
1228 graph.forIsSubrelationOf(predicate, graph.getService(Layer0.class).IsComposedOf, new AsyncProcedure<Boolean>() {
1231 public void exception(AsyncReadGraph graph, Throwable throwable) {
1232 throwable.printStackTrace();
1236 public void execute(AsyncReadGraph graph, Boolean composed) {
1237 if (composed) singleResult.add(predicate);
1248 composedPredicates.addAll(singleResult);
1250 final Set<Resource> specialSchedule = cs.createSet();
1253 for(Pair<Resource, Resource> pair : singles) {
1255 Resource single = pair.second;
1256 if(single != null) {
1257 if(composedPredicates.contains(single)) {
1258 composedPredicates.add(pair.first);
1261 specialSchedule.add(pair.first);
1266 graph.syncRequest(new AsyncReadRequest() {
1269 public void run(AsyncReadGraph graph) {
1271 for (final Resource predicate : specialSchedule) {
1273 graph.forIsSubrelationOf(predicate, graph.getService(Layer0.class).IsComposedOf, new AsyncProcedure<Boolean>() {
1276 public void exception(AsyncReadGraph graph, Throwable throwable) {
1277 throwable.printStackTrace();
1281 public void execute(AsyncReadGraph graph, Boolean composed) {
1282 if (composed) composedResult.add(predicate);
1293 composedPredicates.addAll(composedResult);
1295 composedPredicateTime += (System.nanoTime() - start);
1300 * Composed objects are internal. Mark them for expansion.
1303 public void collectComposedObjects(ReadGraph graph, Collection<DirectStatements>[] expansion, Set<Resource> typeTodo, Set<Resource> objectTodo,
1304 Set<Resource> predicateTodo) throws DatabaseException {
1306 long start = System.nanoTime();
1308 Layer0 l0 = Layer0.getInstance(graph);
1310 for (Collection<DirectStatements> coll : expansion)
1311 for (DirectStatements stms : coll)
1312 for(Statement stm : stms) {
1314 Resource predicate = stm.getPredicate();
1315 Resource object = stm.getObject();
1317 if (composedPredicates.contains(predicate)) {
1319 ExtentStatus existing = status.put(object, ExtentStatus.INTERNAL);
1320 if(existing == null) {
1321 ids.put(object, id++);
1322 composedObjectCounter++;
1323 expansionSeeds.add(object);
1324 // System.err.println("internal: " + NameUtils.getSafeName(graph, object, true));
1325 if(LOG) log("[INTERNAL] (composed object) " + NameUtils.getSafeName(graph, object, true));
1326 } else if (existing == ExtentStatus.EXCLUDED) {
1327 System.err.println("preExcluded: " + NameUtils.getSafeName(graph, object, true));
1328 status.put(object, ExtentStatus.EXCLUDED);
1329 } else if (existing == ExtentStatus.EXTERNAL) {
1330 System.err.println("preExternal: " + NameUtils.getSafeName(graph, object, true));
1331 status.put(object, ExtentStatus.EXTERNAL);
1336 // System.err.println("internal2: " + NameUtils.getSafeName(graph, object, true));
1338 if (!status.containsKey(object)) {
1339 if (l0.InstanceOf.equalsResource(predicate)) {
1340 typeTodo.add(object);
1342 objectTodo.add(object);
1346 if (!status.containsKey(predicate)) {
1347 predicateTodo.add(predicate);
1353 composedObjectTime += System.nanoTime() - start;
1357 public void writeOtherStatements(ReadGraph graph, Collection<Collection<DirectStatements>[]> expansion, ObjectOutputStream composedStatementsOutput, ObjectOutputStream otherStatementsOutput,
1358 ObjectOutputStream valueOutput) throws DatabaseException {
1360 long start = System.nanoTime();
1362 Layer0 l0 = Layer0.getInstance(graph);
1363 SerialisationSupport support = graph.getService(SerialisationSupport.class);
1364 TransferableGraphSupport tgs = graph.getService(TransferableGraphSupport.class);
1366 TIntArrayList other = new TIntArrayList();
1367 TIntArrayList composed = new TIntArrayList();
1371 for (Collection<DirectStatements>[] colls : expansion)
1372 for (Collection<DirectStatements> coll : colls)
1373 for (DirectStatements stms : coll) {
1375 Resource subject = stms.getSubject();
1376 composed.resetQuick();
1378 int sId = support.getTransientId(subject);
1380 composedStatementsOutput.writeInt(sId);
1382 if(graph.hasValue(subject)) {
1383 Datatype dt = graph.getRelatedValue(subject, l0.HasDataType, Bindings.getBindingUnchecked(Datatype.class));
1384 Binding b = Bindings.getBinding(dt);
1385 Object _value = graph.getValue(subject, b);
1386 Variant variant = new Variant(b, _value);
1387 byte[] value = variantSerializer.serialize(variant);
1388 if(LOG) log("[VALUE] " + NameUtils.getSafeName(graph, subject));
1389 valueOutput.writeInt(sId);
1390 valueOutput.writeInt(value.length);
1391 assert (value.length > 0);
1392 valueOutput.write(value);
1395 for (Statement s : stms) {
1397 Resource object = s.getObject();
1398 Resource predicate = s.getPredicate();
1400 ExtentStatus objectStatus = status.get(object);
1402 if(objectStatus == ExtentStatus.INTERNAL) {
1403 composed.add(support.getTransientId(predicate));
1404 composed.add(support.getTransientId(object));
1405 if(LOG) log("[COMPOSED] (internal object) " + NameUtils.toIdString(graph, s));
1406 } else if (l0.InstanceOf.equalsResource(predicate)) {
1407 composed.add(support.getTransientId(predicate));
1408 composed.add(support.getTransientId(object));
1409 if(LOG) log("[COMPOSED] (instanceOf) " + NameUtils.toIdString(graph, s));
1410 } else if (l0.SubrelationOf.equalsResource(predicate)) {
1411 composed.add(support.getTransientId(predicate));
1412 composed.add(support.getTransientId(object));
1413 if(LOG) log("[COMPOSED] (subrelationOf) " + NameUtils.toIdString(graph, s));
1415 if(objectStatus == ExtentStatus.EXTERNAL) {
1417 System.out.println("other " + NameUtils.toIdString(graph, s));
1418 //System.out.println("other.add " + predicate + " - " + object);
1419 other.add(support.getTransientId(predicate));
1420 other.add(support.getTransientId(object));
1421 if(LOG) log("[OTHER] (object is external) " + NameUtils.toIdString(graph, s));
1427 if(!other.isEmpty()) {
1428 otherStatementsOutput.writeInt(sId);
1429 otherStatementsOutput.writeInt(other.size() / 2);
1430 for (int i = 0; i < other.size(); i++)
1431 otherStatementsOutput.writeInt(other.getQuick(i));
1435 composedStatementsOutput.writeInt(composed.size() / 2);
1436 for (int i = 0; i < composed.size(); i++)
1437 composedStatementsOutput.writeInt(composed.getQuick(i));
1441 } catch (IOException e) {
1442 e.printStackTrace();
1445 otherStatementTime += (System.nanoTime() - start);
1449 boolean hasStrictParents(ReadGraph g, Resource r)
1450 throws DatabaseException {
1451 if (g.getPossibleURI(r) != null)
1456 public boolean getExpansionSeedsFromExtents(ReadGraph graph, final Collection<DirectStatements>[] expansion) throws DatabaseException {
1458 long start = System.nanoTime();
1460 final ConcurrentLinkedQueue<Resource> accepts = new ConcurrentLinkedQueue<Resource>();
1463 * Determine statements which could accept statements with todo
1466 search: for (Double priority : priorityList) {
1468 for (final SubgraphAdvisor advisor : advisors) {
1470 if (advisor.priority() > 0)
1473 if (advisor.priority() == priority) {
1475 graph.syncRequest(new ReadRequest() {
1478 public void run(ReadGraph graph) throws DatabaseException {
1480 for (Collection<DirectStatements> coll : expansion)
1481 for (DirectStatements stms : coll)
1482 for(final Statement stm : stms) {
1484 advisor.advise(graph, stm, new AsyncProcedure<Boolean>() {
1487 public void exception(AsyncReadGraph graph, Throwable throwable) {
1488 throwable.printStackTrace();
1492 public void execute(AsyncReadGraph graph, Boolean accept) {
1494 accepts.add(stm.getObject());
1506 if (!accepts.isEmpty())
1511 CollectionSupport cs = graph.getService(CollectionSupport.class);
1512 Set<Resource> schedule = cs.createSet();
1513 for (Resource r : accepts) {
1514 if(!status.containsKey(r))
1518 extentSeedTime += (System.nanoTime() - start);
1520 if (schedule.isEmpty())
1523 fastResolve(graph, schedule);
1524 uriResolve(graph, schedule);
1525 fullResolve(graph, schedule, "accepts");
1531 ConcurrentLinkedQueue<Resource> fastInternals = new ConcurrentLinkedQueue<Resource>();
1533 public void fastResolve(ReadGraph graph, final Set<Resource> rs)
1534 throws DatabaseException {
1535 // This collects and resolves weaks
1536 if(fastResolveLoop(graph, rs))
1537 // Weaks are now resolved
1538 fastResolveLoop(graph, rs);
1541 public boolean fastResolveLoop(ReadGraph graph, final Set<Resource> rs)
1542 throws DatabaseException {
1544 long start = System.nanoTime();
1546 final ConcurrentLinkedQueue<Resource> weakSchedule = new ConcurrentLinkedQueue<Resource>();
1548 graph.syncRequest(new AsyncRead<Boolean>() {
1551 public int threadHash() {
1556 public int getFlags() {
1561 public void perform(AsyncReadGraph graph,
1562 AsyncProcedure<Boolean> procedure) {
1564 QueryControl control = graph.getService(QueryControl.class);
1565 final DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class);
1567 int slice = (int) (rs.size() / control
1568 .getAmountOfQueryThreads()) + 1;
1570 final Resource[] rootArray = rs.toArray(Resource.NONE);
1571 for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
1573 final int start = i * slice;
1574 final int end = Math.min(start + slice,
1577 control.schedule(graph, i, new ControlProcedure() {
1580 public void execute(AsyncReadGraph graph) {
1582 for (int index = start; index < end; index++) {
1584 final Resource r = rootArray[index];
1586 //if (status.containsKey(r)) continue;
1588 graph.asyncRequest(new FastInternalRequest(dqs, r, status, weakInverses, weakSchedule),new AsyncProcedure<Boolean>() {
1591 public void exception(AsyncReadGraph graph,Throwable throwable) {
1592 throwable.printStackTrace();
1596 public void execute(AsyncReadGraph graph,Boolean isInternal) {
1598 fastInternals.add(r);
1612 procedure.execute(graph, true);
1618 if (!weakSchedule.isEmpty()) {
1619 THashSet<Resource> weaks = new THashSet<Resource>(weakSchedule);
1621 for (Resource p : weakSchedule)
1622 System.out.println("classify "
1623 + NameUtils.getSafeName(graph, p));
1624 graph.syncRequest(new ClassifyStatementsRequest(weaks, weakInverses));
1627 for (Resource r : fastInternals) {
1629 if (status.put(r, ExtentStatus.INTERNAL) == null) {
1630 if(LOG) log("[INTERNAL] (fast) " + NameUtils.getSafeName(graph, r, true));
1632 fastInternalCounter++;
1633 expansionSeeds.add(r);
1637 fastResolveTime += (System.nanoTime() - start);
1639 return !weakSchedule.isEmpty();
1643 private ExtentStatus resolveExtent(ReadGraph graph, Resource r, Map<Resource, ExtentStatus> statuses, Set<Resource> expansionSeeds, THashSet<Resource> pending,
1644 ArrayList<Resource> stack) throws DatabaseException {
1646 ExtentStatus current = statuses.get(r);
1647 if(current != null) return current;
1649 if (pending.contains(r))
1650 return ExtentStatus.PENDING;
1652 // In order to break cyclic dependencies
1656 System.out.println("resolveExtent "
1657 + NameUtils.getSafeName(graph, r));
1659 ExtentStatus status = ExtentStatus.INTERNAL;
1660 for (Resource p : getParents(graph, r)) {
1662 ExtentStatus ps = statuses.get(p);
1663 System.out.println(" parent " + NameUtils.getSafeName(graph, p) + "(" + ps + ")");
1665 switch (resolveExtent(graph, p, statuses,
1666 expansionSeeds, pending, stack)) {
1668 return ExtentStatus.EXTERNAL;
1670 status = ExtentStatus.PENDING;
1673 if (status == ExtentStatus.INTERNAL) {
1677 System.out.println(NameUtils.getSafeName(graph, r, true)
1683 public void uriResolve(ReadGraph graph, final Set<Resource> todo)
1684 throws DatabaseException {
1686 long start = System.nanoTime();
1688 for(Resource r : todo) System.out.println("uriResolve " +
1689 NameUtils.getSafeName(graph, r));
1691 final ConcurrentSkipListSet<Resource> found = new ConcurrentSkipListSet<Resource>();
1693 graph.syncRequest(new AsyncReadRequest() {
1696 public void run(AsyncReadGraph graph) {
1698 for (final Resource r : todo) {
1700 // System.out.println("uriresolve before " + r);
1702 if (status.containsKey(r)) continue;
1704 // System.out.println("uriresolve " + r);
1706 graph.forURI(r, new AsyncProcedure<String>() {
1709 public void exception(AsyncReadGraph graph, Throwable throwable) {
1710 throwable.printStackTrace();
1714 public void execute(AsyncReadGraph graph, String uri) {
1718 // System.out.println("uriresolve has uri "
1722 parentExternalCounter++;
1727 // System.out.println("uriresolve ask inverse "
1730 graph.forPossibleInverse(r, new AsyncProcedure<Resource>() {
1733 public void exception(AsyncReadGraph graph, Throwable throwable) {
1734 throwable.printStackTrace();
1738 public void execute(AsyncReadGraph graph, Resource inverse) {
1740 if (inverse != null) {
1742 graph.forURI(inverse, new AsyncProcedure<String>() {
1745 public void exception(AsyncReadGraph graph, Throwable throwable) {
1746 throwable.printStackTrace();
1750 public void execute(AsyncReadGraph graph, String uri) {
1755 parentExternalCounter++;
1781 todo.removeAll(found);
1782 for(Resource r : found) {
1783 status.put(r, ExtentStatus.EXTERNAL);
1784 if(LOG) log("[EXTERNAL] (uriResolve) " + NameUtils.getSafeName(graph, r, true));
1787 parentResolveTime += System.nanoTime() - start;
1791 public void fullResolve(ReadGraph graph, Collection<Resource> rs,
1792 String koss) throws DatabaseException {
1794 long start = System.nanoTime();
1796 for (final Resource r : rs) {
1798 if(status.containsKey(r)) continue;
1800 THashSet<Resource> pending = new THashSet<Resource>();
1801 ArrayList<Resource> stack = new ArrayList<Resource>();
1803 ExtentStatus s = resolveExtent(graph, r, status, expansionSeeds, pending, stack);
1804 if (ExtentStatus.INTERNAL == s || ExtentStatus.PENDING == s) {
1805 if (status.put(r, ExtentStatus.INTERNAL) == null) {
1806 if(LOG) log("[INTERNAL] (resolveExtent) " + NameUtils.getSafeName(graph, r, true));
1808 fullInternalCounter++;
1809 expansionSeeds.add(r);
1812 if (ExtentStatus.EXTERNAL == s) {
1813 if (status.put(r, ExtentStatus.EXTERNAL) == null) {
1814 if(LOG) log("[EXTERNAL] (resolveExtent) " + NameUtils.getSafeName(graph, r, true));
1815 fullExternalCounter++;
1821 fullResolveTime += (System.nanoTime() - start);
1825 public void process(ReadGraph graph,
1826 ObjectOutputStream composedStatementsOutput,
1827 ObjectOutputStream otherStatementsOutput,
1828 ObjectOutputStream valueOutput)
1829 throws DatabaseException {
1831 this.variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT);
1833 CollectionSupport cs = graph.getService(CollectionSupport.class);
1835 Set<Resource> typeTodo = cs.createSet();
1836 Set<Resource> objectTodo = cs.createSet();
1837 Set<Resource> predicateTodo = cs.createSet();
1839 Collection<Collection<DirectStatements>[]> fullExpansion = new ArrayList<Collection<DirectStatements>[]>();
1843 QueryControl control = graph.getService(QueryControl.class);
1844 Collection<DirectStatements>[] expansion = new ArrayList[control.getAmountOfQueryThreads()];
1845 for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
1846 expansion[i] = new ArrayList<DirectStatements>();
1849 // Expand expansionSeeds
1850 expand(graph, expansion, objectTodo);
1852 // Start collecting new seeds
1853 expansionSeeds = cs.createSet();
1855 // Collect predicates which are <R IsComposedOf
1856 extractComposedPredicates(graph, expansion);
1858 // Make composed objects internal and make sure they are further
1860 collectComposedObjects(graph, expansion, typeTodo, objectTodo, predicateTodo);
1863 * Use the expansion seed heuristic to find new resources to
1864 * expand before full analysis.
1866 getExpansionSeedsFromExtents(graph, expansion);
1868 fullExpansion.add(expansion);
1870 } while (!expansionSeeds.isEmpty());
1872 fastResolve(graph, objectTodo);
1873 uriResolve(graph, predicateTodo);
1874 uriResolve(graph, objectTodo);
1875 uriResolve(graph, typeTodo);
1876 fullResolve(graph, objectTodo, "objectTodo");
1877 fullResolve(graph, predicateTodo, "predicateTodo");
1878 fullResolve(graph, typeTodo, "typeTodo");
1880 writeOtherStatements(graph, fullExpansion, composedStatementsOutput, otherStatementsOutput, valueOutput);
1883 System.out.println(composedObjectCounter + " " + fastInternalCounter
1884 + " " + parentExternalCounter + " "
1885 + fullExternalCounter + " " + fullInternalCounter);
1892 public static void getDomain2(ReadGraph graph, TIntIntHashMap ids,
1893 Collection<Resource> roots, Map<Resource, ExtentStatus> preStatus,
1894 Map<Resource, Statement> specials,
1895 ObjectOutputStream otherStatementsOutput,
1896 ObjectOutputStream valueOutput,
1897 TreeMap<String, Variant> extensions,
1898 TIntHashSet excludedShared) throws DatabaseException {
1900 ITask task = ThreadLogger.getInstance().begin("getDomain2");
1902 final DomainProcessor2 processor = new DomainProcessor2();
1904 processor.startupTime = System.nanoTime();
1906 Layer0 l0 = Layer0.getInstance(graph);
1908 CollectionSupport cs = graph.getService(CollectionSupport.class);
1909 SerialisationSupport support = graph.getService(SerialisationSupport.class);
1911 processor.ids = ids;
1912 processor.specials = specials;
1913 processor.status = cs.createMap(ExtentStatus.class);
1914 processor.weakInverses = cs.createMap(WeakStatus.class);
1915 processor.predicates = cs.createSet();
1916 processor.isRelatedToPredicates = cs.createSet();
1917 processor.sharedPredicates = cs.createSet();
1918 // processor.expansionSeeds = cs.createSet();
1920 for(Map.Entry<Resource, ExtentStatus> entry : preStatus.entrySet()) {
1921 processor.status.put(entry.getKey(), entry.getValue());
1922 if(ExtentStatus.EXCLUDED.equals(entry.getValue())) processor.exclusions.add(entry.getKey());
1925 // for (Resource r : excluded) {
1926 // processor.status.put(r, ExtentStatus.EXCLUDED);
1929 Resource rootLibrary = graph.getResource("http:/");
1931 if (!roots.contains(rootLibrary))
1932 processor.status.put(rootLibrary, ExtentStatus.EXTERNAL);
1934 for (Resource root : roots) {
1935 processor.status.put(root, ExtentStatus.INTERNAL);
1936 //processor.ids.put(support.getTransientId(root), processor.ids.size());
1937 for (Resource owner : graph.getObjects(root, l0.IsOwnedBy)) {
1938 processor.status.put(owner, ExtentStatus.EXTERNAL);
1942 processor.startupTime = System.nanoTime() - processor.startupTime;
1944 processor.fringe = new HashSet<Resource>();
1945 processor.fringe.addAll(roots);
1947 processor.internalDomain.addAll(roots);
1949 processor.sharedExternalReferences = new HashSet<Resource>();
1950 processor.sharedExternalFringe = new HashSet<Resource>();
1954 processor.process(graph, otherStatementsOutput, valueOutput);
1956 } catch (IOException e) {
1957 e.printStackTrace();
1960 for(Resource r : processor.sharedExternalReferences) excludedShared.add(support.getTransientId(r));
1962 ClusteringSupport cls = graph.getService(ClusteringSupport.class);
1963 TLongObjectHashMap<TIntArrayList> clusterMap = new TLongObjectHashMap<TIntArrayList>();
1964 for(Map.Entry<Resource, ExtentStatus> entry : processor.status.entrySet()) {
1965 if(ExtentStatus.INTERNAL == entry.getValue()) {
1967 long cluster = cls.getCluster(entry.getKey());
1968 TIntArrayList list = clusterMap.get(cluster);
1970 list = new TIntArrayList();
1971 clusterMap.put(cluster, list);
1973 list.add(support.getTransientId(entry.getKey()));
1977 final TIntArrayList clustering = new TIntArrayList();
1978 clusterMap.forEachEntry(new TLongObjectProcedure<TIntArrayList>() {
1981 public boolean execute(long cluster, TIntArrayList b) {
1982 clustering.add(b.size());
1983 b.forEach(new TIntProcedure() {
1986 public boolean execute(int rId) {
1987 processor.ids.put(rId, processor.id++);
1997 extensions.put(Extensions.CLUSTERING, new Variant(Bindings.INT_ARRAY, clustering.toArray()));
1999 long total = processor.startupTime + processor.expandTime
2000 + processor.composedPredicateTime
2001 + processor.composedObjectTime + processor.extentSeedTime
2002 + processor.fullResolveTime + processor.fastResolveTime +
2003 + processor.parentResolveTime + processor.otherStatementTime;
2006 System.out.println("startup took " + 1e-9 * processor.startupTime
2008 System.out.println("expand took " + 1e-9 * processor.expandTime
2010 System.out.println("composedPredicates took " + 1e-9
2011 * processor.composedPredicateTime + "s.");
2012 System.out.println("composedObjects took " + 1e-9
2013 * processor.composedObjectTime + "s.");
2014 System.out.println("extentSeeding took " + 1e-9
2015 * processor.extentSeedTime + "s.");
2016 System.out.println("fullResolve took " + 1e-9
2017 * processor.fullResolveTime + "s.");
2018 System.out.println("fastResolve took " + 1e-9
2019 * processor.fastResolveTime + "s.");
2020 System.out.println("parentResolve took " + 1e-9
2021 * processor.parentResolveTime + "s.");
2022 System.out.println("otherStatements took " + 1e-9
2023 * processor.otherStatementTime + "s.");
2024 System.out.println("total " + 1e-9 * total + "s.");
2033 public static void getDomain(ReadGraph graph, Map<Resource, Integer> ids,
2034 Collection<Resource> roots, Map<Resource, ExtentStatus> preStatus, Set<SubgraphAdvisor> advisors,
2035 ObjectOutputStream composedStatementsOutput,
2036 ObjectOutputStream otherStatementsOutput,
2037 ObjectOutputStream valueOutput) throws DatabaseException {
2039 ITask task = ThreadLogger.getInstance().begin("getDomain");
2041 DomainProcessor processor = new DomainProcessor(advisors);
2043 processor.startupTime = System.nanoTime();
2045 Layer0 l0 = Layer0.getInstance(graph);
2047 CollectionSupport cs = graph.getService(CollectionSupport.class);
2049 processor.ids = ids;
2050 processor.status = cs.createMap(ExtentStatus.class);
2051 processor.weakInverses = cs.createMap(WeakStatus.class);
2052 processor.predicates = cs.createSet();
2053 processor.composedPredicates = cs.createSet();
2054 processor.expansionSeeds = cs.createSet();
2056 for(Map.Entry<Resource, ExtentStatus> entry : preStatus.entrySet()) {
2057 processor.status.put(entry.getKey(), entry.getValue());
2060 // for (Resource r : excluded) {
2061 // processor.status.put(r, ExtentStatus.EXCLUDED);
2064 if (!roots.contains(graph.getRootLibrary()))
2065 processor.status.put(graph.getRootLibrary(), ExtentStatus.EXTERNAL);
2067 for (Resource root : roots) {
2068 processor.status.put(root, ExtentStatus.INTERNAL);
2069 processor.ids.put(root, processor.id++);
2070 for (Resource owner : graph.getObjects(root, l0.IsOwnedBy)) {
2071 processor.status.put(owner, ExtentStatus.EXTERNAL);
2076 processor.expansionSeeds.addAll(roots);
2078 processor.startupTime = System.nanoTime() - processor.startupTime;
2080 while (!processor.expansionSeeds.isEmpty()) {
2082 processor.process(graph, composedStatementsOutput,
2083 otherStatementsOutput, valueOutput);
2087 long total = processor.startupTime + processor.expandTime
2088 + processor.composedPredicateTime
2089 + processor.composedObjectTime + processor.extentSeedTime
2090 + processor.fullResolveTime + processor.fastResolveTime +
2091 + processor.parentResolveTime + processor.otherStatementTime;
2094 System.out.println("startup took " + 1e-9 * processor.startupTime
2096 System.out.println("expand took " + 1e-9 * processor.expandTime
2098 System.out.println("composedPredicates took " + 1e-9
2099 * processor.composedPredicateTime + "s.");
2100 System.out.println("composedObjects took " + 1e-9
2101 * processor.composedObjectTime + "s.");
2102 System.out.println("extentSeeding took " + 1e-9
2103 * processor.extentSeedTime + "s.");
2104 System.out.println("fullResolve took " + 1e-9
2105 * processor.fullResolveTime + "s.");
2106 System.out.println("fastResolve took " + 1e-9
2107 * processor.fastResolveTime + "s.");
2108 System.out.println("parentResolve took " + 1e-9
2109 * processor.parentResolveTime + "s.");
2110 System.out.println("otherStatements took " + 1e-9
2111 * processor.otherStatementTime + "s.");
2112 System.out.println("total " + 1e-9 * total + "s.");