1 /*******************************************************************************
2 * Copyright (c) 2007, 2010 Association for Decentralized Information Management
4 * All rights reserved. This program and the accompanying materials
5 * are made available under the terms of the Eclipse Public License v1.0
6 * which accompanies this distribution, and is available at
7 * http://www.eclipse.org/legal/epl-v10.html
10 * VTT Technical Research Centre of Finland - initial API and implementation
11 *******************************************************************************/
12 package org.simantics.db.layer0.util;
14 import java.io.DataOutput;
15 import java.io.DataOutputStream;
16 import java.io.FileNotFoundException;
17 import java.io.FileOutputStream;
18 import java.io.IOException;
19 import java.io.ObjectOutputStream;
20 import java.util.ArrayList;
21 import java.util.Collection;
22 import java.util.Collections;
23 import java.util.HashSet;
24 import java.util.List;
27 import java.util.TreeMap;
28 import java.util.concurrent.ConcurrentLinkedQueue;
29 import java.util.concurrent.ConcurrentSkipListSet;
31 import org.simantics.databoard.Bindings;
32 import org.simantics.databoard.Databoard;
33 import org.simantics.databoard.binding.Binding;
34 import org.simantics.databoard.binding.mutable.Variant;
35 import org.simantics.databoard.serialization.Serializer;
36 import org.simantics.databoard.type.Datatype;
37 import org.simantics.db.AsyncReadGraph;
38 import org.simantics.db.DirectStatements;
39 import org.simantics.db.ReadGraph;
40 import org.simantics.db.Resource;
41 import org.simantics.db.Statement;
42 import org.simantics.db.common.request.AsyncReadRequest;
43 import org.simantics.db.common.request.ReadRequest;
44 import org.simantics.db.common.request.ResourceAsyncRead;
45 import org.simantics.db.common.utils.NameUtils;
46 import org.simantics.db.exception.DatabaseException;
47 import org.simantics.db.layer0.adapter.SubgraphAdvisor;
48 import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus;
49 import org.simantics.db.layer0.util.ConsistsOfProcess.InternalEntry;
50 import org.simantics.db.procedure.AsyncProcedure;
51 import org.simantics.db.request.AsyncRead;
52 import org.simantics.db.service.ClusteringSupport;
53 import org.simantics.db.service.CollectionSupport;
54 import org.simantics.db.service.DirectQuerySupport;
55 import org.simantics.db.service.QueryControl;
56 import org.simantics.db.service.QueryControl.ControlProcedure;
57 import org.simantics.db.service.SerialisationSupport;
58 import org.simantics.db.service.StatementSupport;
59 import org.simantics.db.service.TransferableGraphSupport;
60 import org.simantics.graph.representation.Extensions;
61 import org.simantics.layer0.Layer0;
62 import org.simantics.utils.datastructures.Pair;
63 import org.simantics.utils.threads.logger.ITask;
64 import org.simantics.utils.threads.logger.ThreadLogger;
66 import gnu.trove.list.array.TIntArrayList;
67 import gnu.trove.map.hash.TIntIntHashMap;
68 import gnu.trove.map.hash.TLongObjectHashMap;
69 import gnu.trove.procedure.TIntProcedure;
70 import gnu.trove.procedure.TLongObjectProcedure;
71 import gnu.trove.set.TIntSet;
72 import gnu.trove.set.hash.THashSet;
73 import gnu.trove.set.hash.TIntHashSet;
75 public class Subgraphs {
77 public static String LOG_FILE = "export.log";
78 final static private boolean LOG = false;
79 final static private boolean DEBUG = false;
80 final static private boolean PARENT_DEBUG = DEBUG | false;
81 final static private boolean EXTERNAL_DEBUG = DEBUG | false;
82 final static private boolean ADVISOR_LOG = LOG & false;
83 final static private boolean EXPANSION_LOG = LOG & false;
84 final static private boolean INTERNAL_LOG = LOG & false;
85 final static private boolean COMPOSED_LOG = LOG & false;
86 final static private boolean RESOLVE_LOG = LOG & false;
87 final static private boolean CLASSIFY_LOG = LOG & false;
88 final static private boolean EXTERNAL_LOG = LOG & false;
89 final static private boolean PROFILE = false;
91 static enum WeakStatus {
96 static DataOutput log;
102 FileOutputStream stream = new FileOutputStream(LOG_FILE, false);
103 log = new DataOutputStream(stream);
104 } catch (FileNotFoundException e) {
111 private static void log(String line) {
114 log.write((line + "\n").getBytes());
115 } catch (IOException e) {
121 public static Collection<Resource> getParents(ReadGraph g, Resource r)
122 throws DatabaseException {
123 return getParents(g, r, false);
126 static class FastInternalRequest extends ResourceAsyncRead<Boolean> {
128 final DirectQuerySupport dqs;
129 final ConcurrentLinkedQueue<Resource> queue;
130 final Map<Resource, WeakStatus> weakInverses;
131 final Map<Resource, ExtentStatus> status;
133 public FastInternalRequest(DirectQuerySupport dqs, Resource resource,
134 Map<Resource, ExtentStatus> status,
135 Map<Resource, WeakStatus> weakInverses,
136 ConcurrentLinkedQueue<Resource> queue) {
139 this.status = status;
140 this.weakInverses = weakInverses;
145 public int getFlags() {
150 public void perform(AsyncReadGraph graph, final AsyncProcedure<Boolean> procedure) {
152 dqs.forEachDirectStatement(graph, resource, new AsyncProcedure<DirectStatements>() {
155 public void execute(AsyncReadGraph graph, DirectStatements ss) {
157 for(Statement statement : ss) {
158 if (status.get(statement.getObject()) == ExtentStatus.INTERNAL) continue;
159 WeakStatus status = weakInverses.get(statement.getPredicate());
160 if(status == WeakStatus.WEAK) continue;
161 else if (status == null) {
162 queue.add(statement.getPredicate());
166 procedure.execute(graph, ok);
170 public void exception(AsyncReadGraph graph, Throwable throwable) {
171 throwable.printStackTrace();
180 static class ClassifyStatementsRequest implements AsyncRead<Boolean> {
182 final Set<Resource> schedule;
183 final Map<Resource, WeakStatus> weakMap;
185 public ClassifyStatementsRequest(Set<Resource> schedule, Map<Resource, WeakStatus> weakMap) {
186 this.weakMap = weakMap;
187 this.schedule = schedule;
191 public int threadHash() {
196 public int getFlags() {
201 public void perform(AsyncReadGraph graph, final AsyncProcedure<Boolean> procedure) {
203 for (final Resource p : schedule) {
205 graph.forPossibleInverse(p, new AsyncProcedure<Resource>() {
207 private void register(AsyncReadGraph graph, Resource predicate, Resource superRelation, WeakStatus status) {
208 synchronized (weakMap) {
209 weakMap.put(predicate, status);
210 if(superRelation != null) weakMap.put(superRelation, status);
215 public void exception(AsyncReadGraph graph, Throwable throwable) {
216 throwable.printStackTrace();
220 public void execute(AsyncReadGraph graph, final Resource inverse) {
222 if (inverse == null) {
224 register(graph, p, null, WeakStatus.WEAK);
228 graph.forPossibleSuperrelation(inverse, new AsyncProcedure<Resource>() {
231 public void exception(AsyncReadGraph graph, Throwable throwable) {
232 throwable.printStackTrace();
236 public void execute(AsyncReadGraph graph, final Resource superRelation) {
238 if(superRelation != null && weakMap.containsKey(superRelation)) {
239 register(graph, p, null, weakMap.get(superRelation));
243 graph.forIsSubrelationOf(inverse, graph.getService(Layer0.class).IsRelatedTo, new AsyncProcedure<Boolean>() {
246 public void exception(AsyncReadGraph graph,Throwable throwable) {
247 throwable.printStackTrace();
251 public void execute(AsyncReadGraph graph,Boolean strong) {
252 register(graph, p, superRelation, strong ? WeakStatus.STRONG : WeakStatus.WEAK);
269 procedure.execute(graph, false);
275 private static Collection<Resource> getParents(ReadGraph g, Resource r, boolean isStrong) throws DatabaseException {
277 System.out.println("getParents " + NameUtils.getSafeName(g, r));
279 Layer0 l0 = Layer0.getInstance(g);
281 Collection<Resource> predicates = g.getPredicates(r);
283 // --- Consists Of ----------------------------------------------------
285 if (predicates.contains(l0.PartOf)) {
286 Collection<Resource> parents = g.getObjects(r, l0.PartOf);
287 if (parents.size() == 1)
289 ArrayList<Resource> libraryParents = new ArrayList<Resource>(1);
290 for (Resource p : parents)
291 if (g.isInstanceOf(p, l0.Library))
292 libraryParents.add(p);
293 if (!libraryParents.isEmpty())
294 return libraryParents;
299 // --- Ordered sets ---------------------------------------------------
302 Collection<Resource> parents = null;
303 for (Resource p : predicates)
304 if (g.isInstanceOf(p, l0.OrderedSet) && !p.equals(r)) {
306 parents = new ArrayList<Resource>(1);
309 if (parents != null) {
311 System.out.println("ORDERED SET");
318 return Collections.emptyList();
321 if (predicates.contains(l0.InverseOf)) {
323 Resource inv = g.getInverse(r);
324 return getParents(g, inv, true);
331 * If there are DependsOn parents, then IsRelatedTo parents are discarded
334 HashSet<Resource> result = new HashSet<Resource>();
335 for(Resource predicate : predicates) {
336 if(g.isSubrelationOf(predicate, l0.IsDependencyOf)) result.addAll(g.getObjects(r, predicate));
338 if(!result.isEmpty()) return result;
343 * At this point all Is Related To are parents.
346 for(Resource predicate : predicates) {
347 Resource inv = g.getPossibleInverse(predicate);
349 if(g.isSubrelationOf(inv, l0.IsRelatedTo)) result.addAll(g.getObjects(r, predicate));
358 Collection<Resource> invR = g.getObjects(r, b.IsRelatedTo_Inverse);
359 if (predicates.contains(b.InverseOf)) {
360 if (invR.size() > 1) {
363 .println("###########################################");
364 Resource inv = g.getInverse(r);
365 Collection<Resource> ret = new ArrayList<Resource>();
366 for (Statement pp : g.getStatements(r,
367 b.IsRelatedTo_Inverse))
368 if (!pp.getPredicate().equals(inv)) {
370 System.out.println("<"
371 + NameUtils.getSafeName(g, pp
374 + NameUtils.getSafeName(g, pp
377 + NameUtils.getSafeName(g, pp
378 .getObject()) + ">");
380 ret.add(pp.getObject());
384 // System.out.println("?????????????????");
385 Collection<Resource> invParents = getParents(g,
386 g.getInverse(r), true);
387 if (!invParents.isEmpty())
391 System.out.print("invR");
392 for (Resource res : invR)
393 System.out.print(" " + NameUtils.getSafeName(g, res));
394 System.out.println();
402 // public static String getIdentifier(ReadGraph g, Resource r)
403 // throws DatabaseException {
404 // Layer0 L0 = Layer0.getInstance(g);
405 // if (r.equals(g.getRootLibrary()))
407 // String name = g.getPossibleRelatedValue(r, L0.HasName);
410 // Collection<Resource> parents = getParents(g, r, true);
411 // if (parents.size() != 1)
413 // for (Resource p : parents) {
414 // String parentIdentifier = getIdentifier(g, p);
415 // if (parentIdentifier == null)
417 // return parentIdentifier + "/" + name;
424 static class Expansion extends AsyncReadRequest {
426 final private Collection<Resource> roots;
427 final Collection<DirectStatements>[] results;
428 final Collection<Resource>[] listElements;
430 public Expansion(Collection<Resource> roots, Collection<DirectStatements>[] results, Collection<Resource>[] listElements) {
432 this.results = results;
433 this.listElements = listElements;
437 public void run(AsyncReadGraph graph) {
439 QueryControl control = graph.getService(QueryControl.class);
440 final DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class);
442 final DomainStatementProcedure proc = new DomainStatementProcedure(dqs, graph.getService(StatementSupport.class), graph.getService(Layer0.class), results, listElements);
444 int slice = (int) (roots.size() / control.getAmountOfQueryThreads()) + 1;
446 final Resource[] rootArray = roots.toArray(Resource.NONE);
447 for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
449 final int start = i * slice;
450 final int end = Math.min(start + slice, rootArray.length);
452 control.schedule(graph, i, new ControlProcedure() {
455 public void execute(AsyncReadGraph graph) {
456 for (int index = start; index < end; index++) {
457 dqs.forEachDirectStatement(graph, rootArray[index], proc);
469 public int getFlags() {
475 static class Expansion2 extends AsyncReadRequest {
477 final private Collection<Resource> roots;
478 final Collection<DirectStatements>[] results;
479 final boolean ignoreVirtual;
481 public Expansion2(Collection<Resource> roots, Collection<DirectStatements>[] results) {
482 this(roots, results, true);
485 public Expansion2(Collection<Resource> roots, Collection<DirectStatements>[] results, boolean ignoreVirtual) {
487 this.results = results;
488 this.ignoreVirtual = ignoreVirtual;
492 public void run(AsyncReadGraph graph) {
494 QueryControl control = graph.getService(QueryControl.class);
495 final DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class);
497 final DomainStatementProcedure2 proc =
498 new DomainStatementProcedure2(results);
500 int slice = (int) (roots.size() / control.getAmountOfQueryThreads()) + 1;
502 final Resource[] rootArray = roots.toArray(Resource.NONE);
503 for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
505 final int start = i * slice;
506 final int end = Math.min(start + slice, rootArray.length);
508 control.schedule(graph, i, new ControlProcedure() {
510 public void execute(AsyncReadGraph graph) {
512 for (int index = start; index < end; index++) {
513 dqs.forEachDirectPersistentStatement(graph, rootArray[index], proc);
516 for (int index = start; index < end; index++) {
517 dqs.forEachDirectStatement(graph, rootArray[index], proc);
528 public int getFlags() {
534 static class DomainProcessor2 {
536 Serializer variantSerializer;
540 Set<Resource> fringe = null;
541 Set<Resource> exclusions = new HashSet<Resource>();
542 Set<Resource> internalDomain = new HashSet<Resource>();
543 Set<Resource> sharedExternalReferences = null;
544 TIntSet sharedExternalIds = null;
545 Set<Resource> sharedExternalFringe = null;
546 Set<Resource> predicates = null;
547 Set<Resource> isRelatedToPredicates = null;
548 Set<Resource> sharedPredicates = null;
549 TIntIntHashMap ids = null;
550 Map<Resource, Statement> specials = null;
551 Map<Resource, ExtentStatus> status = null;
552 Map<Resource, WeakStatus> weakInverses = null;
554 // final ArrayList<Double> priorityList = new ArrayList<Double>();
556 private long composedObjectCounter = 0;
557 private long fastInternalCounter = 0;
558 private long parentExternalCounter = 0;
559 private long fullInternalCounter = 0;
560 private long fullExternalCounter = 0;
562 private long startupTime = 0;
563 private long expandTime = 0;
564 private long fullResolveTime = 0;
565 private long fastResolveTime = 0;
566 private long otherStatementTime = 0;
567 private long parentResolveTime = 0;
568 private long extentSeedTime = 0;
569 private long composedPredicateTime = 0;
570 private long composedObjectTime = 0;
572 public void expand(ReadGraph graph, Set<Resource> fringe, Collection<DirectStatements>[] expansion) throws DatabaseException {
574 long start = System.nanoTime();
576 Collection<Collection<DirectStatements>[]> fullExpansion = new ArrayList<Collection<DirectStatements>[]>();
577 QueryControl control = graph.getService(QueryControl.class);
578 for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
579 expansion[i] = new ArrayList<DirectStatements>();
582 graph.syncRequest(new Expansion2(fringe, expansion));
586 expandTime += (System.nanoTime() - start);
590 public void classifyPredicates(ReadGraph graph, final Set<Resource> schedule) throws DatabaseException {
592 CollectionSupport cs = graph.getService(CollectionSupport.class);
594 final Layer0 L0 = Layer0.getInstance(graph);
596 long start = System.nanoTime();
598 final ConcurrentLinkedQueue<Resource> composedResult = new ConcurrentLinkedQueue<Resource>();
599 final ConcurrentLinkedQueue<Resource> singleResult = new ConcurrentLinkedQueue<Resource>();
600 final ConcurrentLinkedQueue<Resource> sharedResult = new ConcurrentLinkedQueue<Resource>();
601 final ConcurrentLinkedQueue<Pair<Resource, Resource>> singles = new ConcurrentLinkedQueue<Pair<Resource, Resource>>();
604 graph.syncRequest(new AsyncReadRequest() {
607 public void run(AsyncReadGraph graph) {
609 for (final Resource predicate : schedule) {
611 graph.forPossibleSuperrelation(predicate, new AsyncProcedure<Resource>() {
614 public void exception(AsyncReadGraph graph, Throwable throwable) {
615 throwable.printStackTrace();
619 public void execute(AsyncReadGraph graph, final Resource single) {
620 singles.add(Pair.make(predicate, single));
625 graph.forHasStatement(predicate, L0.SharedRange, new AsyncProcedure<Boolean>() {
628 public void exception(AsyncReadGraph graph, Throwable throwable) {
629 throwable.printStackTrace();
633 public void execute(AsyncReadGraph graph, final Boolean shared) {
634 if(shared) sharedResult.add(predicate);
646 final Set<Resource> singleSchedule = cs.createSet();
647 for(Pair<Resource, Resource> pair : singles) {
649 Resource single = pair.second;
650 if(single != null && predicates.add(single)) singleSchedule.add(single);
654 graph.syncRequest(new AsyncReadRequest() {
657 public void run(AsyncReadGraph graph) {
659 for (final Resource predicate : singleSchedule) {
661 graph.forIsSubrelationOf(predicate, L0.IsRelatedTo, new AsyncProcedure<Boolean>() {
664 public void exception(AsyncReadGraph graph, Throwable throwable) {
665 throwable.printStackTrace();
669 public void execute(AsyncReadGraph graph, Boolean strong) {
670 if (strong) singleResult.add(predicate);
681 isRelatedToPredicates.addAll(singleResult);
682 sharedPredicates.addAll(sharedResult);
684 final Set<Resource> specialSchedule = cs.createSet();
687 for(Pair<Resource, Resource> pair : singles) {
689 Resource single = pair.second;
691 if(isRelatedToPredicates.contains(single)) {
692 isRelatedToPredicates.add(pair.first);
695 specialSchedule.add(pair.first);
700 graph.syncRequest(new AsyncReadRequest() {
703 public void run(AsyncReadGraph graph) {
705 for (final Resource predicate : specialSchedule) {
707 graph.forIsSubrelationOf(predicate, L0.IsRelatedTo, new AsyncProcedure<Boolean>() {
710 public void exception(AsyncReadGraph graph, Throwable throwable) {
711 throwable.printStackTrace();
715 public void execute(AsyncReadGraph graph, Boolean composed) {
716 if (composed) composedResult.add(predicate);
727 isRelatedToPredicates.addAll(composedResult);
729 composedPredicateTime += (System.nanoTime() - start);
733 private Set<Resource> strongInverseSet = new HashSet<Resource>();
735 public void classifyPredicates(ReadGraph graph, final Collection<DirectStatements>[] expansion) throws DatabaseException {
737 CollectionSupport cs = graph.getService(CollectionSupport.class);
738 final Set<Resource> schedule = cs.createSet();
739 final Map<Resource, Resource> newPredicates = cs.createMap(Resource.class);
741 for (Collection<DirectStatements> coll : expansion)
742 for (DirectStatements stms : coll)
743 for(Statement stm : stms) {
745 Resource predicate = stm.getPredicate();
747 if(predicates.add(predicate)) {
748 Resource inverse = graph.getPossibleInverse(predicate);
749 schedule.add(predicate);
750 if(inverse != null) {
751 newPredicates.put(predicate, inverse);
752 if(predicates.add(inverse)) schedule.add(inverse);
759 classifyPredicates(graph, schedule);
761 for(Map.Entry<Resource, Resource> entry : newPredicates.entrySet()) {
762 // Inverse is strong => this has strong inverse
763 if(isRelatedToPredicates.contains(entry.getValue())) {
764 strongInverseSet.add(entry.getKey());
766 // This is strong => inverse has strong inverse
767 if(isRelatedToPredicates.contains(entry.getKey())) {
768 strongInverseSet.add(entry.getValue());
775 * Composed objects are internal. Mark them for expansion.
778 public void processFringe(ReadGraph graph, Collection<DirectStatements>[] expansion,
779 ObjectOutputStream otherStatementsOutput, ObjectOutputStream valueOutput) throws DatabaseException, IOException {
781 SerialisationSupport support = graph.getService(SerialisationSupport.class);
782 TransferableGraphSupport tgs = graph.getService(TransferableGraphSupport.class);
784 Layer0 L0 = Layer0.getInstance(graph);
786 long start = System.nanoTime();
788 for (Collection<DirectStatements> coll : expansion)
789 for (DirectStatements stms : coll) {
791 Resource subject = stms.getSubject();
793 boolean partOf = false;
794 for(Statement stm : stms) {
795 Resource predicate = stm.getPredicate();
796 if(L0.PartOf.equals(predicate)) {
802 ExtentStatus subjectStatus = status.get(subject);
803 if(LOG && subjectStatus != null) log("EXISTING STATUS FOR " + graph.getPossibleURI(subject) + " - " + subjectStatus);
804 if(subjectStatus == ExtentStatus.EXTERNAL) continue;
805 if(partOf && (subjectStatus == null) && graph.getPossibleURI(subject) != null) {
807 status.put(subject, ExtentStatus.EXTERNAL);
809 String uri = graph.getPossibleURI(subject);
810 if(uri == null) log("[EXTERNAL]: No URI for " + subject);
811 else log("[EXTERNAL] " + uri);
814 // Check for SharedRange statements
815 for(Statement stm : stms) {
816 Resource predicate = stm.getPredicate();
817 if(sharedPredicates.contains(predicate)) {
818 sharedExternalFringe.add(stm.getObject());
820 log("[SHARED EXTERNAL FRINGE]: " + NameUtils.getSafeName(graph, stm.getObject()));
827 boolean special = specials.containsKey(subject);
830 log("[SPECIAL] " + NameUtils.getSafeName(graph, subject));
834 status.put(subject, ExtentStatus.INTERNAL);
835 if(LOG) log("[INTERNAL] " + NameUtils.getSafeName(graph, subject));
837 int sId = support.getTransientId(subject);
839 if(graph.hasValue(subject)) {
840 Datatype dt = graph.getRelatedValue(subject, L0.HasDataType, Bindings.getBindingUnchecked(Datatype.class));
841 Binding b = Bindings.getBinding(dt);
842 Object _value = graph.getValue(subject, b);
843 Variant variant = new Variant(b, _value);
844 byte[] value = variantSerializer.serialize(variant);
845 if(LOG) log("[VALUE] " + NameUtils.getSafeName(graph, subject));
846 valueOutput.writeInt(sId);
847 valueOutput.writeInt(value.length);
848 assert (value.length > 0);
849 valueOutput.write(value);
852 TIntArrayList stream = new TIntArrayList();
854 for(Statement stm : stms) {
858 // System.err.println("stm=" + stm + " special=" + specials.get(subject));
862 Resource predicate = stm.getPredicate();
863 Resource object = stm.getObject();
865 ExtentStatus objectStatus = status.get(object);
868 if (isRelatedToPredicates.contains(predicate) && (objectStatus != ExtentStatus.EXCLUDED)) {
870 int pId = support.getTransientId(predicate);
871 int oId = support.getTransientId(object);
874 String s = NameUtils.getSafeName(graph, subject);
875 String p = NameUtils.getSafeName(graph, predicate);
876 String o = NameUtils.getSafeName(graph, object);
877 log("related=" + s + " - " + p + " - " + o);
883 if(objectStatus == null)
889 if(objectStatus == ExtentStatus.INTERNAL) {
891 // The inverse is also weak (or there is no inverse)
892 if(!strongInverseSet.contains(predicate)) {
894 int pId = support.getTransientId(predicate);
895 int oId = support.getTransientId(object);
901 String s = NameUtils.getSafeName(graph, subject);
902 String p = NameUtils.getSafeName(graph, predicate);
903 String o = NameUtils.getSafeName(graph, object);
904 log("fully weak internal=" + s + " - " + p + " - " + o + " - " + objectStatus);
910 String s = NameUtils.getSafeName(graph, subject);
911 String p = NameUtils.getSafeName(graph, predicate);
912 String o = NameUtils.getSafeName(graph, object);
913 log("strong inverse internals=" + s + " - " + p + " - " + o + " - " + objectStatus);
922 // System.err.println("stm=" + stm + " special=" + specials.get(subject));
924 Statement spec = specials.get(subject);
926 // This statement can be specially treated
927 if(stm.getPredicate().equals(spec.getPredicate()) && stm.getObject().equals(spec.getObject())) {
929 int pId = support.getTransientId(predicate);
930 int oId = support.getTransientId(object);
933 String s = NameUtils.getSafeName(graph, subject);
934 String p = NameUtils.getSafeName(graph, predicate);
935 String o = NameUtils.getSafeName(graph, object);
936 log("special=" + s + " - " + p + " - " + o);
947 String s = NameUtils.getSafeName(graph, subject);
948 String p = NameUtils.getSafeName(graph, predicate);
949 String o = NameUtils.getSafeName(graph, object);
950 log("weak with unknown object=" + s + " - " + p + " - " + o + " - " + objectStatus);
961 if(!stream.isEmpty()) {
962 otherStatementsOutput.writeInt(sId);
963 otherStatementsOutput.writeInt(stream.size() / 2);
964 for (int i = 0; i < stream.size(); i++)
965 otherStatementsOutput.writeInt(stream.getQuick(i));
972 composedObjectTime += System.nanoTime() - start;
976 public void process(ReadGraph graph,
977 ObjectOutputStream otherStatementsOutput,
978 ObjectOutputStream valueOutput)
979 throws DatabaseException, IOException {
981 this.variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT);
983 QueryControl control = graph.getService(QueryControl.class);
985 // System.err.println("Begin ConsistsOfProcess");
988 * Browse all stm = (s, ConsistsOf, o)
989 * � All internal URIs are found => from now on, if unidentified resource has PartOf it is external.
990 * � All s are internal
991 * � All o are internal
992 * � All stm are included
994 Pair<List<InternalEntry>,Set<Resource>> pair = ConsistsOfProcess.walk(graph, null, fringe, exclusions, true);
995 List<InternalEntry> entries = pair.first;
996 for(InternalEntry entry : entries) {
997 Resource r = entry.resource;
998 if (status.put(r, ExtentStatus.INTERNAL) == null) {
999 String URI = graph.getPossibleURI(r);
1000 if(URI != null) log("URI INTERNAL " + URI);
1001 else log("URI has no URI for " + r);
1003 internalDomain.add(r);
1007 for(Resource unnamedChild : pair.second) {
1008 if (status.put(unnamedChild, ExtentStatus.INTERNAL) == null) {
1009 fringe.add(unnamedChild);
1014 * This loop resolves the transitive closure of all p < IsRelatedTo such that p does not contain the SharedRange tag.
1015 * Such resources are guaranteed to be internal.
1017 while(!fringe.isEmpty()) {
1019 // System.err.println("Process Fringe with " + fringe.size() + ".");
1021 Collection<DirectStatements>[] expansion = new ArrayList[control.getAmountOfQueryThreads()];
1023 // System.err.println("-expand");
1026 expand(graph, fringe, expansion);
1032 * -Internal / External
1035 // System.err.println("-classify");
1037 classifyPredicates(graph, expansion);
1040 * for stms in [stms]
1041 * if stms contains predicate PartOf => s is External
1042 * else s is Internal
1043 * for all stm=(s,p,o) in stms
1044 * if p <R IsRelatedTo => stm is included
1048 // System.err.println("-process");
1050 processFringe(graph, expansion, otherStatementsOutput, valueOutput);
1054 while(!sharedExternalFringe.isEmpty()) {
1056 Collection<DirectStatements>[] expansion = new ArrayList[control.getAmountOfQueryThreads()];
1057 expand(graph, sharedExternalFringe, expansion);
1059 for (Collection<DirectStatements> coll : expansion)
1060 for (DirectStatements stms : coll) {
1062 Resource subject = stms.getSubject();
1063 ExtentStatus subjectStatus = status.get(subject);
1065 if(ExtentStatus.INTERNAL == subjectStatus) {
1067 if(internalDomain.contains(subject)) continue;
1069 status.put(subject, ExtentStatus.EXTERNAL);
1070 sharedExternalReferences.add(subject);
1073 log("[SHARED EXTERNAL REFERENCE]: " + NameUtils.getSafeName(graph, subject));
1076 for(Statement stm : stms) {
1077 Resource predicate = stm.getPredicate();
1078 if (isRelatedToPredicates.contains(predicate)) {
1079 sharedExternalFringe.add(stm.getObject());
1089 System.out.println(composedObjectCounter + " " + fastInternalCounter
1090 + " " + parentExternalCounter + " "
1091 + fullExternalCounter + " " + fullInternalCounter);
1098 static class DomainProcessor {
1100 Serializer variantSerializer;
1104 Set<Resource> predicates = null;
1105 Set<Resource> composedPredicates = null;
1106 Set<Resource> expansionSeeds = null;
1107 Map<Resource, Integer> ids = null;
1108 Map<Resource, ExtentStatus> status = null;
1109 Map<Resource, WeakStatus> weakInverses = null;
1111 final Set<SubgraphAdvisor> advisors;
1112 final ArrayList<Double> priorityList = new ArrayList<Double>();
1114 private long composedObjectCounter = 0;
1115 private long fastInternalCounter = 0;
1116 private long parentExternalCounter = 0;
1117 private long fullInternalCounter = 0;
1118 private long fullExternalCounter = 0;
1120 private long startupTime = 0;
1121 private long expandTime = 0;
1122 private long fullResolveTime = 0;
1123 private long fastResolveTime = 0;
1124 private long otherStatementTime = 0;
1125 private long parentResolveTime = 0;
1126 private long extentSeedTime = 0;
1127 private long composedPredicateTime = 0;
1128 private long composedObjectTime = 0;
1130 public DomainProcessor(Set<SubgraphAdvisor> advisors) {
1131 this.advisors = advisors;
1132 HashSet<Double> prioritySet = new HashSet<Double>();
1133 for (SubgraphAdvisor advisor : advisors)
1134 prioritySet.add(advisor.priority());
1135 priorityList.addAll(prioritySet);
1136 Collections.sort(priorityList);
1139 public void expand(ReadGraph graph, Collection<DirectStatements>[] expansion, Set<Resource> schedule) throws DatabaseException {
1141 long start = System.nanoTime();
1144 // System.out.println("expanding " + expansionSeeds.size() + " resources.");
1146 QueryControl control = graph.getService(QueryControl.class);
1147 // final Collection<DirectStatements>[] results = new ArrayList[control.getAmountOfQueryThreads()];
1148 final ArrayList<Resource>[] listElements = new ArrayList[control.getAmountOfQueryThreads()];
1149 for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
1150 // results[i] = new ArrayList<DirectStatements>();
1151 listElements[i] = new ArrayList<Resource>();
1155 // for(Resource r : expansionSeeds)
1156 // System.out.println("Expanding " + NameUtils.getSafeName(graph, r, true));
1159 graph.syncRequest(new Expansion(expansionSeeds, expansion, listElements));
1160 for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
1161 // for (DirectStatements s : results[i]) {
1162 // expansion.put(s.getSubject(), s);
1164 for (Resource s : listElements[i]) {
1166 // if(status.put(s, ExtentStatus.INTERNAL) == null) {
1167 // ids.put(s, id++);
1172 expandTime += (System.nanoTime() - start);
1176 public void extractComposedPredicates(ReadGraph graph, final Collection<DirectStatements>[] expansion) throws DatabaseException {
1178 long start = System.nanoTime();
1180 CollectionSupport cs = graph.getService(CollectionSupport.class);
1182 final ConcurrentLinkedQueue<Resource> composedResult = new ConcurrentLinkedQueue<Resource>();
1183 final ConcurrentLinkedQueue<Resource> singleResult = new ConcurrentLinkedQueue<Resource>();
1184 final ConcurrentLinkedQueue<Pair<Resource, Resource>> singles = new ConcurrentLinkedQueue<Pair<Resource, Resource>>();
1186 final Set<Resource> schedule = cs.createSet();
1188 for (Collection<DirectStatements> coll : expansion)
1189 for (DirectStatements stms : coll)
1190 for(Statement stm : stms) {
1191 Resource predicate = stm.getPredicate();
1192 if(predicates.add(predicate)) schedule.add(predicate);
1196 graph.syncRequest(new AsyncReadRequest() {
1199 public void run(AsyncReadGraph graph) {
1201 for (final Resource predicate : schedule) {
1203 graph.forPossibleSuperrelation(predicate, new AsyncProcedure<Resource>() {
1206 public void exception(AsyncReadGraph graph, Throwable throwable) {
1207 throwable.printStackTrace();
1211 public void execute(AsyncReadGraph graph, final Resource single) {
1212 singles.add(Pair.make(predicate, single));
1223 // Determine singles
1224 final Set<Resource> singleSchedule = cs.createSet();
1225 for(Pair<Resource, Resource> pair : singles) {
1227 Resource single = pair.second;
1228 if(single != null && predicates.add(single)) singleSchedule.add(single);
1232 graph.syncRequest(new AsyncReadRequest() {
1235 public void run(AsyncReadGraph graph) {
1237 for (final Resource predicate : singleSchedule) {
1239 graph.forIsSubrelationOf(predicate, graph.getService(Layer0.class).IsComposedOf, new AsyncProcedure<Boolean>() {
1242 public void exception(AsyncReadGraph graph, Throwable throwable) {
1243 throwable.printStackTrace();
1247 public void execute(AsyncReadGraph graph, Boolean composed) {
1248 if (composed) singleResult.add(predicate);
1259 composedPredicates.addAll(singleResult);
1261 final Set<Resource> specialSchedule = cs.createSet();
1264 for(Pair<Resource, Resource> pair : singles) {
1266 Resource single = pair.second;
1267 if(single != null) {
1268 if(composedPredicates.contains(single)) {
1269 composedPredicates.add(pair.first);
1272 specialSchedule.add(pair.first);
1277 graph.syncRequest(new AsyncReadRequest() {
1280 public void run(AsyncReadGraph graph) {
1282 for (final Resource predicate : specialSchedule) {
1284 graph.forIsSubrelationOf(predicate, graph.getService(Layer0.class).IsComposedOf, new AsyncProcedure<Boolean>() {
1287 public void exception(AsyncReadGraph graph, Throwable throwable) {
1288 throwable.printStackTrace();
1292 public void execute(AsyncReadGraph graph, Boolean composed) {
1293 if (composed) composedResult.add(predicate);
1304 composedPredicates.addAll(composedResult);
1306 composedPredicateTime += (System.nanoTime() - start);
1311 * Composed objects are internal. Mark them for expansion.
1314 public void collectComposedObjects(ReadGraph graph, Collection<DirectStatements>[] expansion, Set<Resource> typeTodo, Set<Resource> objectTodo,
1315 Set<Resource> predicateTodo) throws DatabaseException {
1317 long start = System.nanoTime();
1319 Layer0 l0 = Layer0.getInstance(graph);
1321 for (Collection<DirectStatements> coll : expansion)
1322 for (DirectStatements stms : coll)
1323 for(Statement stm : stms) {
1325 Resource predicate = stm.getPredicate();
1326 Resource object = stm.getObject();
1328 if (composedPredicates.contains(predicate)) {
1330 ExtentStatus existing = status.put(object, ExtentStatus.INTERNAL);
1331 if(existing == null) {
1332 ids.put(object, id++);
1333 composedObjectCounter++;
1334 expansionSeeds.add(object);
1335 // System.err.println("internal: " + NameUtils.getSafeName(graph, object, true));
1336 if(LOG) log("[INTERNAL] (composed object) " + NameUtils.getSafeName(graph, object, true));
1337 } else if (existing == ExtentStatus.EXCLUDED) {
1338 System.err.println("preExcluded: " + NameUtils.getSafeName(graph, object, true));
1339 status.put(object, ExtentStatus.EXCLUDED);
1340 } else if (existing == ExtentStatus.EXTERNAL) {
1341 System.err.println("preExternal: " + NameUtils.getSafeName(graph, object, true));
1342 status.put(object, ExtentStatus.EXTERNAL);
1347 // System.err.println("internal2: " + NameUtils.getSafeName(graph, object, true));
1349 if (!status.containsKey(object)) {
1350 if (l0.InstanceOf.equalsResource(predicate)) {
1351 typeTodo.add(object);
1353 objectTodo.add(object);
1357 if (!status.containsKey(predicate)) {
1358 predicateTodo.add(predicate);
1364 composedObjectTime += System.nanoTime() - start;
1368 public void writeOtherStatements(ReadGraph graph, Collection<Collection<DirectStatements>[]> expansion, ObjectOutputStream composedStatementsOutput, ObjectOutputStream otherStatementsOutput,
1369 ObjectOutputStream valueOutput) throws DatabaseException {
1371 long start = System.nanoTime();
1373 Layer0 l0 = Layer0.getInstance(graph);
1374 SerialisationSupport support = graph.getService(SerialisationSupport.class);
1375 TransferableGraphSupport tgs = graph.getService(TransferableGraphSupport.class);
1377 TIntArrayList other = new TIntArrayList();
1378 TIntArrayList composed = new TIntArrayList();
1382 for (Collection<DirectStatements>[] colls : expansion)
1383 for (Collection<DirectStatements> coll : colls)
1384 for (DirectStatements stms : coll) {
1386 Resource subject = stms.getSubject();
1387 composed.resetQuick();
1389 int sId = support.getTransientId(subject);
1391 composedStatementsOutput.writeInt(sId);
1393 if(graph.hasValue(subject)) {
1394 Datatype dt = graph.getRelatedValue(subject, l0.HasDataType, Bindings.getBindingUnchecked(Datatype.class));
1395 Binding b = Bindings.getBinding(dt);
1396 Object _value = graph.getValue(subject, b);
1397 Variant variant = new Variant(b, _value);
1398 byte[] value = variantSerializer.serialize(variant);
1399 if(LOG) log("[VALUE] " + NameUtils.getSafeName(graph, subject));
1400 valueOutput.writeInt(sId);
1401 valueOutput.writeInt(value.length);
1402 assert (value.length > 0);
1403 valueOutput.write(value);
1406 for (Statement s : stms) {
1408 Resource object = s.getObject();
1409 Resource predicate = s.getPredicate();
1411 ExtentStatus objectStatus = status.get(object);
1413 if(objectStatus == ExtentStatus.INTERNAL) {
1414 composed.add(support.getTransientId(predicate));
1415 composed.add(support.getTransientId(object));
1416 if(LOG) log("[COMPOSED] (internal object) " + NameUtils.toIdString(graph, s));
1417 } else if (l0.InstanceOf.equalsResource(predicate)) {
1418 composed.add(support.getTransientId(predicate));
1419 composed.add(support.getTransientId(object));
1420 if(LOG) log("[COMPOSED] (instanceOf) " + NameUtils.toIdString(graph, s));
1421 } else if (l0.SubrelationOf.equalsResource(predicate)) {
1422 composed.add(support.getTransientId(predicate));
1423 composed.add(support.getTransientId(object));
1424 if(LOG) log("[COMPOSED] (subrelationOf) " + NameUtils.toIdString(graph, s));
1426 if(objectStatus == ExtentStatus.EXTERNAL) {
1428 System.out.println("other " + NameUtils.toIdString(graph, s));
1429 //System.out.println("other.add " + predicate + " - " + object);
1430 other.add(support.getTransientId(predicate));
1431 other.add(support.getTransientId(object));
1432 if(LOG) log("[OTHER] (object is external) " + NameUtils.toIdString(graph, s));
1438 if(!other.isEmpty()) {
1439 otherStatementsOutput.writeInt(sId);
1440 otherStatementsOutput.writeInt(other.size() / 2);
1441 for (int i = 0; i < other.size(); i++)
1442 otherStatementsOutput.writeInt(other.getQuick(i));
1446 composedStatementsOutput.writeInt(composed.size() / 2);
1447 for (int i = 0; i < composed.size(); i++)
1448 composedStatementsOutput.writeInt(composed.getQuick(i));
1452 } catch (IOException e) {
1453 e.printStackTrace();
1456 otherStatementTime += (System.nanoTime() - start);
1460 boolean hasStrictParents(ReadGraph g, Resource r)
1461 throws DatabaseException {
1462 if (g.getPossibleURI(r) != null)
1467 public boolean getExpansionSeedsFromExtents(ReadGraph graph, final Collection<DirectStatements>[] expansion) throws DatabaseException {
1469 long start = System.nanoTime();
1471 final ConcurrentLinkedQueue<Resource> accepts = new ConcurrentLinkedQueue<Resource>();
1474 * Determine statements which could accept statements with todo
1477 search: for (Double priority : priorityList) {
1479 for (final SubgraphAdvisor advisor : advisors) {
1481 if (advisor.priority() > 0)
1484 if (advisor.priority() == priority) {
1486 graph.syncRequest(new ReadRequest() {
1489 public void run(ReadGraph graph) throws DatabaseException {
1491 for (Collection<DirectStatements> coll : expansion)
1492 for (DirectStatements stms : coll)
1493 for(final Statement stm : stms) {
1495 advisor.advise(graph, stm, new AsyncProcedure<Boolean>() {
1498 public void exception(AsyncReadGraph graph, Throwable throwable) {
1499 throwable.printStackTrace();
1503 public void execute(AsyncReadGraph graph, Boolean accept) {
1505 accepts.add(stm.getObject());
1517 if (!accepts.isEmpty())
1522 CollectionSupport cs = graph.getService(CollectionSupport.class);
1523 Set<Resource> schedule = cs.createSet();
1524 for (Resource r : accepts) {
1525 if(!status.containsKey(r))
1529 extentSeedTime += (System.nanoTime() - start);
1531 if (schedule.isEmpty())
1534 fastResolve(graph, schedule);
1535 uriResolve(graph, schedule);
1536 fullResolve(graph, schedule, "accepts");
1542 ConcurrentLinkedQueue<Resource> fastInternals = new ConcurrentLinkedQueue<Resource>();
1544 public void fastResolve(ReadGraph graph, final Set<Resource> rs)
1545 throws DatabaseException {
1546 // This collects and resolves weaks
1547 if(fastResolveLoop(graph, rs))
1548 // Weaks are now resolved
1549 fastResolveLoop(graph, rs);
1552 public boolean fastResolveLoop(ReadGraph graph, final Set<Resource> rs)
1553 throws DatabaseException {
1555 long start = System.nanoTime();
1557 final ConcurrentLinkedQueue<Resource> weakSchedule = new ConcurrentLinkedQueue<Resource>();
1559 graph.syncRequest(new AsyncRead<Boolean>() {
1562 public int threadHash() {
1567 public int getFlags() {
1572 public void perform(AsyncReadGraph graph,
1573 AsyncProcedure<Boolean> procedure) {
1575 QueryControl control = graph.getService(QueryControl.class);
1576 final DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class);
1578 int slice = (int) (rs.size() / control
1579 .getAmountOfQueryThreads()) + 1;
1581 final Resource[] rootArray = rs.toArray(Resource.NONE);
1582 for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
1584 final int start = i * slice;
1585 final int end = Math.min(start + slice,
1588 control.schedule(graph, i, new ControlProcedure() {
1591 public void execute(AsyncReadGraph graph) {
1593 for (int index = start; index < end; index++) {
1595 final Resource r = rootArray[index];
1597 //if (status.containsKey(r)) continue;
1599 graph.asyncRequest(new FastInternalRequest(dqs, r, status, weakInverses, weakSchedule),new AsyncProcedure<Boolean>() {
1602 public void exception(AsyncReadGraph graph,Throwable throwable) {
1603 throwable.printStackTrace();
1607 public void execute(AsyncReadGraph graph,Boolean isInternal) {
1609 fastInternals.add(r);
1623 procedure.execute(graph, true);
1629 if (!weakSchedule.isEmpty()) {
1630 THashSet<Resource> weaks = new THashSet<Resource>(weakSchedule);
1632 for (Resource p : weakSchedule)
1633 System.out.println("classify "
1634 + NameUtils.getSafeName(graph, p));
1635 graph.syncRequest(new ClassifyStatementsRequest(weaks, weakInverses));
1638 for (Resource r : fastInternals) {
1640 if (status.put(r, ExtentStatus.INTERNAL) == null) {
1641 if(LOG) log("[INTERNAL] (fast) " + NameUtils.getSafeName(graph, r, true));
1643 fastInternalCounter++;
1644 expansionSeeds.add(r);
1648 fastResolveTime += (System.nanoTime() - start);
1650 return !weakSchedule.isEmpty();
1654 private ExtentStatus resolveExtent(ReadGraph graph, Resource r, Map<Resource, ExtentStatus> statuses, Set<Resource> expansionSeeds, THashSet<Resource> pending,
1655 ArrayList<Resource> stack) throws DatabaseException {
1657 ExtentStatus current = statuses.get(r);
1658 if(current != null) return current;
1660 if (pending.contains(r))
1661 return ExtentStatus.PENDING;
1663 // In order to break cyclic dependencies
1667 System.out.println("resolveExtent "
1668 + NameUtils.getSafeName(graph, r));
1670 ExtentStatus status = ExtentStatus.INTERNAL;
1671 for (Resource p : getParents(graph, r)) {
1673 ExtentStatus ps = statuses.get(p);
1674 System.out.println(" parent " + NameUtils.getSafeName(graph, p) + "(" + ps + ")");
1676 switch (resolveExtent(graph, p, statuses,
1677 expansionSeeds, pending, stack)) {
1679 return ExtentStatus.EXTERNAL;
1681 status = ExtentStatus.PENDING;
1684 if (status == ExtentStatus.INTERNAL) {
1688 System.out.println(NameUtils.getSafeName(graph, r, true)
1694 public void uriResolve(ReadGraph graph, final Set<Resource> todo)
1695 throws DatabaseException {
1697 long start = System.nanoTime();
1699 for(Resource r : todo) System.out.println("uriResolve " +
1700 NameUtils.getSafeName(graph, r));
1702 final ConcurrentSkipListSet<Resource> found = new ConcurrentSkipListSet<Resource>();
1704 graph.syncRequest(new AsyncReadRequest() {
1707 public void run(AsyncReadGraph graph) {
1709 for (final Resource r : todo) {
1711 // System.out.println("uriresolve before " + r);
1713 if (status.containsKey(r)) continue;
1715 // System.out.println("uriresolve " + r);
1717 graph.forURI(r, new AsyncProcedure<String>() {
1720 public void exception(AsyncReadGraph graph, Throwable throwable) {
1721 throwable.printStackTrace();
1725 public void execute(AsyncReadGraph graph, String uri) {
1729 // System.out.println("uriresolve has uri "
1733 parentExternalCounter++;
1738 // System.out.println("uriresolve ask inverse "
1741 graph.forPossibleInverse(r, new AsyncProcedure<Resource>() {
1744 public void exception(AsyncReadGraph graph, Throwable throwable) {
1745 throwable.printStackTrace();
1749 public void execute(AsyncReadGraph graph, Resource inverse) {
1751 if (inverse != null) {
1753 graph.forURI(inverse, new AsyncProcedure<String>() {
1756 public void exception(AsyncReadGraph graph, Throwable throwable) {
1757 throwable.printStackTrace();
1761 public void execute(AsyncReadGraph graph, String uri) {
1766 parentExternalCounter++;
1792 todo.removeAll(found);
1793 for(Resource r : found) {
1794 status.put(r, ExtentStatus.EXTERNAL);
1795 if(LOG) log("[EXTERNAL] (uriResolve) " + NameUtils.getSafeName(graph, r, true));
1798 parentResolveTime += System.nanoTime() - start;
1802 public void fullResolve(ReadGraph graph, Collection<Resource> rs,
1803 String koss) throws DatabaseException {
1805 long start = System.nanoTime();
1807 for (final Resource r : rs) {
1809 if(status.containsKey(r)) continue;
1811 THashSet<Resource> pending = new THashSet<Resource>();
1812 ArrayList<Resource> stack = new ArrayList<Resource>();
1814 ExtentStatus s = resolveExtent(graph, r, status, expansionSeeds, pending, stack);
1815 if (ExtentStatus.INTERNAL == s || ExtentStatus.PENDING == s) {
1816 if (status.put(r, ExtentStatus.INTERNAL) == null) {
1817 if(LOG) log("[INTERNAL] (resolveExtent) " + NameUtils.getSafeName(graph, r, true));
1819 fullInternalCounter++;
1820 expansionSeeds.add(r);
1823 if (ExtentStatus.EXTERNAL == s) {
1824 if (status.put(r, ExtentStatus.EXTERNAL) == null) {
1825 if(LOG) log("[EXTERNAL] (resolveExtent) " + NameUtils.getSafeName(graph, r, true));
1826 fullExternalCounter++;
1832 fullResolveTime += (System.nanoTime() - start);
1836 public void process(ReadGraph graph,
1837 ObjectOutputStream composedStatementsOutput,
1838 ObjectOutputStream otherStatementsOutput,
1839 ObjectOutputStream valueOutput)
1840 throws DatabaseException {
1842 this.variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT);
1844 CollectionSupport cs = graph.getService(CollectionSupport.class);
1846 Set<Resource> typeTodo = cs.createSet();
1847 Set<Resource> objectTodo = cs.createSet();
1848 Set<Resource> predicateTodo = cs.createSet();
1850 Collection<Collection<DirectStatements>[]> fullExpansion = new ArrayList<Collection<DirectStatements>[]>();
1854 QueryControl control = graph.getService(QueryControl.class);
1855 Collection<DirectStatements>[] expansion = new ArrayList[control.getAmountOfQueryThreads()];
1856 for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
1857 expansion[i] = new ArrayList<DirectStatements>();
1860 // Expand expansionSeeds
1861 expand(graph, expansion, objectTodo);
1863 // Start collecting new seeds
1864 expansionSeeds = cs.createSet();
1866 // Collect predicates which are <R IsComposedOf
1867 extractComposedPredicates(graph, expansion);
1869 // Make composed objects internal and make sure they are further
1871 collectComposedObjects(graph, expansion, typeTodo, objectTodo, predicateTodo);
1874 * Use the expansion seed heuristic to find new resources to
1875 * expand before full analysis.
1877 getExpansionSeedsFromExtents(graph, expansion);
1879 fullExpansion.add(expansion);
1881 } while (!expansionSeeds.isEmpty());
1883 fastResolve(graph, objectTodo);
1884 uriResolve(graph, predicateTodo);
1885 uriResolve(graph, objectTodo);
1886 uriResolve(graph, typeTodo);
1887 fullResolve(graph, objectTodo, "objectTodo");
1888 fullResolve(graph, predicateTodo, "predicateTodo");
1889 fullResolve(graph, typeTodo, "typeTodo");
1891 writeOtherStatements(graph, fullExpansion, composedStatementsOutput, otherStatementsOutput, valueOutput);
1894 System.out.println(composedObjectCounter + " " + fastInternalCounter
1895 + " " + parentExternalCounter + " "
1896 + fullExternalCounter + " " + fullInternalCounter);
1903 public static void getDomain2(ReadGraph graph, TIntIntHashMap ids,
1904 Collection<Resource> roots, Map<Resource, ExtentStatus> preStatus,
1905 Map<Resource, Statement> specials,
1906 ObjectOutputStream otherStatementsOutput,
1907 ObjectOutputStream valueOutput,
1908 TreeMap<String, Variant> extensions,
1909 TIntHashSet excludedShared) throws DatabaseException {
1911 ITask task = ThreadLogger.getInstance().begin("getDomain2");
1913 final DomainProcessor2 processor = new DomainProcessor2();
1915 processor.startupTime = System.nanoTime();
1917 Layer0 l0 = Layer0.getInstance(graph);
1919 CollectionSupport cs = graph.getService(CollectionSupport.class);
1920 SerialisationSupport support = graph.getService(SerialisationSupport.class);
1922 processor.ids = ids;
1923 processor.specials = specials;
1924 processor.status = cs.createMap(ExtentStatus.class);
1925 processor.weakInverses = cs.createMap(WeakStatus.class);
1926 processor.predicates = cs.createSet();
1927 processor.isRelatedToPredicates = cs.createSet();
1928 processor.sharedPredicates = cs.createSet();
1929 // processor.expansionSeeds = cs.createSet();
1931 for(Map.Entry<Resource, ExtentStatus> entry : preStatus.entrySet()) {
1932 processor.status.put(entry.getKey(), entry.getValue());
1933 if(ExtentStatus.EXCLUDED.equals(entry.getValue())) processor.exclusions.add(entry.getKey());
1936 // for (Resource r : excluded) {
1937 // processor.status.put(r, ExtentStatus.EXCLUDED);
1940 Resource rootLibrary = graph.getResource("http:/");
1942 if (!roots.contains(rootLibrary))
1943 processor.status.put(rootLibrary, ExtentStatus.EXTERNAL);
1945 for (Resource root : roots) {
1946 processor.status.put(root, ExtentStatus.INTERNAL);
1947 //processor.ids.put(support.getTransientId(root), processor.ids.size());
1948 for (Resource owner : graph.getObjects(root, l0.IsOwnedBy)) {
1949 processor.status.put(owner, ExtentStatus.EXTERNAL);
1953 processor.startupTime = System.nanoTime() - processor.startupTime;
1955 processor.fringe = new HashSet<Resource>();
1956 processor.fringe.addAll(roots);
1958 processor.internalDomain.addAll(roots);
1960 processor.sharedExternalReferences = new HashSet<Resource>();
1961 processor.sharedExternalFringe = new HashSet<Resource>();
1965 processor.process(graph, otherStatementsOutput, valueOutput);
1967 } catch (IOException e) {
1968 e.printStackTrace();
1971 for(Resource r : processor.sharedExternalReferences) excludedShared.add(support.getTransientId(r));
1973 ClusteringSupport cls = graph.getService(ClusteringSupport.class);
1974 TLongObjectHashMap<TIntArrayList> clusterMap = new TLongObjectHashMap<TIntArrayList>();
1975 for(Map.Entry<Resource, ExtentStatus> entry : processor.status.entrySet()) {
1976 if(ExtentStatus.INTERNAL == entry.getValue()) {
1978 long cluster = cls.getCluster(entry.getKey());
1979 TIntArrayList list = clusterMap.get(cluster);
1981 list = new TIntArrayList();
1982 clusterMap.put(cluster, list);
1984 list.add(support.getTransientId(entry.getKey()));
1988 final TIntArrayList clustering = new TIntArrayList();
1989 clusterMap.forEachEntry(new TLongObjectProcedure<TIntArrayList>() {
1992 public boolean execute(long cluster, TIntArrayList b) {
1993 clustering.add(b.size());
1994 b.forEach(new TIntProcedure() {
1997 public boolean execute(int rId) {
1998 processor.ids.put(rId, processor.id++);
2008 extensions.put(Extensions.CLUSTERING, new Variant(Bindings.INT_ARRAY, clustering.toArray()));
2010 long total = processor.startupTime + processor.expandTime
2011 + processor.composedPredicateTime
2012 + processor.composedObjectTime + processor.extentSeedTime
2013 + processor.fullResolveTime + processor.fastResolveTime +
2014 + processor.parentResolveTime + processor.otherStatementTime;
2017 System.out.println("startup took " + 1e-9 * processor.startupTime
2019 System.out.println("expand took " + 1e-9 * processor.expandTime
2021 System.out.println("composedPredicates took " + 1e-9
2022 * processor.composedPredicateTime + "s.");
2023 System.out.println("composedObjects took " + 1e-9
2024 * processor.composedObjectTime + "s.");
2025 System.out.println("extentSeeding took " + 1e-9
2026 * processor.extentSeedTime + "s.");
2027 System.out.println("fullResolve took " + 1e-9
2028 * processor.fullResolveTime + "s.");
2029 System.out.println("fastResolve took " + 1e-9
2030 * processor.fastResolveTime + "s.");
2031 System.out.println("parentResolve took " + 1e-9
2032 * processor.parentResolveTime + "s.");
2033 System.out.println("otherStatements took " + 1e-9
2034 * processor.otherStatementTime + "s.");
2035 System.out.println("total " + 1e-9 * total + "s.");
2044 public static void getDomain(ReadGraph graph, Map<Resource, Integer> ids,
2045 Collection<Resource> roots, Map<Resource, ExtentStatus> preStatus, Set<SubgraphAdvisor> advisors,
2046 ObjectOutputStream composedStatementsOutput,
2047 ObjectOutputStream otherStatementsOutput,
2048 ObjectOutputStream valueOutput) throws DatabaseException {
2050 ITask task = ThreadLogger.getInstance().begin("getDomain");
2052 DomainProcessor processor = new DomainProcessor(advisors);
2054 processor.startupTime = System.nanoTime();
2056 Layer0 l0 = Layer0.getInstance(graph);
2058 CollectionSupport cs = graph.getService(CollectionSupport.class);
2060 processor.ids = ids;
2061 processor.status = cs.createMap(ExtentStatus.class);
2062 processor.weakInverses = cs.createMap(WeakStatus.class);
2063 processor.predicates = cs.createSet();
2064 processor.composedPredicates = cs.createSet();
2065 processor.expansionSeeds = cs.createSet();
2067 for(Map.Entry<Resource, ExtentStatus> entry : preStatus.entrySet()) {
2068 processor.status.put(entry.getKey(), entry.getValue());
2071 // for (Resource r : excluded) {
2072 // processor.status.put(r, ExtentStatus.EXCLUDED);
2075 if (!roots.contains(graph.getRootLibrary()))
2076 processor.status.put(graph.getRootLibrary(), ExtentStatus.EXTERNAL);
2078 for (Resource root : roots) {
2079 processor.status.put(root, ExtentStatus.INTERNAL);
2080 processor.ids.put(root, processor.id++);
2081 for (Resource owner : graph.getObjects(root, l0.IsOwnedBy)) {
2082 processor.status.put(owner, ExtentStatus.EXTERNAL);
2087 processor.expansionSeeds.addAll(roots);
2089 processor.startupTime = System.nanoTime() - processor.startupTime;
2091 while (!processor.expansionSeeds.isEmpty()) {
2093 processor.process(graph, composedStatementsOutput,
2094 otherStatementsOutput, valueOutput);
2098 long total = processor.startupTime + processor.expandTime
2099 + processor.composedPredicateTime
2100 + processor.composedObjectTime + processor.extentSeedTime
2101 + processor.fullResolveTime + processor.fastResolveTime +
2102 + processor.parentResolveTime + processor.otherStatementTime;
2105 System.out.println("startup took " + 1e-9 * processor.startupTime
2107 System.out.println("expand took " + 1e-9 * processor.expandTime
2109 System.out.println("composedPredicates took " + 1e-9
2110 * processor.composedPredicateTime + "s.");
2111 System.out.println("composedObjects took " + 1e-9
2112 * processor.composedObjectTime + "s.");
2113 System.out.println("extentSeeding took " + 1e-9
2114 * processor.extentSeedTime + "s.");
2115 System.out.println("fullResolve took " + 1e-9
2116 * processor.fullResolveTime + "s.");
2117 System.out.println("fastResolve took " + 1e-9
2118 * processor.fastResolveTime + "s.");
2119 System.out.println("parentResolve took " + 1e-9
2120 * processor.parentResolveTime + "s.");
2121 System.out.println("otherStatements took " + 1e-9
2122 * processor.otherStatementTime + "s.");
2123 System.out.println("total " + 1e-9 * total + "s.");