c7e697e6291773b58e44d8bd3a8c2afbf0952442
[simantics/platform.git] / bundles / org.simantics.db.layer0 / src / org / simantics / db / layer0 / util / Subgraphs.java
1 /*******************************************************************************
2  * Copyright (c) 2007, 2010 Association for Decentralized Information Management
3  * in Industry THTH ry.
4  * All rights reserved. This program and the accompanying materials
5  * are made available under the terms of the Eclipse Public License v1.0
6  * which accompanies this distribution, and is available at
7  * http://www.eclipse.org/legal/epl-v10.html
8  *
9  * Contributors:
10  *     VTT Technical Research Centre of Finland - initial API and implementation
11  *******************************************************************************/
12 package org.simantics.db.layer0.util;
13
14 import java.io.DataOutput;
15 import java.io.DataOutputStream;
16 import java.io.FileNotFoundException;
17 import java.io.FileOutputStream;
18 import java.io.IOException;
19 import java.io.ObjectOutputStream;
20 import java.util.ArrayList;
21 import java.util.Collection;
22 import java.util.Collections;
23 import java.util.HashSet;
24 import java.util.List;
25 import java.util.Map;
26 import java.util.Set;
27 import java.util.TreeMap;
28 import java.util.concurrent.ConcurrentLinkedQueue;
29 import java.util.concurrent.ConcurrentSkipListSet;
30
31 import org.simantics.databoard.Bindings;
32 import org.simantics.databoard.Databoard;
33 import org.simantics.databoard.binding.Binding;
34 import org.simantics.databoard.binding.mutable.Variant;
35 import org.simantics.databoard.serialization.Serializer;
36 import org.simantics.databoard.type.Datatype;
37 import org.simantics.db.AsyncReadGraph;
38 import org.simantics.db.DirectStatements;
39 import org.simantics.db.ReadGraph;
40 import org.simantics.db.Resource;
41 import org.simantics.db.Statement;
42 import org.simantics.db.common.request.AsyncReadRequest;
43 import org.simantics.db.common.request.ReadRequest;
44 import org.simantics.db.common.request.ResourceAsyncRead;
45 import org.simantics.db.common.utils.NameUtils;
46 import org.simantics.db.exception.DatabaseException;
47 import org.simantics.db.layer0.adapter.SubgraphAdvisor;
48 import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus;
49 import org.simantics.db.layer0.util.ConsistsOfProcess.InternalEntry;
50 import org.simantics.db.procedure.AsyncProcedure;
51 import org.simantics.db.request.AsyncRead;
52 import org.simantics.db.service.ClusteringSupport;
53 import org.simantics.db.service.CollectionSupport;
54 import org.simantics.db.service.DirectQuerySupport;
55 import org.simantics.db.service.QueryControl;
56 import org.simantics.db.service.QueryControl.ControlProcedure;
57 import org.simantics.db.service.SerialisationSupport;
58 import org.simantics.db.service.StatementSupport;
59 import org.simantics.db.service.TransferableGraphSupport;
60 import org.simantics.graph.representation.Extensions;
61 import org.simantics.layer0.Layer0;
62 import org.simantics.utils.datastructures.Pair;
63 import org.simantics.utils.threads.logger.ITask;
64 import org.simantics.utils.threads.logger.ThreadLogger;
65
66 import gnu.trove.list.array.TIntArrayList;
67 import gnu.trove.map.hash.TIntIntHashMap;
68 import gnu.trove.map.hash.TLongObjectHashMap;
69 import gnu.trove.procedure.TIntProcedure;
70 import gnu.trove.procedure.TLongObjectProcedure;
71 import gnu.trove.set.TIntSet;
72 import gnu.trove.set.hash.THashSet;
73 import gnu.trove.set.hash.TIntHashSet;
74
75 public class Subgraphs {
76
77         public static String LOG_FILE = "export.log";
78         final static private boolean LOG = false;
79         final static private boolean DEBUG = false;
80         final static private boolean PARENT_DEBUG = DEBUG | false;
81         final static private boolean EXTERNAL_DEBUG = DEBUG | false;
82         final static private boolean ADVISOR_LOG = LOG & false;
83         final static private boolean EXPANSION_LOG = LOG & false;
84         final static private boolean INTERNAL_LOG = LOG & false;
85         final static private boolean COMPOSED_LOG = LOG & false;
86         final static private boolean RESOLVE_LOG = LOG & false;
87         final static private boolean CLASSIFY_LOG = LOG & false;
88         final static private boolean EXTERNAL_LOG = LOG & false;
89         final static private boolean PROFILE = false;
90
91         static enum WeakStatus {
92                 STRONG, WEAK
93         }
94
95         
96         static DataOutput log;
97
98         static {
99
100                 if (LOG) {
101                         try {
102                                 FileOutputStream stream = new FileOutputStream(LOG_FILE, false);
103                                 log = new DataOutputStream(stream);
104                         } catch (FileNotFoundException e) {
105                                 e.printStackTrace();
106                         }
107                 }
108
109         }
110         
111         private static void log(String line) {
112                 if (LOG) {
113                         try {
114                                 log.write((line + "\n").getBytes());
115                         } catch (IOException e) {
116                                 e.printStackTrace();
117                         }
118                 }
119         }
120
121         public static Collection<Resource> getParents(ReadGraph g, Resource r)
122                         throws DatabaseException {
123                 return getParents(g, r, false);
124         }
125
126         static class FastInternalRequest extends ResourceAsyncRead<Boolean> {
127
128                 final DirectQuerySupport dqs;
129                 final ConcurrentLinkedQueue<Resource> queue;
130                 final Map<Resource, WeakStatus> weakInverses;
131                 final Map<Resource, ExtentStatus> status;
132
133                 public FastInternalRequest(DirectQuerySupport dqs, Resource resource,
134                                 Map<Resource, ExtentStatus> status,
135                                 Map<Resource, WeakStatus> weakInverses,
136                                 ConcurrentLinkedQueue<Resource> queue) {
137                         super(resource);
138                         this.dqs = dqs;
139                         this.status = status;
140                         this.weakInverses = weakInverses;
141                         this.queue = queue;
142                 }
143
144                 @Override
145                 public int getFlags() {
146                         return 0;
147                 }
148
149                 @Override
150                 public void perform(AsyncReadGraph graph, final AsyncProcedure<Boolean> procedure) {
151
152                         dqs.forEachDirectStatement(graph, resource, new AsyncProcedure<DirectStatements>() {
153
154                                 @Override
155                                 public void execute(AsyncReadGraph graph, DirectStatements ss) {
156                                         boolean ok = true;
157                                         for(Statement statement : ss) {
158                                                 if (status.get(statement.getObject()) == ExtentStatus.INTERNAL) continue;
159                                                 WeakStatus status = weakInverses.get(statement.getPredicate()); 
160                                                 if(status == WeakStatus.WEAK) continue;
161                                                 else if (status == null) {
162                                                         queue.add(statement.getPredicate());
163                                                 }
164                                                 ok = false;
165                                         }
166                                         procedure.execute(graph, ok);
167                                 }
168
169                                 @Override
170                                 public void exception(AsyncReadGraph graph, Throwable throwable) {
171                                         throwable.printStackTrace();
172                                 }
173
174                         });
175                         
176                 }
177
178         }
179
180         static class ClassifyStatementsRequest implements AsyncRead<Boolean> {
181
182                 final Set<Resource> schedule;
183                 final Map<Resource, WeakStatus> weakMap;
184
185                 public ClassifyStatementsRequest(Set<Resource> schedule, Map<Resource, WeakStatus> weakMap) {
186                         this.weakMap = weakMap;
187                         this.schedule = schedule;
188                 }
189
190             @Override
191             public int threadHash() {
192                 return hashCode();
193             }
194                 
195                 @Override
196                 public int getFlags() {
197                         return 0;
198                 }
199
200                 @Override
201                 public void perform(AsyncReadGraph graph, final AsyncProcedure<Boolean> procedure) {
202
203                         for (final Resource p : schedule) {
204
205                                 graph.forPossibleInverse(p, new AsyncProcedure<Resource>() {
206
207                                         private void register(AsyncReadGraph graph, Resource predicate, Resource superRelation, WeakStatus status) {
208                                                 synchronized (weakMap) {
209                                                         weakMap.put(predicate, status);
210                                                         if(superRelation != null) weakMap.put(superRelation, status);
211                                                 }
212                                         }
213
214                                         @Override
215                                         public void exception(AsyncReadGraph graph, Throwable throwable) {
216                                                 throwable.printStackTrace();
217                                         }
218
219                                         @Override
220                                         public void execute(AsyncReadGraph graph, final Resource inverse) {
221
222                                                 if (inverse == null) {
223                                                         
224                                                         register(graph, p, null, WeakStatus.WEAK);
225                                                         
226                                                 } else {
227                                                         
228                                                         graph.forPossibleSuperrelation(inverse, new AsyncProcedure<Resource>() {
229
230                                                                 @Override
231                                                                 public void exception(AsyncReadGraph graph, Throwable throwable) {
232                                                                         throwable.printStackTrace();
233                                                                 }
234
235                                                                 @Override
236                                                                 public void execute(AsyncReadGraph graph, final Resource superRelation) {
237                                                                         
238                                                                         if(superRelation != null && weakMap.containsKey(superRelation)) {
239                                                                                 register(graph, p, null, weakMap.get(superRelation));
240                                                                                 return;
241                                                                         }
242
243                                                                         graph.forIsSubrelationOf(inverse, graph.getService(Layer0.class).IsRelatedTo, new AsyncProcedure<Boolean>() {
244
245                                                                                 @Override
246                                                                                 public void exception(AsyncReadGraph graph,Throwable throwable) {
247                                                                                         throwable.printStackTrace();
248                                                                                 }
249
250                                                                                 @Override
251                                                                                 public void execute(AsyncReadGraph graph,Boolean strong) {
252                                                                                         register(graph, p, superRelation, strong ? WeakStatus.STRONG : WeakStatus.WEAK);
253                                                                                 }
254
255                                                                         });
256                                                                         
257                                                                 }
258                                                                 
259                                                         });
260                                                         
261                                                 }
262
263                                         }
264
265                                 });
266
267                         }
268
269                         procedure.execute(graph, false);
270
271                 }
272
273         }
274
275         private static Collection<Resource> getParents(ReadGraph g, Resource r, boolean isStrong) throws DatabaseException {
276
277                 System.out.println("getParents " + NameUtils.getSafeName(g, r));
278
279                 Layer0 l0 = Layer0.getInstance(g);
280
281                 Collection<Resource> predicates = g.getPredicates(r);
282
283                 // --- Consists Of ----------------------------------------------------
284
285                 if (predicates.contains(l0.PartOf)) {
286                         Collection<Resource> parents = g.getObjects(r, l0.PartOf);
287                         if (parents.size() == 1)
288                                 return parents;
289                         ArrayList<Resource> libraryParents = new ArrayList<Resource>(1);
290                         for (Resource p : parents)
291                                 if (g.isInstanceOf(p, l0.Library))
292                                         libraryParents.add(p);
293                         if (!libraryParents.isEmpty())
294                                 return libraryParents;
295                         else
296                                 return parents;
297                 }
298
299                 // --- Ordered sets ---------------------------------------------------
300
301                 {
302                         Collection<Resource> parents = null;
303                         for (Resource p : predicates)
304                                 if (g.isInstanceOf(p, l0.OrderedSet) && !p.equals(r)) {
305                                         if (parents == null)
306                                                 parents = new ArrayList<Resource>(1);
307                                         parents.add(p);
308                                 }
309                         if (parents != null) {
310                                 if (DEBUG)
311                                         System.out.println("ORDERED SET");
312                                 return parents;
313                         }
314                 }
315
316
317                 if (isStrong)
318                         return Collections.emptyList();
319                 else {
320
321                         if (predicates.contains(l0.InverseOf)) {
322                                 
323                                 Resource inv = g.getInverse(r);
324                                 return getParents(g, inv, true);
325                                 
326                         } else {
327                                 
328                                 /*
329                                  * Depends On
330                                  * 
331                                  * If there are DependsOn parents, then IsRelatedTo parents are discarded
332                                  * 
333                                  */
334                                 HashSet<Resource> result = new HashSet<Resource>();
335                                 for(Resource predicate : predicates) {
336                                         if(g.isSubrelationOf(predicate, l0.IsDependencyOf)) result.addAll(g.getObjects(r, predicate));
337                                 }
338                                 if(!result.isEmpty()) return result;
339                                 
340                                 /*
341                                  * Is Related To
342                                  * 
343                                  * At this point all Is Related To are parents.
344                                  * 
345                                  */
346                                 for(Resource predicate : predicates) {
347                                         Resource inv = g.getPossibleInverse(predicate);
348                                         if(inv != null) {
349                                                 if(g.isSubrelationOf(inv, l0.IsRelatedTo)) result.addAll(g.getObjects(r, predicate)); 
350                                         }
351                                 }
352                                 
353                                 return result;
354                                 
355                         }
356                         
357                         /*
358                         Collection<Resource> invR = g.getObjects(r, b.IsRelatedTo_Inverse);
359                         if (predicates.contains(b.InverseOf)) {
360                                 if (invR.size() > 1) {
361                                         if (DEBUG)
362                                                 System.out
363                                                                 .println("###########################################");
364                                         Resource inv = g.getInverse(r);
365                                         Collection<Resource> ret = new ArrayList<Resource>();
366                                         for (Statement pp : g.getStatements(r,
367                                                         b.IsRelatedTo_Inverse))
368                                                 if (!pp.getPredicate().equals(inv)) {
369                                                         if (DEBUG) {
370                                                                 System.out.println("<"
371                                                                                 + NameUtils.getSafeName(g, pp
372                                                                                                 .getSubject())
373                                                                                 + ","
374                                                                                 + NameUtils.getSafeName(g, pp
375                                                                                                 .getPredicate())
376                                                                                 + ","
377                                                                                 + NameUtils.getSafeName(g, pp
378                                                                                                 .getObject()) + ">");
379                                                         }
380                                                         ret.add(pp.getObject());
381                                                 }
382                                         return ret;
383                                 }
384                                 // System.out.println("?????????????????");
385                                 Collection<Resource> invParents = getParents(g,
386                                                 g.getInverse(r), true);
387                                 if (!invParents.isEmpty())
388                                         return invParents;
389                         }
390                         if (DEBUG) {
391                                 System.out.print("invR");
392                                 for (Resource res : invR)
393                                         System.out.print(" " + NameUtils.getSafeName(g, res));
394                                 System.out.println();
395                         }
396                         return invR;
397                         */
398                 }
399
400         }
401
402 //      public static String getIdentifier(ReadGraph g, Resource r)
403 //                      throws DatabaseException {
404 //      Layer0 L0 = Layer0.getInstance(g);
405 //              if (r.equals(g.getRootLibrary()))
406 //                      return "";
407 //              String name = g.getPossibleRelatedValue(r, L0.HasName);
408 //              if (name == null)
409 //                      return null;
410 //              Collection<Resource> parents = getParents(g, r, true);
411 //              if (parents.size() != 1)
412 //                      return null;
413 //              for (Resource p : parents) {
414 //                      String parentIdentifier = getIdentifier(g, p);
415 //                      if (parentIdentifier == null)
416 //                              return null;
417 //                      return parentIdentifier + "/" + name;
418 //              }
419 //              return null;
420 //      }
421
422         static int kess = 0;
423
424         static class Expansion extends AsyncReadRequest {
425
426                 final private Collection<Resource> roots;
427                 final Collection<DirectStatements>[] results;
428                 final Collection<Resource>[] listElements;
429                 
430                 public Expansion(Collection<Resource> roots, Collection<DirectStatements>[] results, Collection<Resource>[] listElements) {
431                         this.roots = roots;
432                         this.results = results;
433                         this.listElements = listElements;
434                 }
435
436                 @Override
437                 public void run(AsyncReadGraph graph) {
438
439                         QueryControl control = graph.getService(QueryControl.class);
440                         final DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class);
441
442                         final DomainStatementProcedure proc = new DomainStatementProcedure(dqs, graph.getService(StatementSupport.class), graph.getService(Layer0.class), results, listElements);
443
444                         int slice = (int) (roots.size() / control.getAmountOfQueryThreads()) + 1;
445
446                         final Resource[] rootArray = roots.toArray(Resource.NONE);
447                         for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
448
449                                 final int start = i * slice;
450                                 final int end = Math.min(start + slice, rootArray.length);
451
452                                 control.schedule(graph, i, new ControlProcedure() {
453
454                                         @Override
455                                         public void execute(AsyncReadGraph graph) {
456                                                 for (int index = start; index < end; index++) {
457                                                         dqs.forEachDirectStatement(graph, rootArray[index], proc);
458                                                 }
459
460                                         }
461
462                                 });
463
464                         }
465
466                 }
467
468                 @Override
469                 public int getFlags() {
470                         return 0;
471                 }
472
473         }
474
475         static class Expansion2 extends AsyncReadRequest {
476
477                 final private Collection<Resource> roots;
478                 final Collection<DirectStatements>[] results;
479                 final boolean ignoreVirtual;
480
481                 public Expansion2(Collection<Resource> roots, Collection<DirectStatements>[] results) {
482                         this(roots, results, true);
483                 }
484
485                 public Expansion2(Collection<Resource> roots, Collection<DirectStatements>[] results, boolean ignoreVirtual) {
486                         this.roots = roots;
487                         this.results = results;
488                         this.ignoreVirtual = ignoreVirtual;
489                 }
490
491                 @Override
492                 public void run(AsyncReadGraph graph) {
493
494                         QueryControl control = graph.getService(QueryControl.class);
495                         final DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class);
496
497                         final DomainStatementProcedure2 proc = 
498                                 new DomainStatementProcedure2(results);
499
500                         int slice = (int) (roots.size() / control.getAmountOfQueryThreads()) + 1;
501
502                         final Resource[] rootArray = roots.toArray(Resource.NONE);
503                         for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
504
505                                 final int start = i * slice;
506                                 final int end = Math.min(start + slice, rootArray.length);
507
508                                 control.schedule(graph, i, new ControlProcedure() {
509                                         @Override
510                                         public void execute(AsyncReadGraph graph) {
511                                                 if (ignoreVirtual) {
512                                                         for (int index = start; index < end; index++) {
513                                                                 dqs.forEachDirectPersistentStatement(graph, rootArray[index], proc);
514                                                         }
515                                                 } else {
516                                                         for (int index = start; index < end; index++) {
517                                                                 dqs.forEachDirectStatement(graph, rootArray[index], proc);
518                                                         }
519                                                 }
520                                         }
521                                 });
522
523                         }
524
525                 }
526
527                 @Override
528                 public int getFlags() {
529                         return 0;
530                 }
531
532         }
533         
534         static class DomainProcessor2 {
535
536                 Serializer variantSerializer;
537
538                 int id = 0;
539
540                 Set<Resource> fringe = null;
541                 Set<Resource> exclusions = new HashSet<Resource>();
542                 Set<Resource> internalDomain = new HashSet<Resource>();
543                 Set<Resource> sharedExternalReferences = null;
544                 TIntSet sharedExternalIds = null;
545                 Set<Resource> sharedExternalFringe = null;
546                 Set<Resource> predicates = null;
547                 Set<Resource> isRelatedToPredicates = null;
548                 Set<Resource> sharedPredicates = null;
549                 TIntIntHashMap ids = null;
550                 Map<Resource, Statement> specials = null;
551                 Map<Resource, ExtentStatus> status = null;
552                 Map<Resource, WeakStatus> weakInverses = null;
553                 
554 //              final ArrayList<Double> priorityList = new ArrayList<Double>();
555
556                 private long composedObjectCounter = 0;
557                 private long fastInternalCounter = 0;
558                 private long parentExternalCounter = 0;
559                 private long fullInternalCounter = 0;
560                 private long fullExternalCounter = 0;
561
562                 private long startupTime = 0;
563                 private long expandTime = 0;
564                 private long fullResolveTime = 0;
565                 private long fastResolveTime = 0;
566                 private long otherStatementTime = 0;
567                 private long parentResolveTime = 0;
568                 private long extentSeedTime = 0;
569                 private long composedPredicateTime = 0;
570                 private long composedObjectTime = 0;
571
572                 public void expand(ReadGraph graph, Set<Resource> fringe, Collection<DirectStatements>[] expansion) throws DatabaseException {
573
574                         long start = System.nanoTime();
575
576                         Collection<Collection<DirectStatements>[]> fullExpansion = new ArrayList<Collection<DirectStatements>[]>();
577                         QueryControl control = graph.getService(QueryControl.class);
578                         for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
579                                 expansion[i] = new ArrayList<DirectStatements>();
580                         }
581                         
582                         graph.syncRequest(new Expansion2(fringe, expansion));
583
584                         fringe.clear();
585
586                         expandTime += (System.nanoTime() - start);
587
588                 }
589
590                 public void classifyPredicates(ReadGraph graph, final Set<Resource> schedule) throws DatabaseException {
591
592                         CollectionSupport cs = graph.getService(CollectionSupport.class);
593
594                         final Layer0 L0 = Layer0.getInstance(graph);
595                         
596                         long start = System.nanoTime();
597
598                         final ConcurrentLinkedQueue<Resource> composedResult = new ConcurrentLinkedQueue<Resource>();
599                         final ConcurrentLinkedQueue<Resource> singleResult = new ConcurrentLinkedQueue<Resource>();
600                         final ConcurrentLinkedQueue<Resource> sharedResult = new ConcurrentLinkedQueue<Resource>();
601                         final ConcurrentLinkedQueue<Pair<Resource, Resource>> singles = new ConcurrentLinkedQueue<Pair<Resource, Resource>>();
602
603                         // Discover singles
604                         graph.syncRequest(new AsyncReadRequest() {
605
606                                 @Override
607                                 public void run(AsyncReadGraph graph) {
608
609                                         for (final Resource predicate : schedule) {
610                                                 
611                                                 graph.forPossibleSuperrelation(predicate, new AsyncProcedure<Resource>() {
612
613                                                         @Override
614                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {
615                                                                 throwable.printStackTrace();
616                                                         }
617
618                                                         @Override
619                                                         public void execute(AsyncReadGraph graph, final Resource single) {
620                                                                 singles.add(Pair.make(predicate, single));
621                                                         }
622
623                                                 });
624
625                                                 graph.forHasStatement(predicate, L0.SharedRange, new AsyncProcedure<Boolean>() {
626
627                                                         @Override
628                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {
629                                                                 throwable.printStackTrace();
630                                                         }
631
632                                                         @Override
633                                                         public void execute(AsyncReadGraph graph, final Boolean shared) {
634                                                                 if(shared) sharedResult.add(predicate);
635                                                         }
636
637                                                 });
638                                                 
639                                         }
640
641                                 }
642                                 
643                         });
644
645                         // Determine singles
646                         final Set<Resource> singleSchedule = cs.createSet();
647                         for(Pair<Resource, Resource> pair : singles) {
648                                 
649                                 Resource single = pair.second;
650                                 if(single != null && predicates.add(single)) singleSchedule.add(single);
651                                 
652                         }
653                         
654                         graph.syncRequest(new AsyncReadRequest() {
655
656                                 @Override
657                                 public void run(AsyncReadGraph graph) {
658
659                                         for (final Resource predicate : singleSchedule) {
660
661                                                 graph.forIsSubrelationOf(predicate, L0.IsRelatedTo, new AsyncProcedure<Boolean>() {
662
663                                                         @Override
664                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {
665                                                                 throwable.printStackTrace();
666                                                         }
667
668                                                         @Override
669                                                         public void execute(AsyncReadGraph graph, Boolean strong) {
670                                                                 if (strong) singleResult.add(predicate);
671                                                         }
672
673                                                 });
674                                                 
675                                         }
676
677                                 }
678                                 
679                         });
680
681                         isRelatedToPredicates.addAll(singleResult);
682                         sharedPredicates.addAll(sharedResult);
683
684                         final Set<Resource> specialSchedule = cs.createSet();
685                         
686                         // Classify
687                         for(Pair<Resource, Resource> pair : singles) {
688                                 
689                                 Resource single = pair.second;
690                                 if(single != null) {
691                                         if(isRelatedToPredicates.contains(single)) {
692                                                 isRelatedToPredicates.add(pair.first);
693                                         }
694                                 } else {
695                                         specialSchedule.add(pair.first);
696                                 }
697                                 
698                         }
699                         
700                         graph.syncRequest(new AsyncReadRequest() {
701
702                                 @Override
703                                 public void run(AsyncReadGraph graph) {
704
705                                         for (final Resource predicate : specialSchedule) {
706
707                                                 graph.forIsSubrelationOf(predicate, L0.IsRelatedTo, new AsyncProcedure<Boolean>() {
708
709                                                         @Override
710                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {
711                                                                 throwable.printStackTrace();
712                                                         }
713
714                                                         @Override
715                                                         public void execute(AsyncReadGraph graph, Boolean composed) {
716                                                                 if (composed) composedResult.add(predicate);
717                                                         }
718
719                                                 });
720                                                 
721                                         }
722
723                                 }
724                                 
725                         });
726                         
727                         isRelatedToPredicates.addAll(composedResult);
728
729                         composedPredicateTime += (System.nanoTime() - start);
730                         
731                 }
732                 
733                 private Set<Resource> strongInverseSet = new HashSet<Resource>();
734
735                 public void classifyPredicates(ReadGraph graph, final Collection<DirectStatements>[] expansion) throws DatabaseException {
736
737                         CollectionSupport cs = graph.getService(CollectionSupport.class);
738                         final Set<Resource> schedule = cs.createSet();
739                         final Map<Resource, Resource> newPredicates = cs.createMap(Resource.class);
740                         
741                         for (Collection<DirectStatements> coll : expansion)
742                                 for (DirectStatements stms : coll)
743                                         for(Statement stm : stms) {
744                                                 
745                                                 Resource predicate = stm.getPredicate();
746                                                 
747                                                 if(predicates.add(predicate)) {
748                                                         Resource inverse = graph.getPossibleInverse(predicate);
749                                                         schedule.add(predicate);
750                                                         if(inverse != null) {
751                                                                 newPredicates.put(predicate, inverse);
752                                                                 if(predicates.add(inverse)) schedule.add(inverse);
753                                                         }
754                                                         
755                                                 }
756                                                 
757                                         }
758
759                         classifyPredicates(graph, schedule);
760
761                         for(Map.Entry<Resource, Resource> entry : newPredicates.entrySet()) {
762                                 // Inverse is strong => this has strong inverse
763                                 if(isRelatedToPredicates.contains(entry.getValue())) {
764                                         strongInverseSet.add(entry.getKey());
765                                 }
766                                 // This is strong => inverse has strong inverse
767                                 if(isRelatedToPredicates.contains(entry.getKey())) {
768                                         strongInverseSet.add(entry.getValue());
769                                 }
770                         }
771                         
772                 }
773
774                 /*
775                  * Composed objects are internal. Mark them for expansion.
776                  */
777
778                 public void processFringe(ReadGraph graph, Collection<DirectStatements>[] expansion,
779                                 ObjectOutputStream otherStatementsOutput, ObjectOutputStream valueOutput) throws DatabaseException, IOException {
780
781                         SerialisationSupport support = graph.getService(SerialisationSupport.class);
782                         TransferableGraphSupport tgs = graph.getService(TransferableGraphSupport.class);
783
784                         Layer0 L0 = Layer0.getInstance(graph);
785                         
786                         long start = System.nanoTime();
787                         
788                         for (Collection<DirectStatements> coll : expansion)
789                                 for (DirectStatements stms : coll) {
790                                         
791                                         Resource subject = stms.getSubject();
792                                         
793                                         boolean partOf = false;
794                                         for(Statement stm : stms) {
795                                                 Resource predicate = stm.getPredicate();
796                                                 if(L0.PartOf.equals(predicate)) {
797                                                         partOf = true;
798                                                         break;
799                                                 }
800                                         }
801                                         
802                                         ExtentStatus subjectStatus = status.get(subject);
803                                         if(LOG && subjectStatus != null) log("EXISTING STATUS FOR " + graph.getPossibleURI(subject) + " - " + subjectStatus);
804                                         if(subjectStatus == ExtentStatus.EXTERNAL) continue;
805                                         if(partOf && (subjectStatus == null) && graph.getPossibleURI(subject) != null) {
806                                                 
807                                                 status.put(subject, ExtentStatus.EXTERNAL);
808                                                 if(LOG) {
809                                                         String uri = graph.getPossibleURI(subject);
810                                                         if(uri == null) log("[EXTERNAL]: No URI for " + subject);
811                                                         else log("[EXTERNAL] " + uri);
812                                                 }
813
814                                                 // Check for SharedRange statements
815                                                 for(Statement stm : stms) {
816                                                         Resource predicate = stm.getPredicate();
817                                                         if(sharedPredicates.contains(predicate)) {
818                                                                 sharedExternalFringe.add(stm.getObject());
819                                                                 if(LOG) {
820                                                                         log("[SHARED EXTERNAL FRINGE]: " + NameUtils.getSafeName(graph, stm.getObject()));
821                                                                 }
822                                                         }
823                                                 }
824                                                 
825                                         } else {
826                                                 
827                                                 boolean special = specials.containsKey(subject);
828                                                 if(LOG) {
829                                                         if(special) {
830                                                                 log("[SPECIAL] " + NameUtils.getSafeName(graph, subject));
831                                                         }
832                                                 }
833                                                 
834                                                 status.put(subject, ExtentStatus.INTERNAL);
835                                                 if(LOG) log("[INTERNAL] " + NameUtils.getSafeName(graph, subject));
836
837                                                 int sId = support.getTransientId(subject);
838
839                                                 if(graph.hasValue(subject)) {
840                                                         Datatype dt = graph.getRelatedValue(subject, L0.HasDataType, Bindings.getBindingUnchecked(Datatype.class));
841                                                         Binding b = Bindings.getBinding(dt);
842                                                         Object _value = graph.getValue(subject, b);
843                                                         Variant variant = new Variant(b, _value);
844                                                         byte[] value = variantSerializer.serialize(variant);
845                                                         if(LOG) log("[VALUE] " + NameUtils.getSafeName(graph, subject));
846                                                         valueOutput.writeInt(sId);
847                                                         valueOutput.writeInt(value.length);
848                                                         assert (value.length > 0);
849                                                         valueOutput.write(value);
850                                                 }
851                                                 
852                                                 TIntArrayList stream = new TIntArrayList();
853                                                 
854                                                 for(Statement stm : stms) {
855
856                                                         if(special) {
857                                                                 
858 //                                                              System.err.println("stm=" + stm + " special=" + specials.get(subject));
859                                                                 
860                                                         }
861
862                                                         Resource predicate = stm.getPredicate();
863                                                         Resource object = stm.getObject();
864
865                                                         ExtentStatus objectStatus = status.get(object);
866
867                                                         // Strong predicate
868                                                         if (isRelatedToPredicates.contains(predicate) && (objectStatus !=  ExtentStatus.EXCLUDED)) {
869                                                 
870                                                                 int pId = support.getTransientId(predicate);
871                                                                 int oId = support.getTransientId(object);
872                                                                 
873                                                                 if(LOG) {
874                                                                         String s = NameUtils.getSafeName(graph, subject);
875                                                                         String p = NameUtils.getSafeName(graph, predicate);
876                                                                         String o = NameUtils.getSafeName(graph, object);
877                                                                         log("related=" + s + " - " + p + " - " + o);
878                                                                 }
879                                                                 
880                                                                 stream.add(pId);
881                                                                 stream.add(oId);
882                                                                 
883                                                                 if(objectStatus == null)
884                                                                         fringe.add(object);
885                                                                 
886                                                         } else {
887                                                                 
888                                                                 // Weak predicate
889                                                                 if(objectStatus == ExtentStatus.INTERNAL) {
890
891                                                                         // The inverse is also weak (or there is no inverse)
892                                                                         if(!strongInverseSet.contains(predicate)) {
893
894                                                                                 int pId = support.getTransientId(predicate);
895                                                                                 int oId = support.getTransientId(object);
896                                                                                 
897                                                                                 stream.add(pId);
898                                                                                 stream.add(oId);
899                                                                         
900                                                                                 if(LOG) {
901                                                                                         String s = NameUtils.getSafeName(graph, subject);
902                                                                                         String p = NameUtils.getSafeName(graph, predicate);
903                                                                                         String o = NameUtils.getSafeName(graph, object);
904                                                                                         log("fully weak internal=" + s + " - " + p + " - " + o + " - " + objectStatus);
905                                                                                 }
906                                                                                 
907                                                                         } else {
908
909                                                                                 if(LOG) {
910                                                                                         String s = NameUtils.getSafeName(graph, subject);
911                                                                                         String p = NameUtils.getSafeName(graph, predicate);
912                                                                                         String o = NameUtils.getSafeName(graph, object);
913                                                                                         log("strong inverse internals=" + s + " - " + p + " - " + o + " - " + objectStatus);
914                                                                                 }
915                                                                                 
916                                                                         }
917                                                                         
918                                                                 } else {
919
920                                                                         if(special) {
921                                                                                 
922 //                                                                              System.err.println("stm=" + stm + " special=" + specials.get(subject));
923                                                                                 
924                                                                                 Statement spec = specials.get(subject);
925                                                                                 
926                                                                                 // This statement can be specially treated
927                                                                                 if(stm.getPredicate().equals(spec.getPredicate()) && stm.getObject().equals(spec.getObject())) {
928
929                                                                                         int pId = support.getTransientId(predicate);
930                                                                                         int oId = support.getTransientId(object);
931                                                                                         
932                                                                                         if(LOG) {
933                                                                                                 String s = NameUtils.getSafeName(graph, subject);
934                                                                                                 String p = NameUtils.getSafeName(graph, predicate);
935                                                                                                 String o = NameUtils.getSafeName(graph, object);
936                                                                                                 log("special=" + s + " - " + p + " - " + o);
937                                                                                         }
938                                                                                         
939                                                                                         stream.add(pId);
940                                                                                         stream.add(oId);
941                                                                                         
942                                                                                 }
943                                                                                 
944                                                                         } else {
945                                                                         
946                                                                                 if(LOG) {
947                                                                                         String s = NameUtils.getSafeName(graph, subject);
948                                                                                         String p = NameUtils.getSafeName(graph, predicate);
949                                                                                         String o = NameUtils.getSafeName(graph, object);
950                                                                                         log("weak with unknown object=" + s + " - " + p + " - " + o + " - " + objectStatus);
951                                                                                 }
952                                                                                 
953                                                                         }
954                                                                         
955                                                                 }
956                                                                 
957                                                         }
958                                                         
959                                                 }
960                                                 
961                                                 if(!stream.isEmpty()) {
962                                                         otherStatementsOutput.writeInt(sId);
963                                                         otherStatementsOutput.writeInt(stream.size() / 2);
964                                                         for (int i = 0; i < stream.size(); i++)
965                                                                 otherStatementsOutput.writeInt(stream.getQuick(i));
966                                                 }
967                                                 
968                                         }
969                         
970                                 }
971
972                         composedObjectTime += System.nanoTime() - start;
973
974                 }
975
976                 public void process(ReadGraph graph,
977                                 ObjectOutputStream otherStatementsOutput,
978                                 ObjectOutputStream valueOutput)
979                                 throws DatabaseException, IOException {
980
981                         this.variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT);
982
983                         QueryControl control = graph.getService(QueryControl.class);
984
985 //                      System.err.println("Begin ConsistsOfProcess");
986                         
987                         /*
988                          * Browse all stm = (s, ConsistsOf, o)
989                          * � All internal URIs are found => from now on, if unidentified resource has PartOf it is external.
990              * � All s are internal
991              * � All o are internal
992              * � All stm are included
993                          */
994                         List<InternalEntry> entries = ConsistsOfProcess.walk(graph, null, fringe, exclusions, true); 
995                         for(InternalEntry entry : entries) {
996                                 Resource r = entry.resource;
997                                 if (status.put(r, ExtentStatus.INTERNAL) == null) {
998                                         String URI = graph.getPossibleURI(r);
999                                         if(URI != null) log("URI INTERNAL " + URI);
1000                                         else log("URI has no URI for " + r);
1001                                         fringe.add(r);
1002                                         internalDomain.add(r);
1003                                 }
1004                         }
1005                         
1006                         /*
1007                          * This loop resolves the transitive closure of all p < IsRelatedTo such that p does not contain the SharedRange tag.
1008                          * Such resources are guaranteed to be internal.
1009                          */
1010                         while(!fringe.isEmpty()) {
1011
1012 //                              System.err.println("Process Fringe with " + fringe.size() + ".");
1013
1014                                 Collection<DirectStatements>[] expansion = new ArrayList[control.getAmountOfQueryThreads()];
1015
1016 //                              System.err.println("-expand");
1017
1018                                 // Expand fringe
1019                                 expand(graph, fringe, expansion);
1020
1021                                 /*
1022                                  * classify all p
1023                                  * -IsRelatedTo
1024                                  * -SharedRange
1025                                  * -Internal / External
1026                                  */
1027                                 
1028 //                              System.err.println("-classify");
1029
1030                                 classifyPredicates(graph, expansion);
1031
1032                                 /*
1033                                  * for stms in [stms] 
1034                                  *   if stms contains predicate PartOf => s is External
1035                                  *   else s is Internal
1036                                  *   for all stm=(s,p,o) in stms
1037                                  *     if p <R IsRelatedTo => stm is included
1038                                  *     Fringe <- o
1039                                  */
1040
1041 //                              System.err.println("-process");
1042
1043                                 processFringe(graph, expansion, otherStatementsOutput, valueOutput);
1044                                 
1045                         }
1046
1047                         while(!sharedExternalFringe.isEmpty()) {
1048
1049                                 Collection<DirectStatements>[] expansion = new ArrayList[control.getAmountOfQueryThreads()];
1050                                 expand(graph, sharedExternalFringe, expansion);
1051                                 
1052                                 for (Collection<DirectStatements> coll : expansion)
1053                                         for (DirectStatements stms : coll) {
1054
1055                                                 Resource subject = stms.getSubject();
1056                                                 ExtentStatus subjectStatus = status.get(subject);
1057                                                 
1058                                                 if(ExtentStatus.INTERNAL == subjectStatus) {
1059
1060                                                         if(internalDomain.contains(subject)) continue;
1061                                                         
1062                                                         status.put(subject, ExtentStatus.EXTERNAL);
1063                                                         sharedExternalReferences.add(subject);
1064
1065                                                         if(LOG) {
1066                                                                 log("[SHARED EXTERNAL REFERENCE]: " + NameUtils.getSafeName(graph, subject));
1067                                                         }
1068                                                         
1069                                                         for(Statement stm : stms) {
1070                                                                 Resource predicate = stm.getPredicate();
1071                                                                 if (isRelatedToPredicates.contains(predicate)) {
1072                                                                         sharedExternalFringe.add(stm.getObject());
1073                                                                 }
1074                                                         }
1075                                                         
1076                                                 }
1077                                         }
1078                                 
1079                         }
1080
1081                         if (PROFILE) {
1082                                 System.out.println(composedObjectCounter + " " + fastInternalCounter
1083                                                 + " " + parentExternalCounter + " "
1084                                                 + fullExternalCounter + " " + fullInternalCounter);
1085                         }
1086
1087                 }
1088
1089         }
1090
1091         static class DomainProcessor {
1092
1093                 Serializer variantSerializer;
1094
1095                 int id = 0;
1096
1097                 Set<Resource> predicates = null;
1098                 Set<Resource> composedPredicates = null;
1099                 Set<Resource> expansionSeeds = null;
1100                 Map<Resource, Integer> ids = null;
1101                 Map<Resource, ExtentStatus> status = null;
1102                 Map<Resource, WeakStatus> weakInverses = null;
1103                 
1104                 final Set<SubgraphAdvisor> advisors;
1105                 final ArrayList<Double> priorityList = new ArrayList<Double>();
1106
1107                 private long composedObjectCounter = 0;
1108                 private long fastInternalCounter = 0;
1109                 private long parentExternalCounter = 0;
1110                 private long fullInternalCounter = 0;
1111                 private long fullExternalCounter = 0;
1112
1113                 private long startupTime = 0;
1114                 private long expandTime = 0;
1115                 private long fullResolveTime = 0;
1116                 private long fastResolveTime = 0;
1117                 private long otherStatementTime = 0;
1118                 private long parentResolveTime = 0;
1119                 private long extentSeedTime = 0;
1120                 private long composedPredicateTime = 0;
1121                 private long composedObjectTime = 0;
1122
1123                 public DomainProcessor(Set<SubgraphAdvisor> advisors) {
1124                         this.advisors = advisors;
1125                         HashSet<Double> prioritySet = new HashSet<Double>();
1126                         for (SubgraphAdvisor advisor : advisors)
1127                                 prioritySet.add(advisor.priority());
1128                         priorityList.addAll(prioritySet);
1129                         Collections.sort(priorityList);
1130                 }
1131
1132                 public void expand(ReadGraph graph, Collection<DirectStatements>[] expansion, Set<Resource> schedule) throws DatabaseException {
1133
1134                         long start = System.nanoTime();
1135
1136 //                      if (DEBUG)
1137 //                              System.out.println("expanding " + expansionSeeds.size() + " resources.");
1138
1139                         QueryControl control = graph.getService(QueryControl.class);
1140 //                      final Collection<DirectStatements>[] results = new ArrayList[control.getAmountOfQueryThreads()];
1141                         final ArrayList<Resource>[] listElements = new ArrayList[control.getAmountOfQueryThreads()];
1142                         for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
1143 //                              results[i] = new ArrayList<DirectStatements>();
1144                                 listElements[i] = new ArrayList<Resource>();
1145                         }
1146                         
1147 //                      if(DEBUG) {
1148 //                              for(Resource r : expansionSeeds)
1149 //                                      System.out.println("Expanding " + NameUtils.getSafeName(graph, r, true));
1150 //                      }
1151                         
1152                         graph.syncRequest(new Expansion(expansionSeeds, expansion, listElements));
1153                         for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
1154 //                              for (DirectStatements s : results[i]) {
1155 //                                      expansion.put(s.getSubject(), s);
1156 //                              }
1157                                 for (Resource s : listElements[i]) {
1158                                         schedule.add(s);
1159 //                                      if(status.put(s, ExtentStatus.INTERNAL) == null) {
1160 //                                              ids.put(s, id++);
1161 //                                      }
1162                                 }
1163                         }
1164
1165                         expandTime += (System.nanoTime() - start);
1166
1167                 }
1168
1169                 public void extractComposedPredicates(ReadGraph graph, final Collection<DirectStatements>[] expansion) throws DatabaseException {
1170
1171                         long start = System.nanoTime();
1172
1173                         CollectionSupport cs = graph.getService(CollectionSupport.class);
1174                         
1175                         final ConcurrentLinkedQueue<Resource> composedResult = new ConcurrentLinkedQueue<Resource>();
1176                         final ConcurrentLinkedQueue<Resource> singleResult = new ConcurrentLinkedQueue<Resource>();
1177                         final ConcurrentLinkedQueue<Pair<Resource, Resource>> singles = new ConcurrentLinkedQueue<Pair<Resource, Resource>>();
1178                         
1179                         final Set<Resource> schedule = cs.createSet();
1180                         
1181                         for (Collection<DirectStatements> coll : expansion)
1182                                 for (DirectStatements stms : coll)
1183                                         for(Statement stm : stms) {
1184                                                 Resource predicate = stm.getPredicate();
1185                                                 if(predicates.add(predicate)) schedule.add(predicate);  
1186                                         }
1187
1188                         // Discover singles
1189                         graph.syncRequest(new AsyncReadRequest() {
1190
1191                                 @Override
1192                                 public void run(AsyncReadGraph graph) {
1193
1194                                         for (final Resource predicate : schedule) {
1195                                                 
1196                                                 graph.forPossibleSuperrelation(predicate, new AsyncProcedure<Resource>() {
1197
1198                                                         @Override
1199                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {
1200                                                                 throwable.printStackTrace();
1201                                                         }
1202
1203                                                         @Override
1204                                                         public void execute(AsyncReadGraph graph, final Resource single) {
1205                                                                 singles.add(Pair.make(predicate, single));
1206                                                         }
1207
1208                                                 });
1209
1210                                         }
1211
1212                                 }
1213                                 
1214                         });
1215
1216                         // Determine singles
1217                         final Set<Resource> singleSchedule = cs.createSet();
1218                         for(Pair<Resource, Resource> pair : singles) {
1219                                 
1220                                 Resource single = pair.second;
1221                                 if(single != null && predicates.add(single)) singleSchedule.add(single);
1222                                 
1223                         }
1224                         
1225                         graph.syncRequest(new AsyncReadRequest() {
1226
1227                                 @Override
1228                                 public void run(AsyncReadGraph graph) {
1229
1230                                         for (final Resource predicate : singleSchedule) {
1231
1232                                                 graph.forIsSubrelationOf(predicate, graph.getService(Layer0.class).IsComposedOf, new AsyncProcedure<Boolean>() {
1233
1234                                                         @Override
1235                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {
1236                                                                 throwable.printStackTrace();
1237                                                         }
1238
1239                                                         @Override
1240                                                         public void execute(AsyncReadGraph graph, Boolean composed) {
1241                                                                 if (composed) singleResult.add(predicate);
1242                                                         }
1243
1244                                                 });
1245                                                 
1246                                         }
1247
1248                                 }
1249                                 
1250                         });
1251
1252                         composedPredicates.addAll(singleResult);
1253
1254                         final Set<Resource> specialSchedule = cs.createSet();
1255                         
1256                         // Classify
1257                         for(Pair<Resource, Resource> pair : singles) {
1258                                 
1259                                 Resource single = pair.second;
1260                                 if(single != null) {
1261                                         if(composedPredicates.contains(single)) {
1262                                                 composedPredicates.add(pair.first);
1263                                         }
1264                                 } else {
1265                                         specialSchedule.add(pair.first);
1266                                 }
1267                                 
1268                         }
1269                         
1270                         graph.syncRequest(new AsyncReadRequest() {
1271
1272                                 @Override
1273                                 public void run(AsyncReadGraph graph) {
1274
1275                                         for (final Resource predicate : specialSchedule) {
1276
1277                                                 graph.forIsSubrelationOf(predicate, graph.getService(Layer0.class).IsComposedOf, new AsyncProcedure<Boolean>() {
1278
1279                                                         @Override
1280                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {
1281                                                                 throwable.printStackTrace();
1282                                                         }
1283
1284                                                         @Override
1285                                                         public void execute(AsyncReadGraph graph, Boolean composed) {
1286                                                                 if (composed) composedResult.add(predicate);
1287                                                         }
1288
1289                                                 });
1290                                                 
1291                                         }
1292
1293                                 }
1294                                 
1295                         });
1296                         
1297                         composedPredicates.addAll(composedResult);
1298
1299                         composedPredicateTime += (System.nanoTime() - start);
1300
1301                 }
1302
1303                 /*
1304                  * Composed objects are internal. Mark them for expansion.
1305                  */
1306
1307                 public void collectComposedObjects(ReadGraph graph, Collection<DirectStatements>[] expansion, Set<Resource> typeTodo, Set<Resource> objectTodo,
1308                                 Set<Resource> predicateTodo) throws DatabaseException {
1309
1310                         long start = System.nanoTime();
1311
1312                         Layer0 l0 = Layer0.getInstance(graph);
1313                         
1314                         for (Collection<DirectStatements> coll : expansion)
1315                                 for (DirectStatements stms : coll)
1316                                         for(Statement stm : stms) {
1317
1318                                                 Resource predicate = stm.getPredicate();
1319                                                 Resource object = stm.getObject();
1320                                                 
1321                                                 if (composedPredicates.contains(predicate)) {
1322                                                         
1323                                                         ExtentStatus existing = status.put(object, ExtentStatus.INTERNAL);
1324                                                         if(existing == null) {
1325                                                                 ids.put(object, id++);
1326                                                                 composedObjectCounter++;
1327                                                                 expansionSeeds.add(object);
1328 //                                                              System.err.println("internal: " + NameUtils.getSafeName(graph, object, true));
1329                                                                 if(LOG) log("[INTERNAL] (composed object) " + NameUtils.getSafeName(graph, object, true));
1330                                                         } else if (existing == ExtentStatus.EXCLUDED) {
1331                                                                 System.err.println("preExcluded: " + NameUtils.getSafeName(graph, object, true));
1332                                                                 status.put(object, ExtentStatus.EXCLUDED);
1333                                                         } else if (existing == ExtentStatus.EXTERNAL) {
1334                                                                 System.err.println("preExternal: " + NameUtils.getSafeName(graph, object, true));
1335                                                                 status.put(object, ExtentStatus.EXTERNAL);
1336                                                         }
1337                                                         
1338                                                 } else {
1339
1340 //                                                      System.err.println("internal2: " + NameUtils.getSafeName(graph, object, true));
1341                                                         
1342                                                         if (!status.containsKey(object)) {
1343                                                                 if (l0.InstanceOf.equalsResource(predicate)) {
1344                                                                         typeTodo.add(object);
1345                                                                 } else {
1346                                                                         objectTodo.add(object);
1347                                                                 }
1348                                                         }
1349
1350                                                         if (!status.containsKey(predicate)) {
1351                                                                 predicateTodo.add(predicate);
1352                                                         }
1353                                                         
1354                                                 }
1355                                         }
1356
1357                         composedObjectTime += System.nanoTime() - start;
1358
1359                 }
1360
1361                 public void writeOtherStatements(ReadGraph graph, Collection<Collection<DirectStatements>[]> expansion, ObjectOutputStream composedStatementsOutput, ObjectOutputStream otherStatementsOutput,
1362                                 ObjectOutputStream valueOutput) throws DatabaseException {
1363
1364                         long start = System.nanoTime();
1365
1366                         Layer0 l0 = Layer0.getInstance(graph);
1367                         SerialisationSupport support = graph.getService(SerialisationSupport.class);
1368                         TransferableGraphSupport tgs = graph.getService(TransferableGraphSupport.class);
1369
1370                         TIntArrayList other = new TIntArrayList();
1371                         TIntArrayList composed = new TIntArrayList();
1372
1373                         try {
1374
1375                                 for (Collection<DirectStatements>[] colls : expansion)
1376                                         for (Collection<DirectStatements> coll : colls)
1377                                                 for (DirectStatements stms : coll) {
1378
1379                                                         Resource subject = stms.getSubject();
1380                                                         composed.resetQuick();
1381
1382                                                         int sId = support.getTransientId(subject);
1383
1384                                                         composedStatementsOutput.writeInt(sId);
1385
1386                                                         if(graph.hasValue(subject)) {
1387                                                                 Datatype dt = graph.getRelatedValue(subject, l0.HasDataType, Bindings.getBindingUnchecked(Datatype.class));
1388                                                                 Binding b = Bindings.getBinding(dt);
1389                                                                 Object _value = graph.getValue(subject, b);
1390                                                                 Variant variant = new Variant(b, _value);
1391                                                                 byte[] value = variantSerializer.serialize(variant);
1392                                                                 if(LOG) log("[VALUE] " + NameUtils.getSafeName(graph, subject));
1393                                                                 valueOutput.writeInt(sId);
1394                                                                 valueOutput.writeInt(value.length);
1395                                                                 assert (value.length > 0);
1396                                                                 valueOutput.write(value);
1397                                                         }
1398
1399                                                         for (Statement s : stms) {
1400
1401                                                                 Resource object = s.getObject();
1402                                                                 Resource predicate = s.getPredicate();
1403
1404                                                                 ExtentStatus objectStatus = status.get(object); 
1405                                                                 
1406                                                                 if(objectStatus == ExtentStatus.INTERNAL) {
1407                                                                         composed.add(support.getTransientId(predicate));
1408                                                                         composed.add(support.getTransientId(object));
1409                                                                         if(LOG) log("[COMPOSED] (internal object) " + NameUtils.toIdString(graph, s));
1410                                                                 } else if (l0.InstanceOf.equalsResource(predicate)) {
1411                                                                         composed.add(support.getTransientId(predicate));
1412                                                                         composed.add(support.getTransientId(object));
1413                                                                         if(LOG) log("[COMPOSED] (instanceOf) " + NameUtils.toIdString(graph, s));
1414                                                                 } else if (l0.SubrelationOf.equalsResource(predicate)) {
1415                                                                         composed.add(support.getTransientId(predicate));
1416                                                                         composed.add(support.getTransientId(object));
1417                                                                         if(LOG) log("[COMPOSED] (subrelationOf) " + NameUtils.toIdString(graph, s));
1418                                                                 } else {
1419                                                                         if(objectStatus == ExtentStatus.EXTERNAL) {
1420                                                                                 if(DEBUG)
1421                                                                                         System.out.println("other " + NameUtils.toIdString(graph, s));
1422                                                                                         //System.out.println("other.add " + predicate + " - " + object);
1423                                                                                 other.add(support.getTransientId(predicate));
1424                                                                                 other.add(support.getTransientId(object));
1425                                                                                 if(LOG) log("[OTHER] (object is external) " + NameUtils.toIdString(graph, s));
1426                                                                         }
1427                                                                 }
1428
1429                                                         }
1430
1431                                                         if(!other.isEmpty()) {
1432                                                                 otherStatementsOutput.writeInt(sId);
1433                                                                 otherStatementsOutput.writeInt(other.size() / 2);
1434                                                                 for (int i = 0; i < other.size(); i++)
1435                                                                         otherStatementsOutput.writeInt(other.getQuick(i));
1436                                                                 other.resetQuick();
1437                                                         }
1438
1439                                                         composedStatementsOutput.writeInt(composed.size() / 2);
1440                                                         for (int i = 0; i < composed.size(); i++)
1441                                                                 composedStatementsOutput.writeInt(composed.getQuick(i));
1442
1443                                                 }
1444
1445                         } catch (IOException e) {
1446                                 e.printStackTrace();
1447                         }
1448
1449                         otherStatementTime += (System.nanoTime() - start);
1450
1451                 }
1452
1453                 boolean hasStrictParents(ReadGraph g, Resource r)
1454                                 throws DatabaseException {
1455                         if (g.getPossibleURI(r) != null)
1456                                 return true;
1457                         return false;
1458                 }
1459
1460                 public boolean getExpansionSeedsFromExtents(ReadGraph graph, final Collection<DirectStatements>[] expansion) throws DatabaseException {
1461
1462                         long start = System.nanoTime();
1463
1464                         final ConcurrentLinkedQueue<Resource> accepts = new ConcurrentLinkedQueue<Resource>();
1465
1466                         /*
1467                          * Determine statements which could accept statements with todo
1468                          * objects
1469                          */
1470                         search: for (Double priority : priorityList) {
1471                                 
1472                                 for (final SubgraphAdvisor advisor : advisors) {
1473                                         
1474                                         if (advisor.priority() > 0)
1475                                                 continue;
1476                                         
1477                                         if (advisor.priority() == priority) {
1478                                                 
1479                                                 graph.syncRequest(new ReadRequest() {
1480
1481                                                         @Override
1482                                                         public void run(ReadGraph graph) throws DatabaseException {
1483
1484                                                                 for (Collection<DirectStatements> coll : expansion)
1485                                                                         for (DirectStatements stms : coll)
1486                                                                                 for(final Statement stm : stms) {
1487
1488                                                                                         advisor.advise(graph, stm, new AsyncProcedure<Boolean>() {
1489
1490                                                                                                 @Override
1491                                                                                                 public void exception(AsyncReadGraph graph, Throwable throwable) {
1492                                                                                                         throwable.printStackTrace();
1493                                                                                                 }
1494
1495                                                                                                 @Override
1496                                                                                                 public void execute(AsyncReadGraph graph, Boolean accept) {
1497                                                                                                         if (accept) {
1498                                                                                                                 accepts.add(stm.getObject());
1499                                                                                                         }
1500                                                                                                 }
1501
1502                                                                                         });
1503
1504                                                                                 }
1505
1506                                                         }
1507
1508                                                 });
1509                                         }
1510                                         if (!accepts.isEmpty())
1511                                                 break search;
1512                                 }
1513                         }
1514
1515                         CollectionSupport cs = graph.getService(CollectionSupport.class);
1516                         Set<Resource> schedule = cs.createSet();
1517                         for (Resource r : accepts) {
1518                                 if(!status.containsKey(r))
1519                                         schedule.add(r);
1520                         }
1521
1522                         extentSeedTime += (System.nanoTime() - start);
1523
1524                         if (schedule.isEmpty())
1525                                 return false;
1526
1527                         fastResolve(graph, schedule);
1528                         uriResolve(graph, schedule);
1529                         fullResolve(graph, schedule, "accepts");
1530
1531                         return true;
1532
1533                 }
1534
1535                 ConcurrentLinkedQueue<Resource> fastInternals = new ConcurrentLinkedQueue<Resource>();
1536
1537                 public void fastResolve(ReadGraph graph, final Set<Resource> rs)
1538                                 throws DatabaseException {
1539                         // This collects and resolves weaks
1540                         if(fastResolveLoop(graph, rs))
1541                                 // Weaks are now resolved
1542                                 fastResolveLoop(graph, rs);
1543                 }
1544
1545                 public boolean fastResolveLoop(ReadGraph graph, final Set<Resource> rs)
1546                                 throws DatabaseException {
1547
1548                         long start = System.nanoTime();
1549
1550                         final ConcurrentLinkedQueue<Resource> weakSchedule = new ConcurrentLinkedQueue<Resource>();
1551
1552                         graph.syncRequest(new AsyncRead<Boolean>() {
1553
1554                             @Override
1555                             public int threadHash() {
1556                                 return hashCode();
1557                             }
1558                                 
1559                                 @Override
1560                                 public int getFlags() {
1561                                         return 0;
1562                                 }
1563
1564                                 @Override
1565                                 public void perform(AsyncReadGraph graph,
1566                                                 AsyncProcedure<Boolean> procedure) {
1567
1568                                         QueryControl control = graph.getService(QueryControl.class);
1569                                         final DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class);
1570
1571                                         int slice = (int) (rs.size() / control
1572                                                         .getAmountOfQueryThreads()) + 1;
1573
1574                                         final Resource[] rootArray = rs.toArray(Resource.NONE);
1575                                         for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
1576
1577                                                 final int start = i * slice;
1578                                                 final int end = Math.min(start + slice,
1579                                                                 rootArray.length);
1580
1581                                                 control.schedule(graph, i, new ControlProcedure() {
1582
1583                                                         @Override
1584                                                         public void execute(AsyncReadGraph graph) {
1585
1586                                                                 for (int index = start; index < end; index++) {
1587
1588                                                                         final Resource r = rootArray[index];
1589
1590                                                                         //if (status.containsKey(r)) continue;
1591
1592                                                                         graph.asyncRequest(new FastInternalRequest(dqs, r, status, weakInverses, weakSchedule),new AsyncProcedure<Boolean>() {
1593
1594                                                                                 @Override
1595                                                                                 public void exception(AsyncReadGraph graph,Throwable throwable) {
1596                                                                                         throwable.printStackTrace();
1597                                                                                 }
1598
1599                                                                                 @Override
1600                                                                                 public void execute(AsyncReadGraph graph,Boolean isInternal) {
1601                                                                                         if (isInternal) {
1602                                                                                                 fastInternals.add(r);
1603                                                                                         }
1604                                                                                 }
1605
1606                                                                         });
1607
1608                                                                 }
1609
1610                                                         }
1611
1612                                                 });
1613
1614                                         }
1615
1616                                         procedure.execute(graph, true);
1617
1618                                 }
1619
1620                         });
1621
1622                         if (!weakSchedule.isEmpty()) {
1623                                 THashSet<Resource> weaks = new THashSet<Resource>(weakSchedule);                                
1624                                 if (CLASSIFY_LOG)
1625                                         for (Resource p : weakSchedule)
1626                                                 System.out.println("classify "
1627                                                                 + NameUtils.getSafeName(graph, p));
1628                                 graph.syncRequest(new ClassifyStatementsRequest(weaks, weakInverses));
1629                         } 
1630
1631                         for (Resource r : fastInternals) {
1632                                 rs.remove(r);
1633                                 if (status.put(r, ExtentStatus.INTERNAL) == null) {
1634                                         if(LOG) log("[INTERNAL] (fast) " + NameUtils.getSafeName(graph, r, true));
1635                                         ids.put(r, id++);
1636                                         fastInternalCounter++;
1637                                         expansionSeeds.add(r);
1638                                 }
1639                         }
1640
1641                         fastResolveTime += (System.nanoTime() - start);
1642                         
1643                         return !weakSchedule.isEmpty();                 
1644
1645                 }
1646
1647                 private ExtentStatus resolveExtent(ReadGraph graph, Resource r, Map<Resource, ExtentStatus> statuses, Set<Resource> expansionSeeds, THashSet<Resource> pending,
1648                                 ArrayList<Resource> stack) throws DatabaseException {
1649
1650                         ExtentStatus current = statuses.get(r);
1651                         if(current != null) return current;
1652                         
1653                         if (pending.contains(r))
1654                                 return ExtentStatus.PENDING;
1655
1656                         // In order to break cyclic dependencies
1657                         pending.add(r);
1658
1659                         if (PARENT_DEBUG)
1660                                 System.out.println("resolveExtent "
1661                                                 + NameUtils.getSafeName(graph, r));
1662
1663                         ExtentStatus status = ExtentStatus.INTERNAL;
1664                         for (Resource p : getParents(graph, r)) {
1665                                 if (PARENT_DEBUG) {
1666                                         ExtentStatus ps = statuses.get(p);
1667                                         System.out.println(" parent " + NameUtils.getSafeName(graph, p) + "(" + ps + ")");
1668                                 }
1669                                 switch (resolveExtent(graph, p, statuses,
1670                                                 expansionSeeds, pending, stack)) {
1671                                 case EXTERNAL:
1672                                         return ExtentStatus.EXTERNAL;
1673                                 case PENDING:
1674                                         status = ExtentStatus.PENDING;
1675                                 }
1676                         }
1677                         if (status == ExtentStatus.INTERNAL) {
1678                                 pending.remove(r);
1679                                 stack.add(r);
1680                                 if (DEBUG)
1681                                         System.out.println(NameUtils.getSafeName(graph, r, true)
1682                                                         + " is internal.");
1683                         }
1684                         return status;
1685                 }
1686
1687                 public void uriResolve(ReadGraph graph, final Set<Resource> todo)
1688                                 throws DatabaseException {
1689
1690                         long start = System.nanoTime();
1691
1692                         for(Resource r : todo) System.out.println("uriResolve " +
1693                                         NameUtils.getSafeName(graph, r));
1694
1695                         final ConcurrentSkipListSet<Resource> found = new ConcurrentSkipListSet<Resource>();
1696
1697                         graph.syncRequest(new AsyncReadRequest() {
1698                                 
1699                                 @Override
1700                                 public void run(AsyncReadGraph graph) {
1701
1702                                         for (final Resource r : todo) {
1703
1704                                                 // System.out.println("uriresolve before " + r);
1705
1706                                                 if (status.containsKey(r)) continue;
1707
1708                                                 // System.out.println("uriresolve " + r);
1709
1710                                                 graph.forURI(r, new AsyncProcedure<String>() {
1711
1712                                                         @Override
1713                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {
1714                                                                 throwable.printStackTrace();
1715                                                         }
1716
1717                                                         @Override
1718                                                         public void execute(AsyncReadGraph graph, String uri) {
1719
1720                                                                 if (uri != null) {
1721
1722                                                                         // System.out.println("uriresolve has uri "
1723                                                                         // + r);
1724
1725                                                                         if(found.add(r)) {
1726                                                                                 parentExternalCounter++;
1727                                                                         }
1728
1729                                                                 } else {
1730
1731                                                                         // System.out.println("uriresolve ask inverse "
1732                                                                         // + r);
1733
1734                                                                         graph.forPossibleInverse(r, new AsyncProcedure<Resource>() {
1735
1736                                                                                 @Override
1737                                                                                 public void exception(AsyncReadGraph graph, Throwable throwable) {
1738                                                                                         throwable.printStackTrace();
1739                                                                                 }
1740
1741                                                                                 @Override
1742                                                                                 public void execute(AsyncReadGraph graph, Resource inverse) {
1743
1744                                                                                         if (inverse != null) {
1745
1746                                                                                                 graph.forURI(inverse, new AsyncProcedure<String>() {
1747
1748                                                                                                         @Override
1749                                                                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {
1750                                                                                                                 throwable.printStackTrace();
1751                                                                                                         }
1752
1753                                                                                                         @Override
1754                                                                                                         public void execute(AsyncReadGraph graph, String uri) {
1755
1756                                                                                                                 if (uri != null) {
1757
1758                                                                                                                         if(found.add(r)) {
1759                                                                                                                                 parentExternalCounter++;
1760                                                                                                                         }
1761
1762                                                                                                                 }
1763
1764                                                                                                         }
1765
1766                                                                                                 });
1767
1768                                                                                         }
1769
1770                                                                                 }
1771                                                                                 
1772                                                                         });
1773
1774                                                                 }
1775                                                         }
1776
1777                                                 });
1778
1779                                         }
1780
1781                                 }
1782
1783                         });
1784
1785                         todo.removeAll(found);
1786                         for(Resource r : found) {
1787                                 status.put(r, ExtentStatus.EXTERNAL);
1788                                 if(LOG) log("[EXTERNAL] (uriResolve) " + NameUtils.getSafeName(graph, r, true));
1789                         }
1790                         
1791                         parentResolveTime += System.nanoTime() - start;
1792
1793                 }
1794
1795                 public void fullResolve(ReadGraph graph, Collection<Resource> rs,
1796                                 String koss) throws DatabaseException {
1797
1798                         long start = System.nanoTime();
1799
1800                         for (final Resource r : rs) {
1801
1802                                 if(status.containsKey(r)) continue;
1803
1804                                 THashSet<Resource> pending = new THashSet<Resource>();
1805                                 ArrayList<Resource> stack = new ArrayList<Resource>();
1806
1807                                 ExtentStatus s = resolveExtent(graph, r, status, expansionSeeds, pending, stack);
1808                                 if (ExtentStatus.INTERNAL == s || ExtentStatus.PENDING == s) {
1809                                         if (status.put(r, ExtentStatus.INTERNAL) == null) {
1810                                                 if(LOG) log("[INTERNAL] (resolveExtent) " + NameUtils.getSafeName(graph, r, true));
1811                                                 ids.put(r, id++);
1812                                                 fullInternalCounter++;
1813                                                 expansionSeeds.add(r);
1814                                         }
1815                                 }
1816                                 if (ExtentStatus.EXTERNAL == s) {
1817                                         if (status.put(r, ExtentStatus.EXTERNAL) == null) {
1818                                                 if(LOG) log("[EXTERNAL] (resolveExtent) " + NameUtils.getSafeName(graph, r, true));
1819                                                 fullExternalCounter++;
1820                                         }
1821                                 }
1822
1823                         }
1824
1825                         fullResolveTime += (System.nanoTime() - start);
1826
1827                 }
1828
1829                 public void process(ReadGraph graph,
1830                                 ObjectOutputStream composedStatementsOutput,
1831                                 ObjectOutputStream otherStatementsOutput,
1832                                 ObjectOutputStream valueOutput)
1833                                 throws DatabaseException {
1834
1835                         this.variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT);
1836
1837                         CollectionSupport cs = graph.getService(CollectionSupport.class);
1838
1839                         Set<Resource> typeTodo = cs.createSet();
1840                         Set<Resource> objectTodo = cs.createSet();
1841                         Set<Resource> predicateTodo = cs.createSet();
1842                         
1843                         Collection<Collection<DirectStatements>[]> fullExpansion = new ArrayList<Collection<DirectStatements>[]>();
1844                         
1845                         do {
1846
1847                                 QueryControl control = graph.getService(QueryControl.class);
1848                                 Collection<DirectStatements>[] expansion = new ArrayList[control.getAmountOfQueryThreads()];
1849                                 for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
1850                                         expansion[i] = new ArrayList<DirectStatements>();
1851                                 }
1852                                 
1853                                 // Expand expansionSeeds
1854                                 expand(graph, expansion, objectTodo);
1855                                 
1856                                 // Start collecting new seeds
1857                                 expansionSeeds = cs.createSet();
1858
1859                                 // Collect predicates which are <R IsComposedOf
1860                                 extractComposedPredicates(graph, expansion);
1861
1862                                 // Make composed objects internal and make sure they are further
1863                                 // expanded
1864                                 collectComposedObjects(graph, expansion, typeTodo, objectTodo, predicateTodo);
1865
1866                                 /*
1867                                  * Use the expansion seed heuristic to find new resources to
1868                                  * expand before full analysis.
1869                                  */
1870                                 getExpansionSeedsFromExtents(graph, expansion);
1871
1872                                 fullExpansion.add(expansion);
1873
1874                         } while (!expansionSeeds.isEmpty());
1875
1876                         fastResolve(graph, objectTodo);
1877                         uriResolve(graph, predicateTodo);
1878                         uriResolve(graph, objectTodo);
1879                         uriResolve(graph, typeTodo);
1880                         fullResolve(graph, objectTodo, "objectTodo");
1881                         fullResolve(graph, predicateTodo, "predicateTodo");
1882                         fullResolve(graph, typeTodo, "typeTodo");
1883
1884                         writeOtherStatements(graph, fullExpansion, composedStatementsOutput, otherStatementsOutput, valueOutput);
1885
1886                         if (PROFILE) {
1887                                 System.out.println(composedObjectCounter + " " + fastInternalCounter
1888                                                 + " " + parentExternalCounter + " "
1889                                                 + fullExternalCounter + " " + fullInternalCounter);
1890                         }
1891
1892                 }
1893
1894         }
1895
1896         public static void getDomain2(ReadGraph graph, TIntIntHashMap ids,
1897                         Collection<Resource> roots, Map<Resource, ExtentStatus> preStatus,
1898                         Map<Resource, Statement> specials,
1899                         ObjectOutputStream otherStatementsOutput,
1900                         ObjectOutputStream valueOutput,
1901                         TreeMap<String, Variant> extensions,
1902                         TIntHashSet excludedShared) throws DatabaseException {
1903
1904                 ITask task = ThreadLogger.getInstance().begin("getDomain2");
1905
1906                 final DomainProcessor2 processor = new DomainProcessor2();
1907                 
1908                 processor.startupTime = System.nanoTime();
1909
1910                 Layer0 l0 = Layer0.getInstance(graph);
1911                 
1912                 CollectionSupport cs = graph.getService(CollectionSupport.class);
1913                 SerialisationSupport support = graph.getService(SerialisationSupport.class);
1914
1915                 processor.ids = ids;
1916                 processor.specials = specials;
1917                 processor.status = cs.createMap(ExtentStatus.class);
1918                 processor.weakInverses = cs.createMap(WeakStatus.class);
1919                 processor.predicates = cs.createSet();
1920                 processor.isRelatedToPredicates = cs.createSet();
1921                 processor.sharedPredicates = cs.createSet();
1922 //              processor.expansionSeeds = cs.createSet();
1923
1924                 for(Map.Entry<Resource, ExtentStatus> entry : preStatus.entrySet()) {
1925                         processor.status.put(entry.getKey(), entry.getValue());
1926                         if(ExtentStatus.EXCLUDED.equals(entry.getValue())) processor.exclusions.add(entry.getKey());
1927                 }
1928                 
1929 //              for (Resource r : excluded) {
1930 //                      processor.status.put(r, ExtentStatus.EXCLUDED);
1931 //              }
1932                 
1933                 Resource rootLibrary = graph.getResource("http:/");
1934                 
1935                 if (!roots.contains(rootLibrary))
1936                         processor.status.put(rootLibrary, ExtentStatus.EXTERNAL);
1937
1938                 for (Resource root : roots) {
1939                         processor.status.put(root, ExtentStatus.INTERNAL);
1940                         //processor.ids.put(support.getTransientId(root), processor.ids.size());
1941                         for (Resource owner : graph.getObjects(root, l0.IsOwnedBy)) {
1942                                 processor.status.put(owner, ExtentStatus.EXTERNAL);
1943                         }
1944                 }
1945
1946                 processor.startupTime = System.nanoTime() - processor.startupTime;
1947
1948                 processor.fringe = new HashSet<Resource>();
1949                 processor.fringe.addAll(roots);
1950                 
1951                 processor.internalDomain.addAll(roots);
1952                 
1953                 processor.sharedExternalReferences = new HashSet<Resource>();
1954                 processor.sharedExternalFringe = new HashSet<Resource>();
1955
1956                 try {
1957                 
1958                         processor.process(graph, otherStatementsOutput, valueOutput);
1959                 
1960                 } catch (IOException e) {
1961                         e.printStackTrace();
1962                 }
1963                 
1964                 for(Resource r : processor.sharedExternalReferences) excludedShared.add(support.getTransientId(r));
1965                 
1966         ClusteringSupport cls = graph.getService(ClusteringSupport.class);
1967         TLongObjectHashMap<TIntArrayList> clusterMap = new TLongObjectHashMap<TIntArrayList>();
1968                 for(Map.Entry<Resource, ExtentStatus> entry : processor.status.entrySet()) {
1969                         if(ExtentStatus.INTERNAL == entry.getValue()) {
1970                                 
1971                                 long cluster = cls.getCluster(entry.getKey());
1972                                 TIntArrayList list = clusterMap.get(cluster);
1973                                 if(list == null) {
1974                                         list = new TIntArrayList();
1975                                         clusterMap.put(cluster, list);
1976                                 }
1977                                 list.add(support.getTransientId(entry.getKey()));
1978                                 
1979                         }
1980                 }
1981                 final TIntArrayList clustering = new TIntArrayList();
1982                 clusterMap.forEachEntry(new TLongObjectProcedure<TIntArrayList>() {
1983                         
1984                         @Override
1985                         public boolean execute(long cluster, TIntArrayList b) {
1986                                 clustering.add(b.size());
1987                                 b.forEach(new TIntProcedure() {
1988                                         
1989                                         @Override
1990                                         public boolean execute(int rId) {
1991                                                 processor.ids.put(rId, processor.id++);
1992                                                 return true;
1993                                         }
1994                                         
1995                                 });
1996                                 return true;
1997                         }
1998                         
1999                 });
2000                 
2001                 extensions.put(Extensions.CLUSTERING, new Variant(Bindings.INT_ARRAY, clustering.toArray()));
2002                 
2003                 long total = processor.startupTime + processor.expandTime
2004                                 + processor.composedPredicateTime
2005                                 + processor.composedObjectTime + processor.extentSeedTime
2006                                 + processor.fullResolveTime + processor.fastResolveTime + 
2007                                 + processor.parentResolveTime + processor.otherStatementTime;
2008
2009                 if (PROFILE) {
2010                         System.out.println("startup took " + 1e-9 * processor.startupTime
2011                                         + "s.");
2012                         System.out.println("expand took " + 1e-9 * processor.expandTime
2013                                         + "s.");
2014                         System.out.println("composedPredicates took " + 1e-9
2015                                         * processor.composedPredicateTime + "s.");
2016                         System.out.println("composedObjects took " + 1e-9
2017                                         * processor.composedObjectTime + "s.");
2018                         System.out.println("extentSeeding took " + 1e-9
2019                                         * processor.extentSeedTime + "s.");
2020                         System.out.println("fullResolve took " + 1e-9
2021                                         * processor.fullResolveTime + "s.");
2022                         System.out.println("fastResolve took " + 1e-9
2023                                         * processor.fastResolveTime + "s.");
2024                         System.out.println("parentResolve took " + 1e-9
2025                                         * processor.parentResolveTime + "s.");
2026                         System.out.println("otherStatements took " + 1e-9
2027                                         * processor.otherStatementTime + "s.");
2028                         System.out.println("total " + 1e-9 * total + "s.");
2029                 }
2030
2031                 task.finish();
2032
2033                 
2034         }
2035
2036         
2037         public static void getDomain(ReadGraph graph, Map<Resource, Integer> ids,
2038                         Collection<Resource> roots, Map<Resource, ExtentStatus> preStatus, Set<SubgraphAdvisor> advisors,
2039                         ObjectOutputStream composedStatementsOutput,
2040                         ObjectOutputStream otherStatementsOutput,
2041                         ObjectOutputStream valueOutput) throws DatabaseException {
2042
2043                 ITask task = ThreadLogger.getInstance().begin("getDomain");
2044
2045                 DomainProcessor processor = new DomainProcessor(advisors);
2046                 
2047                 processor.startupTime = System.nanoTime();
2048
2049                 Layer0 l0 = Layer0.getInstance(graph);
2050
2051                 CollectionSupport cs = graph.getService(CollectionSupport.class);
2052
2053                 processor.ids = ids;
2054                 processor.status = cs.createMap(ExtentStatus.class);
2055                 processor.weakInverses = cs.createMap(WeakStatus.class);
2056                 processor.predicates = cs.createSet();
2057                 processor.composedPredicates = cs.createSet();
2058                 processor.expansionSeeds = cs.createSet();
2059
2060                 for(Map.Entry<Resource, ExtentStatus> entry : preStatus.entrySet()) {
2061                         processor.status.put(entry.getKey(), entry.getValue());
2062                 }
2063
2064 //              for (Resource r : excluded) {
2065 //                      processor.status.put(r, ExtentStatus.EXCLUDED);
2066 //              }
2067                 
2068                 if (!roots.contains(graph.getRootLibrary()))
2069                         processor.status.put(graph.getRootLibrary(), ExtentStatus.EXTERNAL);
2070
2071                 for (Resource root : roots) {
2072                         processor.status.put(root, ExtentStatus.INTERNAL);
2073                         processor.ids.put(root, processor.id++);
2074                         for (Resource owner : graph.getObjects(root, l0.IsOwnedBy)) {
2075                                 processor.status.put(owner, ExtentStatus.EXTERNAL);
2076                         }
2077                 }
2078
2079
2080                 processor.expansionSeeds.addAll(roots);
2081
2082                 processor.startupTime = System.nanoTime() - processor.startupTime;
2083
2084                 while (!processor.expansionSeeds.isEmpty()) {
2085
2086                         processor.process(graph, composedStatementsOutput,
2087                                         otherStatementsOutput, valueOutput);
2088
2089                 }
2090
2091                 long total = processor.startupTime + processor.expandTime
2092                                 + processor.composedPredicateTime
2093                                 + processor.composedObjectTime + processor.extentSeedTime
2094                                 + processor.fullResolveTime + processor.fastResolveTime + 
2095                                 + processor.parentResolveTime + processor.otherStatementTime;
2096
2097                 if (PROFILE) {
2098                         System.out.println("startup took " + 1e-9 * processor.startupTime
2099                                         + "s.");
2100                         System.out.println("expand took " + 1e-9 * processor.expandTime
2101                                         + "s.");
2102                         System.out.println("composedPredicates took " + 1e-9
2103                                         * processor.composedPredicateTime + "s.");
2104                         System.out.println("composedObjects took " + 1e-9
2105                                         * processor.composedObjectTime + "s.");
2106                         System.out.println("extentSeeding took " + 1e-9
2107                                         * processor.extentSeedTime + "s.");
2108                         System.out.println("fullResolve took " + 1e-9
2109                                         * processor.fullResolveTime + "s.");
2110                         System.out.println("fastResolve took " + 1e-9
2111                                         * processor.fastResolveTime + "s.");
2112                         System.out.println("parentResolve took " + 1e-9
2113                                         * processor.parentResolveTime + "s.");
2114                         System.out.println("otherStatements took " + 1e-9
2115                                         * processor.otherStatementTime + "s.");
2116                         System.out.println("total " + 1e-9 * total + "s.");
2117                 }
2118
2119                 task.finish();
2120
2121         }
2122
2123 }