bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/Subgraphs.java

   1 /*******************************************************************************
   2  * Copyright (c) 2007, 2010 Association for Decentralized Information Management
   3  * in Industry THTH ry.
   4  * All rights reserved. This program and the accompanying materials
   5  * are made available under the terms of the Eclipse Public License v1.0
   6  * which accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *     VTT Technical Research Centre of Finland - initial API and implementation
  11  *******************************************************************************/
  12 package org.simantics.db.layer0.util;
  13
  14 import gnu.trove.list.array.TIntArrayList;
  15 import gnu.trove.map.hash.TIntIntHashMap;
  16 import gnu.trove.map.hash.TLongObjectHashMap;
  17 import gnu.trove.procedure.TIntProcedure;
  18 import gnu.trove.procedure.TLongObjectProcedure;
  19 import gnu.trove.set.TIntSet;
  20 import gnu.trove.set.hash.THashSet;
  21 import gnu.trove.set.hash.TIntHashSet;
  22
  23 import java.io.DataOutput;
  24 import java.io.DataOutputStream;
  25 import java.io.FileNotFoundException;
  26 import java.io.FileOutputStream;
  27 import java.io.IOException;
  28 import java.io.ObjectOutputStream;
  29 import java.util.ArrayList;
  30 import java.util.Collection;
  31 import java.util.Collections;
  32 import java.util.HashSet;
  33 import java.util.Map;
  34 import java.util.Set;
  35 import java.util.TreeMap;
  36 import java.util.concurrent.ConcurrentLinkedQueue;
  37 import java.util.concurrent.ConcurrentSkipListSet;
  38
  39 import org.simantics.databoard.Bindings;
  40 import org.simantics.databoard.Databoard;
  41 import org.simantics.databoard.binding.Binding;
  42 import org.simantics.databoard.binding.mutable.Variant;
  43 import org.simantics.databoard.serialization.Serializer;
  44 import org.simantics.databoard.type.Datatype;
  45 import org.simantics.db.AsyncReadGraph;
  46 import org.simantics.db.DirectStatements;
  47 import org.simantics.db.ReadGraph;
  48 import org.simantics.db.Resource;
  49 import org.simantics.db.Statement;
  50 import org.simantics.db.common.request.AsyncReadRequest;
  51 import org.simantics.db.common.request.ReadRequest;
  52 import org.simantics.db.common.request.ResourceAsyncRead;
  53 import org.simantics.db.common.utils.NameUtils;
  54 import org.simantics.db.exception.DatabaseException;
  55 import org.simantics.db.layer0.adapter.SubgraphAdvisor;
  56 import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus;
  57 import org.simantics.db.procedure.AsyncProcedure;
  58 import org.simantics.db.request.AsyncRead;
  59 import org.simantics.db.service.ClusteringSupport;
  60 import org.simantics.db.service.CollectionSupport;
  61 import org.simantics.db.service.DirectQuerySupport;
  62 import org.simantics.db.service.QueryControl;
  63 import org.simantics.db.service.QueryControl.ControlProcedure;
  64 import org.simantics.db.service.SerialisationSupport;
  65 import org.simantics.db.service.StatementSupport;
  66 import org.simantics.db.service.TransferableGraphSupport;
  67 import org.simantics.graph.representation.Extensions;
  68 import org.simantics.layer0.Layer0;
  69 import org.simantics.utils.datastructures.Pair;
  70 import org.simantics.utils.threads.logger.ITask;
  71 import org.simantics.utils.threads.logger.ThreadLogger;
  72
  73 public class Subgraphs {
  74
  75         public static String LOG_FILE = "export.log";
  76         final static private boolean LOG = false;
  77         final static private boolean DEBUG = false;
  78         final static private boolean PARENT_DEBUG = DEBUG | false;
  79         final static private boolean EXTERNAL_DEBUG = DEBUG | false;
  80         final static private boolean ADVISOR_LOG = LOG & false;
  81         final static private boolean EXPANSION_LOG = LOG & false;
  82         final static private boolean INTERNAL_LOG = LOG & false;
  83         final static private boolean COMPOSED_LOG = LOG & false;
  84         final static private boolean RESOLVE_LOG = LOG & false;
  85         final static private boolean CLASSIFY_LOG = LOG & false;
  86         final static private boolean EXTERNAL_LOG = LOG & false;
  87         final static private boolean PROFILE = false;
  88
  89         static enum WeakStatus {
  90                 STRONG, WEAK
  91         }
  92
  93
  94         static DataOutput log;
  95
  96         static {
  97
  98                 if (LOG) {
  99                         try {
 100                                 FileOutputStream stream = new FileOutputStream(LOG_FILE, false);
 101                                 log = new DataOutputStream(stream);
 102                         } catch (FileNotFoundException e) {
 103                                 e.printStackTrace();
 104                         }
 105                 }
 106
 107         }
 108
 109         private static void log(String line) {
 110                 if (LOG) {
 111                         try {
 112                                 log.write((line + "\n").getBytes());
 113                         } catch (IOException e) {
 114                                 e.printStackTrace();
 115                         }
 116                 }
 117         }
 118
 119         public static Collection<Resource> getParents(ReadGraph g, Resource r)
 120                         throws DatabaseException {
 121                 return getParents(g, r, false);
 122         }
 123
 124         static class FastInternalRequest extends ResourceAsyncRead<Boolean> {
 125
 126                 final DirectQuerySupport dqs;
 127                 final ConcurrentLinkedQueue<Resource> queue;
 128                 final Map<Resource, WeakStatus> weakInverses;
 129                 final Map<Resource, ExtentStatus> status;
 130
 131                 public FastInternalRequest(DirectQuerySupport dqs, Resource resource,
 132                                 Map<Resource, ExtentStatus> status,
 133                                 Map<Resource, WeakStatus> weakInverses,
 134                                 ConcurrentLinkedQueue<Resource> queue) {
 135                         super(resource);
 136                         this.dqs = dqs;
 137                         this.status = status;
 138                         this.weakInverses = weakInverses;
 139                         this.queue = queue;
 140                 }
 141
 142                 @Override
 143                 public int getFlags() {
 144                         return 0;
 145                 }
 146
 147                 @Override
 148                 public void perform(AsyncReadGraph graph, final AsyncProcedure<Boolean> procedure) {
 149
 150                         dqs.forEachDirectStatement(graph, resource, new AsyncProcedure<DirectStatements>() {
 151
 152                                 @Override
 153                                 public void execute(AsyncReadGraph graph, DirectStatements ss) {
 154                                         boolean ok = true;
 155                                         for(Statement statement : ss) {
 156                                                 if (status.get(statement.getObject()) == ExtentStatus.INTERNAL) continue;
 157                                                 WeakStatus status = weakInverses.get(statement.getPredicate());
 158                                                 if(status == WeakStatus.WEAK) continue;
 159                                                 else if (status == null) {
 160                                                         queue.add(statement.getPredicate());
 161                                                 }
 162                                                 ok = false;
 163                                         }
 164                                         procedure.execute(graph, ok);
 165                                 }
 166
 167                                 @Override
 168                                 public void exception(AsyncReadGraph graph, Throwable throwable) {
 169                                         throwable.printStackTrace();
 170                                 }
 171
 172                         });
 173
 174                 }
 175
 176         }
 177
 178         static class ClassifyStatementsRequest implements AsyncRead<Boolean> {
 179
 180                 final Set<Resource> schedule;
 181                 final Map<Resource, WeakStatus> weakMap;
 182
 183                 public ClassifyStatementsRequest(Set<Resource> schedule, Map<Resource, WeakStatus> weakMap) {
 184                         this.weakMap = weakMap;
 185                         this.schedule = schedule;
 186                 }
 187
 188             @Override
 189             public int threadHash() {
 190                 return hashCode();
 191             }
 192
 193                 @Override
 194                 public int getFlags() {
 195                         return 0;
 196                 }
 197
 198                 @Override
 199                 public void perform(AsyncReadGraph graph, final AsyncProcedure<Boolean> procedure) {
 200
 201                         for (final Resource p : schedule) {
 202
 203                                 graph.forPossibleInverse(p, new AsyncProcedure<Resource>() {
 204
 205                                         private void register(AsyncReadGraph graph, Resource predicate, Resource superRelation, WeakStatus status) {
 206                                                 synchronized (weakMap) {
 207                                                         weakMap.put(predicate, status);
 208                                                         if(superRelation != null) weakMap.put(superRelation, status);
 209                                                 }
 210                                         }
 211
 212                                         @Override
 213                                         public void exception(AsyncReadGraph graph, Throwable throwable) {
 214                                                 throwable.printStackTrace();
 215                                         }
 216
 217                                         @Override
 218                                         public void execute(AsyncReadGraph graph, final Resource inverse) {
 219
 220                                                 if (inverse == null) {
 221
 222                                                         register(graph, p, null, WeakStatus.WEAK);
 223
 224                                                 } else {
 225
 226                                                         graph.forPossibleSuperrelation(inverse, new AsyncProcedure<Resource>() {
 227
 228                                                                 @Override
 229                                                                 public void exception(AsyncReadGraph graph, Throwable throwable) {
 230                                                                         throwable.printStackTrace();
 231                                                                 }
 232
 233                                                                 @Override
 234                                                                 public void execute(AsyncReadGraph graph, final Resource superRelation) {
 235
 236                                                                         if(superRelation != null && weakMap.containsKey(superRelation)) {
 237                                                                                 register(graph, p, null, weakMap.get(superRelation));
 238                                                                                 return;
 239                                                                         }
 240
 241                                                                         graph.forIsSubrelationOf(inverse, graph.getService(Layer0.class).IsRelatedTo, new AsyncProcedure<Boolean>() {
 242
 243                                                                                 @Override
 244                                                                                 public void exception(AsyncReadGraph graph,Throwable throwable) {
 245                                                                                         throwable.printStackTrace();
 246                                                                                 }
 247
 248                                                                                 @Override
 249                                                                                 public void execute(AsyncReadGraph graph,Boolean strong) {
 250                                                                                         register(graph, p, superRelation, strong ? WeakStatus.STRONG : WeakStatus.WEAK);
 251                                                                                 }
 252
 253                                                                         });
 254
 255                                                                 }
 256
 257                                                         });
 258
 259                                                 }
 260
 261                                         }
 262
 263                                 });
 264
 265                         }
 266
 267                         procedure.execute(graph, false);
 268
 269                 }
 270
 271         }
 272
 273         private static Collection<Resource> getParents(ReadGraph g, Resource r, boolean isStrong) throws DatabaseException {
 274
 275                 System.out.println("getParents " + NameUtils.getSafeName(g, r));
 276
 277                 Layer0 l0 = Layer0.getInstance(g);
 278
 279                 Collection<Resource> predicates = g.getPredicates(r);
 280
 281                 // --- Consists Of ----------------------------------------------------
 282
 283                 if (predicates.contains(l0.PartOf)) {
 284                         Collection<Resource> parents = g.getObjects(r, l0.PartOf);
 285                         if (parents.size() == 1)
 286                                 return parents;
 287                         ArrayList<Resource> libraryParents = new ArrayList<Resource>(1);
 288                         for (Resource p : parents)
 289                                 if (g.isInstanceOf(p, l0.Library))
 290                                         libraryParents.add(p);
 291                         if (!libraryParents.isEmpty())
 292                                 return libraryParents;
 293                         else
 294                                 return parents;
 295                 }
 296
 297                 // --- Ordered sets ---------------------------------------------------
 298
 299                 {
 300                         Collection<Resource> parents = null;
 301                         for (Resource p : predicates)
 302                                 if (g.isInstanceOf(p, l0.OrderedSet) && !p.equals(r)) {
 303                                         if (parents == null)
 304                                                 parents = new ArrayList<Resource>(1);
 305                                         parents.add(p);
 306                                 }
 307                         if (parents != null) {
 308                                 if (DEBUG)
 309                                         System.out.println("ORDERED SET");
 310                                 return parents;
 311                         }
 312                 }
 313
 314
 315                 if (isStrong)
 316                         return Collections.emptyList();
 317                 else {
 318
 319                         if (predicates.contains(l0.InverseOf)) {
 320
 321                                 Resource inv = g.getInverse(r);
 322                                 return getParents(g, inv, true);
 323
 324                         } else {
 325
 326                                 /*
 327                                  * Depends On
 328                                  *
 329                                  * If there are DependsOn parents, then IsRelatedTo parents are discarded
 330                                  *
 331                                  */
 332                                 HashSet<Resource> result = new HashSet<Resource>();
 333                                 for(Resource predicate : predicates) {
 334                                         if(g.isSubrelationOf(predicate, l0.IsDependencyOf)) result.addAll(g.getObjects(r, predicate));
 335                                 }
 336                                 if(!result.isEmpty()) return result;
 337
 338                                 /*
 339                                  * Is Related To
 340                                  *
 341                                  * At this point all Is Related To are parents.
 342                                  *
 343                                  */
 344                                 for(Resource predicate : predicates) {
 345                                         Resource inv = g.getPossibleInverse(predicate);
 346                                         if(inv != null) {
 347                                                 if(g.isSubrelationOf(inv, l0.IsRelatedTo)) result.addAll(g.getObjects(r, predicate));
 348                                         }
 349                                 }
 350
 351                                 return result;
 352
 353                         }
 354
 355                         /*
 356                         Collection<Resource> invR = g.getObjects(r, b.IsRelatedTo_Inverse);
 357                         if (predicates.contains(b.InverseOf)) {
 358                                 if (invR.size() > 1) {
 359                                         if (DEBUG)
 360                                                 System.out
 361                                                                 .println("###########################################");
 362                                         Resource inv = g.getInverse(r);
 363                                         Collection<Resource> ret = new ArrayList<Resource>();
 364                                         for (Statement pp : g.getStatements(r,
 365                                                         b.IsRelatedTo_Inverse))
 366                                                 if (!pp.getPredicate().equals(inv)) {
 367                                                         if (DEBUG) {
 368                                                                 System.out.println("<"
 369                                                                                 + NameUtils.getSafeName(g, pp
 370                                                                                                 .getSubject())
 371                                                                                 + ","
 372                                                                                 + NameUtils.getSafeName(g, pp
 373                                                                                                 .getPredicate())
 374                                                                                 + ","
 375                                                                                 + NameUtils.getSafeName(g, pp
 376                                                                                                 .getObject()) + ">");
 377                                                         }
 378                                                         ret.add(pp.getObject());
 379                                                 }
 380                                         return ret;
 381                                 }
 382                                 // System.out.println("?????????????????");
 383                                 Collection<Resource> invParents = getParents(g,
 384                                                 g.getInverse(r), true);
 385                                 if (!invParents.isEmpty())
 386                                         return invParents;
 387                         }
 388                         if (DEBUG) {
 389                                 System.out.print("invR");
 390                                 for (Resource res : invR)
 391                                         System.out.print(" " + NameUtils.getSafeName(g, res));
 392                                 System.out.println();
 393                         }
 394                         return invR;
 395                         */
 396                 }
 397
 398         }
 399
 400 //      public static String getIdentifier(ReadGraph g, Resource r)
 401 //                      throws DatabaseException {
 402 //      Layer0 L0 = Layer0.getInstance(g);
 403 //              if (r.equals(g.getRootLibrary()))
 404 //                      return "";
 405 //              String name = g.getPossibleRelatedValue(r, L0.HasName);
 406 //              if (name == null)
 407 //                      return null;
 408 //              Collection<Resource> parents = getParents(g, r, true);
 409 //              if (parents.size() != 1)
 410 //                      return null;
 411 //              for (Resource p : parents) {
 412 //                      String parentIdentifier = getIdentifier(g, p);
 413 //                      if (parentIdentifier == null)
 414 //                              return null;
 415 //                      return parentIdentifier + "/" + name;
 416 //              }
 417 //              return null;
 418 //      }
 419
 420         static int kess = 0;
 421
 422         static class Expansion extends AsyncReadRequest {
 423
 424                 final private Collection<Resource> roots;
 425                 final Collection<DirectStatements>[] results;
 426                 final Collection<Resource>[] listElements;
 427
 428                 public Expansion(Collection<Resource> roots, Collection<DirectStatements>[] results, Collection<Resource>[] listElements) {
 429                         this.roots = roots;
 430                         this.results = results;
 431                         this.listElements = listElements;
 432                 }
 433
 434                 @Override
 435                 public void run(AsyncReadGraph graph) {
 436
 437                         QueryControl control = graph.getService(QueryControl.class);
 438                         final DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class);
 439
 440                         final DomainStatementProcedure proc = new DomainStatementProcedure(dqs, graph.getService(StatementSupport.class), graph.getService(Layer0.class), results, listElements);
 441
 442                         int slice = (int) (roots.size() / control.getAmountOfQueryThreads()) + 1;
 443
 444                         final Resource[] rootArray = roots.toArray(Resource.NONE);
 445                         for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
 446
 447                                 final int start = i * slice;
 448                                 final int end = Math.min(start + slice, rootArray.length);
 449
 450                                 control.schedule(graph, i, new ControlProcedure() {
 451
 452                                         @Override
 453                                         public void execute(AsyncReadGraph graph) {
 454                                                 for (int index = start; index < end; index++) {
 455                                                         dqs.forEachDirectStatement(graph, rootArray[index], proc);
 456                                                 }
 457
 458                                         }
 459
 460                                 });
 461
 462                         }
 463
 464                 }
 465
 466                 @Override
 467                 public int getFlags() {
 468                         return 0;
 469                 }
 470
 471         }
 472
 473         static class Expansion2 extends AsyncReadRequest {
 474
 475                 final private Collection<Resource> roots;
 476                 final Collection<DirectStatements>[] results;
 477                 final boolean ignoreVirtual;
 478
 479                 public Expansion2(Collection<Resource> roots, Collection<DirectStatements>[] results) {
 480                         this(roots, results, true);
 481                 }
 482
 483                 public Expansion2(Collection<Resource> roots, Collection<DirectStatements>[] results, boolean ignoreVirtual) {
 484                         this.roots = roots;
 485                         this.results = results;
 486                         this.ignoreVirtual = ignoreVirtual;
 487                 }
 488
 489                 @Override
 490                 public void run(AsyncReadGraph graph) {
 491
 492                         QueryControl control = graph.getService(QueryControl.class);
 493                         final DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class);
 494
 495                         final DomainStatementProcedure2 proc =
 496                                 new DomainStatementProcedure2(results);
 497
 498                         int slice = (int) (roots.size() / control.getAmountOfQueryThreads()) + 1;
 499
 500                         final Resource[] rootArray = roots.toArray(Resource.NONE);
 501                         for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
 502
 503                                 final int start = i * slice;
 504                                 final int end = Math.min(start + slice, rootArray.length);
 505
 506                                 control.schedule(graph, i, new ControlProcedure() {
 507                                         @Override
 508                                         public void execute(AsyncReadGraph graph) {
 509                                                 if (ignoreVirtual) {
 510                                                         for (int index = start; index < end; index++) {
 511                                                                 dqs.forEachDirectPersistentStatement(graph, rootArray[index], proc);
 512                                                         }
 513                                                 } else {
 514                                                         for (int index = start; index < end; index++) {
 515                                                                 dqs.forEachDirectStatement(graph, rootArray[index], proc);
 516                                                         }
 517                                                 }
 518                                         }
 519                                 });
 520
 521                         }
 522
 523                 }
 524
 525                 @Override
 526                 public int getFlags() {
 527                         return 0;
 528                 }
 529
 530         }
 531
 532         static class DomainProcessor2 {
 533
 534                 Serializer variantSerializer;
 535
 536                 int id = 0;
 537
 538                 Set<Resource> fringe = null;
 539                 Set<Resource> exclusions = new HashSet<Resource>();
 540                 Set<Resource> internalDomain = new HashSet<Resource>();
 541                 Set<Resource> sharedExternalReferences = null;
 542                 TIntSet sharedExternalIds = null;
 543                 Set<Resource> sharedExternalFringe = null;
 544                 Set<Resource> predicates = null;
 545                 Set<Resource> isRelatedToPredicates = null;
 546                 Set<Resource> sharedPredicates = null;
 547                 TIntIntHashMap ids = null;
 548                 Map<Resource, Statement> specials = null;
 549                 Map<Resource, ExtentStatus> status = null;
 550                 Map<Resource, WeakStatus> weakInverses = null;
 551
 552 //              final ArrayList<Double> priorityList = new ArrayList<Double>();
 553
 554                 private long composedObjectCounter = 0;
 555                 private long fastInternalCounter = 0;
 556                 private long parentExternalCounter = 0;
 557                 private long fullInternalCounter = 0;
 558                 private long fullExternalCounter = 0;
 559
 560                 private long startupTime = 0;
 561                 private long expandTime = 0;
 562                 private long fullResolveTime = 0;
 563                 private long fastResolveTime = 0;
 564                 private long otherStatementTime = 0;
 565                 private long parentResolveTime = 0;
 566                 private long extentSeedTime = 0;
 567                 private long composedPredicateTime = 0;
 568                 private long composedObjectTime = 0;
 569
 570                 public void expand(ReadGraph graph, Set<Resource> fringe, Collection<DirectStatements>[] expansion) throws DatabaseException {
 571
 572                         long start = System.nanoTime();
 573
 574                         Collection<Collection<DirectStatements>[]> fullExpansion = new ArrayList<Collection<DirectStatements>[]>();
 575                         QueryControl control = graph.getService(QueryControl.class);
 576                         for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
 577                                 expansion[i] = new ArrayList<DirectStatements>();
 578                         }
 579
 580                         graph.syncRequest(new Expansion2(fringe, expansion));
 581
 582                         fringe.clear();
 583
 584                         expandTime += (System.nanoTime() - start);
 585
 586                 }
 587
 588                 public void classifyPredicates(ReadGraph graph, final Set<Resource> schedule) throws DatabaseException {
 589
 590                         CollectionSupport cs = graph.getService(CollectionSupport.class);
 591
 592                         final Layer0 L0 = Layer0.getInstance(graph);
 593
 594                         long start = System.nanoTime();
 595
 596                         final ConcurrentLinkedQueue<Resource> composedResult = new ConcurrentLinkedQueue<Resource>();
 597                         final ConcurrentLinkedQueue<Resource> singleResult = new ConcurrentLinkedQueue<Resource>();
 598                         final ConcurrentLinkedQueue<Resource> sharedResult = new ConcurrentLinkedQueue<Resource>();
 599                         final ConcurrentLinkedQueue<Pair<Resource, Resource>> singles = new ConcurrentLinkedQueue<Pair<Resource, Resource>>();
 600
 601                         // Discover singles
 602                         graph.syncRequest(new AsyncReadRequest() {
 603
 604                                 @Override
 605                                 public void run(AsyncReadGraph graph) {
 606
 607                                         for (final Resource predicate : schedule) {
 608
 609                                                 graph.forPossibleSuperrelation(predicate, new AsyncProcedure<Resource>() {
 610
 611                                                         @Override
 612                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {
 613                                                                 throwable.printStackTrace();
 614                                                         }
 615
 616                                                         @Override
 617                                                         public void execute(AsyncReadGraph graph, final Resource single) {
 618                                                                 singles.add(Pair.make(predicate, single));
 619                                                         }
 620
 621                                                 });
 622
 623                                                 graph.forHasStatement(predicate, L0.SharedRange, new AsyncProcedure<Boolean>() {
 624
 625                                                         @Override
 626                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {
 627                                                                 throwable.printStackTrace();
 628                                                         }
 629
 630                                                         @Override
 631                                                         public void execute(AsyncReadGraph graph, final Boolean shared) {
 632                                                                 if(shared) sharedResult.add(predicate);
 633                                                         }
 634
 635                                                 });
 636
 637                                         }
 638
 639                                 }
 640
 641                         });
 642
 643                         // Determine singles
 644                         final Set<Resource> singleSchedule = cs.createSet();
 645                         for(Pair<Resource, Resource> pair : singles) {
 646
 647                                 Resource single = pair.second;
 648                                 if(single != null && predicates.add(single)) singleSchedule.add(single);
 649
 650                         }
 651
 652                         graph.syncRequest(new AsyncReadRequest() {
 653
 654                                 @Override
 655                                 public void run(AsyncReadGraph graph) {
 656
 657                                         for (final Resource predicate : singleSchedule) {
 658
 659                                                 graph.forIsSubrelationOf(predicate, L0.IsRelatedTo, new AsyncProcedure<Boolean>() {
 660
 661                                                         @Override
 662                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {
 663                                                                 throwable.printStackTrace();
 664                                                         }
 665
 666                                                         @Override
 667                                                         public void execute(AsyncReadGraph graph, Boolean strong) {
 668                                                                 if (strong) singleResult.add(predicate);
 669                                                         }
 670
 671                                                 });
 672
 673                                         }
 674
 675                                 }
 676
 677                         });
 678
 679                         isRelatedToPredicates.addAll(singleResult);
 680                         sharedPredicates.addAll(sharedResult);
 681
 682                         final Set<Resource> specialSchedule = cs.createSet();
 683
 684                         // Classify
 685                         for(Pair<Resource, Resource> pair : singles) {
 686
 687                                 Resource single = pair.second;
 688                                 if(single != null) {
 689                                         if(isRelatedToPredicates.contains(single)) {
 690                                                 isRelatedToPredicates.add(pair.first);
 691                                         }
 692                                 } else {
 693                                         specialSchedule.add(pair.first);
 694                                 }
 695
 696                         }
 697
 698                         graph.syncRequest(new AsyncReadRequest() {
 699
 700                                 @Override
 701                                 public void run(AsyncReadGraph graph) {
 702
 703                                         for (final Resource predicate : specialSchedule) {
 704
 705                                                 graph.forIsSubrelationOf(predicate, L0.IsRelatedTo, new AsyncProcedure<Boolean>() {
 706
 707                                                         @Override
 708                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {
 709                                                                 throwable.printStackTrace();
 710                                                         }
 711
 712                                                         @Override
 713                                                         public void execute(AsyncReadGraph graph, Boolean composed) {
 714                                                                 if (composed) composedResult.add(predicate);
 715                                                         }
 716
 717                                                 });
 718
 719                                         }
 720
 721                                 }
 722
 723                         });
 724
 725                         isRelatedToPredicates.addAll(composedResult);
 726
 727                         composedPredicateTime += (System.nanoTime() - start);
 728
 729                 }
 730
 731                 private Set<Resource> strongInverseSet = new HashSet<Resource>();
 732
 733                 public void classifyPredicates(ReadGraph graph, final Collection<DirectStatements>[] expansion) throws DatabaseException {
 734
 735                         CollectionSupport cs = graph.getService(CollectionSupport.class);
 736                         final Set<Resource> schedule = cs.createSet();
 737                         final Map<Resource, Resource> newPredicates = cs.createMap(Resource.class);
 738
 739                         for (Collection<DirectStatements> coll : expansion)
 740                                 for (DirectStatements stms : coll)
 741                                         for(Statement stm : stms) {
 742
 743                                                 Resource predicate = stm.getPredicate();
 744
 745                                                 if(predicates.add(predicate)) {
 746                                                         Resource inverse = graph.getPossibleInverse(predicate);
 747                                                         schedule.add(predicate);
 748                                                         if(inverse != null) {
 749                                                                 newPredicates.put(predicate, inverse);
 750                                                                 if(predicates.add(inverse)) schedule.add(inverse);
 751                                                         }
 752
 753                                                 }
 754
 755                                         }
 756
 757                         classifyPredicates(graph, schedule);
 758
 759                         for(Map.Entry<Resource, Resource> entry : newPredicates.entrySet()) {
 760                                 // Inverse is strong => this has strong inverse
 761                                 if(isRelatedToPredicates.contains(entry.getValue())) {
 762                                         strongInverseSet.add(entry.getKey());
 763                                 }
 764                                 // This is strong => inverse has strong inverse
 765                                 if(isRelatedToPredicates.contains(entry.getKey())) {
 766                                         strongInverseSet.add(entry.getValue());
 767                                 }
 768                         }
 769
 770                 }
 771
 772                 /*
 773                  * Composed objects are internal. Mark them for expansion.
 774                  */
 775
 776                 public void processFringe(ReadGraph graph, Collection<DirectStatements>[] expansion,
 777                                 ObjectOutputStream otherStatementsOutput, ObjectOutputStream valueOutput) throws DatabaseException, IOException {
 778
 779                         SerialisationSupport support = graph.getService(SerialisationSupport.class);
 780                         TransferableGraphSupport tgs = graph.getService(TransferableGraphSupport.class);
 781
 782                         Layer0 L0 = Layer0.getInstance(graph);
 783
 784                         long start = System.nanoTime();
 785
 786                         for (Collection<DirectStatements> coll : expansion)
 787                                 for (DirectStatements stms : coll) {
 788
 789                                         Resource subject = stms.getSubject();
 790
 791                                         boolean partOf = false;
 792                                         for(Statement stm : stms) {
 793                                                 Resource predicate = stm.getPredicate();
 794                                                 if(L0.PartOf.equals(predicate)) {
 795                                                         partOf = true;
 796                                                         break;
 797                                                 }
 798                                         }
 799
 800                                         ExtentStatus subjectStatus = status.get(subject);
 801                                         if(LOG && subjectStatus != null) log("EXISTING STATUS FOR " + graph.getPossibleURI(subject) + " - " + subjectStatus);
 802                                         if(subjectStatus == ExtentStatus.EXTERNAL) continue;
 803                                         if(partOf && (subjectStatus == null) && graph.getPossibleURI(subject) != null) {
 804
 805                                                 status.put(subject, ExtentStatus.EXTERNAL);
 806                                                 if(LOG) {
 807                                                         String uri = graph.getPossibleURI(subject);
 808                                                         if(uri == null) log("[EXTERNAL]: No URI for " + subject);
 809                                                         else log("[EXTERNAL] " + uri);
 810                                                 }
 811
 812                                                 // Check for SharedRange statements
 813                                                 for(Statement stm : stms) {
 814                                                         Resource predicate = stm.getPredicate();
 815                                                         if(sharedPredicates.contains(predicate)) {
 816                                                                 sharedExternalFringe.add(stm.getObject());
 817                                                                 if(LOG) {
 818                                                                         log("[SHARED EXTERNAL FRINGE]: " + NameUtils.getSafeName(graph, stm.getObject()));
 819                                                                 }
 820                                                         }
 821                                                 }
 822
 823                                         } else {
 824
 825                                                 boolean special = specials.containsKey(subject);
 826                                                 if(LOG) {
 827                                                         if(special) {
 828                                                                 log("[SPECIAL] " + NameUtils.getSafeName(graph, subject));
 829                                                         }
 830                                                 }
 831
 832                                                 status.put(subject, ExtentStatus.INTERNAL);
 833                                                 if(LOG) log("[INTERNAL] " + NameUtils.getSafeName(graph, subject));
 834
 835                                                 int sId = support.getTransientId(subject);
 836
 837                                                 if(graph.hasValue(subject)) {
 838                                                         Datatype dt = graph.getRelatedValue(subject, L0.HasDataType, Bindings.getBindingUnchecked(Datatype.class));
 839                                                         Binding b = Bindings.getBinding(dt);
 840                                                         Object _value = graph.getValue(subject, b);
 841                                                         Variant variant = new Variant(b, _value);
 842                                                         byte[] value = variantSerializer.serialize(variant);
 843                                                         if(LOG) log("[VALUE] " + NameUtils.getSafeName(graph, subject));
 844                                                         valueOutput.writeInt(sId);
 845                                                         valueOutput.writeInt(value.length);
 846                                                         assert (value.length > 0);
 847                                                         valueOutput.write(value);
 848                                                 }
 849
 850                                                 TIntArrayList stream = new TIntArrayList();
 851
 852                                                 for(Statement stm : stms) {
 853
 854                                                         if(special) {
 855
 856 //                                                              System.err.println("stm=" + stm + " special=" + specials.get(subject));
 857
 858                                                         }
 859
 860                                                         Resource predicate = stm.getPredicate();
 861                                                         Resource object = stm.getObject();
 862
 863                                                         ExtentStatus objectStatus = status.get(object);
 864
 865                                                         // Strong predicate
 866                                                         if (isRelatedToPredicates.contains(predicate) && (objectStatus !=  ExtentStatus.EXCLUDED)) {
 867
 868                                                                 int pId = support.getTransientId(predicate);
 869                                                                 int oId = support.getTransientId(object);
 870
 871                                                                 if(LOG) {
 872                                                                         String s = NameUtils.getSafeName(graph, subject);
 873                                                                         String p = NameUtils.getSafeName(graph, predicate);
 874                                                                         String o = NameUtils.getSafeName(graph, object);
 875                                                                         log("related=" + s + " - " + p + " - " + o);
 876                                                                 }
 877
 878                                                                 stream.add(pId);
 879                                                                 stream.add(oId);
 880
 881                                                                 if(objectStatus == null)
 882                                                                         fringe.add(object);
 883
 884                                                         } else {
 885
 886                                                                 // Weak predicate
 887                                                                 if(objectStatus == ExtentStatus.INTERNAL) {
 888
 889                                                                         // The inverse is also weak (or there is no inverse)
 890                                                                         if(!strongInverseSet.contains(predicate)) {
 891
 892                                                                                 int pId = support.getTransientId(predicate);
 893                                                                                 int oId = support.getTransientId(object);
 894
 895                                                                                 stream.add(pId);
 896                                                                                 stream.add(oId);
 897
 898                                                                                 if(LOG) {
 899                                                                                         String s = NameUtils.getSafeName(graph, subject);
 900                                                                                         String p = NameUtils.getSafeName(graph, predicate);
 901                                                                                         String o = NameUtils.getSafeName(graph, object);
 902                                                                                         log("fully weak internal=" + s + " - " + p + " - " + o + " - " + objectStatus);
 903                                                                                 }
 904
 905                                                                         } else {
 906
 907                                                                                 if(LOG) {
 908                                                                                         String s = NameUtils.getSafeName(graph, subject);
 909                                                                                         String p = NameUtils.getSafeName(graph, predicate);
 910                                                                                         String o = NameUtils.getSafeName(graph, object);
 911                                                                                         log("strong inverse internals=" + s + " - " + p + " - " + o + " - " + objectStatus);
 912                                                                                 }
 913
 914                                                                         }
 915
 916                                                                 } else {
 917
 918                                                                         if(special) {
 919
 920 //                                                                              System.err.println("stm=" + stm + " special=" + specials.get(subject));
 921
 922                                                                                 Statement spec = specials.get(subject);
 923
 924                                                                                 // This statement can be specially treated
 925                                                                                 if(stm.getPredicate().equals(spec.getPredicate()) && stm.getObject().equals(spec.getObject())) {
 926
 927                                                                                         int pId = support.getTransientId(predicate);
 928                                                                                         int oId = support.getTransientId(object);
 929
 930                                                                                         if(LOG) {
 931                                                                                                 String s = NameUtils.getSafeName(graph, subject);
 932                                                                                                 String p = NameUtils.getSafeName(graph, predicate);
 933                                                                                                 String o = NameUtils.getSafeName(graph, object);
 934                                                                                                 log("special=" + s + " - " + p + " - " + o);
 935                                                                                         }
 936
 937                                                                                         stream.add(pId);
 938                                                                                         stream.add(oId);
 939
 940                                                                                 }
 941
 942                                                                         } else {
 943
 944                                                                                 if(LOG) {
 945                                                                                         String s = NameUtils.getSafeName(graph, subject);
 946                                                                                         String p = NameUtils.getSafeName(graph, predicate);
 947                                                                                         String o = NameUtils.getSafeName(graph, object);
 948                                                                                         log("weak with unknown object=" + s + " - " + p + " - " + o + " - " + objectStatus);
 949                                                                                 }
 950
 951                                                                         }
 952
 953                                                                 }
 954
 955                                                         }
 956
 957                                                 }
 958
 959                                                 if(!stream.isEmpty()) {
 960                                                         otherStatementsOutput.writeInt(sId);
 961                                                         otherStatementsOutput.writeInt(stream.size() / 2);
 962                                                         for (int i = 0; i < stream.size(); i++)
 963                                                                 otherStatementsOutput.writeInt(stream.getQuick(i));
 964                                                 }
 965
 966                                         }
 967
 968                                 }
 969
 970                         composedObjectTime += System.nanoTime() - start;
 971
 972                 }
 973
 974                 public void process(ReadGraph graph,
 975                                 ObjectOutputStream otherStatementsOutput,
 976                                 ObjectOutputStream valueOutput)
 977                                 throws DatabaseException, IOException {
 978
 979                         this.variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT);
 980
 981                         QueryControl control = graph.getService(QueryControl.class);
 982
 983 //                      System.err.println("Begin ConsistsOfProcess");
 984
 985                         /*
 986                          * Browse all stm = (s, ConsistsOf, o)
 987                          * � All internal URIs are found => from now on, if unidentified resource has PartOf it is external.
 988              * � All s are internal
 989              * � All o are internal
 990              * � All stm are included
 991                          */
 992                         for(Resource r : ConsistsOfProcess.walk(graph, fringe, exclusions, true)) {
 993                                 if (status.put(r, ExtentStatus.INTERNAL) == null) {
 994                                         String URI = graph.getPossibleURI(r);
 995                                         if(URI != null) log("URI INTERNAL " + URI);
 996                                         else log("URI has no URI for " + r);
 997                                         fringe.add(r);
 998                                         internalDomain.add(r);
 999                                 }
1000                         }
1001
1002                         /*
1003                          * This loop resolves the transitive closure of all p < IsRelatedTo such that p does not contain the SharedRange tag.
1004                          * Such resources are guaranteed to be internal.
1005                          */
1006                         while(!fringe.isEmpty()) {
1007
1008 //                              System.err.println("Process Fringe with " + fringe.size() + ".");
1009
1010                                 Collection<DirectStatements>[] expansion = new ArrayList[control.getAmountOfQueryThreads()];
1011
1012 //                              System.err.println("-expand");
1013
1014                                 // Expand fringe
1015                                 expand(graph, fringe, expansion);
1016
1017                                 /*
1018                                  * classify all p
1019                                  * -IsRelatedTo
1020                                  * -SharedRange
1021                                  * -Internal / External
1022                                  */
1023
1024 //                              System.err.println("-classify");
1025
1026                                 classifyPredicates(graph, expansion);
1027
1028                                 /*
1029                                  * for stms in [stms]
1030                                  *   if stms contains predicate PartOf => s is External
1031                                  *   else s is Internal
1032                                  *   for all stm=(s,p,o) in stms
1033                                  *     if p <R IsRelatedTo => stm is included
1034                                  *     Fringe <- o
1035                                  */
1036
1037 //                              System.err.println("-process");
1038
1039                                 processFringe(graph, expansion, otherStatementsOutput, valueOutput);
1040
1041                         }
1042
1043                         while(!sharedExternalFringe.isEmpty()) {
1044
1045                                 Collection<DirectStatements>[] expansion = new ArrayList[control.getAmountOfQueryThreads()];
1046                                 expand(graph, sharedExternalFringe, expansion);
1047
1048                                 for (Collection<DirectStatements> coll : expansion)
1049                                         for (DirectStatements stms : coll) {
1050
1051                                                 Resource subject = stms.getSubject();
1052                                                 ExtentStatus subjectStatus = status.get(subject);
1053
1054                                                 if(ExtentStatus.INTERNAL == subjectStatus) {
1055
1056                                                         if(internalDomain.contains(subject)) continue;
1057
1058                                                         status.put(subject, ExtentStatus.EXTERNAL);
1059                                                         sharedExternalReferences.add(subject);
1060
1061                                                         if(LOG) {
1062                                                                 log("[SHARED EXTERNAL REFERENCE]: " + NameUtils.getSafeName(graph, subject));
1063                                                         }
1064
1065                                                         for(Statement stm : stms) {
1066                                                                 Resource predicate = stm.getPredicate();
1067                                                                 if (isRelatedToPredicates.contains(predicate)) {
1068                                                                         sharedExternalFringe.add(stm.getObject());
1069                                                                 }
1070                                                         }
1071
1072                                                 }
1073                                         }
1074
1075                         }
1076
1077                         if (PROFILE) {
1078                                 System.out.println(composedObjectCounter + " " + fastInternalCounter
1079                                                 + " " + parentExternalCounter + " "
1080                                                 + fullExternalCounter + " " + fullInternalCounter);
1081                         }
1082
1083                 }
1084
1085         }
1086
1087         static class DomainProcessor {
1088
1089                 Serializer variantSerializer;
1090
1091                 int id = 0;
1092
1093                 Set<Resource> predicates = null;
1094                 Set<Resource> composedPredicates = null;
1095                 Set<Resource> expansionSeeds = null;
1096                 Map<Resource, Integer> ids = null;
1097                 Map<Resource, ExtentStatus> status = null;
1098                 Map<Resource, WeakStatus> weakInverses = null;
1099
1100                 final Set<SubgraphAdvisor> advisors;
1101                 final ArrayList<Double> priorityList = new ArrayList<Double>();
1102
1103                 private long composedObjectCounter = 0;
1104                 private long fastInternalCounter = 0;
1105                 private long parentExternalCounter = 0;
1106                 private long fullInternalCounter = 0;
1107                 private long fullExternalCounter = 0;
1108
1109                 private long startupTime = 0;
1110                 private long expandTime = 0;
1111                 private long fullResolveTime = 0;
1112                 private long fastResolveTime = 0;
1113                 private long otherStatementTime = 0;
1114                 private long parentResolveTime = 0;
1115                 private long extentSeedTime = 0;
1116                 private long composedPredicateTime = 0;
1117                 private long composedObjectTime = 0;
1118
1119                 public DomainProcessor(Set<SubgraphAdvisor> advisors) {
1120                         this.advisors = advisors;
1121                         HashSet<Double> prioritySet = new HashSet<Double>();
1122                         for (SubgraphAdvisor advisor : advisors)
1123                                 prioritySet.add(advisor.priority());
1124                         priorityList.addAll(prioritySet);
1125                         Collections.sort(priorityList);
1126                 }
1127
1128                 public void expand(ReadGraph graph, Collection<DirectStatements>[] expansion, Set<Resource> schedule) throws DatabaseException {
1129
1130                         long start = System.nanoTime();
1131
1132 //                      if (DEBUG)
1133 //                              System.out.println("expanding " + expansionSeeds.size() + " resources.");
1134
1135                         QueryControl control = graph.getService(QueryControl.class);
1136 //                      final Collection<DirectStatements>[] results = new ArrayList[control.getAmountOfQueryThreads()];
1137                         final ArrayList<Resource>[] listElements = new ArrayList[control.getAmountOfQueryThreads()];
1138                         for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
1139 //                              results[i] = new ArrayList<DirectStatements>();
1140                                 listElements[i] = new ArrayList<Resource>();
1141                         }
1142
1143 //                      if(DEBUG) {
1144 //                              for(Resource r : expansionSeeds)
1145 //                                      System.out.println("Expanding " + NameUtils.getSafeName(graph, r, true));
1146 //                      }
1147
1148                         graph.syncRequest(new Expansion(expansionSeeds, expansion, listElements));
1149                         for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
1150 //                              for (DirectStatements s : results[i]) {
1151 //                                      expansion.put(s.getSubject(), s);
1152 //                              }
1153                                 for (Resource s : listElements[i]) {
1154                                         schedule.add(s);
1155 //                                      if(status.put(s, ExtentStatus.INTERNAL) == null) {
1156 //                                              ids.put(s, id++);
1157 //                                      }
1158                                 }
1159                         }
1160
1161                         expandTime += (System.nanoTime() - start);
1162
1163                 }
1164
1165                 public void extractComposedPredicates(ReadGraph graph, final Collection<DirectStatements>[] expansion) throws DatabaseException {
1166
1167                         long start = System.nanoTime();
1168
1169                         CollectionSupport cs = graph.getService(CollectionSupport.class);
1170
1171                         final ConcurrentLinkedQueue<Resource> composedResult = new ConcurrentLinkedQueue<Resource>();
1172                         final ConcurrentLinkedQueue<Resource> singleResult = new ConcurrentLinkedQueue<Resource>();
1173                         final ConcurrentLinkedQueue<Pair<Resource, Resource>> singles = new ConcurrentLinkedQueue<Pair<Resource, Resource>>();
1174
1175                         final Set<Resource> schedule = cs.createSet();
1176
1177                         for (Collection<DirectStatements> coll : expansion)
1178                                 for (DirectStatements stms : coll)
1179                                         for(Statement stm : stms) {
1180                                                 Resource predicate = stm.getPredicate();
1181                                                 if(predicates.add(predicate)) schedule.add(predicate);
1182                                         }
1183
1184                         // Discover singles
1185                         graph.syncRequest(new AsyncReadRequest() {
1186
1187                                 @Override
1188                                 public void run(AsyncReadGraph graph) {
1189
1190                                         for (final Resource predicate : schedule) {
1191
1192                                                 graph.forPossibleSuperrelation(predicate, new AsyncProcedure<Resource>() {
1193
1194                                                         @Override
1195                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {
1196                                                                 throwable.printStackTrace();
1197                                                         }
1198
1199                                                         @Override
1200                                                         public void execute(AsyncReadGraph graph, final Resource single) {
1201                                                                 singles.add(Pair.make(predicate, single));
1202                                                         }
1203
1204                                                 });
1205
1206                                         }
1207
1208                                 }
1209
1210                         });
1211
1212                         // Determine singles
1213                         final Set<Resource> singleSchedule = cs.createSet();
1214                         for(Pair<Resource, Resource> pair : singles) {
1215
1216                                 Resource single = pair.second;
1217                                 if(single != null && predicates.add(single)) singleSchedule.add(single);
1218
1219                         }
1220
1221                         graph.syncRequest(new AsyncReadRequest() {
1222
1223                                 @Override
1224                                 public void run(AsyncReadGraph graph) {
1225
1226                                         for (final Resource predicate : singleSchedule) {
1227
1228                                                 graph.forIsSubrelationOf(predicate, graph.getService(Layer0.class).IsComposedOf, new AsyncProcedure<Boolean>() {
1229
1230                                                         @Override
1231                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {
1232                                                                 throwable.printStackTrace();
1233                                                         }
1234
1235                                                         @Override
1236                                                         public void execute(AsyncReadGraph graph, Boolean composed) {
1237                                                                 if (composed) singleResult.add(predicate);
1238                                                         }
1239
1240                                                 });
1241
1242                                         }
1243
1244                                 }
1245
1246                         });
1247
1248                         composedPredicates.addAll(singleResult);
1249
1250                         final Set<Resource> specialSchedule = cs.createSet();
1251
1252                         // Classify
1253                         for(Pair<Resource, Resource> pair : singles) {
1254
1255                                 Resource single = pair.second;
1256                                 if(single != null) {
1257                                         if(composedPredicates.contains(single)) {
1258                                                 composedPredicates.add(pair.first);
1259                                         }
1260                                 } else {
1261                                         specialSchedule.add(pair.first);
1262                                 }
1263
1264                         }
1265
1266                         graph.syncRequest(new AsyncReadRequest() {
1267
1268                                 @Override
1269                                 public void run(AsyncReadGraph graph) {
1270
1271                                         for (final Resource predicate : specialSchedule) {
1272
1273                                                 graph.forIsSubrelationOf(predicate, graph.getService(Layer0.class).IsComposedOf, new AsyncProcedure<Boolean>() {
1274
1275                                                         @Override
1276                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {
1277                                                                 throwable.printStackTrace();
1278                                                         }
1279
1280                                                         @Override
1281                                                         public void execute(AsyncReadGraph graph, Boolean composed) {
1282                                                                 if (composed) composedResult.add(predicate);
1283                                                         }
1284
1285                                                 });
1286
1287                                         }
1288
1289                                 }
1290
1291                         });
1292
1293                         composedPredicates.addAll(composedResult);
1294
1295                         composedPredicateTime += (System.nanoTime() - start);
1296
1297                 }
1298
1299                 /*
1300                  * Composed objects are internal. Mark them for expansion.
1301                  */
1302
1303                 public void collectComposedObjects(ReadGraph graph, Collection<DirectStatements>[] expansion, Set<Resource> typeTodo, Set<Resource> objectTodo,
1304                                 Set<Resource> predicateTodo) throws DatabaseException {
1305
1306                         long start = System.nanoTime();
1307
1308                         Layer0 l0 = Layer0.getInstance(graph);
1309
1310                         for (Collection<DirectStatements> coll : expansion)
1311                                 for (DirectStatements stms : coll)
1312                                         for(Statement stm : stms) {
1313
1314                                                 Resource predicate = stm.getPredicate();
1315                                                 Resource object = stm.getObject();
1316
1317                                                 if (composedPredicates.contains(predicate)) {
1318
1319                                                         ExtentStatus existing = status.put(object, ExtentStatus.INTERNAL);
1320                                                         if(existing == null) {
1321                                                                 ids.put(object, id++);
1322                                                                 composedObjectCounter++;
1323                                                                 expansionSeeds.add(object);
1324 //                                                              System.err.println("internal: " + NameUtils.getSafeName(graph, object, true));
1325                                                                 if(LOG) log("[INTERNAL] (composed object) " + NameUtils.getSafeName(graph, object, true));
1326                                                         } else if (existing == ExtentStatus.EXCLUDED) {
1327                                                                 System.err.println("preExcluded: " + NameUtils.getSafeName(graph, object, true));
1328                                                                 status.put(object, ExtentStatus.EXCLUDED);
1329                                                         } else if (existing == ExtentStatus.EXTERNAL) {
1330                                                                 System.err.println("preExternal: " + NameUtils.getSafeName(graph, object, true));
1331                                                                 status.put(object, ExtentStatus.EXTERNAL);
1332                                                         }
1333
1334                                                 } else {
1335
1336 //                                                      System.err.println("internal2: " + NameUtils.getSafeName(graph, object, true));
1337
1338                                                         if (!status.containsKey(object)) {
1339                                                                 if (l0.InstanceOf.equalsResource(predicate)) {
1340                                                                         typeTodo.add(object);
1341                                                                 } else {
1342                                                                         objectTodo.add(object);
1343                                                                 }
1344                                                         }
1345
1346                                                         if (!status.containsKey(predicate)) {
1347                                                                 predicateTodo.add(predicate);
1348                                                         }
1349
1350                                                 }
1351                                         }
1352
1353                         composedObjectTime += System.nanoTime() - start;
1354
1355                 }
1356
1357                 public void writeOtherStatements(ReadGraph graph, Collection<Collection<DirectStatements>[]> expansion, ObjectOutputStream composedStatementsOutput, ObjectOutputStream otherStatementsOutput,
1358                                 ObjectOutputStream valueOutput) throws DatabaseException {
1359
1360                         long start = System.nanoTime();
1361
1362                         Layer0 l0 = Layer0.getInstance(graph);
1363                         SerialisationSupport support = graph.getService(SerialisationSupport.class);
1364                         TransferableGraphSupport tgs = graph.getService(TransferableGraphSupport.class);
1365
1366                         TIntArrayList other = new TIntArrayList();
1367                         TIntArrayList composed = new TIntArrayList();
1368
1369                         try {
1370
1371                                 for (Collection<DirectStatements>[] colls : expansion)
1372                                         for (Collection<DirectStatements> coll : colls)
1373                                                 for (DirectStatements stms : coll) {
1374
1375                                                         Resource subject = stms.getSubject();
1376                                                         composed.resetQuick();
1377
1378                                                         int sId = support.getTransientId(subject);
1379
1380                                                         composedStatementsOutput.writeInt(sId);
1381
1382                                                         if(graph.hasValue(subject)) {
1383                                                                 Datatype dt = graph.getRelatedValue(subject, l0.HasDataType, Bindings.getBindingUnchecked(Datatype.class));
1384                                                                 Binding b = Bindings.getBinding(dt);
1385                                                                 Object _value = graph.getValue(subject, b);
1386                                                                 Variant variant = new Variant(b, _value);
1387                                                                 byte[] value = variantSerializer.serialize(variant);
1388                                                                 if(LOG) log("[VALUE] " + NameUtils.getSafeName(graph, subject));
1389                                                                 valueOutput.writeInt(sId);
1390                                                                 valueOutput.writeInt(value.length);
1391                                                                 assert (value.length > 0);
1392                                                                 valueOutput.write(value);
1393                                                         }
1394
1395                                                         for (Statement s : stms) {
1396
1397                                                                 Resource object = s.getObject();
1398                                                                 Resource predicate = s.getPredicate();
1399
1400                                                                 ExtentStatus objectStatus = status.get(object);
1401
1402                                                                 if(objectStatus == ExtentStatus.INTERNAL) {
1403                                                                         composed.add(support.getTransientId(predicate));
1404                                                                         composed.add(support.getTransientId(object));
1405                                                                         if(LOG) log("[COMPOSED] (internal object) " + NameUtils.toIdString(graph, s));
1406                                                                 } else if (l0.InstanceOf.equalsResource(predicate)) {
1407                                                                         composed.add(support.getTransientId(predicate));
1408                                                                         composed.add(support.getTransientId(object));
1409                                                                         if(LOG) log("[COMPOSED] (instanceOf) " + NameUtils.toIdString(graph, s));
1410                                                                 } else if (l0.SubrelationOf.equalsResource(predicate)) {
1411                                                                         composed.add(support.getTransientId(predicate));
1412                                                                         composed.add(support.getTransientId(object));
1413                                                                         if(LOG) log("[COMPOSED] (subrelationOf) " + NameUtils.toIdString(graph, s));
1414                                                                 } else {
1415                                                                         if(objectStatus == ExtentStatus.EXTERNAL) {
1416                                                                                 if(DEBUG)
1417                                                                                         System.out.println("other " + NameUtils.toIdString(graph, s));
1418                                                                                         //System.out.println("other.add " + predicate + " - " + object);
1419                                                                                 other.add(support.getTransientId(predicate));
1420                                                                                 other.add(support.getTransientId(object));
1421                                                                                 if(LOG) log("[OTHER] (object is external) " + NameUtils.toIdString(graph, s));
1422                                                                         }
1423                                                                 }
1424
1425                                                         }
1426
1427                                                         if(!other.isEmpty()) {
1428                                                                 otherStatementsOutput.writeInt(sId);
1429                                                                 otherStatementsOutput.writeInt(other.size() / 2);
1430                                                                 for (int i = 0; i < other.size(); i++)
1431                                                                         otherStatementsOutput.writeInt(other.getQuick(i));
1432                                                                 other.resetQuick();
1433                                                         }
1434
1435                                                         composedStatementsOutput.writeInt(composed.size() / 2);
1436                                                         for (int i = 0; i < composed.size(); i++)
1437                                                                 composedStatementsOutput.writeInt(composed.getQuick(i));
1438
1439                                                 }
1440
1441                         } catch (IOException e) {
1442                                 e.printStackTrace();
1443                         }
1444
1445                         otherStatementTime += (System.nanoTime() - start);
1446
1447                 }
1448
1449                 boolean hasStrictParents(ReadGraph g, Resource r)
1450                                 throws DatabaseException {
1451                         if (g.getPossibleURI(r) != null)
1452                                 return true;
1453                         return false;
1454                 }
1455
1456                 public boolean getExpansionSeedsFromExtents(ReadGraph graph, final Collection<DirectStatements>[] expansion) throws DatabaseException {
1457
1458                         long start = System.nanoTime();
1459
1460                         final ConcurrentLinkedQueue<Resource> accepts = new ConcurrentLinkedQueue<Resource>();
1461
1462                         /*
1463                          * Determine statements which could accept statements with todo
1464                          * objects
1465                          */
1466                         search: for (Double priority : priorityList) {
1467
1468                                 for (final SubgraphAdvisor advisor : advisors) {
1469
1470                                         if (advisor.priority() > 0)
1471                                                 continue;
1472
1473                                         if (advisor.priority() == priority) {
1474
1475                                                 graph.syncRequest(new ReadRequest() {
1476
1477                                                         @Override
1478                                                         public void run(ReadGraph graph) throws DatabaseException {
1479
1480                                                                 for (Collection<DirectStatements> coll : expansion)
1481                                                                         for (DirectStatements stms : coll)
1482                                                                                 for(final Statement stm : stms) {
1483
1484                                                                                         advisor.advise(graph, stm, new AsyncProcedure<Boolean>() {
1485
1486                                                                                                 @Override
1487                                                                                                 public void exception(AsyncReadGraph graph, Throwable throwable) {
1488                                                                                                         throwable.printStackTrace();
1489                                                                                                 }
1490
1491                                                                                                 @Override
1492                                                                                                 public void execute(AsyncReadGraph graph, Boolean accept) {
1493                                                                                                         if (accept) {
1494                                                                                                                 accepts.add(stm.getObject());
1495                                                                                                         }
1496                                                                                                 }
1497
1498                                                                                         });
1499
1500                                                                                 }
1501
1502                                                         }
1503
1504                                                 });
1505                                         }
1506                                         if (!accepts.isEmpty())
1507                                                 break search;
1508                                 }
1509                         }
1510
1511                         CollectionSupport cs = graph.getService(CollectionSupport.class);
1512                         Set<Resource> schedule = cs.createSet();
1513                         for (Resource r : accepts) {
1514                                 if(!status.containsKey(r))
1515                                         schedule.add(r);
1516                         }
1517
1518                         extentSeedTime += (System.nanoTime() - start);
1519
1520                         if (schedule.isEmpty())
1521                                 return false;
1522
1523                         fastResolve(graph, schedule);
1524                         uriResolve(graph, schedule);
1525                         fullResolve(graph, schedule, "accepts");
1526
1527                         return true;
1528
1529                 }
1530
1531                 ConcurrentLinkedQueue<Resource> fastInternals = new ConcurrentLinkedQueue<Resource>();
1532
1533                 public void fastResolve(ReadGraph graph, final Set<Resource> rs)
1534                                 throws DatabaseException {
1535                         // This collects and resolves weaks
1536                         if(fastResolveLoop(graph, rs))
1537                                 // Weaks are now resolved
1538                                 fastResolveLoop(graph, rs);
1539                 }
1540
1541                 public boolean fastResolveLoop(ReadGraph graph, final Set<Resource> rs)
1542                                 throws DatabaseException {
1543
1544                         long start = System.nanoTime();
1545
1546                         final ConcurrentLinkedQueue<Resource> weakSchedule = new ConcurrentLinkedQueue<Resource>();
1547
1548                         graph.syncRequest(new AsyncRead<Boolean>() {
1549
1550                             @Override
1551                             public int threadHash() {
1552                                 return hashCode();
1553                             }
1554
1555                                 @Override
1556                                 public int getFlags() {
1557                                         return 0;
1558                                 }
1559
1560                                 @Override
1561                                 public void perform(AsyncReadGraph graph,
1562                                                 AsyncProcedure<Boolean> procedure) {
1563
1564                                         QueryControl control = graph.getService(QueryControl.class);
1565                                         final DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class);
1566
1567                                         int slice = (int) (rs.size() / control
1568                                                         .getAmountOfQueryThreads()) + 1;
1569
1570                                         final Resource[] rootArray = rs.toArray(Resource.NONE);
1571                                         for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
1572
1573                                                 final int start = i * slice;
1574                                                 final int end = Math.min(start + slice,
1575                                                                 rootArray.length);
1576
1577                                                 control.schedule(graph, i, new ControlProcedure() {
1578
1579                                                         @Override
1580                                                         public void execute(AsyncReadGraph graph) {
1581
1582                                                                 for (int index = start; index < end; index++) {
1583
1584                                                                         final Resource r = rootArray[index];
1585
1586                                                                         //if (status.containsKey(r)) continue;
1587
1588                                                                         graph.asyncRequest(new FastInternalRequest(dqs, r, status, weakInverses, weakSchedule),new AsyncProcedure<Boolean>() {
1589
1590                                                                                 @Override
1591                                                                                 public void exception(AsyncReadGraph graph,Throwable throwable) {
1592                                                                                         throwable.printStackTrace();
1593                                                                                 }
1594
1595                                                                                 @Override
1596                                                                                 public void execute(AsyncReadGraph graph,Boolean isInternal) {
1597                                                                                         if (isInternal) {
1598                                                                                                 fastInternals.add(r);
1599                                                                                         }
1600                                                                                 }
1601
1602                                                                         });
1603
1604                                                                 }
1605
1606                                                         }
1607
1608                                                 });
1609
1610                                         }
1611
1612                                         procedure.execute(graph, true);
1613
1614                                 }
1615
1616                         });
1617
1618                         if (!weakSchedule.isEmpty()) {
1619                                 THashSet<Resource> weaks = new THashSet<Resource>(weakSchedule);
1620                                 if (CLASSIFY_LOG)
1621                                         for (Resource p : weakSchedule)
1622                                                 System.out.println("classify "
1623                                                                 + NameUtils.getSafeName(graph, p));
1624                                 graph.syncRequest(new ClassifyStatementsRequest(weaks, weakInverses));
1625                         }
1626
1627                         for (Resource r : fastInternals) {
1628                                 rs.remove(r);
1629                                 if (status.put(r, ExtentStatus.INTERNAL) == null) {
1630                                         if(LOG) log("[INTERNAL] (fast) " + NameUtils.getSafeName(graph, r, true));
1631                                         ids.put(r, id++);
1632                                         fastInternalCounter++;
1633                                         expansionSeeds.add(r);
1634                                 }
1635                         }
1636
1637                         fastResolveTime += (System.nanoTime() - start);
1638
1639                         return !weakSchedule.isEmpty();
1640
1641                 }
1642
1643                 private ExtentStatus resolveExtent(ReadGraph graph, Resource r, Map<Resource, ExtentStatus> statuses, Set<Resource> expansionSeeds, THashSet<Resource> pending,
1644                                 ArrayList<Resource> stack) throws DatabaseException {
1645
1646                         ExtentStatus current = statuses.get(r);
1647                         if(current != null) return current;
1648
1649                         if (pending.contains(r))
1650                                 return ExtentStatus.PENDING;
1651
1652                         // In order to break cyclic dependencies
1653                         pending.add(r);
1654
1655                         if (PARENT_DEBUG)
1656                                 System.out.println("resolveExtent "
1657                                                 + NameUtils.getSafeName(graph, r));
1658
1659                         ExtentStatus status = ExtentStatus.INTERNAL;
1660                         for (Resource p : getParents(graph, r)) {
1661                                 if (PARENT_DEBUG) {
1662                                         ExtentStatus ps = statuses.get(p);
1663                                         System.out.println(" parent " + NameUtils.getSafeName(graph, p) + "(" + ps + ")");
1664                                 }
1665                                 switch (resolveExtent(graph, p, statuses,
1666                                                 expansionSeeds, pending, stack)) {
1667                                 case EXTERNAL:
1668                                         return ExtentStatus.EXTERNAL;
1669                                 case PENDING:
1670                                         status = ExtentStatus.PENDING;
1671                                 }
1672                         }
1673                         if (status == ExtentStatus.INTERNAL) {
1674                                 pending.remove(r);
1675                                 stack.add(r);
1676                                 if (DEBUG)
1677                                         System.out.println(NameUtils.getSafeName(graph, r, true)
1678                                                         + " is internal.");
1679                         }
1680                         return status;
1681                 }
1682
1683                 public void uriResolve(ReadGraph graph, final Set<Resource> todo)
1684                                 throws DatabaseException {
1685
1686                         long start = System.nanoTime();
1687
1688                         for(Resource r : todo) System.out.println("uriResolve " +
1689                                         NameUtils.getSafeName(graph, r));
1690
1691                         final ConcurrentSkipListSet<Resource> found = new ConcurrentSkipListSet<Resource>();
1692
1693                         graph.syncRequest(new AsyncReadRequest() {
1694
1695                                 @Override
1696                                 public void run(AsyncReadGraph graph) {
1697
1698                                         for (final Resource r : todo) {
1699
1700                                                 // System.out.println("uriresolve before " + r);
1701
1702                                                 if (status.containsKey(r)) continue;
1703
1704                                                 // System.out.println("uriresolve " + r);
1705
1706                                                 graph.forURI(r, new AsyncProcedure<String>() {
1707
1708                                                         @Override
1709                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {
1710                                                                 throwable.printStackTrace();
1711                                                         }
1712
1713                                                         @Override
1714                                                         public void execute(AsyncReadGraph graph, String uri) {
1715
1716                                                                 if (uri != null) {
1717
1718                                                                         // System.out.println("uriresolve has uri "
1719                                                                         // + r);
1720
1721                                                                         if(found.add(r)) {
1722                                                                                 parentExternalCounter++;
1723                                                                         }
1724
1725                                                                 } else {
1726
1727                                                                         // System.out.println("uriresolve ask inverse "
1728                                                                         // + r);
1729
1730                                                                         graph.forPossibleInverse(r, new AsyncProcedure<Resource>() {
1731
1732                                                                                 @Override
1733                                                                                 public void exception(AsyncReadGraph graph, Throwable throwable) {
1734                                                                                         throwable.printStackTrace();
1735                                                                                 }
1736
1737                                                                                 @Override
1738                                                                                 public void execute(AsyncReadGraph graph, Resource inverse) {
1739
1740                                                                                         if (inverse != null) {
1741
1742                                                                                                 graph.forURI(inverse, new AsyncProcedure<String>() {
1743
1744                                                                                                         @Override
1745                                                                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {
1746                                                                                                                 throwable.printStackTrace();
1747                                                                                                         }
1748
1749                                                                                                         @Override
1750                                                                                                         public void execute(AsyncReadGraph graph, String uri) {
1751
1752                                                                                                                 if (uri != null) {
1753
1754                                                                                                                         if(found.add(r)) {
1755                                                                                                                                 parentExternalCounter++;
1756                                                                                                                         }
1757
1758                                                                                                                 }
1759
1760                                                                                                         }
1761
1762                                                                                                 });
1763
1764                                                                                         }
1765
1766                                                                                 }
1767
1768                                                                         });
1769
1770                                                                 }
1771                                                         }
1772
1773                                                 });
1774
1775                                         }
1776
1777                                 }
1778
1779                         });
1780
1781                         todo.removeAll(found);
1782                         for(Resource r : found) {
1783                                 status.put(r, ExtentStatus.EXTERNAL);
1784                                 if(LOG) log("[EXTERNAL] (uriResolve) " + NameUtils.getSafeName(graph, r, true));
1785                         }
1786
1787                         parentResolveTime += System.nanoTime() - start;
1788
1789                 }
1790
1791                 public void fullResolve(ReadGraph graph, Collection<Resource> rs,
1792                                 String koss) throws DatabaseException {
1793
1794                         long start = System.nanoTime();
1795
1796                         for (final Resource r : rs) {
1797
1798                                 if(status.containsKey(r)) continue;
1799
1800                                 THashSet<Resource> pending = new THashSet<Resource>();
1801                                 ArrayList<Resource> stack = new ArrayList<Resource>();
1802
1803                                 ExtentStatus s = resolveExtent(graph, r, status, expansionSeeds, pending, stack);
1804                                 if (ExtentStatus.INTERNAL == s || ExtentStatus.PENDING == s) {
1805                                         if (status.put(r, ExtentStatus.INTERNAL) == null) {
1806                                                 if(LOG) log("[INTERNAL] (resolveExtent) " + NameUtils.getSafeName(graph, r, true));
1807                                                 ids.put(r, id++);
1808                                                 fullInternalCounter++;
1809                                                 expansionSeeds.add(r);
1810                                         }
1811                                 }
1812                                 if (ExtentStatus.EXTERNAL == s) {
1813                                         if (status.put(r, ExtentStatus.EXTERNAL) == null) {
1814                                                 if(LOG) log("[EXTERNAL] (resolveExtent) " + NameUtils.getSafeName(graph, r, true));
1815                                                 fullExternalCounter++;
1816                                         }
1817                                 }
1818
1819                         }
1820
1821                         fullResolveTime += (System.nanoTime() - start);
1822
1823                 }
1824
1825                 public void process(ReadGraph graph,
1826                                 ObjectOutputStream composedStatementsOutput,
1827                                 ObjectOutputStream otherStatementsOutput,
1828                                 ObjectOutputStream valueOutput)
1829                                 throws DatabaseException {
1830
1831                         this.variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT);
1832
1833                         CollectionSupport cs = graph.getService(CollectionSupport.class);
1834
1835                         Set<Resource> typeTodo = cs.createSet();
1836                         Set<Resource> objectTodo = cs.createSet();
1837                         Set<Resource> predicateTodo = cs.createSet();
1838
1839                         Collection<Collection<DirectStatements>[]> fullExpansion = new ArrayList<Collection<DirectStatements>[]>();
1840
1841                         do {
1842
1843                                 QueryControl control = graph.getService(QueryControl.class);
1844                                 Collection<DirectStatements>[] expansion = new ArrayList[control.getAmountOfQueryThreads()];
1845                                 for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
1846                                         expansion[i] = new ArrayList<DirectStatements>();
1847                                 }
1848
1849                                 // Expand expansionSeeds
1850                                 expand(graph, expansion, objectTodo);
1851
1852                                 // Start collecting new seeds
1853                                 expansionSeeds = cs.createSet();
1854
1855                                 // Collect predicates which are <R IsComposedOf
1856                                 extractComposedPredicates(graph, expansion);
1857
1858                                 // Make composed objects internal and make sure they are further
1859                                 // expanded
1860                                 collectComposedObjects(graph, expansion, typeTodo, objectTodo, predicateTodo);
1861
1862                                 /*
1863                                  * Use the expansion seed heuristic to find new resources to
1864                                  * expand before full analysis.
1865                                  */
1866                                 getExpansionSeedsFromExtents(graph, expansion);
1867
1868                                 fullExpansion.add(expansion);
1869
1870                         } while (!expansionSeeds.isEmpty());
1871
1872                         fastResolve(graph, objectTodo);
1873                         uriResolve(graph, predicateTodo);
1874                         uriResolve(graph, objectTodo);
1875                         uriResolve(graph, typeTodo);
1876                         fullResolve(graph, objectTodo, "objectTodo");
1877                         fullResolve(graph, predicateTodo, "predicateTodo");
1878                         fullResolve(graph, typeTodo, "typeTodo");
1879
1880                         writeOtherStatements(graph, fullExpansion, composedStatementsOutput, otherStatementsOutput, valueOutput);
1881
1882                         if (PROFILE) {
1883                                 System.out.println(composedObjectCounter + " " + fastInternalCounter
1884                                                 + " " + parentExternalCounter + " "
1885                                                 + fullExternalCounter + " " + fullInternalCounter);
1886                         }
1887
1888                 }
1889
1890         }
1891
1892         public static void getDomain2(ReadGraph graph, TIntIntHashMap ids,
1893                         Collection<Resource> roots, Map<Resource, ExtentStatus> preStatus,
1894                         Map<Resource, Statement> specials,
1895                         ObjectOutputStream otherStatementsOutput,
1896                         ObjectOutputStream valueOutput,
1897                         TreeMap<String, Variant> extensions,
1898                         TIntHashSet excludedShared) throws DatabaseException {
1899
1900                 ITask task = ThreadLogger.getInstance().begin("getDomain2");
1901
1902                 final DomainProcessor2 processor = new DomainProcessor2();
1903
1904                 processor.startupTime = System.nanoTime();
1905
1906                 Layer0 l0 = Layer0.getInstance(graph);
1907
1908                 CollectionSupport cs = graph.getService(CollectionSupport.class);
1909                 SerialisationSupport support = graph.getService(SerialisationSupport.class);
1910
1911                 processor.ids = ids;
1912                 processor.specials = specials;
1913                 processor.status = cs.createMap(ExtentStatus.class);
1914                 processor.weakInverses = cs.createMap(WeakStatus.class);
1915                 processor.predicates = cs.createSet();
1916                 processor.isRelatedToPredicates = cs.createSet();
1917                 processor.sharedPredicates = cs.createSet();
1918 //              processor.expansionSeeds = cs.createSet();
1919
1920                 for(Map.Entry<Resource, ExtentStatus> entry : preStatus.entrySet()) {
1921                         processor.status.put(entry.getKey(), entry.getValue());
1922                         if(ExtentStatus.EXCLUDED.equals(entry.getValue())) processor.exclusions.add(entry.getKey());
1923                 }
1924
1925 //              for (Resource r : excluded) {
1926 //                      processor.status.put(r, ExtentStatus.EXCLUDED);
1927 //              }
1928
1929                 Resource rootLibrary = graph.getResource("http:/");
1930
1931                 if (!roots.contains(rootLibrary))
1932                         processor.status.put(rootLibrary, ExtentStatus.EXTERNAL);
1933
1934                 for (Resource root : roots) {
1935                         processor.status.put(root, ExtentStatus.INTERNAL);
1936                         //processor.ids.put(support.getTransientId(root), processor.ids.size());
1937                         for (Resource owner : graph.getObjects(root, l0.IsOwnedBy)) {
1938                                 processor.status.put(owner, ExtentStatus.EXTERNAL);
1939                         }
1940                 }
1941
1942                 processor.startupTime = System.nanoTime() - processor.startupTime;
1943
1944                 processor.fringe = new HashSet<Resource>();
1945                 processor.fringe.addAll(roots);
1946
1947                 processor.internalDomain.addAll(roots);
1948
1949                 processor.sharedExternalReferences = new HashSet<Resource>();
1950                 processor.sharedExternalFringe = new HashSet<Resource>();
1951
1952                 try {
1953
1954                         processor.process(graph, otherStatementsOutput, valueOutput);
1955
1956                 } catch (IOException e) {
1957                         e.printStackTrace();
1958                 }
1959
1960                 for(Resource r : processor.sharedExternalReferences) excludedShared.add(support.getTransientId(r));
1961
1962         ClusteringSupport cls = graph.getService(ClusteringSupport.class);
1963         TLongObjectHashMap<TIntArrayList> clusterMap = new TLongObjectHashMap<TIntArrayList>();
1964                 for(Map.Entry<Resource, ExtentStatus> entry : processor.status.entrySet()) {
1965                         if(ExtentStatus.INTERNAL == entry.getValue()) {
1966
1967                                 long cluster = cls.getCluster(entry.getKey());
1968                                 TIntArrayList list = clusterMap.get(cluster);
1969                                 if(list == null) {
1970                                         list = new TIntArrayList();
1971                                         clusterMap.put(cluster, list);
1972                                 }
1973                                 list.add(support.getTransientId(entry.getKey()));
1974
1975                         }
1976                 }
1977                 final TIntArrayList clustering = new TIntArrayList();
1978                 clusterMap.forEachEntry(new TLongObjectProcedure<TIntArrayList>() {
1979
1980                         @Override
1981                         public boolean execute(long cluster, TIntArrayList b) {
1982                                 clustering.add(b.size());
1983                                 b.forEach(new TIntProcedure() {
1984
1985                                         @Override
1986                                         public boolean execute(int rId) {
1987                                                 processor.ids.put(rId, processor.id++);
1988                                                 return true;
1989                                         }
1990
1991                                 });
1992                                 return true;
1993                         }
1994
1995                 });
1996
1997                 extensions.put(Extensions.CLUSTERING, new Variant(Bindings.INT_ARRAY, clustering.toArray()));
1998
1999                 long total = processor.startupTime + processor.expandTime
2000                                 + processor.composedPredicateTime
2001                                 + processor.composedObjectTime + processor.extentSeedTime
2002                                 + processor.fullResolveTime + processor.fastResolveTime +
2003                                 + processor.parentResolveTime + processor.otherStatementTime;
2004
2005                 if (PROFILE) {
2006                         System.out.println("startup took " + 1e-9 * processor.startupTime
2007                                         + "s.");
2008                         System.out.println("expand took " + 1e-9 * processor.expandTime
2009                                         + "s.");
2010                         System.out.println("composedPredicates took " + 1e-9
2011                                         * processor.composedPredicateTime + "s.");
2012                         System.out.println("composedObjects took " + 1e-9
2013                                         * processor.composedObjectTime + "s.");
2014                         System.out.println("extentSeeding took " + 1e-9
2015                                         * processor.extentSeedTime + "s.");
2016                         System.out.println("fullResolve took " + 1e-9
2017                                         * processor.fullResolveTime + "s.");
2018                         System.out.println("fastResolve took " + 1e-9
2019                                         * processor.fastResolveTime + "s.");
2020                         System.out.println("parentResolve took " + 1e-9
2021                                         * processor.parentResolveTime + "s.");
2022                         System.out.println("otherStatements took " + 1e-9
2023                                         * processor.otherStatementTime + "s.");
2024                         System.out.println("total " + 1e-9 * total + "s.");
2025                 }
2026
2027                 task.finish();
2028
2029
2030         }
2031
2032
2033         public static void getDomain(ReadGraph graph, Map<Resource, Integer> ids,
2034                         Collection<Resource> roots, Map<Resource, ExtentStatus> preStatus, Set<SubgraphAdvisor> advisors,
2035                         ObjectOutputStream composedStatementsOutput,
2036                         ObjectOutputStream otherStatementsOutput,
2037                         ObjectOutputStream valueOutput) throws DatabaseException {
2038
2039                 ITask task = ThreadLogger.getInstance().begin("getDomain");
2040
2041                 DomainProcessor processor = new DomainProcessor(advisors);
2042
2043                 processor.startupTime = System.nanoTime();
2044
2045                 Layer0 l0 = Layer0.getInstance(graph);
2046
2047                 CollectionSupport cs = graph.getService(CollectionSupport.class);
2048
2049                 processor.ids = ids;
2050                 processor.status = cs.createMap(ExtentStatus.class);
2051                 processor.weakInverses = cs.createMap(WeakStatus.class);
2052                 processor.predicates = cs.createSet();
2053                 processor.composedPredicates = cs.createSet();
2054                 processor.expansionSeeds = cs.createSet();
2055
2056                 for(Map.Entry<Resource, ExtentStatus> entry : preStatus.entrySet()) {
2057                         processor.status.put(entry.getKey(), entry.getValue());
2058                 }
2059
2060 //              for (Resource r : excluded) {
2061 //                      processor.status.put(r, ExtentStatus.EXCLUDED);
2062 //              }
2063
2064                 if (!roots.contains(graph.getRootLibrary()))
2065                         processor.status.put(graph.getRootLibrary(), ExtentStatus.EXTERNAL);
2066
2067                 for (Resource root : roots) {
2068                         processor.status.put(root, ExtentStatus.INTERNAL);
2069                         processor.ids.put(root, processor.id++);
2070                         for (Resource owner : graph.getObjects(root, l0.IsOwnedBy)) {
2071                                 processor.status.put(owner, ExtentStatus.EXTERNAL);
2072                         }
2073                 }
2074
2075
2076                 processor.expansionSeeds.addAll(roots);
2077
2078                 processor.startupTime = System.nanoTime() - processor.startupTime;
2079
2080                 while (!processor.expansionSeeds.isEmpty()) {
2081
2082                         processor.process(graph, composedStatementsOutput,
2083                                         otherStatementsOutput, valueOutput);
2084
2085                 }
2086
2087                 long total = processor.startupTime + processor.expandTime
2088                                 + processor.composedPredicateTime
2089                                 + processor.composedObjectTime + processor.extentSeedTime
2090                                 + processor.fullResolveTime + processor.fastResolveTime +
2091                                 + processor.parentResolveTime + processor.otherStatementTime;
2092
2093                 if (PROFILE) {
2094                         System.out.println("startup took " + 1e-9 * processor.startupTime
2095                                         + "s.");
2096                         System.out.println("expand took " + 1e-9 * processor.expandTime
2097                                         + "s.");
2098                         System.out.println("composedPredicates took " + 1e-9
2099                                         * processor.composedPredicateTime + "s.");
2100                         System.out.println("composedObjects took " + 1e-9
2101                                         * processor.composedObjectTime + "s.");
2102                         System.out.println("extentSeeding took " + 1e-9
2103                                         * processor.extentSeedTime + "s.");
2104                         System.out.println("fullResolve took " + 1e-9
2105                                         * processor.fullResolveTime + "s.");
2106                         System.out.println("fastResolve took " + 1e-9
2107                                         * processor.fastResolveTime + "s.");
2108                         System.out.println("parentResolve took " + 1e-9
2109                                         * processor.parentResolveTime + "s.");
2110                         System.out.println("otherStatements took " + 1e-9
2111                                         * processor.otherStatementTime + "s.");
2112                         System.out.println("total " + 1e-9 * total + "s.");
2113                 }
2114
2115                 task.finish();
2116
2117         }
2118
2119 }