bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/Subgraphs.java

   1 /*******************************************************************************
   2  * Copyright (c) 2007, 2010 Association for Decentralized Information Management
   3  * in Industry THTH ry.
   4  * All rights reserved. This program and the accompanying materials
   5  * are made available under the terms of the Eclipse Public License v1.0
   6  * which accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *     VTT Technical Research Centre of Finland - initial API and implementation
  11  *******************************************************************************/
  12 package org.simantics.db.layer0.util;
  13
  14 import java.io.DataOutput;
  15 import java.io.DataOutputStream;
  16 import java.io.FileNotFoundException;
  17 import java.io.FileOutputStream;
  18 import java.io.IOException;
  19 import java.io.ObjectOutputStream;
  20 import java.util.ArrayList;
  21 import java.util.Collection;
  22 import java.util.Collections;
  23 import java.util.HashSet;
  24 import java.util.List;
  25 import java.util.Map;
  26 import java.util.Set;
  27 import java.util.TreeMap;
  28 import java.util.concurrent.ConcurrentLinkedQueue;
  29 import java.util.concurrent.ConcurrentSkipListSet;
  30
  31 import org.simantics.databoard.Bindings;
  32 import org.simantics.databoard.Databoard;
  33 import org.simantics.databoard.binding.Binding;
  34 import org.simantics.databoard.binding.mutable.Variant;
  35 import org.simantics.databoard.serialization.Serializer;
  36 import org.simantics.databoard.type.Datatype;
  37 import org.simantics.db.AsyncReadGraph;
  38 import org.simantics.db.DirectStatements;
  39 import org.simantics.db.ReadGraph;
  40 import org.simantics.db.Resource;
  41 import org.simantics.db.Statement;
  42 import org.simantics.db.common.request.AsyncReadRequest;
  43 import org.simantics.db.common.request.ReadRequest;
  44 import org.simantics.db.common.request.ResourceAsyncRead;
  45 import org.simantics.db.common.utils.NameUtils;
  46 import org.simantics.db.exception.DatabaseException;
  47 import org.simantics.db.layer0.adapter.SubgraphAdvisor;
  48 import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus;
  49 import org.simantics.db.layer0.util.ConsistsOfProcess.InternalEntry;
  50 import org.simantics.db.procedure.AsyncProcedure;
  51 import org.simantics.db.request.AsyncRead;
  52 import org.simantics.db.service.ClusteringSupport;
  53 import org.simantics.db.service.CollectionSupport;
  54 import org.simantics.db.service.DirectQuerySupport;
  55 import org.simantics.db.service.QueryControl;
  56 import org.simantics.db.service.QueryControl.ControlProcedure;
  57 import org.simantics.db.service.SerialisationSupport;
  58 import org.simantics.db.service.StatementSupport;
  59 import org.simantics.db.service.TransferableGraphSupport;
  60 import org.simantics.graph.representation.Extensions;
  61 import org.simantics.layer0.Layer0;
  62 import org.simantics.utils.datastructures.Pair;
  63 import org.simantics.utils.threads.logger.ITask;
  64 import org.simantics.utils.threads.logger.ThreadLogger;
  65
  66 import gnu.trove.list.array.TIntArrayList;
  67 import gnu.trove.map.hash.TIntIntHashMap;
  68 import gnu.trove.map.hash.TLongObjectHashMap;
  69 import gnu.trove.procedure.TIntProcedure;
  70 import gnu.trove.procedure.TLongObjectProcedure;
  71 import gnu.trove.set.TIntSet;
  72 import gnu.trove.set.hash.THashSet;
  73 import gnu.trove.set.hash.TIntHashSet;
  74
  75 public class Subgraphs {
  76
  77         public static String LOG_FILE = "export.log";
  78         final static private boolean LOG = false;
  79         final static private boolean DEBUG = false;
  80         final static private boolean PARENT_DEBUG = DEBUG | false;
  81         final static private boolean EXTERNAL_DEBUG = DEBUG | false;
  82         final static private boolean ADVISOR_LOG = LOG & false;
  83         final static private boolean EXPANSION_LOG = LOG & false;
  84         final static private boolean INTERNAL_LOG = LOG & false;
  85         final static private boolean COMPOSED_LOG = LOG & false;
  86         final static private boolean RESOLVE_LOG = LOG & false;
  87         final static private boolean CLASSIFY_LOG = LOG & false;
  88         final static private boolean EXTERNAL_LOG = LOG & false;
  89         final static private boolean PROFILE = false;
  90
  91         static enum WeakStatus {
  92                 STRONG, WEAK
  93         }
  94
  95
  96         static DataOutput log;
  97
  98         static {
  99
 100                 if (LOG) {
 101                         try {
 102                                 FileOutputStream stream = new FileOutputStream(LOG_FILE, false);
 103                                 log = new DataOutputStream(stream);
 104                         } catch (FileNotFoundException e) {
 105                                 e.printStackTrace();
 106                         }
 107                 }
 108
 109         }
 110
 111         private static void log(String line) {
 112                 if (LOG) {
 113                         try {
 114                                 log.write((line + "\n").getBytes());
 115                         } catch (IOException e) {
 116                                 e.printStackTrace();
 117                         }
 118                 }
 119         }
 120
 121         public static Collection<Resource> getParents(ReadGraph g, Resource r)
 122                         throws DatabaseException {
 123                 return getParents(g, r, false);
 124         }
 125
 126         static class FastInternalRequest extends ResourceAsyncRead<Boolean> {
 127
 128                 final DirectQuerySupport dqs;
 129                 final ConcurrentLinkedQueue<Resource> queue;
 130                 final Map<Resource, WeakStatus> weakInverses;
 131                 final Map<Resource, ExtentStatus> status;
 132
 133                 public FastInternalRequest(DirectQuerySupport dqs, Resource resource,
 134                                 Map<Resource, ExtentStatus> status,
 135                                 Map<Resource, WeakStatus> weakInverses,
 136                                 ConcurrentLinkedQueue<Resource> queue) {
 137                         super(resource);
 138                         this.dqs = dqs;
 139                         this.status = status;
 140                         this.weakInverses = weakInverses;
 141                         this.queue = queue;
 142                 }
 143
 144                 @Override
 145                 public int getFlags() {
 146                         return 0;
 147                 }
 148
 149                 @Override
 150                 public void perform(AsyncReadGraph graph, final AsyncProcedure<Boolean> procedure) {
 151
 152                         dqs.forEachDirectStatement(graph, resource, new AsyncProcedure<DirectStatements>() {
 153
 154                                 @Override
 155                                 public void execute(AsyncReadGraph graph, DirectStatements ss) {
 156                                         boolean ok = true;
 157                                         for(Statement statement : ss) {
 158                                                 if (status.get(statement.getObject()) == ExtentStatus.INTERNAL) continue;
 159                                                 WeakStatus status = weakInverses.get(statement.getPredicate());
 160                                                 if(status == WeakStatus.WEAK) continue;
 161                                                 else if (status == null) {
 162                                                         queue.add(statement.getPredicate());
 163                                                 }
 164                                                 ok = false;
 165                                         }
 166                                         procedure.execute(graph, ok);
 167                                 }
 168
 169                                 @Override
 170                                 public void exception(AsyncReadGraph graph, Throwable throwable) {
 171                                         throwable.printStackTrace();
 172                                 }
 173
 174                         });
 175
 176                 }
 177
 178         }
 179
 180         static class ClassifyStatementsRequest implements AsyncRead<Boolean> {
 181
 182                 final Set<Resource> schedule;
 183                 final Map<Resource, WeakStatus> weakMap;
 184
 185                 public ClassifyStatementsRequest(Set<Resource> schedule, Map<Resource, WeakStatus> weakMap) {
 186                         this.weakMap = weakMap;
 187                         this.schedule = schedule;
 188                 }
 189
 190             @Override
 191             public int threadHash() {
 192                 return hashCode();
 193             }
 194
 195                 @Override
 196                 public int getFlags() {
 197                         return 0;
 198                 }
 199
 200                 @Override
 201                 public void perform(AsyncReadGraph graph, final AsyncProcedure<Boolean> procedure) {
 202
 203                         for (final Resource p : schedule) {
 204
 205                                 graph.forPossibleInverse(p, new AsyncProcedure<Resource>() {
 206
 207                                         private void register(AsyncReadGraph graph, Resource predicate, Resource superRelation, WeakStatus status) {
 208                                                 synchronized (weakMap) {
 209                                                         weakMap.put(predicate, status);
 210                                                         if(superRelation != null) weakMap.put(superRelation, status);
 211                                                 }
 212                                         }
 213
 214                                         @Override
 215                                         public void exception(AsyncReadGraph graph, Throwable throwable) {
 216                                                 throwable.printStackTrace();
 217                                         }
 218
 219                                         @Override
 220                                         public void execute(AsyncReadGraph graph, final Resource inverse) {
 221
 222                                                 if (inverse == null) {
 223
 224                                                         register(graph, p, null, WeakStatus.WEAK);
 225
 226                                                 } else {
 227
 228                                                         graph.forPossibleSuperrelation(inverse, new AsyncProcedure<Resource>() {
 229
 230                                                                 @Override
 231                                                                 public void exception(AsyncReadGraph graph, Throwable throwable) {
 232                                                                         throwable.printStackTrace();
 233                                                                 }
 234
 235                                                                 @Override
 236                                                                 public void execute(AsyncReadGraph graph, final Resource superRelation) {
 237
 238                                                                         if(superRelation != null && weakMap.containsKey(superRelation)) {
 239                                                                                 register(graph, p, null, weakMap.get(superRelation));
 240                                                                                 return;
 241                                                                         }
 242
 243                                                                         graph.forIsSubrelationOf(inverse, graph.getService(Layer0.class).IsRelatedTo, new AsyncProcedure<Boolean>() {
 244
 245                                                                                 @Override
 246                                                                                 public void exception(AsyncReadGraph graph,Throwable throwable) {
 247                                                                                         throwable.printStackTrace();
 248                                                                                 }
 249
 250                                                                                 @Override
 251                                                                                 public void execute(AsyncReadGraph graph,Boolean strong) {
 252                                                                                         register(graph, p, superRelation, strong ? WeakStatus.STRONG : WeakStatus.WEAK);
 253                                                                                 }
 254
 255                                                                         });
 256
 257                                                                 }
 258
 259                                                         });
 260
 261                                                 }
 262
 263                                         }
 264
 265                                 });
 266
 267                         }
 268
 269                         procedure.execute(graph, false);
 270
 271                 }
 272
 273         }
 274
 275         private static Collection<Resource> getParents(ReadGraph g, Resource r, boolean isStrong) throws DatabaseException {
 276
 277                 System.out.println("getParents " + NameUtils.getSafeName(g, r));
 278
 279                 Layer0 l0 = Layer0.getInstance(g);
 280
 281                 Collection<Resource> predicates = g.getPredicates(r);
 282
 283                 // --- Consists Of ----------------------------------------------------
 284
 285                 if (predicates.contains(l0.PartOf)) {
 286                         Collection<Resource> parents = g.getObjects(r, l0.PartOf);
 287                         if (parents.size() == 1)
 288                                 return parents;
 289                         ArrayList<Resource> libraryParents = new ArrayList<Resource>(1);
 290                         for (Resource p : parents)
 291                                 if (g.isInstanceOf(p, l0.Library))
 292                                         libraryParents.add(p);
 293                         if (!libraryParents.isEmpty())
 294                                 return libraryParents;
 295                         else
 296                                 return parents;
 297                 }
 298
 299                 // --- Ordered sets ---------------------------------------------------
 300
 301                 {
 302                         Collection<Resource> parents = null;
 303                         for (Resource p : predicates)
 304                                 if (g.isInstanceOf(p, l0.OrderedSet) && !p.equals(r)) {
 305                                         if (parents == null)
 306                                                 parents = new ArrayList<Resource>(1);
 307                                         parents.add(p);
 308                                 }
 309                         if (parents != null) {
 310                                 if (DEBUG)
 311                                         System.out.println("ORDERED SET");
 312                                 return parents;
 313                         }
 314                 }
 315
 316
 317                 if (isStrong)
 318                         return Collections.emptyList();
 319                 else {
 320
 321                         if (predicates.contains(l0.InverseOf)) {
 322
 323                                 Resource inv = g.getInverse(r);
 324                                 return getParents(g, inv, true);
 325
 326                         } else {
 327
 328                                 /*
 329                                  * Depends On
 330                                  *
 331                                  * If there are DependsOn parents, then IsRelatedTo parents are discarded
 332                                  *
 333                                  */
 334                                 HashSet<Resource> result = new HashSet<Resource>();
 335                                 for(Resource predicate : predicates) {
 336                                         if(g.isSubrelationOf(predicate, l0.IsDependencyOf)) result.addAll(g.getObjects(r, predicate));
 337                                 }
 338                                 if(!result.isEmpty()) return result;
 339
 340                                 /*
 341                                  * Is Related To
 342                                  *
 343                                  * At this point all Is Related To are parents.
 344                                  *
 345                                  */
 346                                 for(Resource predicate : predicates) {
 347                                         Resource inv = g.getPossibleInverse(predicate);
 348                                         if(inv != null) {
 349                                                 if(g.isSubrelationOf(inv, l0.IsRelatedTo)) result.addAll(g.getObjects(r, predicate));
 350                                         }
 351                                 }
 352
 353                                 return result;
 354
 355                         }
 356
 357                         /*
 358                         Collection<Resource> invR = g.getObjects(r, b.IsRelatedTo_Inverse);
 359                         if (predicates.contains(b.InverseOf)) {
 360                                 if (invR.size() > 1) {
 361                                         if (DEBUG)
 362                                                 System.out
 363                                                                 .println("###########################################");
 364                                         Resource inv = g.getInverse(r);
 365                                         Collection<Resource> ret = new ArrayList<Resource>();
 366                                         for (Statement pp : g.getStatements(r,
 367                                                         b.IsRelatedTo_Inverse))
 368                                                 if (!pp.getPredicate().equals(inv)) {
 369                                                         if (DEBUG) {
 370                                                                 System.out.println("<"
 371                                                                                 + NameUtils.getSafeName(g, pp
 372                                                                                                 .getSubject())
 373                                                                                 + ","
 374                                                                                 + NameUtils.getSafeName(g, pp
 375                                                                                                 .getPredicate())
 376                                                                                 + ","
 377                                                                                 + NameUtils.getSafeName(g, pp
 378                                                                                                 .getObject()) + ">");
 379                                                         }
 380                                                         ret.add(pp.getObject());
 381                                                 }
 382                                         return ret;
 383                                 }
 384                                 // System.out.println("?????????????????");
 385                                 Collection<Resource> invParents = getParents(g,
 386                                                 g.getInverse(r), true);
 387                                 if (!invParents.isEmpty())
 388                                         return invParents;
 389                         }
 390                         if (DEBUG) {
 391                                 System.out.print("invR");
 392                                 for (Resource res : invR)
 393                                         System.out.print(" " + NameUtils.getSafeName(g, res));
 394                                 System.out.println();
 395                         }
 396                         return invR;
 397                         */
 398                 }
 399
 400         }
 401
 402 //      public static String getIdentifier(ReadGraph g, Resource r)
 403 //                      throws DatabaseException {
 404 //      Layer0 L0 = Layer0.getInstance(g);
 405 //              if (r.equals(g.getRootLibrary()))
 406 //                      return "";
 407 //              String name = g.getPossibleRelatedValue(r, L0.HasName);
 408 //              if (name == null)
 409 //                      return null;
 410 //              Collection<Resource> parents = getParents(g, r, true);
 411 //              if (parents.size() != 1)
 412 //                      return null;
 413 //              for (Resource p : parents) {
 414 //                      String parentIdentifier = getIdentifier(g, p);
 415 //                      if (parentIdentifier == null)
 416 //                              return null;
 417 //                      return parentIdentifier + "/" + name;
 418 //              }
 419 //              return null;
 420 //      }
 421
 422         static int kess = 0;
 423
 424         static class Expansion extends AsyncReadRequest {
 425
 426                 final private Collection<Resource> roots;
 427                 final Collection<DirectStatements>[] results;
 428                 final Collection<Resource>[] listElements;
 429
 430                 public Expansion(Collection<Resource> roots, Collection<DirectStatements>[] results, Collection<Resource>[] listElements) {
 431                         this.roots = roots;
 432                         this.results = results;
 433                         this.listElements = listElements;
 434                 }
 435
 436                 @Override
 437                 public void run(AsyncReadGraph graph) {
 438
 439                         QueryControl control = graph.getService(QueryControl.class);
 440                         final DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class);
 441
 442                         final DomainStatementProcedure proc = new DomainStatementProcedure(dqs, graph.getService(StatementSupport.class), graph.getService(Layer0.class), results, listElements);
 443
 444                         int slice = (int) (roots.size() / control.getAmountOfQueryThreads()) + 1;
 445
 446                         final Resource[] rootArray = roots.toArray(Resource.NONE);
 447                         for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
 448
 449                                 final int start = i * slice;
 450                                 final int end = Math.min(start + slice, rootArray.length);
 451
 452                                 control.schedule(graph, i, new ControlProcedure() {
 453
 454                                         @Override
 455                                         public void execute(AsyncReadGraph graph) {
 456                                                 for (int index = start; index < end; index++) {
 457                                                         dqs.forEachDirectStatement(graph, rootArray[index], proc);
 458                                                 }
 459
 460                                         }
 461
 462                                 });
 463
 464                         }
 465
 466                 }
 467
 468                 @Override
 469                 public int getFlags() {
 470                         return 0;
 471                 }
 472
 473         }
 474
 475         static class Expansion2 extends AsyncReadRequest {
 476
 477                 final private Collection<Resource> roots;
 478                 final Collection<DirectStatements>[] results;
 479                 final boolean ignoreVirtual;
 480
 481                 public Expansion2(Collection<Resource> roots, Collection<DirectStatements>[] results) {
 482                         this(roots, results, true);
 483                 }
 484
 485                 public Expansion2(Collection<Resource> roots, Collection<DirectStatements>[] results, boolean ignoreVirtual) {
 486                         this.roots = roots;
 487                         this.results = results;
 488                         this.ignoreVirtual = ignoreVirtual;
 489                 }
 490
 491                 @Override
 492                 public void run(AsyncReadGraph graph) {
 493
 494                         QueryControl control = graph.getService(QueryControl.class);
 495                         final DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class);
 496
 497                         final DomainStatementProcedure2 proc =
 498                                 new DomainStatementProcedure2(results);
 499
 500                         int slice = (int) (roots.size() / control.getAmountOfQueryThreads()) + 1;
 501
 502                         final Resource[] rootArray = roots.toArray(Resource.NONE);
 503                         for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
 504
 505                                 final int start = i * slice;
 506                                 final int end = Math.min(start + slice, rootArray.length);
 507
 508                                 control.schedule(graph, i, new ControlProcedure() {
 509                                         @Override
 510                                         public void execute(AsyncReadGraph graph) {
 511                                                 if (ignoreVirtual) {
 512                                                         for (int index = start; index < end; index++) {
 513                                                                 dqs.forEachDirectPersistentStatement(graph, rootArray[index], proc);
 514                                                         }
 515                                                 } else {
 516                                                         for (int index = start; index < end; index++) {
 517                                                                 dqs.forEachDirectStatement(graph, rootArray[index], proc);
 518                                                         }
 519                                                 }
 520                                         }
 521                                 });
 522
 523                         }
 524
 525                 }
 526
 527                 @Override
 528                 public int getFlags() {
 529                         return 0;
 530                 }
 531
 532         }
 533
 534         static class DomainProcessor2 {
 535
 536                 Serializer variantSerializer;
 537
 538                 int id = 0;
 539
 540                 Set<Resource> fringe = null;
 541                 Set<Resource> exclusions = new HashSet<Resource>();
 542                 Set<Resource> internalDomain = new HashSet<Resource>();
 543                 Set<Resource> sharedExternalReferences = null;
 544                 TIntSet sharedExternalIds = null;
 545                 Set<Resource> sharedExternalFringe = null;
 546                 Set<Resource> predicates = null;
 547                 Set<Resource> isRelatedToPredicates = null;
 548                 Set<Resource> sharedPredicates = null;
 549                 TIntIntHashMap ids = null;
 550                 Map<Resource, Statement> specials = null;
 551                 Map<Resource, ExtentStatus> status = null;
 552                 Map<Resource, WeakStatus> weakInverses = null;
 553
 554 //              final ArrayList<Double> priorityList = new ArrayList<Double>();
 555
 556                 private long composedObjectCounter = 0;
 557                 private long fastInternalCounter = 0;
 558                 private long parentExternalCounter = 0;
 559                 private long fullInternalCounter = 0;
 560                 private long fullExternalCounter = 0;
 561
 562                 private long startupTime = 0;
 563                 private long expandTime = 0;
 564                 private long fullResolveTime = 0;
 565                 private long fastResolveTime = 0;
 566                 private long otherStatementTime = 0;
 567                 private long parentResolveTime = 0;
 568                 private long extentSeedTime = 0;
 569                 private long composedPredicateTime = 0;
 570                 private long composedObjectTime = 0;
 571
 572                 public void expand(ReadGraph graph, Set<Resource> fringe, Collection<DirectStatements>[] expansion) throws DatabaseException {
 573
 574                         long start = System.nanoTime();
 575
 576                         Collection<Collection<DirectStatements>[]> fullExpansion = new ArrayList<Collection<DirectStatements>[]>();
 577                         QueryControl control = graph.getService(QueryControl.class);
 578                         for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
 579                                 expansion[i] = new ArrayList<DirectStatements>();
 580                         }
 581
 582                         graph.syncRequest(new Expansion2(fringe, expansion));
 583
 584                         fringe.clear();
 585
 586                         expandTime += (System.nanoTime() - start);
 587
 588                 }
 589
 590                 public void classifyPredicates(ReadGraph graph, final Set<Resource> schedule) throws DatabaseException {
 591
 592                         CollectionSupport cs = graph.getService(CollectionSupport.class);
 593
 594                         final Layer0 L0 = Layer0.getInstance(graph);
 595
 596                         long start = System.nanoTime();
 597
 598                         final ConcurrentLinkedQueue<Resource> composedResult = new ConcurrentLinkedQueue<Resource>();
 599                         final ConcurrentLinkedQueue<Resource> singleResult = new ConcurrentLinkedQueue<Resource>();
 600                         final ConcurrentLinkedQueue<Resource> sharedResult = new ConcurrentLinkedQueue<Resource>();
 601                         final ConcurrentLinkedQueue<Pair<Resource, Resource>> singles = new ConcurrentLinkedQueue<Pair<Resource, Resource>>();
 602
 603                         // Discover singles
 604                         graph.syncRequest(new AsyncReadRequest() {
 605
 606                                 @Override
 607                                 public void run(AsyncReadGraph graph) {
 608
 609                                         for (final Resource predicate : schedule) {
 610
 611                                                 graph.forPossibleSuperrelation(predicate, new AsyncProcedure<Resource>() {
 612
 613                                                         @Override
 614                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {
 615                                                                 throwable.printStackTrace();
 616                                                         }
 617
 618                                                         @Override
 619                                                         public void execute(AsyncReadGraph graph, final Resource single) {
 620                                                                 singles.add(Pair.make(predicate, single));
 621                                                         }
 622
 623                                                 });
 624
 625                                                 graph.forHasStatement(predicate, L0.SharedRange, new AsyncProcedure<Boolean>() {
 626
 627                                                         @Override
 628                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {
 629                                                                 throwable.printStackTrace();
 630                                                         }
 631
 632                                                         @Override
 633                                                         public void execute(AsyncReadGraph graph, final Boolean shared) {
 634                                                                 if(shared) sharedResult.add(predicate);
 635                                                         }
 636
 637                                                 });
 638
 639                                         }
 640
 641                                 }
 642
 643                         });
 644
 645                         // Determine singles
 646                         final Set<Resource> singleSchedule = cs.createSet();
 647                         for(Pair<Resource, Resource> pair : singles) {
 648
 649                                 Resource single = pair.second;
 650                                 if(single != null && predicates.add(single)) singleSchedule.add(single);
 651
 652                         }
 653
 654                         graph.syncRequest(new AsyncReadRequest() {
 655
 656                                 @Override
 657                                 public void run(AsyncReadGraph graph) {
 658
 659                                         for (final Resource predicate : singleSchedule) {
 660
 661                                                 graph.forIsSubrelationOf(predicate, L0.IsRelatedTo, new AsyncProcedure<Boolean>() {
 662
 663                                                         @Override
 664                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {
 665                                                                 throwable.printStackTrace();
 666                                                         }
 667
 668                                                         @Override
 669                                                         public void execute(AsyncReadGraph graph, Boolean strong) {
 670                                                                 if (strong) singleResult.add(predicate);
 671                                                         }
 672
 673                                                 });
 674
 675                                         }
 676
 677                                 }
 678
 679                         });
 680
 681                         isRelatedToPredicates.addAll(singleResult);
 682                         sharedPredicates.addAll(sharedResult);
 683
 684                         final Set<Resource> specialSchedule = cs.createSet();
 685
 686                         // Classify
 687                         for(Pair<Resource, Resource> pair : singles) {
 688
 689                                 Resource single = pair.second;
 690                                 if(single != null) {
 691                                         if(isRelatedToPredicates.contains(single)) {
 692                                                 isRelatedToPredicates.add(pair.first);
 693                                         }
 694                                 } else {
 695                                         specialSchedule.add(pair.first);
 696                                 }
 697
 698                         }
 699
 700                         graph.syncRequest(new AsyncReadRequest() {
 701
 702                                 @Override
 703                                 public void run(AsyncReadGraph graph) {
 704
 705                                         for (final Resource predicate : specialSchedule) {
 706
 707                                                 graph.forIsSubrelationOf(predicate, L0.IsRelatedTo, new AsyncProcedure<Boolean>() {
 708
 709                                                         @Override
 710                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {
 711                                                                 throwable.printStackTrace();
 712                                                         }
 713
 714                                                         @Override
 715                                                         public void execute(AsyncReadGraph graph, Boolean composed) {
 716                                                                 if (composed) composedResult.add(predicate);
 717                                                         }
 718
 719                                                 });
 720
 721                                         }
 722
 723                                 }
 724
 725                         });
 726
 727                         isRelatedToPredicates.addAll(composedResult);
 728
 729                         composedPredicateTime += (System.nanoTime() - start);
 730
 731                 }
 732
 733                 private Set<Resource> strongInverseSet = new HashSet<Resource>();
 734
 735                 public void classifyPredicates(ReadGraph graph, final Collection<DirectStatements>[] expansion) throws DatabaseException {
 736
 737                         CollectionSupport cs = graph.getService(CollectionSupport.class);
 738                         final Set<Resource> schedule = cs.createSet();
 739                         final Map<Resource, Resource> newPredicates = cs.createMap(Resource.class);
 740
 741                         for (Collection<DirectStatements> coll : expansion)
 742                                 for (DirectStatements stms : coll)
 743                                         for(Statement stm : stms) {
 744
 745                                                 Resource predicate = stm.getPredicate();
 746
 747                                                 if(predicates.add(predicate)) {
 748                                                         Resource inverse = graph.getPossibleInverse(predicate);
 749                                                         schedule.add(predicate);
 750                                                         if(inverse != null) {
 751                                                                 newPredicates.put(predicate, inverse);
 752                                                                 if(predicates.add(inverse)) schedule.add(inverse);
 753                                                         }
 754
 755                                                 }
 756
 757                                         }
 758
 759                         classifyPredicates(graph, schedule);
 760
 761                         for(Map.Entry<Resource, Resource> entry : newPredicates.entrySet()) {
 762                                 // Inverse is strong => this has strong inverse
 763                                 if(isRelatedToPredicates.contains(entry.getValue())) {
 764                                         strongInverseSet.add(entry.getKey());
 765                                 }
 766                                 // This is strong => inverse has strong inverse
 767                                 if(isRelatedToPredicates.contains(entry.getKey())) {
 768                                         strongInverseSet.add(entry.getValue());
 769                                 }
 770                         }
 771
 772                 }
 773
 774                 /*
 775                  * Composed objects are internal. Mark them for expansion.
 776                  */
 777
 778                 public void processFringe(ReadGraph graph, Collection<DirectStatements>[] expansion,
 779                                 ObjectOutputStream otherStatementsOutput, ObjectOutputStream valueOutput) throws DatabaseException, IOException {
 780
 781                         SerialisationSupport support = graph.getService(SerialisationSupport.class);
 782                         TransferableGraphSupport tgs = graph.getService(TransferableGraphSupport.class);
 783
 784                         Layer0 L0 = Layer0.getInstance(graph);
 785
 786                         long start = System.nanoTime();
 787
 788                         for (Collection<DirectStatements> coll : expansion)
 789                                 for (DirectStatements stms : coll) {
 790
 791                                         Resource subject = stms.getSubject();
 792
 793                                         boolean partOf = false;
 794                                         for(Statement stm : stms) {
 795                                                 Resource predicate = stm.getPredicate();
 796                                                 if(L0.PartOf.equals(predicate)) {
 797                                                         partOf = true;
 798                                                         break;
 799                                                 }
 800                                         }
 801
 802                                         ExtentStatus subjectStatus = status.get(subject);
 803                                         if(LOG && subjectStatus != null) log("EXISTING STATUS FOR " + graph.getPossibleURI(subject) + " - " + subjectStatus);
 804                                         if(subjectStatus == ExtentStatus.EXTERNAL) continue;
 805                                         if(partOf && (subjectStatus == null) && graph.getPossibleURI(subject) != null) {
 806
 807                                                 status.put(subject, ExtentStatus.EXTERNAL);
 808                                                 if(LOG) {
 809                                                         String uri = graph.getPossibleURI(subject);
 810                                                         if(uri == null) log("[EXTERNAL]: No URI for " + subject);
 811                                                         else log("[EXTERNAL] " + uri);
 812                                                 }
 813
 814                                                 // Check for SharedRange statements
 815                                                 for(Statement stm : stms) {
 816                                                         Resource predicate = stm.getPredicate();
 817                                                         if(sharedPredicates.contains(predicate)) {
 818                                                                 sharedExternalFringe.add(stm.getObject());
 819                                                                 if(LOG) {
 820                                                                         log("[SHARED EXTERNAL FRINGE]: " + NameUtils.getSafeName(graph, stm.getObject()));
 821                                                                 }
 822                                                         }
 823                                                 }
 824
 825                                         } else {
 826
 827                                                 boolean special = specials.containsKey(subject);
 828                                                 if(LOG) {
 829                                                         if(special) {
 830                                                                 log("[SPECIAL] " + NameUtils.getSafeName(graph, subject));
 831                                                         }
 832                                                 }
 833
 834                                                 status.put(subject, ExtentStatus.INTERNAL);
 835                                                 if(LOG) log("[INTERNAL] " + NameUtils.getSafeName(graph, subject));
 836
 837                                                 int sId = support.getTransientId(subject);
 838
 839                                                 if(graph.hasValue(subject)) {
 840                                                         Datatype dt = graph.getRelatedValue(subject, L0.HasDataType, Bindings.getBindingUnchecked(Datatype.class));
 841                                                         Binding b = Bindings.getBinding(dt);
 842                                                         Object _value = graph.getValue(subject, b);
 843                                                         Variant variant = new Variant(b, _value);
 844                                                         byte[] value = variantSerializer.serialize(variant);
 845                                                         if(LOG) log("[VALUE] " + NameUtils.getSafeName(graph, subject));
 846                                                         valueOutput.writeInt(sId);
 847                                                         valueOutput.writeInt(value.length);
 848                                                         assert (value.length > 0);
 849                                                         valueOutput.write(value);
 850                                                 }
 851
 852                                                 TIntArrayList stream = new TIntArrayList();
 853
 854                                                 for(Statement stm : stms) {
 855
 856                                                         if(special) {
 857
 858 //                                                              System.err.println("stm=" + stm + " special=" + specials.get(subject));
 859
 860                                                         }
 861
 862                                                         Resource predicate = stm.getPredicate();
 863                                                         Resource object = stm.getObject();
 864
 865                                                         ExtentStatus objectStatus = status.get(object);
 866
 867                                                         // Strong predicate
 868                                                         if (isRelatedToPredicates.contains(predicate) && (objectStatus !=  ExtentStatus.EXCLUDED)) {
 869
 870                                                                 int pId = support.getTransientId(predicate);
 871                                                                 int oId = support.getTransientId(object);
 872
 873                                                                 if(LOG) {
 874                                                                         String s = NameUtils.getSafeName(graph, subject);
 875                                                                         String p = NameUtils.getSafeName(graph, predicate);
 876                                                                         String o = NameUtils.getSafeName(graph, object);
 877                                                                         log("related=" + s + " - " + p + " - " + o);
 878                                                                 }
 879
 880                                                                 stream.add(pId);
 881                                                                 stream.add(oId);
 882
 883                                                                 if(objectStatus == null)
 884                                                                         fringe.add(object);
 885
 886                                                         } else {
 887
 888                                                                 // Weak predicate
 889                                                                 if(objectStatus == ExtentStatus.INTERNAL) {
 890
 891                                                                         // The inverse is also weak (or there is no inverse)
 892                                                                         if(!strongInverseSet.contains(predicate)) {
 893
 894                                                                                 int pId = support.getTransientId(predicate);
 895                                                                                 int oId = support.getTransientId(object);
 896
 897                                                                                 stream.add(pId);
 898                                                                                 stream.add(oId);
 899
 900                                                                                 if(LOG) {
 901                                                                                         String s = NameUtils.getSafeName(graph, subject);
 902                                                                                         String p = NameUtils.getSafeName(graph, predicate);
 903                                                                                         String o = NameUtils.getSafeName(graph, object);
 904                                                                                         log("fully weak internal=" + s + " - " + p + " - " + o + " - " + objectStatus);
 905                                                                                 }
 906
 907                                                                         } else {
 908
 909                                                                                 if(LOG) {
 910                                                                                         String s = NameUtils.getSafeName(graph, subject);
 911                                                                                         String p = NameUtils.getSafeName(graph, predicate);
 912                                                                                         String o = NameUtils.getSafeName(graph, object);
 913                                                                                         log("strong inverse internals=" + s + " - " + p + " - " + o + " - " + objectStatus);
 914                                                                                 }
 915
 916                                                                         }
 917
 918                                                                 } else {
 919
 920                                                                         if(special) {
 921
 922 //                                                                              System.err.println("stm=" + stm + " special=" + specials.get(subject));
 923
 924                                                                                 Statement spec = specials.get(subject);
 925
 926                                                                                 // This statement can be specially treated
 927                                                                                 if(stm.getPredicate().equals(spec.getPredicate()) && stm.getObject().equals(spec.getObject())) {
 928
 929                                                                                         int pId = support.getTransientId(predicate);
 930                                                                                         int oId = support.getTransientId(object);
 931
 932                                                                                         if(LOG) {
 933                                                                                                 String s = NameUtils.getSafeName(graph, subject);
 934                                                                                                 String p = NameUtils.getSafeName(graph, predicate);
 935                                                                                                 String o = NameUtils.getSafeName(graph, object);
 936                                                                                                 log("special=" + s + " - " + p + " - " + o);
 937                                                                                         }
 938
 939                                                                                         stream.add(pId);
 940                                                                                         stream.add(oId);
 941
 942                                                                                 }
 943
 944                                                                         } else {
 945
 946                                                                                 if(LOG) {
 947                                                                                         String s = NameUtils.getSafeName(graph, subject);
 948                                                                                         String p = NameUtils.getSafeName(graph, predicate);
 949                                                                                         String o = NameUtils.getSafeName(graph, object);
 950                                                                                         log("weak with unknown object=" + s + " - " + p + " - " + o + " - " + objectStatus);
 951                                                                                 }
 952
 953                                                                         }
 954
 955                                                                 }
 956
 957                                                         }
 958
 959                                                 }
 960
 961                                                 if(!stream.isEmpty()) {
 962                                                         otherStatementsOutput.writeInt(sId);
 963                                                         otherStatementsOutput.writeInt(stream.size() / 2);
 964                                                         for (int i = 0; i < stream.size(); i++)
 965                                                                 otherStatementsOutput.writeInt(stream.getQuick(i));
 966                                                 }
 967
 968                                         }
 969
 970                                 }
 971
 972                         composedObjectTime += System.nanoTime() - start;
 973
 974                 }
 975
 976                 public void process(ReadGraph graph,
 977                                 ObjectOutputStream otherStatementsOutput,
 978                                 ObjectOutputStream valueOutput)
 979                                 throws DatabaseException, IOException {
 980
 981                         this.variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT);
 982
 983                         QueryControl control = graph.getService(QueryControl.class);
 984
 985 //                      System.err.println("Begin ConsistsOfProcess");
 986
 987                         /*
 988                          * Browse all stm = (s, ConsistsOf, o)
 989                          * � All internal URIs are found => from now on, if unidentified resource has PartOf it is external.
 990              * � All s are internal
 991              * � All o are internal
 992              * � All stm are included
 993                          */
 994                         Pair<List<InternalEntry>,Set<Resource>> pair = ConsistsOfProcess.walk(graph, null, fringe, exclusions, true);
 995                         List<InternalEntry> entries = pair.first;
 996                         for(InternalEntry entry : entries) {
 997                                 Resource r = entry.resource;
 998                                 if (status.put(r, ExtentStatus.INTERNAL) == null) {
 999                                         String URI = graph.getPossibleURI(r);
1000                                         if(URI != null) log("URI INTERNAL " + URI);
1001                                         else log("URI has no URI for " + r);
1002                                         fringe.add(r);
1003                                         internalDomain.add(r);
1004                                 }
1005                         }
1006
1007                         for(Resource unnamedChild : pair.second) {
1008                             if (status.put(unnamedChild, ExtentStatus.INTERNAL) == null) {
1009                                 fringe.add(unnamedChild);
1010                             }
1011                         }
1012
1013                         /*
1014                          * This loop resolves the transitive closure of all p < IsRelatedTo such that p does not contain the SharedRange tag.
1015                          * Such resources are guaranteed to be internal.
1016                          */
1017                         while(!fringe.isEmpty()) {
1018
1019 //                              System.err.println("Process Fringe with " + fringe.size() + ".");
1020
1021                                 Collection<DirectStatements>[] expansion = new ArrayList[control.getAmountOfQueryThreads()];
1022
1023 //                              System.err.println("-expand");
1024
1025                                 // Expand fringe
1026                                 expand(graph, fringe, expansion);
1027
1028                                 /*
1029                                  * classify all p
1030                                  * -IsRelatedTo
1031                                  * -SharedRange
1032                                  * -Internal / External
1033                                  */
1034
1035 //                              System.err.println("-classify");
1036
1037                                 classifyPredicates(graph, expansion);
1038
1039                                 /*
1040                                  * for stms in [stms]
1041                                  *   if stms contains predicate PartOf => s is External
1042                                  *   else s is Internal
1043                                  *   for all stm=(s,p,o) in stms
1044                                  *     if p <R IsRelatedTo => stm is included
1045                                  *     Fringe <- o
1046                                  */
1047
1048 //                              System.err.println("-process");
1049
1050                                 processFringe(graph, expansion, otherStatementsOutput, valueOutput);
1051
1052                         }
1053
1054                         while(!sharedExternalFringe.isEmpty()) {
1055
1056                                 Collection<DirectStatements>[] expansion = new ArrayList[control.getAmountOfQueryThreads()];
1057                                 expand(graph, sharedExternalFringe, expansion);
1058
1059                                 for (Collection<DirectStatements> coll : expansion)
1060                                         for (DirectStatements stms : coll) {
1061
1062                                                 Resource subject = stms.getSubject();
1063                                                 ExtentStatus subjectStatus = status.get(subject);
1064
1065                                                 if(ExtentStatus.INTERNAL == subjectStatus) {
1066
1067                                                         if(internalDomain.contains(subject)) continue;
1068
1069                                                         status.put(subject, ExtentStatus.EXTERNAL);
1070                                                         sharedExternalReferences.add(subject);
1071
1072                                                         if(LOG) {
1073                                                                 log("[SHARED EXTERNAL REFERENCE]: " + NameUtils.getSafeName(graph, subject));
1074                                                         }
1075
1076                                                         for(Statement stm : stms) {
1077                                                                 Resource predicate = stm.getPredicate();
1078                                                                 if (isRelatedToPredicates.contains(predicate)) {
1079                                                                         sharedExternalFringe.add(stm.getObject());
1080                                                                 }
1081                                                         }
1082
1083                                                 }
1084                                         }
1085
1086                         }
1087
1088                         if (PROFILE) {
1089                                 System.out.println(composedObjectCounter + " " + fastInternalCounter
1090                                                 + " " + parentExternalCounter + " "
1091                                                 + fullExternalCounter + " " + fullInternalCounter);
1092                         }
1093
1094                 }
1095
1096         }
1097
1098         static class DomainProcessor {
1099
1100                 Serializer variantSerializer;
1101
1102                 int id = 0;
1103
1104                 Set<Resource> predicates = null;
1105                 Set<Resource> composedPredicates = null;
1106                 Set<Resource> expansionSeeds = null;
1107                 Map<Resource, Integer> ids = null;
1108                 Map<Resource, ExtentStatus> status = null;
1109                 Map<Resource, WeakStatus> weakInverses = null;
1110
1111                 final Set<SubgraphAdvisor> advisors;
1112                 final ArrayList<Double> priorityList = new ArrayList<Double>();
1113
1114                 private long composedObjectCounter = 0;
1115                 private long fastInternalCounter = 0;
1116                 private long parentExternalCounter = 0;
1117                 private long fullInternalCounter = 0;
1118                 private long fullExternalCounter = 0;
1119
1120                 private long startupTime = 0;
1121                 private long expandTime = 0;
1122                 private long fullResolveTime = 0;
1123                 private long fastResolveTime = 0;
1124                 private long otherStatementTime = 0;
1125                 private long parentResolveTime = 0;
1126                 private long extentSeedTime = 0;
1127                 private long composedPredicateTime = 0;
1128                 private long composedObjectTime = 0;
1129
1130                 public DomainProcessor(Set<SubgraphAdvisor> advisors) {
1131                         this.advisors = advisors;
1132                         HashSet<Double> prioritySet = new HashSet<Double>();
1133                         for (SubgraphAdvisor advisor : advisors)
1134                                 prioritySet.add(advisor.priority());
1135                         priorityList.addAll(prioritySet);
1136                         Collections.sort(priorityList);
1137                 }
1138
1139                 public void expand(ReadGraph graph, Collection<DirectStatements>[] expansion, Set<Resource> schedule) throws DatabaseException {
1140
1141                         long start = System.nanoTime();
1142
1143 //                      if (DEBUG)
1144 //                              System.out.println("expanding " + expansionSeeds.size() + " resources.");
1145
1146                         QueryControl control = graph.getService(QueryControl.class);
1147 //                      final Collection<DirectStatements>[] results = new ArrayList[control.getAmountOfQueryThreads()];
1148                         final ArrayList<Resource>[] listElements = new ArrayList[control.getAmountOfQueryThreads()];
1149                         for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
1150 //                              results[i] = new ArrayList<DirectStatements>();
1151                                 listElements[i] = new ArrayList<Resource>();
1152                         }
1153
1154 //                      if(DEBUG) {
1155 //                              for(Resource r : expansionSeeds)
1156 //                                      System.out.println("Expanding " + NameUtils.getSafeName(graph, r, true));
1157 //                      }
1158
1159                         graph.syncRequest(new Expansion(expansionSeeds, expansion, listElements));
1160                         for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
1161 //                              for (DirectStatements s : results[i]) {
1162 //                                      expansion.put(s.getSubject(), s);
1163 //                              }
1164                                 for (Resource s : listElements[i]) {
1165                                         schedule.add(s);
1166 //                                      if(status.put(s, ExtentStatus.INTERNAL) == null) {
1167 //                                              ids.put(s, id++);
1168 //                                      }
1169                                 }
1170                         }
1171
1172                         expandTime += (System.nanoTime() - start);
1173
1174                 }
1175
1176                 public void extractComposedPredicates(ReadGraph graph, final Collection<DirectStatements>[] expansion) throws DatabaseException {
1177
1178                         long start = System.nanoTime();
1179
1180                         CollectionSupport cs = graph.getService(CollectionSupport.class);
1181
1182                         final ConcurrentLinkedQueue<Resource> composedResult = new ConcurrentLinkedQueue<Resource>();
1183                         final ConcurrentLinkedQueue<Resource> singleResult = new ConcurrentLinkedQueue<Resource>();
1184                         final ConcurrentLinkedQueue<Pair<Resource, Resource>> singles = new ConcurrentLinkedQueue<Pair<Resource, Resource>>();
1185
1186                         final Set<Resource> schedule = cs.createSet();
1187
1188                         for (Collection<DirectStatements> coll : expansion)
1189                                 for (DirectStatements stms : coll)
1190                                         for(Statement stm : stms) {
1191                                                 Resource predicate = stm.getPredicate();
1192                                                 if(predicates.add(predicate)) schedule.add(predicate);
1193                                         }
1194
1195                         // Discover singles
1196                         graph.syncRequest(new AsyncReadRequest() {
1197
1198                                 @Override
1199                                 public void run(AsyncReadGraph graph) {
1200
1201                                         for (final Resource predicate : schedule) {
1202
1203                                                 graph.forPossibleSuperrelation(predicate, new AsyncProcedure<Resource>() {
1204
1205                                                         @Override
1206                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {
1207                                                                 throwable.printStackTrace();
1208                                                         }
1209
1210                                                         @Override
1211                                                         public void execute(AsyncReadGraph graph, final Resource single) {
1212                                                                 singles.add(Pair.make(predicate, single));
1213                                                         }
1214
1215                                                 });
1216
1217                                         }
1218
1219                                 }
1220
1221                         });
1222
1223                         // Determine singles
1224                         final Set<Resource> singleSchedule = cs.createSet();
1225                         for(Pair<Resource, Resource> pair : singles) {
1226
1227                                 Resource single = pair.second;
1228                                 if(single != null && predicates.add(single)) singleSchedule.add(single);
1229
1230                         }
1231
1232                         graph.syncRequest(new AsyncReadRequest() {
1233
1234                                 @Override
1235                                 public void run(AsyncReadGraph graph) {
1236
1237                                         for (final Resource predicate : singleSchedule) {
1238
1239                                                 graph.forIsSubrelationOf(predicate, graph.getService(Layer0.class).IsComposedOf, new AsyncProcedure<Boolean>() {
1240
1241                                                         @Override
1242                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {
1243                                                                 throwable.printStackTrace();
1244                                                         }
1245
1246                                                         @Override
1247                                                         public void execute(AsyncReadGraph graph, Boolean composed) {
1248                                                                 if (composed) singleResult.add(predicate);
1249                                                         }
1250
1251                                                 });
1252
1253                                         }
1254
1255                                 }
1256
1257                         });
1258
1259                         composedPredicates.addAll(singleResult);
1260
1261                         final Set<Resource> specialSchedule = cs.createSet();
1262
1263                         // Classify
1264                         for(Pair<Resource, Resource> pair : singles) {
1265
1266                                 Resource single = pair.second;
1267                                 if(single != null) {
1268                                         if(composedPredicates.contains(single)) {
1269                                                 composedPredicates.add(pair.first);
1270                                         }
1271                                 } else {
1272                                         specialSchedule.add(pair.first);
1273                                 }
1274
1275                         }
1276
1277                         graph.syncRequest(new AsyncReadRequest() {
1278
1279                                 @Override
1280                                 public void run(AsyncReadGraph graph) {
1281
1282                                         for (final Resource predicate : specialSchedule) {
1283
1284                                                 graph.forIsSubrelationOf(predicate, graph.getService(Layer0.class).IsComposedOf, new AsyncProcedure<Boolean>() {
1285
1286                                                         @Override
1287                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {
1288                                                                 throwable.printStackTrace();
1289                                                         }
1290
1291                                                         @Override
1292                                                         public void execute(AsyncReadGraph graph, Boolean composed) {
1293                                                                 if (composed) composedResult.add(predicate);
1294                                                         }
1295
1296                                                 });
1297
1298                                         }
1299
1300                                 }
1301
1302                         });
1303
1304                         composedPredicates.addAll(composedResult);
1305
1306                         composedPredicateTime += (System.nanoTime() - start);
1307
1308                 }
1309
1310                 /*
1311                  * Composed objects are internal. Mark them for expansion.
1312                  */
1313
1314                 public void collectComposedObjects(ReadGraph graph, Collection<DirectStatements>[] expansion, Set<Resource> typeTodo, Set<Resource> objectTodo,
1315                                 Set<Resource> predicateTodo) throws DatabaseException {
1316
1317                         long start = System.nanoTime();
1318
1319                         Layer0 l0 = Layer0.getInstance(graph);
1320
1321                         for (Collection<DirectStatements> coll : expansion)
1322                                 for (DirectStatements stms : coll)
1323                                         for(Statement stm : stms) {
1324
1325                                                 Resource predicate = stm.getPredicate();
1326                                                 Resource object = stm.getObject();
1327
1328                                                 if (composedPredicates.contains(predicate)) {
1329
1330                                                         ExtentStatus existing = status.put(object, ExtentStatus.INTERNAL);
1331                                                         if(existing == null) {
1332                                                                 ids.put(object, id++);
1333                                                                 composedObjectCounter++;
1334                                                                 expansionSeeds.add(object);
1335 //                                                              System.err.println("internal: " + NameUtils.getSafeName(graph, object, true));
1336                                                                 if(LOG) log("[INTERNAL] (composed object) " + NameUtils.getSafeName(graph, object, true));
1337                                                         } else if (existing == ExtentStatus.EXCLUDED) {
1338                                                                 System.err.println("preExcluded: " + NameUtils.getSafeName(graph, object, true));
1339                                                                 status.put(object, ExtentStatus.EXCLUDED);
1340                                                         } else if (existing == ExtentStatus.EXTERNAL) {
1341                                                                 System.err.println("preExternal: " + NameUtils.getSafeName(graph, object, true));
1342                                                                 status.put(object, ExtentStatus.EXTERNAL);
1343                                                         }
1344
1345                                                 } else {
1346
1347 //                                                      System.err.println("internal2: " + NameUtils.getSafeName(graph, object, true));
1348
1349                                                         if (!status.containsKey(object)) {
1350                                                                 if (l0.InstanceOf.equalsResource(predicate)) {
1351                                                                         typeTodo.add(object);
1352                                                                 } else {
1353                                                                         objectTodo.add(object);
1354                                                                 }
1355                                                         }
1356
1357                                                         if (!status.containsKey(predicate)) {
1358                                                                 predicateTodo.add(predicate);
1359                                                         }
1360
1361                                                 }
1362                                         }
1363
1364                         composedObjectTime += System.nanoTime() - start;
1365
1366                 }
1367
1368                 public void writeOtherStatements(ReadGraph graph, Collection<Collection<DirectStatements>[]> expansion, ObjectOutputStream composedStatementsOutput, ObjectOutputStream otherStatementsOutput,
1369                                 ObjectOutputStream valueOutput) throws DatabaseException {
1370
1371                         long start = System.nanoTime();
1372
1373                         Layer0 l0 = Layer0.getInstance(graph);
1374                         SerialisationSupport support = graph.getService(SerialisationSupport.class);
1375                         TransferableGraphSupport tgs = graph.getService(TransferableGraphSupport.class);
1376
1377                         TIntArrayList other = new TIntArrayList();
1378                         TIntArrayList composed = new TIntArrayList();
1379
1380                         try {
1381
1382                                 for (Collection<DirectStatements>[] colls : expansion)
1383                                         for (Collection<DirectStatements> coll : colls)
1384                                                 for (DirectStatements stms : coll) {
1385
1386                                                         Resource subject = stms.getSubject();
1387                                                         composed.resetQuick();
1388
1389                                                         int sId = support.getTransientId(subject);
1390
1391                                                         composedStatementsOutput.writeInt(sId);
1392
1393                                                         if(graph.hasValue(subject)) {
1394                                                                 Datatype dt = graph.getRelatedValue(subject, l0.HasDataType, Bindings.getBindingUnchecked(Datatype.class));
1395                                                                 Binding b = Bindings.getBinding(dt);
1396                                                                 Object _value = graph.getValue(subject, b);
1397                                                                 Variant variant = new Variant(b, _value);
1398                                                                 byte[] value = variantSerializer.serialize(variant);
1399                                                                 if(LOG) log("[VALUE] " + NameUtils.getSafeName(graph, subject));
1400                                                                 valueOutput.writeInt(sId);
1401                                                                 valueOutput.writeInt(value.length);
1402                                                                 assert (value.length > 0);
1403                                                                 valueOutput.write(value);
1404                                                         }
1405
1406                                                         for (Statement s : stms) {
1407
1408                                                                 Resource object = s.getObject();
1409                                                                 Resource predicate = s.getPredicate();
1410
1411                                                                 ExtentStatus objectStatus = status.get(object);
1412
1413                                                                 if(objectStatus == ExtentStatus.INTERNAL) {
1414                                                                         composed.add(support.getTransientId(predicate));
1415                                                                         composed.add(support.getTransientId(object));
1416                                                                         if(LOG) log("[COMPOSED] (internal object) " + NameUtils.toIdString(graph, s));
1417                                                                 } else if (l0.InstanceOf.equalsResource(predicate)) {
1418                                                                         composed.add(support.getTransientId(predicate));
1419                                                                         composed.add(support.getTransientId(object));
1420                                                                         if(LOG) log("[COMPOSED] (instanceOf) " + NameUtils.toIdString(graph, s));
1421                                                                 } else if (l0.SubrelationOf.equalsResource(predicate)) {
1422                                                                         composed.add(support.getTransientId(predicate));
1423                                                                         composed.add(support.getTransientId(object));
1424                                                                         if(LOG) log("[COMPOSED] (subrelationOf) " + NameUtils.toIdString(graph, s));
1425                                                                 } else {
1426                                                                         if(objectStatus == ExtentStatus.EXTERNAL) {
1427                                                                                 if(DEBUG)
1428                                                                                         System.out.println("other " + NameUtils.toIdString(graph, s));
1429                                                                                         //System.out.println("other.add " + predicate + " - " + object);
1430                                                                                 other.add(support.getTransientId(predicate));
1431                                                                                 other.add(support.getTransientId(object));
1432                                                                                 if(LOG) log("[OTHER] (object is external) " + NameUtils.toIdString(graph, s));
1433                                                                         }
1434                                                                 }
1435
1436                                                         }
1437
1438                                                         if(!other.isEmpty()) {
1439                                                                 otherStatementsOutput.writeInt(sId);
1440                                                                 otherStatementsOutput.writeInt(other.size() / 2);
1441                                                                 for (int i = 0; i < other.size(); i++)
1442                                                                         otherStatementsOutput.writeInt(other.getQuick(i));
1443                                                                 other.resetQuick();
1444                                                         }
1445
1446                                                         composedStatementsOutput.writeInt(composed.size() / 2);
1447                                                         for (int i = 0; i < composed.size(); i++)
1448                                                                 composedStatementsOutput.writeInt(composed.getQuick(i));
1449
1450                                                 }
1451
1452                         } catch (IOException e) {
1453                                 e.printStackTrace();
1454                         }
1455
1456                         otherStatementTime += (System.nanoTime() - start);
1457
1458                 }
1459
1460                 boolean hasStrictParents(ReadGraph g, Resource r)
1461                                 throws DatabaseException {
1462                         if (g.getPossibleURI(r) != null)
1463                                 return true;
1464                         return false;
1465                 }
1466
1467                 public boolean getExpansionSeedsFromExtents(ReadGraph graph, final Collection<DirectStatements>[] expansion) throws DatabaseException {
1468
1469                         long start = System.nanoTime();
1470
1471                         final ConcurrentLinkedQueue<Resource> accepts = new ConcurrentLinkedQueue<Resource>();
1472
1473                         /*
1474                          * Determine statements which could accept statements with todo
1475                          * objects
1476                          */
1477                         search: for (Double priority : priorityList) {
1478
1479                                 for (final SubgraphAdvisor advisor : advisors) {
1480
1481                                         if (advisor.priority() > 0)
1482                                                 continue;
1483
1484                                         if (advisor.priority() == priority) {
1485
1486                                                 graph.syncRequest(new ReadRequest() {
1487
1488                                                         @Override
1489                                                         public void run(ReadGraph graph) throws DatabaseException {
1490
1491                                                                 for (Collection<DirectStatements> coll : expansion)
1492                                                                         for (DirectStatements stms : coll)
1493                                                                                 for(final Statement stm : stms) {
1494
1495                                                                                         advisor.advise(graph, stm, new AsyncProcedure<Boolean>() {
1496
1497                                                                                                 @Override
1498                                                                                                 public void exception(AsyncReadGraph graph, Throwable throwable) {
1499                                                                                                         throwable.printStackTrace();
1500                                                                                                 }
1501
1502                                                                                                 @Override
1503                                                                                                 public void execute(AsyncReadGraph graph, Boolean accept) {
1504                                                                                                         if (accept) {
1505                                                                                                                 accepts.add(stm.getObject());
1506                                                                                                         }
1507                                                                                                 }
1508
1509                                                                                         });
1510
1511                                                                                 }
1512
1513                                                         }
1514
1515                                                 });
1516                                         }
1517                                         if (!accepts.isEmpty())
1518                                                 break search;
1519                                 }
1520                         }
1521
1522                         CollectionSupport cs = graph.getService(CollectionSupport.class);
1523                         Set<Resource> schedule = cs.createSet();
1524                         for (Resource r : accepts) {
1525                                 if(!status.containsKey(r))
1526                                         schedule.add(r);
1527                         }
1528
1529                         extentSeedTime += (System.nanoTime() - start);
1530
1531                         if (schedule.isEmpty())
1532                                 return false;
1533
1534                         fastResolve(graph, schedule);
1535                         uriResolve(graph, schedule);
1536                         fullResolve(graph, schedule, "accepts");
1537
1538                         return true;
1539
1540                 }
1541
1542                 ConcurrentLinkedQueue<Resource> fastInternals = new ConcurrentLinkedQueue<Resource>();
1543
1544                 public void fastResolve(ReadGraph graph, final Set<Resource> rs)
1545                                 throws DatabaseException {
1546                         // This collects and resolves weaks
1547                         if(fastResolveLoop(graph, rs))
1548                                 // Weaks are now resolved
1549                                 fastResolveLoop(graph, rs);
1550                 }
1551
1552                 public boolean fastResolveLoop(ReadGraph graph, final Set<Resource> rs)
1553                                 throws DatabaseException {
1554
1555                         long start = System.nanoTime();
1556
1557                         final ConcurrentLinkedQueue<Resource> weakSchedule = new ConcurrentLinkedQueue<Resource>();
1558
1559                         graph.syncRequest(new AsyncRead<Boolean>() {
1560
1561                             @Override
1562                             public int threadHash() {
1563                                 return hashCode();
1564                             }
1565
1566                                 @Override
1567                                 public int getFlags() {
1568                                         return 0;
1569                                 }
1570
1571                                 @Override
1572                                 public void perform(AsyncReadGraph graph,
1573                                                 AsyncProcedure<Boolean> procedure) {
1574
1575                                         QueryControl control = graph.getService(QueryControl.class);
1576                                         final DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class);
1577
1578                                         int slice = (int) (rs.size() / control
1579                                                         .getAmountOfQueryThreads()) + 1;
1580
1581                                         final Resource[] rootArray = rs.toArray(Resource.NONE);
1582                                         for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
1583
1584                                                 final int start = i * slice;
1585                                                 final int end = Math.min(start + slice,
1586                                                                 rootArray.length);
1587
1588                                                 control.schedule(graph, i, new ControlProcedure() {
1589
1590                                                         @Override
1591                                                         public void execute(AsyncReadGraph graph) {
1592
1593                                                                 for (int index = start; index < end; index++) {
1594
1595                                                                         final Resource r = rootArray[index];
1596
1597                                                                         //if (status.containsKey(r)) continue;
1598
1599                                                                         graph.asyncRequest(new FastInternalRequest(dqs, r, status, weakInverses, weakSchedule),new AsyncProcedure<Boolean>() {
1600
1601                                                                                 @Override
1602                                                                                 public void exception(AsyncReadGraph graph,Throwable throwable) {
1603                                                                                         throwable.printStackTrace();
1604                                                                                 }
1605
1606                                                                                 @Override
1607                                                                                 public void execute(AsyncReadGraph graph,Boolean isInternal) {
1608                                                                                         if (isInternal) {
1609                                                                                                 fastInternals.add(r);
1610                                                                                         }
1611                                                                                 }
1612
1613                                                                         });
1614
1615                                                                 }
1616
1617                                                         }
1618
1619                                                 });
1620
1621                                         }
1622
1623                                         procedure.execute(graph, true);
1624
1625                                 }
1626
1627                         });
1628
1629                         if (!weakSchedule.isEmpty()) {
1630                                 THashSet<Resource> weaks = new THashSet<Resource>(weakSchedule);
1631                                 if (CLASSIFY_LOG)
1632                                         for (Resource p : weakSchedule)
1633                                                 System.out.println("classify "
1634                                                                 + NameUtils.getSafeName(graph, p));
1635                                 graph.syncRequest(new ClassifyStatementsRequest(weaks, weakInverses));
1636                         }
1637
1638                         for (Resource r : fastInternals) {
1639                                 rs.remove(r);
1640                                 if (status.put(r, ExtentStatus.INTERNAL) == null) {
1641                                         if(LOG) log("[INTERNAL] (fast) " + NameUtils.getSafeName(graph, r, true));
1642                                         ids.put(r, id++);
1643                                         fastInternalCounter++;
1644                                         expansionSeeds.add(r);
1645                                 }
1646                         }
1647
1648                         fastResolveTime += (System.nanoTime() - start);
1649
1650                         return !weakSchedule.isEmpty();
1651
1652                 }
1653
1654                 private ExtentStatus resolveExtent(ReadGraph graph, Resource r, Map<Resource, ExtentStatus> statuses, Set<Resource> expansionSeeds, THashSet<Resource> pending,
1655                                 ArrayList<Resource> stack) throws DatabaseException {
1656
1657                         ExtentStatus current = statuses.get(r);
1658                         if(current != null) return current;
1659
1660                         if (pending.contains(r))
1661                                 return ExtentStatus.PENDING;
1662
1663                         // In order to break cyclic dependencies
1664                         pending.add(r);
1665
1666                         if (PARENT_DEBUG)
1667                                 System.out.println("resolveExtent "
1668                                                 + NameUtils.getSafeName(graph, r));
1669
1670                         ExtentStatus status = ExtentStatus.INTERNAL;
1671                         for (Resource p : getParents(graph, r)) {
1672                                 if (PARENT_DEBUG) {
1673                                         ExtentStatus ps = statuses.get(p);
1674                                         System.out.println(" parent " + NameUtils.getSafeName(graph, p) + "(" + ps + ")");
1675                                 }
1676                                 switch (resolveExtent(graph, p, statuses,
1677                                                 expansionSeeds, pending, stack)) {
1678                                 case EXTERNAL:
1679                                         return ExtentStatus.EXTERNAL;
1680                                 case PENDING:
1681                                         status = ExtentStatus.PENDING;
1682                                 }
1683                         }
1684                         if (status == ExtentStatus.INTERNAL) {
1685                                 pending.remove(r);
1686                                 stack.add(r);
1687                                 if (DEBUG)
1688                                         System.out.println(NameUtils.getSafeName(graph, r, true)
1689                                                         + " is internal.");
1690                         }
1691                         return status;
1692                 }
1693
1694                 public void uriResolve(ReadGraph graph, final Set<Resource> todo)
1695                                 throws DatabaseException {
1696
1697                         long start = System.nanoTime();
1698
1699                         for(Resource r : todo) System.out.println("uriResolve " +
1700                                         NameUtils.getSafeName(graph, r));
1701
1702                         final ConcurrentSkipListSet<Resource> found = new ConcurrentSkipListSet<Resource>();
1703
1704                         graph.syncRequest(new AsyncReadRequest() {
1705
1706                                 @Override
1707                                 public void run(AsyncReadGraph graph) {
1708
1709                                         for (final Resource r : todo) {
1710
1711                                                 // System.out.println("uriresolve before " + r);
1712
1713                                                 if (status.containsKey(r)) continue;
1714
1715                                                 // System.out.println("uriresolve " + r);
1716
1717                                                 graph.forURI(r, new AsyncProcedure<String>() {
1718
1719                                                         @Override
1720                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {
1721                                                                 throwable.printStackTrace();
1722                                                         }
1723
1724                                                         @Override
1725                                                         public void execute(AsyncReadGraph graph, String uri) {
1726
1727                                                                 if (uri != null) {
1728
1729                                                                         // System.out.println("uriresolve has uri "
1730                                                                         // + r);
1731
1732                                                                         if(found.add(r)) {
1733                                                                                 parentExternalCounter++;
1734                                                                         }
1735
1736                                                                 } else {
1737
1738                                                                         // System.out.println("uriresolve ask inverse "
1739                                                                         // + r);
1740
1741                                                                         graph.forPossibleInverse(r, new AsyncProcedure<Resource>() {
1742
1743                                                                                 @Override
1744                                                                                 public void exception(AsyncReadGraph graph, Throwable throwable) {
1745                                                                                         throwable.printStackTrace();
1746                                                                                 }
1747
1748                                                                                 @Override
1749                                                                                 public void execute(AsyncReadGraph graph, Resource inverse) {
1750
1751                                                                                         if (inverse != null) {
1752
1753                                                                                                 graph.forURI(inverse, new AsyncProcedure<String>() {
1754
1755                                                                                                         @Override
1756                                                                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {
1757                                                                                                                 throwable.printStackTrace();
1758                                                                                                         }
1759
1760                                                                                                         @Override
1761                                                                                                         public void execute(AsyncReadGraph graph, String uri) {
1762
1763                                                                                                                 if (uri != null) {
1764
1765                                                                                                                         if(found.add(r)) {
1766                                                                                                                                 parentExternalCounter++;
1767                                                                                                                         }
1768
1769                                                                                                                 }
1770
1771                                                                                                         }
1772
1773                                                                                                 });
1774
1775                                                                                         }
1776
1777                                                                                 }
1778
1779                                                                         });
1780
1781                                                                 }
1782                                                         }
1783
1784                                                 });
1785
1786                                         }
1787
1788                                 }
1789
1790                         });
1791
1792                         todo.removeAll(found);
1793                         for(Resource r : found) {
1794                                 status.put(r, ExtentStatus.EXTERNAL);
1795                                 if(LOG) log("[EXTERNAL] (uriResolve) " + NameUtils.getSafeName(graph, r, true));
1796                         }
1797
1798                         parentResolveTime += System.nanoTime() - start;
1799
1800                 }
1801
1802                 public void fullResolve(ReadGraph graph, Collection<Resource> rs,
1803                                 String koss) throws DatabaseException {
1804
1805                         long start = System.nanoTime();
1806
1807                         for (final Resource r : rs) {
1808
1809                                 if(status.containsKey(r)) continue;
1810
1811                                 THashSet<Resource> pending = new THashSet<Resource>();
1812                                 ArrayList<Resource> stack = new ArrayList<Resource>();
1813
1814                                 ExtentStatus s = resolveExtent(graph, r, status, expansionSeeds, pending, stack);
1815                                 if (ExtentStatus.INTERNAL == s || ExtentStatus.PENDING == s) {
1816                                         if (status.put(r, ExtentStatus.INTERNAL) == null) {
1817                                                 if(LOG) log("[INTERNAL] (resolveExtent) " + NameUtils.getSafeName(graph, r, true));
1818                                                 ids.put(r, id++);
1819                                                 fullInternalCounter++;
1820                                                 expansionSeeds.add(r);
1821                                         }
1822                                 }
1823                                 if (ExtentStatus.EXTERNAL == s) {
1824                                         if (status.put(r, ExtentStatus.EXTERNAL) == null) {
1825                                                 if(LOG) log("[EXTERNAL] (resolveExtent) " + NameUtils.getSafeName(graph, r, true));
1826                                                 fullExternalCounter++;
1827                                         }
1828                                 }
1829
1830                         }
1831
1832                         fullResolveTime += (System.nanoTime() - start);
1833
1834                 }
1835
1836                 public void process(ReadGraph graph,
1837                                 ObjectOutputStream composedStatementsOutput,
1838                                 ObjectOutputStream otherStatementsOutput,
1839                                 ObjectOutputStream valueOutput)
1840                                 throws DatabaseException {
1841
1842                         this.variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT);
1843
1844                         CollectionSupport cs = graph.getService(CollectionSupport.class);
1845
1846                         Set<Resource> typeTodo = cs.createSet();
1847                         Set<Resource> objectTodo = cs.createSet();
1848                         Set<Resource> predicateTodo = cs.createSet();
1849
1850                         Collection<Collection<DirectStatements>[]> fullExpansion = new ArrayList<Collection<DirectStatements>[]>();
1851
1852                         do {
1853
1854                                 QueryControl control = graph.getService(QueryControl.class);
1855                                 Collection<DirectStatements>[] expansion = new ArrayList[control.getAmountOfQueryThreads()];
1856                                 for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
1857                                         expansion[i] = new ArrayList<DirectStatements>();
1858                                 }
1859
1860                                 // Expand expansionSeeds
1861                                 expand(graph, expansion, objectTodo);
1862
1863                                 // Start collecting new seeds
1864                                 expansionSeeds = cs.createSet();
1865
1866                                 // Collect predicates which are <R IsComposedOf
1867                                 extractComposedPredicates(graph, expansion);
1868
1869                                 // Make composed objects internal and make sure they are further
1870                                 // expanded
1871                                 collectComposedObjects(graph, expansion, typeTodo, objectTodo, predicateTodo);
1872
1873                                 /*
1874                                  * Use the expansion seed heuristic to find new resources to
1875                                  * expand before full analysis.
1876                                  */
1877                                 getExpansionSeedsFromExtents(graph, expansion);
1878
1879                                 fullExpansion.add(expansion);
1880
1881                         } while (!expansionSeeds.isEmpty());
1882
1883                         fastResolve(graph, objectTodo);
1884                         uriResolve(graph, predicateTodo);
1885                         uriResolve(graph, objectTodo);
1886                         uriResolve(graph, typeTodo);
1887                         fullResolve(graph, objectTodo, "objectTodo");
1888                         fullResolve(graph, predicateTodo, "predicateTodo");
1889                         fullResolve(graph, typeTodo, "typeTodo");
1890
1891                         writeOtherStatements(graph, fullExpansion, composedStatementsOutput, otherStatementsOutput, valueOutput);
1892
1893                         if (PROFILE) {
1894                                 System.out.println(composedObjectCounter + " " + fastInternalCounter
1895                                                 + " " + parentExternalCounter + " "
1896                                                 + fullExternalCounter + " " + fullInternalCounter);
1897                         }
1898
1899                 }
1900
1901         }
1902
1903         public static void getDomain2(ReadGraph graph, TIntIntHashMap ids,
1904                         Collection<Resource> roots, Map<Resource, ExtentStatus> preStatus,
1905                         Map<Resource, Statement> specials,
1906                         ObjectOutputStream otherStatementsOutput,
1907                         ObjectOutputStream valueOutput,
1908                         TreeMap<String, Variant> extensions,
1909                         TIntHashSet excludedShared) throws DatabaseException {
1910
1911                 ITask task = ThreadLogger.getInstance().begin("getDomain2");
1912
1913                 final DomainProcessor2 processor = new DomainProcessor2();
1914
1915                 processor.startupTime = System.nanoTime();
1916
1917                 Layer0 l0 = Layer0.getInstance(graph);
1918
1919                 CollectionSupport cs = graph.getService(CollectionSupport.class);
1920                 SerialisationSupport support = graph.getService(SerialisationSupport.class);
1921
1922                 processor.ids = ids;
1923                 processor.specials = specials;
1924                 processor.status = cs.createMap(ExtentStatus.class);
1925                 processor.weakInverses = cs.createMap(WeakStatus.class);
1926                 processor.predicates = cs.createSet();
1927                 processor.isRelatedToPredicates = cs.createSet();
1928                 processor.sharedPredicates = cs.createSet();
1929 //              processor.expansionSeeds = cs.createSet();
1930
1931                 for(Map.Entry<Resource, ExtentStatus> entry : preStatus.entrySet()) {
1932                         processor.status.put(entry.getKey(), entry.getValue());
1933                         if(ExtentStatus.EXCLUDED.equals(entry.getValue())) processor.exclusions.add(entry.getKey());
1934                 }
1935
1936 //              for (Resource r : excluded) {
1937 //                      processor.status.put(r, ExtentStatus.EXCLUDED);
1938 //              }
1939
1940                 Resource rootLibrary = graph.getResource("http:/");
1941
1942                 if (!roots.contains(rootLibrary))
1943                         processor.status.put(rootLibrary, ExtentStatus.EXTERNAL);
1944
1945                 for (Resource root : roots) {
1946                         processor.status.put(root, ExtentStatus.INTERNAL);
1947                         //processor.ids.put(support.getTransientId(root), processor.ids.size());
1948                         for (Resource owner : graph.getObjects(root, l0.IsOwnedBy)) {
1949                                 processor.status.put(owner, ExtentStatus.EXTERNAL);
1950                         }
1951                 }
1952
1953                 processor.startupTime = System.nanoTime() - processor.startupTime;
1954
1955                 processor.fringe = new HashSet<Resource>();
1956                 processor.fringe.addAll(roots);
1957
1958                 processor.internalDomain.addAll(roots);
1959
1960                 processor.sharedExternalReferences = new HashSet<Resource>();
1961                 processor.sharedExternalFringe = new HashSet<Resource>();
1962
1963                 try {
1964
1965                         processor.process(graph, otherStatementsOutput, valueOutput);
1966
1967                 } catch (IOException e) {
1968                         e.printStackTrace();
1969                 }
1970
1971                 for(Resource r : processor.sharedExternalReferences) excludedShared.add(support.getTransientId(r));
1972
1973         ClusteringSupport cls = graph.getService(ClusteringSupport.class);
1974         TLongObjectHashMap<TIntArrayList> clusterMap = new TLongObjectHashMap<TIntArrayList>();
1975                 for(Map.Entry<Resource, ExtentStatus> entry : processor.status.entrySet()) {
1976                         if(ExtentStatus.INTERNAL == entry.getValue()) {
1977
1978                                 long cluster = cls.getCluster(entry.getKey());
1979                                 TIntArrayList list = clusterMap.get(cluster);
1980                                 if(list == null) {
1981                                         list = new TIntArrayList();
1982                                         clusterMap.put(cluster, list);
1983                                 }
1984                                 list.add(support.getTransientId(entry.getKey()));
1985
1986                         }
1987                 }
1988                 final TIntArrayList clustering = new TIntArrayList();
1989                 clusterMap.forEachEntry(new TLongObjectProcedure<TIntArrayList>() {
1990
1991                         @Override
1992                         public boolean execute(long cluster, TIntArrayList b) {
1993                                 clustering.add(b.size());
1994                                 b.forEach(new TIntProcedure() {
1995
1996                                         @Override
1997                                         public boolean execute(int rId) {
1998                                                 processor.ids.put(rId, processor.id++);
1999                                                 return true;
2000                                         }
2001
2002                                 });
2003                                 return true;
2004                         }
2005
2006                 });
2007
2008                 extensions.put(Extensions.CLUSTERING, new Variant(Bindings.INT_ARRAY, clustering.toArray()));
2009
2010                 long total = processor.startupTime + processor.expandTime
2011                                 + processor.composedPredicateTime
2012                                 + processor.composedObjectTime + processor.extentSeedTime
2013                                 + processor.fullResolveTime + processor.fastResolveTime +
2014                                 + processor.parentResolveTime + processor.otherStatementTime;
2015
2016                 if (PROFILE) {
2017                         System.out.println("startup took " + 1e-9 * processor.startupTime
2018                                         + "s.");
2019                         System.out.println("expand took " + 1e-9 * processor.expandTime
2020                                         + "s.");
2021                         System.out.println("composedPredicates took " + 1e-9
2022                                         * processor.composedPredicateTime + "s.");
2023                         System.out.println("composedObjects took " + 1e-9
2024                                         * processor.composedObjectTime + "s.");
2025                         System.out.println("extentSeeding took " + 1e-9
2026                                         * processor.extentSeedTime + "s.");
2027                         System.out.println("fullResolve took " + 1e-9
2028                                         * processor.fullResolveTime + "s.");
2029                         System.out.println("fastResolve took " + 1e-9
2030                                         * processor.fastResolveTime + "s.");
2031                         System.out.println("parentResolve took " + 1e-9
2032                                         * processor.parentResolveTime + "s.");
2033                         System.out.println("otherStatements took " + 1e-9
2034                                         * processor.otherStatementTime + "s.");
2035                         System.out.println("total " + 1e-9 * total + "s.");
2036                 }
2037
2038                 task.finish();
2039
2040
2041         }
2042
2043
2044         public static void getDomain(ReadGraph graph, Map<Resource, Integer> ids,
2045                         Collection<Resource> roots, Map<Resource, ExtentStatus> preStatus, Set<SubgraphAdvisor> advisors,
2046                         ObjectOutputStream composedStatementsOutput,
2047                         ObjectOutputStream otherStatementsOutput,
2048                         ObjectOutputStream valueOutput) throws DatabaseException {
2049
2050                 ITask task = ThreadLogger.getInstance().begin("getDomain");
2051
2052                 DomainProcessor processor = new DomainProcessor(advisors);
2053
2054                 processor.startupTime = System.nanoTime();
2055
2056                 Layer0 l0 = Layer0.getInstance(graph);
2057
2058                 CollectionSupport cs = graph.getService(CollectionSupport.class);
2059
2060                 processor.ids = ids;
2061                 processor.status = cs.createMap(ExtentStatus.class);
2062                 processor.weakInverses = cs.createMap(WeakStatus.class);
2063                 processor.predicates = cs.createSet();
2064                 processor.composedPredicates = cs.createSet();
2065                 processor.expansionSeeds = cs.createSet();
2066
2067                 for(Map.Entry<Resource, ExtentStatus> entry : preStatus.entrySet()) {
2068                         processor.status.put(entry.getKey(), entry.getValue());
2069                 }
2070
2071 //              for (Resource r : excluded) {
2072 //                      processor.status.put(r, ExtentStatus.EXCLUDED);
2073 //              }
2074
2075                 if (!roots.contains(graph.getRootLibrary()))
2076                         processor.status.put(graph.getRootLibrary(), ExtentStatus.EXTERNAL);
2077
2078                 for (Resource root : roots) {
2079                         processor.status.put(root, ExtentStatus.INTERNAL);
2080                         processor.ids.put(root, processor.id++);
2081                         for (Resource owner : graph.getObjects(root, l0.IsOwnedBy)) {
2082                                 processor.status.put(owner, ExtentStatus.EXTERNAL);
2083                         }
2084                 }
2085
2086
2087                 processor.expansionSeeds.addAll(roots);
2088
2089                 processor.startupTime = System.nanoTime() - processor.startupTime;
2090
2091                 while (!processor.expansionSeeds.isEmpty()) {
2092
2093                         processor.process(graph, composedStatementsOutput,
2094                                         otherStatementsOutput, valueOutput);
2095
2096                 }
2097
2098                 long total = processor.startupTime + processor.expandTime
2099                                 + processor.composedPredicateTime
2100                                 + processor.composedObjectTime + processor.extentSeedTime
2101                                 + processor.fullResolveTime + processor.fastResolveTime +
2102                                 + processor.parentResolveTime + processor.otherStatementTime;
2103
2104                 if (PROFILE) {
2105                         System.out.println("startup took " + 1e-9 * processor.startupTime
2106                                         + "s.");
2107                         System.out.println("expand took " + 1e-9 * processor.expandTime
2108                                         + "s.");
2109                         System.out.println("composedPredicates took " + 1e-9
2110                                         * processor.composedPredicateTime + "s.");
2111                         System.out.println("composedObjects took " + 1e-9
2112                                         * processor.composedObjectTime + "s.");
2113                         System.out.println("extentSeeding took " + 1e-9
2114                                         * processor.extentSeedTime + "s.");
2115                         System.out.println("fullResolve took " + 1e-9
2116                                         * processor.fullResolveTime + "s.");
2117                         System.out.println("fastResolve took " + 1e-9
2118                                         * processor.fastResolveTime + "s.");
2119                         System.out.println("parentResolve took " + 1e-9
2120                                         * processor.parentResolveTime + "s.");
2121                         System.out.println("otherStatements took " + 1e-9
2122                                         * processor.otherStatementTime + "s.");
2123                         System.out.println("total " + 1e-9 * total + "s.");
2124                 }
2125
2126                 task.finish();
2127
2128         }
2129
2130 }