bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/Subgraphs.java

   1 /*******************************************************************************\r
   2  * Copyright (c) 2007, 2010 Association for Decentralized Information Management\r
   3  * in Industry THTH ry.\r
   4  * All rights reserved. This program and the accompanying materials\r
   5  * are made available under the terms of the Eclipse Public License v1.0\r
   6  * which accompanies this distribution, and is available at\r
   7  * http://www.eclipse.org/legal/epl-v10.html\r
   8  *\r
   9  * Contributors:\r
  10  *     VTT Technical Research Centre of Finland - initial API and implementation\r
  11  *******************************************************************************/\r
  12 package org.simantics.db.layer0.util;\r
  13 \r
  14 import gnu.trove.list.array.TIntArrayList;\r
  15 import gnu.trove.map.hash.TIntIntHashMap;\r
  16 import gnu.trove.map.hash.TLongObjectHashMap;\r
  17 import gnu.trove.procedure.TIntProcedure;\r
  18 import gnu.trove.procedure.TLongObjectProcedure;\r
  19 import gnu.trove.set.TIntSet;\r
  20 import gnu.trove.set.hash.THashSet;\r
  21 import gnu.trove.set.hash.TIntHashSet;\r
  22 \r
  23 import java.io.DataOutput;\r
  24 import java.io.DataOutputStream;\r
  25 import java.io.FileNotFoundException;\r
  26 import java.io.FileOutputStream;\r
  27 import java.io.IOException;\r
  28 import java.io.ObjectOutputStream;\r
  29 import java.util.ArrayList;\r
  30 import java.util.Collection;\r
  31 import java.util.Collections;\r
  32 import java.util.HashSet;\r
  33 import java.util.Map;\r
  34 import java.util.Set;\r
  35 import java.util.TreeMap;\r
  36 import java.util.concurrent.ConcurrentLinkedQueue;\r
  37 import java.util.concurrent.ConcurrentSkipListSet;\r
  38 \r
  39 import org.simantics.databoard.Bindings;\r
  40 import org.simantics.databoard.Databoard;\r
  41 import org.simantics.databoard.binding.Binding;\r
  42 import org.simantics.databoard.binding.mutable.Variant;\r
  43 import org.simantics.databoard.serialization.Serializer;\r
  44 import org.simantics.databoard.type.Datatype;\r
  45 import org.simantics.db.AsyncReadGraph;\r
  46 import org.simantics.db.DirectStatements;\r
  47 import org.simantics.db.ReadGraph;\r
  48 import org.simantics.db.Resource;\r
  49 import org.simantics.db.Statement;\r
  50 import org.simantics.db.common.request.AsyncReadRequest;\r
  51 import org.simantics.db.common.request.ReadRequest;\r
  52 import org.simantics.db.common.request.ResourceAsyncRead;\r
  53 import org.simantics.db.common.utils.NameUtils;\r
  54 import org.simantics.db.exception.DatabaseException;\r
  55 import org.simantics.db.layer0.adapter.SubgraphAdvisor;\r
  56 import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus;\r
  57 import org.simantics.db.procedure.AsyncProcedure;\r
  58 import org.simantics.db.request.AsyncRead;\r
  59 import org.simantics.db.service.ClusteringSupport;\r
  60 import org.simantics.db.service.CollectionSupport;\r
  61 import org.simantics.db.service.DirectQuerySupport;\r
  62 import org.simantics.db.service.QueryControl;\r
  63 import org.simantics.db.service.QueryControl.ControlProcedure;\r
  64 import org.simantics.db.service.SerialisationSupport;\r
  65 import org.simantics.db.service.StatementSupport;\r
  66 import org.simantics.db.service.TransferableGraphSupport;\r
  67 import org.simantics.graph.representation.Extensions;\r
  68 import org.simantics.layer0.Layer0;\r
  69 import org.simantics.utils.datastructures.Pair;\r
  70 import org.simantics.utils.threads.logger.ITask;\r
  71 import org.simantics.utils.threads.logger.ThreadLogger;\r
  72 \r
  73 public class Subgraphs {\r
  74 \r
  75         public static String LOG_FILE = "export.log";\r
  76         final static private boolean LOG = false;\r
  77         final static private boolean DEBUG = false;\r
  78         final static private boolean PARENT_DEBUG = DEBUG | false;\r
  79         final static private boolean EXTERNAL_DEBUG = DEBUG | false;\r
  80         final static private boolean ADVISOR_LOG = LOG & false;\r
  81         final static private boolean EXPANSION_LOG = LOG & false;\r
  82         final static private boolean INTERNAL_LOG = LOG & false;\r
  83         final static private boolean COMPOSED_LOG = LOG & false;\r
  84         final static private boolean RESOLVE_LOG = LOG & false;\r
  85         final static private boolean CLASSIFY_LOG = LOG & false;\r
  86         final static private boolean EXTERNAL_LOG = LOG & false;\r
  87         final static private boolean PROFILE = false;\r
  88 \r
  89         static enum WeakStatus {\r
  90                 STRONG, WEAK\r
  91         }\r
  92 \r
  93         \r
  94         static DataOutput log;\r
  95 \r
  96         static {\r
  97 \r
  98                 if (LOG) {\r
  99                         try {\r
 100                                 FileOutputStream stream = new FileOutputStream(LOG_FILE, false);\r
 101                                 log = new DataOutputStream(stream);\r
 102                         } catch (FileNotFoundException e) {\r
 103                                 e.printStackTrace();\r
 104                         }\r
 105                 }\r
 106 \r
 107         }\r
 108         \r
 109         private static void log(String line) {\r
 110                 if (LOG) {\r
 111                         try {\r
 112                                 log.write((line + "\n").getBytes());\r
 113                         } catch (IOException e) {\r
 114                                 e.printStackTrace();\r
 115                         }\r
 116                 }\r
 117         }\r
 118 \r
 119         public static Collection<Resource> getParents(ReadGraph g, Resource r)\r
 120                         throws DatabaseException {\r
 121                 return getParents(g, r, false);\r
 122         }\r
 123 \r
 124         static class FastInternalRequest extends ResourceAsyncRead<Boolean> {\r
 125 \r
 126                 final DirectQuerySupport dqs;\r
 127                 final ConcurrentLinkedQueue<Resource> queue;\r
 128                 final Map<Resource, WeakStatus> weakInverses;\r
 129                 final Map<Resource, ExtentStatus> status;\r
 130 \r
 131                 public FastInternalRequest(DirectQuerySupport dqs, Resource resource,\r
 132                                 Map<Resource, ExtentStatus> status,\r
 133                                 Map<Resource, WeakStatus> weakInverses,\r
 134                                 ConcurrentLinkedQueue<Resource> queue) {\r
 135                         super(resource);\r
 136                         this.dqs = dqs;\r
 137                         this.status = status;\r
 138                         this.weakInverses = weakInverses;\r
 139                         this.queue = queue;\r
 140                 }\r
 141 \r
 142                 @Override\r
 143                 public int getFlags() {\r
 144                         return 0;\r
 145                 }\r
 146 \r
 147                 @Override\r
 148                 public void perform(AsyncReadGraph graph, final AsyncProcedure<Boolean> procedure) {\r
 149 \r
 150                         dqs.forEachDirectStatement(graph, resource, new AsyncProcedure<DirectStatements>() {\r
 151 \r
 152                                 @Override\r
 153                                 public void execute(AsyncReadGraph graph, DirectStatements ss) {\r
 154                                         boolean ok = true;\r
 155                                         for(Statement statement : ss) {\r
 156                                                 if (status.get(statement.getObject()) == ExtentStatus.INTERNAL) continue;\r
 157                                                 WeakStatus status = weakInverses.get(statement.getPredicate()); \r
 158                                                 if(status == WeakStatus.WEAK) continue;\r
 159                                                 else if (status == null) {\r
 160                                                         queue.add(statement.getPredicate());\r
 161                                                 }\r
 162                                                 ok = false;\r
 163                                         }\r
 164                                         procedure.execute(graph, ok);\r
 165                                 }\r
 166 \r
 167                                 @Override\r
 168                                 public void exception(AsyncReadGraph graph, Throwable throwable) {\r
 169                                         throwable.printStackTrace();\r
 170                                 }\r
 171 \r
 172                         });\r
 173                         \r
 174                 }\r
 175 \r
 176         }\r
 177 \r
 178         static class ClassifyStatementsRequest implements AsyncRead<Boolean> {\r
 179 \r
 180                 final Set<Resource> schedule;\r
 181                 final Map<Resource, WeakStatus> weakMap;\r
 182 \r
 183                 public ClassifyStatementsRequest(Set<Resource> schedule, Map<Resource, WeakStatus> weakMap) {\r
 184                         this.weakMap = weakMap;\r
 185                         this.schedule = schedule;\r
 186                 }\r
 187 \r
 188             @Override\r
 189             public int threadHash() {\r
 190                 return hashCode();\r
 191             }\r
 192                 \r
 193                 @Override\r
 194                 public int getFlags() {\r
 195                         return 0;\r
 196                 }\r
 197 \r
 198                 @Override\r
 199                 public void perform(AsyncReadGraph graph, final AsyncProcedure<Boolean> procedure) {\r
 200 \r
 201                         for (final Resource p : schedule) {\r
 202 \r
 203                                 graph.forPossibleInverse(p, new AsyncProcedure<Resource>() {\r
 204 \r
 205                                         private void register(AsyncReadGraph graph, Resource predicate, Resource superRelation, WeakStatus status) {\r
 206                                                 synchronized (weakMap) {\r
 207                                                         weakMap.put(predicate, status);\r
 208                                                         if(superRelation != null) weakMap.put(superRelation, status);\r
 209                                                 }\r
 210                                         }\r
 211 \r
 212                                         @Override\r
 213                                         public void exception(AsyncReadGraph graph, Throwable throwable) {\r
 214                                                 throwable.printStackTrace();\r
 215                                         }\r
 216 \r
 217                                         @Override\r
 218                                         public void execute(AsyncReadGraph graph, final Resource inverse) {\r
 219 \r
 220                                                 if (inverse == null) {\r
 221                                                         \r
 222                                                         register(graph, p, null, WeakStatus.WEAK);\r
 223                                                         \r
 224                                                 } else {\r
 225                                                         \r
 226                                                         graph.forPossibleSuperrelation(inverse, new AsyncProcedure<Resource>() {\r
 227 \r
 228                                                                 @Override\r
 229                                                                 public void exception(AsyncReadGraph graph, Throwable throwable) {\r
 230                                                                         throwable.printStackTrace();\r
 231                                                                 }\r
 232 \r
 233                                                                 @Override\r
 234                                                                 public void execute(AsyncReadGraph graph, final Resource superRelation) {\r
 235                                                                         \r
 236                                                                         if(superRelation != null && weakMap.containsKey(superRelation)) {\r
 237                                                                                 register(graph, p, null, weakMap.get(superRelation));\r
 238                                                                                 return;\r
 239                                                                         }\r
 240 \r
 241                                                                         graph.forIsSubrelationOf(inverse, graph.getService(Layer0.class).IsRelatedTo, new AsyncProcedure<Boolean>() {\r
 242 \r
 243                                                                                 @Override\r
 244                                                                                 public void exception(AsyncReadGraph graph,Throwable throwable) {\r
 245                                                                                         throwable.printStackTrace();\r
 246                                                                                 }\r
 247 \r
 248                                                                                 @Override\r
 249                                                                                 public void execute(AsyncReadGraph graph,Boolean strong) {\r
 250                                                                                         register(graph, p, superRelation, strong ? WeakStatus.STRONG : WeakStatus.WEAK);\r
 251                                                                                 }\r
 252 \r
 253                                                                         });\r
 254                                                                         \r
 255                                                                 }\r
 256                                                                 \r
 257                                                         });\r
 258                                                         \r
 259                                                 }\r
 260 \r
 261                                         }\r
 262 \r
 263                                 });\r
 264 \r
 265                         }\r
 266 \r
 267                         procedure.execute(graph, false);\r
 268 \r
 269                 }\r
 270 \r
 271         }\r
 272 \r
 273         private static Collection<Resource> getParents(ReadGraph g, Resource r, boolean isStrong) throws DatabaseException {\r
 274 \r
 275                 System.out.println("getParents " + NameUtils.getSafeName(g, r));\r
 276 \r
 277                 Layer0 l0 = Layer0.getInstance(g);\r
 278 \r
 279                 Collection<Resource> predicates = g.getPredicates(r);\r
 280 \r
 281                 // --- Consists Of ----------------------------------------------------\r
 282 \r
 283                 if (predicates.contains(l0.PartOf)) {\r
 284                         Collection<Resource> parents = g.getObjects(r, l0.PartOf);\r
 285                         if (parents.size() == 1)\r
 286                                 return parents;\r
 287                         ArrayList<Resource> libraryParents = new ArrayList<Resource>(1);\r
 288                         for (Resource p : parents)\r
 289                                 if (g.isInstanceOf(p, l0.Library))\r
 290                                         libraryParents.add(p);\r
 291                         if (!libraryParents.isEmpty())\r
 292                                 return libraryParents;\r
 293                         else\r
 294                                 return parents;\r
 295                 }\r
 296 \r
 297                 // --- Ordered sets ---------------------------------------------------\r
 298 \r
 299                 {\r
 300                         Collection<Resource> parents = null;\r
 301                         for (Resource p : predicates)\r
 302                                 if (g.isInstanceOf(p, l0.OrderedSet) && !p.equals(r)) {\r
 303                                         if (parents == null)\r
 304                                                 parents = new ArrayList<Resource>(1);\r
 305                                         parents.add(p);\r
 306                                 }\r
 307                         if (parents != null) {\r
 308                                 if (DEBUG)\r
 309                                         System.out.println("ORDERED SET");\r
 310                                 return parents;\r
 311                         }\r
 312                 }\r
 313 \r
 314 \r
 315                 if (isStrong)\r
 316                         return Collections.emptyList();\r
 317                 else {\r
 318 \r
 319                         if (predicates.contains(l0.InverseOf)) {\r
 320                                 \r
 321                                 Resource inv = g.getInverse(r);\r
 322                                 return getParents(g, inv, true);\r
 323                                 \r
 324                         } else {\r
 325                                 \r
 326                                 /*\r
 327                                  * Depends On\r
 328                                  * \r
 329                                  * If there are DependsOn parents, then IsRelatedTo parents are discarded\r
 330                                  * \r
 331                                  */\r
 332                                 HashSet<Resource> result = new HashSet<Resource>();\r
 333                                 for(Resource predicate : predicates) {\r
 334                                         if(g.isSubrelationOf(predicate, l0.IsDependencyOf)) result.addAll(g.getObjects(r, predicate));\r
 335                                 }\r
 336                                 if(!result.isEmpty()) return result;\r
 337                                 \r
 338                                 /*\r
 339                                  * Is Related To\r
 340                                  * \r
 341                                  * At this point all Is Related To are parents.\r
 342                                  * \r
 343                                  */\r
 344                                 for(Resource predicate : predicates) {\r
 345                                         Resource inv = g.getPossibleInverse(predicate);\r
 346                                         if(inv != null) {\r
 347                                                 if(g.isSubrelationOf(inv, l0.IsRelatedTo)) result.addAll(g.getObjects(r, predicate)); \r
 348                                         }\r
 349                                 }\r
 350                                 \r
 351                                 return result;\r
 352                                 \r
 353                         }\r
 354                         \r
 355                         /*\r
 356                         Collection<Resource> invR = g.getObjects(r, b.IsRelatedTo_Inverse);\r
 357                         if (predicates.contains(b.InverseOf)) {\r
 358                                 if (invR.size() > 1) {\r
 359                                         if (DEBUG)\r
 360                                                 System.out\r
 361                                                                 .println("###########################################");\r
 362                                         Resource inv = g.getInverse(r);\r
 363                                         Collection<Resource> ret = new ArrayList<Resource>();\r
 364                                         for (Statement pp : g.getStatements(r,\r
 365                                                         b.IsRelatedTo_Inverse))\r
 366                                                 if (!pp.getPredicate().equals(inv)) {\r
 367                                                         if (DEBUG) {\r
 368                                                                 System.out.println("<"\r
 369                                                                                 + NameUtils.getSafeName(g, pp\r
 370                                                                                                 .getSubject())\r
 371                                                                                 + ","\r
 372                                                                                 + NameUtils.getSafeName(g, pp\r
 373                                                                                                 .getPredicate())\r
 374                                                                                 + ","\r
 375                                                                                 + NameUtils.getSafeName(g, pp\r
 376                                                                                                 .getObject()) + ">");\r
 377                                                         }\r
 378                                                         ret.add(pp.getObject());\r
 379                                                 }\r
 380                                         return ret;\r
 381                                 }\r
 382                                 // System.out.println("?????????????????");\r
 383                                 Collection<Resource> invParents = getParents(g,\r
 384                                                 g.getInverse(r), true);\r
 385                                 if (!invParents.isEmpty())\r
 386                                         return invParents;\r
 387                         }\r
 388                         if (DEBUG) {\r
 389                                 System.out.print("invR");\r
 390                                 for (Resource res : invR)\r
 391                                         System.out.print(" " + NameUtils.getSafeName(g, res));\r
 392                                 System.out.println();\r
 393                         }\r
 394                         return invR;\r
 395                         */\r
 396                 }\r
 397 \r
 398         }\r
 399 \r
 400 //      public static String getIdentifier(ReadGraph g, Resource r)\r
 401 //                      throws DatabaseException {\r
 402 //      Layer0 L0 = Layer0.getInstance(g);\r
 403 //              if (r.equals(g.getRootLibrary()))\r
 404 //                      return "";\r
 405 //              String name = g.getPossibleRelatedValue(r, L0.HasName);\r
 406 //              if (name == null)\r
 407 //                      return null;\r
 408 //              Collection<Resource> parents = getParents(g, r, true);\r
 409 //              if (parents.size() != 1)\r
 410 //                      return null;\r
 411 //              for (Resource p : parents) {\r
 412 //                      String parentIdentifier = getIdentifier(g, p);\r
 413 //                      if (parentIdentifier == null)\r
 414 //                              return null;\r
 415 //                      return parentIdentifier + "/" + name;\r
 416 //              }\r
 417 //              return null;\r
 418 //      }\r
 419 \r
 420         static int kess = 0;\r
 421 \r
 422         static class Expansion extends AsyncReadRequest {\r
 423 \r
 424                 final private Collection<Resource> roots;\r
 425                 final Collection<DirectStatements>[] results;\r
 426                 final Collection<Resource>[] listElements;\r
 427                 \r
 428                 public Expansion(Collection<Resource> roots, Collection<DirectStatements>[] results, Collection<Resource>[] listElements) {\r
 429                         this.roots = roots;\r
 430                         this.results = results;\r
 431                         this.listElements = listElements;\r
 432                 }\r
 433 \r
 434                 @Override\r
 435                 public void run(AsyncReadGraph graph) {\r
 436 \r
 437                         QueryControl control = graph.getService(QueryControl.class);\r
 438                         final DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class);\r
 439 \r
 440                         final DomainStatementProcedure proc = new DomainStatementProcedure(dqs, graph.getService(StatementSupport.class), graph.getService(Layer0.class), results, listElements);\r
 441 \r
 442                         int slice = (int) (roots.size() / control.getAmountOfQueryThreads()) + 1;\r
 443 \r
 444                         final Resource[] rootArray = roots.toArray(Resource.NONE);\r
 445                         for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {\r
 446 \r
 447                                 final int start = i * slice;\r
 448                                 final int end = Math.min(start + slice, rootArray.length);\r
 449 \r
 450                                 control.schedule(graph, i, new ControlProcedure() {\r
 451 \r
 452                                         @Override\r
 453                                         public void execute(AsyncReadGraph graph) {\r
 454                                                 for (int index = start; index < end; index++) {\r
 455                                                         dqs.forEachDirectStatement(graph, rootArray[index], proc);\r
 456                                                 }\r
 457 \r
 458                                         }\r
 459 \r
 460                                 });\r
 461 \r
 462                         }\r
 463 \r
 464                 }\r
 465 \r
 466                 @Override\r
 467                 public int getFlags() {\r
 468                         return 0;\r
 469                 }\r
 470 \r
 471         }\r
 472 \r
 473         static class Expansion2 extends AsyncReadRequest {\r
 474 \r
 475                 final private Collection<Resource> roots;\r
 476                 final Collection<DirectStatements>[] results;\r
 477                 final boolean ignoreVirtual;\r
 478 \r
 479                 public Expansion2(Collection<Resource> roots, Collection<DirectStatements>[] results) {\r
 480                         this(roots, results, true);\r
 481                 }\r
 482 \r
 483                 public Expansion2(Collection<Resource> roots, Collection<DirectStatements>[] results, boolean ignoreVirtual) {\r
 484                         this.roots = roots;\r
 485                         this.results = results;\r
 486                         this.ignoreVirtual = ignoreVirtual;\r
 487                 }\r
 488 \r
 489                 @Override\r
 490                 public void run(AsyncReadGraph graph) {\r
 491 \r
 492                         QueryControl control = graph.getService(QueryControl.class);\r
 493                         final DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class);\r
 494 \r
 495                         final DomainStatementProcedure2 proc = \r
 496                                 new DomainStatementProcedure2(results);\r
 497 \r
 498                         int slice = (int) (roots.size() / control.getAmountOfQueryThreads()) + 1;\r
 499 \r
 500                         final Resource[] rootArray = roots.toArray(Resource.NONE);\r
 501                         for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {\r
 502 \r
 503                                 final int start = i * slice;\r
 504                                 final int end = Math.min(start + slice, rootArray.length);\r
 505 \r
 506                                 control.schedule(graph, i, new ControlProcedure() {\r
 507                                         @Override\r
 508                                         public void execute(AsyncReadGraph graph) {\r
 509                                                 if (ignoreVirtual) {\r
 510                                                         for (int index = start; index < end; index++) {\r
 511                                                                 dqs.forEachDirectPersistentStatement(graph, rootArray[index], proc);\r
 512                                                         }\r
 513                                                 } else {\r
 514                                                         for (int index = start; index < end; index++) {\r
 515                                                                 dqs.forEachDirectStatement(graph, rootArray[index], proc);\r
 516                                                         }\r
 517                                                 }\r
 518                                         }\r
 519                                 });\r
 520 \r
 521                         }\r
 522 \r
 523                 }\r
 524 \r
 525                 @Override\r
 526                 public int getFlags() {\r
 527                         return 0;\r
 528                 }\r
 529 \r
 530         }\r
 531         \r
 532         static class DomainProcessor2 {\r
 533 \r
 534                 Serializer variantSerializer;\r
 535 \r
 536                 int id = 0;\r
 537 \r
 538                 Set<Resource> fringe = null;\r
 539                 Set<Resource> exclusions = new HashSet<Resource>();\r
 540                 Set<Resource> internalDomain = new HashSet<Resource>();\r
 541                 Set<Resource> sharedExternalReferences = null;\r
 542                 TIntSet sharedExternalIds = null;\r
 543                 Set<Resource> sharedExternalFringe = null;\r
 544                 Set<Resource> predicates = null;\r
 545                 Set<Resource> isRelatedToPredicates = null;\r
 546                 Set<Resource> sharedPredicates = null;\r
 547                 TIntIntHashMap ids = null;\r
 548                 Map<Resource, Statement> specials = null;\r
 549                 Map<Resource, ExtentStatus> status = null;\r
 550                 Map<Resource, WeakStatus> weakInverses = null;\r
 551                 \r
 552 //              final ArrayList<Double> priorityList = new ArrayList<Double>();\r
 553 \r
 554                 private long composedObjectCounter = 0;\r
 555                 private long fastInternalCounter = 0;\r
 556                 private long parentExternalCounter = 0;\r
 557                 private long fullInternalCounter = 0;\r
 558                 private long fullExternalCounter = 0;\r
 559 \r
 560                 private long startupTime = 0;\r
 561                 private long expandTime = 0;\r
 562                 private long fullResolveTime = 0;\r
 563                 private long fastResolveTime = 0;\r
 564                 private long otherStatementTime = 0;\r
 565                 private long parentResolveTime = 0;\r
 566                 private long extentSeedTime = 0;\r
 567                 private long composedPredicateTime = 0;\r
 568                 private long composedObjectTime = 0;\r
 569 \r
 570                 public void expand(ReadGraph graph, Set<Resource> fringe, Collection<DirectStatements>[] expansion) throws DatabaseException {\r
 571 \r
 572                         long start = System.nanoTime();\r
 573 \r
 574                         Collection<Collection<DirectStatements>[]> fullExpansion = new ArrayList<Collection<DirectStatements>[]>();\r
 575                         QueryControl control = graph.getService(QueryControl.class);\r
 576                         for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {\r
 577                                 expansion[i] = new ArrayList<DirectStatements>();\r
 578                         }\r
 579                         \r
 580                         graph.syncRequest(new Expansion2(fringe, expansion));\r
 581 \r
 582                         fringe.clear();\r
 583 \r
 584                         expandTime += (System.nanoTime() - start);\r
 585 \r
 586                 }\r
 587 \r
 588                 public void classifyPredicates(ReadGraph graph, final Set<Resource> schedule) throws DatabaseException {\r
 589 \r
 590                         CollectionSupport cs = graph.getService(CollectionSupport.class);\r
 591 \r
 592                         final Layer0 L0 = Layer0.getInstance(graph);\r
 593                         \r
 594                         long start = System.nanoTime();\r
 595 \r
 596                         final ConcurrentLinkedQueue<Resource> composedResult = new ConcurrentLinkedQueue<Resource>();\r
 597                         final ConcurrentLinkedQueue<Resource> singleResult = new ConcurrentLinkedQueue<Resource>();\r
 598                         final ConcurrentLinkedQueue<Resource> sharedResult = new ConcurrentLinkedQueue<Resource>();\r
 599                         final ConcurrentLinkedQueue<Pair<Resource, Resource>> singles = new ConcurrentLinkedQueue<Pair<Resource, Resource>>();\r
 600 \r
 601                         // Discover singles\r
 602                         graph.syncRequest(new AsyncReadRequest() {\r
 603 \r
 604                                 @Override\r
 605                                 public void run(AsyncReadGraph graph) {\r
 606 \r
 607                                         for (final Resource predicate : schedule) {\r
 608                                                 \r
 609                                                 graph.forPossibleSuperrelation(predicate, new AsyncProcedure<Resource>() {\r
 610 \r
 611                                                         @Override\r
 612                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {\r
 613                                                                 throwable.printStackTrace();\r
 614                                                         }\r
 615 \r
 616                                                         @Override\r
 617                                                         public void execute(AsyncReadGraph graph, final Resource single) {\r
 618                                                                 singles.add(Pair.make(predicate, single));\r
 619                                                         }\r
 620 \r
 621                                                 });\r
 622 \r
 623                                                 graph.forHasStatement(predicate, L0.SharedRange, new AsyncProcedure<Boolean>() {\r
 624 \r
 625                                                         @Override\r
 626                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {\r
 627                                                                 throwable.printStackTrace();\r
 628                                                         }\r
 629 \r
 630                                                         @Override\r
 631                                                         public void execute(AsyncReadGraph graph, final Boolean shared) {\r
 632                                                                 if(shared) sharedResult.add(predicate);\r
 633                                                         }\r
 634 \r
 635                                                 });\r
 636                                                 \r
 637                                         }\r
 638 \r
 639                                 }\r
 640                                 \r
 641                         });\r
 642 \r
 643                         // Determine singles\r
 644                         final Set<Resource> singleSchedule = cs.createSet();\r
 645                         for(Pair<Resource, Resource> pair : singles) {\r
 646                                 \r
 647                                 Resource single = pair.second;\r
 648                                 if(single != null && predicates.add(single)) singleSchedule.add(single);\r
 649                                 \r
 650                         }\r
 651                         \r
 652                         graph.syncRequest(new AsyncReadRequest() {\r
 653 \r
 654                                 @Override\r
 655                                 public void run(AsyncReadGraph graph) {\r
 656 \r
 657                                         for (final Resource predicate : singleSchedule) {\r
 658 \r
 659                                                 graph.forIsSubrelationOf(predicate, L0.IsRelatedTo, new AsyncProcedure<Boolean>() {\r
 660 \r
 661                                                         @Override\r
 662                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {\r
 663                                                                 throwable.printStackTrace();\r
 664                                                         }\r
 665 \r
 666                                                         @Override\r
 667                                                         public void execute(AsyncReadGraph graph, Boolean strong) {\r
 668                                                                 if (strong) singleResult.add(predicate);\r
 669                                                         }\r
 670 \r
 671                                                 });\r
 672                                                 \r
 673                                         }\r
 674 \r
 675                                 }\r
 676                                 \r
 677                         });\r
 678 \r
 679                         isRelatedToPredicates.addAll(singleResult);\r
 680                         sharedPredicates.addAll(sharedResult);\r
 681 \r
 682                         final Set<Resource> specialSchedule = cs.createSet();\r
 683                         \r
 684                         // Classify\r
 685                         for(Pair<Resource, Resource> pair : singles) {\r
 686                                 \r
 687                                 Resource single = pair.second;\r
 688                                 if(single != null) {\r
 689                                         if(isRelatedToPredicates.contains(single)) {\r
 690                                                 isRelatedToPredicates.add(pair.first);\r
 691                                         }\r
 692                                 } else {\r
 693                                         specialSchedule.add(pair.first);\r
 694                                 }\r
 695                                 \r
 696                         }\r
 697                         \r
 698                         graph.syncRequest(new AsyncReadRequest() {\r
 699 \r
 700                                 @Override\r
 701                                 public void run(AsyncReadGraph graph) {\r
 702 \r
 703                                         for (final Resource predicate : specialSchedule) {\r
 704 \r
 705                                                 graph.forIsSubrelationOf(predicate, L0.IsRelatedTo, new AsyncProcedure<Boolean>() {\r
 706 \r
 707                                                         @Override\r
 708                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {\r
 709                                                                 throwable.printStackTrace();\r
 710                                                         }\r
 711 \r
 712                                                         @Override\r
 713                                                         public void execute(AsyncReadGraph graph, Boolean composed) {\r
 714                                                                 if (composed) composedResult.add(predicate);\r
 715                                                         }\r
 716 \r
 717                                                 });\r
 718                                                 \r
 719                                         }\r
 720 \r
 721                                 }\r
 722                                 \r
 723                         });\r
 724                         \r
 725                         isRelatedToPredicates.addAll(composedResult);\r
 726 \r
 727                         composedPredicateTime += (System.nanoTime() - start);\r
 728                         \r
 729                 }\r
 730                 \r
 731                 private Set<Resource> strongInverseSet = new HashSet<Resource>();\r
 732 \r
 733                 public void classifyPredicates(ReadGraph graph, final Collection<DirectStatements>[] expansion) throws DatabaseException {\r
 734 \r
 735                         CollectionSupport cs = graph.getService(CollectionSupport.class);\r
 736                         final Set<Resource> schedule = cs.createSet();\r
 737                         final Map<Resource, Resource> newPredicates = cs.createMap(Resource.class);\r
 738                         \r
 739                         for (Collection<DirectStatements> coll : expansion)\r
 740                                 for (DirectStatements stms : coll)\r
 741                                         for(Statement stm : stms) {\r
 742                                                 \r
 743                                                 Resource predicate = stm.getPredicate();\r
 744                                                 \r
 745                                                 if(predicates.add(predicate)) {\r
 746                                                         Resource inverse = graph.getPossibleInverse(predicate);\r
 747                                                         schedule.add(predicate);\r
 748                                                         if(inverse != null) {\r
 749                                                                 newPredicates.put(predicate, inverse);\r
 750                                                                 if(predicates.add(inverse)) schedule.add(inverse);\r
 751                                                         }\r
 752                                                         \r
 753                                                 }\r
 754                                                 \r
 755                                         }\r
 756 \r
 757                         classifyPredicates(graph, schedule);\r
 758 \r
 759                         for(Map.Entry<Resource, Resource> entry : newPredicates.entrySet()) {\r
 760                                 // Inverse is strong => this has strong inverse\r
 761                                 if(isRelatedToPredicates.contains(entry.getValue())) {\r
 762                                         strongInverseSet.add(entry.getKey());\r
 763                                 }\r
 764                                 // This is strong => inverse has strong inverse\r
 765                                 if(isRelatedToPredicates.contains(entry.getKey())) {\r
 766                                         strongInverseSet.add(entry.getValue());\r
 767                                 }\r
 768                         }\r
 769                         \r
 770                 }\r
 771 \r
 772                 /*\r
 773                  * Composed objects are internal. Mark them for expansion.\r
 774                  */\r
 775 \r
 776                 public void processFringe(ReadGraph graph, Collection<DirectStatements>[] expansion,\r
 777                                 ObjectOutputStream otherStatementsOutput, ObjectOutputStream valueOutput) throws DatabaseException, IOException {\r
 778 \r
 779                         SerialisationSupport support = graph.getService(SerialisationSupport.class);\r
 780                         TransferableGraphSupport tgs = graph.getService(TransferableGraphSupport.class);\r
 781 \r
 782                         Layer0 L0 = Layer0.getInstance(graph);\r
 783                         \r
 784                         long start = System.nanoTime();\r
 785                         \r
 786                         for (Collection<DirectStatements> coll : expansion)\r
 787                                 for (DirectStatements stms : coll) {\r
 788                                         \r
 789                                         Resource subject = stms.getSubject();\r
 790                                         \r
 791                                         boolean partOf = false;\r
 792                                         for(Statement stm : stms) {\r
 793                                                 Resource predicate = stm.getPredicate();\r
 794                                                 if(L0.PartOf.equals(predicate)) {\r
 795                                                         partOf = true;\r
 796                                                         break;\r
 797                                                 }\r
 798                                         }\r
 799                                         \r
 800                                         ExtentStatus subjectStatus = status.get(subject);\r
 801                                         if(LOG && subjectStatus != null) log("EXISTING STATUS FOR " + graph.getPossibleURI(subject) + " - " + subjectStatus);\r
 802                                         if(subjectStatus == ExtentStatus.EXTERNAL) continue;\r
 803                                         if(partOf && (subjectStatus == null) && graph.getPossibleURI(subject) != null) {\r
 804                                                 \r
 805                                                 status.put(subject, ExtentStatus.EXTERNAL);\r
 806                                                 if(LOG) {\r
 807                                                         String uri = graph.getPossibleURI(subject);\r
 808                                                         if(uri == null) log("[EXTERNAL]: No URI for " + subject);\r
 809                                                         else log("[EXTERNAL] " + uri);\r
 810                                                 }\r
 811 \r
 812                                                 // Check for SharedRange statements\r
 813                                                 for(Statement stm : stms) {\r
 814                                                         Resource predicate = stm.getPredicate();\r
 815                                                         if(sharedPredicates.contains(predicate)) {\r
 816                                                                 sharedExternalFringe.add(stm.getObject());\r
 817                                                                 if(LOG) {\r
 818                                                                         log("[SHARED EXTERNAL FRINGE]: " + NameUtils.getSafeName(graph, stm.getObject()));\r
 819                                                                 }\r
 820                                                         }\r
 821                                                 }\r
 822                                                 \r
 823                                         } else {\r
 824                                                 \r
 825                                                 boolean special = specials.containsKey(subject);\r
 826                                                 if(LOG) {\r
 827                                                         if(special) {\r
 828                                                                 log("[SPECIAL] " + NameUtils.getSafeName(graph, subject));\r
 829                                                         }\r
 830                                                 }\r
 831                                                 \r
 832                                                 status.put(subject, ExtentStatus.INTERNAL);\r
 833                                                 if(LOG) log("[INTERNAL] " + NameUtils.getSafeName(graph, subject));\r
 834 \r
 835                                                 int sId = support.getTransientId(subject);\r
 836 \r
 837                                                 if(graph.hasValue(subject)) {\r
 838                                                         Datatype dt = graph.getRelatedValue(subject, L0.HasDataType, Bindings.getBindingUnchecked(Datatype.class));\r
 839                                                         Binding b = Bindings.getBinding(dt);\r
 840                                                         Object _value = graph.getValue(subject, b);\r
 841                                                         Variant variant = new Variant(b, _value);\r
 842                                                         byte[] value = variantSerializer.serialize(variant);\r
 843                                                         if(LOG) log("[VALUE] " + NameUtils.getSafeName(graph, subject));\r
 844                                                         valueOutput.writeInt(sId);\r
 845                                                         valueOutput.writeInt(value.length);\r
 846                                                         assert (value.length > 0);\r
 847                                                         valueOutput.write(value);\r
 848                                                 }\r
 849                                                 \r
 850                                                 TIntArrayList stream = new TIntArrayList();\r
 851                                                 \r
 852                                                 for(Statement stm : stms) {\r
 853 \r
 854                                                         if(special) {\r
 855                                                                 \r
 856 //                                                              System.err.println("stm=" + stm + " special=" + specials.get(subject));\r
 857                                                                 \r
 858                                                         }\r
 859 \r
 860                                                         Resource predicate = stm.getPredicate();\r
 861                                                         Resource object = stm.getObject();\r
 862 \r
 863                                                         ExtentStatus objectStatus = status.get(object);\r
 864 \r
 865                                                         // Strong predicate\r
 866                                                         if (isRelatedToPredicates.contains(predicate) && (objectStatus !=  ExtentStatus.EXCLUDED)) {\r
 867                                                 \r
 868                                                                 int pId = support.getTransientId(predicate);\r
 869                                                                 int oId = support.getTransientId(object);\r
 870                                                                 \r
 871                                                                 if(LOG) {\r
 872                                                                         String s = NameUtils.getSafeName(graph, subject);\r
 873                                                                         String p = NameUtils.getSafeName(graph, predicate);\r
 874                                                                         String o = NameUtils.getSafeName(graph, object);\r
 875                                                                         log("related=" + s + " - " + p + " - " + o);\r
 876                                                                 }\r
 877                                                                 \r
 878                                                                 stream.add(pId);\r
 879                                                                 stream.add(oId);\r
 880                                                                 \r
 881                                                                 if(objectStatus == null)\r
 882                                                                         fringe.add(object);\r
 883                                                                 \r
 884                                                         } else {\r
 885                                                                 \r
 886                                                                 // Weak predicate\r
 887                                                                 if(objectStatus == ExtentStatus.INTERNAL) {\r
 888 \r
 889                                                                         // The inverse is also weak (or there is no inverse)\r
 890                                                                         if(!strongInverseSet.contains(predicate)) {\r
 891 \r
 892                                                                                 int pId = support.getTransientId(predicate);\r
 893                                                                                 int oId = support.getTransientId(object);\r
 894                                                                                 \r
 895                                                                                 stream.add(pId);\r
 896                                                                                 stream.add(oId);\r
 897                                                                         \r
 898                                                                                 if(LOG) {\r
 899                                                                                         String s = NameUtils.getSafeName(graph, subject);\r
 900                                                                                         String p = NameUtils.getSafeName(graph, predicate);\r
 901                                                                                         String o = NameUtils.getSafeName(graph, object);\r
 902                                                                                         log("fully weak internal=" + s + " - " + p + " - " + o + " - " + objectStatus);\r
 903                                                                                 }\r
 904                                                                                 \r
 905                                                                         } else {\r
 906 \r
 907                                                                                 if(LOG) {\r
 908                                                                                         String s = NameUtils.getSafeName(graph, subject);\r
 909                                                                                         String p = NameUtils.getSafeName(graph, predicate);\r
 910                                                                                         String o = NameUtils.getSafeName(graph, object);\r
 911                                                                                         log("strong inverse internals=" + s + " - " + p + " - " + o + " - " + objectStatus);\r
 912                                                                                 }\r
 913                                                                                 \r
 914                                                                         }\r
 915                                                                         \r
 916                                                                 } else {\r
 917 \r
 918                                                                         if(special) {\r
 919                                                                                 \r
 920 //                                                                              System.err.println("stm=" + stm + " special=" + specials.get(subject));\r
 921                                                                                 \r
 922                                                                                 Statement spec = specials.get(subject);\r
 923                                                                                 \r
 924                                                                                 // This statement can be specially treated\r
 925                                                                                 if(stm.getPredicate().equals(spec.getPredicate()) && stm.getObject().equals(spec.getObject())) {\r
 926 \r
 927                                                                                         int pId = support.getTransientId(predicate);\r
 928                                                                                         int oId = support.getTransientId(object);\r
 929                                                                                         \r
 930                                                                                         if(LOG) {\r
 931                                                                                                 String s = NameUtils.getSafeName(graph, subject);\r
 932                                                                                                 String p = NameUtils.getSafeName(graph, predicate);\r
 933                                                                                                 String o = NameUtils.getSafeName(graph, object);\r
 934                                                                                                 log("special=" + s + " - " + p + " - " + o);\r
 935                                                                                         }\r
 936                                                                                         \r
 937                                                                                         stream.add(pId);\r
 938                                                                                         stream.add(oId);\r
 939                                                                                         \r
 940                                                                                 }\r
 941                                                                                 \r
 942                                                                         } else {\r
 943                                                                         \r
 944                                                                                 if(LOG) {\r
 945                                                                                         String s = NameUtils.getSafeName(graph, subject);\r
 946                                                                                         String p = NameUtils.getSafeName(graph, predicate);\r
 947                                                                                         String o = NameUtils.getSafeName(graph, object);\r
 948                                                                                         log("weak with unknown object=" + s + " - " + p + " - " + o + " - " + objectStatus);\r
 949                                                                                 }\r
 950                                                                                 \r
 951                                                                         }\r
 952                                                                         \r
 953                                                                 }\r
 954                                                                 \r
 955                                                         }\r
 956                                                         \r
 957                                                 }\r
 958                                                 \r
 959                                                 if(!stream.isEmpty()) {\r
 960                                                         otherStatementsOutput.writeInt(sId);\r
 961                                                         otherStatementsOutput.writeInt(stream.size() / 2);\r
 962                                                         for (int i = 0; i < stream.size(); i++)\r
 963                                                                 otherStatementsOutput.writeInt(stream.getQuick(i));\r
 964                                                 }\r
 965                                                 \r
 966                                         }\r
 967                         \r
 968                                 }\r
 969 \r
 970                         composedObjectTime += System.nanoTime() - start;\r
 971 \r
 972                 }\r
 973 \r
 974                 public void process(ReadGraph graph,\r
 975                                 ObjectOutputStream otherStatementsOutput,\r
 976                                 ObjectOutputStream valueOutput)\r
 977                                 throws DatabaseException, IOException {\r
 978 \r
 979                         this.variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT);\r
 980 \r
 981                         QueryControl control = graph.getService(QueryControl.class);\r
 982 \r
 983 //                      System.err.println("Begin ConsistsOfProcess");\r
 984                         \r
 985                         /*\r
 986                          * Browse all stm = (s, ConsistsOf, o)\r
 987                          * � All internal URIs are found => from now on, if unidentified resource has PartOf it is external.\r
 988              * � All s are internal\r
 989              * � All o are internal\r
 990              * � All stm are included\r
 991                          */\r
 992                         for(Resource r : ConsistsOfProcess.walk(graph, fringe, exclusions, true)) {\r
 993                                 if (status.put(r, ExtentStatus.INTERNAL) == null) {\r
 994                                         String URI = graph.getPossibleURI(r);\r
 995                                         if(URI != null) log("URI INTERNAL " + URI);\r
 996                                         else log("URI has no URI for " + r);\r
 997                                         fringe.add(r);\r
 998                                         internalDomain.add(r);\r
 999                                 }\r
1000                         }\r
1001                         \r
1002                         /*\r
1003                          * This loop resolves the transitive closure of all p < IsRelatedTo such that p does not contain the SharedRange tag.\r
1004                          * Such resources are guaranteed to be internal.\r
1005                          */\r
1006                         while(!fringe.isEmpty()) {\r
1007 \r
1008 //                              System.err.println("Process Fringe with " + fringe.size() + ".");\r
1009 \r
1010                                 Collection<DirectStatements>[] expansion = new ArrayList[control.getAmountOfQueryThreads()];\r
1011 \r
1012 //                              System.err.println("-expand");\r
1013 \r
1014                                 // Expand fringe\r
1015                                 expand(graph, fringe, expansion);\r
1016 \r
1017                                 /*\r
1018                                  * classify all p\r
1019                                  * -IsRelatedTo\r
1020                                  * -SharedRange\r
1021                                  * -Internal / External\r
1022                                  */\r
1023                                 \r
1024 //                              System.err.println("-classify");\r
1025 \r
1026                                 classifyPredicates(graph, expansion);\r
1027 \r
1028                                 /*\r
1029                                  * for stms in [stms] \r
1030                                  *   if stms contains predicate PartOf => s is External\r
1031                                  *   else s is Internal\r
1032                                  *   for all stm=(s,p,o) in stms\r
1033                                  *     if p <R IsRelatedTo => stm is included\r
1034                                  *     Fringe <- o\r
1035                                  */\r
1036 \r
1037 //                              System.err.println("-process");\r
1038 \r
1039                                 processFringe(graph, expansion, otherStatementsOutput, valueOutput);\r
1040                                 \r
1041                         }\r
1042 \r
1043                         while(!sharedExternalFringe.isEmpty()) {\r
1044 \r
1045                                 Collection<DirectStatements>[] expansion = new ArrayList[control.getAmountOfQueryThreads()];\r
1046                                 expand(graph, sharedExternalFringe, expansion);\r
1047                                 \r
1048                                 for (Collection<DirectStatements> coll : expansion)\r
1049                                         for (DirectStatements stms : coll) {\r
1050 \r
1051                                                 Resource subject = stms.getSubject();\r
1052                                                 ExtentStatus subjectStatus = status.get(subject);\r
1053                                                 \r
1054                                                 if(ExtentStatus.INTERNAL == subjectStatus) {\r
1055 \r
1056                                                         if(internalDomain.contains(subject)) continue;\r
1057                                                         \r
1058                                                         status.put(subject, ExtentStatus.EXTERNAL);\r
1059                                                         sharedExternalReferences.add(subject);\r
1060 \r
1061                                                         if(LOG) {\r
1062                                                                 log("[SHARED EXTERNAL REFERENCE]: " + NameUtils.getSafeName(graph, subject));\r
1063                                                         }\r
1064                                                         \r
1065                                                         for(Statement stm : stms) {\r
1066                                                                 Resource predicate = stm.getPredicate();\r
1067                                                                 if (isRelatedToPredicates.contains(predicate)) {\r
1068                                                                         sharedExternalFringe.add(stm.getObject());\r
1069                                                                 }\r
1070                                                         }\r
1071                                                         \r
1072                                                 }\r
1073                                         }\r
1074                                 \r
1075                         }\r
1076 \r
1077                         if (PROFILE) {\r
1078                                 System.out.println(composedObjectCounter + " " + fastInternalCounter\r
1079                                                 + " " + parentExternalCounter + " "\r
1080                                                 + fullExternalCounter + " " + fullInternalCounter);\r
1081                         }\r
1082 \r
1083                 }\r
1084 \r
1085         }\r
1086 \r
1087         static class DomainProcessor {\r
1088 \r
1089                 Serializer variantSerializer;\r
1090 \r
1091                 int id = 0;\r
1092 \r
1093                 Set<Resource> predicates = null;\r
1094                 Set<Resource> composedPredicates = null;\r
1095                 Set<Resource> expansionSeeds = null;\r
1096                 Map<Resource, Integer> ids = null;\r
1097                 Map<Resource, ExtentStatus> status = null;\r
1098                 Map<Resource, WeakStatus> weakInverses = null;\r
1099                 \r
1100                 final Set<SubgraphAdvisor> advisors;\r
1101                 final ArrayList<Double> priorityList = new ArrayList<Double>();\r
1102 \r
1103                 private long composedObjectCounter = 0;\r
1104                 private long fastInternalCounter = 0;\r
1105                 private long parentExternalCounter = 0;\r
1106                 private long fullInternalCounter = 0;\r
1107                 private long fullExternalCounter = 0;\r
1108 \r
1109                 private long startupTime = 0;\r
1110                 private long expandTime = 0;\r
1111                 private long fullResolveTime = 0;\r
1112                 private long fastResolveTime = 0;\r
1113                 private long otherStatementTime = 0;\r
1114                 private long parentResolveTime = 0;\r
1115                 private long extentSeedTime = 0;\r
1116                 private long composedPredicateTime = 0;\r
1117                 private long composedObjectTime = 0;\r
1118 \r
1119                 public DomainProcessor(Set<SubgraphAdvisor> advisors) {\r
1120                         this.advisors = advisors;\r
1121                         HashSet<Double> prioritySet = new HashSet<Double>();\r
1122                         for (SubgraphAdvisor advisor : advisors)\r
1123                                 prioritySet.add(advisor.priority());\r
1124                         priorityList.addAll(prioritySet);\r
1125                         Collections.sort(priorityList);\r
1126                 }\r
1127 \r
1128                 public void expand(ReadGraph graph, Collection<DirectStatements>[] expansion, Set<Resource> schedule) throws DatabaseException {\r
1129 \r
1130                         long start = System.nanoTime();\r
1131 \r
1132 //                      if (DEBUG)\r
1133 //                              System.out.println("expanding " + expansionSeeds.size() + " resources.");\r
1134 \r
1135                         QueryControl control = graph.getService(QueryControl.class);\r
1136 //                      final Collection<DirectStatements>[] results = new ArrayList[control.getAmountOfQueryThreads()];\r
1137                         final ArrayList<Resource>[] listElements = new ArrayList[control.getAmountOfQueryThreads()];\r
1138                         for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {\r
1139 //                              results[i] = new ArrayList<DirectStatements>();\r
1140                                 listElements[i] = new ArrayList<Resource>();\r
1141                         }\r
1142                         \r
1143 //                      if(DEBUG) {\r
1144 //                              for(Resource r : expansionSeeds)\r
1145 //                                      System.out.println("Expanding " + NameUtils.getSafeName(graph, r, true));\r
1146 //                      }\r
1147                         \r
1148                         graph.syncRequest(new Expansion(expansionSeeds, expansion, listElements));\r
1149                         for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {\r
1150 //                              for (DirectStatements s : results[i]) {\r
1151 //                                      expansion.put(s.getSubject(), s);\r
1152 //                              }\r
1153                                 for (Resource s : listElements[i]) {\r
1154                                         schedule.add(s);\r
1155 //                                      if(status.put(s, ExtentStatus.INTERNAL) == null) {\r
1156 //                                              ids.put(s, id++);\r
1157 //                                      }\r
1158                                 }\r
1159                         }\r
1160 \r
1161                         expandTime += (System.nanoTime() - start);\r
1162 \r
1163                 }\r
1164 \r
1165                 public void extractComposedPredicates(ReadGraph graph, final Collection<DirectStatements>[] expansion) throws DatabaseException {\r
1166 \r
1167                         long start = System.nanoTime();\r
1168 \r
1169                         CollectionSupport cs = graph.getService(CollectionSupport.class);\r
1170                         \r
1171                         final ConcurrentLinkedQueue<Resource> composedResult = new ConcurrentLinkedQueue<Resource>();\r
1172                         final ConcurrentLinkedQueue<Resource> singleResult = new ConcurrentLinkedQueue<Resource>();\r
1173                         final ConcurrentLinkedQueue<Pair<Resource, Resource>> singles = new ConcurrentLinkedQueue<Pair<Resource, Resource>>();\r
1174                         \r
1175                         final Set<Resource> schedule = cs.createSet();\r
1176                         \r
1177                         for (Collection<DirectStatements> coll : expansion)\r
1178                                 for (DirectStatements stms : coll)\r
1179                                         for(Statement stm : stms) {\r
1180                                                 Resource predicate = stm.getPredicate();\r
1181                                                 if(predicates.add(predicate)) schedule.add(predicate);  \r
1182                                         }\r
1183 \r
1184                         // Discover singles\r
1185                         graph.syncRequest(new AsyncReadRequest() {\r
1186 \r
1187                                 @Override\r
1188                                 public void run(AsyncReadGraph graph) {\r
1189 \r
1190                                         for (final Resource predicate : schedule) {\r
1191                                                 \r
1192                                                 graph.forPossibleSuperrelation(predicate, new AsyncProcedure<Resource>() {\r
1193 \r
1194                                                         @Override\r
1195                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {\r
1196                                                                 throwable.printStackTrace();\r
1197                                                         }\r
1198 \r
1199                                                         @Override\r
1200                                                         public void execute(AsyncReadGraph graph, final Resource single) {\r
1201                                                                 singles.add(Pair.make(predicate, single));\r
1202                                                         }\r
1203 \r
1204                                                 });\r
1205 \r
1206                                         }\r
1207 \r
1208                                 }\r
1209                                 \r
1210                         });\r
1211 \r
1212                         // Determine singles\r
1213                         final Set<Resource> singleSchedule = cs.createSet();\r
1214                         for(Pair<Resource, Resource> pair : singles) {\r
1215                                 \r
1216                                 Resource single = pair.second;\r
1217                                 if(single != null && predicates.add(single)) singleSchedule.add(single);\r
1218                                 \r
1219                         }\r
1220                         \r
1221                         graph.syncRequest(new AsyncReadRequest() {\r
1222 \r
1223                                 @Override\r
1224                                 public void run(AsyncReadGraph graph) {\r
1225 \r
1226                                         for (final Resource predicate : singleSchedule) {\r
1227 \r
1228                                                 graph.forIsSubrelationOf(predicate, graph.getService(Layer0.class).IsComposedOf, new AsyncProcedure<Boolean>() {\r
1229 \r
1230                                                         @Override\r
1231                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {\r
1232                                                                 throwable.printStackTrace();\r
1233                                                         }\r
1234 \r
1235                                                         @Override\r
1236                                                         public void execute(AsyncReadGraph graph, Boolean composed) {\r
1237                                                                 if (composed) singleResult.add(predicate);\r
1238                                                         }\r
1239 \r
1240                                                 });\r
1241                                                 \r
1242                                         }\r
1243 \r
1244                                 }\r
1245                                 \r
1246                         });\r
1247 \r
1248                         composedPredicates.addAll(singleResult);\r
1249 \r
1250                         final Set<Resource> specialSchedule = cs.createSet();\r
1251                         \r
1252                         // Classify\r
1253                         for(Pair<Resource, Resource> pair : singles) {\r
1254                                 \r
1255                                 Resource single = pair.second;\r
1256                                 if(single != null) {\r
1257                                         if(composedPredicates.contains(single)) {\r
1258                                                 composedPredicates.add(pair.first);\r
1259                                         }\r
1260                                 } else {\r
1261                                         specialSchedule.add(pair.first);\r
1262                                 }\r
1263                                 \r
1264                         }\r
1265                         \r
1266                         graph.syncRequest(new AsyncReadRequest() {\r
1267 \r
1268                                 @Override\r
1269                                 public void run(AsyncReadGraph graph) {\r
1270 \r
1271                                         for (final Resource predicate : specialSchedule) {\r
1272 \r
1273                                                 graph.forIsSubrelationOf(predicate, graph.getService(Layer0.class).IsComposedOf, new AsyncProcedure<Boolean>() {\r
1274 \r
1275                                                         @Override\r
1276                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {\r
1277                                                                 throwable.printStackTrace();\r
1278                                                         }\r
1279 \r
1280                                                         @Override\r
1281                                                         public void execute(AsyncReadGraph graph, Boolean composed) {\r
1282                                                                 if (composed) composedResult.add(predicate);\r
1283                                                         }\r
1284 \r
1285                                                 });\r
1286                                                 \r
1287                                         }\r
1288 \r
1289                                 }\r
1290                                 \r
1291                         });\r
1292                         \r
1293                         composedPredicates.addAll(composedResult);\r
1294 \r
1295                         composedPredicateTime += (System.nanoTime() - start);\r
1296 \r
1297                 }\r
1298 \r
1299                 /*\r
1300                  * Composed objects are internal. Mark them for expansion.\r
1301                  */\r
1302 \r
1303                 public void collectComposedObjects(ReadGraph graph, Collection<DirectStatements>[] expansion, Set<Resource> typeTodo, Set<Resource> objectTodo,\r
1304                                 Set<Resource> predicateTodo) throws DatabaseException {\r
1305 \r
1306                         long start = System.nanoTime();\r
1307 \r
1308                         Layer0 l0 = Layer0.getInstance(graph);\r
1309                         \r
1310                         for (Collection<DirectStatements> coll : expansion)\r
1311                                 for (DirectStatements stms : coll)\r
1312                                         for(Statement stm : stms) {\r
1313 \r
1314                                                 Resource predicate = stm.getPredicate();\r
1315                                                 Resource object = stm.getObject();\r
1316                                                 \r
1317                                                 if (composedPredicates.contains(predicate)) {\r
1318                                                         \r
1319                                                         ExtentStatus existing = status.put(object, ExtentStatus.INTERNAL);\r
1320                                                         if(existing == null) {\r
1321                                                                 ids.put(object, id++);\r
1322                                                                 composedObjectCounter++;\r
1323                                                                 expansionSeeds.add(object);\r
1324 //                                                              System.err.println("internal: " + NameUtils.getSafeName(graph, object, true));\r
1325                                                                 if(LOG) log("[INTERNAL] (composed object) " + NameUtils.getSafeName(graph, object, true));\r
1326                                                         } else if (existing == ExtentStatus.EXCLUDED) {\r
1327                                                                 System.err.println("preExcluded: " + NameUtils.getSafeName(graph, object, true));\r
1328                                                                 status.put(object, ExtentStatus.EXCLUDED);\r
1329                                                         } else if (existing == ExtentStatus.EXTERNAL) {\r
1330                                                                 System.err.println("preExternal: " + NameUtils.getSafeName(graph, object, true));\r
1331                                                                 status.put(object, ExtentStatus.EXTERNAL);\r
1332                                                         }\r
1333                                                         \r
1334                                                 } else {\r
1335 \r
1336 //                                                      System.err.println("internal2: " + NameUtils.getSafeName(graph, object, true));\r
1337                                                         \r
1338                                                         if (!status.containsKey(object)) {\r
1339                                                                 if (l0.InstanceOf.equalsResource(predicate)) {\r
1340                                                                         typeTodo.add(object);\r
1341                                                                 } else {\r
1342                                                                         objectTodo.add(object);\r
1343                                                                 }\r
1344                                                         }\r
1345 \r
1346                                                         if (!status.containsKey(predicate)) {\r
1347                                                                 predicateTodo.add(predicate);\r
1348                                                         }\r
1349                                                         \r
1350                                                 }\r
1351                                         }\r
1352 \r
1353                         composedObjectTime += System.nanoTime() - start;\r
1354 \r
1355                 }\r
1356 \r
1357                 public void writeOtherStatements(ReadGraph graph, Collection<Collection<DirectStatements>[]> expansion, ObjectOutputStream composedStatementsOutput, ObjectOutputStream otherStatementsOutput,\r
1358                                 ObjectOutputStream valueOutput) throws DatabaseException {\r
1359 \r
1360                         long start = System.nanoTime();\r
1361 \r
1362                         Layer0 l0 = Layer0.getInstance(graph);\r
1363                         SerialisationSupport support = graph.getService(SerialisationSupport.class);\r
1364                         TransferableGraphSupport tgs = graph.getService(TransferableGraphSupport.class);\r
1365 \r
1366                         TIntArrayList other = new TIntArrayList();\r
1367                         TIntArrayList composed = new TIntArrayList();\r
1368 \r
1369                         try {\r
1370 \r
1371                                 for (Collection<DirectStatements>[] colls : expansion)\r
1372                                         for (Collection<DirectStatements> coll : colls)\r
1373                                                 for (DirectStatements stms : coll) {\r
1374 \r
1375                                                         Resource subject = stms.getSubject();\r
1376                                                         composed.resetQuick();\r
1377 \r
1378                                                         int sId = support.getTransientId(subject);\r
1379 \r
1380                                                         composedStatementsOutput.writeInt(sId);\r
1381 \r
1382                                                         if(graph.hasValue(subject)) {\r
1383                                                                 Datatype dt = graph.getRelatedValue(subject, l0.HasDataType, Bindings.getBindingUnchecked(Datatype.class));\r
1384                                                                 Binding b = Bindings.getBinding(dt);\r
1385                                                                 Object _value = graph.getValue(subject, b);\r
1386                                                                 Variant variant = new Variant(b, _value);\r
1387                                                                 byte[] value = variantSerializer.serialize(variant);\r
1388                                                                 if(LOG) log("[VALUE] " + NameUtils.getSafeName(graph, subject));\r
1389                                                                 valueOutput.writeInt(sId);\r
1390                                                                 valueOutput.writeInt(value.length);\r
1391                                                                 assert (value.length > 0);\r
1392                                                                 valueOutput.write(value);\r
1393                                                         }\r
1394 \r
1395                                                         for (Statement s : stms) {\r
1396 \r
1397                                                                 Resource object = s.getObject();\r
1398                                                                 Resource predicate = s.getPredicate();\r
1399 \r
1400                                                                 ExtentStatus objectStatus = status.get(object); \r
1401                                                                 \r
1402                                                                 if(objectStatus == ExtentStatus.INTERNAL) {\r
1403                                                                         composed.add(support.getTransientId(predicate));\r
1404                                                                         composed.add(support.getTransientId(object));\r
1405                                                                         if(LOG) log("[COMPOSED] (internal object) " + NameUtils.toIdString(graph, s));\r
1406                                                                 } else if (l0.InstanceOf.equalsResource(predicate)) {\r
1407                                                                         composed.add(support.getTransientId(predicate));\r
1408                                                                         composed.add(support.getTransientId(object));\r
1409                                                                         if(LOG) log("[COMPOSED] (instanceOf) " + NameUtils.toIdString(graph, s));\r
1410                                                                 } else if (l0.SubrelationOf.equalsResource(predicate)) {\r
1411                                                                         composed.add(support.getTransientId(predicate));\r
1412                                                                         composed.add(support.getTransientId(object));\r
1413                                                                         if(LOG) log("[COMPOSED] (subrelationOf) " + NameUtils.toIdString(graph, s));\r
1414                                                                 } else {\r
1415                                                                         if(objectStatus == ExtentStatus.EXTERNAL) {\r
1416                                                                                 if(DEBUG)\r
1417                                                                                         System.out.println("other " + NameUtils.toIdString(graph, s));\r
1418                                                                                         //System.out.println("other.add " + predicate + " - " + object);\r
1419                                                                                 other.add(support.getTransientId(predicate));\r
1420                                                                                 other.add(support.getTransientId(object));\r
1421                                                                                 if(LOG) log("[OTHER] (object is external) " + NameUtils.toIdString(graph, s));\r
1422                                                                         }\r
1423                                                                 }\r
1424 \r
1425                                                         }\r
1426 \r
1427                                                         if(!other.isEmpty()) {\r
1428                                                                 otherStatementsOutput.writeInt(sId);\r
1429                                                                 otherStatementsOutput.writeInt(other.size() / 2);\r
1430                                                                 for (int i = 0; i < other.size(); i++)\r
1431                                                                         otherStatementsOutput.writeInt(other.getQuick(i));\r
1432                                                                 other.resetQuick();\r
1433                                                         }\r
1434 \r
1435                                                         composedStatementsOutput.writeInt(composed.size() / 2);\r
1436                                                         for (int i = 0; i < composed.size(); i++)\r
1437                                                                 composedStatementsOutput.writeInt(composed.getQuick(i));\r
1438 \r
1439                                                 }\r
1440 \r
1441                         } catch (IOException e) {\r
1442                                 e.printStackTrace();\r
1443                         }\r
1444 \r
1445                         otherStatementTime += (System.nanoTime() - start);\r
1446 \r
1447                 }\r
1448 \r
1449                 boolean hasStrictParents(ReadGraph g, Resource r)\r
1450                                 throws DatabaseException {\r
1451                         if (g.getPossibleURI(r) != null)\r
1452                                 return true;\r
1453                         return false;\r
1454                 }\r
1455 \r
1456                 public boolean getExpansionSeedsFromExtents(ReadGraph graph, final Collection<DirectStatements>[] expansion) throws DatabaseException {\r
1457 \r
1458                         long start = System.nanoTime();\r
1459 \r
1460                         final ConcurrentLinkedQueue<Resource> accepts = new ConcurrentLinkedQueue<Resource>();\r
1461 \r
1462                         /*\r
1463                          * Determine statements which could accept statements with todo\r
1464                          * objects\r
1465                          */\r
1466                         search: for (Double priority : priorityList) {\r
1467                                 \r
1468                                 for (final SubgraphAdvisor advisor : advisors) {\r
1469                                         \r
1470                                         if (advisor.priority() > 0)\r
1471                                                 continue;\r
1472                                         \r
1473                                         if (advisor.priority() == priority) {\r
1474                                                 \r
1475                                                 graph.syncRequest(new ReadRequest() {\r
1476 \r
1477                                                         @Override\r
1478                                                         public void run(ReadGraph graph) throws DatabaseException {\r
1479 \r
1480                                                                 for (Collection<DirectStatements> coll : expansion)\r
1481                                                                         for (DirectStatements stms : coll)\r
1482                                                                                 for(final Statement stm : stms) {\r
1483 \r
1484                                                                                         advisor.advise(graph, stm, new AsyncProcedure<Boolean>() {\r
1485 \r
1486                                                                                                 @Override\r
1487                                                                                                 public void exception(AsyncReadGraph graph, Throwable throwable) {\r
1488                                                                                                         throwable.printStackTrace();\r
1489                                                                                                 }\r
1490 \r
1491                                                                                                 @Override\r
1492                                                                                                 public void execute(AsyncReadGraph graph, Boolean accept) {\r
1493                                                                                                         if (accept) {\r
1494                                                                                                                 accepts.add(stm.getObject());\r
1495                                                                                                         }\r
1496                                                                                                 }\r
1497 \r
1498                                                                                         });\r
1499 \r
1500                                                                                 }\r
1501 \r
1502                                                         }\r
1503 \r
1504                                                 });\r
1505                                         }\r
1506                                         if (!accepts.isEmpty())\r
1507                                                 break search;\r
1508                                 }\r
1509                         }\r
1510 \r
1511                         CollectionSupport cs = graph.getService(CollectionSupport.class);\r
1512                         Set<Resource> schedule = cs.createSet();\r
1513                         for (Resource r : accepts) {\r
1514                                 if(!status.containsKey(r))\r
1515                                         schedule.add(r);\r
1516                         }\r
1517 \r
1518                         extentSeedTime += (System.nanoTime() - start);\r
1519 \r
1520                         if (schedule.isEmpty())\r
1521                                 return false;\r
1522 \r
1523                         fastResolve(graph, schedule);\r
1524                         uriResolve(graph, schedule);\r
1525                         fullResolve(graph, schedule, "accepts");\r
1526 \r
1527                         return true;\r
1528 \r
1529                 }\r
1530 \r
1531                 ConcurrentLinkedQueue<Resource> fastInternals = new ConcurrentLinkedQueue<Resource>();\r
1532 \r
1533                 public void fastResolve(ReadGraph graph, final Set<Resource> rs)\r
1534                                 throws DatabaseException {\r
1535                         // This collects and resolves weaks\r
1536                         if(fastResolveLoop(graph, rs))\r
1537                                 // Weaks are now resolved\r
1538                                 fastResolveLoop(graph, rs);\r
1539                 }\r
1540 \r
1541                 public boolean fastResolveLoop(ReadGraph graph, final Set<Resource> rs)\r
1542                                 throws DatabaseException {\r
1543 \r
1544                         long start = System.nanoTime();\r
1545 \r
1546                         final ConcurrentLinkedQueue<Resource> weakSchedule = new ConcurrentLinkedQueue<Resource>();\r
1547 \r
1548                         graph.syncRequest(new AsyncRead<Boolean>() {\r
1549 \r
1550                             @Override\r
1551                             public int threadHash() {\r
1552                                 return hashCode();\r
1553                             }\r
1554                                 \r
1555                                 @Override\r
1556                                 public int getFlags() {\r
1557                                         return 0;\r
1558                                 }\r
1559 \r
1560                                 @Override\r
1561                                 public void perform(AsyncReadGraph graph,\r
1562                                                 AsyncProcedure<Boolean> procedure) {\r
1563 \r
1564                                         QueryControl control = graph.getService(QueryControl.class);\r
1565                                         final DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class);\r
1566 \r
1567                                         int slice = (int) (rs.size() / control\r
1568                                                         .getAmountOfQueryThreads()) + 1;\r
1569 \r
1570                                         final Resource[] rootArray = rs.toArray(Resource.NONE);\r
1571                                         for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {\r
1572 \r
1573                                                 final int start = i * slice;\r
1574                                                 final int end = Math.min(start + slice,\r
1575                                                                 rootArray.length);\r
1576 \r
1577                                                 control.schedule(graph, i, new ControlProcedure() {\r
1578 \r
1579                                                         @Override\r
1580                                                         public void execute(AsyncReadGraph graph) {\r
1581 \r
1582                                                                 for (int index = start; index < end; index++) {\r
1583 \r
1584                                                                         final Resource r = rootArray[index];\r
1585 \r
1586                                                                         //if (status.containsKey(r)) continue;\r
1587 \r
1588                                                                         graph.asyncRequest(new FastInternalRequest(dqs, r, status, weakInverses, weakSchedule),new AsyncProcedure<Boolean>() {\r
1589 \r
1590                                                                                 @Override\r
1591                                                                                 public void exception(AsyncReadGraph graph,Throwable throwable) {\r
1592                                                                                         throwable.printStackTrace();\r
1593                                                                                 }\r
1594 \r
1595                                                                                 @Override\r
1596                                                                                 public void execute(AsyncReadGraph graph,Boolean isInternal) {\r
1597                                                                                         if (isInternal) {\r
1598                                                                                                 fastInternals.add(r);\r
1599                                                                                         }\r
1600                                                                                 }\r
1601 \r
1602                                                                         });\r
1603 \r
1604                                                                 }\r
1605 \r
1606                                                         }\r
1607 \r
1608                                                 });\r
1609 \r
1610                                         }\r
1611 \r
1612                                         procedure.execute(graph, true);\r
1613 \r
1614                                 }\r
1615 \r
1616                         });\r
1617 \r
1618                         if (!weakSchedule.isEmpty()) {\r
1619                                 THashSet<Resource> weaks = new THashSet<Resource>(weakSchedule);                                \r
1620                                 if (CLASSIFY_LOG)\r
1621                                         for (Resource p : weakSchedule)\r
1622                                                 System.out.println("classify "\r
1623                                                                 + NameUtils.getSafeName(graph, p));\r
1624                                 graph.syncRequest(new ClassifyStatementsRequest(weaks, weakInverses));\r
1625                         } \r
1626 \r
1627                         for (Resource r : fastInternals) {\r
1628                                 rs.remove(r);\r
1629                                 if (status.put(r, ExtentStatus.INTERNAL) == null) {\r
1630                                         if(LOG) log("[INTERNAL] (fast) " + NameUtils.getSafeName(graph, r, true));\r
1631                                         ids.put(r, id++);\r
1632                                         fastInternalCounter++;\r
1633                                         expansionSeeds.add(r);\r
1634                                 }\r
1635                         }\r
1636 \r
1637                         fastResolveTime += (System.nanoTime() - start);\r
1638                         \r
1639                         return !weakSchedule.isEmpty();                 \r
1640 \r
1641                 }\r
1642 \r
1643                 private ExtentStatus resolveExtent(ReadGraph graph, Resource r, Map<Resource, ExtentStatus> statuses, Set<Resource> expansionSeeds, THashSet<Resource> pending,\r
1644                                 ArrayList<Resource> stack) throws DatabaseException {\r
1645 \r
1646                         ExtentStatus current = statuses.get(r);\r
1647                         if(current != null) return current;\r
1648                         \r
1649                         if (pending.contains(r))\r
1650                                 return ExtentStatus.PENDING;\r
1651 \r
1652                         // In order to break cyclic dependencies\r
1653                         pending.add(r);\r
1654 \r
1655                         if (PARENT_DEBUG)\r
1656                                 System.out.println("resolveExtent "\r
1657                                                 + NameUtils.getSafeName(graph, r));\r
1658 \r
1659                         ExtentStatus status = ExtentStatus.INTERNAL;\r
1660                         for (Resource p : getParents(graph, r)) {\r
1661                                 if (PARENT_DEBUG) {\r
1662                                         ExtentStatus ps = statuses.get(p);\r
1663                                         System.out.println(" parent " + NameUtils.getSafeName(graph, p) + "(" + ps + ")");\r
1664                                 }\r
1665                                 switch (resolveExtent(graph, p, statuses,\r
1666                                                 expansionSeeds, pending, stack)) {\r
1667                                 case EXTERNAL:\r
1668                                         return ExtentStatus.EXTERNAL;\r
1669                                 case PENDING:\r
1670                                         status = ExtentStatus.PENDING;\r
1671                                 }\r
1672                         }\r
1673                         if (status == ExtentStatus.INTERNAL) {\r
1674                                 pending.remove(r);\r
1675                                 stack.add(r);\r
1676                                 if (DEBUG)\r
1677                                         System.out.println(NameUtils.getSafeName(graph, r, true)\r
1678                                                         + " is internal.");\r
1679                         }\r
1680                         return status;\r
1681                 }\r
1682 \r
1683                 public void uriResolve(ReadGraph graph, final Set<Resource> todo)\r
1684                                 throws DatabaseException {\r
1685 \r
1686                         long start = System.nanoTime();\r
1687 \r
1688                         for(Resource r : todo) System.out.println("uriResolve " +\r
1689                                         NameUtils.getSafeName(graph, r));\r
1690 \r
1691                         final ConcurrentSkipListSet<Resource> found = new ConcurrentSkipListSet<Resource>();\r
1692 \r
1693                         graph.syncRequest(new AsyncReadRequest() {\r
1694                                 \r
1695                                 @Override\r
1696                                 public void run(AsyncReadGraph graph) {\r
1697 \r
1698                                         for (final Resource r : todo) {\r
1699 \r
1700                                                 // System.out.println("uriresolve before " + r);\r
1701 \r
1702                                                 if (status.containsKey(r)) continue;\r
1703 \r
1704                                                 // System.out.println("uriresolve " + r);\r
1705 \r
1706                                                 graph.forURI(r, new AsyncProcedure<String>() {\r
1707 \r
1708                                                         @Override\r
1709                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {\r
1710                                                                 throwable.printStackTrace();\r
1711                                                         }\r
1712 \r
1713                                                         @Override\r
1714                                                         public void execute(AsyncReadGraph graph, String uri) {\r
1715 \r
1716                                                                 if (uri != null) {\r
1717 \r
1718                                                                         // System.out.println("uriresolve has uri "\r
1719                                                                         // + r);\r
1720 \r
1721                                                                         if(found.add(r)) {\r
1722                                                                                 parentExternalCounter++;\r
1723                                                                         }\r
1724 \r
1725                                                                 } else {\r
1726 \r
1727                                                                         // System.out.println("uriresolve ask inverse "\r
1728                                                                         // + r);\r
1729 \r
1730                                                                         graph.forPossibleInverse(r, new AsyncProcedure<Resource>() {\r
1731 \r
1732                                                                                 @Override\r
1733                                                                                 public void exception(AsyncReadGraph graph, Throwable throwable) {\r
1734                                                                                         throwable.printStackTrace();\r
1735                                                                                 }\r
1736 \r
1737                                                                                 @Override\r
1738                                                                                 public void execute(AsyncReadGraph graph, Resource inverse) {\r
1739 \r
1740                                                                                         if (inverse != null) {\r
1741 \r
1742                                                                                                 graph.forURI(inverse, new AsyncProcedure<String>() {\r
1743 \r
1744                                                                                                         @Override\r
1745                                                                                                         public void exception(AsyncReadGraph graph, Throwable throwable) {\r
1746                                                                                                                 throwable.printStackTrace();\r
1747                                                                                                         }\r
1748 \r
1749                                                                                                         @Override\r
1750                                                                                                         public void execute(AsyncReadGraph graph, String uri) {\r
1751 \r
1752                                                                                                                 if (uri != null) {\r
1753 \r
1754                                                                                                                         if(found.add(r)) {\r
1755                                                                                                                                 parentExternalCounter++;\r
1756                                                                                                                         }\r
1757 \r
1758                                                                                                                 }\r
1759 \r
1760                                                                                                         }\r
1761 \r
1762                                                                                                 });\r
1763 \r
1764                                                                                         }\r
1765 \r
1766                                                                                 }\r
1767                                                                                 \r
1768                                                                         });\r
1769 \r
1770                                                                 }\r
1771                                                         }\r
1772 \r
1773                                                 });\r
1774 \r
1775                                         }\r
1776 \r
1777                                 }\r
1778 \r
1779                         });\r
1780 \r
1781                         todo.removeAll(found);\r
1782                         for(Resource r : found) {\r
1783                                 status.put(r, ExtentStatus.EXTERNAL);\r
1784                                 if(LOG) log("[EXTERNAL] (uriResolve) " + NameUtils.getSafeName(graph, r, true));\r
1785                         }\r
1786                         \r
1787                         parentResolveTime += System.nanoTime() - start;\r
1788 \r
1789                 }\r
1790 \r
1791                 public void fullResolve(ReadGraph graph, Collection<Resource> rs,\r
1792                                 String koss) throws DatabaseException {\r
1793 \r
1794                         long start = System.nanoTime();\r
1795 \r
1796                         for (final Resource r : rs) {\r
1797 \r
1798                                 if(status.containsKey(r)) continue;\r
1799 \r
1800                                 THashSet<Resource> pending = new THashSet<Resource>();\r
1801                                 ArrayList<Resource> stack = new ArrayList<Resource>();\r
1802 \r
1803                                 ExtentStatus s = resolveExtent(graph, r, status, expansionSeeds, pending, stack);\r
1804                                 if (ExtentStatus.INTERNAL == s || ExtentStatus.PENDING == s) {\r
1805                                         if (status.put(r, ExtentStatus.INTERNAL) == null) {\r
1806                                                 if(LOG) log("[INTERNAL] (resolveExtent) " + NameUtils.getSafeName(graph, r, true));\r
1807                                                 ids.put(r, id++);\r
1808                                                 fullInternalCounter++;\r
1809                                                 expansionSeeds.add(r);\r
1810                                         }\r
1811                                 }\r
1812                                 if (ExtentStatus.EXTERNAL == s) {\r
1813                                         if (status.put(r, ExtentStatus.EXTERNAL) == null) {\r
1814                                                 if(LOG) log("[EXTERNAL] (resolveExtent) " + NameUtils.getSafeName(graph, r, true));\r
1815                                                 fullExternalCounter++;\r
1816                                         }\r
1817                                 }\r
1818 \r
1819                         }\r
1820 \r
1821                         fullResolveTime += (System.nanoTime() - start);\r
1822 \r
1823                 }\r
1824 \r
1825                 public void process(ReadGraph graph,\r
1826                                 ObjectOutputStream composedStatementsOutput,\r
1827                                 ObjectOutputStream otherStatementsOutput,\r
1828                                 ObjectOutputStream valueOutput)\r
1829                                 throws DatabaseException {\r
1830 \r
1831                         this.variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT);\r
1832 \r
1833                         CollectionSupport cs = graph.getService(CollectionSupport.class);\r
1834 \r
1835                         Set<Resource> typeTodo = cs.createSet();\r
1836                         Set<Resource> objectTodo = cs.createSet();\r
1837                         Set<Resource> predicateTodo = cs.createSet();\r
1838                         \r
1839                         Collection<Collection<DirectStatements>[]> fullExpansion = new ArrayList<Collection<DirectStatements>[]>();\r
1840                         \r
1841                         do {\r
1842 \r
1843                                 QueryControl control = graph.getService(QueryControl.class);\r
1844                                 Collection<DirectStatements>[] expansion = new ArrayList[control.getAmountOfQueryThreads()];\r
1845                                 for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {\r
1846                                         expansion[i] = new ArrayList<DirectStatements>();\r
1847                                 }\r
1848                                 \r
1849                                 // Expand expansionSeeds\r
1850                                 expand(graph, expansion, objectTodo);\r
1851                                 \r
1852                                 // Start collecting new seeds\r
1853                                 expansionSeeds = cs.createSet();\r
1854 \r
1855                                 // Collect predicates which are <R IsComposedOf\r
1856                                 extractComposedPredicates(graph, expansion);\r
1857 \r
1858                                 // Make composed objects internal and make sure they are further\r
1859                                 // expanded\r
1860                                 collectComposedObjects(graph, expansion, typeTodo, objectTodo, predicateTodo);\r
1861 \r
1862                                 /*\r
1863                                  * Use the expansion seed heuristic to find new resources to\r
1864                                  * expand before full analysis.\r
1865                                  */\r
1866                                 getExpansionSeedsFromExtents(graph, expansion);\r
1867 \r
1868                                 fullExpansion.add(expansion);\r
1869 \r
1870                         } while (!expansionSeeds.isEmpty());\r
1871 \r
1872                         fastResolve(graph, objectTodo);\r
1873                         uriResolve(graph, predicateTodo);\r
1874                         uriResolve(graph, objectTodo);\r
1875                         uriResolve(graph, typeTodo);\r
1876                         fullResolve(graph, objectTodo, "objectTodo");\r
1877                         fullResolve(graph, predicateTodo, "predicateTodo");\r
1878                         fullResolve(graph, typeTodo, "typeTodo");\r
1879 \r
1880                         writeOtherStatements(graph, fullExpansion, composedStatementsOutput, otherStatementsOutput, valueOutput);\r
1881 \r
1882                         if (PROFILE) {\r
1883                                 System.out.println(composedObjectCounter + " " + fastInternalCounter\r
1884                                                 + " " + parentExternalCounter + " "\r
1885                                                 + fullExternalCounter + " " + fullInternalCounter);\r
1886                         }\r
1887 \r
1888                 }\r
1889 \r
1890         }\r
1891 \r
1892         public static void getDomain2(ReadGraph graph, TIntIntHashMap ids,\r
1893                         Collection<Resource> roots, Map<Resource, ExtentStatus> preStatus,\r
1894                         Map<Resource, Statement> specials,\r
1895                         ObjectOutputStream otherStatementsOutput,\r
1896                         ObjectOutputStream valueOutput,\r
1897                         TreeMap<String, Variant> extensions,\r
1898                         TIntHashSet excludedShared) throws DatabaseException {\r
1899 \r
1900                 ITask task = ThreadLogger.getInstance().begin("getDomain2");\r
1901 \r
1902                 final DomainProcessor2 processor = new DomainProcessor2();\r
1903                 \r
1904                 processor.startupTime = System.nanoTime();\r
1905 \r
1906                 Layer0 l0 = Layer0.getInstance(graph);\r
1907                 \r
1908                 CollectionSupport cs = graph.getService(CollectionSupport.class);\r
1909                 SerialisationSupport support = graph.getService(SerialisationSupport.class);\r
1910 \r
1911                 processor.ids = ids;\r
1912                 processor.specials = specials;\r
1913                 processor.status = cs.createMap(ExtentStatus.class);\r
1914                 processor.weakInverses = cs.createMap(WeakStatus.class);\r
1915                 processor.predicates = cs.createSet();\r
1916                 processor.isRelatedToPredicates = cs.createSet();\r
1917                 processor.sharedPredicates = cs.createSet();\r
1918 //              processor.expansionSeeds = cs.createSet();\r
1919 \r
1920                 for(Map.Entry<Resource, ExtentStatus> entry : preStatus.entrySet()) {\r
1921                         processor.status.put(entry.getKey(), entry.getValue());\r
1922                         if(ExtentStatus.EXCLUDED.equals(entry.getValue())) processor.exclusions.add(entry.getKey());\r
1923                 }\r
1924                 \r
1925 //              for (Resource r : excluded) {\r
1926 //                      processor.status.put(r, ExtentStatus.EXCLUDED);\r
1927 //              }\r
1928                 \r
1929                 Resource rootLibrary = graph.getResource("http:/");\r
1930                 \r
1931                 if (!roots.contains(rootLibrary))\r
1932                         processor.status.put(rootLibrary, ExtentStatus.EXTERNAL);\r
1933 \r
1934                 for (Resource root : roots) {\r
1935                         processor.status.put(root, ExtentStatus.INTERNAL);\r
1936                         //processor.ids.put(support.getTransientId(root), processor.ids.size());\r
1937                         for (Resource owner : graph.getObjects(root, l0.IsOwnedBy)) {\r
1938                                 processor.status.put(owner, ExtentStatus.EXTERNAL);\r
1939                         }\r
1940                 }\r
1941 \r
1942                 processor.startupTime = System.nanoTime() - processor.startupTime;\r
1943 \r
1944                 processor.fringe = new HashSet<Resource>();\r
1945                 processor.fringe.addAll(roots);\r
1946                 \r
1947                 processor.internalDomain.addAll(roots);\r
1948                 \r
1949                 processor.sharedExternalReferences = new HashSet<Resource>();\r
1950                 processor.sharedExternalFringe = new HashSet<Resource>();\r
1951 \r
1952                 try {\r
1953                 \r
1954                         processor.process(graph, otherStatementsOutput, valueOutput);\r
1955                 \r
1956                 } catch (IOException e) {\r
1957                         e.printStackTrace();\r
1958                 }\r
1959                 \r
1960                 for(Resource r : processor.sharedExternalReferences) excludedShared.add(support.getTransientId(r));\r
1961                 \r
1962         ClusteringSupport cls = graph.getService(ClusteringSupport.class);\r
1963         TLongObjectHashMap<TIntArrayList> clusterMap = new TLongObjectHashMap<TIntArrayList>();\r
1964                 for(Map.Entry<Resource, ExtentStatus> entry : processor.status.entrySet()) {\r
1965                         if(ExtentStatus.INTERNAL == entry.getValue()) {\r
1966                                 \r
1967                                 long cluster = cls.getCluster(entry.getKey());\r
1968                                 TIntArrayList list = clusterMap.get(cluster);\r
1969                                 if(list == null) {\r
1970                                         list = new TIntArrayList();\r
1971                                         clusterMap.put(cluster, list);\r
1972                                 }\r
1973                                 list.add(support.getTransientId(entry.getKey()));\r
1974                                 \r
1975                         }\r
1976                 }\r
1977                 final TIntArrayList clustering = new TIntArrayList();\r
1978                 clusterMap.forEachEntry(new TLongObjectProcedure<TIntArrayList>() {\r
1979                         \r
1980                         @Override\r
1981                         public boolean execute(long cluster, TIntArrayList b) {\r
1982                                 clustering.add(b.size());\r
1983                                 b.forEach(new TIntProcedure() {\r
1984                                         \r
1985                                         @Override\r
1986                                         public boolean execute(int rId) {\r
1987                                                 processor.ids.put(rId, processor.id++);\r
1988                                                 return true;\r
1989                                         }\r
1990                                         \r
1991                                 });\r
1992                                 return true;\r
1993                         }\r
1994                         \r
1995                 });\r
1996                 \r
1997                 extensions.put(Extensions.CLUSTERING, new Variant(Bindings.INT_ARRAY, clustering.toArray()));\r
1998                 \r
1999                 long total = processor.startupTime + processor.expandTime\r
2000                                 + processor.composedPredicateTime\r
2001                                 + processor.composedObjectTime + processor.extentSeedTime\r
2002                                 + processor.fullResolveTime + processor.fastResolveTime + \r
2003                                 + processor.parentResolveTime + processor.otherStatementTime;\r
2004 \r
2005                 if (PROFILE) {\r
2006                         System.out.println("startup took " + 1e-9 * processor.startupTime\r
2007                                         + "s.");\r
2008                         System.out.println("expand took " + 1e-9 * processor.expandTime\r
2009                                         + "s.");\r
2010                         System.out.println("composedPredicates took " + 1e-9\r
2011                                         * processor.composedPredicateTime + "s.");\r
2012                         System.out.println("composedObjects took " + 1e-9\r
2013                                         * processor.composedObjectTime + "s.");\r
2014                         System.out.println("extentSeeding took " + 1e-9\r
2015                                         * processor.extentSeedTime + "s.");\r
2016                         System.out.println("fullResolve took " + 1e-9\r
2017                                         * processor.fullResolveTime + "s.");\r
2018                         System.out.println("fastResolve took " + 1e-9\r
2019                                         * processor.fastResolveTime + "s.");\r
2020                         System.out.println("parentResolve took " + 1e-9\r
2021                                         * processor.parentResolveTime + "s.");\r
2022                         System.out.println("otherStatements took " + 1e-9\r
2023                                         * processor.otherStatementTime + "s.");\r
2024                         System.out.println("total " + 1e-9 * total + "s.");\r
2025                 }\r
2026 \r
2027                 task.finish();\r
2028 \r
2029                 \r
2030         }\r
2031 \r
2032         \r
2033         public static void getDomain(ReadGraph graph, Map<Resource, Integer> ids,\r
2034                         Collection<Resource> roots, Map<Resource, ExtentStatus> preStatus, Set<SubgraphAdvisor> advisors,\r
2035                         ObjectOutputStream composedStatementsOutput,\r
2036                         ObjectOutputStream otherStatementsOutput,\r
2037                         ObjectOutputStream valueOutput) throws DatabaseException {\r
2038 \r
2039                 ITask task = ThreadLogger.getInstance().begin("getDomain");\r
2040 \r
2041                 DomainProcessor processor = new DomainProcessor(advisors);\r
2042                 \r
2043                 processor.startupTime = System.nanoTime();\r
2044 \r
2045                 Layer0 l0 = Layer0.getInstance(graph);\r
2046 \r
2047                 CollectionSupport cs = graph.getService(CollectionSupport.class);\r
2048 \r
2049                 processor.ids = ids;\r
2050                 processor.status = cs.createMap(ExtentStatus.class);\r
2051                 processor.weakInverses = cs.createMap(WeakStatus.class);\r
2052                 processor.predicates = cs.createSet();\r
2053                 processor.composedPredicates = cs.createSet();\r
2054                 processor.expansionSeeds = cs.createSet();\r
2055 \r
2056                 for(Map.Entry<Resource, ExtentStatus> entry : preStatus.entrySet()) {\r
2057                         processor.status.put(entry.getKey(), entry.getValue());\r
2058                 }\r
2059 \r
2060 //              for (Resource r : excluded) {\r
2061 //                      processor.status.put(r, ExtentStatus.EXCLUDED);\r
2062 //              }\r
2063                 \r
2064                 if (!roots.contains(graph.getRootLibrary()))\r
2065                         processor.status.put(graph.getRootLibrary(), ExtentStatus.EXTERNAL);\r
2066 \r
2067                 for (Resource root : roots) {\r
2068                         processor.status.put(root, ExtentStatus.INTERNAL);\r
2069                         processor.ids.put(root, processor.id++);\r
2070                         for (Resource owner : graph.getObjects(root, l0.IsOwnedBy)) {\r
2071                                 processor.status.put(owner, ExtentStatus.EXTERNAL);\r
2072                         }\r
2073                 }\r
2074 \r
2075 \r
2076                 processor.expansionSeeds.addAll(roots);\r
2077 \r
2078                 processor.startupTime = System.nanoTime() - processor.startupTime;\r
2079 \r
2080                 while (!processor.expansionSeeds.isEmpty()) {\r
2081 \r
2082                         processor.process(graph, composedStatementsOutput,\r
2083                                         otherStatementsOutput, valueOutput);\r
2084 \r
2085                 }\r
2086 \r
2087                 long total = processor.startupTime + processor.expandTime\r
2088                                 + processor.composedPredicateTime\r
2089                                 + processor.composedObjectTime + processor.extentSeedTime\r
2090                                 + processor.fullResolveTime + processor.fastResolveTime + \r
2091                                 + processor.parentResolveTime + processor.otherStatementTime;\r
2092 \r
2093                 if (PROFILE) {\r
2094                         System.out.println("startup took " + 1e-9 * processor.startupTime\r
2095                                         + "s.");\r
2096                         System.out.println("expand took " + 1e-9 * processor.expandTime\r
2097                                         + "s.");\r
2098                         System.out.println("composedPredicates took " + 1e-9\r
2099                                         * processor.composedPredicateTime + "s.");\r
2100                         System.out.println("composedObjects took " + 1e-9\r
2101                                         * processor.composedObjectTime + "s.");\r
2102                         System.out.println("extentSeeding took " + 1e-9\r
2103                                         * processor.extentSeedTime + "s.");\r
2104                         System.out.println("fullResolve took " + 1e-9\r
2105                                         * processor.fullResolveTime + "s.");\r
2106                         System.out.println("fastResolve took " + 1e-9\r
2107                                         * processor.fastResolveTime + "s.");\r
2108                         System.out.println("parentResolve took " + 1e-9\r
2109                                         * processor.parentResolveTime + "s.");\r
2110                         System.out.println("otherStatements took " + 1e-9\r
2111                                         * processor.otherStatementTime + "s.");\r
2112                         System.out.println("total " + 1e-9 * total + "s.");\r
2113                 }\r
2114 \r
2115                 task.finish();\r
2116 \r
2117         }\r
2118 \r
2119 }\r