+++ /dev/null
-/*******************************************************************************
- * Copyright (c) 2007, 2010 Association for Decentralized Information Management
- * in Industry THTH ry.
- * All rights reserved. This program and the accompanying materials
- * are made available under the terms of the Eclipse Public License v1.0
- * which accompanies this distribution, and is available at
- * http://www.eclipse.org/legal/epl-v10.html
- *
- * Contributors:
- * VTT Technical Research Centre of Finland - initial API and implementation
- *******************************************************************************/
-package org.simantics.db.layer0.util;
-
-import java.io.DataOutput;
-import java.io.DataOutputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.ObjectOutputStream;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ConcurrentSkipListSet;
-
-import org.simantics.databoard.Bindings;
-import org.simantics.databoard.Databoard;
-import org.simantics.databoard.binding.Binding;
-import org.simantics.databoard.binding.mutable.Variant;
-import org.simantics.databoard.serialization.Serializer;
-import org.simantics.databoard.type.Datatype;
-import org.simantics.db.AsyncReadGraph;
-import org.simantics.db.DirectStatements;
-import org.simantics.db.ReadGraph;
-import org.simantics.db.Resource;
-import org.simantics.db.Statement;
-import org.simantics.db.common.request.AsyncReadRequest;
-import org.simantics.db.common.request.ReadRequest;
-import org.simantics.db.common.request.ResourceAsyncRead;
-import org.simantics.db.common.utils.NameUtils;
-import org.simantics.db.exception.DatabaseException;
-import org.simantics.db.layer0.adapter.SubgraphAdvisor;
-import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus;
-import org.simantics.db.layer0.util.ConsistsOfProcess.InternalEntry;
-import org.simantics.db.procedure.AsyncProcedure;
-import org.simantics.db.request.AsyncRead;
-import org.simantics.db.service.ClusteringSupport;
-import org.simantics.db.service.CollectionSupport;
-import org.simantics.db.service.DirectQuerySupport;
-import org.simantics.db.service.QueryControl;
-import org.simantics.db.service.QueryControl.ControlProcedure;
-import org.simantics.db.service.SerialisationSupport;
-import org.simantics.db.service.StatementSupport;
-import org.simantics.db.service.TransferableGraphSupport;
-import org.simantics.graph.representation.Extensions;
-import org.simantics.layer0.Layer0;
-import org.simantics.utils.datastructures.Pair;
-import org.simantics.utils.threads.logger.ITask;
-import org.simantics.utils.threads.logger.ThreadLogger;
-
-import gnu.trove.list.array.TIntArrayList;
-import gnu.trove.map.hash.TIntIntHashMap;
-import gnu.trove.map.hash.TLongObjectHashMap;
-import gnu.trove.procedure.TIntProcedure;
-import gnu.trove.procedure.TLongObjectProcedure;
-import gnu.trove.set.TIntSet;
-import gnu.trove.set.hash.THashSet;
-import gnu.trove.set.hash.TIntHashSet;
-
-public class Subgraphs {
-
- public static String LOG_FILE = "export.log";
- final static private boolean LOG = false;
- final static private boolean DEBUG = false;
- final static private boolean PARENT_DEBUG = DEBUG | false;
- final static private boolean EXTERNAL_DEBUG = DEBUG | false;
- final static private boolean ADVISOR_LOG = LOG & false;
- final static private boolean EXPANSION_LOG = LOG & false;
- final static private boolean INTERNAL_LOG = LOG & false;
- final static private boolean COMPOSED_LOG = LOG & false;
- final static private boolean RESOLVE_LOG = LOG & false;
- final static private boolean CLASSIFY_LOG = LOG & false;
- final static private boolean EXTERNAL_LOG = LOG & false;
- final static private boolean PROFILE = false;
-
- static enum WeakStatus {
- STRONG, WEAK
- }
-
-
- static DataOutput log;
-
- static {
-
- if (LOG) {
- try {
- FileOutputStream stream = new FileOutputStream(LOG_FILE, false);
- log = new DataOutputStream(stream);
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- }
- }
-
- }
-
- private static void log(String line) {
- if (LOG) {
- try {
- log.write((line + "\n").getBytes());
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }
-
- public static Collection<Resource> getParents(ReadGraph g, Resource r)
- throws DatabaseException {
- return getParents(g, r, false);
- }
-
- static class FastInternalRequest extends ResourceAsyncRead<Boolean> {
-
- final DirectQuerySupport dqs;
- final ConcurrentLinkedQueue<Resource> queue;
- final Map<Resource, WeakStatus> weakInverses;
- final Map<Resource, ExtentStatus> status;
-
- public FastInternalRequest(DirectQuerySupport dqs, Resource resource,
- Map<Resource, ExtentStatus> status,
- Map<Resource, WeakStatus> weakInverses,
- ConcurrentLinkedQueue<Resource> queue) {
- super(resource);
- this.dqs = dqs;
- this.status = status;
- this.weakInverses = weakInverses;
- this.queue = queue;
- }
-
- @Override
- public int getFlags() {
- return 0;
- }
-
- @Override
- public void perform(AsyncReadGraph graph, final AsyncProcedure<Boolean> procedure) {
-
- dqs.forEachDirectStatement(graph, resource, new AsyncProcedure<DirectStatements>() {
-
- @Override
- public void execute(AsyncReadGraph graph, DirectStatements ss) {
- boolean ok = true;
- for(Statement statement : ss) {
- if (status.get(statement.getObject()) == ExtentStatus.INTERNAL) continue;
- WeakStatus status = weakInverses.get(statement.getPredicate());
- if(status == WeakStatus.WEAK) continue;
- else if (status == null) {
- queue.add(statement.getPredicate());
- }
- ok = false;
- }
- procedure.execute(graph, ok);
- }
-
- @Override
- public void exception(AsyncReadGraph graph, Throwable throwable) {
- throwable.printStackTrace();
- }
-
- });
-
- }
-
- }
-
- static class ClassifyStatementsRequest implements AsyncRead<Boolean> {
-
- final Set<Resource> schedule;
- final Map<Resource, WeakStatus> weakMap;
-
- public ClassifyStatementsRequest(Set<Resource> schedule, Map<Resource, WeakStatus> weakMap) {
- this.weakMap = weakMap;
- this.schedule = schedule;
- }
-
- @Override
- public int threadHash() {
- return hashCode();
- }
-
- @Override
- public int getFlags() {
- return 0;
- }
-
- @Override
- public void perform(AsyncReadGraph graph, final AsyncProcedure<Boolean> procedure) {
-
- for (final Resource p : schedule) {
-
- graph.forPossibleInverse(p, new AsyncProcedure<Resource>() {
-
- private void register(AsyncReadGraph graph, Resource predicate, Resource superRelation, WeakStatus status) {
- synchronized (weakMap) {
- weakMap.put(predicate, status);
- if(superRelation != null) weakMap.put(superRelation, status);
- }
- }
-
- @Override
- public void exception(AsyncReadGraph graph, Throwable throwable) {
- throwable.printStackTrace();
- }
-
- @Override
- public void execute(AsyncReadGraph graph, final Resource inverse) {
-
- if (inverse == null) {
-
- register(graph, p, null, WeakStatus.WEAK);
-
- } else {
-
- graph.forPossibleSuperrelation(inverse, new AsyncProcedure<Resource>() {
-
- @Override
- public void exception(AsyncReadGraph graph, Throwable throwable) {
- throwable.printStackTrace();
- }
-
- @Override
- public void execute(AsyncReadGraph graph, final Resource superRelation) {
-
- if(superRelation != null && weakMap.containsKey(superRelation)) {
- register(graph, p, null, weakMap.get(superRelation));
- return;
- }
-
- graph.forIsSubrelationOf(inverse, graph.getService(Layer0.class).IsRelatedTo, new AsyncProcedure<Boolean>() {
-
- @Override
- public void exception(AsyncReadGraph graph,Throwable throwable) {
- throwable.printStackTrace();
- }
-
- @Override
- public void execute(AsyncReadGraph graph,Boolean strong) {
- register(graph, p, superRelation, strong ? WeakStatus.STRONG : WeakStatus.WEAK);
- }
-
- });
-
- }
-
- });
-
- }
-
- }
-
- });
-
- }
-
- procedure.execute(graph, false);
-
- }
-
- }
-
- private static Collection<Resource> getParents(ReadGraph g, Resource r, boolean isStrong) throws DatabaseException {
-
- System.out.println("getParents " + NameUtils.getSafeName(g, r));
-
- Layer0 l0 = Layer0.getInstance(g);
-
- Collection<Resource> predicates = g.getPredicates(r);
-
- // --- Consists Of ----------------------------------------------------
-
- if (predicates.contains(l0.PartOf)) {
- Collection<Resource> parents = g.getObjects(r, l0.PartOf);
- if (parents.size() == 1)
- return parents;
- ArrayList<Resource> libraryParents = new ArrayList<Resource>(1);
- for (Resource p : parents)
- if (g.isInstanceOf(p, l0.Library))
- libraryParents.add(p);
- if (!libraryParents.isEmpty())
- return libraryParents;
- else
- return parents;
- }
-
- // --- Ordered sets ---------------------------------------------------
-
- {
- Collection<Resource> parents = null;
- for (Resource p : predicates)
- if (g.isInstanceOf(p, l0.OrderedSet) && !p.equals(r)) {
- if (parents == null)
- parents = new ArrayList<Resource>(1);
- parents.add(p);
- }
- if (parents != null) {
- if (DEBUG)
- System.out.println("ORDERED SET");
- return parents;
- }
- }
-
-
- if (isStrong)
- return Collections.emptyList();
- else {
-
- if (predicates.contains(l0.InverseOf)) {
-
- Resource inv = g.getInverse(r);
- return getParents(g, inv, true);
-
- } else {
-
- /*
- * Depends On
- *
- * If there are DependsOn parents, then IsRelatedTo parents are discarded
- *
- */
- HashSet<Resource> result = new HashSet<Resource>();
- for(Resource predicate : predicates) {
- if(g.isSubrelationOf(predicate, l0.IsDependencyOf)) result.addAll(g.getObjects(r, predicate));
- }
- if(!result.isEmpty()) return result;
-
- /*
- * Is Related To
- *
- * At this point all Is Related To are parents.
- *
- */
- for(Resource predicate : predicates) {
- Resource inv = g.getPossibleInverse(predicate);
- if(inv != null) {
- if(g.isSubrelationOf(inv, l0.IsRelatedTo)) result.addAll(g.getObjects(r, predicate));
- }
- }
-
- return result;
-
- }
-
- /*
- Collection<Resource> invR = g.getObjects(r, b.IsRelatedTo_Inverse);
- if (predicates.contains(b.InverseOf)) {
- if (invR.size() > 1) {
- if (DEBUG)
- System.out
- .println("###########################################");
- Resource inv = g.getInverse(r);
- Collection<Resource> ret = new ArrayList<Resource>();
- for (Statement pp : g.getStatements(r,
- b.IsRelatedTo_Inverse))
- if (!pp.getPredicate().equals(inv)) {
- if (DEBUG) {
- System.out.println("<"
- + NameUtils.getSafeName(g, pp
- .getSubject())
- + ","
- + NameUtils.getSafeName(g, pp
- .getPredicate())
- + ","
- + NameUtils.getSafeName(g, pp
- .getObject()) + ">");
- }
- ret.add(pp.getObject());
- }
- return ret;
- }
- // System.out.println("?????????????????");
- Collection<Resource> invParents = getParents(g,
- g.getInverse(r), true);
- if (!invParents.isEmpty())
- return invParents;
- }
- if (DEBUG) {
- System.out.print("invR");
- for (Resource res : invR)
- System.out.print(" " + NameUtils.getSafeName(g, res));
- System.out.println();
- }
- return invR;
- */
- }
-
- }
-
-// public static String getIdentifier(ReadGraph g, Resource r)
-// throws DatabaseException {
-// Layer0 L0 = Layer0.getInstance(g);
-// if (r.equals(g.getRootLibrary()))
-// return "";
-// String name = g.getPossibleRelatedValue(r, L0.HasName);
-// if (name == null)
-// return null;
-// Collection<Resource> parents = getParents(g, r, true);
-// if (parents.size() != 1)
-// return null;
-// for (Resource p : parents) {
-// String parentIdentifier = getIdentifier(g, p);
-// if (parentIdentifier == null)
-// return null;
-// return parentIdentifier + "/" + name;
-// }
-// return null;
-// }
-
- static int kess = 0;
-
- static class Expansion extends AsyncReadRequest {
-
- final private Collection<Resource> roots;
- final Collection<DirectStatements>[] results;
- final Collection<Resource>[] listElements;
-
- public Expansion(Collection<Resource> roots, Collection<DirectStatements>[] results, Collection<Resource>[] listElements) {
- this.roots = roots;
- this.results = results;
- this.listElements = listElements;
- }
-
- @Override
- public void run(AsyncReadGraph graph) {
-
- QueryControl control = graph.getService(QueryControl.class);
- final DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class);
-
- final DomainStatementProcedure proc = new DomainStatementProcedure(dqs, graph.getService(StatementSupport.class), graph.getService(Layer0.class), results, listElements);
-
- int slice = (int) (roots.size() / control.getAmountOfQueryThreads()) + 1;
-
- final Resource[] rootArray = roots.toArray(Resource.NONE);
- for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
-
- final int start = i * slice;
- final int end = Math.min(start + slice, rootArray.length);
-
- control.schedule(graph, i, new ControlProcedure() {
-
- @Override
- public void execute(AsyncReadGraph graph) {
- for (int index = start; index < end; index++) {
- dqs.forEachDirectStatement(graph, rootArray[index], proc);
- }
-
- }
-
- });
-
- }
-
- }
-
- @Override
- public int getFlags() {
- return 0;
- }
-
- }
-
- static class Expansion2 extends AsyncReadRequest {
-
- final private Collection<Resource> roots;
- final Collection<DirectStatements>[] results;
- final boolean ignoreVirtual;
-
- public Expansion2(Collection<Resource> roots, Collection<DirectStatements>[] results) {
- this(roots, results, true);
- }
-
- public Expansion2(Collection<Resource> roots, Collection<DirectStatements>[] results, boolean ignoreVirtual) {
- this.roots = roots;
- this.results = results;
- this.ignoreVirtual = ignoreVirtual;
- }
-
- @Override
- public void run(AsyncReadGraph graph) {
-
- QueryControl control = graph.getService(QueryControl.class);
- final DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class);
-
- final DomainStatementProcedure2 proc =
- new DomainStatementProcedure2(results);
-
- int slice = (int) (roots.size() / control.getAmountOfQueryThreads()) + 1;
-
- final Resource[] rootArray = roots.toArray(Resource.NONE);
- for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
-
- final int start = i * slice;
- final int end = Math.min(start + slice, rootArray.length);
-
- control.schedule(graph, i, new ControlProcedure() {
- @Override
- public void execute(AsyncReadGraph graph) {
- if (ignoreVirtual) {
- for (int index = start; index < end; index++) {
- dqs.forEachDirectPersistentStatement(graph, rootArray[index], proc);
- }
- } else {
- for (int index = start; index < end; index++) {
- dqs.forEachDirectStatement(graph, rootArray[index], proc);
- }
- }
- }
- });
-
- }
-
- }
-
- @Override
- public int getFlags() {
- return 0;
- }
-
- }
-
- static class DomainProcessor2 {
-
- Serializer variantSerializer;
-
- int id = 0;
-
- Set<Resource> fringe = null;
- Set<Resource> exclusions = new HashSet<Resource>();
- Set<Resource> internalDomain = new HashSet<Resource>();
- Set<Resource> sharedExternalReferences = null;
- TIntSet sharedExternalIds = null;
- Set<Resource> sharedExternalFringe = null;
- Set<Resource> predicates = null;
- Set<Resource> isRelatedToPredicates = null;
- Set<Resource> sharedPredicates = null;
- TIntIntHashMap ids = null;
- Map<Resource, Statement> specials = null;
- Map<Resource, ExtentStatus> status = null;
- Map<Resource, WeakStatus> weakInverses = null;
-
-// final ArrayList<Double> priorityList = new ArrayList<Double>();
-
- private long composedObjectCounter = 0;
- private long fastInternalCounter = 0;
- private long parentExternalCounter = 0;
- private long fullInternalCounter = 0;
- private long fullExternalCounter = 0;
-
- private long startupTime = 0;
- private long expandTime = 0;
- private long fullResolveTime = 0;
- private long fastResolveTime = 0;
- private long otherStatementTime = 0;
- private long parentResolveTime = 0;
- private long extentSeedTime = 0;
- private long composedPredicateTime = 0;
- private long composedObjectTime = 0;
-
- public void expand(ReadGraph graph, Set<Resource> fringe, Collection<DirectStatements>[] expansion) throws DatabaseException {
-
- long start = System.nanoTime();
-
- Collection<Collection<DirectStatements>[]> fullExpansion = new ArrayList<Collection<DirectStatements>[]>();
- QueryControl control = graph.getService(QueryControl.class);
- for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
- expansion[i] = new ArrayList<DirectStatements>();
- }
-
- graph.syncRequest(new Expansion2(fringe, expansion));
-
- fringe.clear();
-
- expandTime += (System.nanoTime() - start);
-
- }
-
- public void classifyPredicates(ReadGraph graph, final Set<Resource> schedule) throws DatabaseException {
-
- CollectionSupport cs = graph.getService(CollectionSupport.class);
-
- final Layer0 L0 = Layer0.getInstance(graph);
-
- long start = System.nanoTime();
-
- final ConcurrentLinkedQueue<Resource> composedResult = new ConcurrentLinkedQueue<Resource>();
- final ConcurrentLinkedQueue<Resource> singleResult = new ConcurrentLinkedQueue<Resource>();
- final ConcurrentLinkedQueue<Resource> sharedResult = new ConcurrentLinkedQueue<Resource>();
- final ConcurrentLinkedQueue<Pair<Resource, Resource>> singles = new ConcurrentLinkedQueue<Pair<Resource, Resource>>();
-
- // Discover singles
- graph.syncRequest(new AsyncReadRequest() {
-
- @Override
- public void run(AsyncReadGraph graph) {
-
- for (final Resource predicate : schedule) {
-
- graph.forPossibleSuperrelation(predicate, new AsyncProcedure<Resource>() {
-
- @Override
- public void exception(AsyncReadGraph graph, Throwable throwable) {
- throwable.printStackTrace();
- }
-
- @Override
- public void execute(AsyncReadGraph graph, final Resource single) {
- singles.add(Pair.make(predicate, single));
- }
-
- });
-
- graph.forHasStatement(predicate, L0.SharedRange, new AsyncProcedure<Boolean>() {
-
- @Override
- public void exception(AsyncReadGraph graph, Throwable throwable) {
- throwable.printStackTrace();
- }
-
- @Override
- public void execute(AsyncReadGraph graph, final Boolean shared) {
- if(shared) sharedResult.add(predicate);
- }
-
- });
-
- }
-
- }
-
- });
-
- // Determine singles
- final Set<Resource> singleSchedule = cs.createSet();
- for(Pair<Resource, Resource> pair : singles) {
-
- Resource single = pair.second;
- if(single != null && predicates.add(single)) singleSchedule.add(single);
-
- }
-
- graph.syncRequest(new AsyncReadRequest() {
-
- @Override
- public void run(AsyncReadGraph graph) {
-
- for (final Resource predicate : singleSchedule) {
-
- graph.forIsSubrelationOf(predicate, L0.IsRelatedTo, new AsyncProcedure<Boolean>() {
-
- @Override
- public void exception(AsyncReadGraph graph, Throwable throwable) {
- throwable.printStackTrace();
- }
-
- @Override
- public void execute(AsyncReadGraph graph, Boolean strong) {
- if (strong) singleResult.add(predicate);
- }
-
- });
-
- }
-
- }
-
- });
-
- isRelatedToPredicates.addAll(singleResult);
- sharedPredicates.addAll(sharedResult);
-
- final Set<Resource> specialSchedule = cs.createSet();
-
- // Classify
- for(Pair<Resource, Resource> pair : singles) {
-
- Resource single = pair.second;
- if(single != null) {
- if(isRelatedToPredicates.contains(single)) {
- isRelatedToPredicates.add(pair.first);
- }
- } else {
- specialSchedule.add(pair.first);
- }
-
- }
-
- graph.syncRequest(new AsyncReadRequest() {
-
- @Override
- public void run(AsyncReadGraph graph) {
-
- for (final Resource predicate : specialSchedule) {
-
- graph.forIsSubrelationOf(predicate, L0.IsRelatedTo, new AsyncProcedure<Boolean>() {
-
- @Override
- public void exception(AsyncReadGraph graph, Throwable throwable) {
- throwable.printStackTrace();
- }
-
- @Override
- public void execute(AsyncReadGraph graph, Boolean composed) {
- if (composed) composedResult.add(predicate);
- }
-
- });
-
- }
-
- }
-
- });
-
- isRelatedToPredicates.addAll(composedResult);
-
- composedPredicateTime += (System.nanoTime() - start);
-
- }
-
- private Set<Resource> strongInverseSet = new HashSet<Resource>();
-
- public void classifyPredicates(ReadGraph graph, final Collection<DirectStatements>[] expansion) throws DatabaseException {
-
- CollectionSupport cs = graph.getService(CollectionSupport.class);
- final Set<Resource> schedule = cs.createSet();
- final Map<Resource, Resource> newPredicates = cs.createMap(Resource.class);
-
- for (Collection<DirectStatements> coll : expansion)
- for (DirectStatements stms : coll)
- for(Statement stm : stms) {
-
- Resource predicate = stm.getPredicate();
-
- if(predicates.add(predicate)) {
- Resource inverse = graph.getPossibleInverse(predicate);
- schedule.add(predicate);
- if(inverse != null) {
- newPredicates.put(predicate, inverse);
- if(predicates.add(inverse)) schedule.add(inverse);
- }
-
- }
-
- }
-
- classifyPredicates(graph, schedule);
-
- for(Map.Entry<Resource, Resource> entry : newPredicates.entrySet()) {
- // Inverse is strong => this has strong inverse
- if(isRelatedToPredicates.contains(entry.getValue())) {
- strongInverseSet.add(entry.getKey());
- }
- // This is strong => inverse has strong inverse
- if(isRelatedToPredicates.contains(entry.getKey())) {
- strongInverseSet.add(entry.getValue());
- }
- }
-
- }
-
- /*
- * Composed objects are internal. Mark them for expansion.
- */
-
- public void processFringe(ReadGraph graph, Collection<DirectStatements>[] expansion,
- ObjectOutputStream otherStatementsOutput, ObjectOutputStream valueOutput) throws DatabaseException, IOException {
-
- SerialisationSupport support = graph.getService(SerialisationSupport.class);
- TransferableGraphSupport tgs = graph.getService(TransferableGraphSupport.class);
-
- Layer0 L0 = Layer0.getInstance(graph);
-
- long start = System.nanoTime();
-
- for (Collection<DirectStatements> coll : expansion)
- for (DirectStatements stms : coll) {
-
- Resource subject = stms.getSubject();
-
- boolean partOf = false;
- for(Statement stm : stms) {
- Resource predicate = stm.getPredicate();
- if(L0.PartOf.equals(predicate)) {
- partOf = true;
- break;
- }
- }
-
- ExtentStatus subjectStatus = status.get(subject);
- if(LOG && subjectStatus != null) log("EXISTING STATUS FOR " + graph.getPossibleURI(subject) + " - " + subjectStatus);
- if(subjectStatus == ExtentStatus.EXTERNAL) continue;
- if(partOf && (subjectStatus == null) && graph.getPossibleURI(subject) != null) {
-
- status.put(subject, ExtentStatus.EXTERNAL);
- if(LOG) {
- String uri = graph.getPossibleURI(subject);
- if(uri == null) log("[EXTERNAL]: No URI for " + subject);
- else log("[EXTERNAL] " + uri);
- }
-
- // Check for SharedRange statements
- for(Statement stm : stms) {
- Resource predicate = stm.getPredicate();
- if(sharedPredicates.contains(predicate)) {
- sharedExternalFringe.add(stm.getObject());
- if(LOG) {
- log("[SHARED EXTERNAL FRINGE]: " + NameUtils.getSafeName(graph, stm.getObject()));
- }
- }
- }
-
- } else {
-
- boolean special = specials.containsKey(subject);
- if(LOG) {
- if(special) {
- log("[SPECIAL] " + NameUtils.getSafeName(graph, subject));
- }
- }
-
- status.put(subject, ExtentStatus.INTERNAL);
- if(LOG) log("[INTERNAL] " + NameUtils.getSafeName(graph, subject));
-
- int sId = support.getTransientId(subject);
-
- if(graph.hasValue(subject)) {
- Datatype dt = graph.getRelatedValue(subject, L0.HasDataType, Bindings.getBindingUnchecked(Datatype.class));
- Binding b = Bindings.getBinding(dt);
- Object _value = graph.getValue(subject, b);
- Variant variant = new Variant(b, _value);
- byte[] value = variantSerializer.serialize(variant);
- if(LOG) log("[VALUE] " + NameUtils.getSafeName(graph, subject));
- valueOutput.writeInt(sId);
- valueOutput.writeInt(value.length);
- assert (value.length > 0);
- valueOutput.write(value);
- }
-
- TIntArrayList stream = new TIntArrayList();
-
- for(Statement stm : stms) {
-
- if(special) {
-
-// System.err.println("stm=" + stm + " special=" + specials.get(subject));
-
- }
-
- Resource predicate = stm.getPredicate();
- Resource object = stm.getObject();
-
- ExtentStatus objectStatus = status.get(object);
-
- // Strong predicate
- if (isRelatedToPredicates.contains(predicate) && (objectStatus != ExtentStatus.EXCLUDED)) {
-
- int pId = support.getTransientId(predicate);
- int oId = support.getTransientId(object);
-
- if(LOG) {
- String s = NameUtils.getSafeName(graph, subject);
- String p = NameUtils.getSafeName(graph, predicate);
- String o = NameUtils.getSafeName(graph, object);
- log("related=" + s + " - " + p + " - " + o);
- }
-
- stream.add(pId);
- stream.add(oId);
-
- if(objectStatus == null)
- fringe.add(object);
-
- } else {
-
- // Weak predicate
- if(objectStatus == ExtentStatus.INTERNAL) {
-
- // The inverse is also weak (or there is no inverse)
- if(!strongInverseSet.contains(predicate)) {
-
- int pId = support.getTransientId(predicate);
- int oId = support.getTransientId(object);
-
- stream.add(pId);
- stream.add(oId);
-
- if(LOG) {
- String s = NameUtils.getSafeName(graph, subject);
- String p = NameUtils.getSafeName(graph, predicate);
- String o = NameUtils.getSafeName(graph, object);
- log("fully weak internal=" + s + " - " + p + " - " + o + " - " + objectStatus);
- }
-
- } else {
-
- if(LOG) {
- String s = NameUtils.getSafeName(graph, subject);
- String p = NameUtils.getSafeName(graph, predicate);
- String o = NameUtils.getSafeName(graph, object);
- log("strong inverse internals=" + s + " - " + p + " - " + o + " - " + objectStatus);
- }
-
- }
-
- } else {
-
- if(special) {
-
-// System.err.println("stm=" + stm + " special=" + specials.get(subject));
-
- Statement spec = specials.get(subject);
-
- // This statement can be specially treated
- if(stm.getPredicate().equals(spec.getPredicate()) && stm.getObject().equals(spec.getObject())) {
-
- int pId = support.getTransientId(predicate);
- int oId = support.getTransientId(object);
-
- if(LOG) {
- String s = NameUtils.getSafeName(graph, subject);
- String p = NameUtils.getSafeName(graph, predicate);
- String o = NameUtils.getSafeName(graph, object);
- log("special=" + s + " - " + p + " - " + o);
- }
-
- stream.add(pId);
- stream.add(oId);
-
- }
-
- } else {
-
- if(LOG) {
- String s = NameUtils.getSafeName(graph, subject);
- String p = NameUtils.getSafeName(graph, predicate);
- String o = NameUtils.getSafeName(graph, object);
- log("weak with unknown object=" + s + " - " + p + " - " + o + " - " + objectStatus);
- }
-
- }
-
- }
-
- }
-
- }
-
- if(!stream.isEmpty()) {
- otherStatementsOutput.writeInt(sId);
- otherStatementsOutput.writeInt(stream.size() / 2);
- for (int i = 0; i < stream.size(); i++)
- otherStatementsOutput.writeInt(stream.getQuick(i));
- }
-
- }
-
- }
-
- composedObjectTime += System.nanoTime() - start;
-
- }
-
- public void process(ReadGraph graph,
- ObjectOutputStream otherStatementsOutput,
- ObjectOutputStream valueOutput)
- throws DatabaseException, IOException {
-
- this.variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT);
-
- QueryControl control = graph.getService(QueryControl.class);
-
-// System.err.println("Begin ConsistsOfProcess");
-
- /*
- * Browse all stm = (s, ConsistsOf, o)
- * � All internal URIs are found => from now on, if unidentified resource has PartOf it is external.
- * � All s are internal
- * � All o are internal
- * � All stm are included
- */
- Pair<List<InternalEntry>,Set<Resource>> pair = ConsistsOfProcess.walk(graph, null, fringe, exclusions, true);
- List<InternalEntry> entries = pair.first;
- for(InternalEntry entry : entries) {
- Resource r = entry.resource;
- if (status.put(r, ExtentStatus.INTERNAL) == null) {
- String URI = graph.getPossibleURI(r);
- if(URI != null) log("URI INTERNAL " + URI);
- else log("URI has no URI for " + r);
- fringe.add(r);
- internalDomain.add(r);
- }
- }
-
- for(Resource unnamedChild : pair.second) {
- if (status.put(unnamedChild, ExtentStatus.INTERNAL) == null) {
- fringe.add(unnamedChild);
- }
- }
-
- /*
- * This loop resolves the transitive closure of all p < IsRelatedTo such that p does not contain the SharedRange tag.
- * Such resources are guaranteed to be internal.
- */
- while(!fringe.isEmpty()) {
-
-// System.err.println("Process Fringe with " + fringe.size() + ".");
-
- Collection<DirectStatements>[] expansion = new ArrayList[control.getAmountOfQueryThreads()];
-
-// System.err.println("-expand");
-
- // Expand fringe
- expand(graph, fringe, expansion);
-
- /*
- * classify all p
- * -IsRelatedTo
- * -SharedRange
- * -Internal / External
- */
-
-// System.err.println("-classify");
-
- classifyPredicates(graph, expansion);
-
- /*
- * for stms in [stms]
- * if stms contains predicate PartOf => s is External
- * else s is Internal
- * for all stm=(s,p,o) in stms
- * if p <R IsRelatedTo => stm is included
- * Fringe <- o
- */
-
-// System.err.println("-process");
-
- processFringe(graph, expansion, otherStatementsOutput, valueOutput);
-
- }
-
- while(!sharedExternalFringe.isEmpty()) {
-
- Collection<DirectStatements>[] expansion = new ArrayList[control.getAmountOfQueryThreads()];
- expand(graph, sharedExternalFringe, expansion);
-
- for (Collection<DirectStatements> coll : expansion)
- for (DirectStatements stms : coll) {
-
- Resource subject = stms.getSubject();
- ExtentStatus subjectStatus = status.get(subject);
-
- if(ExtentStatus.INTERNAL == subjectStatus) {
-
- if(internalDomain.contains(subject)) continue;
-
- status.put(subject, ExtentStatus.EXTERNAL);
- sharedExternalReferences.add(subject);
-
- if(LOG) {
- log("[SHARED EXTERNAL REFERENCE]: " + NameUtils.getSafeName(graph, subject));
- }
-
- for(Statement stm : stms) {
- Resource predicate = stm.getPredicate();
- if (isRelatedToPredicates.contains(predicate)) {
- sharedExternalFringe.add(stm.getObject());
- }
- }
-
- }
- }
-
- }
-
- if (PROFILE) {
- System.out.println(composedObjectCounter + " " + fastInternalCounter
- + " " + parentExternalCounter + " "
- + fullExternalCounter + " " + fullInternalCounter);
- }
-
- }
-
- }
-
- static class DomainProcessor {
-
- Serializer variantSerializer;
-
- int id = 0;
-
- Set<Resource> predicates = null;
- Set<Resource> composedPredicates = null;
- Set<Resource> expansionSeeds = null;
- Map<Resource, Integer> ids = null;
- Map<Resource, ExtentStatus> status = null;
- Map<Resource, WeakStatus> weakInverses = null;
-
- final Set<SubgraphAdvisor> advisors;
- final ArrayList<Double> priorityList = new ArrayList<Double>();
-
- private long composedObjectCounter = 0;
- private long fastInternalCounter = 0;
- private long parentExternalCounter = 0;
- private long fullInternalCounter = 0;
- private long fullExternalCounter = 0;
-
- private long startupTime = 0;
- private long expandTime = 0;
- private long fullResolveTime = 0;
- private long fastResolveTime = 0;
- private long otherStatementTime = 0;
- private long parentResolveTime = 0;
- private long extentSeedTime = 0;
- private long composedPredicateTime = 0;
- private long composedObjectTime = 0;
-
- public DomainProcessor(Set<SubgraphAdvisor> advisors) {
- this.advisors = advisors;
- HashSet<Double> prioritySet = new HashSet<Double>();
- for (SubgraphAdvisor advisor : advisors)
- prioritySet.add(advisor.priority());
- priorityList.addAll(prioritySet);
- Collections.sort(priorityList);
- }
-
- public void expand(ReadGraph graph, Collection<DirectStatements>[] expansion, Set<Resource> schedule) throws DatabaseException {
-
- long start = System.nanoTime();
-
-// if (DEBUG)
-// System.out.println("expanding " + expansionSeeds.size() + " resources.");
-
- QueryControl control = graph.getService(QueryControl.class);
-// final Collection<DirectStatements>[] results = new ArrayList[control.getAmountOfQueryThreads()];
- final ArrayList<Resource>[] listElements = new ArrayList[control.getAmountOfQueryThreads()];
- for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
-// results[i] = new ArrayList<DirectStatements>();
- listElements[i] = new ArrayList<Resource>();
- }
-
-// if(DEBUG) {
-// for(Resource r : expansionSeeds)
-// System.out.println("Expanding " + NameUtils.getSafeName(graph, r, true));
-// }
-
- graph.syncRequest(new Expansion(expansionSeeds, expansion, listElements));
- for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
-// for (DirectStatements s : results[i]) {
-// expansion.put(s.getSubject(), s);
-// }
- for (Resource s : listElements[i]) {
- schedule.add(s);
-// if(status.put(s, ExtentStatus.INTERNAL) == null) {
-// ids.put(s, id++);
-// }
- }
- }
-
- expandTime += (System.nanoTime() - start);
-
- }
-
- public void extractComposedPredicates(ReadGraph graph, final Collection<DirectStatements>[] expansion) throws DatabaseException {
-
- long start = System.nanoTime();
-
- CollectionSupport cs = graph.getService(CollectionSupport.class);
-
- final ConcurrentLinkedQueue<Resource> composedResult = new ConcurrentLinkedQueue<Resource>();
- final ConcurrentLinkedQueue<Resource> singleResult = new ConcurrentLinkedQueue<Resource>();
- final ConcurrentLinkedQueue<Pair<Resource, Resource>> singles = new ConcurrentLinkedQueue<Pair<Resource, Resource>>();
-
- final Set<Resource> schedule = cs.createSet();
-
- for (Collection<DirectStatements> coll : expansion)
- for (DirectStatements stms : coll)
- for(Statement stm : stms) {
- Resource predicate = stm.getPredicate();
- if(predicates.add(predicate)) schedule.add(predicate);
- }
-
- // Discover singles
- graph.syncRequest(new AsyncReadRequest() {
-
- @Override
- public void run(AsyncReadGraph graph) {
-
- for (final Resource predicate : schedule) {
-
- graph.forPossibleSuperrelation(predicate, new AsyncProcedure<Resource>() {
-
- @Override
- public void exception(AsyncReadGraph graph, Throwable throwable) {
- throwable.printStackTrace();
- }
-
- @Override
- public void execute(AsyncReadGraph graph, final Resource single) {
- singles.add(Pair.make(predicate, single));
- }
-
- });
-
- }
-
- }
-
- });
-
- // Determine singles
- final Set<Resource> singleSchedule = cs.createSet();
- for(Pair<Resource, Resource> pair : singles) {
-
- Resource single = pair.second;
- if(single != null && predicates.add(single)) singleSchedule.add(single);
-
- }
-
- graph.syncRequest(new AsyncReadRequest() {
-
- @Override
- public void run(AsyncReadGraph graph) {
-
- for (final Resource predicate : singleSchedule) {
-
- graph.forIsSubrelationOf(predicate, graph.getService(Layer0.class).IsComposedOf, new AsyncProcedure<Boolean>() {
-
- @Override
- public void exception(AsyncReadGraph graph, Throwable throwable) {
- throwable.printStackTrace();
- }
-
- @Override
- public void execute(AsyncReadGraph graph, Boolean composed) {
- if (composed) singleResult.add(predicate);
- }
-
- });
-
- }
-
- }
-
- });
-
- composedPredicates.addAll(singleResult);
-
- final Set<Resource> specialSchedule = cs.createSet();
-
- // Classify
- for(Pair<Resource, Resource> pair : singles) {
-
- Resource single = pair.second;
- if(single != null) {
- if(composedPredicates.contains(single)) {
- composedPredicates.add(pair.first);
- }
- } else {
- specialSchedule.add(pair.first);
- }
-
- }
-
- graph.syncRequest(new AsyncReadRequest() {
-
- @Override
- public void run(AsyncReadGraph graph) {
-
- for (final Resource predicate : specialSchedule) {
-
- graph.forIsSubrelationOf(predicate, graph.getService(Layer0.class).IsComposedOf, new AsyncProcedure<Boolean>() {
-
- @Override
- public void exception(AsyncReadGraph graph, Throwable throwable) {
- throwable.printStackTrace();
- }
-
- @Override
- public void execute(AsyncReadGraph graph, Boolean composed) {
- if (composed) composedResult.add(predicate);
- }
-
- });
-
- }
-
- }
-
- });
-
- composedPredicates.addAll(composedResult);
-
- composedPredicateTime += (System.nanoTime() - start);
-
- }
-
- /*
- * Composed objects are internal. Mark them for expansion.
- */
-
- public void collectComposedObjects(ReadGraph graph, Collection<DirectStatements>[] expansion, Set<Resource> typeTodo, Set<Resource> objectTodo,
- Set<Resource> predicateTodo) throws DatabaseException {
-
- long start = System.nanoTime();
-
- Layer0 l0 = Layer0.getInstance(graph);
-
- for (Collection<DirectStatements> coll : expansion)
- for (DirectStatements stms : coll)
- for(Statement stm : stms) {
-
- Resource predicate = stm.getPredicate();
- Resource object = stm.getObject();
-
- if (composedPredicates.contains(predicate)) {
-
- ExtentStatus existing = status.put(object, ExtentStatus.INTERNAL);
- if(existing == null) {
- ids.put(object, id++);
- composedObjectCounter++;
- expansionSeeds.add(object);
-// System.err.println("internal: " + NameUtils.getSafeName(graph, object, true));
- if(LOG) log("[INTERNAL] (composed object) " + NameUtils.getSafeName(graph, object, true));
- } else if (existing == ExtentStatus.EXCLUDED) {
- System.err.println("preExcluded: " + NameUtils.getSafeName(graph, object, true));
- status.put(object, ExtentStatus.EXCLUDED);
- } else if (existing == ExtentStatus.EXTERNAL) {
- System.err.println("preExternal: " + NameUtils.getSafeName(graph, object, true));
- status.put(object, ExtentStatus.EXTERNAL);
- }
-
- } else {
-
-// System.err.println("internal2: " + NameUtils.getSafeName(graph, object, true));
-
- if (!status.containsKey(object)) {
- if (l0.InstanceOf.equalsResource(predicate)) {
- typeTodo.add(object);
- } else {
- objectTodo.add(object);
- }
- }
-
- if (!status.containsKey(predicate)) {
- predicateTodo.add(predicate);
- }
-
- }
- }
-
- composedObjectTime += System.nanoTime() - start;
-
- }
-
- public void writeOtherStatements(ReadGraph graph, Collection<Collection<DirectStatements>[]> expansion, ObjectOutputStream composedStatementsOutput, ObjectOutputStream otherStatementsOutput,
- ObjectOutputStream valueOutput) throws DatabaseException {
-
- long start = System.nanoTime();
-
- Layer0 l0 = Layer0.getInstance(graph);
- SerialisationSupport support = graph.getService(SerialisationSupport.class);
- TransferableGraphSupport tgs = graph.getService(TransferableGraphSupport.class);
-
- TIntArrayList other = new TIntArrayList();
- TIntArrayList composed = new TIntArrayList();
-
- try {
-
- for (Collection<DirectStatements>[] colls : expansion)
- for (Collection<DirectStatements> coll : colls)
- for (DirectStatements stms : coll) {
-
- Resource subject = stms.getSubject();
- composed.resetQuick();
-
- int sId = support.getTransientId(subject);
-
- composedStatementsOutput.writeInt(sId);
-
- if(graph.hasValue(subject)) {
- Datatype dt = graph.getRelatedValue(subject, l0.HasDataType, Bindings.getBindingUnchecked(Datatype.class));
- Binding b = Bindings.getBinding(dt);
- Object _value = graph.getValue(subject, b);
- Variant variant = new Variant(b, _value);
- byte[] value = variantSerializer.serialize(variant);
- if(LOG) log("[VALUE] " + NameUtils.getSafeName(graph, subject));
- valueOutput.writeInt(sId);
- valueOutput.writeInt(value.length);
- assert (value.length > 0);
- valueOutput.write(value);
- }
-
- for (Statement s : stms) {
-
- Resource object = s.getObject();
- Resource predicate = s.getPredicate();
-
- ExtentStatus objectStatus = status.get(object);
-
- if(objectStatus == ExtentStatus.INTERNAL) {
- composed.add(support.getTransientId(predicate));
- composed.add(support.getTransientId(object));
- if(LOG) log("[COMPOSED] (internal object) " + NameUtils.toIdString(graph, s));
- } else if (l0.InstanceOf.equalsResource(predicate)) {
- composed.add(support.getTransientId(predicate));
- composed.add(support.getTransientId(object));
- if(LOG) log("[COMPOSED] (instanceOf) " + NameUtils.toIdString(graph, s));
- } else if (l0.SubrelationOf.equalsResource(predicate)) {
- composed.add(support.getTransientId(predicate));
- composed.add(support.getTransientId(object));
- if(LOG) log("[COMPOSED] (subrelationOf) " + NameUtils.toIdString(graph, s));
- } else {
- if(objectStatus == ExtentStatus.EXTERNAL) {
- if(DEBUG)
- System.out.println("other " + NameUtils.toIdString(graph, s));
- //System.out.println("other.add " + predicate + " - " + object);
- other.add(support.getTransientId(predicate));
- other.add(support.getTransientId(object));
- if(LOG) log("[OTHER] (object is external) " + NameUtils.toIdString(graph, s));
- }
- }
-
- }
-
- if(!other.isEmpty()) {
- otherStatementsOutput.writeInt(sId);
- otherStatementsOutput.writeInt(other.size() / 2);
- for (int i = 0; i < other.size(); i++)
- otherStatementsOutput.writeInt(other.getQuick(i));
- other.resetQuick();
- }
-
- composedStatementsOutput.writeInt(composed.size() / 2);
- for (int i = 0; i < composed.size(); i++)
- composedStatementsOutput.writeInt(composed.getQuick(i));
-
- }
-
- } catch (IOException e) {
- e.printStackTrace();
- }
-
- otherStatementTime += (System.nanoTime() - start);
-
- }
-
- boolean hasStrictParents(ReadGraph g, Resource r)
- throws DatabaseException {
- if (g.getPossibleURI(r) != null)
- return true;
- return false;
- }
-
- public boolean getExpansionSeedsFromExtents(ReadGraph graph, final Collection<DirectStatements>[] expansion) throws DatabaseException {
-
- long start = System.nanoTime();
-
- final ConcurrentLinkedQueue<Resource> accepts = new ConcurrentLinkedQueue<Resource>();
-
- /*
- * Determine statements which could accept statements with todo
- * objects
- */
- search: for (Double priority : priorityList) {
-
- for (final SubgraphAdvisor advisor : advisors) {
-
- if (advisor.priority() > 0)
- continue;
-
- if (advisor.priority() == priority) {
-
- graph.syncRequest(new ReadRequest() {
-
- @Override
- public void run(ReadGraph graph) throws DatabaseException {
-
- for (Collection<DirectStatements> coll : expansion)
- for (DirectStatements stms : coll)
- for(final Statement stm : stms) {
-
- advisor.advise(graph, stm, new AsyncProcedure<Boolean>() {
-
- @Override
- public void exception(AsyncReadGraph graph, Throwable throwable) {
- throwable.printStackTrace();
- }
-
- @Override
- public void execute(AsyncReadGraph graph, Boolean accept) {
- if (accept) {
- accepts.add(stm.getObject());
- }
- }
-
- });
-
- }
-
- }
-
- });
- }
- if (!accepts.isEmpty())
- break search;
- }
- }
-
- CollectionSupport cs = graph.getService(CollectionSupport.class);
- Set<Resource> schedule = cs.createSet();
- for (Resource r : accepts) {
- if(!status.containsKey(r))
- schedule.add(r);
- }
-
- extentSeedTime += (System.nanoTime() - start);
-
- if (schedule.isEmpty())
- return false;
-
- fastResolve(graph, schedule);
- uriResolve(graph, schedule);
- fullResolve(graph, schedule, "accepts");
-
- return true;
-
- }
-
- ConcurrentLinkedQueue<Resource> fastInternals = new ConcurrentLinkedQueue<Resource>();
-
- public void fastResolve(ReadGraph graph, final Set<Resource> rs)
- throws DatabaseException {
- // This collects and resolves weaks
- if(fastResolveLoop(graph, rs))
- // Weaks are now resolved
- fastResolveLoop(graph, rs);
- }
-
- public boolean fastResolveLoop(ReadGraph graph, final Set<Resource> rs)
- throws DatabaseException {
-
- long start = System.nanoTime();
-
- final ConcurrentLinkedQueue<Resource> weakSchedule = new ConcurrentLinkedQueue<Resource>();
-
- graph.syncRequest(new AsyncRead<Boolean>() {
-
- @Override
- public int threadHash() {
- return hashCode();
- }
-
- @Override
- public int getFlags() {
- return 0;
- }
-
- @Override
- public void perform(AsyncReadGraph graph,
- AsyncProcedure<Boolean> procedure) {
-
- QueryControl control = graph.getService(QueryControl.class);
- final DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class);
-
- int slice = (int) (rs.size() / control
- .getAmountOfQueryThreads()) + 1;
-
- final Resource[] rootArray = rs.toArray(Resource.NONE);
- for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
-
- final int start = i * slice;
- final int end = Math.min(start + slice,
- rootArray.length);
-
- control.schedule(graph, i, new ControlProcedure() {
-
- @Override
- public void execute(AsyncReadGraph graph) {
-
- for (int index = start; index < end; index++) {
-
- final Resource r = rootArray[index];
-
- //if (status.containsKey(r)) continue;
-
- graph.asyncRequest(new FastInternalRequest(dqs, r, status, weakInverses, weakSchedule),new AsyncProcedure<Boolean>() {
-
- @Override
- public void exception(AsyncReadGraph graph,Throwable throwable) {
- throwable.printStackTrace();
- }
-
- @Override
- public void execute(AsyncReadGraph graph,Boolean isInternal) {
- if (isInternal) {
- fastInternals.add(r);
- }
- }
-
- });
-
- }
-
- }
-
- });
-
- }
-
- procedure.execute(graph, true);
-
- }
-
- });
-
- if (!weakSchedule.isEmpty()) {
- THashSet<Resource> weaks = new THashSet<Resource>(weakSchedule);
- if (CLASSIFY_LOG)
- for (Resource p : weakSchedule)
- System.out.println("classify "
- + NameUtils.getSafeName(graph, p));
- graph.syncRequest(new ClassifyStatementsRequest(weaks, weakInverses));
- }
-
- for (Resource r : fastInternals) {
- rs.remove(r);
- if (status.put(r, ExtentStatus.INTERNAL) == null) {
- if(LOG) log("[INTERNAL] (fast) " + NameUtils.getSafeName(graph, r, true));
- ids.put(r, id++);
- fastInternalCounter++;
- expansionSeeds.add(r);
- }
- }
-
- fastResolveTime += (System.nanoTime() - start);
-
- return !weakSchedule.isEmpty();
-
- }
-
- private ExtentStatus resolveExtent(ReadGraph graph, Resource r, Map<Resource, ExtentStatus> statuses, Set<Resource> expansionSeeds, THashSet<Resource> pending,
- ArrayList<Resource> stack) throws DatabaseException {
-
- ExtentStatus current = statuses.get(r);
- if(current != null) return current;
-
- if (pending.contains(r))
- return ExtentStatus.PENDING;
-
- // In order to break cyclic dependencies
- pending.add(r);
-
- if (PARENT_DEBUG)
- System.out.println("resolveExtent "
- + NameUtils.getSafeName(graph, r));
-
- ExtentStatus status = ExtentStatus.INTERNAL;
- for (Resource p : getParents(graph, r)) {
- if (PARENT_DEBUG) {
- ExtentStatus ps = statuses.get(p);
- System.out.println(" parent " + NameUtils.getSafeName(graph, p) + "(" + ps + ")");
- }
- switch (resolveExtent(graph, p, statuses,
- expansionSeeds, pending, stack)) {
- case EXTERNAL:
- return ExtentStatus.EXTERNAL;
- case PENDING:
- status = ExtentStatus.PENDING;
- }
- }
- if (status == ExtentStatus.INTERNAL) {
- pending.remove(r);
- stack.add(r);
- if (DEBUG)
- System.out.println(NameUtils.getSafeName(graph, r, true)
- + " is internal.");
- }
- return status;
- }
-
- public void uriResolve(ReadGraph graph, final Set<Resource> todo)
- throws DatabaseException {
-
- long start = System.nanoTime();
-
- for(Resource r : todo) System.out.println("uriResolve " +
- NameUtils.getSafeName(graph, r));
-
- final ConcurrentSkipListSet<Resource> found = new ConcurrentSkipListSet<Resource>();
-
- graph.syncRequest(new AsyncReadRequest() {
-
- @Override
- public void run(AsyncReadGraph graph) {
-
- for (final Resource r : todo) {
-
- // System.out.println("uriresolve before " + r);
-
- if (status.containsKey(r)) continue;
-
- // System.out.println("uriresolve " + r);
-
- graph.forURI(r, new AsyncProcedure<String>() {
-
- @Override
- public void exception(AsyncReadGraph graph, Throwable throwable) {
- throwable.printStackTrace();
- }
-
- @Override
- public void execute(AsyncReadGraph graph, String uri) {
-
- if (uri != null) {
-
- // System.out.println("uriresolve has uri "
- // + r);
-
- if(found.add(r)) {
- parentExternalCounter++;
- }
-
- } else {
-
- // System.out.println("uriresolve ask inverse "
- // + r);
-
- graph.forPossibleInverse(r, new AsyncProcedure<Resource>() {
-
- @Override
- public void exception(AsyncReadGraph graph, Throwable throwable) {
- throwable.printStackTrace();
- }
-
- @Override
- public void execute(AsyncReadGraph graph, Resource inverse) {
-
- if (inverse != null) {
-
- graph.forURI(inverse, new AsyncProcedure<String>() {
-
- @Override
- public void exception(AsyncReadGraph graph, Throwable throwable) {
- throwable.printStackTrace();
- }
-
- @Override
- public void execute(AsyncReadGraph graph, String uri) {
-
- if (uri != null) {
-
- if(found.add(r)) {
- parentExternalCounter++;
- }
-
- }
-
- }
-
- });
-
- }
-
- }
-
- });
-
- }
- }
-
- });
-
- }
-
- }
-
- });
-
- todo.removeAll(found);
- for(Resource r : found) {
- status.put(r, ExtentStatus.EXTERNAL);
- if(LOG) log("[EXTERNAL] (uriResolve) " + NameUtils.getSafeName(graph, r, true));
- }
-
- parentResolveTime += System.nanoTime() - start;
-
- }
-
- public void fullResolve(ReadGraph graph, Collection<Resource> rs,
- String koss) throws DatabaseException {
-
- long start = System.nanoTime();
-
- for (final Resource r : rs) {
-
- if(status.containsKey(r)) continue;
-
- THashSet<Resource> pending = new THashSet<Resource>();
- ArrayList<Resource> stack = new ArrayList<Resource>();
-
- ExtentStatus s = resolveExtent(graph, r, status, expansionSeeds, pending, stack);
- if (ExtentStatus.INTERNAL == s || ExtentStatus.PENDING == s) {
- if (status.put(r, ExtentStatus.INTERNAL) == null) {
- if(LOG) log("[INTERNAL] (resolveExtent) " + NameUtils.getSafeName(graph, r, true));
- ids.put(r, id++);
- fullInternalCounter++;
- expansionSeeds.add(r);
- }
- }
- if (ExtentStatus.EXTERNAL == s) {
- if (status.put(r, ExtentStatus.EXTERNAL) == null) {
- if(LOG) log("[EXTERNAL] (resolveExtent) " + NameUtils.getSafeName(graph, r, true));
- fullExternalCounter++;
- }
- }
-
- }
-
- fullResolveTime += (System.nanoTime() - start);
-
- }
-
- public void process(ReadGraph graph,
- ObjectOutputStream composedStatementsOutput,
- ObjectOutputStream otherStatementsOutput,
- ObjectOutputStream valueOutput)
- throws DatabaseException {
-
- this.variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT);
-
- CollectionSupport cs = graph.getService(CollectionSupport.class);
-
- Set<Resource> typeTodo = cs.createSet();
- Set<Resource> objectTodo = cs.createSet();
- Set<Resource> predicateTodo = cs.createSet();
-
- Collection<Collection<DirectStatements>[]> fullExpansion = new ArrayList<Collection<DirectStatements>[]>();
-
- do {
-
- QueryControl control = graph.getService(QueryControl.class);
- Collection<DirectStatements>[] expansion = new ArrayList[control.getAmountOfQueryThreads()];
- for (int i = 0; i < control.getAmountOfQueryThreads(); i++) {
- expansion[i] = new ArrayList<DirectStatements>();
- }
-
- // Expand expansionSeeds
- expand(graph, expansion, objectTodo);
-
- // Start collecting new seeds
- expansionSeeds = cs.createSet();
-
- // Collect predicates which are <R IsComposedOf
- extractComposedPredicates(graph, expansion);
-
- // Make composed objects internal and make sure they are further
- // expanded
- collectComposedObjects(graph, expansion, typeTodo, objectTodo, predicateTodo);
-
- /*
- * Use the expansion seed heuristic to find new resources to
- * expand before full analysis.
- */
- getExpansionSeedsFromExtents(graph, expansion);
-
- fullExpansion.add(expansion);
-
- } while (!expansionSeeds.isEmpty());
-
- fastResolve(graph, objectTodo);
- uriResolve(graph, predicateTodo);
- uriResolve(graph, objectTodo);
- uriResolve(graph, typeTodo);
- fullResolve(graph, objectTodo, "objectTodo");
- fullResolve(graph, predicateTodo, "predicateTodo");
- fullResolve(graph, typeTodo, "typeTodo");
-
- writeOtherStatements(graph, fullExpansion, composedStatementsOutput, otherStatementsOutput, valueOutput);
-
- if (PROFILE) {
- System.out.println(composedObjectCounter + " " + fastInternalCounter
- + " " + parentExternalCounter + " "
- + fullExternalCounter + " " + fullInternalCounter);
- }
-
- }
-
- }
-
- public static void getDomain2(ReadGraph graph, TIntIntHashMap ids,
- Collection<Resource> roots, Map<Resource, ExtentStatus> preStatus,
- Map<Resource, Statement> specials,
- ObjectOutputStream otherStatementsOutput,
- ObjectOutputStream valueOutput,
- TreeMap<String, Variant> extensions,
- TIntHashSet excludedShared) throws DatabaseException {
-
- ITask task = ThreadLogger.getInstance().begin("getDomain2");
-
- final DomainProcessor2 processor = new DomainProcessor2();
-
- processor.startupTime = System.nanoTime();
-
- Layer0 l0 = Layer0.getInstance(graph);
-
- CollectionSupport cs = graph.getService(CollectionSupport.class);
- SerialisationSupport support = graph.getService(SerialisationSupport.class);
-
- processor.ids = ids;
- processor.specials = specials;
- processor.status = cs.createMap(ExtentStatus.class);
- processor.weakInverses = cs.createMap(WeakStatus.class);
- processor.predicates = cs.createSet();
- processor.isRelatedToPredicates = cs.createSet();
- processor.sharedPredicates = cs.createSet();
-// processor.expansionSeeds = cs.createSet();
-
- for(Map.Entry<Resource, ExtentStatus> entry : preStatus.entrySet()) {
- processor.status.put(entry.getKey(), entry.getValue());
- if(ExtentStatus.EXCLUDED.equals(entry.getValue())) processor.exclusions.add(entry.getKey());
- }
-
-// for (Resource r : excluded) {
-// processor.status.put(r, ExtentStatus.EXCLUDED);
-// }
-
- Resource rootLibrary = graph.getResource("http:/");
-
- if (!roots.contains(rootLibrary))
- processor.status.put(rootLibrary, ExtentStatus.EXTERNAL);
-
- for (Resource root : roots) {
- processor.status.put(root, ExtentStatus.INTERNAL);
- //processor.ids.put(support.getTransientId(root), processor.ids.size());
- for (Resource owner : graph.getObjects(root, l0.IsOwnedBy)) {
- processor.status.put(owner, ExtentStatus.EXTERNAL);
- }
- }
-
- processor.startupTime = System.nanoTime() - processor.startupTime;
-
- processor.fringe = new HashSet<Resource>();
- processor.fringe.addAll(roots);
-
- processor.internalDomain.addAll(roots);
-
- processor.sharedExternalReferences = new HashSet<Resource>();
- processor.sharedExternalFringe = new HashSet<Resource>();
-
- try {
-
- processor.process(graph, otherStatementsOutput, valueOutput);
-
- } catch (IOException e) {
- e.printStackTrace();
- }
-
- for(Resource r : processor.sharedExternalReferences) excludedShared.add(support.getTransientId(r));
-
- ClusteringSupport cls = graph.getService(ClusteringSupport.class);
- TLongObjectHashMap<TIntArrayList> clusterMap = new TLongObjectHashMap<TIntArrayList>();
- for(Map.Entry<Resource, ExtentStatus> entry : processor.status.entrySet()) {
- if(ExtentStatus.INTERNAL == entry.getValue()) {
-
- long cluster = cls.getCluster(entry.getKey());
- TIntArrayList list = clusterMap.get(cluster);
- if(list == null) {
- list = new TIntArrayList();
- clusterMap.put(cluster, list);
- }
- list.add(support.getTransientId(entry.getKey()));
-
- }
- }
- final TIntArrayList clustering = new TIntArrayList();
- clusterMap.forEachEntry(new TLongObjectProcedure<TIntArrayList>() {
-
- @Override
- public boolean execute(long cluster, TIntArrayList b) {
- clustering.add(b.size());
- b.forEach(new TIntProcedure() {
-
- @Override
- public boolean execute(int rId) {
- processor.ids.put(rId, processor.id++);
- return true;
- }
-
- });
- return true;
- }
-
- });
-
- extensions.put(Extensions.CLUSTERING, new Variant(Bindings.INT_ARRAY, clustering.toArray()));
-
- long total = processor.startupTime + processor.expandTime
- + processor.composedPredicateTime
- + processor.composedObjectTime + processor.extentSeedTime
- + processor.fullResolveTime + processor.fastResolveTime +
- + processor.parentResolveTime + processor.otherStatementTime;
-
- if (PROFILE) {
- System.out.println("startup took " + 1e-9 * processor.startupTime
- + "s.");
- System.out.println("expand took " + 1e-9 * processor.expandTime
- + "s.");
- System.out.println("composedPredicates took " + 1e-9
- * processor.composedPredicateTime + "s.");
- System.out.println("composedObjects took " + 1e-9
- * processor.composedObjectTime + "s.");
- System.out.println("extentSeeding took " + 1e-9
- * processor.extentSeedTime + "s.");
- System.out.println("fullResolve took " + 1e-9
- * processor.fullResolveTime + "s.");
- System.out.println("fastResolve took " + 1e-9
- * processor.fastResolveTime + "s.");
- System.out.println("parentResolve took " + 1e-9
- * processor.parentResolveTime + "s.");
- System.out.println("otherStatements took " + 1e-9
- * processor.otherStatementTime + "s.");
- System.out.println("total " + 1e-9 * total + "s.");
- }
-
- task.finish();
-
-
- }
-
-
- public static void getDomain(ReadGraph graph, Map<Resource, Integer> ids,
- Collection<Resource> roots, Map<Resource, ExtentStatus> preStatus, Set<SubgraphAdvisor> advisors,
- ObjectOutputStream composedStatementsOutput,
- ObjectOutputStream otherStatementsOutput,
- ObjectOutputStream valueOutput) throws DatabaseException {
-
- ITask task = ThreadLogger.getInstance().begin("getDomain");
-
- DomainProcessor processor = new DomainProcessor(advisors);
-
- processor.startupTime = System.nanoTime();
-
- Layer0 l0 = Layer0.getInstance(graph);
-
- CollectionSupport cs = graph.getService(CollectionSupport.class);
-
- processor.ids = ids;
- processor.status = cs.createMap(ExtentStatus.class);
- processor.weakInverses = cs.createMap(WeakStatus.class);
- processor.predicates = cs.createSet();
- processor.composedPredicates = cs.createSet();
- processor.expansionSeeds = cs.createSet();
-
- for(Map.Entry<Resource, ExtentStatus> entry : preStatus.entrySet()) {
- processor.status.put(entry.getKey(), entry.getValue());
- }
-
-// for (Resource r : excluded) {
-// processor.status.put(r, ExtentStatus.EXCLUDED);
-// }
-
- if (!roots.contains(graph.getRootLibrary()))
- processor.status.put(graph.getRootLibrary(), ExtentStatus.EXTERNAL);
-
- for (Resource root : roots) {
- processor.status.put(root, ExtentStatus.INTERNAL);
- processor.ids.put(root, processor.id++);
- for (Resource owner : graph.getObjects(root, l0.IsOwnedBy)) {
- processor.status.put(owner, ExtentStatus.EXTERNAL);
- }
- }
-
-
- processor.expansionSeeds.addAll(roots);
-
- processor.startupTime = System.nanoTime() - processor.startupTime;
-
- while (!processor.expansionSeeds.isEmpty()) {
-
- processor.process(graph, composedStatementsOutput,
- otherStatementsOutput, valueOutput);
-
- }
-
- long total = processor.startupTime + processor.expandTime
- + processor.composedPredicateTime
- + processor.composedObjectTime + processor.extentSeedTime
- + processor.fullResolveTime + processor.fastResolveTime +
- + processor.parentResolveTime + processor.otherStatementTime;
-
- if (PROFILE) {
- System.out.println("startup took " + 1e-9 * processor.startupTime
- + "s.");
- System.out.println("expand took " + 1e-9 * processor.expandTime
- + "s.");
- System.out.println("composedPredicates took " + 1e-9
- * processor.composedPredicateTime + "s.");
- System.out.println("composedObjects took " + 1e-9
- * processor.composedObjectTime + "s.");
- System.out.println("extentSeeding took " + 1e-9
- * processor.extentSeedTime + "s.");
- System.out.println("fullResolve took " + 1e-9
- * processor.fullResolveTime + "s.");
- System.out.println("fastResolve took " + 1e-9
- * processor.fastResolveTime + "s.");
- System.out.println("parentResolve took " + 1e-9
- * processor.parentResolveTime + "s.");
- System.out.println("otherStatements took " + 1e-9
- * processor.otherStatementTime + "s.");
- System.out.println("total " + 1e-9 * total + "s.");
- }
-
- task.finish();
-
- }
-
-}
Code Review / simantics / platform.git / blobdiff