X-Git-Url: https://gerrit.simantics.org/r/gitweb?p=simantics%2Fplatform.git;a=blobdiff_plain;f=bundles%2Forg.simantics.db.layer0%2Fsrc%2Forg%2Fsimantics%2Fdb%2Flayer0%2Futil%2FTransferableGraphRequest.java;fp=bundles%2Forg.simantics.db.layer0%2Fsrc%2Forg%2Fsimantics%2Fdb%2Flayer0%2Futil%2FTransferableGraphRequest.java;h=6b31692a1661bd8752c5ad91605ba1ef3ee5d8e6;hp=29533371e0947e9b7effc0fc44af5267185030c2;hb=0ae2b770234dfc3cbb18bd38f324125cf0faca07;hpb=24e2b34260f219f0d1644ca7a138894980e25b14 diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TransferableGraphRequest.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TransferableGraphRequest.java index 29533371e..6b31692a1 100644 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TransferableGraphRequest.java +++ b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TransferableGraphRequest.java @@ -1,748 +1,748 @@ -/******************************************************************************* - * Copyright (c) 2007, 2010 Association for Decentralized Information Management - * in Industry THTH ry. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Eclipse Public License v1.0 - * which accompanies this distribution, and is available at - * http://www.eclipse.org/legal/epl-v10.html - * - * Contributors: - * VTT Technical Research Centre of Finland - initial API and implementation - *******************************************************************************/ -package org.simantics.db.layer0.util; - -import gnu.trove.list.array.TIntArrayList; - -import java.io.ByteArrayInputStream; -import java.io.DataOutput; -import java.io.DataOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ConcurrentSkipListSet; -import java.util.concurrent.atomic.AtomicInteger; - -import org.apache.commons.io.output.DeferredFileOutputStream; -import org.simantics.databoard.Bindings; -import org.simantics.databoard.Databoard; -import org.simantics.databoard.binding.mutable.Variant; -import org.simantics.databoard.serialization.Serializer; -import org.simantics.db.AsyncReadGraph; -import org.simantics.db.ReadGraph; -import org.simantics.db.Resource; -import org.simantics.db.Statement; -import org.simantics.db.common.StandardStatement; -import org.simantics.db.common.utils.NameUtils; -import org.simantics.db.exception.DatabaseException; -import org.simantics.db.exception.ValidationException; -import org.simantics.db.layer0.adapter.Instances; -import org.simantics.db.layer0.adapter.SubgraphAdvisor; -import org.simantics.db.layer0.adapter.SubgraphExtent; -import org.simantics.db.layer0.adapter.SubgraphExtent.Classifier; -import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus; -import org.simantics.db.procedure.AsyncProcedure; -import org.simantics.db.request.AsyncRead; -import org.simantics.db.request.Read; -import org.simantics.db.service.CollectionSupport; -import org.simantics.db.service.QueryControl; -import org.simantics.db.service.QueryControl.ControlProcedure; -import org.simantics.db.service.SerialisationSupport; -import org.simantics.graph.representation.External; -import org.simantics.graph.representation.Identity; -import org.simantics.graph.representation.Root; -import org.simantics.graph.representation.TransferableGraph1; -import org.simantics.graph.representation.Value; -import org.simantics.layer0.Layer0; -import org.simantics.operation.Layer0X; -import org.simantics.utils.datastructures.Pair; -import org.simantics.utils.datastructures.Triple; - -/** - * IsComposedOf objects are always in domain - * Ordered set elements are always in domain - * r is in domain if for all statements (s, IsRelatedTo, r) s is in domain - * - * IsWeaklyRelatedTo statements where subject is in domain are only accepted if object is in domain or has URI - * - * @deprecated in favor of {@link TransferableGraphRequest2} - */ -public class TransferableGraphRequest implements Read { - - public static String LOG_FILE = "transferableGraph.log"; - final static private boolean LOG = false; - final static private boolean DEBUG = false; - final static private boolean PROFILE = false; - -// final private Collection> roots; -// private Collection preExternals = Collections.emptyList(); -// private Resource model; - - private TransferableGraphConfiguration configuration; - - static DataOutput log; - - static { - - if (LOG) { - try { - FileOutputStream stream = new FileOutputStream(LOG_FILE); - log = new DataOutputStream(stream); - } catch (FileNotFoundException e) { - e.printStackTrace(); - } - } - - } - - private static void log(String line) { - if (LOG) { - try { - log.writeUTF(line + "\n"); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - - - - public TransferableGraphRequest(Collection> roots, Resource model) { - - configuration = new TransferableGraphConfiguration(); - configuration.roots = roots; - configuration.model = model; - -// this.roots = roots; -// this.model = model; - - } - - - public TransferableGraphRequest(Collection> roots) { - this(roots, null); - } - - public TransferableGraphRequest(TransferableGraphConfiguration conf) { - this.configuration = conf; - } - - Layer0 l0; - - TIntArrayList inverses = new TIntArrayList(); - int statements[]; - int statementIndex = 0; - Map ids; - Map values; - TIntArrayList externalParents = new TIntArrayList(); - ArrayList externalNames = new ArrayList(); -// ConcurrentLinkedQueue> values = new ConcurrentLinkedQueue>(); - int id = 0; - int internalCount; - - int indent = 0; - - private Serializer variantSerializer; - - private boolean validateExternal(Resource r) { - if(configuration.disallowedExternals != null) { - System.err.println("validateExternal agains " + configuration.disallowedExternals); - return !configuration.disallowedExternals.contains(r); - } - return true; - } - - public int getId(ReadGraph graph, Resource r, Resource predicate) throws DatabaseException { -// for(int i=0;i parents = graph.getObjects(r, l0.PartOf); - if(parents.size() != 1) { - if(parents.size() == 0) { - Resource inv = graph.getPossibleObject(r, l0.InverseOf); - if(inv != null) { - ++indent; - if(DEBUG) System.out.println("inverse " + NameUtils.getSafeName(graph, inv)); - int invId = getId(graph, inv, null); - externalParents.add(invId); - --indent; - externalNames.add("@inverse"); - ids.put(r, id); - return id++; - } - } -// for(Statement stat : graph.getStatements(r, b.IsWeaklyRelatedTo)) -// System.out.println(NameUtils.getSafeName(graph, stat.getPredicate()) -// + " -> " + NameUtils.getSafeName(graph, stat.getObject())); -// if(predicate != null) { -// if(!graph.isSubrelationOf(predicate, b.IsRelatedTo)) { -// return -2; -// } -// } - throw new ValidationException("Reference to external resource " - + NameUtils.getSafeName(graph, r, true) + " without unique uri (" + parents.size() + " parents)."); - } - for(Resource p : parents) { -// System.out.println("Parent " + NameUtils.getSafeName(graph, p)); - ++indent; - if(!validateExternal(p)) throw new ValidationException("References to '" + graph.getURI(p) + "' are not allowed."); - externalParents.add(getId(graph, p, null)); - --indent; - } -// System.out.println("Request named for " + GraphUtils.getReadableName(g, r)); -// String name = graph.getPossibleRelatedValue(r, b.HasName); - externalNames.add((String)graph.getRelatedValue(r, l0.HasName)); -// if(name != null) externalNames.add(name); -// else externalNames.add("@resource"); - ids.put(r, id); - return id++; - } - } - - public void addId(ReadGraph graph, Resource r, Resource predicate) throws DatabaseException { - statements[statementIndex++] = getId(graph, r, predicate); - } - - private void searchStatementPart(ReadGraph graph, final ConcurrentSkipListSet statementSet, final ArrayList> part, final Set extents) throws DatabaseException { - - final SubgraphExtent.Callback callback = new SubgraphExtent.Callback() { - - @Override - public void statement(Statement statement, boolean accept) { - if(accept) { - statementSet.add(statement); - } - } - - }; - - graph.syncRequest(new AsyncRead() { - - @Override - public int threadHash() { - return hashCode(); - } - - @Override - public void perform(AsyncReadGraph graph, AsyncProcedure procedure) { - - QueryControl control = graph.getService(QueryControl.class); - - int slice = (int)(part.size() / control.getAmountOfQueryThreads()) + 1; - - final Pair[] rootArray = (Pair[])part.toArray(new Pair[part.size()]); - for(int i=0;i r = (Pair)rootArray[index]; - - final AtomicInteger position = new AtomicInteger(0); - final SubgraphExtent.Classifier[] classifiers = new SubgraphExtent.Classifier[extents.size()]; - - for(SubgraphExtent extent : extents) { - - extent.accept(graph, r.first, new AsyncProcedure() { - - public void execute(AsyncReadGraph graph, Classifier c) { - - int current = position.incrementAndGet(); - classifiers[current-1] = c; - - if(current == extents.size()) { - - for(Triple statement : r.second) { - if(!r.first.isPersistent()) continue; - for(Classifier classifier : classifiers) { - classifier.classify(graph, new StandardStatement(r.first, (Resource)statement.first, (Resource)statement.second), (ExtentStatus)statement.third, callback); - } - } - -// graph.forEachDirectStatement(r, new AsyncMultiProcedure() { -// -// @Override -// public void exception(AsyncReadGraph graph, Throwable throwable) { -// throwable.printStackTrace(); -// } -// -// @Override -// public void execute(AsyncReadGraph graph, Statement statement) { -// } -// -// @Override -// public void finished(AsyncReadGraph graph) { -// } -// -// }); - - } - - } - - public void exception(AsyncReadGraph graph, Throwable throwable) { - throwable.printStackTrace(); - } - - }, callback); - - } - - } - - } - - }); - - } - - procedure.execute(graph, false); - - } - - @Override - public int getFlags() { - return 0; - } - - }); - - } - - class StatementSetBuilder { - - private int SLICE = 100000; - - final private CollectionSupport cs; - final private LinkedList> sets = new LinkedList>(); - - private Collection current; - - StatementSetBuilder(CollectionSupport cs) { - this.cs = cs; - current = cs.createStatementList(); - } - - LinkedList> get() { - sets.add(current); - return sets; - } - - void add(Statement stm) { - current.add(stm); - if(current.size() == SLICE) { - sets.add(current); - current = cs.createStatementList(); - } - } - - int size() { - int result = 0; - for(Collection c : sets) result += c.size(); - return result; - } - - - void addAll(Collection ss) { - for(Statement s : ss) add(s); - } - - } - - private void searchStatements(ReadGraph graph, ObjectInputStream composedInput, ObjectInputStream statementInput, ObjectInputStream valueInput, final Set extents) throws DatabaseException, IOException { - - CollectionSupport cs = graph.getService(CollectionSupport.class); - - ConcurrentSkipListSet statementSet = new ConcurrentSkipListSet(); - - StatementSetBuilder builder = new StatementSetBuilder(cs); - - SerialisationSupport support = graph.getService(SerialisationSupport.class); - - while(composedInput.available() > 0) { - - int s = composedInput.readInt(); - Resource subject = support.getResource(s); - - int size = composedInput.readInt(); - for(int i=0;i 0) { - - ArrayList> stms = new ArrayList>(); - - int counter = 0; - - while(statementInput.available() > 0 && counter++ < 1000) { - - int s = statementInput.readInt(); - Resource subject = support.getResource(s); - int size = statementInput.readInt(); - Triple[] list = new Triple[size]; - stms.add(Pair.make(subject, list)); - for(int i=0;i(support.getResource(p), support.getResource(o), ExtentStatus.EXTERNAL); - } - } - - searchStatementPart(graph, statementSet, stms, extents); - - } - - if(DEBUG) { - for(Statement stm : statementSet) - System.out.println("other " + NameUtils.toString(graph, stm)); - } - - builder.addAll(statementSet); - - if(DEBUG) System.out.println("total " + statementSet.size() + " statements found."); - - statementSet = null; - - LinkedList> statementSets = builder.get(); - - Map inverses = cs.createMap(Resource.class); - Set predicateSet = cs.createSet(); - for(Collection set : statementSets) - for(Statement s : set) { - Resource predicate = s.getPredicate(); - if(predicateSet.add(predicate)) { - Resource inverse = graph.getPossibleInverse(predicate); - inverses.put(predicate, inverse); - } - } - - predicateSet = null; - - statements = new int[4 * builder.size()]; - - while(!statementSets.isEmpty()) { - Collection set = statementSets.pop(); - for(Statement s : set) { - Resource subject = s.getSubject(); - Resource predicate = s.getPredicate(); - Resource object = s.getObject(); - int subjectId = ids.get(subject); - int objectId = getId(graph, object, predicate); - // The statement can be denied still - if(objectId != -2) { - statements[statementIndex++] = subjectId; - addId(graph, predicate, null); - Resource inverse = inverses.get(predicate); - if(inverse != null) { - if(LOG) log("[STM] " + subject.getResourceId() + ", " + predicate.getResourceId() + ", " + inverse.getResourceId() + ", " + object.getResourceId()); - addId(graph, inverse, null); - } else { - if(LOG) log("[STM] " + subject.getResourceId() + ", " + predicate.getResourceId() + ", -1, " + object.getResourceId()); - statements[statementIndex++] = -1; - } - statements[statementIndex++] = objectId; - - - - } else { - System.out.println("denied"); - } - } - } - - inverses = null; - - while(valueInput.available() > 0) { - - int s = valueInput.readInt(); - Resource subject = support.getResource(s); - int valueSize = valueInput.readInt(); - byte[] value = new byte[valueSize]; - valueInput.readFully(value); - Variant variant = (Variant) variantSerializer.deserialize(value); - values.put(subject, variant); - - } - - } - - public void setExternals(Collection rs) { - configuration.externals = rs; - } - - @Override - public TransferableGraph1 perform(ReadGraph graph) throws DatabaseException { - - this.variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT); - - this.l0 = Layer0.getInstance(graph); - Layer0X L0X = Layer0X.getInstance(graph); - - long total = System.nanoTime(); - - long startupTime = System.nanoTime(); - - CollectionSupport cs = graph.getService(CollectionSupport.class); - - ids = cs.createMap(Integer.class); - values = cs.createMap(byte[].class); - - ArrayList rootResources = new ArrayList(); - for(Pair p : configuration.roots) rootResources.add(p.first); - -// if(model == null) { -// SimulationResource SIMU = SimulationResource.getInstance(graph); -// for(Resource root : rootResources) { -// if(graph.isInstanceOf(root, SIMU.Model)) { -// model = root; -// break; -// } -// model = graph.syncRequest(new TypedParent(root, SIMU.Model)); -// if(model != null) break; -// } -// } - - Map preStatus = new HashMap(); - - for(Resource root : rootResources) { - Resource name = graph.getPossibleObject(root, l0.HasName); - if(name != null) { - preStatus.put(name, ExtentStatus.EXCLUDED); - } - } - - for(Resource r : configuration.externals) preStatus.put(r, ExtentStatus.EXTERNAL); - - Set extents = configuration.extents; - if(extents == null) { - - Instances extentInstances = graph.adapt(L0X.SubgraphExtent, Instances.class); - Collection extentResources = extentInstances.find(graph, configuration.model); - extents = new HashSet(); - for(Resource r : extentResources) { - extents.add(graph.getPossibleAdapter(r, SubgraphExtent.class)); - } - - if(DEBUG) { - System.out.println("Extents " + extents.size()); - for(Resource extent : extentResources) System.out.println("-" + NameUtils.getSafeName(graph, extent)); - } - - } - - Set advisors = configuration.advisors; - if(advisors == null) { - - Instances advisorInstances = graph.adapt(L0X.SubgraphAdvisor, Instances.class); - Collection advisorResources = advisorInstances.find(graph, configuration.model); - advisors = new HashSet(); - for(Resource r : advisorResources) { - advisors.add(graph.getPossibleAdapter(r, SubgraphAdvisor.class)); - } - - if(DEBUG) { - System.out.println("Advisors " + advisors.size()); - for(Resource advisor : advisorResources) System.out.println("-" + NameUtils.getSafeName(graph, advisor)); - } - - } - - long startupTimeEnd = System.nanoTime(); - - long domainTime = System.nanoTime(); - - String composedStatements = "composed" + UUID.randomUUID().toString(); - String otherStatements = "other" + UUID.randomUUID().toString(); - String valueFileName = "value" + UUID.randomUUID().toString(); - - File composedStatementsFile = new File(composedStatements); - File otherStatementsFile = new File(otherStatements); - File valueFile = new File(valueFileName); - - if(DEBUG) - System.out.println("getDomain writes " + composedStatementsFile.getAbsolutePath()); - - try { - - DeferredFileOutputStream composedStatementsStream = new DeferredFileOutputStream(5 * 1024*1024, composedStatementsFile); - DeferredFileOutputStream otherStatementsStream = new DeferredFileOutputStream(1024*1024, otherStatementsFile); - DeferredFileOutputStream valueStream = new DeferredFileOutputStream(1024*1024, valueFile); - - ObjectOutputStream composedStatementsOutput = new ObjectOutputStream(composedStatementsStream); - ObjectOutputStream otherStatementsOutput = new ObjectOutputStream(otherStatementsStream); - ObjectOutputStream valueOutput = new ObjectOutputStream(valueStream); - - Subgraphs.getDomain(graph, ids, rootResources, preStatus, advisors, composedStatementsOutput, otherStatementsOutput, valueOutput); - id = ids.size(); - - composedStatementsOutput.flush(); - otherStatementsOutput.flush(); - valueOutput.flush(); - composedStatementsStream.close(); - otherStatementsStream.close(); - valueStream.close(); - - long domainTimeEnd = System.nanoTime(); - - long fileTime = System.nanoTime(); - - internalCount = id; - - ids.put(graph.getRootLibrary(), id++); - externalNames.add("http:/"); - externalParents.add(-1); - -// if(model == null) return null; - - InputStream composedStatementsInputStream = null; - InputStream otherStatementsInputStream = null; - InputStream valueInputStream = null; - - if(composedStatementsStream.isInMemory()) { - composedStatementsInputStream = new ByteArrayInputStream(composedStatementsStream.getData()); - } else { - composedStatementsInputStream = new FileInputStream(composedStatementsFile); - } - - if(otherStatementsStream.isInMemory()) { - otherStatementsInputStream = new ByteArrayInputStream(otherStatementsStream.getData()); - } else { - otherStatementsInputStream = new FileInputStream(otherStatementsFile); - } - - if(valueStream.isInMemory()) { - valueInputStream = new ByteArrayInputStream(valueStream.getData()); - } else { - valueInputStream = new FileInputStream(valueFile); - } - - composedStatementsStream = null; - otherStatementsStream = null; - valueStream = null; - - ObjectInputStream composedStatementsInput = new ObjectInputStream(composedStatementsInputStream); - ObjectInputStream otherStatementsInput = new ObjectInputStream(otherStatementsInputStream); - ObjectInputStream valueInput = new ObjectInputStream(valueInputStream); - - long fileTimeEnd = System.nanoTime(); - - long statementTime = System.nanoTime(); - searchStatements(graph, composedStatementsInput, otherStatementsInput, valueInput, extents); - long statementTimeEnd = System.nanoTime(); - - // domainParts = null; - - long buildTime = System.nanoTime(); - - int resourceCount = ids.size(); - - Identity[] identityArray; - { // Identities - ArrayList identities = new ArrayList(); - - for(Pair r : configuration.roots) { - Resource type = graph.getPossibleType(r.first, l0.Entity); - if(type == null) type = l0.Entity; - identities.add(new Identity( - ids.get(r.first), - new Root(r.second, graph.getURI(type)) - )); - } - - int internalsPlusExternals = ids.size(); - for(int i = internalCount; i < internalsPlusExternals ; i++) { - int parent = externalParents.get(i - internalCount); - String name = externalNames.get(i - internalCount); - identities.add(new Identity( - i, - new External(parent, name) - )); - } - identityArray = identities.toArray(new Identity[identities.size()]); - } - - Value[] valueArray = new Value[values.size()]; - { // Values - int index = 0; - for(Map.Entry entry : values.entrySet()) { - if(LOG) log("[VALUE] " + entry.getKey().getResourceId()); - int r = getId(graph, entry.getKey(), null); - valueArray[index++] = new Value(r, entry.getValue()); - } - } - ids = null; - values = null; - - long buildTimeEnd = System.nanoTime(); - - TransferableGraph1 result = - new TransferableGraph1(resourceCount, - identityArray, - statements, - valueArray); - - if(DEBUG) { - System.out.println("transferable graph content: " + result); - } - - long totalEnd = System.nanoTime(); - - if(PROFILE) { - System.out.println("startup in " + 1e-9*(startupTimeEnd - startupTime) + "s."); - System.out.println("domain was found in " + 1e-9*(domainTimeEnd - domainTime) + "s."); - System.out.println("files were written in " + 1e-9*(fileTimeEnd - fileTime) + "s."); - System.out.println("statements were found in " + 1e-9*(statementTimeEnd - statementTime) + "s."); - System.out.println("tg was built in " + 1e-9*(buildTimeEnd - buildTime) + "s."); - System.out.println("total time for building subgraph was " + 1e-9*(totalEnd-total) + "s."); - } - - return result; - } catch (IOException e) { - e.printStackTrace(); - } - - return null; - - } - -} +/******************************************************************************* + * Copyright (c) 2007, 2010 Association for Decentralized Information Management + * in Industry THTH ry. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * VTT Technical Research Centre of Finland - initial API and implementation + *******************************************************************************/ +package org.simantics.db.layer0.util; + +import gnu.trove.list.array.TIntArrayList; + +import java.io.ByteArrayInputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.io.output.DeferredFileOutputStream; +import org.simantics.databoard.Bindings; +import org.simantics.databoard.Databoard; +import org.simantics.databoard.binding.mutable.Variant; +import org.simantics.databoard.serialization.Serializer; +import org.simantics.db.AsyncReadGraph; +import org.simantics.db.ReadGraph; +import org.simantics.db.Resource; +import org.simantics.db.Statement; +import org.simantics.db.common.StandardStatement; +import org.simantics.db.common.utils.NameUtils; +import org.simantics.db.exception.DatabaseException; +import org.simantics.db.exception.ValidationException; +import org.simantics.db.layer0.adapter.Instances; +import org.simantics.db.layer0.adapter.SubgraphAdvisor; +import org.simantics.db.layer0.adapter.SubgraphExtent; +import org.simantics.db.layer0.adapter.SubgraphExtent.Classifier; +import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus; +import org.simantics.db.procedure.AsyncProcedure; +import org.simantics.db.request.AsyncRead; +import org.simantics.db.request.Read; +import org.simantics.db.service.CollectionSupport; +import org.simantics.db.service.QueryControl; +import org.simantics.db.service.QueryControl.ControlProcedure; +import org.simantics.db.service.SerialisationSupport; +import org.simantics.graph.representation.External; +import org.simantics.graph.representation.Identity; +import org.simantics.graph.representation.Root; +import org.simantics.graph.representation.TransferableGraph1; +import org.simantics.graph.representation.Value; +import org.simantics.layer0.Layer0; +import org.simantics.operation.Layer0X; +import org.simantics.utils.datastructures.Pair; +import org.simantics.utils.datastructures.Triple; + +/** + * IsComposedOf objects are always in domain + * Ordered set elements are always in domain + * r is in domain if for all statements (s, IsRelatedTo, r) s is in domain + * + * IsWeaklyRelatedTo statements where subject is in domain are only accepted if object is in domain or has URI + * + * @deprecated in favor of {@link TransferableGraphRequest2} + */ +public class TransferableGraphRequest implements Read { + + public static String LOG_FILE = "transferableGraph.log"; + final static private boolean LOG = false; + final static private boolean DEBUG = false; + final static private boolean PROFILE = false; + +// final private Collection> roots; +// private Collection preExternals = Collections.emptyList(); +// private Resource model; + + private TransferableGraphConfiguration configuration; + + static DataOutput log; + + static { + + if (LOG) { + try { + FileOutputStream stream = new FileOutputStream(LOG_FILE); + log = new DataOutputStream(stream); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } + } + + } + + private static void log(String line) { + if (LOG) { + try { + log.writeUTF(line + "\n"); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + + + public TransferableGraphRequest(Collection> roots, Resource model) { + + configuration = new TransferableGraphConfiguration(); + configuration.roots = roots; + configuration.model = model; + +// this.roots = roots; +// this.model = model; + + } + + + public TransferableGraphRequest(Collection> roots) { + this(roots, null); + } + + public TransferableGraphRequest(TransferableGraphConfiguration conf) { + this.configuration = conf; + } + + Layer0 l0; + + TIntArrayList inverses = new TIntArrayList(); + int statements[]; + int statementIndex = 0; + Map ids; + Map values; + TIntArrayList externalParents = new TIntArrayList(); + ArrayList externalNames = new ArrayList(); +// ConcurrentLinkedQueue> values = new ConcurrentLinkedQueue>(); + int id = 0; + int internalCount; + + int indent = 0; + + private Serializer variantSerializer; + + private boolean validateExternal(Resource r) { + if(configuration.disallowedExternals != null) { + System.err.println("validateExternal agains " + configuration.disallowedExternals); + return !configuration.disallowedExternals.contains(r); + } + return true; + } + + public int getId(ReadGraph graph, Resource r, Resource predicate) throws DatabaseException { +// for(int i=0;i parents = graph.getObjects(r, l0.PartOf); + if(parents.size() != 1) { + if(parents.size() == 0) { + Resource inv = graph.getPossibleObject(r, l0.InverseOf); + if(inv != null) { + ++indent; + if(DEBUG) System.out.println("inverse " + NameUtils.getSafeName(graph, inv)); + int invId = getId(graph, inv, null); + externalParents.add(invId); + --indent; + externalNames.add("@inverse"); + ids.put(r, id); + return id++; + } + } +// for(Statement stat : graph.getStatements(r, b.IsWeaklyRelatedTo)) +// System.out.println(NameUtils.getSafeName(graph, stat.getPredicate()) +// + " -> " + NameUtils.getSafeName(graph, stat.getObject())); +// if(predicate != null) { +// if(!graph.isSubrelationOf(predicate, b.IsRelatedTo)) { +// return -2; +// } +// } + throw new ValidationException("Reference to external resource " + + NameUtils.getSafeName(graph, r, true) + " without unique uri (" + parents.size() + " parents)."); + } + for(Resource p : parents) { +// System.out.println("Parent " + NameUtils.getSafeName(graph, p)); + ++indent; + if(!validateExternal(p)) throw new ValidationException("References to '" + graph.getURI(p) + "' are not allowed."); + externalParents.add(getId(graph, p, null)); + --indent; + } +// System.out.println("Request named for " + GraphUtils.getReadableName(g, r)); +// String name = graph.getPossibleRelatedValue(r, b.HasName); + externalNames.add((String)graph.getRelatedValue(r, l0.HasName)); +// if(name != null) externalNames.add(name); +// else externalNames.add("@resource"); + ids.put(r, id); + return id++; + } + } + + public void addId(ReadGraph graph, Resource r, Resource predicate) throws DatabaseException { + statements[statementIndex++] = getId(graph, r, predicate); + } + + private void searchStatementPart(ReadGraph graph, final ConcurrentSkipListSet statementSet, final ArrayList> part, final Set extents) throws DatabaseException { + + final SubgraphExtent.Callback callback = new SubgraphExtent.Callback() { + + @Override + public void statement(Statement statement, boolean accept) { + if(accept) { + statementSet.add(statement); + } + } + + }; + + graph.syncRequest(new AsyncRead() { + + @Override + public int threadHash() { + return hashCode(); + } + + @Override + public void perform(AsyncReadGraph graph, AsyncProcedure procedure) { + + QueryControl control = graph.getService(QueryControl.class); + + int slice = (int)(part.size() / control.getAmountOfQueryThreads()) + 1; + + final Pair[] rootArray = (Pair[])part.toArray(new Pair[part.size()]); + for(int i=0;i r = (Pair)rootArray[index]; + + final AtomicInteger position = new AtomicInteger(0); + final SubgraphExtent.Classifier[] classifiers = new SubgraphExtent.Classifier[extents.size()]; + + for(SubgraphExtent extent : extents) { + + extent.accept(graph, r.first, new AsyncProcedure() { + + public void execute(AsyncReadGraph graph, Classifier c) { + + int current = position.incrementAndGet(); + classifiers[current-1] = c; + + if(current == extents.size()) { + + for(Triple statement : r.second) { + if(!r.first.isPersistent()) continue; + for(Classifier classifier : classifiers) { + classifier.classify(graph, new StandardStatement(r.first, (Resource)statement.first, (Resource)statement.second), (ExtentStatus)statement.third, callback); + } + } + +// graph.forEachDirectStatement(r, new AsyncMultiProcedure() { +// +// @Override +// public void exception(AsyncReadGraph graph, Throwable throwable) { +// throwable.printStackTrace(); +// } +// +// @Override +// public void execute(AsyncReadGraph graph, Statement statement) { +// } +// +// @Override +// public void finished(AsyncReadGraph graph) { +// } +// +// }); + + } + + } + + public void exception(AsyncReadGraph graph, Throwable throwable) { + throwable.printStackTrace(); + } + + }, callback); + + } + + } + + } + + }); + + } + + procedure.execute(graph, false); + + } + + @Override + public int getFlags() { + return 0; + } + + }); + + } + + class StatementSetBuilder { + + private int SLICE = 100000; + + final private CollectionSupport cs; + final private LinkedList> sets = new LinkedList>(); + + private Collection current; + + StatementSetBuilder(CollectionSupport cs) { + this.cs = cs; + current = cs.createStatementList(); + } + + LinkedList> get() { + sets.add(current); + return sets; + } + + void add(Statement stm) { + current.add(stm); + if(current.size() == SLICE) { + sets.add(current); + current = cs.createStatementList(); + } + } + + int size() { + int result = 0; + for(Collection c : sets) result += c.size(); + return result; + } + + + void addAll(Collection ss) { + for(Statement s : ss) add(s); + } + + } + + private void searchStatements(ReadGraph graph, ObjectInputStream composedInput, ObjectInputStream statementInput, ObjectInputStream valueInput, final Set extents) throws DatabaseException, IOException { + + CollectionSupport cs = graph.getService(CollectionSupport.class); + + ConcurrentSkipListSet statementSet = new ConcurrentSkipListSet(); + + StatementSetBuilder builder = new StatementSetBuilder(cs); + + SerialisationSupport support = graph.getService(SerialisationSupport.class); + + while(composedInput.available() > 0) { + + int s = composedInput.readInt(); + Resource subject = support.getResource(s); + + int size = composedInput.readInt(); + for(int i=0;i 0) { + + ArrayList> stms = new ArrayList>(); + + int counter = 0; + + while(statementInput.available() > 0 && counter++ < 1000) { + + int s = statementInput.readInt(); + Resource subject = support.getResource(s); + int size = statementInput.readInt(); + Triple[] list = new Triple[size]; + stms.add(Pair.make(subject, list)); + for(int i=0;i(support.getResource(p), support.getResource(o), ExtentStatus.EXTERNAL); + } + } + + searchStatementPart(graph, statementSet, stms, extents); + + } + + if(DEBUG) { + for(Statement stm : statementSet) + System.out.println("other " + NameUtils.toString(graph, stm)); + } + + builder.addAll(statementSet); + + if(DEBUG) System.out.println("total " + statementSet.size() + " statements found."); + + statementSet = null; + + LinkedList> statementSets = builder.get(); + + Map inverses = cs.createMap(Resource.class); + Set predicateSet = cs.createSet(); + for(Collection set : statementSets) + for(Statement s : set) { + Resource predicate = s.getPredicate(); + if(predicateSet.add(predicate)) { + Resource inverse = graph.getPossibleInverse(predicate); + inverses.put(predicate, inverse); + } + } + + predicateSet = null; + + statements = new int[4 * builder.size()]; + + while(!statementSets.isEmpty()) { + Collection set = statementSets.pop(); + for(Statement s : set) { + Resource subject = s.getSubject(); + Resource predicate = s.getPredicate(); + Resource object = s.getObject(); + int subjectId = ids.get(subject); + int objectId = getId(graph, object, predicate); + // The statement can be denied still + if(objectId != -2) { + statements[statementIndex++] = subjectId; + addId(graph, predicate, null); + Resource inverse = inverses.get(predicate); + if(inverse != null) { + if(LOG) log("[STM] " + subject.getResourceId() + ", " + predicate.getResourceId() + ", " + inverse.getResourceId() + ", " + object.getResourceId()); + addId(graph, inverse, null); + } else { + if(LOG) log("[STM] " + subject.getResourceId() + ", " + predicate.getResourceId() + ", -1, " + object.getResourceId()); + statements[statementIndex++] = -1; + } + statements[statementIndex++] = objectId; + + + + } else { + System.out.println("denied"); + } + } + } + + inverses = null; + + while(valueInput.available() > 0) { + + int s = valueInput.readInt(); + Resource subject = support.getResource(s); + int valueSize = valueInput.readInt(); + byte[] value = new byte[valueSize]; + valueInput.readFully(value); + Variant variant = (Variant) variantSerializer.deserialize(value); + values.put(subject, variant); + + } + + } + + public void setExternals(Collection rs) { + configuration.externals = rs; + } + + @Override + public TransferableGraph1 perform(ReadGraph graph) throws DatabaseException { + + this.variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT); + + this.l0 = Layer0.getInstance(graph); + Layer0X L0X = Layer0X.getInstance(graph); + + long total = System.nanoTime(); + + long startupTime = System.nanoTime(); + + CollectionSupport cs = graph.getService(CollectionSupport.class); + + ids = cs.createMap(Integer.class); + values = cs.createMap(byte[].class); + + ArrayList rootResources = new ArrayList(); + for(Pair p : configuration.roots) rootResources.add(p.first); + +// if(model == null) { +// SimulationResource SIMU = SimulationResource.getInstance(graph); +// for(Resource root : rootResources) { +// if(graph.isInstanceOf(root, SIMU.Model)) { +// model = root; +// break; +// } +// model = graph.syncRequest(new TypedParent(root, SIMU.Model)); +// if(model != null) break; +// } +// } + + Map preStatus = new HashMap(); + + for(Resource root : rootResources) { + Resource name = graph.getPossibleObject(root, l0.HasName); + if(name != null) { + preStatus.put(name, ExtentStatus.EXCLUDED); + } + } + + for(Resource r : configuration.externals) preStatus.put(r, ExtentStatus.EXTERNAL); + + Set extents = configuration.extents; + if(extents == null) { + + Instances extentInstances = graph.adapt(L0X.SubgraphExtent, Instances.class); + Collection extentResources = extentInstances.find(graph, configuration.model); + extents = new HashSet(); + for(Resource r : extentResources) { + extents.add(graph.getPossibleAdapter(r, SubgraphExtent.class)); + } + + if(DEBUG) { + System.out.println("Extents " + extents.size()); + for(Resource extent : extentResources) System.out.println("-" + NameUtils.getSafeName(graph, extent)); + } + + } + + Set advisors = configuration.advisors; + if(advisors == null) { + + Instances advisorInstances = graph.adapt(L0X.SubgraphAdvisor, Instances.class); + Collection advisorResources = advisorInstances.find(graph, configuration.model); + advisors = new HashSet(); + for(Resource r : advisorResources) { + advisors.add(graph.getPossibleAdapter(r, SubgraphAdvisor.class)); + } + + if(DEBUG) { + System.out.println("Advisors " + advisors.size()); + for(Resource advisor : advisorResources) System.out.println("-" + NameUtils.getSafeName(graph, advisor)); + } + + } + + long startupTimeEnd = System.nanoTime(); + + long domainTime = System.nanoTime(); + + String composedStatements = "composed" + UUID.randomUUID().toString(); + String otherStatements = "other" + UUID.randomUUID().toString(); + String valueFileName = "value" + UUID.randomUUID().toString(); + + File composedStatementsFile = new File(composedStatements); + File otherStatementsFile = new File(otherStatements); + File valueFile = new File(valueFileName); + + if(DEBUG) + System.out.println("getDomain writes " + composedStatementsFile.getAbsolutePath()); + + try { + + DeferredFileOutputStream composedStatementsStream = new DeferredFileOutputStream(5 * 1024*1024, composedStatementsFile); + DeferredFileOutputStream otherStatementsStream = new DeferredFileOutputStream(1024*1024, otherStatementsFile); + DeferredFileOutputStream valueStream = new DeferredFileOutputStream(1024*1024, valueFile); + + ObjectOutputStream composedStatementsOutput = new ObjectOutputStream(composedStatementsStream); + ObjectOutputStream otherStatementsOutput = new ObjectOutputStream(otherStatementsStream); + ObjectOutputStream valueOutput = new ObjectOutputStream(valueStream); + + Subgraphs.getDomain(graph, ids, rootResources, preStatus, advisors, composedStatementsOutput, otherStatementsOutput, valueOutput); + id = ids.size(); + + composedStatementsOutput.flush(); + otherStatementsOutput.flush(); + valueOutput.flush(); + composedStatementsStream.close(); + otherStatementsStream.close(); + valueStream.close(); + + long domainTimeEnd = System.nanoTime(); + + long fileTime = System.nanoTime(); + + internalCount = id; + + ids.put(graph.getRootLibrary(), id++); + externalNames.add("http:/"); + externalParents.add(-1); + +// if(model == null) return null; + + InputStream composedStatementsInputStream = null; + InputStream otherStatementsInputStream = null; + InputStream valueInputStream = null; + + if(composedStatementsStream.isInMemory()) { + composedStatementsInputStream = new ByteArrayInputStream(composedStatementsStream.getData()); + } else { + composedStatementsInputStream = new FileInputStream(composedStatementsFile); + } + + if(otherStatementsStream.isInMemory()) { + otherStatementsInputStream = new ByteArrayInputStream(otherStatementsStream.getData()); + } else { + otherStatementsInputStream = new FileInputStream(otherStatementsFile); + } + + if(valueStream.isInMemory()) { + valueInputStream = new ByteArrayInputStream(valueStream.getData()); + } else { + valueInputStream = new FileInputStream(valueFile); + } + + composedStatementsStream = null; + otherStatementsStream = null; + valueStream = null; + + ObjectInputStream composedStatementsInput = new ObjectInputStream(composedStatementsInputStream); + ObjectInputStream otherStatementsInput = new ObjectInputStream(otherStatementsInputStream); + ObjectInputStream valueInput = new ObjectInputStream(valueInputStream); + + long fileTimeEnd = System.nanoTime(); + + long statementTime = System.nanoTime(); + searchStatements(graph, composedStatementsInput, otherStatementsInput, valueInput, extents); + long statementTimeEnd = System.nanoTime(); + + // domainParts = null; + + long buildTime = System.nanoTime(); + + int resourceCount = ids.size(); + + Identity[] identityArray; + { // Identities + ArrayList identities = new ArrayList(); + + for(Pair r : configuration.roots) { + Resource type = graph.getPossibleType(r.first, l0.Entity); + if(type == null) type = l0.Entity; + identities.add(new Identity( + ids.get(r.first), + new Root(r.second, graph.getURI(type)) + )); + } + + int internalsPlusExternals = ids.size(); + for(int i = internalCount; i < internalsPlusExternals ; i++) { + int parent = externalParents.get(i - internalCount); + String name = externalNames.get(i - internalCount); + identities.add(new Identity( + i, + new External(parent, name) + )); + } + identityArray = identities.toArray(new Identity[identities.size()]); + } + + Value[] valueArray = new Value[values.size()]; + { // Values + int index = 0; + for(Map.Entry entry : values.entrySet()) { + if(LOG) log("[VALUE] " + entry.getKey().getResourceId()); + int r = getId(graph, entry.getKey(), null); + valueArray[index++] = new Value(r, entry.getValue()); + } + } + ids = null; + values = null; + + long buildTimeEnd = System.nanoTime(); + + TransferableGraph1 result = + new TransferableGraph1(resourceCount, + identityArray, + statements, + valueArray); + + if(DEBUG) { + System.out.println("transferable graph content: " + result); + } + + long totalEnd = System.nanoTime(); + + if(PROFILE) { + System.out.println("startup in " + 1e-9*(startupTimeEnd - startupTime) + "s."); + System.out.println("domain was found in " + 1e-9*(domainTimeEnd - domainTime) + "s."); + System.out.println("files were written in " + 1e-9*(fileTimeEnd - fileTime) + "s."); + System.out.println("statements were found in " + 1e-9*(statementTimeEnd - statementTime) + "s."); + System.out.println("tg was built in " + 1e-9*(buildTimeEnd - buildTime) + "s."); + System.out.println("total time for building subgraph was " + 1e-9*(totalEnd-total) + "s."); + } + + return result; + } catch (IOException e) { + e.printStackTrace(); + } + + return null; + + } + +}