+++ /dev/null
-/*******************************************************************************
- * Copyright (c) 2007, 2010 Association for Decentralized Information Management
- * in Industry THTH ry.
- * All rights reserved. This program and the accompanying materials
- * are made available under the terms of the Eclipse Public License v1.0
- * which accompanies this distribution, and is available at
- * http://www.eclipse.org/legal/epl-v10.html
- *
- * Contributors:
- * VTT Technical Research Centre of Finland - initial API and implementation
- *******************************************************************************/
-package org.simantics.db.layer0.util;
-
-import gnu.trove.list.array.TIntArrayList;
-
-import java.io.ByteArrayInputStream;
-import java.io.DataOutput;
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentSkipListSet;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.commons.io.output.DeferredFileOutputStream;
-import org.simantics.databoard.Bindings;
-import org.simantics.databoard.Databoard;
-import org.simantics.databoard.binding.mutable.Variant;
-import org.simantics.databoard.serialization.Serializer;
-import org.simantics.db.AsyncReadGraph;
-import org.simantics.db.ReadGraph;
-import org.simantics.db.Resource;
-import org.simantics.db.Statement;
-import org.simantics.db.common.StandardStatement;
-import org.simantics.db.common.utils.NameUtils;
-import org.simantics.db.exception.DatabaseException;
-import org.simantics.db.exception.ValidationException;
-import org.simantics.db.layer0.adapter.Instances;
-import org.simantics.db.layer0.adapter.SubgraphAdvisor;
-import org.simantics.db.layer0.adapter.SubgraphExtent;
-import org.simantics.db.layer0.adapter.SubgraphExtent.Classifier;
-import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus;
-import org.simantics.db.procedure.AsyncProcedure;
-import org.simantics.db.request.AsyncRead;
-import org.simantics.db.request.Read;
-import org.simantics.db.service.CollectionSupport;
-import org.simantics.db.service.QueryControl;
-import org.simantics.db.service.QueryControl.ControlProcedure;
-import org.simantics.db.service.SerialisationSupport;
-import org.simantics.graph.representation.External;
-import org.simantics.graph.representation.Identity;
-import org.simantics.graph.representation.Root;
-import org.simantics.graph.representation.TransferableGraph1;
-import org.simantics.graph.representation.Value;
-import org.simantics.layer0.Layer0;
-import org.simantics.operation.Layer0X;
-import org.simantics.utils.datastructures.Pair;
-import org.simantics.utils.datastructures.Triple;
-
-/**
- * IsComposedOf objects are always in domain
- * Ordered set elements are always in domain
- * r is in domain if for all statements (s, IsRelatedTo, r) s is in domain
- *
- * IsWeaklyRelatedTo statements where subject is in domain are only accepted if object is in domain or has URI
- *
- * @deprecated in favor of {@link TransferableGraphRequest2}
- */
-public class TransferableGraphRequest implements Read<TransferableGraph1> {
-
- public static String LOG_FILE = "transferableGraph.log";
- final static private boolean LOG = false;
- final static private boolean DEBUG = false;
- final static private boolean PROFILE = false;
-
-// final private Collection<Pair<Resource, String>> roots;
-// private Collection<Resource> preExternals = Collections.emptyList();
-// private Resource model;
-
- private TransferableGraphConfiguration configuration;
-
- static DataOutput log;
-
- static {
-
- if (LOG) {
- try {
- FileOutputStream stream = new FileOutputStream(LOG_FILE);
- log = new DataOutputStream(stream);
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- }
- }
-
- }
-
- private static void log(String line) {
- if (LOG) {
- try {
- log.writeUTF(line + "\n");
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }
-
-
-
- public TransferableGraphRequest(Collection<Pair<Resource, String>> roots, Resource model) {
-
- configuration = new TransferableGraphConfiguration();
- configuration.roots = roots;
- configuration.model = model;
-
-// this.roots = roots;
-// this.model = model;
-
- }
-
-
- public TransferableGraphRequest(Collection<Pair<Resource, String>> roots) {
- this(roots, null);
- }
-
- public TransferableGraphRequest(TransferableGraphConfiguration conf) {
- this.configuration = conf;
- }
-
- Layer0 l0;
-
- TIntArrayList inverses = new TIntArrayList();
- int statements[];
- int statementIndex = 0;
- Map<Resource, Integer> ids;
- Map<Resource, Variant> values;
- TIntArrayList externalParents = new TIntArrayList();
- ArrayList<String> externalNames = new ArrayList<String>();
-// ConcurrentLinkedQueue<Pair<Resource, byte[]>> values = new ConcurrentLinkedQueue<Pair<Resource, byte[]>>();
- int id = 0;
- int internalCount;
-
- int indent = 0;
-
- private Serializer variantSerializer;
-
- private boolean validateExternal(Resource r) {
- if(configuration.disallowedExternals != null) {
- System.err.println("validateExternal agains " + configuration.disallowedExternals);
- return !configuration.disallowedExternals.contains(r);
- }
- return true;
- }
-
- public int getId(ReadGraph graph, Resource r, Resource predicate) throws DatabaseException {
-// for(int i=0;i<indent;++i)
-// System.out.print(" ");
-// System.out.println(NameUtils.getSafeName(graph, r));
- if(ids.containsKey(r)) {
- int ret = ids.get(r);
- if(ret == -1) {
- for(int i=0;i<=indent;++i)
- System.out.print(" ");
- System.out.println("Cycle!!!"); // with " + GraphUtils.getReadableName(g, r));
- }
- return ret;
- }
- else {
- //ids.put(r, -1);
- Collection<Resource> parents = graph.getObjects(r, l0.PartOf);
- if(parents.size() != 1) {
- if(parents.size() == 0) {
- Resource inv = graph.getPossibleObject(r, l0.InverseOf);
- if(inv != null) {
- ++indent;
- if(DEBUG) System.out.println("inverse " + NameUtils.getSafeName(graph, inv));
- int invId = getId(graph, inv, null);
- externalParents.add(invId);
- --indent;
- externalNames.add("@inverse");
- ids.put(r, id);
- return id++;
- }
- }
-// for(Statement stat : graph.getStatements(r, b.IsWeaklyRelatedTo))
-// System.out.println(NameUtils.getSafeName(graph, stat.getPredicate())
-// + " -> " + NameUtils.getSafeName(graph, stat.getObject()));
-// if(predicate != null) {
-// if(!graph.isSubrelationOf(predicate, b.IsRelatedTo)) {
-// return -2;
-// }
-// }
- throw new ValidationException("Reference to external resource "
- + NameUtils.getSafeName(graph, r, true) + " without unique uri (" + parents.size() + " parents).");
- }
- for(Resource p : parents) {
-// System.out.println("Parent " + NameUtils.getSafeName(graph, p));
- ++indent;
- if(!validateExternal(p)) throw new ValidationException("References to '" + graph.getURI(p) + "' are not allowed.");
- externalParents.add(getId(graph, p, null));
- --indent;
- }
-// System.out.println("Request named for " + GraphUtils.getReadableName(g, r));
-// String name = graph.getPossibleRelatedValue(r, b.HasName);
- externalNames.add((String)graph.getRelatedValue(r, l0.HasName));
-// if(name != null) externalNames.add(name);
-// else externalNames.add("@resource");
- ids.put(r, id);
- return id++;
- }
- }
-
- public void addId(ReadGraph graph, Resource r, Resource predicate) throws DatabaseException {
- statements[statementIndex++] = getId(graph, r, predicate);
- }
-
- private void searchStatementPart(ReadGraph graph, final ConcurrentSkipListSet<Statement> statementSet, final ArrayList<Pair<Resource, Triple[]>> part, final Set<SubgraphExtent> extents) throws DatabaseException {
-
- final SubgraphExtent.Callback callback = new SubgraphExtent.Callback() {
-
- @Override
- public void statement(Statement statement, boolean accept) {
- if(accept) {
- statementSet.add(statement);
- }
- }
-
- };
-
- graph.syncRequest(new AsyncRead<Boolean>() {
-
- @Override
- public int threadHash() {
- return hashCode();
- }
-
- @Override
- public void perform(AsyncReadGraph graph, AsyncProcedure<Boolean> procedure) {
-
- QueryControl control = graph.getService(QueryControl.class);
-
- int slice = (int)(part.size() / control.getAmountOfQueryThreads()) + 1;
-
- final Pair[] rootArray = (Pair[])part.toArray(new Pair[part.size()]);
- for(int i=0;i<control.getAmountOfQueryThreads();i++) {
-
- final int start = i * slice;
- final int end = Math.min(start + slice, rootArray.length);
-
- control.schedule(graph, i, new ControlProcedure() {
-
- @Override
- public void execute(AsyncReadGraph graph) {
-
- for(int index = start;index < end;index++) {
-
- final Pair<Resource, Triple[]> r = (Pair<Resource, Triple[]>)rootArray[index];
-
- final AtomicInteger position = new AtomicInteger(0);
- final SubgraphExtent.Classifier[] classifiers = new SubgraphExtent.Classifier[extents.size()];
-
- for(SubgraphExtent extent : extents) {
-
- extent.accept(graph, r.first, new AsyncProcedure<Classifier>() {
-
- public void execute(AsyncReadGraph graph, Classifier c) {
-
- int current = position.incrementAndGet();
- classifiers[current-1] = c;
-
- if(current == extents.size()) {
-
- for(Triple statement : r.second) {
- if(!r.first.isPersistent()) continue;
- for(Classifier classifier : classifiers) {
- classifier.classify(graph, new StandardStatement(r.first, (Resource)statement.first, (Resource)statement.second), (ExtentStatus)statement.third, callback);
- }
- }
-
-// graph.forEachDirectStatement(r, new AsyncMultiProcedure<Statement>() {
-//
-// @Override
-// public void exception(AsyncReadGraph graph, Throwable throwable) {
-// throwable.printStackTrace();
-// }
-//
-// @Override
-// public void execute(AsyncReadGraph graph, Statement statement) {
-// }
-//
-// @Override
-// public void finished(AsyncReadGraph graph) {
-// }
-//
-// });
-
- }
-
- }
-
- public void exception(AsyncReadGraph graph, Throwable throwable) {
- throwable.printStackTrace();
- }
-
- }, callback);
-
- }
-
- }
-
- }
-
- });
-
- }
-
- procedure.execute(graph, false);
-
- }
-
- @Override
- public int getFlags() {
- return 0;
- }
-
- });
-
- }
-
- class StatementSetBuilder {
-
- private int SLICE = 100000;
-
- final private CollectionSupport cs;
- final private LinkedList<Collection<Statement>> sets = new LinkedList<Collection<Statement>>();
-
- private Collection<Statement> current;
-
- StatementSetBuilder(CollectionSupport cs) {
- this.cs = cs;
- current = cs.createStatementList();
- }
-
- LinkedList<Collection<Statement>> get() {
- sets.add(current);
- return sets;
- }
-
- void add(Statement stm) {
- current.add(stm);
- if(current.size() == SLICE) {
- sets.add(current);
- current = cs.createStatementList();
- }
- }
-
- int size() {
- int result = 0;
- for(Collection<Statement> c : sets) result += c.size();
- return result;
- }
-
-
- void addAll(Collection<Statement> ss) {
- for(Statement s : ss) add(s);
- }
-
- }
-
- private void searchStatements(ReadGraph graph, ObjectInputStream composedInput, ObjectInputStream statementInput, ObjectInputStream valueInput, final Set<SubgraphExtent> extents) throws DatabaseException, IOException {
-
- CollectionSupport cs = graph.getService(CollectionSupport.class);
-
- ConcurrentSkipListSet<Statement> statementSet = new ConcurrentSkipListSet<Statement>();
-
- StatementSetBuilder builder = new StatementSetBuilder(cs);
-
- SerialisationSupport support = graph.getService(SerialisationSupport.class);
-
- while(composedInput.available() > 0) {
-
- int s = composedInput.readInt();
- Resource subject = support.getResource(s);
-
- int size = composedInput.readInt();
- for(int i=0;i<size;i++) {
- int p = composedInput.readInt();
- int o = composedInput.readInt();
- StandardStatement stm = new StandardStatement(subject, support.getResource(p), support.getResource(o));
-// if(DEBUG) System.out.println("composed " + NameUtils.toString(graph, stm));
- builder.add(stm);
- }
-
- }
-
- while(statementInput.available() > 0) {
-
- ArrayList<Pair<Resource, Triple[]>> stms = new ArrayList<Pair<Resource, Triple[]>>();
-
- int counter = 0;
-
- while(statementInput.available() > 0 && counter++ < 1000) {
-
- int s = statementInput.readInt();
- Resource subject = support.getResource(s);
- int size = statementInput.readInt();
- Triple[] list = new Triple[size];
- stms.add(Pair.make(subject, list));
- for(int i=0;i<size;i++) {
- int p = statementInput.readInt();
- int o = statementInput.readInt();
- if(DEBUG) {
- System.out.println("otherInput " + NameUtils.toString(graph, new StandardStatement(support.getResource(s), support.getResource(p), support.getResource(o))));
- }
- list[i] = new Triple<Resource, Resource, ExtentStatus>(support.getResource(p), support.getResource(o), ExtentStatus.EXTERNAL);
- }
- }
-
- searchStatementPart(graph, statementSet, stms, extents);
-
- }
-
- if(DEBUG) {
- for(Statement stm : statementSet)
- System.out.println("other " + NameUtils.toString(graph, stm));
- }
-
- builder.addAll(statementSet);
-
- if(DEBUG) System.out.println("total " + statementSet.size() + " statements found.");
-
- statementSet = null;
-
- LinkedList<Collection<Statement>> statementSets = builder.get();
-
- Map<Resource, Resource> inverses = cs.createMap(Resource.class);
- Set<Resource> predicateSet = cs.createSet();
- for(Collection<Statement> set : statementSets)
- for(Statement s : set) {
- Resource predicate = s.getPredicate();
- if(predicateSet.add(predicate)) {
- Resource inverse = graph.getPossibleInverse(predicate);
- inverses.put(predicate, inverse);
- }
- }
-
- predicateSet = null;
-
- statements = new int[4 * builder.size()];
-
- while(!statementSets.isEmpty()) {
- Collection<Statement> set = statementSets.pop();
- for(Statement s : set) {
- Resource subject = s.getSubject();
- Resource predicate = s.getPredicate();
- Resource object = s.getObject();
- int subjectId = ids.get(subject);
- int objectId = getId(graph, object, predicate);
- // The statement can be denied still
- if(objectId != -2) {
- statements[statementIndex++] = subjectId;
- addId(graph, predicate, null);
- Resource inverse = inverses.get(predicate);
- if(inverse != null) {
- if(LOG) log("[STM] " + subject.getResourceId() + ", " + predicate.getResourceId() + ", " + inverse.getResourceId() + ", " + object.getResourceId());
- addId(graph, inverse, null);
- } else {
- if(LOG) log("[STM] " + subject.getResourceId() + ", " + predicate.getResourceId() + ", -1, " + object.getResourceId());
- statements[statementIndex++] = -1;
- }
- statements[statementIndex++] = objectId;
-
-
-
- } else {
- System.out.println("denied");
- }
- }
- }
-
- inverses = null;
-
- while(valueInput.available() > 0) {
-
- int s = valueInput.readInt();
- Resource subject = support.getResource(s);
- int valueSize = valueInput.readInt();
- byte[] value = new byte[valueSize];
- valueInput.readFully(value);
- Variant variant = (Variant) variantSerializer.deserialize(value);
- values.put(subject, variant);
-
- }
-
- }
-
- public void setExternals(Collection<Resource> rs) {
- configuration.externals = rs;
- }
-
- @Override
- public TransferableGraph1 perform(ReadGraph graph) throws DatabaseException {
-
- this.variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT);
-
- this.l0 = Layer0.getInstance(graph);
- Layer0X L0X = Layer0X.getInstance(graph);
-
- long total = System.nanoTime();
-
- long startupTime = System.nanoTime();
-
- CollectionSupport cs = graph.getService(CollectionSupport.class);
-
- ids = cs.createMap(Integer.class);
- values = cs.createMap(byte[].class);
-
- ArrayList<Resource> rootResources = new ArrayList<Resource>();
- for(Pair<Resource, String> p : configuration.roots) rootResources.add(p.first);
-
-// if(model == null) {
-// SimulationResource SIMU = SimulationResource.getInstance(graph);
-// for(Resource root : rootResources) {
-// if(graph.isInstanceOf(root, SIMU.Model)) {
-// model = root;
-// break;
-// }
-// model = graph.syncRequest(new TypedParent(root, SIMU.Model));
-// if(model != null) break;
-// }
-// }
-
- Map<Resource, ExtentStatus> preStatus = new HashMap<Resource, ExtentStatus>();
-
- for(Resource root : rootResources) {
- Resource name = graph.getPossibleObject(root, l0.HasName);
- if(name != null) {
- preStatus.put(name, ExtentStatus.EXCLUDED);
- }
- }
-
- for(Resource r : configuration.externals) preStatus.put(r, ExtentStatus.EXTERNAL);
-
- Set<SubgraphExtent> extents = configuration.extents;
- if(extents == null) {
-
- Instances extentInstances = graph.adapt(L0X.SubgraphExtent, Instances.class);
- Collection<Resource> extentResources = extentInstances.find(graph, configuration.model);
- extents = new HashSet<SubgraphExtent>();
- for(Resource r : extentResources) {
- extents.add(graph.getPossibleAdapter(r, SubgraphExtent.class));
- }
-
- if(DEBUG) {
- System.out.println("Extents " + extents.size());
- for(Resource extent : extentResources) System.out.println("-" + NameUtils.getSafeName(graph, extent));
- }
-
- }
-
- Set<SubgraphAdvisor> advisors = configuration.advisors;
- if(advisors == null) {
-
- Instances advisorInstances = graph.adapt(L0X.SubgraphAdvisor, Instances.class);
- Collection<Resource> advisorResources = advisorInstances.find(graph, configuration.model);
- advisors = new HashSet<SubgraphAdvisor>();
- for(Resource r : advisorResources) {
- advisors.add(graph.getPossibleAdapter(r, SubgraphAdvisor.class));
- }
-
- if(DEBUG) {
- System.out.println("Advisors " + advisors.size());
- for(Resource advisor : advisorResources) System.out.println("-" + NameUtils.getSafeName(graph, advisor));
- }
-
- }
-
- long startupTimeEnd = System.nanoTime();
-
- long domainTime = System.nanoTime();
-
- String composedStatements = "composed" + UUID.randomUUID().toString();
- String otherStatements = "other" + UUID.randomUUID().toString();
- String valueFileName = "value" + UUID.randomUUID().toString();
-
- File composedStatementsFile = new File(composedStatements);
- File otherStatementsFile = new File(otherStatements);
- File valueFile = new File(valueFileName);
-
- if(DEBUG)
- System.out.println("getDomain writes " + composedStatementsFile.getAbsolutePath());
-
- try {
-
- DeferredFileOutputStream composedStatementsStream = new DeferredFileOutputStream(5 * 1024*1024, composedStatementsFile);
- DeferredFileOutputStream otherStatementsStream = new DeferredFileOutputStream(1024*1024, otherStatementsFile);
- DeferredFileOutputStream valueStream = new DeferredFileOutputStream(1024*1024, valueFile);
-
- ObjectOutputStream composedStatementsOutput = new ObjectOutputStream(composedStatementsStream);
- ObjectOutputStream otherStatementsOutput = new ObjectOutputStream(otherStatementsStream);
- ObjectOutputStream valueOutput = new ObjectOutputStream(valueStream);
-
- Subgraphs.getDomain(graph, ids, rootResources, preStatus, advisors, composedStatementsOutput, otherStatementsOutput, valueOutput);
- id = ids.size();
-
- composedStatementsOutput.flush();
- otherStatementsOutput.flush();
- valueOutput.flush();
- composedStatementsStream.close();
- otherStatementsStream.close();
- valueStream.close();
-
- long domainTimeEnd = System.nanoTime();
-
- long fileTime = System.nanoTime();
-
- internalCount = id;
-
- ids.put(graph.getRootLibrary(), id++);
- externalNames.add("http:/");
- externalParents.add(-1);
-
-// if(model == null) return null;
-
- InputStream composedStatementsInputStream = null;
- InputStream otherStatementsInputStream = null;
- InputStream valueInputStream = null;
-
- if(composedStatementsStream.isInMemory()) {
- composedStatementsInputStream = new ByteArrayInputStream(composedStatementsStream.getData());
- } else {
- composedStatementsInputStream = new FileInputStream(composedStatementsFile);
- }
-
- if(otherStatementsStream.isInMemory()) {
- otherStatementsInputStream = new ByteArrayInputStream(otherStatementsStream.getData());
- } else {
- otherStatementsInputStream = new FileInputStream(otherStatementsFile);
- }
-
- if(valueStream.isInMemory()) {
- valueInputStream = new ByteArrayInputStream(valueStream.getData());
- } else {
- valueInputStream = new FileInputStream(valueFile);
- }
-
- composedStatementsStream = null;
- otherStatementsStream = null;
- valueStream = null;
-
- ObjectInputStream composedStatementsInput = new ObjectInputStream(composedStatementsInputStream);
- ObjectInputStream otherStatementsInput = new ObjectInputStream(otherStatementsInputStream);
- ObjectInputStream valueInput = new ObjectInputStream(valueInputStream);
-
- long fileTimeEnd = System.nanoTime();
-
- long statementTime = System.nanoTime();
- searchStatements(graph, composedStatementsInput, otherStatementsInput, valueInput, extents);
- long statementTimeEnd = System.nanoTime();
-
- // domainParts = null;
-
- long buildTime = System.nanoTime();
-
- int resourceCount = ids.size();
-
- Identity[] identityArray;
- { // Identities
- ArrayList<Identity> identities = new ArrayList<Identity>();
-
- for(Pair<Resource, String> r : configuration.roots) {
- Resource type = graph.getPossibleType(r.first, l0.Entity);
- if(type == null) type = l0.Entity;
- identities.add(new Identity(
- ids.get(r.first),
- new Root(r.second, graph.getURI(type))
- ));
- }
-
- int internalsPlusExternals = ids.size();
- for(int i = internalCount; i < internalsPlusExternals ; i++) {
- int parent = externalParents.get(i - internalCount);
- String name = externalNames.get(i - internalCount);
- identities.add(new Identity(
- i,
- new External(parent, name)
- ));
- }
- identityArray = identities.toArray(new Identity[identities.size()]);
- }
-
- Value[] valueArray = new Value[values.size()];
- { // Values
- int index = 0;
- for(Map.Entry<Resource, Variant> entry : values.entrySet()) {
- if(LOG) log("[VALUE] " + entry.getKey().getResourceId());
- int r = getId(graph, entry.getKey(), null);
- valueArray[index++] = new Value(r, entry.getValue());
- }
- }
- ids = null;
- values = null;
-
- long buildTimeEnd = System.nanoTime();
-
- TransferableGraph1 result =
- new TransferableGraph1(resourceCount,
- identityArray,
- statements,
- valueArray);
-
- if(DEBUG) {
- System.out.println("transferable graph content: " + result);
- }
-
- long totalEnd = System.nanoTime();
-
- if(PROFILE) {
- System.out.println("startup in " + 1e-9*(startupTimeEnd - startupTime) + "s.");
- System.out.println("domain was found in " + 1e-9*(domainTimeEnd - domainTime) + "s.");
- System.out.println("files were written in " + 1e-9*(fileTimeEnd - fileTime) + "s.");
- System.out.println("statements were found in " + 1e-9*(statementTimeEnd - statementTime) + "s.");
- System.out.println("tg was built in " + 1e-9*(buildTimeEnd - buildTime) + "s.");
- System.out.println("total time for building subgraph was " + 1e-9*(totalEnd-total) + "s.");
- }
-
- return result;
- } catch (IOException e) {
- e.printStackTrace();
- }
-
- return null;
-
- }
-
-}
Code Review / simantics / platform.git / blobdiff