1 /*******************************************************************************
2 * Copyright (c) 2007, 2010 Association for Decentralized Information Management
4 * All rights reserved. This program and the accompanying materials
5 * are made available under the terms of the Eclipse Public License v1.0
6 * which accompanies this distribution, and is available at
7 * http://www.eclipse.org/legal/epl-v10.html
10 * VTT Technical Research Centre of Finland - initial API and implementation
11 *******************************************************************************/
12 package org.simantics.db.layer0.util;
14 import gnu.trove.list.array.TIntArrayList;
16 import java.io.ByteArrayInputStream;
17 import java.io.DataOutput;
18 import java.io.DataOutputStream;
20 import java.io.FileInputStream;
21 import java.io.FileNotFoundException;
22 import java.io.FileOutputStream;
23 import java.io.IOException;
24 import java.io.InputStream;
25 import java.io.ObjectInputStream;
26 import java.io.ObjectOutputStream;
27 import java.util.ArrayList;
28 import java.util.Collection;
29 import java.util.HashMap;
30 import java.util.HashSet;
31 import java.util.LinkedList;
34 import java.util.UUID;
35 import java.util.concurrent.ConcurrentSkipListSet;
36 import java.util.concurrent.atomic.AtomicInteger;
38 import org.apache.commons.io.output.DeferredFileOutputStream;
39 import org.simantics.databoard.Bindings;
40 import org.simantics.databoard.Databoard;
41 import org.simantics.databoard.binding.mutable.Variant;
42 import org.simantics.databoard.serialization.Serializer;
43 import org.simantics.db.AsyncReadGraph;
44 import org.simantics.db.ReadGraph;
45 import org.simantics.db.Resource;
46 import org.simantics.db.Statement;
47 import org.simantics.db.common.StandardStatement;
48 import org.simantics.db.common.utils.NameUtils;
49 import org.simantics.db.exception.DatabaseException;
50 import org.simantics.db.exception.ValidationException;
51 import org.simantics.db.layer0.adapter.Instances;
52 import org.simantics.db.layer0.adapter.SubgraphAdvisor;
53 import org.simantics.db.layer0.adapter.SubgraphExtent;
54 import org.simantics.db.layer0.adapter.SubgraphExtent.Classifier;
55 import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus;
56 import org.simantics.db.procedure.AsyncProcedure;
57 import org.simantics.db.request.AsyncRead;
58 import org.simantics.db.request.Read;
59 import org.simantics.db.service.CollectionSupport;
60 import org.simantics.db.service.QueryControl;
61 import org.simantics.db.service.QueryControl.ControlProcedure;
62 import org.simantics.db.service.SerialisationSupport;
63 import org.simantics.graph.representation.External;
64 import org.simantics.graph.representation.Identity;
65 import org.simantics.graph.representation.Root;
66 import org.simantics.graph.representation.TransferableGraph1;
67 import org.simantics.graph.representation.Value;
68 import org.simantics.layer0.Layer0;
69 import org.simantics.operation.Layer0X;
70 import org.simantics.utils.datastructures.Pair;
71 import org.simantics.utils.datastructures.Triple;
74 * IsComposedOf objects are always in domain
75 * Ordered set elements are always in domain
76 * r is in domain if for all statements (s, IsRelatedTo, r) s is in domain
78 * IsWeaklyRelatedTo statements where subject is in domain are only accepted if object is in domain or has URI
80 * @deprecated in favor of {@link TransferableGraphRequest2}
82 public class TransferableGraphRequest implements Read<TransferableGraph1> {
84 public static String LOG_FILE = "transferableGraph.log";
85 final static private boolean LOG = false;
86 final static private boolean DEBUG = false;
87 final static private boolean PROFILE = false;
89 // final private Collection<Pair<Resource, String>> roots;
90 // private Collection<Resource> preExternals = Collections.emptyList();
91 // private Resource model;
93 private TransferableGraphConfiguration configuration;
95 static DataOutput log;
101 FileOutputStream stream = new FileOutputStream(LOG_FILE);
102 log = new DataOutputStream(stream);
103 } catch (FileNotFoundException e) {
110 private static void log(String line) {
113 log.writeUTF(line + "\n");
114 } catch (IOException e) {
122 public TransferableGraphRequest(Collection<Pair<Resource, String>> roots, Resource model) {
124 configuration = new TransferableGraphConfiguration();
125 configuration.roots = roots;
126 configuration.model = model;
128 // this.roots = roots;
129 // this.model = model;
134 public TransferableGraphRequest(Collection<Pair<Resource, String>> roots) {
138 public TransferableGraphRequest(TransferableGraphConfiguration conf) {
139 this.configuration = conf;
144 TIntArrayList inverses = new TIntArrayList();
146 int statementIndex = 0;
147 Map<Resource, Integer> ids;
148 Map<Resource, Variant> values;
149 TIntArrayList externalParents = new TIntArrayList();
150 ArrayList<String> externalNames = new ArrayList<String>();
151 // ConcurrentLinkedQueue<Pair<Resource, byte[]>> values = new ConcurrentLinkedQueue<Pair<Resource, byte[]>>();
157 private Serializer variantSerializer;
159 private boolean validateExternal(Resource r) {
160 if(configuration.disallowedExternals != null) {
161 System.err.println("validateExternal agains " + configuration.disallowedExternals);
162 return !configuration.disallowedExternals.contains(r);
167 public int getId(ReadGraph graph, Resource r, Resource predicate) throws DatabaseException {
168 // for(int i=0;i<indent;++i)
169 // System.out.print(" ");
170 // System.out.println(NameUtils.getSafeName(graph, r));
171 if(ids.containsKey(r)) {
172 int ret = ids.get(r);
174 for(int i=0;i<=indent;++i)
175 System.out.print(" ");
176 System.out.println("Cycle!!!"); // with " + GraphUtils.getReadableName(g, r));
182 Collection<Resource> parents = graph.getObjects(r, l0.PartOf);
183 if(parents.size() != 1) {
184 if(parents.size() == 0) {
185 Resource inv = graph.getPossibleObject(r, l0.InverseOf);
188 if(DEBUG) System.out.println("inverse " + NameUtils.getSafeName(graph, inv));
189 int invId = getId(graph, inv, null);
190 externalParents.add(invId);
192 externalNames.add("@inverse");
197 // for(Statement stat : graph.getStatements(r, b.IsWeaklyRelatedTo))
198 // System.out.println(NameUtils.getSafeName(graph, stat.getPredicate())
199 // + " -> " + NameUtils.getSafeName(graph, stat.getObject()));
200 // if(predicate != null) {
201 // if(!graph.isSubrelationOf(predicate, b.IsRelatedTo)) {
205 throw new ValidationException("Reference to external resource "
206 + NameUtils.getSafeName(graph, r, true) + " without unique uri (" + parents.size() + " parents).");
208 for(Resource p : parents) {
209 // System.out.println("Parent " + NameUtils.getSafeName(graph, p));
211 if(!validateExternal(p)) throw new ValidationException("References to '" + graph.getURI(p) + "' are not allowed.");
212 externalParents.add(getId(graph, p, null));
215 // System.out.println("Request named for " + GraphUtils.getReadableName(g, r));
216 // String name = graph.getPossibleRelatedValue(r, b.HasName);
217 externalNames.add((String)graph.getRelatedValue(r, l0.HasName));
218 // if(name != null) externalNames.add(name);
219 // else externalNames.add("@resource");
225 public void addId(ReadGraph graph, Resource r, Resource predicate) throws DatabaseException {
226 statements[statementIndex++] = getId(graph, r, predicate);
229 private void searchStatementPart(ReadGraph graph, final ConcurrentSkipListSet<Statement> statementSet, final ArrayList<Pair<Resource, Triple[]>> part, final Set<SubgraphExtent> extents) throws DatabaseException {
231 final SubgraphExtent.Callback callback = new SubgraphExtent.Callback() {
234 public void statement(Statement statement, boolean accept) {
236 statementSet.add(statement);
242 graph.syncRequest(new AsyncRead<Boolean>() {
245 public int threadHash() {
250 public void perform(AsyncReadGraph graph, AsyncProcedure<Boolean> procedure) {
252 QueryControl control = graph.getService(QueryControl.class);
254 int slice = (int)(part.size() / control.getAmountOfQueryThreads()) + 1;
256 final Pair[] rootArray = (Pair[])part.toArray(new Pair[part.size()]);
257 for(int i=0;i<control.getAmountOfQueryThreads();i++) {
259 final int start = i * slice;
260 final int end = Math.min(start + slice, rootArray.length);
262 control.schedule(graph, i, new ControlProcedure() {
265 public void execute(AsyncReadGraph graph) {
267 for(int index = start;index < end;index++) {
269 final Pair<Resource, Triple[]> r = (Pair<Resource, Triple[]>)rootArray[index];
271 final AtomicInteger position = new AtomicInteger(0);
272 final SubgraphExtent.Classifier[] classifiers = new SubgraphExtent.Classifier[extents.size()];
274 for(SubgraphExtent extent : extents) {
276 extent.accept(graph, r.first, new AsyncProcedure<Classifier>() {
278 public void execute(AsyncReadGraph graph, Classifier c) {
280 int current = position.incrementAndGet();
281 classifiers[current-1] = c;
283 if(current == extents.size()) {
285 for(Triple statement : r.second) {
286 if(!r.first.isPersistent()) continue;
287 for(Classifier classifier : classifiers) {
288 classifier.classify(graph, new StandardStatement(r.first, (Resource)statement.first, (Resource)statement.second), (ExtentStatus)statement.third, callback);
292 // graph.forEachDirectStatement(r, new AsyncMultiProcedure<Statement>() {
295 // public void exception(AsyncReadGraph graph, Throwable throwable) {
296 // throwable.printStackTrace();
300 // public void execute(AsyncReadGraph graph, Statement statement) {
304 // public void finished(AsyncReadGraph graph) {
313 public void exception(AsyncReadGraph graph, Throwable throwable) {
314 throwable.printStackTrace();
329 procedure.execute(graph, false);
334 public int getFlags() {
342 class StatementSetBuilder {
344 private int SLICE = 100000;
346 final private CollectionSupport cs;
347 final private LinkedList<Collection<Statement>> sets = new LinkedList<Collection<Statement>>();
349 private Collection<Statement> current;
351 StatementSetBuilder(CollectionSupport cs) {
353 current = cs.createStatementList();
356 LinkedList<Collection<Statement>> get() {
361 void add(Statement stm) {
363 if(current.size() == SLICE) {
365 current = cs.createStatementList();
371 for(Collection<Statement> c : sets) result += c.size();
376 void addAll(Collection<Statement> ss) {
377 for(Statement s : ss) add(s);
382 private void searchStatements(ReadGraph graph, ObjectInputStream composedInput, ObjectInputStream statementInput, ObjectInputStream valueInput, final Set<SubgraphExtent> extents) throws DatabaseException, IOException {
384 CollectionSupport cs = graph.getService(CollectionSupport.class);
386 ConcurrentSkipListSet<Statement> statementSet = new ConcurrentSkipListSet<Statement>();
388 StatementSetBuilder builder = new StatementSetBuilder(cs);
390 SerialisationSupport support = graph.getService(SerialisationSupport.class);
392 while(composedInput.available() > 0) {
394 int s = composedInput.readInt();
395 Resource subject = support.getResource(s);
397 int size = composedInput.readInt();
398 for(int i=0;i<size;i++) {
399 int p = composedInput.readInt();
400 int o = composedInput.readInt();
401 StandardStatement stm = new StandardStatement(subject, support.getResource(p), support.getResource(o));
402 // if(DEBUG) System.out.println("composed " + NameUtils.toString(graph, stm));
408 while(statementInput.available() > 0) {
410 ArrayList<Pair<Resource, Triple[]>> stms = new ArrayList<Pair<Resource, Triple[]>>();
414 while(statementInput.available() > 0 && counter++ < 1000) {
416 int s = statementInput.readInt();
417 Resource subject = support.getResource(s);
418 int size = statementInput.readInt();
419 Triple[] list = new Triple[size];
420 stms.add(Pair.make(subject, list));
421 for(int i=0;i<size;i++) {
422 int p = statementInput.readInt();
423 int o = statementInput.readInt();
425 System.out.println("otherInput " + NameUtils.toString(graph, new StandardStatement(support.getResource(s), support.getResource(p), support.getResource(o))));
427 list[i] = new Triple<Resource, Resource, ExtentStatus>(support.getResource(p), support.getResource(o), ExtentStatus.EXTERNAL);
431 searchStatementPart(graph, statementSet, stms, extents);
436 for(Statement stm : statementSet)
437 System.out.println("other " + NameUtils.toString(graph, stm));
440 builder.addAll(statementSet);
442 if(DEBUG) System.out.println("total " + statementSet.size() + " statements found.");
446 LinkedList<Collection<Statement>> statementSets = builder.get();
448 Map<Resource, Resource> inverses = cs.createMap(Resource.class);
449 Set<Resource> predicateSet = cs.createSet();
450 for(Collection<Statement> set : statementSets)
451 for(Statement s : set) {
452 Resource predicate = s.getPredicate();
453 if(predicateSet.add(predicate)) {
454 Resource inverse = graph.getPossibleInverse(predicate);
455 inverses.put(predicate, inverse);
461 statements = new int[4 * builder.size()];
463 while(!statementSets.isEmpty()) {
464 Collection<Statement> set = statementSets.pop();
465 for(Statement s : set) {
466 Resource subject = s.getSubject();
467 Resource predicate = s.getPredicate();
468 Resource object = s.getObject();
469 int subjectId = ids.get(subject);
470 int objectId = getId(graph, object, predicate);
471 // The statement can be denied still
473 statements[statementIndex++] = subjectId;
474 addId(graph, predicate, null);
475 Resource inverse = inverses.get(predicate);
476 if(inverse != null) {
477 if(LOG) log("[STM] " + subject.getResourceId() + ", " + predicate.getResourceId() + ", " + inverse.getResourceId() + ", " + object.getResourceId());
478 addId(graph, inverse, null);
480 if(LOG) log("[STM] " + subject.getResourceId() + ", " + predicate.getResourceId() + ", -1, " + object.getResourceId());
481 statements[statementIndex++] = -1;
483 statements[statementIndex++] = objectId;
488 System.out.println("denied");
495 while(valueInput.available() > 0) {
497 int s = valueInput.readInt();
498 Resource subject = support.getResource(s);
499 int valueSize = valueInput.readInt();
500 byte[] value = new byte[valueSize];
501 valueInput.readFully(value);
502 Variant variant = (Variant) variantSerializer.deserialize(value);
503 values.put(subject, variant);
509 public void setExternals(Collection<Resource> rs) {
510 configuration.externals = rs;
514 public TransferableGraph1 perform(ReadGraph graph) throws DatabaseException {
516 this.variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT);
518 this.l0 = Layer0.getInstance(graph);
519 Layer0X L0X = Layer0X.getInstance(graph);
521 long total = System.nanoTime();
523 long startupTime = System.nanoTime();
525 CollectionSupport cs = graph.getService(CollectionSupport.class);
527 ids = cs.createMap(Integer.class);
528 values = cs.createMap(byte[].class);
530 ArrayList<Resource> rootResources = new ArrayList<Resource>();
531 for(Pair<Resource, String> p : configuration.roots) rootResources.add(p.first);
533 // if(model == null) {
534 // SimulationResource SIMU = SimulationResource.getInstance(graph);
535 // for(Resource root : rootResources) {
536 // if(graph.isInstanceOf(root, SIMU.Model)) {
540 // model = graph.syncRequest(new TypedParent(root, SIMU.Model));
541 // if(model != null) break;
545 Map<Resource, ExtentStatus> preStatus = new HashMap<Resource, ExtentStatus>();
547 for(Resource root : rootResources) {
548 Resource name = graph.getPossibleObject(root, l0.HasName);
550 preStatus.put(name, ExtentStatus.EXCLUDED);
554 for(Resource r : configuration.externals) preStatus.put(r, ExtentStatus.EXTERNAL);
556 Set<SubgraphExtent> extents = configuration.extents;
557 if(extents == null) {
559 Instances extentInstances = graph.adapt(L0X.SubgraphExtent, Instances.class);
560 Collection<Resource> extentResources = extentInstances.find(graph, configuration.model);
561 extents = new HashSet<SubgraphExtent>();
562 for(Resource r : extentResources) {
563 extents.add(graph.getPossibleAdapter(r, SubgraphExtent.class));
567 System.out.println("Extents " + extents.size());
568 for(Resource extent : extentResources) System.out.println("-" + NameUtils.getSafeName(graph, extent));
573 Set<SubgraphAdvisor> advisors = configuration.advisors;
574 if(advisors == null) {
576 Instances advisorInstances = graph.adapt(L0X.SubgraphAdvisor, Instances.class);
577 Collection<Resource> advisorResources = advisorInstances.find(graph, configuration.model);
578 advisors = new HashSet<SubgraphAdvisor>();
579 for(Resource r : advisorResources) {
580 advisors.add(graph.getPossibleAdapter(r, SubgraphAdvisor.class));
584 System.out.println("Advisors " + advisors.size());
585 for(Resource advisor : advisorResources) System.out.println("-" + NameUtils.getSafeName(graph, advisor));
590 long startupTimeEnd = System.nanoTime();
592 long domainTime = System.nanoTime();
594 String composedStatements = "composed" + UUID.randomUUID().toString();
595 String otherStatements = "other" + UUID.randomUUID().toString();
596 String valueFileName = "value" + UUID.randomUUID().toString();
598 File composedStatementsFile = new File(composedStatements);
599 File otherStatementsFile = new File(otherStatements);
600 File valueFile = new File(valueFileName);
603 System.out.println("getDomain writes " + composedStatementsFile.getAbsolutePath());
607 DeferredFileOutputStream composedStatementsStream = new DeferredFileOutputStream(5 * 1024*1024, composedStatementsFile);
608 DeferredFileOutputStream otherStatementsStream = new DeferredFileOutputStream(1024*1024, otherStatementsFile);
609 DeferredFileOutputStream valueStream = new DeferredFileOutputStream(1024*1024, valueFile);
611 ObjectOutputStream composedStatementsOutput = new ObjectOutputStream(composedStatementsStream);
612 ObjectOutputStream otherStatementsOutput = new ObjectOutputStream(otherStatementsStream);
613 ObjectOutputStream valueOutput = new ObjectOutputStream(valueStream);
615 Subgraphs.getDomain(graph, ids, rootResources, preStatus, advisors, composedStatementsOutput, otherStatementsOutput, valueOutput);
618 composedStatementsOutput.flush();
619 otherStatementsOutput.flush();
621 composedStatementsStream.close();
622 otherStatementsStream.close();
625 long domainTimeEnd = System.nanoTime();
627 long fileTime = System.nanoTime();
631 ids.put(graph.getRootLibrary(), id++);
632 externalNames.add("http:/");
633 externalParents.add(-1);
635 // if(model == null) return null;
637 InputStream composedStatementsInputStream = null;
638 InputStream otherStatementsInputStream = null;
639 InputStream valueInputStream = null;
641 if(composedStatementsStream.isInMemory()) {
642 composedStatementsInputStream = new ByteArrayInputStream(composedStatementsStream.getData());
644 composedStatementsInputStream = new FileInputStream(composedStatementsFile);
647 if(otherStatementsStream.isInMemory()) {
648 otherStatementsInputStream = new ByteArrayInputStream(otherStatementsStream.getData());
650 otherStatementsInputStream = new FileInputStream(otherStatementsFile);
653 if(valueStream.isInMemory()) {
654 valueInputStream = new ByteArrayInputStream(valueStream.getData());
656 valueInputStream = new FileInputStream(valueFile);
659 composedStatementsStream = null;
660 otherStatementsStream = null;
663 ObjectInputStream composedStatementsInput = new ObjectInputStream(composedStatementsInputStream);
664 ObjectInputStream otherStatementsInput = new ObjectInputStream(otherStatementsInputStream);
665 ObjectInputStream valueInput = new ObjectInputStream(valueInputStream);
667 long fileTimeEnd = System.nanoTime();
669 long statementTime = System.nanoTime();
670 searchStatements(graph, composedStatementsInput, otherStatementsInput, valueInput, extents);
671 long statementTimeEnd = System.nanoTime();
673 // domainParts = null;
675 long buildTime = System.nanoTime();
677 int resourceCount = ids.size();
679 Identity[] identityArray;
681 ArrayList<Identity> identities = new ArrayList<Identity>();
683 for(Pair<Resource, String> r : configuration.roots) {
684 Resource type = graph.getPossibleType(r.first, l0.Entity);
685 if(type == null) type = l0.Entity;
686 identities.add(new Identity(
688 new Root(r.second, graph.getURI(type))
692 int internalsPlusExternals = ids.size();
693 for(int i = internalCount; i < internalsPlusExternals ; i++) {
694 int parent = externalParents.get(i - internalCount);
695 String name = externalNames.get(i - internalCount);
696 identities.add(new Identity(
698 new External(parent, name)
701 identityArray = identities.toArray(new Identity[identities.size()]);
704 Value[] valueArray = new Value[values.size()];
707 for(Map.Entry<Resource, Variant> entry : values.entrySet()) {
708 if(LOG) log("[VALUE] " + entry.getKey().getResourceId());
709 int r = getId(graph, entry.getKey(), null);
710 valueArray[index++] = new Value(r, entry.getValue());
716 long buildTimeEnd = System.nanoTime();
718 TransferableGraph1 result =
719 new TransferableGraph1(resourceCount,
725 System.out.println("transferable graph content: " + result);
728 long totalEnd = System.nanoTime();
731 System.out.println("startup in " + 1e-9*(startupTimeEnd - startupTime) + "s.");
732 System.out.println("domain was found in " + 1e-9*(domainTimeEnd - domainTime) + "s.");
733 System.out.println("files were written in " + 1e-9*(fileTimeEnd - fileTime) + "s.");
734 System.out.println("statements were found in " + 1e-9*(statementTimeEnd - statementTime) + "s.");
735 System.out.println("tg was built in " + 1e-9*(buildTimeEnd - buildTime) + "s.");
736 System.out.println("total time for building subgraph was " + 1e-9*(totalEnd-total) + "s.");
740 } catch (IOException e) {