1 /*******************************************************************************
2 * Copyright (c) 2007, 2010 Association for Decentralized Information Management
4 * All rights reserved. This program and the accompanying materials
5 * are made available under the terms of the Eclipse Public License v1.0
6 * which accompanies this distribution, and is available at
7 * http://www.eclipse.org/legal/epl-v10.html
10 * VTT Technical Research Centre of Finland - initial API and implementation
11 *******************************************************************************/
12 package org.simantics.db.layer0.util;
14 import gnu.trove.list.array.TIntArrayList;
15 import gnu.trove.map.TIntObjectMap;
16 import gnu.trove.map.hash.TIntIntHashMap;
17 import gnu.trove.map.hash.TIntObjectHashMap;
18 import gnu.trove.procedure.TIntObjectProcedure;
19 import gnu.trove.set.hash.TIntHashSet;
21 import java.io.ByteArrayInputStream;
22 import java.io.DataOutput;
23 import java.io.DataOutputStream;
25 import java.io.FileInputStream;
26 import java.io.FileNotFoundException;
27 import java.io.FileOutputStream;
28 import java.io.IOException;
29 import java.io.InputStream;
30 import java.io.ObjectInputStream;
31 import java.io.ObjectOutputStream;
32 import java.lang.management.ManagementFactory;
33 import java.lang.reflect.InvocationTargetException;
34 import java.lang.reflect.Method;
35 import java.util.ArrayList;
36 import java.util.Collection;
37 import java.util.HashMap;
39 import java.util.TreeMap;
40 import java.util.UUID;
42 import org.apache.commons.io.output.DeferredFileOutputStream;
43 import org.simantics.databoard.Bindings;
44 import org.simantics.databoard.binding.mutable.Variant;
45 import org.simantics.db.ReadGraph;
46 import org.simantics.db.Resource;
47 import org.simantics.db.common.utils.NameUtils;
48 import org.simantics.db.exception.DatabaseException;
49 import org.simantics.db.exception.ValidationException;
50 import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus;
51 import org.simantics.db.request.Read;
52 import org.simantics.db.service.ClusterControl;
53 import org.simantics.db.service.ClusterControl.ClusterState;
54 import org.simantics.db.service.SerialisationSupport;
55 import org.simantics.graph.representation.External;
56 import org.simantics.graph.representation.Identity;
57 import org.simantics.graph.representation.Root;
58 import org.simantics.graph.representation.TransferableGraph1;
59 import org.simantics.graph.representation.Value;
60 import org.simantics.layer0.Layer0;
61 import org.simantics.utils.datastructures.Pair;
64 * @deprecated in favor of {@link ModelTransferableGraphSourceRequest}
67 public class TransferableGraphRequest2 implements Read<TransferableGraph1> {
69 public static String LOG_FILE = "transferableGraph.log";
70 final static private boolean LOG = false;
71 final static private boolean DEBUG = false;
72 final static private boolean PROFILE = false;
74 private TransferableGraphConfiguration configuration;
76 static DataOutput log;
82 FileOutputStream stream = new FileOutputStream(LOG_FILE);
83 log = new DataOutputStream(stream);
84 } catch (FileNotFoundException e) {
91 private static void log(String line) {
94 log.writeUTF(line + "\n");
95 } catch (IOException e) {
103 public TransferableGraphRequest2(Collection<Pair<Resource, String>> roots, Resource model) {
105 configuration = new TransferableGraphConfiguration();
106 configuration.roots = roots;
107 configuration.model = model;
112 public TransferableGraphRequest2(Collection<Pair<Resource, String>> roots) {
116 public TransferableGraphRequest2(TransferableGraphConfiguration conf) {
117 this.configuration = conf;
122 TIntArrayList inverses = new TIntArrayList();
124 int statementIndex = 0;
126 TIntObjectMap<Variant> values;
127 TIntArrayList externalParents = new TIntArrayList();
128 ArrayList<String> externalNames = new ArrayList<String>();
134 private SerialisationSupport support;
136 private boolean validateExternal(Resource r) {
137 if(configuration.disallowedExternals != null) {
138 System.err.println("validateExternal agains " + configuration.disallowedExternals);
139 return !configuration.disallowedExternals.contains(r);
144 private Resource getResource(int r) throws DatabaseException {
145 return support.getResource(r);
148 public int getInternalId(int r) {
152 public int getId(ReadGraph graph, int r, int predicate) throws DatabaseException {
153 if(ids.containsKey(r)) {
154 int ret = ids.get(r);
156 for(int i=0;i<=indent;++i)
157 System.out.print(" ");
158 System.out.println("Cycle!!!"); // with " + GraphUtils.getReadableName(g, r));
163 Collection<Resource> parents = graph.getObjects(getResource(r), L0.PartOf);
164 if(parents.size() != 1) {
165 throw new ValidationException("Reference to external resource "
166 + NameUtils.getSafeName(graph, getResource(r), true) + " without unique uri (" + parents.size() + " parents).");
168 for(Resource p : parents) {
170 if(!validateExternal(p)) throw new ValidationException("References to '" + graph.getURI(p) + "' are not allowed.");
171 externalParents.add(getId(graph, support.getTransientId(p), 0));
174 externalNames.add((String)graph.getRelatedValue(getResource(r), L0.HasName));
180 public void addId(ReadGraph graph, int r, int predicate) throws DatabaseException {
181 statements[statementIndex++] = getId(graph, r, predicate);
184 public void setExternals(Collection<Resource> rs) {
185 configuration.externals = rs;
189 public TransferableGraph1 perform(ReadGraph graph) throws DatabaseException {
191 support = graph.getService(SerialisationSupport.class);
193 this.L0 = Layer0.getInstance(graph);
195 long total = System.nanoTime();
197 long startupTime = System.nanoTime();
199 ClusterControl cc = graph.getService(ClusterControl.class);
201 ids = new TIntIntHashMap();
202 values = new TIntObjectHashMap<Variant>();
204 ArrayList<Resource> rootResources = new ArrayList<Resource>();
205 for(Pair<Resource, String> p : configuration.roots) rootResources.add(p.first);
207 Map<Resource, ExtentStatus> preStatus = new HashMap<Resource, ExtentStatus>();
209 for(Resource root : rootResources) {
210 Resource name = graph.getPossibleObject(root, L0.HasName);
212 preStatus.put(name, ExtentStatus.EXCLUDED);
216 for(Resource r : configuration.externals) preStatus.put(r, ExtentStatus.EXTERNAL);
218 long startupTimeEnd = System.nanoTime();
220 long domainTime = System.nanoTime();
222 String otherStatements = "other" + UUID.randomUUID().toString();
223 String valueFileName = "value" + UUID.randomUUID().toString();
225 File otherStatementsFile = new File(otherStatements);
226 File valueFile = new File(valueFileName);
230 DeferredFileOutputStream otherStatementsStream = new DeferredFileOutputStream(1024*1024, otherStatementsFile);
231 DeferredFileOutputStream valueStream = new DeferredFileOutputStream(1024*1024, valueFile);
233 ObjectOutputStream otherStatementsOutput = new ObjectOutputStream(otherStatementsStream);
234 ObjectOutputStream valueOutput = new ObjectOutputStream(valueStream);
236 ClusterState clusterState = cc.getClusterState();
238 TIntHashSet excludedShared = new TIntHashSet();
240 TreeMap<String, Variant> extensions = new TreeMap<String, Variant>();
242 Subgraphs.getDomain2(graph, ids, rootResources, preStatus, configuration.specials, otherStatementsOutput, valueOutput, extensions, excludedShared);
246 cc.restoreClusterState(clusterState);
248 // dumpHeap("domain.hprof");
250 otherStatementsOutput.flush();
252 otherStatementsStream.close();
255 long domainDuration = System.nanoTime() - domainTime;
256 System.err.println("Analysed graph in " + 1e-9*domainDuration + "s.");
260 ids.put(support.getTransientId(graph.getResource("http:/")), id++);
261 externalNames.add("http:/");
262 externalParents.add(-1);
264 InputStream otherStatementsInputStream = null;
265 InputStream valueInputStream = null;
267 if(otherStatementsStream.isInMemory()) {
268 otherStatementsInputStream = new ByteArrayInputStream(otherStatementsStream.getData());
270 otherStatementsInputStream = new FileInputStream(otherStatementsFile);
273 if(valueStream.isInMemory()) {
274 valueInputStream = new ByteArrayInputStream(valueStream.getData());
276 valueInputStream = new FileInputStream(valueFile);
279 otherStatementsStream = null;
282 ObjectInputStream otherStatementsInput = new ObjectInputStream(otherStatementsInputStream);
283 ObjectInputStream valueInput = new ObjectInputStream(valueInputStream);
285 long statementTime = System.nanoTime();
287 TIntArrayList statementSet = new TIntArrayList();
289 while(otherStatementsInput.available() > 0) {
291 int s = otherStatementsInput.readInt();
293 boolean exclude = !ids.contains(s);
295 int size = otherStatementsInput.readInt();
296 for(int i=0;i<size;i++) {
297 int p = otherStatementsInput.readInt();
298 int o = otherStatementsInput.readInt();
300 if(excludedShared.contains(o)) {
301 // System.err.println("excluding shared " + s + " " + p + " " + o);
308 // System.err.println("excluding shared " + s);
314 TIntIntHashMap inverses = new TIntIntHashMap();
315 TIntHashSet predicateSet = new TIntHashSet();
316 for(int i=0;i<statementSet.size();i+=3) {
317 int p = statementSet.getQuick(i+1);
318 if(predicateSet.add(p)) {
319 Resource inverse = graph.getPossibleInverse(getResource(p));
320 if(inverse != null) inverses.put(p, support.getTransientId(inverse));
326 TIntArrayList tgStatements = new TIntArrayList();
328 // dumpHeap("export.hprof");
331 int trim = Math.max(65536,statementSet.size()/12);
333 for(int i=statementSet.size();i>0;i-=3) {
336 statementSet.remove(i, statementSet.size()-i);
337 statementSet.trimToSize();
338 trim = Math.max(65536,statementSet.size()/12);
341 int s = statementSet.getQuick(i-3);
342 int p = statementSet.getQuick(i-2);
343 int o = statementSet.getQuick(i-1);
345 int subjectId = ids.get(s);
346 if(subjectId >= internalCount) System.err.println("Statement for external: " + s + " " + p + " " + o);
348 int objectId = getId(graph, o, p);
349 // The statement can be denied still
351 tgStatements.add(subjectId);
352 tgStatements.add(getId(graph, p, 0));
353 int inverse = inverses.get(p);
355 tgStatements.add(getId(graph, inverse, 0));
357 tgStatements.add(-1);
359 tgStatements.add(objectId);
361 System.out.println("denied");
366 statements = tgStatements.toArray();
368 long statementDuration = System.nanoTime() - statementTime;
369 System.err.println("Built transferable statements in " + 1e-9*statementDuration + "s.");
373 while(valueInput.available() > 0) {
375 int s = valueInput.readInt();
376 // Resource subject = support.getResource(s);
377 int valueSize = valueInput.readInt();
378 byte[] value = new byte[valueSize];
379 valueInput.readFully(value);
380 Variant variant = (Variant)Bindings.VARIANT.serializer().deserialize(value);
381 values.put(s, variant);
385 int resourceCount = ids.size();
387 Identity[] identityArray;
389 ArrayList<Identity> identities = new ArrayList<Identity>();
391 for(Pair<Resource, String> r : configuration.roots) {
392 Resource type = graph.getPossibleType(r.first, L0.Entity);
393 if(type == null) type = L0.Entity;
394 identities.add(new Identity(
395 ids.get(support.getTransientId(r.first)),
396 new Root(r.second, graph.getURI(type))
400 int internalsPlusExternals = ids.size();
401 for(int i = internalCount; i < internalsPlusExternals ; i++) {
402 int parent = externalParents.get(i - internalCount);
403 String name = externalNames.get(i - internalCount);
404 identities.add(new Identity(
406 new External(parent, name)
409 identityArray = identities.toArray(new Identity[identities.size()]);
412 final Value[] valueArray = new Value[values.size()];
414 values.forEachEntry(new TIntObjectProcedure<Variant>() {
419 public boolean execute(int subject, Variant bytes) {
420 //if(LOG) log("[VALUE] " + entry.getKey().getResourceId());
421 int r = getInternalId(subject);
422 if(r==-1) System.err.println("No id for value resource " + subject);
423 else valueArray[index++] = new Value(r, bytes);
432 TransferableGraph1 result =
433 new TransferableGraph1(resourceCount,
436 valueArray, extensions);
439 System.out.println("transferable graph content: " + result);
442 long totalEnd = System.nanoTime();
445 System.out.println("startup in " + 1e-9*(startupTimeEnd - startupTime) + "s.");
446 System.out.println("domain was found in " + 1e-9*(domainDuration) + "s.");
447 System.out.println("statements were found in " + 1e-9*(statementDuration) + "s.");
448 System.out.println("total time for building subgraph was " + 1e-9*(totalEnd-total) + "s.");
453 } catch (IOException e) {
455 } catch (Throwable t) {
457 dumpHeap("crash.hprof");
464 private static void dumpHeap(String path) {
467 Object bean = getBean();
471 Method m = bean.getClass().getMethod("dumpHeap", String.class, boolean.class);
472 m.invoke(bean, path, true);
474 } catch (IllegalArgumentException e) {
475 } catch (IllegalAccessException e) {
476 } catch (SecurityException e) {
477 } catch (NoSuchMethodException e) {
478 } catch (InvocationTargetException e) {
484 private static Object getBean() {
485 Class<?> beanClass = getBeanClass();
486 if (beanClass == null)
489 Object bean = ManagementFactory.newPlatformMXBeanProxy(
490 ManagementFactory.getPlatformMBeanServer(),
491 "com.sun.management:type=HotSpotDiagnostic",
494 } catch (IOException e) {
499 private static Class<?> getBeanClass() {
501 Class<?> clazz = Class.forName("com.sun.management.HotSpotDiagnosticMXBean");
503 } catch (ClassNotFoundException e) {