1 /*******************************************************************************
\r
2 * Copyright (c) 2007, 2010 Association for Decentralized Information Management
\r
3 * in Industry THTH ry.
\r
4 * All rights reserved. This program and the accompanying materials
\r
5 * are made available under the terms of the Eclipse Public License v1.0
\r
6 * which accompanies this distribution, and is available at
\r
7 * http://www.eclipse.org/legal/epl-v10.html
\r
10 * VTT Technical Research Centre of Finland - initial API and implementation
\r
11 *******************************************************************************/
\r
12 package org.simantics.db.layer0.util;
\r
14 import gnu.trove.list.array.TIntArrayList;
\r
15 import gnu.trove.map.TIntObjectMap;
\r
16 import gnu.trove.map.hash.TIntIntHashMap;
\r
17 import gnu.trove.map.hash.TIntObjectHashMap;
\r
18 import gnu.trove.procedure.TIntObjectProcedure;
\r
19 import gnu.trove.set.hash.TIntHashSet;
\r
21 import java.io.ByteArrayInputStream;
\r
22 import java.io.DataOutput;
\r
23 import java.io.DataOutputStream;
\r
24 import java.io.File;
\r
25 import java.io.FileInputStream;
\r
26 import java.io.FileNotFoundException;
\r
27 import java.io.FileOutputStream;
\r
28 import java.io.IOException;
\r
29 import java.io.InputStream;
\r
30 import java.io.ObjectInputStream;
\r
31 import java.io.ObjectOutputStream;
\r
32 import java.lang.management.ManagementFactory;
\r
33 import java.lang.reflect.InvocationTargetException;
\r
34 import java.lang.reflect.Method;
\r
35 import java.util.ArrayList;
\r
36 import java.util.Collection;
\r
37 import java.util.HashMap;
\r
38 import java.util.Map;
\r
39 import java.util.TreeMap;
\r
40 import java.util.UUID;
\r
42 import org.apache.commons.io.output.DeferredFileOutputStream;
\r
43 import org.simantics.databoard.Bindings;
\r
44 import org.simantics.databoard.binding.mutable.Variant;
\r
45 import org.simantics.db.ReadGraph;
\r
46 import org.simantics.db.Resource;
\r
47 import org.simantics.db.common.utils.NameUtils;
\r
48 import org.simantics.db.exception.DatabaseException;
\r
49 import org.simantics.db.exception.ValidationException;
\r
50 import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus;
\r
51 import org.simantics.db.request.Read;
\r
52 import org.simantics.db.service.ClusterControl;
\r
53 import org.simantics.db.service.ClusterControl.ClusterState;
\r
54 import org.simantics.db.service.SerialisationSupport;
\r
55 import org.simantics.graph.representation.External;
\r
56 import org.simantics.graph.representation.Identity;
\r
57 import org.simantics.graph.representation.Root;
\r
58 import org.simantics.graph.representation.TransferableGraph1;
\r
59 import org.simantics.graph.representation.Value;
\r
60 import org.simantics.layer0.Layer0;
\r
61 import org.simantics.utils.datastructures.Pair;
\r
64 * @deprecated in favor of {@link ModelTransferableGraphSourceRequest}
\r
67 public class TransferableGraphRequest2 implements Read<TransferableGraph1> {
\r
69 public static String LOG_FILE = "transferableGraph.log";
\r
70 final static private boolean LOG = false;
\r
71 final static private boolean DEBUG = false;
\r
72 final static private boolean PROFILE = false;
\r
74 private TransferableGraphConfiguration configuration;
\r
76 static DataOutput log;
\r
82 FileOutputStream stream = new FileOutputStream(LOG_FILE);
\r
83 log = new DataOutputStream(stream);
\r
84 } catch (FileNotFoundException e) {
\r
85 e.printStackTrace();
\r
91 private static void log(String line) {
\r
94 log.writeUTF(line + "\n");
\r
95 } catch (IOException e) {
\r
96 e.printStackTrace();
\r
103 public TransferableGraphRequest2(Collection<Pair<Resource, String>> roots, Resource model) {
\r
105 configuration = new TransferableGraphConfiguration();
\r
106 configuration.roots = roots;
\r
107 configuration.model = model;
\r
112 public TransferableGraphRequest2(Collection<Pair<Resource, String>> roots) {
\r
116 public TransferableGraphRequest2(TransferableGraphConfiguration conf) {
\r
117 this.configuration = conf;
\r
122 TIntArrayList inverses = new TIntArrayList();
\r
124 int statementIndex = 0;
\r
125 TIntIntHashMap ids;
\r
126 TIntObjectMap<Variant> values;
\r
127 TIntArrayList externalParents = new TIntArrayList();
\r
128 ArrayList<String> externalNames = new ArrayList<String>();
\r
134 private SerialisationSupport support;
\r
136 private boolean validateExternal(Resource r) {
\r
137 if(configuration.disallowedExternals != null) {
\r
138 System.err.println("validateExternal agains " + configuration.disallowedExternals);
\r
139 return !configuration.disallowedExternals.contains(r);
\r
144 private Resource getResource(int r) throws DatabaseException {
\r
145 return support.getResource(r);
\r
148 public int getInternalId(int r) {
\r
152 public int getId(ReadGraph graph, int r, int predicate) throws DatabaseException {
\r
153 if(ids.containsKey(r)) {
\r
154 int ret = ids.get(r);
\r
156 for(int i=0;i<=indent;++i)
\r
157 System.out.print(" ");
\r
158 System.out.println("Cycle!!!"); // with " + GraphUtils.getReadableName(g, r));
\r
163 Collection<Resource> parents = graph.getObjects(getResource(r), L0.PartOf);
\r
164 if(parents.size() != 1) {
\r
165 throw new ValidationException("Reference to external resource "
\r
166 + NameUtils.getSafeName(graph, getResource(r), true) + " without unique uri (" + parents.size() + " parents).");
\r
168 for(Resource p : parents) {
\r
170 if(!validateExternal(p)) throw new ValidationException("References to '" + graph.getURI(p) + "' are not allowed.");
\r
171 externalParents.add(getId(graph, support.getTransientId(p), 0));
\r
174 externalNames.add((String)graph.getRelatedValue(getResource(r), L0.HasName));
\r
180 public void addId(ReadGraph graph, int r, int predicate) throws DatabaseException {
\r
181 statements[statementIndex++] = getId(graph, r, predicate);
\r
184 public void setExternals(Collection<Resource> rs) {
\r
185 configuration.externals = rs;
\r
189 public TransferableGraph1 perform(ReadGraph graph) throws DatabaseException {
\r
191 support = graph.getService(SerialisationSupport.class);
\r
193 this.L0 = Layer0.getInstance(graph);
\r
195 long total = System.nanoTime();
\r
197 long startupTime = System.nanoTime();
\r
199 ClusterControl cc = graph.getService(ClusterControl.class);
\r
201 ids = new TIntIntHashMap();
\r
202 values = new TIntObjectHashMap<Variant>();
\r
204 ArrayList<Resource> rootResources = new ArrayList<Resource>();
\r
205 for(Pair<Resource, String> p : configuration.roots) rootResources.add(p.first);
\r
207 Map<Resource, ExtentStatus> preStatus = new HashMap<Resource, ExtentStatus>();
\r
209 for(Resource root : rootResources) {
\r
210 Resource name = graph.getPossibleObject(root, L0.HasName);
\r
212 preStatus.put(name, ExtentStatus.EXCLUDED);
\r
216 for(Resource r : configuration.externals) preStatus.put(r, ExtentStatus.EXTERNAL);
\r
218 long startupTimeEnd = System.nanoTime();
\r
220 long domainTime = System.nanoTime();
\r
222 String otherStatements = "other" + UUID.randomUUID().toString();
\r
223 String valueFileName = "value" + UUID.randomUUID().toString();
\r
225 File otherStatementsFile = new File(otherStatements);
\r
226 File valueFile = new File(valueFileName);
\r
230 DeferredFileOutputStream otherStatementsStream = new DeferredFileOutputStream(1024*1024, otherStatementsFile);
\r
231 DeferredFileOutputStream valueStream = new DeferredFileOutputStream(1024*1024, valueFile);
\r
233 ObjectOutputStream otherStatementsOutput = new ObjectOutputStream(otherStatementsStream);
\r
234 ObjectOutputStream valueOutput = new ObjectOutputStream(valueStream);
\r
236 ClusterState clusterState = cc.getClusterState();
\r
238 TIntHashSet excludedShared = new TIntHashSet();
\r
240 TreeMap<String, Variant> extensions = new TreeMap<String, Variant>();
\r
242 Subgraphs.getDomain2(graph, ids, rootResources, preStatus, configuration.specials, otherStatementsOutput, valueOutput, extensions, excludedShared);
\r
246 cc.restoreClusterState(clusterState);
\r
248 // dumpHeap("domain.hprof");
\r
250 otherStatementsOutput.flush();
\r
251 valueOutput.flush();
\r
252 otherStatementsStream.close();
\r
253 valueStream.close();
\r
255 long domainDuration = System.nanoTime() - domainTime;
\r
256 System.err.println("Analysed graph in " + 1e-9*domainDuration + "s.");
\r
258 internalCount = id;
\r
260 ids.put(support.getTransientId(graph.getResource("http:/")), id++);
\r
261 externalNames.add("http:/");
\r
262 externalParents.add(-1);
\r
264 InputStream otherStatementsInputStream = null;
\r
265 InputStream valueInputStream = null;
\r
267 if(otherStatementsStream.isInMemory()) {
\r
268 otherStatementsInputStream = new ByteArrayInputStream(otherStatementsStream.getData());
\r
270 otherStatementsInputStream = new FileInputStream(otherStatementsFile);
\r
273 if(valueStream.isInMemory()) {
\r
274 valueInputStream = new ByteArrayInputStream(valueStream.getData());
\r
276 valueInputStream = new FileInputStream(valueFile);
\r
279 otherStatementsStream = null;
\r
280 valueStream = null;
\r
282 ObjectInputStream otherStatementsInput = new ObjectInputStream(otherStatementsInputStream);
\r
283 ObjectInputStream valueInput = new ObjectInputStream(valueInputStream);
\r
285 long statementTime = System.nanoTime();
\r
287 TIntArrayList statementSet = new TIntArrayList();
\r
289 while(otherStatementsInput.available() > 0) {
\r
291 int s = otherStatementsInput.readInt();
\r
293 boolean exclude = !ids.contains(s);
\r
295 int size = otherStatementsInput.readInt();
\r
296 for(int i=0;i<size;i++) {
\r
297 int p = otherStatementsInput.readInt();
\r
298 int o = otherStatementsInput.readInt();
\r
300 if(excludedShared.contains(o)) {
\r
301 // System.err.println("excluding shared " + s + " " + p + " " + o);
\r
303 statementSet.add(s);
\r
304 statementSet.add(p);
\r
305 statementSet.add(o);
\r
308 // System.err.println("excluding shared " + s);
\r
314 TIntIntHashMap inverses = new TIntIntHashMap();
\r
315 TIntHashSet predicateSet = new TIntHashSet();
\r
316 for(int i=0;i<statementSet.size();i+=3) {
\r
317 int p = statementSet.getQuick(i+1);
\r
318 if(predicateSet.add(p)) {
\r
319 Resource inverse = graph.getPossibleInverse(getResource(p));
\r
320 if(inverse != null) inverses.put(p, support.getTransientId(inverse));
\r
324 predicateSet = null;
\r
326 TIntArrayList tgStatements = new TIntArrayList();
\r
328 // dumpHeap("export.hprof");
\r
331 int trim = Math.max(65536,statementSet.size()/12);
\r
333 for(int i=statementSet.size();i>0;i-=3) {
\r
336 statementSet.remove(i, statementSet.size()-i);
\r
337 statementSet.trimToSize();
\r
338 trim = Math.max(65536,statementSet.size()/12);
\r
341 int s = statementSet.getQuick(i-3);
\r
342 int p = statementSet.getQuick(i-2);
\r
343 int o = statementSet.getQuick(i-1);
\r
345 int subjectId = ids.get(s);
\r
346 if(subjectId >= internalCount) System.err.println("Statement for external: " + s + " " + p + " " + o);
\r
348 int objectId = getId(graph, o, p);
\r
349 // The statement can be denied still
\r
350 if(objectId != -2) {
\r
351 tgStatements.add(subjectId);
\r
352 tgStatements.add(getId(graph, p, 0));
\r
353 int inverse = inverses.get(p);
\r
355 tgStatements.add(getId(graph, inverse, 0));
\r
357 tgStatements.add(-1);
\r
359 tgStatements.add(objectId);
\r
361 System.out.println("denied");
\r
366 statements = tgStatements.toArray();
\r
368 long statementDuration = System.nanoTime() - statementTime;
\r
369 System.err.println("Built transferable statements in " + 1e-9*statementDuration + "s.");
\r
373 while(valueInput.available() > 0) {
\r
375 int s = valueInput.readInt();
\r
376 // Resource subject = support.getResource(s);
\r
377 int valueSize = valueInput.readInt();
\r
378 byte[] value = new byte[valueSize];
\r
379 valueInput.readFully(value);
\r
380 Variant variant = (Variant)Bindings.VARIANT.serializer().deserialize(value);
\r
381 values.put(s, variant);
\r
385 int resourceCount = ids.size();
\r
387 Identity[] identityArray;
\r
389 ArrayList<Identity> identities = new ArrayList<Identity>();
\r
391 for(Pair<Resource, String> r : configuration.roots) {
\r
392 Resource type = graph.getPossibleType(r.first, L0.Entity);
\r
393 if(type == null) type = L0.Entity;
\r
394 identities.add(new Identity(
\r
395 ids.get(support.getTransientId(r.first)),
\r
396 new Root(r.second, graph.getURI(type))
\r
400 int internalsPlusExternals = ids.size();
\r
401 for(int i = internalCount; i < internalsPlusExternals ; i++) {
\r
402 int parent = externalParents.get(i - internalCount);
\r
403 String name = externalNames.get(i - internalCount);
\r
404 identities.add(new Identity(
\r
406 new External(parent, name)
\r
409 identityArray = identities.toArray(new Identity[identities.size()]);
\r
412 final Value[] valueArray = new Value[values.size()];
\r
414 values.forEachEntry(new TIntObjectProcedure<Variant>() {
\r
419 public boolean execute(int subject, Variant bytes) {
\r
420 //if(LOG) log("[VALUE] " + entry.getKey().getResourceId());
\r
421 int r = getInternalId(subject);
\r
422 if(r==-1) System.err.println("No id for value resource " + subject);
\r
423 else valueArray[index++] = new Value(r, bytes);
\r
432 TransferableGraph1 result =
\r
433 new TransferableGraph1(resourceCount,
\r
436 valueArray, extensions);
\r
439 System.out.println("transferable graph content: " + result);
\r
442 long totalEnd = System.nanoTime();
\r
445 System.out.println("startup in " + 1e-9*(startupTimeEnd - startupTime) + "s.");
\r
446 System.out.println("domain was found in " + 1e-9*(domainDuration) + "s.");
\r
447 System.out.println("statements were found in " + 1e-9*(statementDuration) + "s.");
\r
448 System.out.println("total time for building subgraph was " + 1e-9*(totalEnd-total) + "s.");
\r
453 } catch (IOException e) {
\r
454 e.printStackTrace();
\r
455 } catch (Throwable t) {
\r
456 t.printStackTrace();
\r
457 dumpHeap("crash.hprof");
\r
464 private static void dumpHeap(String path) {
\r
467 Object bean = getBean();
\r
471 Method m = bean.getClass().getMethod("dumpHeap", String.class, boolean.class);
\r
472 m.invoke(bean, path, true);
\r
474 } catch (IllegalArgumentException e) {
\r
475 } catch (IllegalAccessException e) {
\r
476 } catch (SecurityException e) {
\r
477 } catch (NoSuchMethodException e) {
\r
478 } catch (InvocationTargetException e) {
\r
484 private static Object getBean() {
\r
485 Class<?> beanClass = getBeanClass();
\r
486 if (beanClass == null)
\r
489 Object bean = ManagementFactory.newPlatformMXBeanProxy(
\r
490 ManagementFactory.getPlatformMBeanServer(),
\r
491 "com.sun.management:type=HotSpotDiagnostic",
\r
494 } catch (IOException e) {
\r
499 private static Class<?> getBeanClass() {
\r
501 Class<?> clazz = Class.forName("com.sun.management.HotSpotDiagnosticMXBean");
\r
503 } catch (ClassNotFoundException e) {
\r