1 /*******************************************************************************
\r
2 * Copyright (c) 2007, 2010 Association for Decentralized Information Management
\r
3 * in Industry THTH ry.
\r
4 * All rights reserved. This program and the accompanying materials
\r
5 * are made available under the terms of the Eclipse Public License v1.0
\r
6 * which accompanies this distribution, and is available at
\r
7 * http://www.eclipse.org/legal/epl-v10.html
\r
10 * VTT Technical Research Centre of Finland - initial API and implementation
\r
11 *******************************************************************************/
\r
12 package org.simantics.db.layer0.util;
\r
14 import gnu.trove.list.array.TIntArrayList;
\r
15 import gnu.trove.map.TIntObjectMap;
\r
16 import gnu.trove.map.hash.TIntIntHashMap;
\r
17 import gnu.trove.map.hash.TIntObjectHashMap;
\r
18 import gnu.trove.procedure.TIntObjectProcedure;
\r
19 import gnu.trove.set.hash.TIntHashSet;
\r
21 import java.io.ByteArrayInputStream;
\r
22 import java.io.DataOutput;
\r
23 import java.io.DataOutputStream;
\r
24 import java.io.File;
\r
25 import java.io.FileInputStream;
\r
26 import java.io.FileNotFoundException;
\r
27 import java.io.FileOutputStream;
\r
28 import java.io.IOException;
\r
29 import java.io.InputStream;
\r
30 import java.io.ObjectInputStream;
\r
31 import java.io.ObjectOutputStream;
\r
32 import java.lang.management.ManagementFactory;
\r
33 import java.lang.reflect.InvocationTargetException;
\r
34 import java.lang.reflect.Method;
\r
35 import java.util.ArrayList;
\r
36 import java.util.Collection;
\r
37 import java.util.HashMap;
\r
38 import java.util.Map;
\r
39 import java.util.TreeMap;
\r
40 import java.util.UUID;
\r
42 import org.apache.commons.io.output.DeferredFileOutputStream;
\r
43 import org.simantics.databoard.Bindings;
\r
44 import org.simantics.databoard.binding.mutable.Variant;
\r
45 import org.simantics.db.ReadGraph;
\r
46 import org.simantics.db.Resource;
\r
47 import org.simantics.db.common.utils.NameUtils;
\r
48 import org.simantics.db.exception.DatabaseException;
\r
49 import org.simantics.db.exception.ValidationException;
\r
50 import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus;
\r
51 import org.simantics.db.request.Read;
\r
52 import org.simantics.db.service.ClusterControl;
\r
53 import org.simantics.db.service.ClusterControl.ClusterState;
\r
54 import org.simantics.db.service.SerialisationSupport;
\r
55 import org.simantics.graph.representation.External;
\r
56 import org.simantics.graph.representation.Identity;
\r
57 import org.simantics.graph.representation.Root;
\r
58 import org.simantics.graph.representation.TransferableGraph1;
\r
59 import org.simantics.graph.representation.Value;
\r
60 import org.simantics.layer0.Layer0;
\r
61 import org.simantics.utils.datastructures.Pair;
\r
63 public class TransferableGraphRequest2 implements Read<TransferableGraph1> {
\r
65 public static String LOG_FILE = "transferableGraph.log";
\r
66 final static private boolean LOG = false;
\r
67 final static private boolean DEBUG = false;
\r
68 final static private boolean PROFILE = false;
\r
70 private TransferableGraphConfiguration configuration;
\r
72 static DataOutput log;
\r
78 FileOutputStream stream = new FileOutputStream(LOG_FILE);
\r
79 log = new DataOutputStream(stream);
\r
80 } catch (FileNotFoundException e) {
\r
81 e.printStackTrace();
\r
87 private static void log(String line) {
\r
90 log.writeUTF(line + "\n");
\r
91 } catch (IOException e) {
\r
92 e.printStackTrace();
\r
99 public TransferableGraphRequest2(Collection<Pair<Resource, String>> roots, Resource model) {
\r
101 configuration = new TransferableGraphConfiguration();
\r
102 configuration.roots = roots;
\r
103 configuration.model = model;
\r
108 public TransferableGraphRequest2(Collection<Pair<Resource, String>> roots) {
\r
112 public TransferableGraphRequest2(TransferableGraphConfiguration conf) {
\r
113 this.configuration = conf;
\r
118 TIntArrayList inverses = new TIntArrayList();
\r
120 int statementIndex = 0;
\r
121 TIntIntHashMap ids;
\r
122 TIntObjectMap<Variant> values;
\r
123 TIntArrayList externalParents = new TIntArrayList();
\r
124 ArrayList<String> externalNames = new ArrayList<String>();
\r
130 private SerialisationSupport support;
\r
132 private boolean validateExternal(Resource r) {
\r
133 if(configuration.disallowedExternals != null) {
\r
134 System.err.println("validateExternal agains " + configuration.disallowedExternals);
\r
135 return !configuration.disallowedExternals.contains(r);
\r
140 private Resource getResource(int r) throws DatabaseException {
\r
141 return support.getResource(r);
\r
144 public int getInternalId(int r) {
\r
148 public int getId(ReadGraph graph, int r, int predicate) throws DatabaseException {
\r
149 if(ids.containsKey(r)) {
\r
150 int ret = ids.get(r);
\r
152 for(int i=0;i<=indent;++i)
\r
153 System.out.print(" ");
\r
154 System.out.println("Cycle!!!"); // with " + GraphUtils.getReadableName(g, r));
\r
159 Collection<Resource> parents = graph.getObjects(getResource(r), L0.PartOf);
\r
160 if(parents.size() != 1) {
\r
161 throw new ValidationException("Reference to external resource "
\r
162 + NameUtils.getSafeName(graph, getResource(r), true) + " without unique uri (" + parents.size() + " parents).");
\r
164 for(Resource p : parents) {
\r
166 if(!validateExternal(p)) throw new ValidationException("References to '" + graph.getURI(p) + "' are not allowed.");
\r
167 externalParents.add(getId(graph, support.getTransientId(p), 0));
\r
170 externalNames.add((String)graph.getRelatedValue(getResource(r), L0.HasName));
\r
176 public void addId(ReadGraph graph, int r, int predicate) throws DatabaseException {
\r
177 statements[statementIndex++] = getId(graph, r, predicate);
\r
180 public void setExternals(Collection<Resource> rs) {
\r
181 configuration.externals = rs;
\r
185 public TransferableGraph1 perform(ReadGraph graph) throws DatabaseException {
\r
187 support = graph.getService(SerialisationSupport.class);
\r
189 this.L0 = Layer0.getInstance(graph);
\r
191 long total = System.nanoTime();
\r
193 long startupTime = System.nanoTime();
\r
195 ClusterControl cc = graph.getService(ClusterControl.class);
\r
197 ids = new TIntIntHashMap();
\r
198 values = new TIntObjectHashMap<Variant>();
\r
200 ArrayList<Resource> rootResources = new ArrayList<Resource>();
\r
201 for(Pair<Resource, String> p : configuration.roots) rootResources.add(p.first);
\r
203 Map<Resource, ExtentStatus> preStatus = new HashMap<Resource, ExtentStatus>();
\r
205 for(Resource root : rootResources) {
\r
206 Resource name = graph.getPossibleObject(root, L0.HasName);
\r
208 preStatus.put(name, ExtentStatus.EXCLUDED);
\r
212 for(Resource r : configuration.externals) preStatus.put(r, ExtentStatus.EXTERNAL);
\r
214 long startupTimeEnd = System.nanoTime();
\r
216 long domainTime = System.nanoTime();
\r
218 String otherStatements = "other" + UUID.randomUUID().toString();
\r
219 String valueFileName = "value" + UUID.randomUUID().toString();
\r
221 File otherStatementsFile = new File(otherStatements);
\r
222 File valueFile = new File(valueFileName);
\r
226 DeferredFileOutputStream otherStatementsStream = new DeferredFileOutputStream(1024*1024, otherStatementsFile);
\r
227 DeferredFileOutputStream valueStream = new DeferredFileOutputStream(1024*1024, valueFile);
\r
229 ObjectOutputStream otherStatementsOutput = new ObjectOutputStream(otherStatementsStream);
\r
230 ObjectOutputStream valueOutput = new ObjectOutputStream(valueStream);
\r
232 ClusterState clusterState = cc.getClusterState();
\r
234 TIntHashSet excludedShared = new TIntHashSet();
\r
236 TreeMap<String, Variant> extensions = new TreeMap<String, Variant>();
\r
238 Subgraphs.getDomain2(graph, ids, rootResources, preStatus, configuration.specials, otherStatementsOutput, valueOutput, extensions, excludedShared);
\r
242 cc.restoreClusterState(clusterState);
\r
244 // dumpHeap("domain.hprof");
\r
246 otherStatementsOutput.flush();
\r
247 valueOutput.flush();
\r
248 otherStatementsStream.close();
\r
249 valueStream.close();
\r
251 long domainDuration = System.nanoTime() - domainTime;
\r
252 System.err.println("Analysed graph in " + 1e-9*domainDuration + "s.");
\r
254 internalCount = id;
\r
256 ids.put(support.getTransientId(graph.getResource("http:/")), id++);
\r
257 externalNames.add("http:/");
\r
258 externalParents.add(-1);
\r
260 InputStream otherStatementsInputStream = null;
\r
261 InputStream valueInputStream = null;
\r
263 if(otherStatementsStream.isInMemory()) {
\r
264 otherStatementsInputStream = new ByteArrayInputStream(otherStatementsStream.getData());
\r
266 otherStatementsInputStream = new FileInputStream(otherStatementsFile);
\r
269 if(valueStream.isInMemory()) {
\r
270 valueInputStream = new ByteArrayInputStream(valueStream.getData());
\r
272 valueInputStream = new FileInputStream(valueFile);
\r
275 otherStatementsStream = null;
\r
276 valueStream = null;
\r
278 ObjectInputStream otherStatementsInput = new ObjectInputStream(otherStatementsInputStream);
\r
279 ObjectInputStream valueInput = new ObjectInputStream(valueInputStream);
\r
281 long statementTime = System.nanoTime();
\r
283 TIntArrayList statementSet = new TIntArrayList();
\r
285 while(otherStatementsInput.available() > 0) {
\r
287 int s = otherStatementsInput.readInt();
\r
289 boolean exclude = !ids.contains(s);
\r
291 int size = otherStatementsInput.readInt();
\r
292 for(int i=0;i<size;i++) {
\r
293 int p = otherStatementsInput.readInt();
\r
294 int o = otherStatementsInput.readInt();
\r
296 if(excludedShared.contains(o)) {
\r
297 // System.err.println("excluding shared " + s + " " + p + " " + o);
\r
299 statementSet.add(s);
\r
300 statementSet.add(p);
\r
301 statementSet.add(o);
\r
304 // System.err.println("excluding shared " + s);
\r
310 TIntIntHashMap inverses = new TIntIntHashMap();
\r
311 TIntHashSet predicateSet = new TIntHashSet();
\r
312 for(int i=0;i<statementSet.size();i+=3) {
\r
313 int p = statementSet.getQuick(i+1);
\r
314 if(predicateSet.add(p)) {
\r
315 Resource inverse = graph.getPossibleInverse(getResource(p));
\r
316 if(inverse != null) inverses.put(p, support.getTransientId(inverse));
\r
320 predicateSet = null;
\r
322 TIntArrayList tgStatements = new TIntArrayList();
\r
324 // dumpHeap("export.hprof");
\r
327 int trim = Math.max(65536,statementSet.size()/12);
\r
329 for(int i=statementSet.size();i>0;i-=3) {
\r
332 statementSet.remove(i, statementSet.size()-i);
\r
333 statementSet.trimToSize();
\r
334 trim = Math.max(65536,statementSet.size()/12);
\r
337 int s = statementSet.getQuick(i-3);
\r
338 int p = statementSet.getQuick(i-2);
\r
339 int o = statementSet.getQuick(i-1);
\r
341 int subjectId = ids.get(s);
\r
342 if(subjectId >= internalCount) System.err.println("Statement for external: " + s + " " + p + " " + o);
\r
344 int objectId = getId(graph, o, p);
\r
345 // The statement can be denied still
\r
346 if(objectId != -2) {
\r
347 tgStatements.add(subjectId);
\r
348 tgStatements.add(getId(graph, p, 0));
\r
349 int inverse = inverses.get(p);
\r
351 tgStatements.add(getId(graph, inverse, 0));
\r
353 tgStatements.add(-1);
\r
355 tgStatements.add(objectId);
\r
357 System.out.println("denied");
\r
362 statements = tgStatements.toArray();
\r
364 long statementDuration = System.nanoTime() - statementTime;
\r
365 System.err.println("Built transferable statements in " + 1e-9*statementDuration + "s.");
\r
369 while(valueInput.available() > 0) {
\r
371 int s = valueInput.readInt();
\r
372 // Resource subject = support.getResource(s);
\r
373 int valueSize = valueInput.readInt();
\r
374 byte[] value = new byte[valueSize];
\r
375 valueInput.readFully(value);
\r
376 Variant variant = (Variant)Bindings.VARIANT.serializer().deserialize(value);
\r
377 values.put(s, variant);
\r
381 int resourceCount = ids.size();
\r
383 Identity[] identityArray;
\r
385 ArrayList<Identity> identities = new ArrayList<Identity>();
\r
387 for(Pair<Resource, String> r : configuration.roots) {
\r
388 Resource type = graph.getPossibleType(r.first, L0.Entity);
\r
389 if(type == null) type = L0.Entity;
\r
390 identities.add(new Identity(
\r
391 ids.get(support.getTransientId(r.first)),
\r
392 new Root(r.second, graph.getURI(type))
\r
396 int internalsPlusExternals = ids.size();
\r
397 for(int i = internalCount; i < internalsPlusExternals ; i++) {
\r
398 int parent = externalParents.get(i - internalCount);
\r
399 String name = externalNames.get(i - internalCount);
\r
400 identities.add(new Identity(
\r
402 new External(parent, name)
\r
405 identityArray = identities.toArray(new Identity[identities.size()]);
\r
408 final Value[] valueArray = new Value[values.size()];
\r
410 values.forEachEntry(new TIntObjectProcedure<Variant>() {
\r
415 public boolean execute(int subject, Variant bytes) {
\r
416 //if(LOG) log("[VALUE] " + entry.getKey().getResourceId());
\r
417 int r = getInternalId(subject);
\r
418 if(r==-1) System.err.println("No id for value resource " + subject);
\r
419 else valueArray[index++] = new Value(r, bytes);
\r
428 TransferableGraph1 result =
\r
429 new TransferableGraph1(resourceCount,
\r
432 valueArray, extensions);
\r
435 System.out.println("transferable graph content: " + result);
\r
438 long totalEnd = System.nanoTime();
\r
441 System.out.println("startup in " + 1e-9*(startupTimeEnd - startupTime) + "s.");
\r
442 System.out.println("domain was found in " + 1e-9*(domainDuration) + "s.");
\r
443 System.out.println("statements were found in " + 1e-9*(statementDuration) + "s.");
\r
444 System.out.println("total time for building subgraph was " + 1e-9*(totalEnd-total) + "s.");
\r
449 } catch (IOException e) {
\r
450 e.printStackTrace();
\r
451 } catch (Throwable t) {
\r
452 t.printStackTrace();
\r
453 dumpHeap("crash.hprof");
\r
460 private static void dumpHeap(String path) {
\r
463 Object bean = getBean();
\r
467 Method m = bean.getClass().getMethod("dumpHeap", String.class, boolean.class);
\r
468 m.invoke(bean, path, true);
\r
470 } catch (IllegalArgumentException e) {
\r
471 } catch (IllegalAccessException e) {
\r
472 } catch (SecurityException e) {
\r
473 } catch (NoSuchMethodException e) {
\r
474 } catch (InvocationTargetException e) {
\r
480 private static Object getBean() {
\r
481 Class<?> beanClass = getBeanClass();
\r
482 if (beanClass == null)
\r
485 Object bean = ManagementFactory.newPlatformMXBeanProxy(
\r
486 ManagementFactory.getPlatformMBeanServer(),
\r
487 "com.sun.management:type=HotSpotDiagnostic",
\r
490 } catch (IOException e) {
\r
495 private static Class<?> getBeanClass() {
\r
497 Class<?> clazz = Class.forName("com.sun.management.HotSpotDiagnosticMXBean");
\r
499 } catch (ClassNotFoundException e) {
\r