bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TransferableGraphRequest2.java

   1 /*******************************************************************************
   2  * Copyright (c) 2007, 2010 Association for Decentralized Information Management
   3  * in Industry THTH ry.
   4  * All rights reserved. This program and the accompanying materials
   5  * are made available under the terms of the Eclipse Public License v1.0
   6  * which accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *     VTT Technical Research Centre of Finland - initial API and implementation
  11  *******************************************************************************/
  12 package org.simantics.db.layer0.util;
  13
  14 import gnu.trove.list.array.TIntArrayList;
  15 import gnu.trove.map.TIntObjectMap;
  16 import gnu.trove.map.hash.TIntIntHashMap;
  17 import gnu.trove.map.hash.TIntObjectHashMap;
  18 import gnu.trove.procedure.TIntObjectProcedure;
  19 import gnu.trove.set.hash.TIntHashSet;
  20
  21 import java.io.ByteArrayInputStream;
  22 import java.io.DataOutput;
  23 import java.io.DataOutputStream;
  24 import java.io.File;
  25 import java.io.FileInputStream;
  26 import java.io.FileNotFoundException;
  27 import java.io.FileOutputStream;
  28 import java.io.IOException;
  29 import java.io.InputStream;
  30 import java.io.ObjectInputStream;
  31 import java.io.ObjectOutputStream;
  32 import java.lang.management.ManagementFactory;
  33 import java.lang.reflect.InvocationTargetException;
  34 import java.lang.reflect.Method;
  35 import java.util.ArrayList;
  36 import java.util.Collection;
  37 import java.util.HashMap;
  38 import java.util.Map;
  39 import java.util.TreeMap;
  40 import java.util.UUID;
  41
  42 import org.apache.commons.io.output.DeferredFileOutputStream;
  43 import org.simantics.databoard.Bindings;
  44 import org.simantics.databoard.binding.mutable.Variant;
  45 import org.simantics.db.ReadGraph;
  46 import org.simantics.db.Resource;
  47 import org.simantics.db.common.utils.NameUtils;
  48 import org.simantics.db.exception.DatabaseException;
  49 import org.simantics.db.exception.ValidationException;
  50 import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus;
  51 import org.simantics.db.request.Read;
  52 import org.simantics.db.service.ClusterControl;
  53 import org.simantics.db.service.ClusterControl.ClusterState;
  54 import org.simantics.db.service.SerialisationSupport;
  55 import org.simantics.graph.representation.External;
  56 import org.simantics.graph.representation.Identity;
  57 import org.simantics.graph.representation.Root;
  58 import org.simantics.graph.representation.TransferableGraph1;
  59 import org.simantics.graph.representation.Value;
  60 import org.simantics.layer0.Layer0;
  61 import org.simantics.utils.datastructures.Pair;
  62
  63 /**
  64  * @deprecated in favor of {@link ModelTransferableGraphSourceRequest}
  65  */
  66 @Deprecated
  67 public class TransferableGraphRequest2 implements Read<TransferableGraph1> {
  68
  69         public static String LOG_FILE = "transferableGraph.log";
  70         final static private boolean LOG = false;
  71         final static private boolean DEBUG = false;
  72         final static private boolean PROFILE = false;
  73
  74         private TransferableGraphConfiguration configuration;
  75
  76         static DataOutput log;
  77
  78         static {
  79
  80                 if (LOG) {
  81                         try {
  82                                 FileOutputStream stream = new FileOutputStream(LOG_FILE);
  83                                 log = new DataOutputStream(stream);
  84                         } catch (FileNotFoundException e) {
  85                                 e.printStackTrace();
  86                         }
  87                 }
  88
  89         }
  90
  91         private static void log(String line) {
  92                 if (LOG) {
  93                         try {
  94                                 log.writeUTF(line + "\n");
  95                         } catch (IOException e) {
  96                                 e.printStackTrace();
  97                         }
  98                 }
  99         }
 100
 101
 102
 103         public TransferableGraphRequest2(Collection<Pair<Resource, String>> roots, Resource model) {
 104
 105                 configuration = new TransferableGraphConfiguration();
 106                 configuration.roots = roots;
 107                 configuration.model = model;
 108
 109         }
 110
 111
 112         public TransferableGraphRequest2(Collection<Pair<Resource, String>> roots) {
 113                 this(roots, null);
 114         }
 115
 116         public TransferableGraphRequest2(TransferableGraphConfiguration conf) {
 117                 this.configuration = conf;
 118         }
 119
 120         Layer0 L0;
 121
 122         TIntArrayList inverses = new TIntArrayList();
 123         int statements[];
 124         int statementIndex = 0;
 125         TIntIntHashMap ids;
 126         TIntObjectMap<Variant> values;
 127         TIntArrayList externalParents = new TIntArrayList();
 128         ArrayList<String> externalNames = new ArrayList<String>();
 129         int id = 0;
 130         int internalCount;
 131
 132         int indent = 0;
 133
 134         private SerialisationSupport support;
 135
 136         private boolean validateExternal(Resource r) {
 137                 if(configuration.disallowedExternals != null) {
 138                         System.err.println("validateExternal agains " + configuration.disallowedExternals);
 139                         return !configuration.disallowedExternals.contains(r);
 140                 }
 141                 return true;
 142         }
 143
 144         private Resource getResource(int r) throws DatabaseException {
 145                 return support.getResource(r);
 146         }
 147
 148         public int getInternalId(int r) {
 149                 return ids.get(r);
 150         }
 151
 152         public int getId(ReadGraph graph, int r, int predicate) throws DatabaseException {
 153                 if(ids.containsKey(r)) {
 154                     int ret = ids.get(r);
 155                     if(ret == -1) {
 156                         for(int i=0;i<=indent;++i)
 157                             System.out.print("  ");
 158                         System.out.println("Cycle!!!"); // with " + GraphUtils.getReadableName(g, r));
 159                     }
 160                         return ret;
 161                 }
 162                 else {
 163                         Collection<Resource> parents = graph.getObjects(getResource(r), L0.PartOf);
 164                         if(parents.size() != 1) {
 165                                 throw new ValidationException("Reference to external resource "
 166                                                 + NameUtils.getSafeName(graph, getResource(r), true) + " without unique uri (" + parents.size() + " parents).");
 167                         }
 168                         for(Resource p : parents) {
 169                             ++indent;
 170                     if(!validateExternal(p)) throw new ValidationException("References to '" + graph.getURI(p) + "' are not allowed.");
 171                                 externalParents.add(getId(graph, support.getTransientId(p), 0));
 172                                 --indent;
 173                         }
 174             externalNames.add((String)graph.getRelatedValue(getResource(r), L0.HasName));
 175                         ids.put(r, id);
 176                         return id++;
 177                 }
 178         }
 179
 180         public void addId(ReadGraph graph, int r, int predicate) throws DatabaseException {
 181                 statements[statementIndex++] = getId(graph, r, predicate);
 182         }
 183
 184         public void setExternals(Collection<Resource> rs) {
 185                 configuration.externals = rs;
 186         }
 187
 188         @Override
 189         public TransferableGraph1 perform(ReadGraph graph) throws DatabaseException {
 190
 191                 support = graph.getService(SerialisationSupport.class);
 192
 193                 this.L0 = Layer0.getInstance(graph);
 194
 195                 long total = System.nanoTime();
 196
 197         long startupTime = System.nanoTime();
 198
 199         ClusterControl cc = graph.getService(ClusterControl.class);
 200
 201         ids = new TIntIntHashMap();
 202         values = new TIntObjectHashMap<Variant>();
 203
 204                 ArrayList<Resource> rootResources = new ArrayList<Resource>();
 205                 for(Pair<Resource, String> p : configuration.roots) rootResources.add(p.first);
 206
 207                 Map<Resource, ExtentStatus> preStatus = new HashMap<Resource, ExtentStatus>();
 208
 209                 for(Resource root : rootResources) {
 210                         Resource name = graph.getPossibleObject(root, L0.HasName);
 211                         if(name != null) {
 212                                 preStatus.put(name, ExtentStatus.EXCLUDED);
 213                         }
 214                 }
 215
 216                 for(Resource r : configuration.externals) preStatus.put(r, ExtentStatus.EXTERNAL);
 217
 218         long startupTimeEnd = System.nanoTime();
 219
 220                 long domainTime = System.nanoTime();
 221
 222                 String otherStatements = "other" + UUID.randomUUID().toString();
 223                 String valueFileName = "value" + UUID.randomUUID().toString();
 224
 225         File otherStatementsFile = new File(otherStatements);
 226         File valueFile = new File(valueFileName);
 227
 228         try {
 229
 230                 DeferredFileOutputStream otherStatementsStream = new DeferredFileOutputStream(1024*1024, otherStatementsFile);
 231                 DeferredFileOutputStream valueStream = new DeferredFileOutputStream(1024*1024, valueFile);
 232
 233                 ObjectOutputStream otherStatementsOutput = new ObjectOutputStream(otherStatementsStream);
 234                 ObjectOutputStream valueOutput = new ObjectOutputStream(valueStream);
 235
 236                 ClusterState clusterState = cc.getClusterState();
 237
 238                 TIntHashSet excludedShared = new TIntHashSet();
 239
 240                 TreeMap<String, Variant> extensions = new TreeMap<String, Variant>();
 241
 242                 Subgraphs.getDomain2(graph, ids, rootResources, preStatus, configuration.specials, otherStatementsOutput, valueOutput, extensions, excludedShared);
 243
 244                 id = ids.size();
 245
 246                 cc.restoreClusterState(clusterState);
 247
 248 //              dumpHeap("domain.hprof");
 249
 250                 otherStatementsOutput.flush();
 251                 valueOutput.flush();
 252                 otherStatementsStream.close();
 253                 valueStream.close();
 254
 255                 long domainDuration = System.nanoTime() - domainTime;
 256                 System.err.println("Analysed graph in " + 1e-9*domainDuration + "s.");
 257
 258                 internalCount = id;
 259
 260                 ids.put(support.getTransientId(graph.getResource("http:/")), id++);
 261                 externalNames.add("http:/");
 262                 externalParents.add(-1);
 263
 264                 InputStream otherStatementsInputStream = null;
 265                 InputStream valueInputStream = null;
 266
 267                 if(otherStatementsStream.isInMemory()) {
 268                         otherStatementsInputStream = new ByteArrayInputStream(otherStatementsStream.getData());
 269                 } else {
 270                         otherStatementsInputStream = new FileInputStream(otherStatementsFile);
 271                 }
 272
 273                 if(valueStream.isInMemory()) {
 274                         valueInputStream = new ByteArrayInputStream(valueStream.getData());
 275                 } else {
 276                         valueInputStream = new FileInputStream(valueFile);
 277                 }
 278
 279                 otherStatementsStream = null;
 280                 valueStream = null;
 281
 282                 ObjectInputStream otherStatementsInput = new ObjectInputStream(otherStatementsInputStream);
 283                 ObjectInputStream valueInput = new ObjectInputStream(valueInputStream);
 284
 285                 long statementTime = System.nanoTime();
 286
 287                 TIntArrayList statementSet = new TIntArrayList();
 288
 289                 while(otherStatementsInput.available() > 0) {
 290
 291                         int s = otherStatementsInput.readInt();
 292
 293                         boolean exclude = !ids.contains(s);
 294
 295                         int size = otherStatementsInput.readInt();
 296                         for(int i=0;i<size;i++) {
 297                                 int p = otherStatementsInput.readInt();
 298                                 int o = otherStatementsInput.readInt();
 299                                 if(!exclude) {
 300                                         if(excludedShared.contains(o)) {
 301 //                                              System.err.println("excluding shared " + s + " " + p + " " + o);
 302                                         } else {
 303                                                 statementSet.add(s);
 304                                                 statementSet.add(p);
 305                                                 statementSet.add(o);
 306                                         }
 307                                 } else {
 308 //                                              System.err.println("excluding shared " + s);
 309                                 }
 310                         }
 311
 312                 }
 313
 314                 TIntIntHashMap inverses = new TIntIntHashMap();
 315                 TIntHashSet predicateSet = new TIntHashSet();
 316                 for(int i=0;i<statementSet.size();i+=3) {
 317                         int p = statementSet.getQuick(i+1);
 318                         if(predicateSet.add(p)) {
 319                                         Resource inverse = graph.getPossibleInverse(getResource(p));
 320                                         if(inverse != null) inverses.put(p, support.getTransientId(inverse));
 321                         }
 322                 }
 323
 324                 predicateSet = null;
 325
 326                 TIntArrayList tgStatements = new TIntArrayList();
 327
 328 //              dumpHeap("export.hprof");
 329
 330                 // 25%
 331                 int trim = Math.max(65536,statementSet.size()/12);
 332
 333                 for(int i=statementSet.size();i>0;i-=3) {
 334
 335                         if(trim-- == 0) {
 336                                 statementSet.remove(i, statementSet.size()-i);
 337                                 statementSet.trimToSize();
 338                                 trim = Math.max(65536,statementSet.size()/12);
 339                         }
 340
 341                         int s = statementSet.getQuick(i-3);
 342                         int p = statementSet.getQuick(i-2);
 343                         int o = statementSet.getQuick(i-1);
 344
 345                         int subjectId = ids.get(s);
 346                         if(subjectId >= internalCount) System.err.println("Statement for external: " + s + " " + p + " " + o);
 347
 348                         int objectId = getId(graph, o, p);
 349                         // The statement can be denied still
 350                         if(objectId != -2) {
 351                                 tgStatements.add(subjectId);
 352                                 tgStatements.add(getId(graph, p, 0));
 353                                 int inverse = inverses.get(p);
 354                                 if(inverse != 0) {
 355                                         tgStatements.add(getId(graph, inverse, 0));
 356                                 } else {
 357                                         tgStatements.add(-1);
 358                                 }
 359                                 tgStatements.add(objectId);
 360                         } else {
 361                                 System.out.println("denied");
 362                         }
 363
 364                 }
 365
 366                 statements = tgStatements.toArray();
 367
 368                 long statementDuration = System.nanoTime() - statementTime;
 369                 System.err.println("Built transferable statements in " + 1e-9*statementDuration + "s.");
 370
 371                 inverses = null;
 372
 373                 while(valueInput.available() > 0) {
 374
 375                         int s = valueInput.readInt();
 376 //                      Resource subject = support.getResource(s);
 377                         int valueSize = valueInput.readInt();
 378                         byte[] value = new byte[valueSize];
 379                         valueInput.readFully(value);
 380                         Variant variant = (Variant)Bindings.VARIANT.serializer().deserialize(value);
 381                         values.put(s, variant);
 382
 383                 }
 384
 385                 int resourceCount = ids.size();
 386
 387                 Identity[] identityArray;
 388                 { // Identities
 389                         ArrayList<Identity> identities = new ArrayList<Identity>();
 390
 391                         for(Pair<Resource, String> r : configuration.roots) {
 392                                 Resource type = graph.getPossibleType(r.first, L0.Entity);
 393                                 if(type == null) type = L0.Entity;
 394                                 identities.add(new Identity(
 395                                                 ids.get(support.getTransientId(r.first)),
 396                                                 new Root(r.second, graph.getURI(type))
 397                                 ));
 398                         }
 399
 400                         int internalsPlusExternals = ids.size();
 401                         for(int i = internalCount; i < internalsPlusExternals ; i++) {
 402                                 int parent = externalParents.get(i - internalCount);
 403                                 String name = externalNames.get(i - internalCount);
 404                                 identities.add(new Identity(
 405                                                 i,
 406                                                 new External(parent, name)
 407                                 ));
 408                         }
 409                         identityArray = identities.toArray(new Identity[identities.size()]);
 410                 }
 411
 412             final Value[] valueArray = new Value[values.size()];
 413                 { // Values
 414                         values.forEachEntry(new TIntObjectProcedure<Variant>() {
 415
 416                         int index = 0;
 417
 418                                         @Override
 419                                         public boolean execute(int subject, Variant bytes) {
 420                                         //if(LOG) log("[VALUE] " + entry.getKey().getResourceId());
 421                                         int r = getInternalId(subject);
 422                                         if(r==-1) System.err.println("No id for value resource " + subject);
 423                                         else valueArray[index++] = new Value(r, bytes);
 424                                         return true;
 425                                         }
 426
 427                                 });
 428                 }
 429             ids = null;
 430                 values = null;
 431
 432             TransferableGraph1 result =
 433                 new TransferableGraph1(resourceCount,
 434                         identityArray,
 435                         statements,
 436                         valueArray, extensions);
 437
 438                 if(DEBUG) {
 439                         System.out.println("transferable graph content: " + result);
 440                 }
 441
 442                 long totalEnd = System.nanoTime();
 443
 444                 if(PROFILE) {
 445                         System.out.println("startup in " + 1e-9*(startupTimeEnd - startupTime) + "s.");
 446                         System.out.println("domain was found in " + 1e-9*(domainDuration) + "s.");
 447                         System.out.println("statements were found in " + 1e-9*(statementDuration) + "s.");
 448                         System.out.println("total time for building subgraph was " + 1e-9*(totalEnd-total) + "s.");
 449                 }
 450
 451                 return result;
 452
 453         } catch (IOException e) {
 454                 e.printStackTrace();
 455         } catch (Throwable t) {
 456                 t.printStackTrace();
 457                 dumpHeap("crash.hprof");
 458         }
 459
 460         return null;
 461
 462         }
 463
 464         private static void dumpHeap(String path) {
 465
 466         try {
 467             Object bean = getBean();
 468             if (bean == null)
 469                 return;
 470
 471             Method m = bean.getClass().getMethod("dumpHeap", String.class, boolean.class);
 472             m.invoke(bean, path, true);
 473
 474         } catch (IllegalArgumentException e) {
 475         } catch (IllegalAccessException e) {
 476         } catch (SecurityException e) {
 477         } catch (NoSuchMethodException e) {
 478         } catch (InvocationTargetException e) {
 479                 } finally {
 480         }
 481
 482         }
 483
 484     private static Object getBean() {
 485         Class<?> beanClass = getBeanClass();
 486         if (beanClass == null)
 487             return null;
 488         try {
 489             Object bean = ManagementFactory.newPlatformMXBeanProxy(
 490                     ManagementFactory.getPlatformMBeanServer(),
 491                     "com.sun.management:type=HotSpotDiagnostic",
 492                     beanClass);
 493             return bean;
 494         } catch (IOException e) {
 495             return null;
 496         }
 497     }
 498
 499     private static Class<?> getBeanClass() {
 500         try {
 501             Class<?> clazz = Class.forName("com.sun.management.HotSpotDiagnosticMXBean");
 502             return clazz;
 503         } catch (ClassNotFoundException e) {
 504             return null;
 505         }
 506     }
 507
 508
 509 }