bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TransferableGraphRequest2.java

   1 /*******************************************************************************\r
   2  * Copyright (c) 2007, 2010 Association for Decentralized Information Management\r
   3  * in Industry THTH ry.\r
   4  * All rights reserved. This program and the accompanying materials\r
   5  * are made available under the terms of the Eclipse Public License v1.0\r
   6  * which accompanies this distribution, and is available at\r
   7  * http://www.eclipse.org/legal/epl-v10.html\r
   8  *\r
   9  * Contributors:\r
  10  *     VTT Technical Research Centre of Finland - initial API and implementation\r
  11  *******************************************************************************/\r
  12 package org.simantics.db.layer0.util;\r
  13 \r
  14 import gnu.trove.list.array.TIntArrayList;\r
  15 import gnu.trove.map.TIntObjectMap;\r
  16 import gnu.trove.map.hash.TIntIntHashMap;\r
  17 import gnu.trove.map.hash.TIntObjectHashMap;\r
  18 import gnu.trove.procedure.TIntObjectProcedure;\r
  19 import gnu.trove.set.hash.TIntHashSet;\r
  20 \r
  21 import java.io.ByteArrayInputStream;\r
  22 import java.io.DataOutput;\r
  23 import java.io.DataOutputStream;\r
  24 import java.io.File;\r
  25 import java.io.FileInputStream;\r
  26 import java.io.FileNotFoundException;\r
  27 import java.io.FileOutputStream;\r
  28 import java.io.IOException;\r
  29 import java.io.InputStream;\r
  30 import java.io.ObjectInputStream;\r
  31 import java.io.ObjectOutputStream;\r
  32 import java.lang.management.ManagementFactory;\r
  33 import java.lang.reflect.InvocationTargetException;\r
  34 import java.lang.reflect.Method;\r
  35 import java.util.ArrayList;\r
  36 import java.util.Collection;\r
  37 import java.util.HashMap;\r
  38 import java.util.Map;\r
  39 import java.util.TreeMap;\r
  40 import java.util.UUID;\r
  41 \r
  42 import org.apache.commons.io.output.DeferredFileOutputStream;\r
  43 import org.simantics.databoard.Bindings;\r
  44 import org.simantics.databoard.binding.mutable.Variant;\r
  45 import org.simantics.db.ReadGraph;\r
  46 import org.simantics.db.Resource;\r
  47 import org.simantics.db.common.utils.NameUtils;\r
  48 import org.simantics.db.exception.DatabaseException;\r
  49 import org.simantics.db.exception.ValidationException;\r
  50 import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus;\r
  51 import org.simantics.db.request.Read;\r
  52 import org.simantics.db.service.ClusterControl;\r
  53 import org.simantics.db.service.ClusterControl.ClusterState;\r
  54 import org.simantics.db.service.SerialisationSupport;\r
  55 import org.simantics.graph.representation.External;\r
  56 import org.simantics.graph.representation.Identity;\r
  57 import org.simantics.graph.representation.Root;\r
  58 import org.simantics.graph.representation.TransferableGraph1;\r
  59 import org.simantics.graph.representation.Value;\r
  60 import org.simantics.layer0.Layer0;\r
  61 import org.simantics.utils.datastructures.Pair;\r
  62 \r
  63 /**\r
  64  * @deprecated in favor of {@link ModelTransferableGraphSourceRequest}\r
  65  */\r
  66 @Deprecated\r
  67 public class TransferableGraphRequest2 implements Read<TransferableGraph1> {\r
  68 \r
  69         public static String LOG_FILE = "transferableGraph.log";\r
  70         final static private boolean LOG = false;\r
  71         final static private boolean DEBUG = false;\r
  72         final static private boolean PROFILE = false;\r
  73         \r
  74         private TransferableGraphConfiguration configuration;\r
  75         \r
  76         static DataOutput log;\r
  77 \r
  78         static {\r
  79 \r
  80                 if (LOG) {\r
  81                         try {\r
  82                                 FileOutputStream stream = new FileOutputStream(LOG_FILE);\r
  83                                 log = new DataOutputStream(stream);\r
  84                         } catch (FileNotFoundException e) {\r
  85                                 e.printStackTrace();\r
  86                         }\r
  87                 }\r
  88 \r
  89         }\r
  90         \r
  91         private static void log(String line) {\r
  92                 if (LOG) {\r
  93                         try {\r
  94                                 log.writeUTF(line + "\n");\r
  95                         } catch (IOException e) {\r
  96                                 e.printStackTrace();\r
  97                         }\r
  98                 }\r
  99         }\r
 100         \r
 101         \r
 102         \r
 103         public TransferableGraphRequest2(Collection<Pair<Resource, String>> roots, Resource model) {\r
 104                 \r
 105                 configuration = new TransferableGraphConfiguration();\r
 106                 configuration.roots = roots;\r
 107                 configuration.model = model;\r
 108                 \r
 109         }\r
 110 \r
 111         \r
 112         public TransferableGraphRequest2(Collection<Pair<Resource, String>> roots) {\r
 113                 this(roots, null);\r
 114         }\r
 115 \r
 116         public TransferableGraphRequest2(TransferableGraphConfiguration conf) {\r
 117                 this.configuration = conf;\r
 118         }\r
 119 \r
 120         Layer0 L0;\r
 121         \r
 122         TIntArrayList inverses = new TIntArrayList();\r
 123         int statements[];\r
 124         int statementIndex = 0;\r
 125         TIntIntHashMap ids;\r
 126         TIntObjectMap<Variant> values;\r
 127         TIntArrayList externalParents = new TIntArrayList();\r
 128         ArrayList<String> externalNames = new ArrayList<String>(); \r
 129         int id = 0;\r
 130         int internalCount;\r
 131         \r
 132         int indent = 0;\r
 133 \r
 134         private SerialisationSupport support;\r
 135 \r
 136         private boolean validateExternal(Resource r) {\r
 137                 if(configuration.disallowedExternals != null) {\r
 138                         System.err.println("validateExternal agains " + configuration.disallowedExternals);\r
 139                         return !configuration.disallowedExternals.contains(r);\r
 140                 }\r
 141                 return true;\r
 142         }\r
 143         \r
 144         private Resource getResource(int r) throws DatabaseException {\r
 145                 return support.getResource(r);\r
 146         }\r
 147         \r
 148         public int getInternalId(int r) {\r
 149                 return ids.get(r);\r
 150         }\r
 151         \r
 152         public int getId(ReadGraph graph, int r, int predicate) throws DatabaseException {\r
 153                 if(ids.containsKey(r)) {\r
 154                     int ret = ids.get(r);\r
 155                     if(ret == -1) {\r
 156                         for(int i=0;i<=indent;++i)\r
 157                             System.out.print("  ");\r
 158                         System.out.println("Cycle!!!"); // with " + GraphUtils.getReadableName(g, r));\r
 159                     }\r
 160                         return ret;\r
 161                 }\r
 162                 else {\r
 163                         Collection<Resource> parents = graph.getObjects(getResource(r), L0.PartOf);                     \r
 164                         if(parents.size() != 1) {\r
 165                                 throw new ValidationException("Reference to external resource " \r
 166                                                 + NameUtils.getSafeName(graph, getResource(r), true) + " without unique uri (" + parents.size() + " parents).");\r
 167                         }\r
 168                         for(Resource p : parents) {\r
 169                             ++indent;\r
 170                     if(!validateExternal(p)) throw new ValidationException("References to '" + graph.getURI(p) + "' are not allowed.");\r
 171                                 externalParents.add(getId(graph, support.getTransientId(p), 0));\r
 172                                 --indent;\r
 173                         }\r
 174             externalNames.add((String)graph.getRelatedValue(getResource(r), L0.HasName));\r
 175                         ids.put(r, id);\r
 176                         return id++;\r
 177                 }\r
 178         }\r
 179         \r
 180         public void addId(ReadGraph graph, int r, int predicate) throws DatabaseException {\r
 181                 statements[statementIndex++] = getId(graph, r, predicate); \r
 182         }\r
 183         \r
 184         public void setExternals(Collection<Resource> rs) {\r
 185                 configuration.externals = rs;\r
 186         }\r
 187         \r
 188         @Override\r
 189         public TransferableGraph1 perform(ReadGraph graph) throws DatabaseException {\r
 190 \r
 191                 support = graph.getService(SerialisationSupport.class);\r
 192 \r
 193                 this.L0 = Layer0.getInstance(graph);    \r
 194 \r
 195                 long total = System.nanoTime();\r
 196 \r
 197         long startupTime = System.nanoTime();\r
 198 \r
 199         ClusterControl cc = graph.getService(ClusterControl.class);\r
 200         \r
 201         ids = new TIntIntHashMap();\r
 202         values = new TIntObjectHashMap<Variant>();\r
 203         \r
 204                 ArrayList<Resource> rootResources = new ArrayList<Resource>();\r
 205                 for(Pair<Resource, String> p : configuration.roots) rootResources.add(p.first);\r
 206 \r
 207                 Map<Resource, ExtentStatus> preStatus = new HashMap<Resource, ExtentStatus>();\r
 208                 \r
 209                 for(Resource root : rootResources) {\r
 210                         Resource name = graph.getPossibleObject(root, L0.HasName);\r
 211                         if(name != null) {\r
 212                                 preStatus.put(name, ExtentStatus.EXCLUDED);\r
 213                         }\r
 214                 }\r
 215                 \r
 216                 for(Resource r : configuration.externals) preStatus.put(r, ExtentStatus.EXTERNAL); \r
 217 \r
 218         long startupTimeEnd = System.nanoTime();\r
 219 \r
 220                 long domainTime = System.nanoTime();\r
 221                 \r
 222                 String otherStatements = "other" + UUID.randomUUID().toString();\r
 223                 String valueFileName = "value" + UUID.randomUUID().toString();\r
 224                 \r
 225         File otherStatementsFile = new File(otherStatements);\r
 226         File valueFile = new File(valueFileName);\r
 227         \r
 228         try {\r
 229         \r
 230                 DeferredFileOutputStream otherStatementsStream = new DeferredFileOutputStream(1024*1024, otherStatementsFile);\r
 231                 DeferredFileOutputStream valueStream = new DeferredFileOutputStream(1024*1024, valueFile);\r
 232                 \r
 233                 ObjectOutputStream otherStatementsOutput = new ObjectOutputStream(otherStatementsStream);\r
 234                 ObjectOutputStream valueOutput = new ObjectOutputStream(valueStream);\r
 235 \r
 236                 ClusterState clusterState = cc.getClusterState();\r
 237 \r
 238                 TIntHashSet excludedShared = new TIntHashSet();\r
 239                 \r
 240                 TreeMap<String, Variant> extensions = new TreeMap<String, Variant>();\r
 241                 \r
 242                 Subgraphs.getDomain2(graph, ids, rootResources, preStatus, configuration.specials, otherStatementsOutput, valueOutput, extensions, excludedShared);\r
 243                 \r
 244                 id = ids.size();\r
 245 \r
 246                 cc.restoreClusterState(clusterState);\r
 247 \r
 248 //              dumpHeap("domain.hprof");\r
 249 \r
 250                 otherStatementsOutput.flush();\r
 251                 valueOutput.flush();\r
 252                 otherStatementsStream.close();\r
 253                 valueStream.close();\r
 254                 \r
 255                 long domainDuration = System.nanoTime() - domainTime;\r
 256                 System.err.println("Analysed graph in " + 1e-9*domainDuration + "s.");\r
 257 \r
 258                 internalCount = id;\r
 259 \r
 260                 ids.put(support.getTransientId(graph.getResource("http:/")), id++);\r
 261                 externalNames.add("http:/");\r
 262                 externalParents.add(-1);\r
 263 \r
 264                 InputStream otherStatementsInputStream = null;\r
 265                 InputStream valueInputStream = null;\r
 266                 \r
 267                 if(otherStatementsStream.isInMemory()) {\r
 268                         otherStatementsInputStream = new ByteArrayInputStream(otherStatementsStream.getData());\r
 269                 } else {\r
 270                         otherStatementsInputStream = new FileInputStream(otherStatementsFile);\r
 271                 }\r
 272 \r
 273                 if(valueStream.isInMemory()) {\r
 274                         valueInputStream = new ByteArrayInputStream(valueStream.getData());\r
 275                 } else {\r
 276                         valueInputStream = new FileInputStream(valueFile);\r
 277                 }\r
 278 \r
 279                 otherStatementsStream = null;\r
 280                 valueStream = null;\r
 281                 \r
 282                 ObjectInputStream otherStatementsInput = new ObjectInputStream(otherStatementsInputStream);\r
 283                 ObjectInputStream valueInput = new ObjectInputStream(valueInputStream);\r
 284 \r
 285                 long statementTime = System.nanoTime();\r
 286                 \r
 287                 TIntArrayList statementSet = new TIntArrayList();\r
 288                 \r
 289                 while(otherStatementsInput.available() > 0) {\r
 290                         \r
 291                         int s = otherStatementsInput.readInt();\r
 292                         \r
 293                         boolean exclude = !ids.contains(s);\r
 294                         \r
 295                         int size = otherStatementsInput.readInt();\r
 296                         for(int i=0;i<size;i++) {\r
 297                                 int p = otherStatementsInput.readInt();\r
 298                                 int o = otherStatementsInput.readInt();\r
 299                                 if(!exclude) {\r
 300                                         if(excludedShared.contains(o)) {\r
 301 //                                              System.err.println("excluding shared " + s + " " + p + " " + o);\r
 302                                         } else {\r
 303                                                 statementSet.add(s);\r
 304                                                 statementSet.add(p);\r
 305                                                 statementSet.add(o);\r
 306                                         }\r
 307                                 } else {\r
 308 //                                              System.err.println("excluding shared " + s);\r
 309                                 }\r
 310                         }\r
 311                         \r
 312                 }\r
 313 \r
 314                 TIntIntHashMap inverses = new TIntIntHashMap();\r
 315                 TIntHashSet predicateSet = new TIntHashSet();\r
 316                 for(int i=0;i<statementSet.size();i+=3) {\r
 317                         int p = statementSet.getQuick(i+1);\r
 318                         if(predicateSet.add(p)) {\r
 319                                         Resource inverse = graph.getPossibleInverse(getResource(p));\r
 320                                         if(inverse != null) inverses.put(p, support.getTransientId(inverse));\r
 321                         }\r
 322                 }\r
 323                         \r
 324                 predicateSet = null;\r
 325                 \r
 326                 TIntArrayList tgStatements = new TIntArrayList();\r
 327                 \r
 328 //              dumpHeap("export.hprof");\r
 329 \r
 330                 // 25%\r
 331                 int trim = Math.max(65536,statementSet.size()/12);\r
 332                 \r
 333                 for(int i=statementSet.size();i>0;i-=3) {\r
 334 \r
 335                         if(trim-- == 0) {\r
 336                                 statementSet.remove(i, statementSet.size()-i);\r
 337                                 statementSet.trimToSize();\r
 338                                 trim = Math.max(65536,statementSet.size()/12);\r
 339                         }\r
 340 \r
 341                         int s = statementSet.getQuick(i-3);\r
 342                         int p = statementSet.getQuick(i-2);\r
 343                         int o = statementSet.getQuick(i-1);\r
 344                 \r
 345                         int subjectId = ids.get(s);\r
 346                         if(subjectId >= internalCount) System.err.println("Statement for external: " + s + " " + p + " " + o);\r
 347                         \r
 348                         int objectId = getId(graph, o, p);\r
 349                         // The statement can be denied still\r
 350                         if(objectId != -2) {\r
 351                                 tgStatements.add(subjectId);\r
 352                                 tgStatements.add(getId(graph, p, 0));\r
 353                                 int inverse = inverses.get(p);\r
 354                                 if(inverse != 0) {\r
 355                                         tgStatements.add(getId(graph, inverse, 0));\r
 356                                 } else {\r
 357                                         tgStatements.add(-1);\r
 358                                 }\r
 359                                 tgStatements.add(objectId);\r
 360                         } else {\r
 361                                 System.out.println("denied");\r
 362                         }\r
 363                         \r
 364                 }\r
 365                 \r
 366                 statements = tgStatements.toArray();\r
 367 \r
 368                 long statementDuration = System.nanoTime() - statementTime;\r
 369                 System.err.println("Built transferable statements in " + 1e-9*statementDuration + "s.");\r
 370                 \r
 371                 inverses = null;\r
 372                 \r
 373                 while(valueInput.available() > 0) {\r
 374 \r
 375                         int s = valueInput.readInt();\r
 376 //                      Resource subject = support.getResource(s);\r
 377                         int valueSize = valueInput.readInt();\r
 378                         byte[] value = new byte[valueSize];\r
 379                         valueInput.readFully(value);\r
 380                         Variant variant = (Variant)Bindings.VARIANT.serializer().deserialize(value);\r
 381                         values.put(s, variant);\r
 382                         \r
 383                 }\r
 384 \r
 385                 int resourceCount = ids.size();\r
 386 \r
 387                 Identity[] identityArray;\r
 388                 { // Identities\r
 389                         ArrayList<Identity> identities = new ArrayList<Identity>();\r
 390 \r
 391                         for(Pair<Resource, String> r : configuration.roots) {\r
 392                                 Resource type = graph.getPossibleType(r.first, L0.Entity);\r
 393                                 if(type == null) type = L0.Entity;\r
 394                                 identities.add(new Identity(\r
 395                                                 ids.get(support.getTransientId(r.first)),\r
 396                                                 new Root(r.second, graph.getURI(type))\r
 397                                 ));\r
 398                         }\r
 399 \r
 400                         int internalsPlusExternals = ids.size();                \r
 401                         for(int i = internalCount; i < internalsPlusExternals ; i++) {\r
 402                                 int parent = externalParents.get(i - internalCount);\r
 403                                 String name = externalNames.get(i - internalCount);\r
 404                                 identities.add(new Identity(\r
 405                                                 i, \r
 406                                                 new External(parent, name)\r
 407                                 ));\r
 408                         }\r
 409                         identityArray = identities.toArray(new Identity[identities.size()]);\r
 410                 }       \r
 411             \r
 412             final Value[] valueArray = new Value[values.size()];\r
 413                 { // Values\r
 414                         values.forEachEntry(new TIntObjectProcedure<Variant>() {\r
 415 \r
 416                         int index = 0;\r
 417 \r
 418                                         @Override\r
 419                                         public boolean execute(int subject, Variant bytes) {\r
 420                                         //if(LOG) log("[VALUE] " + entry.getKey().getResourceId());\r
 421                                         int r = getInternalId(subject);\r
 422                                         if(r==-1) System.err.println("No id for value resource " + subject);\r
 423                                         else valueArray[index++] = new Value(r, bytes);\r
 424                                         return true;\r
 425                                         }\r
 426                                         \r
 427                                 });\r
 428                 }\r
 429             ids = null;\r
 430                 values = null;\r
 431 \r
 432             TransferableGraph1 result = \r
 433                 new TransferableGraph1(resourceCount, \r
 434                         identityArray, \r
 435                         statements, \r
 436                         valueArray, extensions);\r
 437             \r
 438                 if(DEBUG) {\r
 439                         System.out.println("transferable graph content: " + result);\r
 440                 }\r
 441 \r
 442                 long totalEnd = System.nanoTime();\r
 443 \r
 444                 if(PROFILE) {   \r
 445                         System.out.println("startup in " + 1e-9*(startupTimeEnd - startupTime) + "s.");\r
 446                         System.out.println("domain was found in " + 1e-9*(domainDuration) + "s.");\r
 447                         System.out.println("statements were found in " + 1e-9*(statementDuration) + "s.");\r
 448                         System.out.println("total time for building subgraph was " + 1e-9*(totalEnd-total) + "s.");\r
 449                 }\r
 450 \r
 451                 return result;\r
 452 \r
 453         } catch (IOException e) {\r
 454                 e.printStackTrace();\r
 455         } catch (Throwable t) {\r
 456                 t.printStackTrace();\r
 457                 dumpHeap("crash.hprof");\r
 458         }\r
 459                 \r
 460         return null;\r
 461                 \r
 462         }\r
 463 \r
 464         private static void dumpHeap(String path) {\r
 465                 \r
 466         try {\r
 467             Object bean = getBean();\r
 468             if (bean == null)\r
 469                 return;\r
 470 \r
 471             Method m = bean.getClass().getMethod("dumpHeap", String.class, boolean.class);\r
 472             m.invoke(bean, path, true);\r
 473             \r
 474         } catch (IllegalArgumentException e) {\r
 475         } catch (IllegalAccessException e) {\r
 476         } catch (SecurityException e) {\r
 477         } catch (NoSuchMethodException e) {\r
 478         } catch (InvocationTargetException e) {\r
 479                 } finally {\r
 480         }\r
 481                 \r
 482         }\r
 483         \r
 484     private static Object getBean() {\r
 485         Class<?> beanClass = getBeanClass();\r
 486         if (beanClass == null)\r
 487             return null;\r
 488         try {\r
 489             Object bean = ManagementFactory.newPlatformMXBeanProxy(\r
 490                     ManagementFactory.getPlatformMBeanServer(),\r
 491                     "com.sun.management:type=HotSpotDiagnostic",\r
 492                     beanClass);\r
 493             return bean;\r
 494         } catch (IOException e) {\r
 495             return null;\r
 496         }\r
 497     }\r
 498     \r
 499     private static Class<?> getBeanClass() {\r
 500         try {\r
 501             Class<?> clazz = Class.forName("com.sun.management.HotSpotDiagnosticMXBean");\r
 502             return clazz;\r
 503         } catch (ClassNotFoundException e) {\r
 504             return null;\r
 505         }\r
 506     }\r
 507     \r
 508         \r
 509 }\r