bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TransferableGraphRequest2.java

   1 /*******************************************************************************\r
   2  * Copyright (c) 2007, 2010 Association for Decentralized Information Management\r
   3  * in Industry THTH ry.\r
   4  * All rights reserved. This program and the accompanying materials\r
   5  * are made available under the terms of the Eclipse Public License v1.0\r
   6  * which accompanies this distribution, and is available at\r
   7  * http://www.eclipse.org/legal/epl-v10.html\r
   8  *\r
   9  * Contributors:\r
  10  *     VTT Technical Research Centre of Finland - initial API and implementation\r
  11  *******************************************************************************/\r
  12 package org.simantics.db.layer0.util;\r
  13 \r
  14 import gnu.trove.list.array.TIntArrayList;\r
  15 import gnu.trove.map.TIntObjectMap;\r
  16 import gnu.trove.map.hash.TIntIntHashMap;\r
  17 import gnu.trove.map.hash.TIntObjectHashMap;\r
  18 import gnu.trove.procedure.TIntObjectProcedure;\r
  19 import gnu.trove.set.hash.TIntHashSet;\r
  20 \r
  21 import java.io.ByteArrayInputStream;\r
  22 import java.io.DataOutput;\r
  23 import java.io.DataOutputStream;\r
  24 import java.io.File;\r
  25 import java.io.FileInputStream;\r
  26 import java.io.FileNotFoundException;\r
  27 import java.io.FileOutputStream;\r
  28 import java.io.IOException;\r
  29 import java.io.InputStream;\r
  30 import java.io.ObjectInputStream;\r
  31 import java.io.ObjectOutputStream;\r
  32 import java.lang.management.ManagementFactory;\r
  33 import java.lang.reflect.InvocationTargetException;\r
  34 import java.lang.reflect.Method;\r
  35 import java.util.ArrayList;\r
  36 import java.util.Collection;\r
  37 import java.util.HashMap;\r
  38 import java.util.Map;\r
  39 import java.util.TreeMap;\r
  40 import java.util.UUID;\r
  41 \r
  42 import org.apache.commons.io.output.DeferredFileOutputStream;\r
  43 import org.simantics.databoard.Bindings;\r
  44 import org.simantics.databoard.binding.mutable.Variant;\r
  45 import org.simantics.db.ReadGraph;\r
  46 import org.simantics.db.Resource;\r
  47 import org.simantics.db.common.utils.NameUtils;\r
  48 import org.simantics.db.exception.DatabaseException;\r
  49 import org.simantics.db.exception.ValidationException;\r
  50 import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus;\r
  51 import org.simantics.db.request.Read;\r
  52 import org.simantics.db.service.ClusterControl;\r
  53 import org.simantics.db.service.ClusterControl.ClusterState;\r
  54 import org.simantics.db.service.SerialisationSupport;\r
  55 import org.simantics.graph.representation.External;\r
  56 import org.simantics.graph.representation.Identity;\r
  57 import org.simantics.graph.representation.Root;\r
  58 import org.simantics.graph.representation.TransferableGraph1;\r
  59 import org.simantics.graph.representation.Value;\r
  60 import org.simantics.layer0.Layer0;\r
  61 import org.simantics.utils.datastructures.Pair;\r
  62 \r
  63 public class TransferableGraphRequest2 implements Read<TransferableGraph1> {\r
  64 \r
  65         public static String LOG_FILE = "transferableGraph.log";\r
  66         final static private boolean LOG = false;\r
  67         final static private boolean DEBUG = false;\r
  68         final static private boolean PROFILE = false;\r
  69         \r
  70         private TransferableGraphConfiguration configuration;\r
  71         \r
  72         static DataOutput log;\r
  73 \r
  74         static {\r
  75 \r
  76                 if (LOG) {\r
  77                         try {\r
  78                                 FileOutputStream stream = new FileOutputStream(LOG_FILE);\r
  79                                 log = new DataOutputStream(stream);\r
  80                         } catch (FileNotFoundException e) {\r
  81                                 e.printStackTrace();\r
  82                         }\r
  83                 }\r
  84 \r
  85         }\r
  86         \r
  87         private static void log(String line) {\r
  88                 if (LOG) {\r
  89                         try {\r
  90                                 log.writeUTF(line + "\n");\r
  91                         } catch (IOException e) {\r
  92                                 e.printStackTrace();\r
  93                         }\r
  94                 }\r
  95         }\r
  96         \r
  97         \r
  98         \r
  99         public TransferableGraphRequest2(Collection<Pair<Resource, String>> roots, Resource model) {\r
 100                 \r
 101                 configuration = new TransferableGraphConfiguration();\r
 102                 configuration.roots = roots;\r
 103                 configuration.model = model;\r
 104                 \r
 105         }\r
 106 \r
 107         \r
 108         public TransferableGraphRequest2(Collection<Pair<Resource, String>> roots) {\r
 109                 this(roots, null);\r
 110         }\r
 111 \r
 112         public TransferableGraphRequest2(TransferableGraphConfiguration conf) {\r
 113                 this.configuration = conf;\r
 114         }\r
 115 \r
 116         Layer0 L0;\r
 117         \r
 118         TIntArrayList inverses = new TIntArrayList();\r
 119         int statements[];\r
 120         int statementIndex = 0;\r
 121         TIntIntHashMap ids;\r
 122         TIntObjectMap<Variant> values;\r
 123         TIntArrayList externalParents = new TIntArrayList();\r
 124         ArrayList<String> externalNames = new ArrayList<String>(); \r
 125         int id = 0;\r
 126         int internalCount;\r
 127         \r
 128         int indent = 0;\r
 129 \r
 130         private SerialisationSupport support;\r
 131 \r
 132         private boolean validateExternal(Resource r) {\r
 133                 if(configuration.disallowedExternals != null) {\r
 134                         System.err.println("validateExternal agains " + configuration.disallowedExternals);\r
 135                         return !configuration.disallowedExternals.contains(r);\r
 136                 }\r
 137                 return true;\r
 138         }\r
 139         \r
 140         private Resource getResource(int r) throws DatabaseException {\r
 141                 return support.getResource(r);\r
 142         }\r
 143         \r
 144         public int getInternalId(int r) {\r
 145                 return ids.get(r);\r
 146         }\r
 147         \r
 148         public int getId(ReadGraph graph, int r, int predicate) throws DatabaseException {\r
 149                 if(ids.containsKey(r)) {\r
 150                     int ret = ids.get(r);\r
 151                     if(ret == -1) {\r
 152                         for(int i=0;i<=indent;++i)\r
 153                             System.out.print("  ");\r
 154                         System.out.println("Cycle!!!"); // with " + GraphUtils.getReadableName(g, r));\r
 155                     }\r
 156                         return ret;\r
 157                 }\r
 158                 else {\r
 159                         Collection<Resource> parents = graph.getObjects(getResource(r), L0.PartOf);                     \r
 160                         if(parents.size() != 1) {\r
 161                                 throw new ValidationException("Reference to external resource " \r
 162                                                 + NameUtils.getSafeName(graph, getResource(r), true) + " without unique uri (" + parents.size() + " parents).");\r
 163                         }\r
 164                         for(Resource p : parents) {\r
 165                             ++indent;\r
 166                     if(!validateExternal(p)) throw new ValidationException("References to '" + graph.getURI(p) + "' are not allowed.");\r
 167                                 externalParents.add(getId(graph, support.getTransientId(p), 0));\r
 168                                 --indent;\r
 169                         }\r
 170             externalNames.add((String)graph.getRelatedValue(getResource(r), L0.HasName));\r
 171                         ids.put(r, id);\r
 172                         return id++;\r
 173                 }\r
 174         }\r
 175         \r
 176         public void addId(ReadGraph graph, int r, int predicate) throws DatabaseException {\r
 177                 statements[statementIndex++] = getId(graph, r, predicate); \r
 178         }\r
 179         \r
 180         public void setExternals(Collection<Resource> rs) {\r
 181                 configuration.externals = rs;\r
 182         }\r
 183         \r
 184         @Override\r
 185         public TransferableGraph1 perform(ReadGraph graph) throws DatabaseException {\r
 186 \r
 187                 support = graph.getService(SerialisationSupport.class);\r
 188 \r
 189                 this.L0 = Layer0.getInstance(graph);    \r
 190 \r
 191                 long total = System.nanoTime();\r
 192 \r
 193         long startupTime = System.nanoTime();\r
 194 \r
 195         ClusterControl cc = graph.getService(ClusterControl.class);\r
 196         \r
 197         ids = new TIntIntHashMap();\r
 198         values = new TIntObjectHashMap<Variant>();\r
 199         \r
 200                 ArrayList<Resource> rootResources = new ArrayList<Resource>();\r
 201                 for(Pair<Resource, String> p : configuration.roots) rootResources.add(p.first);\r
 202 \r
 203                 Map<Resource, ExtentStatus> preStatus = new HashMap<Resource, ExtentStatus>();\r
 204                 \r
 205                 for(Resource root : rootResources) {\r
 206                         Resource name = graph.getPossibleObject(root, L0.HasName);\r
 207                         if(name != null) {\r
 208                                 preStatus.put(name, ExtentStatus.EXCLUDED);\r
 209                         }\r
 210                 }\r
 211                 \r
 212                 for(Resource r : configuration.externals) preStatus.put(r, ExtentStatus.EXTERNAL); \r
 213 \r
 214         long startupTimeEnd = System.nanoTime();\r
 215 \r
 216                 long domainTime = System.nanoTime();\r
 217                 \r
 218                 String otherStatements = "other" + UUID.randomUUID().toString();\r
 219                 String valueFileName = "value" + UUID.randomUUID().toString();\r
 220                 \r
 221         File otherStatementsFile = new File(otherStatements);\r
 222         File valueFile = new File(valueFileName);\r
 223         \r
 224         try {\r
 225         \r
 226                 DeferredFileOutputStream otherStatementsStream = new DeferredFileOutputStream(1024*1024, otherStatementsFile);\r
 227                 DeferredFileOutputStream valueStream = new DeferredFileOutputStream(1024*1024, valueFile);\r
 228                 \r
 229                 ObjectOutputStream otherStatementsOutput = new ObjectOutputStream(otherStatementsStream);\r
 230                 ObjectOutputStream valueOutput = new ObjectOutputStream(valueStream);\r
 231 \r
 232                 ClusterState clusterState = cc.getClusterState();\r
 233 \r
 234                 TIntHashSet excludedShared = new TIntHashSet();\r
 235                 \r
 236                 TreeMap<String, Variant> extensions = new TreeMap<String, Variant>();\r
 237                 \r
 238                 Subgraphs.getDomain2(graph, ids, rootResources, preStatus, configuration.specials, otherStatementsOutput, valueOutput, extensions, excludedShared);\r
 239                 \r
 240                 id = ids.size();\r
 241 \r
 242                 cc.restoreClusterState(clusterState);\r
 243 \r
 244 //              dumpHeap("domain.hprof");\r
 245 \r
 246                 otherStatementsOutput.flush();\r
 247                 valueOutput.flush();\r
 248                 otherStatementsStream.close();\r
 249                 valueStream.close();\r
 250                 \r
 251                 long domainDuration = System.nanoTime() - domainTime;\r
 252                 System.err.println("Analysed graph in " + 1e-9*domainDuration + "s.");\r
 253 \r
 254                 internalCount = id;\r
 255 \r
 256                 ids.put(support.getTransientId(graph.getResource("http:/")), id++);\r
 257                 externalNames.add("http:/");\r
 258                 externalParents.add(-1);\r
 259 \r
 260                 InputStream otherStatementsInputStream = null;\r
 261                 InputStream valueInputStream = null;\r
 262                 \r
 263                 if(otherStatementsStream.isInMemory()) {\r
 264                         otherStatementsInputStream = new ByteArrayInputStream(otherStatementsStream.getData());\r
 265                 } else {\r
 266                         otherStatementsInputStream = new FileInputStream(otherStatementsFile);\r
 267                 }\r
 268 \r
 269                 if(valueStream.isInMemory()) {\r
 270                         valueInputStream = new ByteArrayInputStream(valueStream.getData());\r
 271                 } else {\r
 272                         valueInputStream = new FileInputStream(valueFile);\r
 273                 }\r
 274 \r
 275                 otherStatementsStream = null;\r
 276                 valueStream = null;\r
 277                 \r
 278                 ObjectInputStream otherStatementsInput = new ObjectInputStream(otherStatementsInputStream);\r
 279                 ObjectInputStream valueInput = new ObjectInputStream(valueInputStream);\r
 280 \r
 281                 long statementTime = System.nanoTime();\r
 282                 \r
 283                 TIntArrayList statementSet = new TIntArrayList();\r
 284                 \r
 285                 while(otherStatementsInput.available() > 0) {\r
 286                         \r
 287                         int s = otherStatementsInput.readInt();\r
 288                         \r
 289                         boolean exclude = !ids.contains(s);\r
 290                         \r
 291                         int size = otherStatementsInput.readInt();\r
 292                         for(int i=0;i<size;i++) {\r
 293                                 int p = otherStatementsInput.readInt();\r
 294                                 int o = otherStatementsInput.readInt();\r
 295                                 if(!exclude) {\r
 296                                         if(excludedShared.contains(o)) {\r
 297 //                                              System.err.println("excluding shared " + s + " " + p + " " + o);\r
 298                                         } else {\r
 299                                                 statementSet.add(s);\r
 300                                                 statementSet.add(p);\r
 301                                                 statementSet.add(o);\r
 302                                         }\r
 303                                 } else {\r
 304 //                                              System.err.println("excluding shared " + s);\r
 305                                 }\r
 306                         }\r
 307                         \r
 308                 }\r
 309 \r
 310                 TIntIntHashMap inverses = new TIntIntHashMap();\r
 311                 TIntHashSet predicateSet = new TIntHashSet();\r
 312                 for(int i=0;i<statementSet.size();i+=3) {\r
 313                         int p = statementSet.getQuick(i+1);\r
 314                         if(predicateSet.add(p)) {\r
 315                                         Resource inverse = graph.getPossibleInverse(getResource(p));\r
 316                                         if(inverse != null) inverses.put(p, support.getTransientId(inverse));\r
 317                         }\r
 318                 }\r
 319                         \r
 320                 predicateSet = null;\r
 321                 \r
 322                 TIntArrayList tgStatements = new TIntArrayList();\r
 323                 \r
 324 //              dumpHeap("export.hprof");\r
 325 \r
 326                 // 25%\r
 327                 int trim = Math.max(65536,statementSet.size()/12);\r
 328                 \r
 329                 for(int i=statementSet.size();i>0;i-=3) {\r
 330 \r
 331                         if(trim-- == 0) {\r
 332                                 statementSet.remove(i, statementSet.size()-i);\r
 333                                 statementSet.trimToSize();\r
 334                                 trim = Math.max(65536,statementSet.size()/12);\r
 335                         }\r
 336 \r
 337                         int s = statementSet.getQuick(i-3);\r
 338                         int p = statementSet.getQuick(i-2);\r
 339                         int o = statementSet.getQuick(i-1);\r
 340                 \r
 341                         int subjectId = ids.get(s);\r
 342                         if(subjectId >= internalCount) System.err.println("Statement for external: " + s + " " + p + " " + o);\r
 343                         \r
 344                         int objectId = getId(graph, o, p);\r
 345                         // The statement can be denied still\r
 346                         if(objectId != -2) {\r
 347                                 tgStatements.add(subjectId);\r
 348                                 tgStatements.add(getId(graph, p, 0));\r
 349                                 int inverse = inverses.get(p);\r
 350                                 if(inverse != 0) {\r
 351                                         tgStatements.add(getId(graph, inverse, 0));\r
 352                                 } else {\r
 353                                         tgStatements.add(-1);\r
 354                                 }\r
 355                                 tgStatements.add(objectId);\r
 356                         } else {\r
 357                                 System.out.println("denied");\r
 358                         }\r
 359                         \r
 360                 }\r
 361                 \r
 362                 statements = tgStatements.toArray();\r
 363 \r
 364                 long statementDuration = System.nanoTime() - statementTime;\r
 365                 System.err.println("Built transferable statements in " + 1e-9*statementDuration + "s.");\r
 366                 \r
 367                 inverses = null;\r
 368                 \r
 369                 while(valueInput.available() > 0) {\r
 370 \r
 371                         int s = valueInput.readInt();\r
 372 //                      Resource subject = support.getResource(s);\r
 373                         int valueSize = valueInput.readInt();\r
 374                         byte[] value = new byte[valueSize];\r
 375                         valueInput.readFully(value);\r
 376                         Variant variant = (Variant)Bindings.VARIANT.serializer().deserialize(value);\r
 377                         values.put(s, variant);\r
 378                         \r
 379                 }\r
 380 \r
 381                 int resourceCount = ids.size();\r
 382 \r
 383                 Identity[] identityArray;\r
 384                 { // Identities\r
 385                         ArrayList<Identity> identities = new ArrayList<Identity>();\r
 386 \r
 387                         for(Pair<Resource, String> r : configuration.roots) {\r
 388                                 Resource type = graph.getPossibleType(r.first, L0.Entity);\r
 389                                 if(type == null) type = L0.Entity;\r
 390                                 identities.add(new Identity(\r
 391                                                 ids.get(support.getTransientId(r.first)),\r
 392                                                 new Root(r.second, graph.getURI(type))\r
 393                                 ));\r
 394                         }\r
 395 \r
 396                         int internalsPlusExternals = ids.size();                \r
 397                         for(int i = internalCount; i < internalsPlusExternals ; i++) {\r
 398                                 int parent = externalParents.get(i - internalCount);\r
 399                                 String name = externalNames.get(i - internalCount);\r
 400                                 identities.add(new Identity(\r
 401                                                 i, \r
 402                                                 new External(parent, name)\r
 403                                 ));\r
 404                         }\r
 405                         identityArray = identities.toArray(new Identity[identities.size()]);\r
 406                 }       \r
 407             \r
 408             final Value[] valueArray = new Value[values.size()];\r
 409                 { // Values\r
 410                         values.forEachEntry(new TIntObjectProcedure<Variant>() {\r
 411 \r
 412                         int index = 0;\r
 413 \r
 414                                         @Override\r
 415                                         public boolean execute(int subject, Variant bytes) {\r
 416                                         //if(LOG) log("[VALUE] " + entry.getKey().getResourceId());\r
 417                                         int r = getInternalId(subject);\r
 418                                         if(r==-1) System.err.println("No id for value resource " + subject);\r
 419                                         else valueArray[index++] = new Value(r, bytes);\r
 420                                         return true;\r
 421                                         }\r
 422                                         \r
 423                                 });\r
 424                 }\r
 425             ids = null;\r
 426                 values = null;\r
 427 \r
 428             TransferableGraph1 result = \r
 429                 new TransferableGraph1(resourceCount, \r
 430                         identityArray, \r
 431                         statements, \r
 432                         valueArray, extensions);\r
 433             \r
 434                 if(DEBUG) {\r
 435                         System.out.println("transferable graph content: " + result);\r
 436                 }\r
 437 \r
 438                 long totalEnd = System.nanoTime();\r
 439 \r
 440                 if(PROFILE) {   \r
 441                         System.out.println("startup in " + 1e-9*(startupTimeEnd - startupTime) + "s.");\r
 442                         System.out.println("domain was found in " + 1e-9*(domainDuration) + "s.");\r
 443                         System.out.println("statements were found in " + 1e-9*(statementDuration) + "s.");\r
 444                         System.out.println("total time for building subgraph was " + 1e-9*(totalEnd-total) + "s.");\r
 445                 }\r
 446 \r
 447                 return result;\r
 448 \r
 449         } catch (IOException e) {\r
 450                 e.printStackTrace();\r
 451         } catch (Throwable t) {\r
 452                 t.printStackTrace();\r
 453                 dumpHeap("crash.hprof");\r
 454         }\r
 455                 \r
 456         return null;\r
 457                 \r
 458         }\r
 459 \r
 460         private static void dumpHeap(String path) {\r
 461                 \r
 462         try {\r
 463             Object bean = getBean();\r
 464             if (bean == null)\r
 465                 return;\r
 466 \r
 467             Method m = bean.getClass().getMethod("dumpHeap", String.class, boolean.class);\r
 468             m.invoke(bean, path, true);\r
 469             \r
 470         } catch (IllegalArgumentException e) {\r
 471         } catch (IllegalAccessException e) {\r
 472         } catch (SecurityException e) {\r
 473         } catch (NoSuchMethodException e) {\r
 474         } catch (InvocationTargetException e) {\r
 475                 } finally {\r
 476         }\r
 477                 \r
 478         }\r
 479         \r
 480     private static Object getBean() {\r
 481         Class<?> beanClass = getBeanClass();\r
 482         if (beanClass == null)\r
 483             return null;\r
 484         try {\r
 485             Object bean = ManagementFactory.newPlatformMXBeanProxy(\r
 486                     ManagementFactory.getPlatformMBeanServer(),\r
 487                     "com.sun.management:type=HotSpotDiagnostic",\r
 488                     beanClass);\r
 489             return bean;\r
 490         } catch (IOException e) {\r
 491             return null;\r
 492         }\r
 493     }\r
 494     \r
 495     private static Class<?> getBeanClass() {\r
 496         try {\r
 497             Class<?> clazz = Class.forName("com.sun.management.HotSpotDiagnosticMXBean");\r
 498             return clazz;\r
 499         } catch (ClassNotFoundException e) {\r
 500             return null;\r
 501         }\r
 502     }\r
 503     \r
 504         \r
 505 }\r