bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/ModelTransferableGraphSource.java

   1 package org.simantics.db.layer0.util;\r
   2 \r
   3 import java.io.Closeable;\r
   4 import java.io.DataInput;\r
   5 import java.io.File;\r
   6 import java.io.IOException;\r
   7 import java.nio.file.Files;\r
   8 import java.util.ArrayList;\r
   9 import java.util.Collection;\r
  10 import java.util.Collections;\r
  11 import java.util.HashSet;\r
  12 import java.util.List;\r
  13 import java.util.TreeMap;\r
  14 \r
  15 import org.simantics.databoard.Bindings;\r
  16 import org.simantics.databoard.Databoard;\r
  17 import org.simantics.databoard.binding.Binding;\r
  18 import org.simantics.databoard.binding.mutable.Variant;\r
  19 import org.simantics.databoard.container.DataContainer;\r
  20 import org.simantics.databoard.serialization.Serializer;\r
  21 import org.simantics.databoard.type.Datatype;\r
  22 import org.simantics.databoard.util.binary.ByteBufferReadable;\r
  23 import org.simantics.db.ReadGraph;\r
  24 import org.simantics.db.Resource;\r
  25 import org.simantics.db.common.utils.NameUtils;\r
  26 import org.simantics.db.exception.DatabaseException;\r
  27 import org.simantics.db.exception.RuntimeDatabaseException;\r
  28 import org.simantics.db.exception.ValidationException;\r
  29 import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus;\r
  30 import org.simantics.db.layer0.util.TransferableGraphConfiguration2.RootSpec;\r
  31 import org.simantics.db.service.SerialisationSupport;\r
  32 import org.simantics.graph.db.TransferableGraphSource;\r
  33 import org.simantics.graph.representation.External;\r
  34 import org.simantics.graph.representation.Identity;\r
  35 import org.simantics.graph.representation.Root;\r
  36 import org.simantics.graph.representation.Value;\r
  37 import org.simantics.layer0.Layer0;\r
  38 \r
  39 import gnu.trove.list.array.TIntArrayList;\r
  40 import gnu.trove.procedure.TIntIntProcedure;\r
  41 \r
  42 public class ModelTransferableGraphSource implements TransferableGraphSource {\r
  43 \r
  44         final private TransferableGraphConfiguration2 configuration;\r
  45         final private DomainProcessorState state;\r
  46         final private int externalBase;\r
  47         final private int resourceCount;\r
  48         final private File[] files;\r
  49         final private TGValueModifier valueModifier;\r
  50 \r
  51         private volatile boolean closed = false;\r
  52 \r
  53         TIntArrayList externalParents = new TIntArrayList();\r
  54         ArrayList<String> externalNames = new ArrayList<String>();\r
  55 \r
  56         public ModelTransferableGraphSource(final ReadGraph graph, TransferableGraphConfiguration2 configuration, final DomainProcessorState state, File ... fs) throws DatabaseException {\r
  57 \r
  58                 this.configuration = configuration;\r
  59                 this.state = state;\r
  60                 this.files = fs;\r
  61                 this.valueModifier = state.valueModifier;\r
  62 \r
  63                 SerialisationSupport ss = graph.getService(SerialisationSupport.class);\r
  64 \r
  65                 // At this point ids contains all internal resources. Now add roots and externals.\r
  66                 \r
  67                 // Root Library\r
  68                 state.ids.put(ss.getTransientId(graph.getRootLibrary()), state.id++);\r
  69                 \r
  70                 // External roots - internal roots were already processed as internal resources by domain processor\r
  71                 for(RootSpec rs : configuration.roots) {\r
  72                         if(!rs.internal) {\r
  73                                 int resourceId = ss.getTransientId(rs.resource);\r
  74                                 state.ids.put(resourceId, state.id++);\r
  75                                 // The fixed roots have been seen as externals by domain processor. Now remove them from external set.\r
  76                                 state.externals.remove(resourceId);\r
  77                         }\r
  78                 }\r
  79                 \r
  80                 this.externalBase = state.id;\r
  81 \r
  82                 final Collection<String> errors = new HashSet<String>();\r
  83 \r
  84                 // All resource considered as not internal by domain processor. Can also contain roots.\r
  85                 int[] externals = state.externals.toArray();\r
  86                 \r
  87                 // Build up the state.externals, externalNames and externalParents\r
  88                 for(int i=0;i<externals.length;i++) {\r
  89                         getId(graph, externals[i], errors);\r
  90                 }\r
  91                 \r
  92                 state.inverses.forEachEntry(new TIntIntProcedure() {\r
  93                         \r
  94                         @Override\r
  95                         public boolean execute(int predicate, int inverse) {\r
  96                                 try {\r
  97                                         getId(graph, predicate, errors);\r
  98                                         if(inverse != 0) getId(graph, inverse, errors);\r
  99                                 } catch (DatabaseException e) {\r
 100                                         throw new RuntimeDatabaseException(e);\r
 101                                 }\r
 102                                 return true;\r
 103                         }\r
 104                         \r
 105                 });\r
 106                 \r
 107                 if(!errors.isEmpty()) {\r
 108                         ArrayList<String> sorted = new ArrayList<String>(errors);\r
 109                         Collections.sort(sorted);\r
 110                         StringBuilder message = new StringBuilder();\r
 111                         message.append("Errors in exported model:\n");\r
 112                         for(String error : sorted) {\r
 113                                 message.append(error);\r
 114                                 message.append("\n");\r
 115                         }\r
 116                         throw new DatabaseException(message.toString());\r
 117                 }\r
 118                 \r
 119                 this.resourceCount = state.id;\r
 120                 \r
 121         }\r
 122 \r
 123         int indent = 0;\r
 124         \r
 125         public boolean validateExternal(Resource ext) {\r
 126                 if(configuration.validate) {\r
 127                         ExtentStatus status = configuration.preStatus.get(ext);\r
 128                         if(status != null) {\r
 129                                 if(ExtentStatus.INTERNAL.equals(status)) return false;\r
 130                                 else if(ExtentStatus.EXCLUDED.equals(status)) return false;\r
 131                         }\r
 132                 }\r
 133                 return true;\r
 134         }\r
 135         \r
 136         private Resource getResource(ReadGraph graph, int r) throws DatabaseException {\r
 137                 SerialisationSupport ss = graph.getService(SerialisationSupport.class);\r
 138                 return ss.getResource(r);\r
 139         }\r
 140         \r
 141         final public int getExistingId(ReadGraph graph, int r) throws DatabaseException {\r
 142                 \r
 143                 int ret = state.ids.get(r);\r
 144                 if(ret != -1) {\r
 145                         return ret;\r
 146                 } else {\r
 147                         SerialisationSupport ss = graph.getService(SerialisationSupport.class);\r
 148                         throw new DatabaseException("Id has not been created for " + NameUtils.getSafeName(graph, ss.getResource(r)));\r
 149                 }\r
 150 \r
 151         }\r
 152         \r
 153         /*\r
 154          * \r
 155          * @return -2 if r is not really external and the statement should be excluded\r
 156          * \r
 157          */\r
 158         public int getId(ReadGraph graph, int r, Collection<String> errors) throws DatabaseException {\r
 159 \r
 160 //              // First external is root library\r
 161 //              if(r == rootId) return internalCount;\r
 162                 \r
 163                 SerialisationSupport ss = graph.getService(SerialisationSupport.class);\r
 164                 Layer0 L0 = Layer0.getInstance(graph);\r
 165                 \r
 166                 if(state.ids.containsKey(r)) {\r
 167                     int ret = state.ids.get(r);\r
 168                     if(ret == -1) {\r
 169                         for(int i=0;i<=indent;++i)\r
 170                             System.out.print("  ");\r
 171                         System.out.println("Cycle!!!"); // with " + GraphUtils.getReadableName(g, r));\r
 172                     }\r
 173                         return ret;\r
 174                 }\r
 175                 else {\r
 176                         Resource res = getResource(graph, r);\r
 177             if(!validateExternal(res)) {\r
 178                 errors.add("Illegal reference to " + graph.getPossibleURI(getResource(graph, r)));\r
 179                 return -2;\r
 180             }\r
 181                         Collection<Resource> parents = graph.getObjects(res, L0.PartOf);                        \r
 182                         if(parents.size() != 1) {\r
 183                                 throw new ValidationException("Reference to external resource " \r
 184                                                 + NameUtils.getSafeName(graph, getResource(graph, r), true) + " without unique uri (" + parents.size() + " parents).");\r
 185                         }\r
 186                         int pid = 0;\r
 187                         for(Resource p : parents) {\r
 188                             ++indent;\r
 189                             pid = getId(graph, ss.getTransientId(p), errors);\r
 190                             if(pid == -2) {\r
 191                         errors.add("Illegal reference to " + graph.getPossibleURI(getResource(graph, r)));\r
 192                                 return -2;\r
 193                             }\r
 194                         }\r
 195                         --indent;\r
 196                         String name = graph.getRelatedValue(res, L0.HasName);\r
 197                         // Record the external entry\r
 198                         externalParents.add(pid);\r
 199             externalNames.add(name);\r
 200                         state.ids.put(r, state.id);\r
 201                         // Ensure that this resource is included into the set of externals to maintain the total number of externals \r
 202                         state.externals.add(r);\r
 203                         return state.id++;\r
 204                 }\r
 205         }\r
 206         \r
 207         @Override\r
 208         public DataContainer getHeader() throws Exception {\r
 209             return null;\r
 210         }\r
 211         \r
 212         @Override\r
 213         public int getResourceCount() {\r
 214                 return resourceCount;\r
 215         }\r
 216         \r
 217         @Override\r
 218         public int getIdentityCount() {\r
 219                 return configuration.roots.size() + state.externals.size() + 1;\r
 220         }\r
 221         \r
 222         @Override\r
 223         public int getStatementCount() {\r
 224                 return state.statementCount;\r
 225         }\r
 226         \r
 227         @Override\r
 228         public int getValueCount() {\r
 229                 return state.valueCount;\r
 230         }\r
 231 \r
 232         @Override\r
 233         public void forStatements(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {\r
 234                 \r
 235                 int[] value = new int[4];\r
 236                 long length = state.otherStatementsInput.length();\r
 237                 state.otherStatementsInput.position(0);\r
 238                 \r
 239                 while(state.otherStatementsInput.position() < length && !state.monitor.isCanceled()) {\r
 240                         \r
 241                         int s = state.otherStatementsInput.readInt();\r
 242                         int subjectId = state.ids.get(s);\r
 243                         \r
 244                         boolean exclude = subjectId == -1;\r
 245                         \r
 246                         int size = state.otherStatementsInput.readInt();\r
 247                         for(int i=0;i<size;i++) {\r
 248                                 int p = state.otherStatementsInput.readInt();\r
 249                                 int o = state.otherStatementsInput.readInt();\r
 250                                 if(!exclude) {\r
 251                                         if(state.excludedShared.contains(o)) {\r
 252                                                 System.err.println("excluding shared " + s + " " + p + " " + o);\r
 253                                         } else {\r
 254                                                 \r
 255                                                 int objectId = getExistingId(graph, o);\r
 256                                                 // The statement can be denied still\r
 257                                                 if(objectId != -2) {\r
 258                                                         value[0] = subjectId;\r
 259                                                         value[1] = getExistingId(graph, p);\r
 260                                                         int inverse = state.inverses.get(p);\r
 261                                                         if(inverse != 0) {\r
 262                                                                 value[2] = getExistingId(graph, inverse);\r
 263                                                         } else {\r
 264                                                                 value[2] = -1;\r
 265                                                         }\r
 266                                                         value[3] = objectId;\r
 267 \r
 268                                                         procedure.execute(value);\r
 269                                                         \r
 270                                                 } else {\r
 271                                                         System.err.println("Denied (" + NameUtils.getSafeName(graph, getResource(graph, s)) + ", " + NameUtils.getSafeName(graph, getResource(graph, p)) + "," + NameUtils.getSafeName(graph, getResource(graph, o)) + ")");\r
 272                                                 }                                               \r
 273                                                 \r
 274                                         }\r
 275                                 } else {\r
 276                                         System.err.println("excluding shared " + s);\r
 277                                 }\r
 278                         }\r
 279                 }\r
 280         }\r
 281 \r
 282         @Override\r
 283         public void forValues(ReadGraph graph, TransferableGraphSourceProcedure<Value> procedure) throws Exception {\r
 284 \r
 285                 Serializer variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT);\r
 286                 List<Object> idContext = new ArrayList<>();\r
 287                 long length = state.valueInput.length();\r
 288                 state.valueInput.position(0);\r
 289 \r
 290                 while(state.valueInput.position() < length && !state.monitor.isCanceled()) {\r
 291 \r
 292                         // Ignore value type tag\r
 293                         int s = state.valueInput.readInt();\r
 294                         byte valueType = state.valueInput.readByte();\r
 295                         switch (valueType) {\r
 296                         case TAG_RAW_COPY_VARIANT_VALUE:\r
 297                                 state.valueInput.readInt();\r
 298                                 // Intentional fallthrough.\r
 299 \r
 300                         case TAG_POTENTIALLY_MODIFIED_VARIANT_VALUE: {\r
 301                                 idContext.clear();\r
 302                                 Variant variant = (Variant) variantSerializer.deserialize((DataInput)state.valueInput, idContext);\r
 303                                 if (valueModifier.mayNeedModification(variant.type())) {\r
 304                                         Object currentObject = variant.getValue();\r
 305                                         Object newObject = valueModifier.modify(state, variant.getBinding(), currentObject);\r
 306                                         if (newObject != currentObject)\r
 307                                                 variant = new Variant(variant.getBinding(), newObject);\r
 308                                 }\r
 309                                 procedure.execute(new Value(state.ids.get(s), variant));\r
 310                                 break;\r
 311                         }\r
 312 \r
 313                         default:\r
 314                                 throw new IllegalArgumentException("Unrecognized variant value type encountered: " + valueType);\r
 315                         }\r
 316                 }\r
 317         }\r
 318 \r
 319     @Override\r
 320     public void forValues2(ReadGraph graph, TransferableGraphSourceValueProcedure procedure) throws Exception {\r
 321 \r
 322         Serializer datatypeSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.getBindingUnchecked(Datatype.class));\r
 323         List<Object> idContext = new ArrayList<>();\r
 324         long length = state.valueInput.length();\r
 325         state.valueInput.position(0);\r
 326 \r
 327         while(state.valueInput.position() < length && !state.monitor.isCanceled()) {\r
 328 \r
 329             int s = state.valueInput.readInt();\r
 330             byte valueType = state.valueInput.readByte();\r
 331 \r
 332             switch (valueType) {\r
 333             case TAG_RAW_COPY_VARIANT_VALUE: {\r
 334                 // Variant data could be copied raw, no need for modifications\r
 335                 // Note that the variant data is a concatenation of the\r
 336                 // variant datatype serialization and the value serialization.\r
 337                 // variantLength contains both datatype and value data.\r
 338                 int variantLength = state.valueInput.readInt();\r
 339                 procedure.rawCopy(state.ids.get(s), variantLength, state.valueInput);\r
 340                 break;\r
 341             }\r
 342 \r
 343             case TAG_POTENTIALLY_MODIFIED_VARIANT_VALUE: {\r
 344                 // Variant data may need to be modified.\r
 345                 // Cannot optimize this case with raw copying.\r
 346                 idContext.clear();\r
 347                 Datatype type = (Datatype)datatypeSerializer.deserialize((DataInput)state.valueInput, idContext);\r
 348 \r
 349                 if (valueModifier.mayNeedModification(type)) {\r
 350                     Binding binding = Bindings.getBinding(type);\r
 351                     Serializer serializer = Bindings.getSerializerUnchecked(binding);\r
 352                     Object value = serializer.deserialize((DataInput)state.valueInput);\r
 353                     value = valueModifier.modify(state, binding, value);\r
 354                     byte[] bytes = serializer.serialize(value);\r
 355                     procedure.execute(state.ids.get(s), type, new ByteBufferReadable(bytes));\r
 356                 } else {\r
 357                     procedure.execute(state.ids.get(s), type, state.valueInput);\r
 358                 }\r
 359                 break;\r
 360             }\r
 361 \r
 362             default:\r
 363                 throw new IllegalArgumentException("Unrecognized variant value type encountered: " + valueType);\r
 364             }\r
 365         }\r
 366     }\r
 367 \r
 368         protected Identity getRootIdentity(DomainProcessorState state, SerialisationSupport support, Resource rootLibrary) throws DatabaseException {\r
 369                 return new Identity(state.ids.get(support.getTransientId(rootLibrary)), new External(-1, ""));\r
 370         }\r
 371 \r
 372         @Override\r
 373         public void forIdentities(ReadGraph graph, TransferableGraphSourceProcedure<Identity> procedure) throws Exception {\r
 374                 \r
 375                 SerialisationSupport support = graph.getService(SerialisationSupport.class);\r
 376                 Layer0 L0 = Layer0.getInstance(graph);\r
 377 \r
 378                 // TODO: this should be Root with name ""\r
 379                 procedure.execute(getRootIdentity(state, support, graph.getRootLibrary()));\r
 380 \r
 381                 // Declare internal and external roots\r
 382                 for(RootSpec r : configuration.roots) {\r
 383                         Resource type = graph.getPossibleType(r.resource, L0.Entity);\r
 384                         if(type == null) type = L0.Entity;\r
 385                         procedure.execute(new Identity(\r
 386                                         state.ids.get(support.getTransientId(r.resource)),\r
 387                                         new Root(r.name, graph.getURI(type))));\r
 388                 }\r
 389 \r
 390                 for(int i = 0; i < state.externals.size() ; i++) {\r
 391                         int parent = externalParents.get(i);\r
 392                         String name = externalNames.get(i);\r
 393                         procedure.execute(new Identity(externalBase + i, new External(parent,name)));\r
 394         }\r
 395         \r
 396         }\r
 397 \r
 398         @Override\r
 399         public TreeMap<String, Variant> getExtensions() {\r
 400                 return state.extensions;\r
 401         }\r
 402 \r
 403         public File[] getFiles() {\r
 404             return files;\r
 405         }\r
 406 \r
 407         private static <T> T tryClose(T c) throws IOException {\r
 408                 if (c != null && c instanceof Closeable)\r
 409                         ((Closeable) c).close();\r
 410                 return null;\r
 411         }\r
 412 \r
 413         public void closeStreams() throws IOException {\r
 414                 state.valueInput = tryClose(state.valueInput);\r
 415                 state.otherStatementsInput = tryClose(state.otherStatementsInput);\r
 416                 state.statementsOutput = tryClose(state.statementsOutput);\r
 417                 state.valueOutput = tryClose(state.valueOutput);\r
 418         }\r
 419 \r
 420         @Override\r
 421         public void reset() throws Exception {\r
 422             throw new UnsupportedOperationException();\r
 423         }\r
 424         \r
 425         public long[] getResourceArray(ReadGraph graph) throws DatabaseException {\r
 426                 final SerialisationSupport ss = graph.getService(SerialisationSupport.class);\r
 427                 final long[] result = new long[state.ids.size()];\r
 428                 state.ids.forEachEntry(new TIntIntProcedure() {\r
 429 \r
 430                         @Override\r
 431                         public boolean execute(int a, int b) {\r
 432                                 \r
 433                                 try {\r
 434                                         Resource r = ss.getResource(a);\r
 435                                         result[b] = r.getResourceId();\r
 436                                 } catch (DatabaseException e) {\r
 437                                         e.printStackTrace();\r
 438                                 }\r
 439                                 \r
 440                                 return true;\r
 441                                 \r
 442                         }\r
 443                 });\r
 444                 return result;\r
 445         }\r
 446         \r
 447         public DomainProcessorState getState() {\r
 448                 return state;\r
 449         }\r
 450         \r
 451         public void forResourceStatements(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {\r
 452                 \r
 453                 int[] value = new int[4];\r
 454                 long length = state.otherStatementsInput.length();\r
 455                 state.otherStatementsInput.position(0);\r
 456                 \r
 457                 while(state.otherStatementsInput.position() < length) {\r
 458                         \r
 459                         int s = state.otherStatementsInput.readInt();\r
 460                         int subjectId = state.ids.get(s);\r
 461                         \r
 462                         boolean exclude = subjectId == -1;\r
 463                         \r
 464                         int size = state.otherStatementsInput.readInt();\r
 465                         for(int i=0;i<size;i++) {\r
 466                                 int p = state.otherStatementsInput.readInt();\r
 467                                 int o = state.otherStatementsInput.readInt();\r
 468                                 if(!exclude) {\r
 469                                         if(state.excludedShared.contains(o)) {\r
 470                                                 System.err.println("excluding shared " + s + " " + p + " " + o);\r
 471                                         } else {\r
 472                                                 \r
 473                                                 int objectId = getExistingId(graph, o);\r
 474                                                 // The statement can be denied still\r
 475                                                 if(objectId != -2) {\r
 476                                                         value[0] = s;\r
 477                                                         value[1] = p;\r
 478                                                         int inverse = state.inverses.get(p);\r
 479                                                         if(inverse != 0) {\r
 480                                                                 value[2] = inverse;\r
 481                                                         } else {\r
 482                                                                 value[2] = -1;\r
 483                                                         }\r
 484                                                         value[3] = o;\r
 485 \r
 486                                                         procedure.execute(value);\r
 487                                                         \r
 488                                                 } else {\r
 489                                                         System.err.println("Denied (" + NameUtils.getSafeName(graph, getResource(graph, s)) + ", " + NameUtils.getSafeName(graph, getResource(graph, p)) + "," + NameUtils.getSafeName(graph, getResource(graph, o)) + ")");\r
 490                                                 }\r
 491                                                 \r
 492                                         }\r
 493                                 } else {\r
 494                                         System.err.println("excluding shared " + s);\r
 495                                 }\r
 496                         }\r
 497                 }\r
 498         }\r
 499 \r
 500         public void forValueResources(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {\r
 501                 int[] value = { 0 };\r
 502                 long length = state.valueInput.length();\r
 503                 while (state.valueInput.position() < length) {\r
 504                         value[0] = state.valueInput.readInt();\r
 505                         procedure.execute(value);\r
 506                 }\r
 507         }\r
 508 \r
 509         public TransferableGraphConfiguration2 getConfiguration() {\r
 510                 return configuration;\r
 511         }\r
 512 \r
 513         @Override\r
 514         public void close() throws IOException {\r
 515                 synchronized (this) {\r
 516                         if (closed)\r
 517                                 return;\r
 518                         closed = true;\r
 519                 }\r
 520                 closeStreams();\r
 521                 if (files != null) {\r
 522                         for (File f : files) {\r
 523                                 Files.deleteIfExists(f.toPath());\r
 524                         }\r
 525                 }\r
 526         }\r
 527 \r
 528 }