bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/ModelTransferableGraphSource.java

   1 package org.simantics.db.layer0.util;
   2
   3 import java.io.Closeable;
   4 import java.io.DataInput;
   5 import java.io.File;
   6 import java.io.IOException;
   7 import java.nio.file.Files;
   8 import java.util.ArrayList;
   9 import java.util.Collection;
  10 import java.util.Collections;
  11 import java.util.HashSet;
  12 import java.util.List;
  13 import java.util.TreeMap;
  14
  15 import org.simantics.databoard.Bindings;
  16 import org.simantics.databoard.Databoard;
  17 import org.simantics.databoard.binding.Binding;
  18 import org.simantics.databoard.binding.mutable.Variant;
  19 import org.simantics.databoard.container.DataContainer;
  20 import org.simantics.databoard.serialization.Serializer;
  21 import org.simantics.databoard.type.Datatype;
  22 import org.simantics.databoard.util.binary.ByteBufferReadable;
  23 import org.simantics.db.ReadGraph;
  24 import org.simantics.db.Resource;
  25 import org.simantics.db.common.utils.NameUtils;
  26 import org.simantics.db.exception.DatabaseException;
  27 import org.simantics.db.exception.RuntimeDatabaseException;
  28 import org.simantics.db.exception.ValidationException;
  29 import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus;
  30 import org.simantics.db.layer0.util.TransferableGraphConfiguration2.RootSpec;
  31 import org.simantics.db.service.SerialisationSupport;
  32 import org.simantics.graph.db.TransferableGraphSource;
  33 import org.simantics.graph.representation.External;
  34 import org.simantics.graph.representation.Identity;
  35 import org.simantics.graph.representation.Root;
  36 import org.simantics.graph.representation.Value;
  37 import org.simantics.layer0.Layer0;
  38
  39 import gnu.trove.list.array.TIntArrayList;
  40 import gnu.trove.procedure.TIntIntProcedure;
  41
  42 public class ModelTransferableGraphSource implements TransferableGraphSource {
  43
  44         final private TransferableGraphConfiguration2 configuration;
  45         final private DomainProcessorState state;
  46         final private int externalBase;
  47         final private int resourceCount;
  48         final private File[] files;
  49         final private TGValueModifier valueModifier;
  50
  51         private volatile boolean closed = false;
  52
  53         TIntArrayList externalParents = new TIntArrayList();
  54         ArrayList<String> externalNames = new ArrayList<String>();
  55
  56         public ModelTransferableGraphSource(final ReadGraph graph, TransferableGraphConfiguration2 configuration, final DomainProcessorState state, File ... fs) throws DatabaseException {
  57
  58                 this.configuration = configuration;
  59                 this.state = state;
  60                 this.files = fs;
  61                 this.valueModifier = state.valueModifier;
  62
  63                 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
  64
  65                 // At this point ids contains all internal resources. Now add roots and externals.
  66
  67                 // Root Library
  68                 state.ids.put(ss.getTransientId(graph.getRootLibrary()), state.id++);
  69
  70                 // External roots - internal roots were already processed as internal resources by domain processor
  71                 for(RootSpec rs : configuration.roots) {
  72                         if(!rs.internal) {
  73                                 int resourceId = ss.getTransientId(rs.resource);
  74                                 state.ids.put(resourceId, state.id++);
  75                                 // The fixed roots have been seen as externals by domain processor. Now remove them from external set.
  76                                 state.externals.remove(resourceId);
  77                         }
  78                 }
  79
  80                 this.externalBase = state.id;
  81
  82                 final Collection<String> errors = new HashSet<String>();
  83
  84                 // All resource considered as not internal by domain processor. Can also contain roots.
  85                 int[] externals = state.externals.toArray();
  86
  87                 // Build up the state.externals, externalNames and externalParents
  88                 for(int i=0;i<externals.length;i++) {
  89                         getId(graph, externals[i], errors);
  90                 }
  91
  92                 state.inverses.forEachEntry(new TIntIntProcedure() {
  93
  94                         @Override
  95                         public boolean execute(int predicate, int inverse) {
  96                                 try {
  97                                         getId(graph, predicate, errors);
  98                                         if(inverse != 0) getId(graph, inverse, errors);
  99                                 } catch (DatabaseException e) {
 100                                         throw new RuntimeDatabaseException(e);
 101                                 }
 102                                 return true;
 103                         }
 104
 105                 });
 106
 107                 if(!errors.isEmpty()) {
 108                         ArrayList<String> sorted = new ArrayList<String>(errors);
 109                         Collections.sort(sorted);
 110                         StringBuilder message = new StringBuilder();
 111                         message.append("Errors in exported model:\n");
 112                         for(String error : sorted) {
 113                                 message.append(error);
 114                                 message.append("\n");
 115                         }
 116                         throw new DatabaseException(message.toString());
 117                 }
 118
 119                 this.resourceCount = state.id;
 120
 121         }
 122
 123         int indent = 0;
 124
 125         public boolean validateExternal(Resource ext) {
 126                 if(configuration.validate) {
 127                         ExtentStatus status = configuration.preStatus.get(ext);
 128                         if(status != null) {
 129                                 if(ExtentStatus.INTERNAL.equals(status)) return false;
 130                                 else if(ExtentStatus.EXCLUDED.equals(status)) return false;
 131                         }
 132                 }
 133                 return true;
 134         }
 135
 136         private Resource getResource(ReadGraph graph, int r) throws DatabaseException {
 137                 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
 138                 return ss.getResource(r);
 139         }
 140
 141         final public int getExistingId(ReadGraph graph, int r) throws DatabaseException {
 142
 143                 int ret = state.ids.get(r);
 144                 if(ret != -1) {
 145                         return ret;
 146                 } else {
 147                         SerialisationSupport ss = graph.getService(SerialisationSupport.class);
 148                         throw new DatabaseException("Id has not been created for " + NameUtils.getSafeName(graph, ss.getResource(r)));
 149                 }
 150
 151         }
 152
 153         /*
 154          *
 155          * @return -2 if r is not really external and the statement should be excluded
 156          *
 157          */
 158         public int getId(ReadGraph graph, int r, Collection<String> errors) throws DatabaseException {
 159
 160 //              // First external is root library
 161 //              if(r == rootId) return internalCount;
 162
 163                 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
 164                 Layer0 L0 = Layer0.getInstance(graph);
 165
 166                 if(state.ids.containsKey(r)) {
 167                     int ret = state.ids.get(r);
 168                     if(ret == -1) {
 169                         for(int i=0;i<=indent;++i)
 170                             System.out.print("  ");
 171                         System.out.println("Cycle!!!"); // with " + GraphUtils.getReadableName(g, r));
 172                     }
 173                         return ret;
 174                 }
 175                 else {
 176                         Resource res = getResource(graph, r);
 177             if(!validateExternal(res)) {
 178                 errors.add("Illegal reference to " + graph.getPossibleURI(getResource(graph, r)));
 179                 return -2;
 180             }
 181                         Collection<Resource> parents = graph.getObjects(res, L0.PartOf);
 182                         if(parents.size() != 1) {
 183                                 throw new ValidationException("Reference to external resource "
 184                                                 + NameUtils.getSafeName(graph, getResource(graph, r), true) + " without unique uri (" + parents.size() + " parents).");
 185                         }
 186                         int pid = 0;
 187                         for(Resource p : parents) {
 188                             ++indent;
 189                             pid = getId(graph, ss.getTransientId(p), errors);
 190                             if(pid == -2) {
 191                         errors.add("Illegal reference to " + graph.getPossibleURI(getResource(graph, r)));
 192                                 return -2;
 193                             }
 194                         }
 195                         --indent;
 196                         String name = graph.getRelatedValue(res, L0.HasName);
 197                         // Record the external entry
 198                         externalParents.add(pid);
 199             externalNames.add(name);
 200                         state.ids.put(r, state.id);
 201                         // Ensure that this resource is included into the set of externals to maintain the total number of externals
 202                         state.externals.add(r);
 203                         return state.id++;
 204                 }
 205         }
 206
 207         @Override
 208         public DataContainer getHeader() throws Exception {
 209             return null;
 210         }
 211
 212         @Override
 213         public int getResourceCount() {
 214                 return resourceCount;
 215         }
 216
 217         @Override
 218         public int getIdentityCount() {
 219                 return configuration.roots.size() + state.externals.size() + 1;
 220         }
 221
 222         @Override
 223         public int getStatementCount() {
 224                 return state.statementCount;
 225         }
 226
 227         @Override
 228         public int getValueCount() {
 229                 return state.valueCount;
 230         }
 231
 232         @Override
 233         public void forStatements(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
 234
 235                 int[] value = new int[4];
 236                 long length = state.otherStatementsInput.length();
 237                 state.otherStatementsInput.position(0);
 238
 239                 while(state.otherStatementsInput.position() < length && !state.monitor.isCanceled()) {
 240
 241                         int s = state.otherStatementsInput.readInt();
 242                         int subjectId = state.ids.get(s);
 243
 244                         boolean exclude = subjectId == -1;
 245
 246                         int size = state.otherStatementsInput.readInt();
 247                         for(int i=0;i<size;i++) {
 248                                 int p = state.otherStatementsInput.readInt();
 249                                 int o = state.otherStatementsInput.readInt();
 250                                 if(!exclude) {
 251                                         if(state.pending.contains(o)) {
 252                                                 System.err.println("excluding garbage statement " + s + " " + p + " " + o + ", object resource is garbage");
 253                                         } else if(state.excludedShared.contains(o)) {
 254                                                 System.err.println("excluding shared " + s + " " + p + " " + o);
 255                                         } else {
 256
 257                                                 int objectId = getExistingId(graph, o);
 258                                                 // The statement can be denied still
 259                                                 if(objectId != -2) {
 260                                                         value[0] = subjectId;
 261                                                         value[1] = getExistingId(graph, p);
 262                                                         int inverse = state.inverses.get(p);
 263                                                         if(inverse != 0) {
 264                                                                 value[2] = getExistingId(graph, inverse);
 265                                                         } else {
 266                                                                 value[2] = -1;
 267                                                         }
 268                                                         value[3] = objectId;
 269
 270                                                         procedure.execute(value);
 271
 272                                                 } else {
 273                                                         System.err.println("Denied (" + NameUtils.getSafeName(graph, getResource(graph, s)) + ", " + NameUtils.getSafeName(graph, getResource(graph, p)) + "," + NameUtils.getSafeName(graph, getResource(graph, o)) + ")");
 274                                                 }
 275
 276                                         }
 277                                 } else {
 278                                         System.err.println("excluding shared " + s);
 279                                 }
 280                         }
 281                 }
 282         }
 283
 284         @Override
 285         public void forValues(ReadGraph graph, TransferableGraphSourceProcedure<Value> procedure) throws Exception {
 286
 287                 Serializer variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT);
 288                 List<Object> idContext = new ArrayList<>();
 289                 long length = state.valueInput.length();
 290                 state.valueInput.position(0);
 291
 292                 while(state.valueInput.position() < length && !state.monitor.isCanceled()) {
 293
 294                         // Ignore value type tag
 295                         int s = state.valueInput.readInt();
 296                         byte valueType = state.valueInput.readByte();
 297                         switch (valueType) {
 298                         case TAG_RAW_COPY_VARIANT_VALUE:
 299                                 state.valueInput.readInt();
 300                                 // Intentional fallthrough.
 301
 302                         case TAG_POTENTIALLY_MODIFIED_VARIANT_VALUE: {
 303                                 idContext.clear();
 304                                 Variant variant = (Variant) variantSerializer.deserialize((DataInput)state.valueInput, idContext);
 305                                 if (valueModifier.mayNeedModification(variant.type())) {
 306                                         Object currentObject = variant.getValue();
 307                                         Object newObject = valueModifier.modify(state, variant.getBinding(), currentObject);
 308                                         if (newObject != currentObject)
 309                                                 variant = new Variant(variant.getBinding(), newObject);
 310                                 }
 311                                 procedure.execute(new Value(state.ids.get(s), variant));
 312                                 break;
 313                         }
 314
 315                         default:
 316                                 throw new IllegalArgumentException("Unrecognized variant value type encountered: " + valueType);
 317                         }
 318                 }
 319         }
 320
 321     @Override
 322     public void forValues2(ReadGraph graph, TransferableGraphSourceValueProcedure procedure) throws Exception {
 323
 324         Serializer datatypeSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.getBindingUnchecked(Datatype.class));
 325         List<Object> idContext = new ArrayList<>();
 326         long length = state.valueInput.length();
 327         state.valueInput.position(0);
 328
 329         while(state.valueInput.position() < length && !state.monitor.isCanceled()) {
 330
 331             int s = state.valueInput.readInt();
 332             byte valueType = state.valueInput.readByte();
 333
 334             switch (valueType) {
 335             case TAG_RAW_COPY_VARIANT_VALUE: {
 336                 // Variant data could be copied raw, no need for modifications
 337                 // Note that the variant data is a concatenation of the
 338                 // variant datatype serialization and the value serialization.
 339                 // variantLength contains both datatype and value data.
 340                 int variantLength = state.valueInput.readInt();
 341                 procedure.rawCopy(state.ids.get(s), variantLength, state.valueInput);
 342                 break;
 343             }
 344
 345             case TAG_POTENTIALLY_MODIFIED_VARIANT_VALUE: {
 346                 // Variant data may need to be modified.
 347                 // Cannot optimize this case with raw copying.
 348                 idContext.clear();
 349                 Datatype type = (Datatype)datatypeSerializer.deserialize((DataInput)state.valueInput, idContext);
 350
 351                 if (valueModifier.mayNeedModification(type)) {
 352                     Binding binding = Bindings.getBinding(type);
 353                     Serializer serializer = Bindings.getSerializerUnchecked(binding);
 354                     Object value = serializer.deserialize((DataInput)state.valueInput);
 355                     value = valueModifier.modify(state, binding, value);
 356                     byte[] bytes = serializer.serialize(value);
 357                     procedure.execute(state.ids.get(s), type, new ByteBufferReadable(bytes));
 358                 } else {
 359                     procedure.execute(state.ids.get(s), type, state.valueInput);
 360                 }
 361                 break;
 362             }
 363
 364             default:
 365                 throw new IllegalArgumentException("Unrecognized variant value type encountered: " + valueType);
 366             }
 367         }
 368     }
 369
 370         protected Identity getRootIdentity(DomainProcessorState state, SerialisationSupport support, Resource rootLibrary) throws DatabaseException {
 371                 return new Identity(state.ids.get(support.getTransientId(rootLibrary)), new External(-1, ""));
 372         }
 373
 374         @Override
 375         public void forIdentities(ReadGraph graph, TransferableGraphSourceProcedure<Identity> procedure) throws Exception {
 376
 377                 SerialisationSupport support = graph.getService(SerialisationSupport.class);
 378                 Layer0 L0 = Layer0.getInstance(graph);
 379
 380                 // TODO: this should be Root with name ""
 381                 procedure.execute(getRootIdentity(state, support, graph.getRootLibrary()));
 382
 383                 // Declare internal and external roots
 384                 for(RootSpec r : configuration.roots) {
 385                         Resource type = graph.getPossibleType(r.resource, L0.Entity);
 386                         if(type == null) type = L0.Entity;
 387                         procedure.execute(new Identity(
 388                                         state.ids.get(support.getTransientId(r.resource)),
 389                                         new Root(r.name, graph.getURI(type))));
 390                 }
 391
 392                 for(int i = 0; i < state.externals.size() ; i++) {
 393                         int parent = externalParents.get(i);
 394                         String name = externalNames.get(i);
 395                         procedure.execute(new Identity(externalBase + i, new External(parent,name)));
 396         }
 397
 398         }
 399
 400         @Override
 401         public TreeMap<String, Variant> getExtensions() {
 402                 return state.extensions;
 403         }
 404
 405         public File[] getFiles() {
 406             return files;
 407         }
 408
 409         private static <T> T tryClose(T c) throws IOException {
 410                 if (c != null && c instanceof Closeable)
 411                         ((Closeable) c).close();
 412                 return null;
 413         }
 414
 415         public void closeStreams() throws IOException {
 416                 state.valueInput = tryClose(state.valueInput);
 417                 state.otherStatementsInput = tryClose(state.otherStatementsInput);
 418                 state.statementsOutput = tryClose(state.statementsOutput);
 419                 state.valueOutput = tryClose(state.valueOutput);
 420         }
 421
 422         @Override
 423         public void reset() throws Exception {
 424             throw new UnsupportedOperationException();
 425         }
 426
 427         public long[] getResourceArray(ReadGraph graph) throws DatabaseException {
 428                 final SerialisationSupport ss = graph.getService(SerialisationSupport.class);
 429                 final long[] result = new long[state.ids.size()];
 430                 state.ids.forEachEntry(new TIntIntProcedure() {
 431
 432                         @Override
 433                         public boolean execute(int a, int b) {
 434
 435                                 try {
 436                                         Resource r = ss.getResource(a);
 437                                         result[b] = r.getResourceId();
 438                                 } catch (DatabaseException e) {
 439                                         e.printStackTrace();
 440                                 }
 441
 442                                 return true;
 443
 444                         }
 445                 });
 446                 return result;
 447         }
 448
 449         public DomainProcessorState getState() {
 450                 return state;
 451         }
 452
 453         public void forResourceStatements(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
 454
 455                 int[] value = new int[4];
 456                 long length = state.otherStatementsInput.length();
 457                 state.otherStatementsInput.position(0);
 458
 459                 while(state.otherStatementsInput.position() < length) {
 460
 461                         int s = state.otherStatementsInput.readInt();
 462                         int subjectId = state.ids.get(s);
 463
 464                         boolean exclude = subjectId == -1;
 465
 466                         int size = state.otherStatementsInput.readInt();
 467                         for(int i=0;i<size;i++) {
 468                                 int p = state.otherStatementsInput.readInt();
 469                                 int o = state.otherStatementsInput.readInt();
 470                                 if(!exclude) {
 471                                         if(state.excludedShared.contains(o)) {
 472                                                 System.err.println("excluding shared " + s + " " + p + " " + o);
 473                                         } else {
 474
 475                                                 int objectId = getExistingId(graph, o);
 476                                                 // The statement can be denied still
 477                                                 if(objectId != -2) {
 478                                                         value[0] = s;
 479                                                         value[1] = p;
 480                                                         int inverse = state.inverses.get(p);
 481                                                         if(inverse != 0) {
 482                                                                 value[2] = inverse;
 483                                                         } else {
 484                                                                 value[2] = -1;
 485                                                         }
 486                                                         value[3] = o;
 487
 488                                                         procedure.execute(value);
 489
 490                                                 } else {
 491                                                         System.err.println("Denied (" + NameUtils.getSafeName(graph, getResource(graph, s)) + ", " + NameUtils.getSafeName(graph, getResource(graph, p)) + "," + NameUtils.getSafeName(graph, getResource(graph, o)) + ")");
 492                                                 }
 493
 494                                         }
 495                                 } else {
 496                                         System.err.println("excluding shared " + s);
 497                                 }
 498                         }
 499                 }
 500         }
 501
 502         public void forValueResources(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
 503                 int[] value = { 0 };
 504                 long length = state.valueInput.length();
 505                 while (state.valueInput.position() < length) {
 506                         value[0] = state.valueInput.readInt();
 507                         procedure.execute(value);
 508                 }
 509         }
 510
 511         public TransferableGraphConfiguration2 getConfiguration() {
 512                 return configuration;
 513         }
 514
 515         @Override
 516         public void close() throws IOException {
 517                 synchronized (this) {
 518                         if (closed)
 519                                 return;
 520                         closed = true;
 521                 }
 522                 closeStreams();
 523                 if (files != null) {
 524                         for (File f : files) {
 525                                 Files.deleteIfExists(f.toPath());
 526                         }
 527                 }
 528         }
 529
 530 }