bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/ModelTransferableGraphSource.java

   1 package org.simantics.db.layer0.util;
   2
   3 import java.io.Closeable;
   4 import java.io.DataInput;
   5 import java.io.File;
   6 import java.io.IOException;
   7 import java.nio.file.Files;
   8 import java.util.ArrayList;
   9 import java.util.Collection;
  10 import java.util.Collections;
  11 import java.util.HashSet;
  12 import java.util.List;
  13 import java.util.TreeMap;
  14
  15 import org.simantics.databoard.Bindings;
  16 import org.simantics.databoard.Databoard;
  17 import org.simantics.databoard.binding.Binding;
  18 import org.simantics.databoard.binding.mutable.Variant;
  19 import org.simantics.databoard.container.DataContainer;
  20 import org.simantics.databoard.serialization.Serializer;
  21 import org.simantics.databoard.type.Datatype;
  22 import org.simantics.databoard.util.binary.ByteBufferReadable;
  23 import org.simantics.db.ReadGraph;
  24 import org.simantics.db.Resource;
  25 import org.simantics.db.common.utils.NameUtils;
  26 import org.simantics.db.exception.DatabaseException;
  27 import org.simantics.db.exception.RuntimeDatabaseException;
  28 import org.simantics.db.exception.ValidationException;
  29 import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus;
  30 import org.simantics.db.layer0.util.ConsistsOfProcess.InternalEntry;
  31 import org.simantics.db.layer0.util.TransferableGraphConfiguration2.RootSpec;
  32 import org.simantics.db.service.SerialisationSupport;
  33 import org.simantics.graph.db.TransferableGraphSource;
  34 import org.simantics.graph.representation.External;
  35 import org.simantics.graph.representation.Identity;
  36 import org.simantics.graph.representation.Internal;
  37 import org.simantics.graph.representation.Root;
  38 import org.simantics.graph.representation.Value;
  39 import org.simantics.layer0.Layer0;
  40
  41 import gnu.trove.list.array.TIntArrayList;
  42 import gnu.trove.map.TIntObjectMap;
  43 import gnu.trove.map.hash.TIntObjectHashMap;
  44 import gnu.trove.procedure.TIntIntProcedure;
  45
  46 public class ModelTransferableGraphSource implements TransferableGraphSource {
  47
  48         final private TransferableGraphConfiguration2 configuration;
  49         final private DomainProcessorState state;
  50         final private int externalBase;
  51         final private int resourceCount;
  52         final private File[] files;
  53         final private TGValueModifier valueModifier;
  54
  55         private volatile boolean closed = false;
  56
  57         TIntArrayList externalParents = new TIntArrayList();
  58         ArrayList<String> externalNames = new ArrayList<>();
  59         TreeMap<String,String> downloads = new TreeMap<String,String>();
  60
  61         public ModelTransferableGraphSource(final ReadGraph graph, TransferableGraphConfiguration2 configuration, final DomainProcessorState state, File ... fs) throws DatabaseException {
  62
  63                 this.configuration = configuration;
  64                 this.state = state;
  65                 this.files = fs;
  66                 this.valueModifier = state.valueModifier;
  67
  68                 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
  69
  70                 // At this point ids contains all internal resources. Now add roots and externals.
  71
  72                 // Root Library
  73                 state.ids.put(ss.getTransientId(graph.getRootLibrary()), state.id++);
  74
  75                 // External roots - internal roots were already processed as internal resources by domain processor
  76                 for(RootSpec rs : configuration.roots) {
  77                         if(!rs.internal) {
  78                                 int resourceId = ss.getTransientId(rs.resource);
  79                                 state.ids.put(resourceId, state.id++);
  80                                 // The fixed roots have been seen as externals by domain processor. Now remove them from external set.
  81                                 state.externals.remove(resourceId);
  82                         }
  83                 }
  84
  85                 this.externalBase = state.id;
  86
  87                 final Collection<String> errors = new HashSet<>();
  88
  89                 // All resource considered as not internal by domain processor. Can also contain roots.
  90                 int[] externals = state.externals.toArray();
  91
  92                 // Build up the state.externals, externalNames and externalParents
  93                 for(int i=0;i<externals.length;i++) {
  94                         getId(graph, externals[i], errors);
  95                 }
  96
  97                 state.inverses.forEachEntry(new TIntIntProcedure() {
  98
  99                         @Override
 100                         public boolean execute(int predicate, int inverse) {
 101                                 try {
 102                                         getId(graph, predicate, errors);
 103                                         if(inverse != 0) getId(graph, inverse, errors);
 104                                 } catch (DatabaseException e) {
 105                                         throw new RuntimeDatabaseException(e);
 106                                 }
 107                                 return true;
 108                         }
 109
 110                 });
 111
 112                 if(!errors.isEmpty()) {
 113                         ArrayList<String> sorted = new ArrayList<>(errors);
 114                         Collections.sort(sorted);
 115                         StringBuilder message = new StringBuilder();
 116                         message.append("Errors in exported model:\n");
 117                         for(String error : sorted) {
 118                                 message.append(error);
 119                                 message.append("\n");
 120                         }
 121                         throw new DatabaseException(message.toString());
 122                 }
 123
 124                 this.resourceCount = state.id;
 125
 126                 state.extensions.put(ExternalDownloadBean.EXTENSION_KEY, new Variant(ExternalDownloadBean.BINDING, new ExternalDownloadBean(downloads)));
 127
 128         }
 129
 130         int indent = 0;
 131
 132         public boolean validateExternal(Resource ext) {
 133                 if(configuration.validate) {
 134                         ExtentStatus status = configuration.preStatus.get(ext);
 135                         if(status != null) {
 136                                 if(ExtentStatus.INTERNAL.equals(status)) return false;
 137                                 else if(ExtentStatus.EXCLUDED.equals(status)) return false;
 138                         }
 139                 }
 140                 return true;
 141         }
 142
 143         private Resource getResource(ReadGraph graph, int r) throws DatabaseException {
 144                 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
 145                 return ss.getResource(r);
 146         }
 147
 148         final public int getExistingId(ReadGraph graph, int r) throws DatabaseException {
 149
 150                 int ret = state.ids.get(r);
 151                 if(ret != -1) {
 152                         return ret;
 153                 } else {
 154                         SerialisationSupport ss = graph.getService(SerialisationSupport.class);
 155                         throw new DatabaseException("Id has not been created for " + NameUtils.getSafeName(graph, ss.getResource(r)));
 156                 }
 157
 158         }
 159
 160         /*
 161          *
 162          * @return -2 if r is not really external and the statement should be excluded
 163          *
 164          */
 165         public int getId(ReadGraph graph, int r, Collection<String> errors) throws DatabaseException {
 166
 167 //              // First external is root library
 168 //              if(r == rootId) return internalCount;
 169
 170                 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
 171                 Layer0 L0 = Layer0.getInstance(graph);
 172
 173                 if(state.ids.containsKey(r)) {
 174                     int ret = state.ids.get(r);
 175                     if(ret == -1) {
 176                         for(int i=0;i<=indent;++i)
 177                             System.out.print("  ");
 178                         System.out.println("Cycle!!!"); // with " + GraphUtils.getReadableName(g, r));
 179                     }
 180                         return ret;
 181                 }
 182                 else {
 183                         Resource res = getResource(graph, r);
 184             if(!validateExternal(res)) {
 185                 errors.add("Illegal reference to " + graph.getPossibleURI(getResource(graph, r)));
 186                 return -2;
 187             }
 188                         Collection<Resource> parents = graph.getObjects(res, L0.PartOf);
 189                         if(parents.size() != 1) {
 190                                 throw new ValidationException("Reference to external resource "
 191                                                 + NameUtils.getSafeName(graph, getResource(graph, r), true) + " without unique uri (" + parents.size() + " parents).");
 192                         }
 193                         int pid = 0;
 194                         for(Resource p : parents) {
 195                             ++indent;
 196                             pid = getId(graph, ss.getTransientId(p), errors);
 197                             if(pid == -2) {
 198                         errors.add("Illegal reference to " + graph.getPossibleURI(getResource(graph, r)));
 199                                 return -2;
 200                             }
 201                         }
 202                         --indent;
 203                         String name = graph.getRelatedValue(res, L0.HasName);
 204                         // Record the external entry
 205                         externalParents.add(pid);
 206             externalNames.add(name);
 207                         state.ids.put(r, state.id);
 208                         // Ensure that this resource is included into the set of externals to maintain the total number of externals
 209                         state.externals.add(r);
 210                         String download = graph.getPossibleRelatedValue(res, L0.Ontology_download, Bindings.STRING);
 211                         if(download != null) {
 212                                 String uri = graph.getURI(res);
 213                                 downloads.put(uri, download);
 214                         }
 215                         return state.id++;
 216                 }
 217         }
 218
 219         @Override
 220         public DataContainer getHeader() throws Exception {
 221             return null;
 222         }
 223
 224         @Override
 225         public int getResourceCount() {
 226                 return resourceCount;
 227         }
 228
 229         @Override
 230         public int getIdentityCount() {
 231                 return configuration.roots.size() + state.externals.size() + state.internalEntries.size() + 1;
 232         }
 233
 234         @Override
 235         public int getStatementCount() {
 236                 return state.statementCount;
 237         }
 238
 239         @Override
 240         public int getValueCount() {
 241                 return state.valueCount;
 242         }
 243
 244         @Override
 245         public void forStatements(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
 246
 247                 int[] value = new int[4];
 248                 long length = state.otherStatementsInput.length();
 249                 state.otherStatementsInput.position(0);
 250
 251                 while(state.otherStatementsInput.position() < length && !state.monitor.isCanceled()) {
 252
 253                         int s = state.otherStatementsInput.readInt();
 254                         int subjectId = state.ids.get(s);
 255
 256                         boolean exclude = subjectId == -1;
 257
 258                         int size = state.otherStatementsInput.readInt();
 259                         for(int i=0;i<size;i++) {
 260                                 int p = state.otherStatementsInput.readInt();
 261                                 int o = state.otherStatementsInput.readInt();
 262                                 if(!exclude) {
 263                                         if(state.pending.contains(o)) {
 264                                                 System.err.println("excluding garbage statement " + s + " " + p + " " + o + ", object resource is garbage");
 265                                         } else if(state.excludedShared.contains(o)) {
 266                                                 System.err.println("excluding shared " + s + " " + p + " " + o);
 267                                         } else {
 268
 269                                                 int objectId = getExistingId(graph, o);
 270                                                 // The statement can be denied still
 271                                                 if(objectId != -2) {
 272                                                         value[0] = subjectId;
 273                                                         value[1] = getExistingId(graph, p);
 274                                                         int inverse = state.inverses.get(p);
 275                                                         if(inverse != 0) {
 276                                                                 value[2] = getExistingId(graph, inverse);
 277                                                         } else {
 278                                                                 value[2] = -1;
 279                                                         }
 280                                                         value[3] = objectId;
 281
 282                                                         procedure.execute(value);
 283
 284                                                 } else {
 285                                                         System.err.println("Denied (" + NameUtils.getSafeName(graph, getResource(graph, s)) + ", " + NameUtils.getSafeName(graph, getResource(graph, p)) + "," + NameUtils.getSafeName(graph, getResource(graph, o)) + ")");
 286                                                 }
 287
 288                                         }
 289                                 } else {
 290                                         System.err.println("excluding shared " + s);
 291                                 }
 292                         }
 293                 }
 294         }
 295
 296         @Override
 297         public void forValues(ReadGraph graph, TransferableGraphSourceProcedure<Value> procedure) throws Exception {
 298
 299                 Serializer variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT);
 300                 List<Object> idContext = new ArrayList<>();
 301                 long length = state.valueInput.length();
 302                 state.valueInput.position(0);
 303
 304                 while(state.valueInput.position() < length && !state.monitor.isCanceled()) {
 305
 306                         // Ignore value type tag
 307                         int s = state.valueInput.readInt();
 308                         byte valueType = state.valueInput.readByte();
 309                         switch (valueType) {
 310                         case TAG_RAW_COPY_VARIANT_VALUE:
 311                                 state.valueInput.readInt();
 312                                 // Intentional fallthrough.
 313
 314                         case TAG_POTENTIALLY_MODIFIED_VARIANT_VALUE: {
 315                                 idContext.clear();
 316                                 Variant variant = (Variant) variantSerializer.deserialize((DataInput)state.valueInput, idContext);
 317                                 if (valueModifier.mayNeedModification(variant.type())) {
 318                                         Object currentObject = variant.getValue();
 319                                         Object newObject = valueModifier.modify(state, variant.getBinding(), currentObject);
 320                                         if (newObject != currentObject)
 321                                                 variant = new Variant(variant.getBinding(), newObject);
 322                                 }
 323                                 procedure.execute(new Value(state.ids.get(s), variant));
 324                                 break;
 325                         }
 326
 327                         default:
 328                                 throw new IllegalArgumentException("Unrecognized variant value type encountered: " + valueType);
 329                         }
 330                 }
 331         }
 332
 333     @Override
 334     public void forValues2(ReadGraph graph, TransferableGraphSourceValueProcedure procedure) throws Exception {
 335
 336         Serializer datatypeSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.getBindingUnchecked(Datatype.class));
 337         List<Object> idContext = new ArrayList<>();
 338         long length = state.valueInput.length();
 339         state.valueInput.position(0);
 340
 341         while(state.valueInput.position() < length && !state.monitor.isCanceled()) {
 342
 343             int s = state.valueInput.readInt();
 344             byte valueType = state.valueInput.readByte();
 345
 346             switch (valueType) {
 347             case TAG_RAW_COPY_VARIANT_VALUE: {
 348                 // Variant data could be copied raw, no need for modifications
 349                 // Note that the variant data is a concatenation of the
 350                 // variant datatype serialization and the value serialization.
 351                 // variantLength contains both datatype and value data.
 352                 int variantLength = state.valueInput.readInt();
 353                 procedure.rawCopy(state.ids.get(s), variantLength, state.valueInput);
 354                 break;
 355             }
 356
 357             case TAG_POTENTIALLY_MODIFIED_VARIANT_VALUE: {
 358                 // Variant data may need to be modified.
 359                 // Cannot optimize this case with raw copying.
 360                 idContext.clear();
 361                 Datatype type = (Datatype)datatypeSerializer.deserialize((DataInput)state.valueInput, idContext);
 362
 363                 if (valueModifier.mayNeedModification(type)) {
 364                     Binding binding = Bindings.getBinding(type);
 365                     Serializer serializer = Bindings.getSerializerUnchecked(binding);
 366                     Object value = serializer.deserialize((DataInput)state.valueInput);
 367                     value = valueModifier.modify(state, binding, value);
 368                     byte[] bytes = serializer.serialize(value);
 369                     procedure.execute(state.ids.get(s), type, new ByteBufferReadable(bytes));
 370                 } else {
 371                     procedure.execute(state.ids.get(s), type, state.valueInput);
 372                 }
 373                 break;
 374             }
 375
 376             default:
 377                 throw new IllegalArgumentException("Unrecognized variant value type encountered: " + valueType);
 378             }
 379         }
 380     }
 381
 382         protected Identity getRootIdentity(DomainProcessorState state, SerialisationSupport support, Resource rootLibrary) throws DatabaseException {
 383                 return new Identity(state.ids.get(support.getTransientId(rootLibrary)), new External(-1, ""));
 384         }
 385
 386         @Override
 387         public void forIdentities(ReadGraph graph, TransferableGraphSourceProcedure<Identity> procedure) throws Exception {
 388
 389                 SerialisationSupport support = graph.getService(SerialisationSupport.class);
 390                 Layer0 L0 = Layer0.getInstance(graph);
 391
 392                 // TODO: this should be Root with name ""
 393                 procedure.execute(getRootIdentity(state, support, graph.getRootLibrary()));
 394
 395                 TIntObjectMap<Identity> internalMap = new TIntObjectHashMap<>(100, 0.5f, Integer.MIN_VALUE);
 396
 397                 // Declare internal and external roots
 398                 for(RootSpec r : configuration.roots) {
 399                         String typeId = r.type;
 400                         if (typeId == null) {
 401                                 Resource type = graph.getPossibleType(r.resource, L0.Entity);
 402                                 typeId = type != null ? graph.getURI(type) : Layer0.URIs.Entity;
 403                         }
 404                         int id = state.ids.get(support.getTransientId(r.resource));
 405                         Root root = new Root(r.name, typeId);
 406                         Identity rootId = new Identity(id,root);
 407                         internalMap.put(id, rootId);
 408                         procedure.execute(rootId);
 409                 }
 410
 411                 for(int i = 0; i < state.externals.size() ; i++) {
 412                         int parent = externalParents.get(i);
 413                         String name = externalNames.get(i);
 414                         procedure.execute(new Identity(externalBase + i, new External(parent,name)));
 415         }
 416
 417                 if(state.internalEntries != null) {
 418                         for(InternalEntry ie : state.internalEntries) {
 419                             if(ie.parent != null) {
 420                                 if(ie.name != null) {
 421                                     procedure.execute(resolveInternal(graph, support, ie, internalMap));
 422                             } else {
 423                                 // In this case there is a child that has no HasName => this should be treated as a blank
 424                                 }
 425                                 } else {
 426                                         throw new DatabaseException("Invalid internal entry " + ie);
 427                                 }
 428                         }
 429                 }
 430
 431         }
 432
 433         private Identity resolveInternal(ReadGraph graph, SerialisationSupport ss, InternalEntry entry, TIntObjectMap<Identity> internalMap) throws DatabaseException {
 434                 int id = state.ids.get(ss.getTransientId(entry.resource));
 435                 Identity existing = internalMap.get(id);
 436                 if(existing != null) return existing;
 437                 Identity parent = resolveInternal(graph, ss, entry.parent, internalMap);
 438                 Identity result = new Identity(id,
 439                                 new Internal(parent.resource, entry.name));
 440                 internalMap.put(id, result);
 441                 return result;
 442         }
 443
 444         @Override
 445         public TreeMap<String, Variant> getExtensions() {
 446                 return state.extensions;
 447         }
 448
 449         public File[] getFiles() {
 450             return files;
 451         }
 452
 453         private static <T> T tryClose(T c) throws IOException {
 454                 if (c != null && c instanceof Closeable)
 455                         ((Closeable) c).close();
 456                 return null;
 457         }
 458
 459         public void closeStreams() throws IOException {
 460                 state.valueInput = tryClose(state.valueInput);
 461                 state.otherStatementsInput = tryClose(state.otherStatementsInput);
 462                 state.statementsOutput = tryClose(state.statementsOutput);
 463                 state.valueOutput = tryClose(state.valueOutput);
 464         }
 465
 466         @Override
 467         public void reset() throws Exception {
 468             throw new UnsupportedOperationException();
 469         }
 470
 471         public long[] getResourceArray(ReadGraph graph) throws DatabaseException {
 472                 final SerialisationSupport ss = graph.getService(SerialisationSupport.class);
 473                 final long[] result = new long[state.ids.size()];
 474                 state.ids.forEachEntry(new TIntIntProcedure() {
 475
 476                         @Override
 477                         public boolean execute(int a, int b) {
 478
 479                                 try {
 480                                         Resource r = ss.getResource(a);
 481                                         result[b] = r.getResourceId();
 482                                 } catch (DatabaseException e) {
 483                                         e.printStackTrace();
 484                                 }
 485
 486                                 return true;
 487
 488                         }
 489                 });
 490                 return result;
 491         }
 492
 493         public DomainProcessorState getState() {
 494                 return state;
 495         }
 496
 497         public void forResourceStatements(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
 498
 499                 int[] value = new int[4];
 500                 long length = state.otherStatementsInput.length();
 501                 state.otherStatementsInput.position(0);
 502
 503                 while(state.otherStatementsInput.position() < length) {
 504
 505                         int s = state.otherStatementsInput.readInt();
 506                         int subjectId = state.ids.get(s);
 507
 508                         boolean exclude = subjectId == -1;
 509
 510                         int size = state.otherStatementsInput.readInt();
 511                         for(int i=0;i<size;i++) {
 512                                 int p = state.otherStatementsInput.readInt();
 513                                 int o = state.otherStatementsInput.readInt();
 514                                 if(!exclude) {
 515                                         if(state.excludedShared.contains(o)) {
 516                                                 System.err.println("excluding shared " + s + " " + p + " " + o);
 517                                         } else {
 518
 519                                                 int objectId = getExistingId(graph, o);
 520                                                 // The statement can be denied still
 521                                                 if(objectId != -2) {
 522                                                         value[0] = s;
 523                                                         value[1] = p;
 524                                                         int inverse = state.inverses.get(p);
 525                                                         if(inverse != 0) {
 526                                                                 value[2] = inverse;
 527                                                         } else {
 528                                                                 value[2] = -1;
 529                                                         }
 530                                                         value[3] = o;
 531
 532                                                         procedure.execute(value);
 533
 534                                                 } else {
 535                                                         System.err.println("Denied (" + NameUtils.getSafeName(graph, getResource(graph, s)) + ", " + NameUtils.getSafeName(graph, getResource(graph, p)) + "," + NameUtils.getSafeName(graph, getResource(graph, o)) + ")");
 536                                                 }
 537
 538                                         }
 539                                 } else {
 540                                         System.err.println("excluding shared " + s);
 541                                 }
 542                         }
 543                 }
 544         }
 545
 546         public void forValueResources(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
 547                 int[] value = { 0 };
 548                 long length = state.valueInput.length();
 549                 while (state.valueInput.position() < length) {
 550                         value[0] = state.valueInput.readInt();
 551                         procedure.execute(value);
 552                 }
 553         }
 554
 555         public TransferableGraphConfiguration2 getConfiguration() {
 556                 return configuration;
 557         }
 558
 559         @Override
 560         public void close() throws IOException {
 561                 synchronized (this) {
 562                         if (closed)
 563                                 return;
 564                         closed = true;
 565                 }
 566                 closeStreams();
 567                 if (files != null) {
 568                         for (File f : files) {
 569                                 Files.deleteIfExists(f.toPath());
 570                         }
 571                 }
 572         }
 573
 574 }