bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/ModelTransferableGraphSource.java

   1 package org.simantics.db.layer0.util;
   2
   3 import java.io.Closeable;
   4 import java.io.DataInput;
   5 import java.io.File;
   6 import java.io.IOException;
   7 import java.nio.file.Files;
   8 import java.util.ArrayList;
   9 import java.util.Collection;
  10 import java.util.Collections;
  11 import java.util.HashSet;
  12 import java.util.List;
  13 import java.util.TreeMap;
  14
  15 import org.simantics.databoard.Bindings;
  16 import org.simantics.databoard.Databoard;
  17 import org.simantics.databoard.binding.Binding;
  18 import org.simantics.databoard.binding.mutable.Variant;
  19 import org.simantics.databoard.container.DataContainer;
  20 import org.simantics.databoard.serialization.Serializer;
  21 import org.simantics.databoard.type.Datatype;
  22 import org.simantics.databoard.util.binary.ByteBufferReadable;
  23 import org.simantics.db.ReadGraph;
  24 import org.simantics.db.Resource;
  25 import org.simantics.db.common.utils.NameUtils;
  26 import org.simantics.db.exception.DatabaseException;
  27 import org.simantics.db.exception.RuntimeDatabaseException;
  28 import org.simantics.db.exception.ValidationException;
  29 import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus;
  30 import org.simantics.db.layer0.util.ConsistsOfProcess.ConsistsOfProcessEntry;
  31 import org.simantics.db.layer0.util.TransferableGraphConfiguration2.SeedSpec;
  32 import org.simantics.db.layer0.util.TransferableGraphConfiguration2.SeedSpec.SeedSpecType;
  33 import org.simantics.db.service.SerialisationSupport;
  34 import org.simantics.graph.db.TransferableGraphSource;
  35 import org.simantics.graph.representation.External;
  36 import org.simantics.graph.representation.Identity;
  37 import org.simantics.graph.representation.Internal;
  38 import org.simantics.graph.representation.Root;
  39 import org.simantics.graph.representation.Value;
  40 import org.simantics.layer0.Layer0;
  41
  42 import gnu.trove.list.array.TIntArrayList;
  43 import gnu.trove.map.TIntObjectMap;
  44 import gnu.trove.map.hash.TIntObjectHashMap;
  45 import gnu.trove.procedure.TIntIntProcedure;
  46
  47 public class ModelTransferableGraphSource implements TransferableGraphSource {
  48
  49         final private TransferableGraphConfiguration2 configuration;
  50         final private DomainProcessorState state;
  51         final private int externalBase;
  52         final private int resourceCount;
  53         final private File[] files;
  54         final private TGValueModifier valueModifier;
  55
  56         private volatile boolean closed = false;
  57
  58         TIntArrayList externalParents = new TIntArrayList();
  59         ArrayList<String> externalNames = new ArrayList<>();
  60         TreeMap<String,String> downloads = new TreeMap<String,String>();
  61
  62         public ModelTransferableGraphSource(final ReadGraph graph, TransferableGraphConfiguration2 configuration, final DomainProcessorState state, File ... fs) throws DatabaseException {
  63
  64                 this.configuration = configuration;
  65                 this.state = state;
  66                 this.files = fs;
  67                 this.valueModifier = state.valueModifier;
  68
  69                 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
  70
  71                 // At this point ids contains all internal resources. Now add roots and externals.
  72
  73                 // Root Library
  74                 state.ids.put(ss.getTransientId(graph.getRootLibrary()), state.id++);
  75
  76                 // External roots - internal roots were already processed as internal resources by domain processor
  77                 for(SeedSpec spec : configuration.seeds) {
  78                         if(SeedSpecType.SPECIAL_ROOT.equals(spec.specType)) {
  79                                 int resourceId = ss.getTransientId(spec.resource);
  80                                 state.ids.put(resourceId, state.id++);
  81                                 // The fixed roots have been seen as externals by domain processor. Now remove them from external set.
  82                                 state.externals.remove(resourceId);
  83                         }
  84                 }
  85
  86                 this.externalBase = state.id;
  87
  88                 final Collection<String> errors = new HashSet<>();
  89
  90                 // All resource considered as not internal by domain processor. Can also contain roots.
  91                 int[] externals = state.externals.toArray();
  92
  93                 // Build up the state.externals, externalNames and externalParents
  94                 for(int i=0;i<externals.length;i++) {
  95                         getId(graph, externals[i], errors);
  96                 }
  97
  98                 state.inverses.forEachEntry(new TIntIntProcedure() {
  99
 100                         @Override
 101                         public boolean execute(int predicate, int inverse) {
 102                                 try {
 103                                         getId(graph, predicate, errors);
 104                                         if(inverse != 0) getId(graph, inverse, errors);
 105                                 } catch (DatabaseException e) {
 106                                         throw new RuntimeDatabaseException(e);
 107                                 }
 108                                 return true;
 109                         }
 110
 111                 });
 112
 113                 if(!errors.isEmpty()) {
 114                         ArrayList<String> sorted = new ArrayList<>(errors);
 115                         Collections.sort(sorted);
 116                         StringBuilder message = new StringBuilder();
 117                         message.append("Errors in exported model:\n");
 118                         for(String error : sorted) {
 119                                 message.append(error);
 120                                 message.append("\n");
 121                         }
 122                         throw new DatabaseException(message.toString());
 123                 }
 124
 125                 this.resourceCount = state.id;
 126
 127                 state.extensions.put(ExternalDownloadBean.EXTENSION_KEY, new Variant(ExternalDownloadBean.BINDING, new ExternalDownloadBean(downloads)));
 128
 129         }
 130
 131         int indent = 0;
 132
 133         public boolean validateExternal(Resource ext) {
 134                 if(configuration.validate) {
 135                         ExtentStatus status = configuration.preStatus.get(ext);
 136                         if(status != null) {
 137                                 if(ExtentStatus.INTERNAL.equals(status)) return false;
 138                                 else if(ExtentStatus.EXCLUDED.equals(status)) return false;
 139                         }
 140                 }
 141                 return true;
 142         }
 143
 144         private Resource getResource(ReadGraph graph, int r) throws DatabaseException {
 145                 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
 146                 return ss.getResource(r);
 147         }
 148
 149         final public int getExistingId(ReadGraph graph, int r) throws DatabaseException {
 150
 151                 int ret = state.ids.get(r);
 152                 if(ret != -1) {
 153                         return ret;
 154                 } else {
 155                         SerialisationSupport ss = graph.getService(SerialisationSupport.class);
 156                         throw new DatabaseException("Id has not been created for " + NameUtils.getSafeName(graph, ss.getResource(r)));
 157                 }
 158
 159         }
 160
 161         /*
 162          *
 163          * @return -2 if r is not really external and the statement should be excluded
 164          *
 165          */
 166         public int getId(ReadGraph graph, int r, Collection<String> errors) throws DatabaseException {
 167
 168 //              // First external is root library
 169 //              if(r == rootId) return internalCount;
 170
 171                 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
 172                 Layer0 L0 = Layer0.getInstance(graph);
 173
 174                 if(state.ids.containsKey(r)) {
 175                     int ret = state.ids.get(r);
 176                     if(ret == -1) {
 177                         for(int i=0;i<=indent;++i)
 178                             System.out.print("  ");
 179                         System.out.println("Cycle!!!"); // with " + GraphUtils.getReadableName(g, r));
 180                     }
 181                         return ret;
 182                 }
 183                 else {
 184                         Resource res = getResource(graph, r);
 185             if(!validateExternal(res)) {
 186                 errors.add("Illegal reference to " + graph.getPossibleURI(getResource(graph, r)));
 187                 return -2;
 188             }
 189                         Collection<Resource> parents = graph.getObjects(res, L0.PartOf);
 190                         if(parents.size() != 1) {
 191                                 throw new ValidationException("Reference to external resource "
 192                                                 + NameUtils.getSafeName(graph, getResource(graph, r), true) + " without unique uri (" + parents.size() + " parents).");
 193                         }
 194                         int pid = 0;
 195                         for(Resource p : parents) {
 196                             ++indent;
 197                             pid = getId(graph, ss.getTransientId(p), errors);
 198                             if(pid == -2) {
 199                         errors.add("Illegal reference to " + graph.getPossibleURI(getResource(graph, r)));
 200                                 return -2;
 201                             }
 202                         }
 203                         --indent;
 204                         String name = graph.getRelatedValue(res, L0.HasName);
 205                         // Record the external entry
 206                         externalParents.add(pid);
 207             externalNames.add(name);
 208                         state.ids.put(r, state.id);
 209                         // Ensure that this resource is included into the set of externals to maintain the total number of externals
 210                         state.externals.add(r);
 211                         String download = graph.getPossibleRelatedValue(res, L0.Ontology_download, Bindings.STRING);
 212                         if(download != null) {
 213                                 String uri = graph.getURI(res);
 214                                 downloads.put(uri, download);
 215                         }
 216                         return state.id++;
 217                 }
 218         }
 219
 220         @Override
 221         public DataContainer getHeader() throws Exception {
 222             return null;
 223         }
 224
 225         @Override
 226         public int getResourceCount() {
 227                 return resourceCount;
 228         }
 229
 230         private int countRootSeeds() {
 231                 int result = 0;
 232                 for(SeedSpec spec : configuration.seeds) {
 233                         if(SeedSpecType.INTERNAL.equals(spec.specType)) continue;
 234                         result++;
 235                 }
 236                 return result;
 237         }
 238
 239         @Override
 240         public int getIdentityCount() {
 241                 return countRootSeeds() + state.externals.size() + state.internalEntries.size() + 1;
 242         }
 243
 244         @Override
 245         public int getStatementCount() {
 246                 return state.statementCount;
 247         }
 248
 249         @Override
 250         public int getValueCount() {
 251                 return state.valueCount;
 252         }
 253
 254         @Override
 255         public void forStatements(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
 256
 257                 int[] value = new int[4];
 258                 long length = state.otherStatementsInput.length();
 259                 state.otherStatementsInput.position(0);
 260
 261                 while(state.otherStatementsInput.position() < length && !state.monitor.isCanceled()) {
 262
 263                         int s = state.otherStatementsInput.readInt();
 264                         int subjectId = state.ids.get(s);
 265
 266                         boolean exclude = subjectId == -1;
 267
 268                         int size = state.otherStatementsInput.readInt();
 269                         for(int i=0;i<size;i++) {
 270                                 int p = state.otherStatementsInput.readInt();
 271                                 int o = state.otherStatementsInput.readInt();
 272                                 if(!exclude) {
 273                                         if(state.pending.contains(o)) {
 274                                                 System.err.println("excluding garbage statement " + s + " " + p + " " + o + ", object resource is garbage");
 275                                         } else if(state.excludedShared.contains(o)) {
 276                                                 System.err.println("excluding shared " + s + " " + p + " " + o);
 277                                         } else {
 278
 279                                                 int objectId = getExistingId(graph, o);
 280                                                 // The statement can be denied still
 281                                                 if(objectId != -2) {
 282                                                         value[0] = subjectId;
 283                                                         value[1] = getExistingId(graph, p);
 284                                                         int inverse = state.inverses.get(p);
 285                                                         if(inverse != 0) {
 286                                                                 value[2] = getExistingId(graph, inverse);
 287                                                         } else {
 288                                                                 value[2] = -1;
 289                                                         }
 290                                                         value[3] = objectId;
 291
 292                                                         procedure.execute(value);
 293
 294                                                 } else {
 295                                                         System.err.println("Denied (" + NameUtils.getSafeName(graph, getResource(graph, s)) + ", " + NameUtils.getSafeName(graph, getResource(graph, p)) + "," + NameUtils.getSafeName(graph, getResource(graph, o)) + ")");
 296                                                 }
 297
 298                                         }
 299                                 } else {
 300                                         System.err.println("excluding shared " + s);
 301                                 }
 302                         }
 303                 }
 304         }
 305
 306         @Override
 307         public void forValues(ReadGraph graph, TransferableGraphSourceProcedure<Value> procedure) throws Exception {
 308
 309                 Serializer variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT);
 310                 List<Object> idContext = new ArrayList<>();
 311                 long length = state.valueInput.length();
 312                 state.valueInput.position(0);
 313
 314                 while(state.valueInput.position() < length && !state.monitor.isCanceled()) {
 315
 316                         // Ignore value type tag
 317                         int s = state.valueInput.readInt();
 318                         byte valueType = state.valueInput.readByte();
 319                         switch (valueType) {
 320                         case TAG_RAW_COPY_VARIANT_VALUE:
 321                                 state.valueInput.readInt();
 322                                 // Intentional fallthrough.
 323
 324                         case TAG_POTENTIALLY_MODIFIED_VARIANT_VALUE: {
 325                                 idContext.clear();
 326                                 Variant variant = (Variant) variantSerializer.deserialize((DataInput)state.valueInput, idContext);
 327                                 if (valueModifier.mayNeedModification(variant.type())) {
 328                                         Object currentObject = variant.getValue();
 329                                         Object newObject = valueModifier.modify(state, variant.getBinding(), currentObject);
 330                                         if (newObject != currentObject)
 331                                                 variant = new Variant(variant.getBinding(), newObject);
 332                                 }
 333                                 procedure.execute(new Value(state.ids.get(s), variant));
 334                                 break;
 335                         }
 336
 337                         default:
 338                                 throw new IllegalArgumentException("Unrecognized variant value type encountered: " + valueType);
 339                         }
 340                 }
 341         }
 342
 343     @Override
 344     public void forValues2(ReadGraph graph, TransferableGraphSourceValueProcedure procedure) throws Exception {
 345
 346         Serializer datatypeSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.getBindingUnchecked(Datatype.class));
 347         List<Object> idContext = new ArrayList<>();
 348         long length = state.valueInput.length();
 349         state.valueInput.position(0);
 350
 351         while(state.valueInput.position() < length && !state.monitor.isCanceled()) {
 352
 353             int s = state.valueInput.readInt();
 354             byte valueType = state.valueInput.readByte();
 355
 356             switch (valueType) {
 357             case TAG_RAW_COPY_VARIANT_VALUE: {
 358                 // Variant data could be copied raw, no need for modifications
 359                 // Note that the variant data is a concatenation of the
 360                 // variant datatype serialization and the value serialization.
 361                 // variantLength contains both datatype and value data.
 362                 int variantLength = state.valueInput.readInt();
 363                 procedure.rawCopy(state.ids.get(s), variantLength, state.valueInput);
 364                 break;
 365             }
 366
 367             case TAG_POTENTIALLY_MODIFIED_VARIANT_VALUE: {
 368                 // Variant data may need to be modified.
 369                 // Cannot optimize this case with raw copying.
 370                 idContext.clear();
 371                 Datatype type = (Datatype)datatypeSerializer.deserialize((DataInput)state.valueInput, idContext);
 372
 373                 if (valueModifier.mayNeedModification(type)) {
 374                     Binding binding = Bindings.getBinding(type);
 375                     Serializer serializer = Bindings.getSerializerUnchecked(binding);
 376                     Object value = serializer.deserialize((DataInput)state.valueInput);
 377                     value = valueModifier.modify(state, binding, value);
 378                     byte[] bytes = serializer.serialize(value);
 379                     procedure.execute(state.ids.get(s), type, new ByteBufferReadable(bytes));
 380                 } else {
 381                     procedure.execute(state.ids.get(s), type, state.valueInput);
 382                 }
 383                 break;
 384             }
 385
 386             default:
 387                 throw new IllegalArgumentException("Unrecognized variant value type encountered: " + valueType);
 388             }
 389         }
 390     }
 391
 392         protected Identity getRootIdentity(DomainProcessorState state, SerialisationSupport support, Resource rootLibrary) throws DatabaseException {
 393                 return new Identity(state.ids.get(support.getTransientId(rootLibrary)), new External(-1, ""));
 394         }
 395
 396         @Override
 397         public void forIdentities(ReadGraph graph, TransferableGraphSourceProcedure<Identity> procedure) throws Exception {
 398
 399                 SerialisationSupport support = graph.getService(SerialisationSupport.class);
 400                 Layer0 L0 = Layer0.getInstance(graph);
 401
 402                 // TODO: this should be Root with name ""
 403                 procedure.execute(getRootIdentity(state, support, graph.getRootLibrary()));
 404
 405                 TIntObjectMap<Identity> internalMap = new TIntObjectHashMap<>(100, 0.5f, Integer.MIN_VALUE);
 406
 407                 // Declare internal and external roots
 408                 for(SeedSpec r : configuration.seeds) {
 409                         if(SeedSpecType.INTERNAL.equals(r.specType)) continue;
 410                         String typeId = r.type;
 411                         if (typeId == null) {
 412                                 Resource type = graph.getPossibleType(r.resource, L0.Entity);
 413                                 typeId = type != null ? graph.getURI(type) : Layer0.URIs.Entity;
 414                         }
 415                         int id = state.ids.get(support.getTransientId(r.resource));
 416                         Root root = new Root(r.name, typeId);
 417                         Identity rootId = new Identity(id,root);
 418                         internalMap.put(id, rootId);
 419                         procedure.execute(rootId);
 420                 }
 421
 422                 for(int i = 0; i < state.externals.size() ; i++) {
 423                         int parent = externalParents.get(i);
 424                         String name = externalNames.get(i);
 425                         procedure.execute(new Identity(externalBase + i, new External(parent,name)));
 426         }
 427
 428                 if(state.internalEntries != null) {
 429                         for(ConsistsOfProcessEntry ie : state.internalEntries) {
 430                                 if(ie.parent != null) {
 431                                         if(ie.name != null) {
 432                                                 procedure.execute(resolveInternal(graph, support, ie, internalMap));
 433                                         } else {
 434                                                 // In this case there is a child that has no HasName => this should be treated as a blank
 435                                         }
 436                                 } else {
 437                                         procedure.execute(resolveInternal(graph, support, ie, internalMap));
 438                                 }
 439                         }
 440                 }
 441
 442         }
 443
 444         private Identity resolveInternal(ReadGraph graph, SerialisationSupport ss, ConsistsOfProcessEntry entry, TIntObjectMap<Identity> internalMap) throws DatabaseException {
 445                 int id = state.ids.get(ss.getTransientId(entry.resource));
 446                 Identity existing = internalMap.get(id);
 447                 if(existing != null) return existing;
 448
 449                 if(entry.parent == null) {
 450                         Layer0 L0 = Layer0.getInstance(graph);
 451                         Resource possibleParent = graph.getPossibleObject(entry.resource, L0.PartOf);
 452                         if(possibleParent == null) throw new DatabaseException("Invalid root or internal parent path: " + entry.resource);
 453                         int externalId = state.ids.get(ss.getTransientId(possibleParent));
 454                         Identity result = new Identity(id,
 455                                         new Internal(externalId, entry.name));
 456                         internalMap.put(id, result);
 457                         return result;
 458                 } else {
 459                         Identity parent = resolveInternal(graph, ss, entry.parent, internalMap);
 460                         Identity result = new Identity(id,
 461                                         new Internal(parent.resource, entry.name));
 462                         internalMap.put(id, result);
 463                         return result;
 464                 }
 465         }
 466
 467         @Override
 468         public TreeMap<String, Variant> getExtensions() {
 469                 return state.extensions;
 470         }
 471
 472         public File[] getFiles() {
 473             return files;
 474         }
 475
 476         private static <T> T tryClose(T c) throws IOException {
 477                 if (c != null && c instanceof Closeable)
 478                         ((Closeable) c).close();
 479                 return null;
 480         }
 481
 482         public void closeStreams() throws IOException {
 483                 state.valueInput = tryClose(state.valueInput);
 484                 state.otherStatementsInput = tryClose(state.otherStatementsInput);
 485                 state.statementsOutput = tryClose(state.statementsOutput);
 486                 state.valueOutput = tryClose(state.valueOutput);
 487         }
 488
 489         @Override
 490         public void reset() throws Exception {
 491             throw new UnsupportedOperationException();
 492         }
 493
 494         public long[] getResourceArray(ReadGraph graph) throws DatabaseException {
 495                 final SerialisationSupport ss = graph.getService(SerialisationSupport.class);
 496                 final long[] result = new long[state.ids.size()];
 497                 state.ids.forEachEntry(new TIntIntProcedure() {
 498
 499                         @Override
 500                         public boolean execute(int a, int b) {
 501
 502                                 try {
 503                                         Resource r = ss.getResource(a);
 504                                         result[b] = r.getResourceId();
 505                                 } catch (DatabaseException e) {
 506                                         e.printStackTrace();
 507                                 }
 508
 509                                 return true;
 510
 511                         }
 512                 });
 513                 return result;
 514         }
 515
 516         public DomainProcessorState getState() {
 517                 return state;
 518         }
 519
 520         public void forResourceStatements(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
 521
 522                 int[] value = new int[4];
 523                 long length = state.otherStatementsInput.length();
 524                 state.otherStatementsInput.position(0);
 525
 526                 while(state.otherStatementsInput.position() < length) {
 527
 528                         int s = state.otherStatementsInput.readInt();
 529                         int subjectId = state.ids.get(s);
 530
 531                         boolean exclude = subjectId == -1;
 532
 533                         int size = state.otherStatementsInput.readInt();
 534                         for(int i=0;i<size;i++) {
 535                                 int p = state.otherStatementsInput.readInt();
 536                                 int o = state.otherStatementsInput.readInt();
 537                                 if(!exclude) {
 538                                         if(state.excludedShared.contains(o)) {
 539                                                 System.err.println("excluding shared " + s + " " + p + " " + o);
 540                                         } else {
 541
 542                                                 int objectId = getExistingId(graph, o);
 543                                                 // The statement can be denied still
 544                                                 if(objectId != -2) {
 545                                                         value[0] = s;
 546                                                         value[1] = p;
 547                                                         int inverse = state.inverses.get(p);
 548                                                         if(inverse != 0) {
 549                                                                 value[2] = inverse;
 550                                                         } else {
 551                                                                 value[2] = -1;
 552                                                         }
 553                                                         value[3] = o;
 554
 555                                                         procedure.execute(value);
 556
 557                                                 } else {
 558                                                         System.err.println("Denied (" + NameUtils.getSafeName(graph, getResource(graph, s)) + ", " + NameUtils.getSafeName(graph, getResource(graph, p)) + "," + NameUtils.getSafeName(graph, getResource(graph, o)) + ")");
 559                                                 }
 560
 561                                         }
 562                                 } else {
 563                                         System.err.println("excluding shared " + s);
 564                                 }
 565                         }
 566                 }
 567         }
 568
 569         public void forValueResources(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
 570                 int[] value = { 0 };
 571                 long length = state.valueInput.length();
 572                 while (state.valueInput.position() < length) {
 573                         value[0] = state.valueInput.readInt();
 574                         procedure.execute(value);
 575                 }
 576         }
 577
 578         public TransferableGraphConfiguration2 getConfiguration() {
 579                 return configuration;
 580         }
 581
 582         @Override
 583         public void close() throws IOException {
 584                 synchronized (this) {
 585                         if (closed)
 586                                 return;
 587                         closed = true;
 588                 }
 589                 closeStreams();
 590                 if (files != null) {
 591                         for (File f : files) {
 592                                 Files.deleteIfExists(f.toPath());
 593                         }
 594                 }
 595         }
 596
 597 }