bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/ModelTransferableGraphSource.java

   1 package org.simantics.db.layer0.util;
   2
   3 import java.io.Closeable;
   4 import java.io.DataInput;
   5 import java.io.File;
   6 import java.io.IOException;
   7 import java.nio.file.Files;
   8 import java.util.ArrayList;
   9 import java.util.Collection;
  10 import java.util.Collections;
  11 import java.util.HashSet;
  12 import java.util.List;
  13 import java.util.TreeMap;
  14
  15 import org.simantics.databoard.Bindings;
  16 import org.simantics.databoard.Databoard;
  17 import org.simantics.databoard.binding.Binding;
  18 import org.simantics.databoard.binding.mutable.Variant;
  19 import org.simantics.databoard.container.DataContainer;
  20 import org.simantics.databoard.serialization.Serializer;
  21 import org.simantics.databoard.type.Datatype;
  22 import org.simantics.databoard.util.binary.ByteBufferReadable;
  23 import org.simantics.db.ReadGraph;
  24 import org.simantics.db.Resource;
  25 import org.simantics.db.common.utils.NameUtils;
  26 import org.simantics.db.exception.DatabaseException;
  27 import org.simantics.db.exception.RuntimeDatabaseException;
  28 import org.simantics.db.exception.ValidationException;
  29 import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus;
  30 import org.simantics.db.layer0.util.TransferableGraphConfiguration2.RootSpec;
  31 import org.simantics.db.service.SerialisationSupport;
  32 import org.simantics.graph.db.TransferableGraphSource;
  33 import org.simantics.graph.representation.External;
  34 import org.simantics.graph.representation.Identity;
  35 import org.simantics.graph.representation.Root;
  36 import org.simantics.graph.representation.Value;
  37 import org.simantics.layer0.Layer0;
  38
  39 import gnu.trove.list.array.TIntArrayList;
  40 import gnu.trove.procedure.TIntIntProcedure;
  41
  42 public class ModelTransferableGraphSource implements TransferableGraphSource {
  43
  44         final private TransferableGraphConfiguration2 configuration;
  45         final private DomainProcessorState state;
  46         final private int externalBase;
  47         final private int resourceCount;
  48         final private File[] files;
  49         final private TGValueModifier valueModifier;
  50
  51         private volatile boolean closed = false;
  52
  53         TIntArrayList externalParents = new TIntArrayList();
  54         ArrayList<String> externalNames = new ArrayList<String>();
  55         TreeMap<String,String> downloads = new TreeMap<String,String>();
  56
  57         public ModelTransferableGraphSource(final ReadGraph graph, TransferableGraphConfiguration2 configuration, final DomainProcessorState state, File ... fs) throws DatabaseException {
  58
  59                 this.configuration = configuration;
  60                 this.state = state;
  61                 this.files = fs;
  62                 this.valueModifier = state.valueModifier;
  63
  64                 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
  65
  66                 // At this point ids contains all internal resources. Now add roots and externals.
  67
  68                 // Root Library
  69                 state.ids.put(ss.getTransientId(graph.getRootLibrary()), state.id++);
  70
  71                 // External roots - internal roots were already processed as internal resources by domain processor
  72                 for(RootSpec rs : configuration.roots) {
  73                         if(!rs.internal) {
  74                                 int resourceId = ss.getTransientId(rs.resource);
  75                                 state.ids.put(resourceId, state.id++);
  76                                 // The fixed roots have been seen as externals by domain processor. Now remove them from external set.
  77                                 state.externals.remove(resourceId);
  78                         }
  79                 }
  80
  81                 this.externalBase = state.id;
  82
  83                 final Collection<String> errors = new HashSet<String>();
  84
  85                 // All resource considered as not internal by domain processor. Can also contain roots.
  86                 int[] externals = state.externals.toArray();
  87
  88                 // Build up the state.externals, externalNames and externalParents
  89                 for(int i=0;i<externals.length;i++) {
  90                         getId(graph, externals[i], errors);
  91                 }
  92
  93                 state.inverses.forEachEntry(new TIntIntProcedure() {
  94
  95                         @Override
  96                         public boolean execute(int predicate, int inverse) {
  97                                 try {
  98                                         getId(graph, predicate, errors);
  99                                         if(inverse != 0) getId(graph, inverse, errors);
 100                                 } catch (DatabaseException e) {
 101                                         throw new RuntimeDatabaseException(e);
 102                                 }
 103                                 return true;
 104                         }
 105
 106                 });
 107
 108                 if(!errors.isEmpty()) {
 109                         ArrayList<String> sorted = new ArrayList<String>(errors);
 110                         Collections.sort(sorted);
 111                         StringBuilder message = new StringBuilder();
 112                         message.append("Errors in exported model:\n");
 113                         for(String error : sorted) {
 114                                 message.append(error);
 115                                 message.append("\n");
 116                         }
 117                         throw new DatabaseException(message.toString());
 118                 }
 119
 120                 this.resourceCount = state.id;
 121
 122                 state.extensions.put(ExternalDownloadBean.EXTENSION_KEY, new Variant(ExternalDownloadBean.BINDING, new ExternalDownloadBean(downloads)));
 123
 124         }
 125
 126         int indent = 0;
 127
 128         public boolean validateExternal(Resource ext) {
 129                 if(configuration.validate) {
 130                         ExtentStatus status = configuration.preStatus.get(ext);
 131                         if(status != null) {
 132                                 if(ExtentStatus.INTERNAL.equals(status)) return false;
 133                                 else if(ExtentStatus.EXCLUDED.equals(status)) return false;
 134                         }
 135                 }
 136                 return true;
 137         }
 138
 139         private Resource getResource(ReadGraph graph, int r) throws DatabaseException {
 140                 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
 141                 return ss.getResource(r);
 142         }
 143
 144         final public int getExistingId(ReadGraph graph, int r) throws DatabaseException {
 145
 146                 int ret = state.ids.get(r);
 147                 if(ret != -1) {
 148                         return ret;
 149                 } else {
 150                         SerialisationSupport ss = graph.getService(SerialisationSupport.class);
 151                         throw new DatabaseException("Id has not been created for " + NameUtils.getSafeName(graph, ss.getResource(r)));
 152                 }
 153
 154         }
 155
 156         /*
 157          *
 158          * @return -2 if r is not really external and the statement should be excluded
 159          *
 160          */
 161         public int getId(ReadGraph graph, int r, Collection<String> errors) throws DatabaseException {
 162
 163 //              // First external is root library
 164 //              if(r == rootId) return internalCount;
 165
 166                 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
 167                 Layer0 L0 = Layer0.getInstance(graph);
 168
 169                 if(state.ids.containsKey(r)) {
 170                     int ret = state.ids.get(r);
 171                     if(ret == -1) {
 172                         for(int i=0;i<=indent;++i)
 173                             System.out.print("  ");
 174                         System.out.println("Cycle!!!"); // with " + GraphUtils.getReadableName(g, r));
 175                     }
 176                         return ret;
 177                 }
 178                 else {
 179                         Resource res = getResource(graph, r);
 180             if(!validateExternal(res)) {
 181                 errors.add("Illegal reference to " + graph.getPossibleURI(getResource(graph, r)));
 182                 return -2;
 183             }
 184                         Collection<Resource> parents = graph.getObjects(res, L0.PartOf);
 185                         if(parents.size() != 1) {
 186                                 throw new ValidationException("Reference to external resource "
 187                                                 + NameUtils.getSafeName(graph, getResource(graph, r), true) + " without unique uri (" + parents.size() + " parents).");
 188                         }
 189                         int pid = 0;
 190                         for(Resource p : parents) {
 191                             ++indent;
 192                             pid = getId(graph, ss.getTransientId(p), errors);
 193                             if(pid == -2) {
 194                         errors.add("Illegal reference to " + graph.getPossibleURI(getResource(graph, r)));
 195                                 return -2;
 196                             }
 197                         }
 198                         --indent;
 199                         String name = graph.getRelatedValue(res, L0.HasName);
 200                         // Record the external entry
 201                         externalParents.add(pid);
 202             externalNames.add(name);
 203                         state.ids.put(r, state.id);
 204                         // Ensure that this resource is included into the set of externals to maintain the total number of externals
 205                         state.externals.add(r);
 206                         String download = graph.getPossibleRelatedValue(res, L0.Ontology_download, Bindings.STRING);
 207                         if(download != null) {
 208                                 String uri = graph.getURI(res);
 209                                 downloads.put(uri, download);
 210                         }
 211                         return state.id++;
 212                 }
 213         }
 214
 215         @Override
 216         public DataContainer getHeader() throws Exception {
 217             return null;
 218         }
 219
 220         @Override
 221         public int getResourceCount() {
 222                 return resourceCount;
 223         }
 224
 225         @Override
 226         public int getIdentityCount() {
 227                 return configuration.roots.size() + state.externals.size() + 1;
 228         }
 229
 230         @Override
 231         public int getStatementCount() {
 232                 return state.statementCount;
 233         }
 234
 235         @Override
 236         public int getValueCount() {
 237                 return state.valueCount;
 238         }
 239
 240         @Override
 241         public void forStatements(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
 242
 243                 int[] value = new int[4];
 244                 long length = state.otherStatementsInput.length();
 245                 state.otherStatementsInput.position(0);
 246
 247                 while(state.otherStatementsInput.position() < length && !state.monitor.isCanceled()) {
 248
 249                         int s = state.otherStatementsInput.readInt();
 250                         int subjectId = state.ids.get(s);
 251
 252                         boolean exclude = subjectId == -1;
 253
 254                         int size = state.otherStatementsInput.readInt();
 255                         for(int i=0;i<size;i++) {
 256                                 int p = state.otherStatementsInput.readInt();
 257                                 int o = state.otherStatementsInput.readInt();
 258                                 if(!exclude) {
 259                                         if(state.pending.contains(o)) {
 260                                                 System.err.println("excluding garbage statement " + s + " " + p + " " + o + ", object resource is garbage");
 261                                         } else if(state.excludedShared.contains(o)) {
 262                                                 System.err.println("excluding shared " + s + " " + p + " " + o);
 263                                         } else {
 264
 265                                                 int objectId = getExistingId(graph, o);
 266                                                 // The statement can be denied still
 267                                                 if(objectId != -2) {
 268                                                         value[0] = subjectId;
 269                                                         value[1] = getExistingId(graph, p);
 270                                                         int inverse = state.inverses.get(p);
 271                                                         if(inverse != 0) {
 272                                                                 value[2] = getExistingId(graph, inverse);
 273                                                         } else {
 274                                                                 value[2] = -1;
 275                                                         }
 276                                                         value[3] = objectId;
 277
 278                                                         procedure.execute(value);
 279
 280                                                 } else {
 281                                                         System.err.println("Denied (" + NameUtils.getSafeName(graph, getResource(graph, s)) + ", " + NameUtils.getSafeName(graph, getResource(graph, p)) + "," + NameUtils.getSafeName(graph, getResource(graph, o)) + ")");
 282                                                 }
 283
 284                                         }
 285                                 } else {
 286                                         System.err.println("excluding shared " + s);
 287                                 }
 288                         }
 289                 }
 290         }
 291
 292         @Override
 293         public void forValues(ReadGraph graph, TransferableGraphSourceProcedure<Value> procedure) throws Exception {
 294
 295                 Serializer variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT);
 296                 List<Object> idContext = new ArrayList<>();
 297                 long length = state.valueInput.length();
 298                 state.valueInput.position(0);
 299
 300                 while(state.valueInput.position() < length && !state.monitor.isCanceled()) {
 301
 302                         // Ignore value type tag
 303                         int s = state.valueInput.readInt();
 304                         byte valueType = state.valueInput.readByte();
 305                         switch (valueType) {
 306                         case TAG_RAW_COPY_VARIANT_VALUE:
 307                                 state.valueInput.readInt();
 308                                 // Intentional fallthrough.
 309
 310                         case TAG_POTENTIALLY_MODIFIED_VARIANT_VALUE: {
 311                                 idContext.clear();
 312                                 Variant variant = (Variant) variantSerializer.deserialize((DataInput)state.valueInput, idContext);
 313                                 if (valueModifier.mayNeedModification(variant.type())) {
 314                                         Object currentObject = variant.getValue();
 315                                         Object newObject = valueModifier.modify(state, variant.getBinding(), currentObject);
 316                                         if (newObject != currentObject)
 317                                                 variant = new Variant(variant.getBinding(), newObject);
 318                                 }
 319                                 procedure.execute(new Value(state.ids.get(s), variant));
 320                                 break;
 321                         }
 322
 323                         default:
 324                                 throw new IllegalArgumentException("Unrecognized variant value type encountered: " + valueType);
 325                         }
 326                 }
 327         }
 328
 329     @Override
 330     public void forValues2(ReadGraph graph, TransferableGraphSourceValueProcedure procedure) throws Exception {
 331
 332         Serializer datatypeSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.getBindingUnchecked(Datatype.class));
 333         List<Object> idContext = new ArrayList<>();
 334         long length = state.valueInput.length();
 335         state.valueInput.position(0);
 336
 337         while(state.valueInput.position() < length && !state.monitor.isCanceled()) {
 338
 339             int s = state.valueInput.readInt();
 340             byte valueType = state.valueInput.readByte();
 341
 342             switch (valueType) {
 343             case TAG_RAW_COPY_VARIANT_VALUE: {
 344                 // Variant data could be copied raw, no need for modifications
 345                 // Note that the variant data is a concatenation of the
 346                 // variant datatype serialization and the value serialization.
 347                 // variantLength contains both datatype and value data.
 348                 int variantLength = state.valueInput.readInt();
 349                 procedure.rawCopy(state.ids.get(s), variantLength, state.valueInput);
 350                 break;
 351             }
 352
 353             case TAG_POTENTIALLY_MODIFIED_VARIANT_VALUE: {
 354                 // Variant data may need to be modified.
 355                 // Cannot optimize this case with raw copying.
 356                 idContext.clear();
 357                 Datatype type = (Datatype)datatypeSerializer.deserialize((DataInput)state.valueInput, idContext);
 358
 359                 if (valueModifier.mayNeedModification(type)) {
 360                     Binding binding = Bindings.getBinding(type);
 361                     Serializer serializer = Bindings.getSerializerUnchecked(binding);
 362                     Object value = serializer.deserialize((DataInput)state.valueInput);
 363                     value = valueModifier.modify(state, binding, value);
 364                     byte[] bytes = serializer.serialize(value);
 365                     procedure.execute(state.ids.get(s), type, new ByteBufferReadable(bytes));
 366                 } else {
 367                     procedure.execute(state.ids.get(s), type, state.valueInput);
 368                 }
 369                 break;
 370             }
 371
 372             default:
 373                 throw new IllegalArgumentException("Unrecognized variant value type encountered: " + valueType);
 374             }
 375         }
 376     }
 377
 378         protected Identity getRootIdentity(DomainProcessorState state, SerialisationSupport support, Resource rootLibrary) throws DatabaseException {
 379                 return new Identity(state.ids.get(support.getTransientId(rootLibrary)), new External(-1, ""));
 380         }
 381
 382         @Override
 383         public void forIdentities(ReadGraph graph, TransferableGraphSourceProcedure<Identity> procedure) throws Exception {
 384
 385                 SerialisationSupport support = graph.getService(SerialisationSupport.class);
 386                 Layer0 L0 = Layer0.getInstance(graph);
 387
 388                 // TODO: this should be Root with name ""
 389                 procedure.execute(getRootIdentity(state, support, graph.getRootLibrary()));
 390
 391                 // Declare internal and external roots
 392                 for(RootSpec r : configuration.roots) {
 393                         String typeId = r.type;
 394                         if (typeId == null) {
 395                                 Resource type = graph.getPossibleType(r.resource, L0.Entity);
 396                                 typeId = type != null ? graph.getURI(type) : Layer0.URIs.Entity;
 397                         }
 398                         procedure.execute(new Identity(
 399                                         state.ids.get(support.getTransientId(r.resource)),
 400                                         new Root(r.name, typeId)));
 401                 }
 402
 403                 for(int i = 0; i < state.externals.size() ; i++) {
 404                         int parent = externalParents.get(i);
 405                         String name = externalNames.get(i);
 406                         procedure.execute(new Identity(externalBase + i, new External(parent,name)));
 407         }
 408
 409         }
 410
 411         @Override
 412         public TreeMap<String, Variant> getExtensions() {
 413                 return state.extensions;
 414         }
 415
 416         public File[] getFiles() {
 417             return files;
 418         }
 419
 420         private static <T> T tryClose(T c) throws IOException {
 421                 if (c != null && c instanceof Closeable)
 422                         ((Closeable) c).close();
 423                 return null;
 424         }
 425
 426         public void closeStreams() throws IOException {
 427                 state.valueInput = tryClose(state.valueInput);
 428                 state.otherStatementsInput = tryClose(state.otherStatementsInput);
 429                 state.statementsOutput = tryClose(state.statementsOutput);
 430                 state.valueOutput = tryClose(state.valueOutput);
 431         }
 432
 433         @Override
 434         public void reset() throws Exception {
 435             throw new UnsupportedOperationException();
 436         }
 437
 438         public long[] getResourceArray(ReadGraph graph) throws DatabaseException {
 439                 final SerialisationSupport ss = graph.getService(SerialisationSupport.class);
 440                 final long[] result = new long[state.ids.size()];
 441                 state.ids.forEachEntry(new TIntIntProcedure() {
 442
 443                         @Override
 444                         public boolean execute(int a, int b) {
 445
 446                                 try {
 447                                         Resource r = ss.getResource(a);
 448                                         result[b] = r.getResourceId();
 449                                 } catch (DatabaseException e) {
 450                                         e.printStackTrace();
 451                                 }
 452
 453                                 return true;
 454
 455                         }
 456                 });
 457                 return result;
 458         }
 459
 460         public DomainProcessorState getState() {
 461                 return state;
 462         }
 463
 464         public void forResourceStatements(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
 465
 466                 int[] value = new int[4];
 467                 long length = state.otherStatementsInput.length();
 468                 state.otherStatementsInput.position(0);
 469
 470                 while(state.otherStatementsInput.position() < length) {
 471
 472                         int s = state.otherStatementsInput.readInt();
 473                         int subjectId = state.ids.get(s);
 474
 475                         boolean exclude = subjectId == -1;
 476
 477                         int size = state.otherStatementsInput.readInt();
 478                         for(int i=0;i<size;i++) {
 479                                 int p = state.otherStatementsInput.readInt();
 480                                 int o = state.otherStatementsInput.readInt();
 481                                 if(!exclude) {
 482                                         if(state.excludedShared.contains(o)) {
 483                                                 System.err.println("excluding shared " + s + " " + p + " " + o);
 484                                         } else {
 485
 486                                                 int objectId = getExistingId(graph, o);
 487                                                 // The statement can be denied still
 488                                                 if(objectId != -2) {
 489                                                         value[0] = s;
 490                                                         value[1] = p;
 491                                                         int inverse = state.inverses.get(p);
 492                                                         if(inverse != 0) {
 493                                                                 value[2] = inverse;
 494                                                         } else {
 495                                                                 value[2] = -1;
 496                                                         }
 497                                                         value[3] = o;
 498
 499                                                         procedure.execute(value);
 500
 501                                                 } else {
 502                                                         System.err.println("Denied (" + NameUtils.getSafeName(graph, getResource(graph, s)) + ", " + NameUtils.getSafeName(graph, getResource(graph, p)) + "," + NameUtils.getSafeName(graph, getResource(graph, o)) + ")");
 503                                                 }
 504
 505                                         }
 506                                 } else {
 507                                         System.err.println("excluding shared " + s);
 508                                 }
 509                         }
 510                 }
 511         }
 512
 513         public void forValueResources(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
 514                 int[] value = { 0 };
 515                 long length = state.valueInput.length();
 516                 while (state.valueInput.position() < length) {
 517                         value[0] = state.valueInput.readInt();
 518                         procedure.execute(value);
 519                 }
 520         }
 521
 522         public TransferableGraphConfiguration2 getConfiguration() {
 523                 return configuration;
 524         }
 525
 526         @Override
 527         public void close() throws IOException {
 528                 synchronized (this) {
 529                         if (closed)
 530                                 return;
 531                         closed = true;
 532                 }
 533                 closeStreams();
 534                 if (files != null) {
 535                         for (File f : files) {
 536                                 Files.deleteIfExists(f.toPath());
 537                         }
 538                 }
 539         }
 540
 541 }