bundles/org.simantics.graph.db/src/org/simantics/graph/db/StreamingTransferableGraphImportProcess.java

   1 /*******************************************************************************
   2  * Copyright (c) 2012, 2016 Association for Decentralized Information Management
   3  * in Industry THTH ry.
   4  * All rights reserved. This program and the accompanying materials
   5  * are made available under the terms of the Eclipse Public License v1.0
   6  * which accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *     VTT Technical Research Centre of Finland - initial API and implementation
  11  *     Semantum Oy
  12  *******************************************************************************/
  13 package org.simantics.graph.db;
  14
  15 import java.io.DataInput;
  16 import java.io.DataOutput;
  17 import java.io.DataOutputStream;
  18 import java.io.FileNotFoundException;
  19 import java.io.FileOutputStream;
  20 import java.io.IOException;
  21 import java.io.InputStream;
  22 import java.util.HashSet;
  23 import java.util.Map;
  24 import java.util.Set;
  25 import java.util.TreeMap;
  26
  27 import org.simantics.databoard.Bindings;
  28 import org.simantics.databoard.adapter.AdaptException;
  29 import org.simantics.databoard.binding.Binding;
  30 import org.simantics.databoard.binding.mutable.Variant;
  31 import org.simantics.databoard.serialization.Serializer;
  32 import org.simantics.databoard.type.Datatype;
  33 import org.simantics.db.ReadGraph;
  34 import org.simantics.db.Resource;
  35 import org.simantics.db.Session;
  36 import org.simantics.db.VirtualGraph;
  37 import org.simantics.db.WriteOnlyGraph;
  38 import org.simantics.db.common.WriteBindings;
  39 import org.simantics.db.common.procedure.adapter.TransientCacheAsyncListener;
  40 import org.simantics.db.common.request.PossibleIndexRoot;
  41 import org.simantics.db.common.uri.UnescapedChildMapOfResource;
  42 import org.simantics.db.common.utils.Logger;
  43 import org.simantics.db.exception.DatabaseException;
  44 import org.simantics.db.service.ClusterBuilder2;
  45 import org.simantics.db.service.ClusterBuilderFactory;
  46 import org.simantics.db.service.ClusteringSupport;
  47 import org.simantics.db.service.SerialisationSupport;
  48 import org.simantics.graph.db.TransferableGraphSource.TransferableGraphSourceProcedure;
  49 import org.simantics.graph.db.TransferableGraphSource.TransferableGraphSourceValueProcedure;
  50 import org.simantics.graph.representation.Extensions;
  51 import org.simantics.graph.representation.External;
  52 import org.simantics.graph.representation.Identity;
  53 import org.simantics.graph.representation.IdentityDefinition;
  54 import org.simantics.graph.representation.Internal;
  55 import org.simantics.graph.representation.Optional;
  56 import org.simantics.graph.representation.Root;
  57 import org.simantics.graph.representation.TransferableGraphUtils;
  58 import org.simantics.graph.utils.TGResourceUtil;
  59 import org.simantics.graph.utils.TGResourceUtil.LongAdapter;
  60
  61 public class StreamingTransferableGraphImportProcess implements TransferableGraphImporter {
  62
  63         public static String LOG_FILE = "transferableGraphs.log";
  64         final static private boolean LOG = false;
  65
  66         static DataOutput log;
  67
  68         static {
  69
  70                 if (LOG) {
  71
  72                         try {
  73                                 FileOutputStream stream = new FileOutputStream(LOG_FILE);
  74                                 log = new DataOutputStream(stream);
  75                         } catch (FileNotFoundException e) {
  76                                 e.printStackTrace();
  77                         }
  78                 }
  79
  80         }
  81
  82         private static void log(String line) {
  83                 if (LOG) {
  84                         try {
  85                                 log.writeUTF(line + "\n");
  86                         } catch (IOException e) {
  87                                 e.printStackTrace();
  88                         }
  89                 }
  90         }
  91
  92         Resource indexRoot;
  93         TransferableGraphSource tg;
  94         VirtualGraph vg;
  95         IImportAdvisor2 advisor;
  96         ClusterBuilder2 builder;
  97         final TGResourceUtil resourceUtil = new TGResourceUtil();
  98
  99         int[] handles;
 100
 101         Set<String> missingExternals = new HashSet<String>();
 102
 103         int resourceCount;
 104         Identity[] identities;
 105         TreeMap<String, Variant> extensions;
 106
 107         // Builtins
 108         Resource RootLibrary;
 109         Resource String;
 110         Resource Library;
 111
 112         Resource InstanceOf;
 113         Resource ConsistsOf;
 114         Resource PartOf;
 115         Resource HasName;
 116         Resource NameOf;
 117
 118         public StreamingTransferableGraphImportProcess(Session session, VirtualGraph vg, TransferableGraphSource tg, IImportAdvisor2 advisor) {
 119                 this.tg = tg;
 120                 this.vg = vg;
 121                 this.advisor = advisor;
 122         }
 123
 124         public void readIdentities(ReadGraph g) throws Exception {
 125                 extensions = tg.getExtensions();
 126                 resourceCount = tg.getResourceCount();
 127                 identities = new Identity[tg.getIdentityCount()];
 128                 tg.forIdentities(g, new TransferableGraphSourceProcedure<Identity>() {
 129
 130                         int counter = 0;
 131
 132                         @Override
 133                         public void execute(Identity value) throws Exception {
 134                                 identities[counter++] = value;
 135                         }
 136                 });
 137         }
 138
 139         public void findBuiltins(WriteOnlyGraph g) throws DatabaseException {
 140                 RootLibrary = g.getBuiltin("http:/");
 141                 String = g.getBuiltin(CoreInitialization.LAYER0 + "String");
 142                 Library = g.getBuiltin(CoreInitialization.LAYER0 + "Library");
 143                 InstanceOf = g.getBuiltin(CoreInitialization.LAYER0 + "InstanceOf");
 144                 ConsistsOf = g.getBuiltin(CoreInitialization.LAYER0 + "ConsistsOf");
 145                 PartOf = g.getBuiltin(CoreInitialization.LAYER0 + "PartOf");
 146                 HasName = g.getBuiltin(CoreInitialization.LAYER0 + "HasName");
 147                 NameOf = g.getBuiltin(CoreInitialization.LAYER0 + "NameOf");
 148         }
 149
 150         public void findBuiltins(ReadGraph g) throws DatabaseException {
 151                 RootLibrary = g.getBuiltin("http:/");
 152                 String = g.getBuiltin(CoreInitialization.LAYER0 + "String");
 153                 Library = g.getBuiltin(CoreInitialization.LAYER0 + "Library");
 154                 InstanceOf = g.getBuiltin(CoreInitialization.LAYER0 + "InstanceOf");
 155                 ConsistsOf = g.getBuiltin(CoreInitialization.LAYER0 + "ConsistsOf");
 156                 PartOf = g.getBuiltin(CoreInitialization.LAYER0 + "PartOf");
 157                 HasName = g.getBuiltin(CoreInitialization.LAYER0 + "HasName");
 158                 NameOf = g.getBuiltin(CoreInitialization.LAYER0 + "NameOf");
 159         }
 160
 161 //      /* Preparation that is used when the core is empty.
 162 //       */
 163 //      void initialPrepare(WriteOnlyGraph graph) throws DatabaseException {
 164 //              findBuiltins(graph);
 165 //
 166 //              resources = new Resource[tg.resourceCount];
 167 //
 168 //              int Root = -1;
 169 //              int SimanticsDomain = -1;
 170 //              int Layer0 = -1;
 171 //
 172 //              for(Identity identity : tg.identities) {
 173 //                      if(identity.definition instanceof Internal) {
 174 //                              Internal def = (Internal)identity.definition;
 175 //                              Resource res = null;
 176 //                              if(def.parent == Layer0) {
 177 //                                      try {
 178 //                                              res = graph.getBuiltin(CoreInitialization.LAYER0 + def.name);
 179 //                                      } catch(ResourceNotFoundException e) {
 180 //                                      }
 181 //                              }
 182 //                              else if(def.parent == SimanticsDomain) {
 183 //                                      if(def.name.equals("Layer0-1.0"))
 184 //                                              Layer0 = identity.resource;
 185 //                              }
 186 //                              else if(def.parent == Root) {
 187 //                                      if(def.name.equals("www.simantics.org"))
 188 //                                              SimanticsDomain = identity.resource;
 189 //                              }
 190 //
 191 //                              if(res == null)
 192 //                                      res = createChild(graph, resources[def.parent], def.name);
 193 //                              else
 194 //                                      createChild(graph, res, resources[def.parent], def.name);
 195 //                              resources[identity.resource] = res;
 196 //                      }
 197 //                      else if(identity.definition instanceof Root) {
 198 //                              Root = identity.resource;
 199 //                              resources[identity.resource] = RootLibrary;
 200 //                      }
 201 //              }
 202 //      }
 203
 204         void addMissing(String external) {
 205                 Set<String> removals = new HashSet<String>();
 206                 for(String ext : missingExternals) if(ext.startsWith(external)) return;
 207                 for(String ext : missingExternals) if(external.startsWith(ext)) removals.add(ext);
 208                 missingExternals.removeAll(removals);
 209                 missingExternals.add(external);
 210         }
 211
 212         void prepare(ReadGraph graph) throws Exception {
 213
 214                 Resource target = advisor.getTarget();
 215                 if(target != null)
 216                         indexRoot = graph.syncRequest(new PossibleIndexRoot(target));
 217
 218                 findBuiltins(graph);
 219                 readIdentities(graph);
 220
 221 //              System.err.println("ext: " + extensions);
 222 //              System.err.println("rc: " + resourceCount);
 223 //              System.err.println("ic: " + identities.length);
 224
 225                 ClusterBuilderFactory factory = graph.getService(ClusterBuilderFactory.class);
 226                 ClusterBuilder2 builder = factory.create(vg, false);
 227
 228         this.handles = new int[resourceCount];
 229
 230                 for(Identity identity : identities) {
 231                         IdentityDefinition definition = identity.definition;
 232                         if(definition instanceof External) {
 233                                 External def = (External)definition;
 234                                 if(def.parent == -1) {
 235                                     handles[identity.resource] = builder.handle(RootLibrary);
 236                                 } else {
 237                                         if("@inverse".equals(def.name)) {
 238                                                 int parent = handles[def.parent];
 239                                                 int child = builder.handle(graph.getInverse(builder.resource(parent)));
 240                                                 handles[identity.resource] = child;
 241                                         } else {
 242                                                 int handle = handles[def.parent];
 243                                                 Resource parent = handle != 0 ? builder.resource(handle) : null;
 244                                                 // TODO: escape should be removed when names become well-behaving
 245                                                 if(parent != null) {
 246                                                     Map<String,Resource> childMap = graph
 247                                             .syncRequest(new UnescapedChildMapOfResource(parent),
 248                                                     new TransientCacheAsyncListener<Map<String, Resource>>());
 249                                                         Resource child = childMap.get(def.name);
 250                                                         if(child == null) {
 251                                                                 addMissing(graph.getURI(parent) + "/" + def.name);
 252                                                         } else {
 253                                                                 handles[identity.resource] = builder.handle(child);
 254                                                         }
 255                                                 } else {
 256                                                     addMissing(TransferableGraphUtils.getURI(resourceCount, identities, def.parent) + "/" + def.name);
 257                                                 }
 258                                         }
 259                                 }
 260                         }
 261                         else if(definition instanceof Internal) {
 262                                 // Do not do anything for now
 263                         }
 264                         else if(definition instanceof Root) {
 265                                 Root root = (Root)definition;
 266                                 if(root.name.equals(""))
 267                                     handles[identity.resource] = builder.handle(RootLibrary);
 268                                 else  {
 269                                         Resource existing = advisor.analyzeRoot(graph, root);
 270                                         if(existing != null)
 271                                             handles[identity.resource] = builder.handle(existing);
 272                                 }
 273                         }
 274                         else if(definition instanceof Optional) {
 275                                 External def = (External)definition;
 276                                 Resource parent = builder.resource(handles[def.parent]);
 277                                 if(parent != null)
 278                                         handles[identity.resource] = builder.handle(graph.syncRequest(new UnescapedChildMapOfResource(parent)).get(def.name));
 279                         }
 280                 }
 281
 282                 if(!missingExternals.isEmpty()) throw new MissingDependencyException(this);
 283
 284         }
 285
 286         @Override
 287         public Resource createChild(WriteOnlyGraph graph, Resource parent, Resource child, String name) throws DatabaseException {
 288             if(child == null) child = graph.newResource();
 289                 Resource nameResource = graph.newResource();
 290                 graph.claim(nameResource, InstanceOf, null, String);
 291                 graph.claimValue(nameResource, name, WriteBindings.STRING);
 292                 graph.claim(child, HasName, NameOf, nameResource);
 293                 return child;
 294         }
 295
 296         int[] getClustering() {
 297                 Variant v = extensions.get(Extensions.CLUSTERING);
 298                 if(v == null) return null;
 299                 try {
 300                         return (int[])v.getValue(Bindings.INT_ARRAY);
 301                 } catch (AdaptException e) {
 302                         Logger.defaultLogError(e);
 303                         return null;
 304                 }
 305         }
 306
 307         int[] getClusterSets() {
 308                 Variant v = extensions.get(Extensions.CLUSTER_SETS);
 309                 if(v == null) return null;
 310                 try {
 311                         return (int[])v.getValue(Bindings.INT_ARRAY);
 312                 } catch (AdaptException e) {
 313                         Logger.defaultLogError(e);
 314                         return null;
 315                 }
 316         }
 317
 318         boolean needTranslation(Datatype type) {
 319             return resourceUtil.mayHaveResource(type);
 320         }
 321
 322         void findClusterSet(WriteOnlyGraph graph, Resource rootLibrary, int[] clustering, int[] clusterSets, long[] clusters, int id) throws DatabaseException {
 323                 ClusteringSupport support = graph.getService(ClusteringSupport.class);
 324                 if(id == Extensions.ROOT_LIBRARY_CLUSTER_SET || id == Extensions.INDEX_ROOT_CLUSTER_SET) return;
 325                 for(int pos=0,index=0;index<clustering.length;index++) {
 326                         pos += clustering[index];
 327                         if(id < pos) {
 328                                 int cs = clusterSets[index];
 329                                 if(handles[id] == 0) {
 330                                         int csHandle = 0;
 331                                         if(cs == Extensions.ROOT_LIBRARY_CLUSTER_SET) csHandle = builder.handle(rootLibrary);
 332                                         else if(cs == Extensions.INDEX_ROOT_CLUSTER_SET) {
 333                                                 if(indexRoot == null) throw new DatabaseException("No index root was available in TG import.");
 334                                                 csHandle = builder.handle(indexRoot);
 335                                         }
 336                                         else {
 337                                                 findClusterSet(graph, rootLibrary, clustering, clusterSets, clusters, cs);
 338                                                 csHandle = handles[cs];
 339                                         }
 340
 341                                         if(clusters[index] != 0)
 342                                                 builder.selectCluster(clusters[index]);
 343                                         else if(cs >= 0)
 344                                                 builder.newCluster(csHandle);
 345
 346                                         handles[id] = builder.newResource(csHandle);
 347                                         clusters[index] = support.getCluster(builder.resource(handles[id]));
 348
 349                                         builder.createClusterSet(handles[id]);
 350                                 }
 351                                 return;
 352                         }
 353                 }
 354         }
 355
 356         void write(final WriteOnlyGraph graph) throws Exception {
 357
 358         final SerialisationSupport ss = graph.getService(SerialisationSupport.class);
 359
 360         ClusterBuilderFactory factory = graph.getService(ClusterBuilderFactory.class);
 361         if(advisor instanceof IImportAdvisor2) {
 362             boolean allowImmutable = ((IImportAdvisor2)advisor).allowImmutableModifications();
 363             builder = factory.create(vg, allowImmutable);
 364         } else {
 365             builder = factory.create(vg, false);
 366         }
 367
 368                 final int[] handles = this.handles;
 369
 370                 int[] clustering = getClustering();
 371                 if(clustering != null) {
 372
 373                         int[] clusterSets = getClusterSets();
 374                         if(clusterSets != null) {
 375
 376                                 assert(clustering.length == clusterSets.length);
 377
 378                                 long[] clusters = new long[clustering.length];
 379
 380                                 // Create clustering
 381                                 for(int i=0;i<clusterSets.length;i++) {
 382                                         findClusterSet(graph, graph.getRootLibrary(), clustering, clusterSets, clusters, clusterSets[i]);
 383                                 }
 384
 385                                 // Then create all resources
 386                                 int i=0;
 387                             for(int j=0;j<clustering.length;j++) {
 388                                 int c = clustering[j];
 389                                 int s = clusterSets[j];
 390                                 int setHandle = 0;
 391                                         if(s == Extensions.ROOT_LIBRARY_CLUSTER_SET)
 392                                                 setHandle = builder.handle(graph.getRootLibrary());
 393                                         else if(s == Extensions.INDEX_ROOT_CLUSTER_SET) {
 394                                                 if(indexRoot == null) throw new DatabaseException("No index root was available in TG import.");
 395                                                 setHandle = builder.handle(indexRoot);
 396                                         }
 397                                         else setHandle = handles[s];
 398                                         // Preserve clustering only for internal resources
 399                                         if(clusters[j] != 0)
 400                                                 builder.selectCluster(clusters[j]);
 401                                         else if(s >= 0)
 402                                                 builder.newCluster(setHandle);
 403                                         for(int r=0;r<c;r++, i++)
 404                                                 if(handles[i] == 0) handles[i] = builder.newResource();
 405                                 }
 406
 407                                 for(;i<handles.length;++i)
 408                                         if(handles[i] == 0) handles[i] = builder.newResource();
 409
 410                         } else {
 411
 412                         int i = 0;
 413                                 for(int c : clustering) {
 414                                         builder.newCluster();
 415                                         for(int r=0;r<c;r++, i++)
 416                                                 if(handles[i] == 0) handles[i] = builder.newResource();
 417                                 }
 418
 419                                 for(;i<handles.length;++i)
 420                                         if(handles[i] == 0) handles[i] = builder.newResource();
 421
 422                         }
 423
 424                 } else {
 425
 426                         // Create blank resources
 427                         for(int i=0;i<handles.length;++i)
 428                                 if(handles[i] == 0) handles[i] = builder.newResource();
 429
 430                 }
 431
 432                 // Internal identities
 433                 for(Identity identity : identities) {
 434                         IdentityDefinition definition = identity.definition;
 435 //                      if(handles[identity.resource] != 0)
 436 //                              continue;
 437                         if(definition instanceof External) {
 438                                 // Already done everything
 439                         }
 440                         else if(definition instanceof Internal) {
 441                                 Internal def = (Internal)definition;
 442                                 if(handles[identity.resource] != 0)
 443                                         handles[identity.resource] = builder.handle(advisor.createChild(graph, this, builder.resource(handles[def.parent]), builder.resource(handles[identity.resource]), def.name));
 444                                 else
 445                                         handles[identity.resource] = builder.handle(advisor.createChild(graph, this, builder.resource(handles[def.parent]), null, def.name));
 446                         }
 447                         else if(definition instanceof Root) {
 448
 449                                 Root root = (Root)definition;
 450                                 if(handles[identity.resource] != 0)
 451                                         handles[identity.resource] = builder.handle(advisor.createRoot(graph, root, builder.resource(handles[identity.resource])));
 452                                 else
 453                                         handles[identity.resource] = builder.handle(advisor.createRoot(graph, root, null));
 454                         }
 455                         else if(definition instanceof Optional) {
 456                                 Optional def = (Optional)definition;
 457                                 if(handles[identity.resource] != 0) {
 458                                         Resource child = advisor.createChild(graph, this, builder.resource(handles[def.parent]), builder.resource(handles[identity.resource]), def.name);
 459                                         graph.claim(child, InstanceOf, null, Library); // ???
 460                                         handles[identity.resource] = builder.handle(child);
 461                                 } else {
 462                                         Resource child = advisor.createChild(graph, this, builder.resource(handles[def.parent]), null, def.name);
 463                                         graph.claim(child, InstanceOf, null, Library); // ???
 464                                         handles[identity.resource] = builder.handle(child);
 465                                 }
 466                         }
 467                 }
 468
 469                 tg.getStatementCount();
 470                 tg.forStatements(null, new TransferableGraphSourceProcedure<int[]>() {
 471
 472                         @Override
 473                         public void execute(int[] value) throws Exception {
 474
 475                                 int sub = value[0];
 476                                 int pred = value[1];
 477                                 int inv = value[2];
 478                                 int obj = value[3];
 479
 480                                 int subject = handles[sub];
 481                                 int predicate = handles[pred];
 482                                 int object = handles[obj];
 483
 484                                 builder.addStatement(graph, subject, predicate, object);
 485                                 if(inv >= 0) {
 486                                     int inverse = handles[inv];
 487                                     builder.addStatement(graph, object, inverse, subject);
 488                                 }
 489
 490                         }
 491
 492                 });
 493
 494                 tg.getValueCount();
 495
 496                 class ValueProcedure extends InputStream implements TransferableGraphSourceValueProcedure {
 497
 498             private TGResourceUtil util = new TGResourceUtil();
 499                     private DataInput source;
 500
 501             @Override
 502             public void execute(int _resource, Datatype type, DataInput stream) throws Exception {
 503
 504                 source = stream;
 505
 506                 //int file = _resource & 0x80000000;
 507                 int resource = _resource & 0x7FFFFFFF;
 508
 509                 Binding binding = Bindings.getBinding(type);
 510                 Serializer s = Bindings.getSerializer(binding);
 511
 512                 builder.beginValue(handles[resource]);
 513                 if(util.mayHaveResource(type)) {
 514                     Object value = s.deserialize(stream);
 515                     util.adaptValue( binding,  value, new LongAdapter() {
 516                                 @Override
 517                                 public long adapt(long in) {
 518                                     try {
 519                                         return ss.getRandomAccessId(handles[(int)in]);
 520                                     } catch (DatabaseException e) {
 521                                         throw new IllegalStateException(e);
 522                                     }
 523                                 }
 524                             });
 525                     byte[] bytes = s.serialize(value);
 526                     for(byte b : bytes) {
 527                         int val = b;
 528                         if(val < 0) val += 256;
 529                         builder.appendValue(val);
 530                     }
 531                 } else {
 532                     s.skip(this);
 533                 }
 534                 builder.endValue();
 535
 536             }
 537
 538             @Override
 539             public int read() throws IOException {
 540                 int value = source.readUnsignedByte();
 541                 try {
 542                     builder.appendValue(value);
 543                 } catch (DatabaseException e) {
 544                     e.printStackTrace();
 545                 }
 546                 return value;
 547             }
 548
 549             @Override
 550             public void rawCopy(int resource, int length, DataInput input) throws Exception {
 551                 builder.beginValue(handles[resource]);
 552                 for (int i = 0; i < length; ++i)
 553                     builder.appendValue(input.readUnsignedByte());
 554                 builder.endValue();
 555             }
 556
 557                 };
 558
 559                 tg.forValues2(null, new ValueProcedure());
 560
 561         }
 562
 563         @Override
 564         public long[] getResourceIds(SerialisationSupport serializer) throws DatabaseException {
 565                 final int count = handles.length;
 566                 long[] resourceIds = new long[count];
 567                 for(int i=0;i<count;++i)
 568                     resourceIds[i] = serializer.getRandomAccessId(handles[i]);
 569                 return resourceIds;
 570         }
 571 }