bundles/org.simantics.graph.db/src/org/simantics/graph/db/StreamingTransferableGraphImportProcess.java

   1 /*******************************************************************************
   2  * Copyright (c) 2012, 2017 Association for Decentralized Information Management
   3  * in Industry THTH ry.
   4  * All rights reserved. This program and the accompanying materials
   5  * are made available under the terms of the Eclipse Public License v1.0
   6  * which accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *     VTT Technical Research Centre of Finland - initial API and implementation
  11  *     Semantum Oy - e.g. #7016
  12  *******************************************************************************/
  13 package org.simantics.graph.db;
  14
  15 import java.io.DataInput;
  16 import java.io.IOException;
  17 import java.io.InputStream;
  18 import java.util.ArrayList;
  19 import java.util.Collections;
  20 import java.util.HashMap;
  21 import java.util.HashSet;
  22 import java.util.Map;
  23 import java.util.Set;
  24 import java.util.TreeMap;
  25
  26 import org.simantics.databoard.Bindings;
  27 import org.simantics.databoard.adapter.AdaptException;
  28 import org.simantics.databoard.binding.Binding;
  29 import org.simantics.databoard.binding.mutable.Variant;
  30 import org.simantics.databoard.serialization.Serializer;
  31 import org.simantics.databoard.type.Datatype;
  32 import org.simantics.databoard.util.URIStringUtils;
  33 import org.simantics.db.ReadGraph;
  34 import org.simantics.db.Resource;
  35 import org.simantics.db.Session;
  36 import org.simantics.db.VirtualGraph;
  37 import org.simantics.db.WriteOnlyGraph;
  38 import org.simantics.db.common.WriteBindings;
  39 import org.simantics.db.common.procedure.adapter.TransientCacheAsyncListener;
  40 import org.simantics.db.common.request.PossibleIndexRoot;
  41 import org.simantics.db.common.uri.UnescapedChildMapOfResource;
  42 import org.simantics.db.common.utils.Logger;
  43 import org.simantics.db.exception.DatabaseException;
  44 import org.simantics.db.service.ClusterBuilder2;
  45 import org.simantics.db.service.ClusterBuilderFactory;
  46 import org.simantics.db.service.ClusteringSupport;
  47 import org.simantics.db.service.SerialisationSupport;
  48 import org.simantics.db.service.XSupport;
  49 import org.simantics.graph.db.TransferableGraphSource.TransferableGraphSourceProcedure;
  50 import org.simantics.graph.db.TransferableGraphSource.TransferableGraphSourceValueProcedure;
  51 import org.simantics.graph.representation.Extensions;
  52 import org.simantics.graph.representation.External;
  53 import org.simantics.graph.representation.Identity;
  54 import org.simantics.graph.representation.IdentityDefinition;
  55 import org.simantics.graph.representation.Internal;
  56 import org.simantics.graph.representation.Optional;
  57 import org.simantics.graph.representation.Root;
  58 import org.simantics.graph.representation.TransferableGraphUtils;
  59 import org.simantics.graph.utils.TGResourceUtil;
  60 import org.simantics.graph.utils.TGResourceUtil.LongAdapter;
  61 import org.simantics.utils.datastructures.Pair;
  62 import org.slf4j.LoggerFactory;
  63
  64 import gnu.trove.map.TIntObjectMap;
  65 import gnu.trove.map.hash.TIntObjectHashMap;
  66
  67 public class StreamingTransferableGraphImportProcess implements TransferableGraphImporter {
  68
  69         private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(StreamingTransferableGraphImportProcess.class);
  70
  71         Resource indexRoot;
  72         TransferableGraphSource tg;
  73         VirtualGraph vg;
  74         IImportAdvisor2 advisor;
  75         ClusterBuilder2 builder;
  76         final TGResourceUtil resourceUtil = new TGResourceUtil();
  77
  78         int[] handles;
  79
  80         Map<String,Integer> allMissingExternals = new HashMap<>();
  81         Set<String> missingExternals = new HashSet<>();
  82         Map<String,Resource> resolvedParents = new HashMap<>();
  83         TIntObjectHashMap<Resource> existingInternalMap = new TIntObjectHashMap<>();
  84
  85         int resourceCount;
  86         Identity[] identities;
  87         TreeMap<String, Variant> extensions;
  88
  89         // Builtins
  90         Resource RootLibrary;
  91         Resource String;
  92         Resource ExternalEntity;
  93         Resource Library;
  94
  95         Resource InstanceOf;
  96         Resource ConsistsOf;
  97         Resource PartOf;
  98         Resource HasName;
  99         Resource NameOf;
 100
 101         public StreamingTransferableGraphImportProcess(Session session, VirtualGraph vg, TransferableGraphSource tg, IImportAdvisor2 advisor) {
 102                 this.tg = tg;
 103                 this.vg = vg;
 104                 this.advisor = advisor;
 105         }
 106
 107         public void readIdentities(ReadGraph g) throws Exception {
 108                 extensions = tg.getExtensions();
 109                 resourceCount = tg.getResourceCount();
 110                 identities = new Identity[tg.getIdentityCount()];
 111                 tg.forIdentities(g, new TransferableGraphSourceProcedure<Identity>() {
 112
 113                         int counter = 0;
 114
 115                         @Override
 116                         public void execute(Identity value) throws Exception {
 117                                 identities[counter++] = value;
 118                         }
 119                 });
 120         }
 121
 122         public void findBuiltins(WriteOnlyGraph g) throws DatabaseException {
 123                 RootLibrary = g.getBuiltin("http:/");
 124                 String = g.getBuiltin(CoreInitialization.LAYER0 + "String");
 125                 Library = g.getBuiltin(CoreInitialization.LAYER0 + "Library");
 126                 InstanceOf = g.getBuiltin(CoreInitialization.LAYER0 + "InstanceOf");
 127                 ConsistsOf = g.getBuiltin(CoreInitialization.LAYER0 + "ConsistsOf");
 128                 PartOf = g.getBuiltin(CoreInitialization.LAYER0 + "PartOf");
 129                 HasName = g.getBuiltin(CoreInitialization.LAYER0 + "HasName");
 130                 NameOf = g.getBuiltin(CoreInitialization.LAYER0 + "NameOf");
 131                 ExternalEntity = g.getBuiltin(CoreInitialization.LAYER0 + "ExternalEntity");
 132         }
 133
 134         public void findBuiltins(ReadGraph g) throws DatabaseException {
 135                 RootLibrary = g.getBuiltin("http:/");
 136                 String = g.getBuiltin(CoreInitialization.LAYER0 + "String");
 137                 Library = g.getBuiltin(CoreInitialization.LAYER0 + "Library");
 138                 InstanceOf = g.getBuiltin(CoreInitialization.LAYER0 + "InstanceOf");
 139                 ConsistsOf = g.getBuiltin(CoreInitialization.LAYER0 + "ConsistsOf");
 140                 PartOf = g.getBuiltin(CoreInitialization.LAYER0 + "PartOf");
 141                 HasName = g.getBuiltin(CoreInitialization.LAYER0 + "HasName");
 142                 NameOf = g.getBuiltin(CoreInitialization.LAYER0 + "NameOf");
 143                 ExternalEntity = g.getBuiltin(CoreInitialization.LAYER0 + "ExternalEntity");
 144         }
 145
 146         void addMissing(int handleIndex, String external) {
 147                 allMissingExternals.put(external, handleIndex);
 148                 Set<String> removals = new HashSet<>();
 149                 for(String ext : missingExternals) if(ext.startsWith(external)) return;
 150                 for(String ext : missingExternals) if(external.startsWith(ext)) removals.add(ext);
 151                 missingExternals.removeAll(removals);
 152                 missingExternals.add(external);
 153         }
 154
 155         void prepare(ReadGraph graph) throws Exception {
 156
 157                 Resource target = advisor.getTarget();
 158                 if(target != null)
 159                         indexRoot = graph.syncRequest(new PossibleIndexRoot(target));
 160
 161                 findBuiltins(graph);
 162                 readIdentities(graph);
 163
 164 //              System.err.println("ext: " + extensions);
 165 //              System.err.println("rc: " + resourceCount);
 166 //              System.err.println("ic: " + identities.length);
 167
 168                 ClusterBuilderFactory factory = graph.getService(ClusterBuilderFactory.class);
 169                 ClusterBuilder2 builder = factory.create(vg, false);
 170
 171                 this.handles = new int[resourceCount];
 172                 TIntObjectMap<Identity> identityMap = TransferableGraphUtils.mapIdentities(identities);
 173
 174                 for(Identity identity : identities) {
 175                         IdentityDefinition definition = identity.definition;
 176                         if(definition instanceof External) {
 177                                 External def = (External)definition;
 178                                 if(def.parent == -1) {
 179                                     handles[identity.resource] = builder.handle(RootLibrary);
 180                                 } else {
 181                                         if("@inverse".equals(def.name)) {
 182                                                 int parent = handles[def.parent];
 183                                                 int child = builder.handle(graph.getInverse(builder.resource(parent)));
 184                                                 handles[identity.resource] = child;
 185                                         } else {
 186                                                 int handle = handles[def.parent];
 187                                                 Resource parent = handle != 0 ? builder.resource(handle) : null;
 188                                                 // TODO: escape should be removed when names become well-behaving
 189                                                 if(parent != null) {
 190                                                         resolvedParents.put(graph.getURI(parent), parent);
 191                                                     Map<String,Resource> childMap = graph
 192                                             .syncRequest(new UnescapedChildMapOfResource(parent),
 193                                                     TransientCacheAsyncListener.instance());
 194                                                         Resource child = childMap.get(def.name);
 195                                                         if(child == null) {
 196                                                                 addMissing(identity.resource, graph.getURI(parent) + "/" + def.name);
 197                                                         } else {
 198                                                                 handles[identity.resource] = builder.handle(child);
 199                                                         }
 200                                                 } else {
 201                                                     addMissing(identity.resource, TransferableGraphUtils.getURI(resourceCount, identityMap, def.parent) + "/" + def.name);
 202                                                 }
 203                                         }
 204                                 }
 205                         }
 206                         else if(definition instanceof Internal) {
 207                                 String uri = TransferableGraphUtils.getURI(resourceCount, identityMap, identity.resource);
 208                                 Resource existing = graph.getPossibleResource(uri);
 209                                 if(existing != null) {
 210                                         existingInternalMap.put(identity.resource, existing);
 211                                 }
 212                         }
 213                         else if(definition instanceof Root) {
 214                                 Root root = (Root)definition;
 215                                 if(root.name.equals(""))
 216                                     handles[identity.resource] = builder.handle(RootLibrary);
 217                                 else  {
 218                                         Resource existing = advisor.analyzeRoot(graph, root);
 219                                         if(existing != null)
 220                                             handles[identity.resource] = builder.handle(existing);
 221                                 }
 222                         }
 223                         else if(definition instanceof Optional) {
 224                                 External def = (External)definition;
 225                                 Resource parent = builder.resource(handles[def.parent]);
 226                                 if(parent != null)
 227                                         handles[identity.resource] = builder.handle(graph.syncRequest(new UnescapedChildMapOfResource(parent)).get(def.name));
 228                         }
 229                 }
 230
 231                 //if(!missingExternals.isEmpty()) throw new MissingDependencyException(this);
 232
 233         }
 234
 235         @Override
 236         public Resource createChild(WriteOnlyGraph graph, Resource parent, Resource child, String name) throws DatabaseException {
 237             if(child == null) child = graph.newResource();
 238                 Resource nameResource = graph.newResource();
 239                 graph.claim(nameResource, InstanceOf, null, String);
 240                 graph.claimValue(nameResource, name, WriteBindings.STRING);
 241                 graph.claim(child, HasName, NameOf, nameResource);
 242                 return child;
 243         }
 244
 245         int[] getClustering() {
 246                 Variant v = extensions.get(Extensions.CLUSTERING);
 247                 if(v == null) return null;
 248                 try {
 249                         return (int[])v.getValue(Bindings.INT_ARRAY);
 250                 } catch (AdaptException e) {
 251                         Logger.defaultLogError(e);
 252                         return null;
 253                 }
 254         }
 255
 256         int[] getClusterSets() {
 257                 Variant v = extensions.get(Extensions.CLUSTER_SETS);
 258                 if(v == null) return null;
 259                 try {
 260                         return (int[])v.getValue(Bindings.INT_ARRAY);
 261                 } catch (AdaptException e) {
 262                         Logger.defaultLogError(e);
 263                         return null;
 264                 }
 265         }
 266
 267         boolean needTranslation(Datatype type) {
 268             return resourceUtil.mayHaveResource(type);
 269         }
 270
 271         void findClusterSet(WriteOnlyGraph graph, Resource rootLibrary, int[] clustering, int[] clusterSets, long[] clusters, int id) throws DatabaseException {
 272                 ClusteringSupport support = graph.getService(ClusteringSupport.class);
 273                 if(id == Extensions.ROOT_LIBRARY_CLUSTER_SET || id == Extensions.INDEX_ROOT_CLUSTER_SET) return;
 274                 for(int pos=0,index=0;index<clustering.length;index++) {
 275                         pos += clustering[index];
 276                         if(id < pos) {
 277                                 int cs = clusterSets[index];
 278                                 if(handles[id] == 0) {
 279                                         int csHandle = 0;
 280                                         if(cs == Extensions.ROOT_LIBRARY_CLUSTER_SET) csHandle = builder.handle(rootLibrary);
 281                                         else if(cs == Extensions.INDEX_ROOT_CLUSTER_SET) {
 282                                                 if(indexRoot == null) throw new DatabaseException("No index root was available in TG import.");
 283                                                 csHandle = builder.handle(indexRoot);
 284                                         }
 285                                         else {
 286                                                 findClusterSet(graph, rootLibrary, clustering, clusterSets, clusters, cs);
 287                                                 csHandle = handles[cs];
 288                                         }
 289
 290                                         if(clusters[index] != 0)
 291                                                 builder.selectCluster(clusters[index]);
 292                                         else if(cs >= 0)
 293                                                 builder.newCluster(csHandle);
 294
 295                                         handles[id] = builder.newResource(csHandle);
 296                                         clusters[index] = support.getCluster(builder.resource(handles[id]));
 297
 298                                         builder.createClusterSet(handles[id]);
 299                                 }
 300                                 return;
 301                         }
 302                 }
 303         }
 304
 305         void createMissing(final WriteOnlyGraph graph) throws Exception {
 306
 307                 if(allMissingExternals.isEmpty()) return;
 308
 309                 XSupport xs = graph.getService(XSupport.class);
 310                 Pair<Boolean,Boolean> serviceMode = xs.getServiceMode();
 311                 xs.setServiceMode(true, false);
 312                 try {
 313                         ArrayList<String> missing = new ArrayList<>(allMissingExternals.keySet());
 314                         Collections.sort(missing);
 315                         for(String uri : missing) {
 316                                 String[] parts = URIStringUtils.splitURI(uri);
 317
 318                                 Resource parent = resolvedParents.get(parts[0]);
 319                                 // TODO: proper exception message
 320                                 if(parent == null) throw new IllegalStateException("!!");
 321
 322                                 Resource childResource = graph.newResource();
 323                                 graph.claim(childResource, InstanceOf, null, ExternalEntity);
 324
 325                                 Resource nameResource = graph.newResource();
 326                                 graph.claim(nameResource, InstanceOf, null, String);
 327                                 graph.claimValue(nameResource, parts[1], WriteBindings.STRING);
 328                                 graph.claim(childResource, HasName, NameOf, nameResource);
 329
 330                                 graph.claim(parent, ConsistsOf, PartOf, childResource);
 331
 332                                 resolvedParents.put(uri, childResource);
 333
 334                                 handles[allMissingExternals.get(uri)] = builder.handle(childResource);
 335                         }
 336                 } finally {
 337                         xs.setServiceMode(serviceMode.first, serviceMode.second);
 338                 }
 339         }
 340
 341         void write(final WriteOnlyGraph graph) throws Exception {
 342
 343         final SerialisationSupport ss = graph.getService(SerialisationSupport.class);
 344
 345         ClusterBuilderFactory factory = graph.getService(ClusterBuilderFactory.class);
 346         if(advisor instanceof IImportAdvisor2) {
 347             boolean allowImmutable = ((IImportAdvisor2)advisor).allowImmutableModifications();
 348             builder = factory.create(vg, allowImmutable);
 349         } else {
 350             builder = factory.create(vg, false);
 351         }
 352
 353         createMissing(graph);
 354
 355                 final int[] handles = this.handles;
 356
 357                 int[] clustering = getClustering();
 358                 if(clustering != null) {
 359
 360                         int[] clusterSets = getClusterSets();
 361                         if(clusterSets != null) {
 362
 363                                 assert(clustering.length == clusterSets.length);
 364
 365                                 long[] clusters = new long[clustering.length];
 366
 367                                 // Create clustering
 368                                 for(int i=0;i<clusterSets.length;i++) {
 369                                         findClusterSet(graph, graph.getRootLibrary(), clustering, clusterSets, clusters, clusterSets[i]);
 370                                 }
 371
 372                                 // Then create all resources
 373                                 int i=0;
 374                             for(int j=0;j<clustering.length;j++) {
 375                                 int c = clustering[j];
 376                                 int s = clusterSets[j];
 377                                 int setHandle = 0;
 378                                         if(s == Extensions.ROOT_LIBRARY_CLUSTER_SET)
 379                                                 setHandle = builder.handle(graph.getRootLibrary());
 380                                         else if(s == Extensions.INDEX_ROOT_CLUSTER_SET) {
 381                                                 if(indexRoot == null) throw new DatabaseException("No index root was available in TG import.");
 382                                                 setHandle = builder.handle(indexRoot);
 383                                         }
 384                                         else setHandle = handles[s];
 385                                         // Preserve clustering only for internal resources
 386                                         if(clusters[j] != 0)
 387                                                 builder.selectCluster(clusters[j]);
 388                                         else if(s >= 0)
 389                                                 builder.newCluster(setHandle);
 390                                         for(int r=0;r<c;r++, i++)
 391                                                 if(handles[i] == 0) handles[i] = builder.newResource();
 392                                 }
 393
 394                                 for(;i<handles.length;++i)
 395                                         if(handles[i] == 0) handles[i] = builder.newResource();
 396
 397                         } else {
 398
 399                         int i = 0;
 400                                 for(int c : clustering) {
 401                                         builder.newCluster();
 402                                         for(int r=0;r<c;r++, i++)
 403                                                 if(handles[i] == 0) handles[i] = builder.newResource();
 404                                 }
 405
 406                                 for(;i<handles.length;++i)
 407                                         if(handles[i] == 0) handles[i] = builder.newResource();
 408
 409                         }
 410
 411                 } else {
 412
 413                         // Create blank resources
 414                         for(int i=0;i<handles.length;++i)
 415                                 if(handles[i] == 0) handles[i] = builder.newResource();
 416
 417                 }
 418
 419                 // Internal identities
 420                 for(Identity identity : identities) {
 421                         IdentityDefinition definition = identity.definition;
 422 //                      if(handles[identity.resource] != 0)
 423 //                              continue;
 424                         if(definition instanceof External) {
 425                                 // Already done everything
 426                         }
 427                         else if(definition instanceof Internal) {
 428                                 Internal def = (Internal)definition;
 429
 430                                 Resource external = existingInternalMap.get(identity.resource);
 431                                 if(external != null) {
 432                                         handles[identity.resource] = builder.handle(external);
 433                                 } else {
 434                                         if(handles[identity.resource] != 0)
 435                                                 handles[identity.resource] = builder.handle(advisor.createChild(graph, this, builder.resource(handles[def.parent]), builder.resource(handles[identity.resource]), def.name));
 436                                         else
 437                                                 handles[identity.resource] = builder.handle(advisor.createChild(graph, this, builder.resource(handles[def.parent]), null, def.name));
 438                                 }
 439
 440                         }
 441                         else if(definition instanceof Root) {
 442
 443                                 Root root = (Root)definition;
 444                                 if(handles[identity.resource] != 0)
 445                                         handles[identity.resource] = builder.handle(advisor.createRoot(graph, root, builder.resource(handles[identity.resource])));
 446                                 else
 447                                         handles[identity.resource] = builder.handle(advisor.createRoot(graph, root, null));
 448                         }
 449                         else if(definition instanceof Optional) {
 450                                 Optional def = (Optional)definition;
 451                                 if(handles[identity.resource] != 0) {
 452                                         Resource child = advisor.createChild(graph, this, builder.resource(handles[def.parent]), builder.resource(handles[identity.resource]), def.name);
 453                                         graph.claim(child, InstanceOf, null, Library); // ???
 454                                         handles[identity.resource] = builder.handle(child);
 455                                 } else {
 456                                         Resource child = advisor.createChild(graph, this, builder.resource(handles[def.parent]), null, def.name);
 457                                         graph.claim(child, InstanceOf, null, Library); // ???
 458                                         handles[identity.resource] = builder.handle(child);
 459                                 }
 460                         }
 461                 }
 462
 463                 tg.getStatementCount();
 464                 tg.forStatements(null, new TransferableGraphSourceProcedure<int[]>() {
 465
 466                         @Override
 467                         public void execute(int[] value) throws Exception {
 468
 469                                 int sub = value[0];
 470                                 int pred = value[1];
 471                                 int inv = value[2];
 472                                 int obj = value[3];
 473
 474                                 int subject = handles[sub];
 475                                 int predicate = handles[pred];
 476                                 int object = handles[obj];
 477
 478                                 builder.addStatement(graph, subject, predicate, object);
 479                                 if(inv >= 0) {
 480                                     int inverse = handles[inv];
 481                                     builder.addStatement(graph, object, inverse, subject);
 482                                 }
 483
 484                         }
 485
 486                 });
 487
 488                 tg.getValueCount();
 489
 490                 class ValueProcedure extends InputStream implements TransferableGraphSourceValueProcedure {
 491
 492             private TGResourceUtil util = new TGResourceUtil();
 493                     private DataInput source;
 494
 495             @Override
 496             public void execute(int _resource, Datatype type, DataInput stream) throws Exception {
 497
 498                 source = stream;
 499
 500                 //int file = _resource & 0x80000000;
 501                 int resource = _resource & 0x7FFFFFFF;
 502
 503                 Binding binding = Bindings.getBinding(type);
 504                 Serializer s = Bindings.getSerializer(binding);
 505
 506                 builder.beginValue(handles[resource]);
 507                 if(util.mayHaveResource(type)) {
 508                     Object value = s.deserialize(stream);
 509                     util.adaptValue( binding,  value, new LongAdapter() {
 510                                 @Override
 511                                 public long adapt(long in) {
 512                                     try {
 513                                         return ss.getRandomAccessId(handles[(int)in]);
 514                                     } catch (DatabaseException e) {
 515                                         throw new IllegalStateException(e);
 516                                     }
 517                                 }
 518                             });
 519                     byte[] bytes = s.serialize(value);
 520                     for(byte b : bytes) {
 521                         int val = b;
 522                         if(val < 0) val += 256;
 523                         builder.appendValue(val);
 524                     }
 525                 } else {
 526                     s.skip(this);
 527                 }
 528                 builder.endValue();
 529
 530             }
 531
 532             @Override
 533             public int read() throws IOException {
 534                 int value = source.readUnsignedByte();
 535                 try {
 536                     builder.appendValue(value);
 537                 } catch (DatabaseException e) {
 538                     LOGGER.error("Failed to write value into database", e);
 539                 }
 540                 return value;
 541             }
 542
 543             @Override
 544             public void rawCopy(int resource, int length, DataInput input) throws Exception {
 545                 builder.beginValue(handles[resource]);
 546                 for (int i = 0; i < length; ++i)
 547                     builder.appendValue(input.readUnsignedByte());
 548                 builder.endValue();
 549             }
 550
 551                 };
 552
 553                 tg.forValues2(null, new ValueProcedure());
 554
 555                 for(Resource r : existingInternalMap.valueCollection()) {
 556                         graph.deny(r, InstanceOf, null, ExternalEntity, null);
 557                 }
 558
 559         }
 560
 561         @Override
 562         public long[] getResourceIds(SerialisationSupport serializer) throws DatabaseException {
 563                 final int count = handles.length;
 564                 long[] resourceIds = new long[count];
 565                 for(int i=0;i<count;++i)
 566                     resourceIds[i] = serializer.getRandomAccessId(handles[i]);
 567                 return resourceIds;
 568         }
 569 }