bundles/org.simantics.graph.db/src/org/simantics/graph/db/StreamingTransferableGraphImportProcess.java

   1 /*******************************************************************************
   2  * Copyright (c) 2012, 2017 Association for Decentralized Information Management
   3  * in Industry THTH ry.
   4  * All rights reserved. This program and the accompanying materials
   5  * are made available under the terms of the Eclipse Public License v1.0
   6  * which accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *     VTT Technical Research Centre of Finland - initial API and implementation
  11  *     Semantum Oy - e.g. #7016
  12  *******************************************************************************/
  13 package org.simantics.graph.db;
  14
  15 import java.io.DataInput;
  16 import java.io.IOException;
  17 import java.io.InputStream;
  18 import java.util.ArrayList;
  19 import java.util.Collections;
  20 import java.util.HashMap;
  21 import java.util.HashSet;
  22 import java.util.Map;
  23 import java.util.Set;
  24 import java.util.TreeMap;
  25
  26 import org.simantics.databoard.Bindings;
  27 import org.simantics.databoard.adapter.AdaptException;
  28 import org.simantics.databoard.binding.Binding;
  29 import org.simantics.databoard.binding.mutable.Variant;
  30 import org.simantics.databoard.serialization.Serializer;
  31 import org.simantics.databoard.type.Datatype;
  32 import org.simantics.databoard.util.URIStringUtils;
  33 import org.simantics.db.ReadGraph;
  34 import org.simantics.db.Resource;
  35 import org.simantics.db.Session;
  36 import org.simantics.db.VirtualGraph;
  37 import org.simantics.db.WriteOnlyGraph;
  38 import org.simantics.db.common.WriteBindings;
  39 import org.simantics.db.common.procedure.adapter.TransientCacheAsyncListener;
  40 import org.simantics.db.common.request.PossibleIndexRoot;
  41 import org.simantics.db.common.uri.UnescapedChildMapOfResource;
  42 import org.simantics.db.common.utils.Logger;
  43 import org.simantics.db.exception.DatabaseException;
  44 import org.simantics.db.service.ClusterBuilder2;
  45 import org.simantics.db.service.ClusterBuilderFactory;
  46 import org.simantics.db.service.ClusteringSupport;
  47 import org.simantics.db.service.SerialisationSupport;
  48 import org.simantics.db.service.XSupport;
  49 import org.simantics.graph.db.TransferableGraphSource.TransferableGraphSourceProcedure;
  50 import org.simantics.graph.db.TransferableGraphSource.TransferableGraphSourceValueProcedure;
  51 import org.simantics.graph.representation.Extensions;
  52 import org.simantics.graph.representation.External;
  53 import org.simantics.graph.representation.Identity;
  54 import org.simantics.graph.representation.IdentityDefinition;
  55 import org.simantics.graph.representation.Internal;
  56 import org.simantics.graph.representation.Optional;
  57 import org.simantics.graph.representation.Root;
  58 import org.simantics.graph.representation.TransferableGraphUtils;
  59 import org.simantics.graph.utils.TGResourceUtil;
  60 import org.simantics.graph.utils.TGResourceUtil.LongAdapter;
  61 import org.simantics.utils.datastructures.Pair;
  62 import org.slf4j.LoggerFactory;
  63
  64 import gnu.trove.map.TIntObjectMap;
  65 import gnu.trove.map.hash.TIntObjectHashMap;
  66
  67 public class StreamingTransferableGraphImportProcess implements TransferableGraphImporter {
  68
  69         private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(StreamingTransferableGraphImportProcess.class);
  70
  71         Resource indexRoot;
  72         TransferableGraphSource tg;
  73         VirtualGraph vg;
  74         IImportAdvisor2 advisor;
  75         TGStatusMonitor monitor;
  76         ClusterBuilder2 builder;
  77         final TGResourceUtil resourceUtil = new TGResourceUtil();
  78
  79         int[] handles;
  80
  81         Map<String,Integer> allMissingExternals = new HashMap<>();
  82         Set<String> missingExternals = new HashSet<>();
  83         Map<String,Resource> resolvedParents = new HashMap<>();
  84         TIntObjectHashMap<Resource> existingInternalMap = new TIntObjectHashMap<>();
  85
  86         int resourceCount;
  87         Identity[] identities;
  88         TreeMap<String, Variant> extensions;
  89
  90         // Builtins
  91         Resource RootLibrary;
  92         Resource String;
  93         Resource ExternalEntity;
  94         Resource Library;
  95
  96         Resource InstanceOf;
  97         Resource ConsistsOf;
  98         Resource PartOf;
  99         Resource HasName;
 100         Resource NameOf;
 101
 102         boolean failOnMissing;
 103
 104         public StreamingTransferableGraphImportProcess(Session session, VirtualGraph vg, TransferableGraphSource tg, IImportAdvisor2 advisor) {
 105                 this(session, vg, tg, advisor, null);
 106         }
 107
 108         public StreamingTransferableGraphImportProcess(Session session, VirtualGraph vg, TransferableGraphSource tg, IImportAdvisor2 advisor, TGStatusMonitor monitor) {
 109                 this(session, vg, tg, advisor, monitor, failOnMissingEntities());
 110         }
 111
 112         public StreamingTransferableGraphImportProcess(Session session, VirtualGraph vg, TransferableGraphSource tg, IImportAdvisor2 advisor, TGStatusMonitor monitor, boolean failOnMissing) {
 113                 this.tg = tg;
 114                 this.vg = vg;
 115                 this.advisor = advisor;
 116                 this.monitor = monitor;
 117                 this.failOnMissing = failOnMissing;
 118         }
 119
 120         private int updatePercentage(int percentage, int done, int total) {
 121                 if (monitor != null && (done & 63) == 0) {
 122                         int current = 100*done / total;
 123                         if (current > percentage) {
 124                                 percentage = current;
 125                                 monitor.status(percentage);
 126                         }
 127                 }
 128                 return percentage;
 129         }
 130
 131         public void readIdentities(ReadGraph g) throws Exception {
 132                 extensions = tg.getExtensions();
 133                 resourceCount = tg.getResourceCount();
 134                 identities = new Identity[tg.getIdentityCount()];
 135                 tg.forIdentities(g, new TransferableGraphSourceProcedure<Identity>() {
 136
 137                         int counter = 0;
 138
 139                         @Override
 140                         public void execute(Identity value) throws Exception {
 141                                 identities[counter++] = value;
 142                         }
 143                 });
 144         }
 145
 146         public void findBuiltins(WriteOnlyGraph g) throws DatabaseException {
 147                 RootLibrary = g.getBuiltin("http:/");
 148                 String = g.getBuiltin(CoreInitialization.LAYER0 + "String");
 149                 Library = g.getBuiltin(CoreInitialization.LAYER0 + "Library");
 150                 InstanceOf = g.getBuiltin(CoreInitialization.LAYER0 + "InstanceOf");
 151                 ConsistsOf = g.getBuiltin(CoreInitialization.LAYER0 + "ConsistsOf");
 152                 PartOf = g.getBuiltin(CoreInitialization.LAYER0 + "PartOf");
 153                 HasName = g.getBuiltin(CoreInitialization.LAYER0 + "HasName");
 154                 NameOf = g.getBuiltin(CoreInitialization.LAYER0 + "NameOf");
 155                 ExternalEntity = g.getBuiltin(CoreInitialization.LAYER0 + "ExternalEntity");
 156         }
 157
 158         public void findBuiltins(ReadGraph g) throws DatabaseException {
 159                 RootLibrary = g.getBuiltin("http:/");
 160                 String = g.getBuiltin(CoreInitialization.LAYER0 + "String");
 161                 Library = g.getBuiltin(CoreInitialization.LAYER0 + "Library");
 162                 InstanceOf = g.getBuiltin(CoreInitialization.LAYER0 + "InstanceOf");
 163                 ConsistsOf = g.getBuiltin(CoreInitialization.LAYER0 + "ConsistsOf");
 164                 PartOf = g.getBuiltin(CoreInitialization.LAYER0 + "PartOf");
 165                 HasName = g.getBuiltin(CoreInitialization.LAYER0 + "HasName");
 166                 NameOf = g.getBuiltin(CoreInitialization.LAYER0 + "NameOf");
 167                 ExternalEntity = g.getBuiltin(CoreInitialization.LAYER0 + "ExternalEntity");
 168         }
 169
 170         void addMissing(int handleIndex, String external) {
 171                 allMissingExternals.put(external, handleIndex);
 172                 Set<String> removals = new HashSet<>();
 173                 for(String ext : missingExternals) if(ext.startsWith(external + "/")) return;
 174                 for(String ext : missingExternals) if(external.startsWith(ext + "/")) removals.add(ext);
 175                 missingExternals.removeAll(removals);
 176                 missingExternals.add(external);
 177         }
 178
 179         void prepare(ReadGraph graph) throws Exception {
 180
 181                 Resource target = advisor.getTarget();
 182                 if(target != null)
 183                         indexRoot = graph.syncRequest(new PossibleIndexRoot(target));
 184
 185                 findBuiltins(graph);
 186                 readIdentities(graph);
 187
 188 //              System.err.println("ext: " + extensions);
 189 //              System.err.println("rc: " + resourceCount);
 190 //              System.err.println("ic: " + identities.length);
 191
 192                 ClusterBuilderFactory factory = graph.getService(ClusterBuilderFactory.class);
 193                 ClusterBuilder2 builder = factory.create(vg, false);
 194
 195                 this.handles = new int[resourceCount];
 196                 TIntObjectMap<Identity> identityMap = TransferableGraphUtils.mapIdentities(identities);
 197
 198                 // We must process roots first, because internal identifiers depend on them.
 199                 for(Identity identity : identities) {
 200                         IdentityDefinition definition = identity.definition;
 201                         if (definition instanceof Root) {
 202                                 Root root = (Root) definition;
 203                                 if (root.name.equals(""))
 204                                         handles[identity.resource] = builder.handle(RootLibrary);
 205                                 else {
 206                                         Resource existing = advisor.analyzeRoot(graph, root);
 207                                         if (existing != null)
 208                                                 handles[identity.resource] = builder.handle(existing);
 209                                 }
 210                         }
 211                 }
 212
 213                 for(Identity identity : identities) {
 214                         IdentityDefinition definition = identity.definition;
 215                         if(definition instanceof External) {
 216                                 External def = (External)definition;
 217                                 if(def.parent == -1) {
 218                                     handles[identity.resource] = builder.handle(RootLibrary);
 219                                 } else {
 220                                         if("@inverse".equals(def.name)) {
 221                                                 int parent = handles[def.parent];
 222                                                 int child = builder.handle(graph.getInverse(builder.resource(parent)));
 223                                                 handles[identity.resource] = child;
 224                                         } else {
 225                                                 int handle = handles[def.parent];
 226                                                 Resource parent = handle != 0 ? builder.resource(handle) : null;
 227                                                 // TODO: escape should be removed when names become well-behaving
 228                                                 if(parent != null) {
 229                                                         resolvedParents.put(graph.getURI(parent), parent);
 230                                                     Map<String,Resource> childMap = graph
 231                                             .syncRequest(new UnescapedChildMapOfResource(parent),
 232                                                     TransientCacheAsyncListener.instance());
 233                                                         Resource child = childMap.get(def.name);
 234                                                         if(child == null) {
 235                                                                 addMissing(identity.resource, graph.getURI(parent) + "/" + URIStringUtils.escape(def.name));
 236                                                         } else {
 237                                                                 handles[identity.resource] = builder.handle(child);
 238                                                         }
 239                                                 } else {
 240                                                     addMissing(identity.resource, TransferableGraphUtils.getURI(resourceCount, identityMap, def.parent) + "/" + URIStringUtils.escape(def.name));
 241                                                 }
 242                                         }
 243                                 }
 244                         }
 245                         else if(definition instanceof Internal) {
 246                                 String uri = TransferableGraphUtils.getURI(resourceCount, identityMap, identity.resource);
 247                                 Resource existing = graph.getPossibleResource(uri);
 248                                 if(existing != null) {
 249                                         existingInternalMap.put(identity.resource, existing);
 250                                 }
 251                         }
 252                         else if(definition instanceof Optional) {
 253                                 External def = (External)definition;
 254                                 Resource parent = builder.resource(handles[def.parent]);
 255                                 if(parent != null)
 256                                         handles[identity.resource] = builder.handle(graph.syncRequest(new UnescapedChildMapOfResource(parent)).get(def.name));
 257                         }
 258                 }
 259
 260                 if (!missingExternals.isEmpty() && failOnMissing)
 261                         throw new MissingDependencyException(this);
 262         }
 263
 264         private static boolean failOnMissingEntities() {
 265                 return "true".equalsIgnoreCase(
 266                                 System.getProperty(
 267                                                 "org.simantics.tg.import.failOnMissingEntities",
 268                                                 "false") );
 269         }
 270
 271         @Override
 272         public Resource createChild(WriteOnlyGraph graph, Resource parent, Resource child, String name) throws DatabaseException {
 273                 //System.err.println("child " + parent + " - " + child + " = " + name);
 274             if(child == null) child = graph.newResource();
 275                 Resource nameResource = graph.newResource();
 276                 graph.claim(nameResource, InstanceOf, null, String);
 277                 graph.claimValue(nameResource, name, WriteBindings.STRING);
 278                 graph.claim(child, HasName, NameOf, nameResource);
 279                 graph.claim(parent,  ConsistsOf, PartOf, child);
 280                 return child;
 281         }
 282
 283         int[] getClustering() {
 284                 if (vg != null)
 285                         return null;
 286                 Variant v = extensions.get(Extensions.CLUSTERING);
 287                 if(v == null) return null;
 288                 try {
 289                         return (int[])v.getValue(Bindings.INT_ARRAY);
 290                 } catch (AdaptException e) {
 291                         Logger.defaultLogError(e);
 292                         return null;
 293                 }
 294         }
 295
 296         int[] getClusterSets() {
 297                 if (vg != null)
 298                         return null;
 299                 Variant v = extensions.get(Extensions.CLUSTER_SETS);
 300                 if(v == null) return null;
 301                 try {
 302                         return (int[])v.getValue(Bindings.INT_ARRAY);
 303                 } catch (AdaptException e) {
 304                         Logger.defaultLogError(e);
 305                         return null;
 306                 }
 307         }
 308
 309         boolean needTranslation(Datatype type) {
 310             return resourceUtil.mayHaveResource(type);
 311         }
 312
 313         void findClusterSet(WriteOnlyGraph graph, Resource rootLibrary, int[] clustering, int[] clusterSets, long[] clusters, int id) throws DatabaseException {
 314                 ClusteringSupport support = graph.getService(ClusteringSupport.class);
 315                 if(id == Extensions.ROOT_LIBRARY_CLUSTER_SET || id == Extensions.INDEX_ROOT_CLUSTER_SET) return;
 316                 Resource indexRootClusterSetResource = rootLibrary;
 317                 if(indexRoot != null && support.isClusterSet(indexRoot)) {
 318                         indexRootClusterSetResource = indexRoot;
 319                 } else {
 320                         graph.setClusterSet4NewResource(rootLibrary);
 321                         graph.flushCluster();
 322                 }
 323                 int indexRootCsHandle = builder.handle(indexRootClusterSetResource);
 324                 for(int pos=0,index=0;index<clustering.length;index++) {
 325                         pos += clustering[index];
 326                         if(id < pos) {
 327                                 int cs = clusterSets[index];
 328                                 if(handles[id] == 0) {
 329                                         int csHandle = 0;
 330                                         if(cs == Extensions.ROOT_LIBRARY_CLUSTER_SET) csHandle = builder.handle(rootLibrary);
 331                                         else if(cs == Extensions.INDEX_ROOT_CLUSTER_SET) {
 332                                                 if(indexRoot == null) throw new DatabaseException("No index root was available in TG import.");
 333                                                 csHandle = indexRootCsHandle;
 334                                         }
 335                                         else {
 336                                                 findClusterSet(graph, rootLibrary, clustering, clusterSets, clusters, cs);
 337                                                 csHandle = handles[cs];
 338                                         }
 339
 340                                         if(clusters[index] != 0)
 341                                                 builder.selectCluster(clusters[index]);
 342                                         else if(cs >= 0)
 343                                                 builder.newCluster(csHandle);
 344
 345                                         handles[id] = builder.newResource(csHandle);
 346                                         clusters[index] = support.getCluster(builder.resource(handles[id]));
 347
 348                                         builder.createClusterSet(handles[id]);
 349                                 }
 350                                 return;
 351                         }
 352                 }
 353         }
 354
 355         void createMissing(final WriteOnlyGraph graph) throws Exception {
 356
 357                 if(allMissingExternals.isEmpty()) return;
 358
 359                 XSupport xs = graph.getService(XSupport.class);
 360                 Pair<Boolean,Boolean> serviceMode = xs.getServiceMode();
 361                 xs.setServiceMode(true, false);
 362                 try {
 363                         ArrayList<String> missing = new ArrayList<>(allMissingExternals.keySet());
 364                         Collections.sort(missing);
 365                         for(String uri : missing) {
 366                                 String[] parts = URIStringUtils.splitURI(uri);
 367                                 // URIStringUtils.splitURI returns root URI in non-standard format, so fix it manually as a workaround
 368                                 if (parts[0].equals("http://")) {
 369                                         parts[0] = "http:/";
 370                                 }
 371
 372                                 Resource parent = resolvedParents.get(parts[0]);
 373                                 // TODO: proper exception message
 374                                 if(parent == null) {
 375                                         throw new IllegalStateException("Missing URI: " + uri);
 376                                 }
 377
 378                                 Resource childResource = graph.newResource();
 379                                 graph.claim(childResource, InstanceOf, null, ExternalEntity);
 380
 381                                 Resource nameResource = graph.newResource();
 382                                 graph.claim(nameResource, InstanceOf, null, String);
 383                                 graph.claimValue(nameResource, URIStringUtils.unescape(parts[1]), WriteBindings.STRING);
 384                                 graph.claim(childResource, HasName, NameOf, nameResource);
 385
 386                                 graph.claim(parent, ConsistsOf, PartOf, childResource);
 387
 388                                 resolvedParents.put(uri, childResource);
 389
 390                                 handles[allMissingExternals.get(uri)] = builder.handle(childResource);
 391                         }
 392                 } finally {
 393                         xs.setServiceMode(serviceMode.first, serviceMode.second);
 394                 }
 395         }
 396
 397         void write(final WriteOnlyGraph graph) throws Exception {
 398
 399         final SerialisationSupport ss = graph.getService(SerialisationSupport.class);
 400
 401         ClusterBuilderFactory factory = graph.getService(ClusterBuilderFactory.class);
 402         if(advisor instanceof IImportAdvisor2) {
 403             boolean allowImmutable = ((IImportAdvisor2)advisor).allowImmutableModifications();
 404             builder = factory.create(vg, allowImmutable);
 405         } else {
 406             builder = factory.create(vg, false);
 407         }
 408
 409         createMissing(graph);
 410
 411                 final int[] handles = this.handles;
 412
 413                 int[] clustering = getClustering();
 414                 if(clustering != null) {
 415
 416                         int[] clusterSets = getClusterSets();
 417                         if(clusterSets != null) {
 418
 419                                 assert(clustering.length == clusterSets.length);
 420
 421                                 long[] clusters = new long[clustering.length];
 422
 423                                 // Create clustering
 424                                 for(int i=0;i<clusterSets.length;i++) {
 425                                         findClusterSet(graph, graph.getRootLibrary(), clustering, clusterSets, clusters, clusterSets[i]);
 426                                 }
 427
 428                                 // Then create all resources
 429                                 int i=0;
 430                             for(int j=0;j<clustering.length;j++) {
 431                                 int c = clustering[j];
 432                                 int s = clusterSets[j];
 433                                 int setHandle = 0;
 434                                         if(s == Extensions.ROOT_LIBRARY_CLUSTER_SET)
 435                                                 setHandle = builder.handle(graph.getRootLibrary());
 436                                         else if(s == Extensions.INDEX_ROOT_CLUSTER_SET) {
 437                                                 if(indexRoot == null) throw new DatabaseException("No index root was available in TG import.");
 438                                                 setHandle = builder.handle(indexRoot);
 439                                         }
 440                                         else setHandle = handles[s];
 441                                         // Preserve clustering only for internal resources
 442                                         if(clusters[j] != 0)
 443                                                 builder.selectCluster(clusters[j]);
 444                                         else if(s >= 0)
 445                                                 builder.newCluster(setHandle);
 446                                         for(int r=0;r<c;r++, i++)
 447                                                 if(handles[i] == 0) handles[i] = builder.newResource();
 448                                 }
 449
 450                                 for(;i<handles.length;++i)
 451                                         if(handles[i] == 0) handles[i] = builder.newResource();
 452
 453                         } else {
 454
 455                         int i = 0;
 456                                 for(int c : clustering) {
 457                                         builder.newCluster();
 458                                         for(int r=0;r<c;r++, i++)
 459                                                 if(handles[i] == 0) handles[i] = builder.newResource();
 460                                 }
 461
 462                                 for(;i<handles.length;++i)
 463                                         if(handles[i] == 0) handles[i] = builder.newResource();
 464
 465                         }
 466
 467                 } else {
 468
 469                         // Create blank resources
 470                         for(int i=0;i<handles.length;++i)
 471                                 if(handles[i] == 0) handles[i] = builder.newResource();
 472
 473                 }
 474
 475                 // Internal identities
 476                 for(Identity identity : identities) {
 477                         IdentityDefinition definition = identity.definition;
 478 //                      if(handles[identity.resource] != 0)
 479 //                              continue;
 480                         if(definition instanceof External) {
 481                                 // Already done everything
 482                         }
 483                         else if(definition instanceof Internal) {
 484                                 Internal def = (Internal)definition;
 485                                 Resource external = existingInternalMap.get(identity.resource);
 486                                 if(external != null) {
 487                                         handles[identity.resource] = builder.handle(external);
 488                                 } else {
 489                                         if(handles[identity.resource] != 0)
 490                                                 handles[identity.resource] = builder.handle(advisor.createChild(graph, this, builder.resource(handles[def.parent]), builder.resource(handles[identity.resource]), def.name));
 491                                         else
 492                                                 handles[identity.resource] = builder.handle(advisor.createChild(graph, this, builder.resource(handles[def.parent]), null, def.name));
 493                                 }
 494
 495                         }
 496                         else if(definition instanceof Root) {
 497
 498                                 Root root = (Root)definition;
 499                                 if(handles[identity.resource] != 0)
 500                                         handles[identity.resource] = builder.handle(advisor.createRoot(graph, root, builder.resource(handles[identity.resource])));
 501                                 else
 502                                         handles[identity.resource] = builder.handle(advisor.createRoot(graph, root, null));
 503                         }
 504                         else if(definition instanceof Optional) {
 505                                 Optional def = (Optional)definition;
 506                                 if(handles[identity.resource] != 0) {
 507                                         Resource child = advisor.createChild(graph, this, builder.resource(handles[def.parent]), builder.resource(handles[identity.resource]), def.name);
 508                                         graph.claim(child, InstanceOf, null, Library); // ???
 509                                         handles[identity.resource] = builder.handle(child);
 510                                 } else {
 511                                         Resource child = advisor.createChild(graph, this, builder.resource(handles[def.parent]), null, def.name);
 512                                         graph.claim(child, InstanceOf, null, Library); // ???
 513                                         handles[identity.resource] = builder.handle(child);
 514                                 }
 515                         }
 516                 }
 517
 518                 int[] done = { 0 };
 519                 int[] percentage = { 0 };
 520
 521                 int statementCount = tg.getStatementCount();
 522                 tg.forStatements(null, new TransferableGraphSourceProcedure<int[]>() {
 523
 524                         @Override
 525                         public void execute(int[] value) throws Exception {
 526
 527                                 int sub = value[0];
 528                                 int pred = value[1];
 529                                 int inv = value[2];
 530                                 int obj = value[3];
 531
 532                                 int subject = handles[sub];
 533                                 int predicate = handles[pred];
 534                                 int object = handles[obj];
 535
 536                                 builder.addStatement(graph, subject, predicate, object);
 537                                 if(inv >= 0) {
 538                                     int inverse = handles[inv];
 539                                     builder.addStatement(graph, object, inverse, subject);
 540                                 }
 541
 542                                 // Count from 0% -> 50% => total = statementCount*2
 543                                 percentage[0] = updatePercentage(percentage[0], done[0]++, statementCount*2);
 544
 545                         }
 546
 547                 });
 548
 549                 int valueCount = tg.getValueCount();
 550                 done[0] = 0;
 551
 552                 class ValueProcedure extends InputStream implements TransferableGraphSourceValueProcedure {
 553
 554             private TGResourceUtil util = new TGResourceUtil();
 555                     private DataInput source;
 556
 557             @Override
 558             public void execute(int _resource, Datatype type, DataInput stream) throws Exception {
 559
 560                 source = stream;
 561
 562                 //int file = _resource & 0x80000000;
 563                 int resource = _resource & 0x7FFFFFFF;
 564
 565                 Binding binding = Bindings.getBinding(type);
 566                 Serializer s = Bindings.getSerializer(binding);
 567
 568                 builder.beginValue(handles[resource]);
 569                 if(util.mayHaveResource(type)) {
 570                     Object value = s.deserialize(stream);
 571                     util.adaptValue( binding,  value, new LongAdapter() {
 572                                 @Override
 573                                 public long adapt(long in) {
 574                                     try {
 575                                         return ss.getRandomAccessId(handles[(int)in]);
 576                                     } catch (DatabaseException e) {
 577                                         throw new IllegalStateException(e);
 578                                     }
 579                                 }
 580                             });
 581                     byte[] bytes = s.serialize(value);
 582                     for(byte b : bytes) {
 583                         int val = b;
 584                         if(val < 0) val += 256;
 585                         builder.appendValue(val);
 586                     }
 587                 } else {
 588                     s.skip(this);
 589                 }
 590                 builder.endValue();
 591                 work();
 592
 593             }
 594
 595             @Override
 596             public int read() throws IOException {
 597                 int value = source.readUnsignedByte();
 598                 try {
 599                     builder.appendValue(value);
 600                 } catch (DatabaseException e) {
 601                     LOGGER.error("Failed to write value into database", e);
 602                 }
 603                 return value;
 604             }
 605
 606             @Override
 607             public void rawCopy(int resource, int length, DataInput input) throws Exception {
 608                 builder.beginValue(handles[resource]);
 609                 for (int i = 0; i < length; ++i)
 610                     builder.appendValue(input.readUnsignedByte());
 611                 builder.endValue();
 612                 work();
 613             }
 614
 615             private void work() {
 616                 // Count from 50% -> 100% => [valueCount, valueCount*2)
 617                 percentage[0] = updatePercentage(percentage[0], valueCount + done[0]++, valueCount*2);
 618             }
 619                 };
 620
 621                 tg.forValues2(null, new ValueProcedure());
 622
 623                 for(Resource r : existingInternalMap.valueCollection()) {
 624                         try {
 625                                 graph.deny(r, InstanceOf, null, ExternalEntity, null);
 626                         } catch (DatabaseException e) {
 627                                 graph.deny(r, InstanceOf, null, ExternalEntity, vg);
 628                         }
 629                 }
 630
 631         }
 632
 633         @Override
 634         public long[] getResourceIds(SerialisationSupport serializer) throws DatabaseException {
 635                 final int count = handles.length;
 636                 long[] resourceIds = new long[count];
 637                 for(int i=0;i<count;++i)
 638                     resourceIds[i] = serializer.getRandomAccessId(handles[i]);
 639                 return resourceIds;
 640         }
 641 }