bundles/org.simantics.graph.db/src/org/simantics/graph/db/TransferableGraphs.java

   1 /*******************************************************************************
   2  * Copyright (c) 2007, 2010 Association for Decentralized Information Management
   3  * in Industry THTH ry.
   4  * All rights reserved. This program and the accompanying materials
   5  * are made available under the terms of the Eclipse Public License v1.0
   6  * which accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *     VTT Technical Research Centre of Finland - initial API and implementation
  11  *******************************************************************************/
  12 package org.simantics.graph.db;
  13
  14 import java.io.DataInput;
  15 import java.io.DataOutput;
  16 import java.io.File;
  17 import java.io.IOException;
  18 import java.io.InputStream;
  19 import java.util.ArrayList;
  20 import java.util.Collection;
  21 import java.util.HashSet;
  22 import java.util.TreeMap;
  23 import java.util.UUID;
  24
  25 import org.simantics.databoard.Accessors;
  26 import org.simantics.databoard.Bindings;
  27 import org.simantics.databoard.Datatypes;
  28 import org.simantics.databoard.accessor.error.AccessorConstructionException;
  29 import org.simantics.databoard.accessor.error.AccessorException;
  30 import org.simantics.databoard.accessor.file.FileVariantAccessor;
  31 import org.simantics.databoard.binding.Binding;
  32 import org.simantics.databoard.binding.error.BindingConstructionException;
  33 import org.simantics.databoard.binding.error.DatatypeConstructionException;
  34 import org.simantics.databoard.binding.mutable.Variant;
  35 import org.simantics.databoard.container.DataContainer;
  36 import org.simantics.databoard.container.DataContainers;
  37 import org.simantics.databoard.serialization.SerializationException;
  38 import org.simantics.databoard.serialization.Serializer;
  39 import org.simantics.databoard.type.Datatype;
  40 import org.simantics.databoard.util.binary.BinaryFile;
  41 import org.simantics.databoard.util.binary.RandomAccessBinary;
  42 import org.simantics.db.ReadGraph;
  43 import org.simantics.db.RequestProcessor;
  44 import org.simantics.db.Resource;
  45 import org.simantics.db.Session;
  46 import org.simantics.db.VirtualGraph;
  47 import org.simantics.db.WriteGraph;
  48 import org.simantics.db.WriteOnlyGraph;
  49 import org.simantics.db.common.CommentMetadata;
  50 import org.simantics.db.common.request.ReadRequest;
  51 import org.simantics.db.common.request.WriteOnlyRequest;
  52 import org.simantics.db.common.request.WriteRequest;
  53 import org.simantics.db.exception.CancelTransactionException;
  54 import org.simantics.db.exception.DatabaseException;
  55 import org.simantics.db.service.SerialisationSupport;
  56 import org.simantics.db.service.VirtualGraphSupport;
  57 import org.simantics.graph.db.TransferableGraphSource.TransferableGraphSourceValueProcedure;
  58 import org.simantics.graph.diff.TransferableGraphDelta1;
  59 import org.simantics.graph.representation.Extensions;
  60 import org.simantics.graph.representation.External;
  61 import org.simantics.graph.representation.Identity;
  62 import org.simantics.graph.representation.TransferableGraph1;
  63 import org.simantics.graph.representation.Value;
  64 import org.simantics.utils.datastructures.BinaryFunction;
  65 import org.slf4j.Logger;
  66 import org.slf4j.LoggerFactory;
  67
  68 import gnu.trove.list.array.TIntArrayList;
  69 import gnu.trove.map.hash.TObjectIntHashMap;
  70
  71 public class TransferableGraphs {
  72     final static Logger LOGGER = LoggerFactory.getLogger(TransferableGraphs.class);
  73
  74         public static long[] importGraph(Session session, Object tg, IImportAdvisor advisor) throws DatabaseException, TransferableGraphException {
  75                 if (tg instanceof TransferableGraph1)
  76                 {
  77                         return importGraph1(session, (TransferableGraph1) tg, advisor);
  78                 }
  79                 throw new TransferableGraphException("Cannot import "+tg.getClass().getName());
  80         }
  81
  82         public static long[] importGraph(WriteGraph g, Object tg, IImportAdvisor advisor) throws DatabaseException, TransferableGraphException {
  83                 if (tg instanceof TransferableGraph1)
  84                 {
  85                         return importGraph1(g, (TransferableGraph1) tg, advisor);
  86                 }
  87                 throw new TransferableGraphException("Cannot import "+tg.getClass().getName());
  88         }
  89
  90         public static long[] importGraph(Session session, Object tg) throws DatabaseException, TransferableGraphException {
  91                 if (tg instanceof TransferableGraph1)
  92                 {
  93                         return importGraph1(session, (TransferableGraph1) tg);
  94                 }
  95                 throw new TransferableGraphException("Cannot import "+tg.getClass().getName());
  96         }
  97
  98         public static long[] importGraph(WriteGraph g, Object tg) throws DatabaseException, TransferableGraphException {
  99                 if (tg instanceof TransferableGraph1)
 100                 {
 101                         return importGraph1(g, (TransferableGraph1) tg);
 102                 }
 103                 throw new TransferableGraphException("Cannot import "+tg.getClass().getName());
 104         }
 105
 106         public static Collection<Resource> collectExternals(RequestProcessor processor, final TransferableGraph1 tg) throws DatabaseException, TransferableGraphException {
 107                 final TransferableGraphImportProcess process = new TransferableGraphImportProcess(tg, new ImportAdvisor());
 108                 processor.syncRequest(new ReadRequest() {
 109                         @Override
 110                         public void run(ReadGraph graph) throws DatabaseException {
 111                                 process.prepare(graph);
 112                         }
 113                 });
 114                 HashSet<Resource> result = new HashSet<Resource>();
 115                 for(Identity id : tg.identities) {
 116                         if(id.definition instanceof External) {
 117                                 result.add(process.resources[id.resource]);
 118                         }
 119                 }
 120                 return result;
 121         }
 122
 123         /**
 124          * Imports transferable graph version 1 to the database. Root advisor is used
 125          * to give identities to roots of the transferable graphs. It may be null,
 126          * in which case new resources are created for all roots but the root library.
 127          *
 128          * @param session
 129          * @param tg
 130      * @param advisor root advisor or <code>null</code>
 131          * @throws DatabaseException
 132          */
 133         public static long[] importGraph1(Session session, final TransferableGraph1 tg, final IImportAdvisor advisor_) throws DatabaseException, TransferableGraphException {
 134
 135                 final IImportAdvisor2 advisor = (advisor_ instanceof IImportAdvisor2) ? ((IImportAdvisor2)advisor_) : new WrapperAdvisor(advisor_);
 136
 137                 final TransferableGraphImportProcess process = new TransferableGraphImportProcess(tg,
 138                                 advisor == null ? new ImportAdvisor() : advisor);
 139                 session.syncRequest(new ReadRequest() {
 140                         @Override
 141                         public void run(ReadGraph graph) throws DatabaseException {
 142                                 process.prepare(graph);
 143                         }
 144                 });
 145                 session.syncRequest(new WriteOnlyRequest() {
 146                         @Override
 147                         public void perform(WriteOnlyGraph graph) throws DatabaseException {
 148                                 advisor.beforeWrite(graph, process);
 149                                 process.write(graph);
 150                                 advisor.afterWrite(graph, process);
 151                         }
 152                 });
 153                 return process.getResourceIds(
 154                                 session.getService(SerialisationSupport.class));
 155         }
 156
 157         public static void importGraph1(Session session, final TransferableGraph1 tg, IImportAdvisor advisor, final BinaryFunction<Boolean, WriteOnlyGraph, TransferableGraphImportProcess> callback) throws DatabaseException, TransferableGraphException {
 158                 final TransferableGraphImportProcess process = new TransferableGraphImportProcess(tg,
 159                                 advisor == null ? new ImportAdvisor() : advisor);
 160                 session.syncRequest(new ReadRequest() {
 161                         @Override
 162                         public void run(ReadGraph graph) throws DatabaseException {
 163                                 process.prepare(graph);
 164                         }
 165                 });
 166                 session.syncRequest(new WriteOnlyRequest() {
 167                         @Override
 168                         public void perform(WriteOnlyGraph graph) throws DatabaseException {
 169                                 process.write(graph);
 170                                 if(callback != null)
 171                                         callback.call(graph, process);
 172                         }
 173                 });
 174         }
 175
 176     public static void importGraph1(Session session, final TransferableGraphSource tg, IImportAdvisor advisor) throws Exception {
 177         importGraph1(session, tg, advisor, null);
 178     }
 179
 180     public static void importGraph1(Session session, final TransferableGraphSource tg, IImportAdvisor advisor, TGStatusMonitor monitor) throws DatabaseException {
 181         importGraph1(session, null, tg, advisor, monitor);
 182     }
 183
 184     public static void importGraph1(Session session, VirtualGraph vg, final TransferableGraphSource tg, IImportAdvisor advisor_, TGStatusMonitor monitor) throws DatabaseException {
 185
 186         final IImportAdvisor2 advisor = (advisor_ instanceof IImportAdvisor2) ? ((IImportAdvisor2)advisor_) : new WrapperAdvisor(advisor_);
 187
 188                 final StreamingTransferableGraphImportProcess process = new StreamingTransferableGraphImportProcess(session, vg, tg, advisor);
 189                 session.syncRequest(new ReadRequest() {
 190                         @Override
 191                         public void run(ReadGraph graph) throws DatabaseException {
 192                                 try {
 193                                         process.prepare(graph);
 194                                 } catch (DatabaseException e) {
 195                                         throw e;
 196                                 } catch (Exception e) {
 197                                         throw new DatabaseException(e);
 198                                 }
 199                         }
 200                 });
 201                 session.syncRequest(new WriteOnlyRequest(vg) {
 202                         @Override
 203                         public void perform(WriteOnlyGraph graph) throws DatabaseException {
 204                                 try {
 205                                     advisor.beforeWrite(graph, process);
 206                                         process.write(graph);
 207                     advisor.afterWrite(graph, process);
 208                                 } catch (Exception e) {
 209                                         throw new DatabaseException(e);
 210                                 }
 211                         }
 212                 });
 213         }
 214
 215         public static void importGraph1WithMonitor(Session session, final TransferableGraph1 tg, IImportAdvisor advisor_, TGStatusMonitor monitor) throws DatabaseException {
 216                 final IImportAdvisor2 advisor = (advisor_ instanceof IImportAdvisor2) ? ((IImportAdvisor2)advisor_) : new WrapperAdvisor(advisor_);
 217                 final TransferableGraphImportProcess process = new TransferableGraphImportProcess(tg,
 218                                 advisor == null ? new ImportAdvisor() : advisor, monitor);
 219                 session.syncRequest(new ReadRequest() {
 220                         @Override
 221                         public void run(ReadGraph graph) throws DatabaseException {
 222                                 process.prepare(graph);
 223                         }
 224                 });
 225                 session.syncRequest(new WriteOnlyRequest() {
 226                         @Override
 227                         public void perform(WriteOnlyGraph graph) throws DatabaseException {
 228                                 advisor.beforeWrite(graph, process);
 229                                 process.write2(graph);
 230                                 advisor.afterWrite(graph, process);
 231                                 CommentMetadata comments = graph.getMetadata(CommentMetadata.class);
 232                                 comments.add("Imported transferable graph with " + tg.resourceCount + " resources");
 233                                 graph.addMetadata(comments);
 234                         }
 235                 });
 236         }
 237
 238         public static void importGraph1WithChanges(Session session, final TransferableGraph1 tg, IImportAdvisor advisor, final BinaryFunction<Boolean, WriteGraph, TransferableGraphImportProcess> callback) throws DatabaseException, TransferableGraphException {
 239                 final TransferableGraphImportProcess process = new TransferableGraphImportProcess(tg,
 240                                 advisor == null ? new ImportAdvisor() : advisor);
 241                 session.syncRequest(new ReadRequest() {
 242                         @Override
 243                         public void run(ReadGraph graph) throws DatabaseException {
 244                                 process.prepare(graph);
 245                         }
 246                 });
 247                 session.syncRequest(new WriteRequest() {
 248                         @Override
 249                         public void perform(WriteGraph graph) throws DatabaseException {
 250                                 process.write2(graph);
 251                 CommentMetadata comments = graph.getMetadata(CommentMetadata.class);
 252                 comments.add("Imported transferable graph with " + tg.resourceCount + " resources");
 253                 graph.addMetadata(comments);
 254                                 if(callback != null)
 255                                         callback.call(graph, process);
 256                         }
 257                 });
 258         }
 259
 260         public static long[] importGraph1(Session session, final TransferableGraph1 tg) throws DatabaseException, TransferableGraphException {
 261                 final TransferableGraphImportProcess process =
 262                         new TransferableGraphImportProcess(tg,
 263                                 null);
 264                 session.syncRequest(new ReadRequest() {
 265                         @Override
 266                         public void run(ReadGraph graph) throws DatabaseException {
 267                                 process.prepare(graph);
 268                         }
 269                 });
 270                 session.syncRequest(new WriteOnlyRequest() {
 271                         @Override
 272                         public void perform(WriteOnlyGraph graph) throws DatabaseException {
 273                                 process.write(graph);
 274                         }
 275                 });
 276                 return process.getResourceIds(
 277                                 session.getService(SerialisationSupport.class));
 278         }
 279
 280         public static long[] importGraph1(WriteGraph graph, final TransferableGraph1 tg) throws DatabaseException, TransferableGraphException {
 281                 final TransferableGraphImportProcess process =
 282                         new TransferableGraphImportProcess(tg,
 283                                 null);
 284                 process.prepare(graph);
 285                 process.write2(graph);
 286                 return process.getResourceIds(
 287                                 graph.getSession().getService(SerialisationSupport.class));
 288         }
 289
 290         /**
 291          * Import transferable graph version 1 to the database. Root advisor is used
 292          * to give identities to roots of the transferable graphs. It may be null,
 293          * in which case new resources are created for all roots but the root library.
 294          *
 295          * @param session
 296          * @param tg
 297      * @param advisor root advisor or <code>null</code>
 298          * @throws DatabaseException
 299          */
 300         public static long[] importGraph1(WriteGraph graph, TransferableGraph1 tg, IImportAdvisor advisor) throws DatabaseException {
 301             return importGraph1(graph, tg, advisor, null);
 302         }
 303
 304         public static long[] importGraph1(WriteGraph graph, TransferableGraph1 tg, IImportAdvisor advisor, TGStatusMonitor monitor) throws DatabaseException {
 305                 TransferableGraphImportProcess process = new TransferableGraphImportProcess(tg,
 306                                 advisor == null ? new ImportAdvisor() : advisor, monitor);
 307                 process.prepare(graph);
 308                 if(advisor instanceof IImportAdvisor2) ((IImportAdvisor2)advisor).beforeWrite(graph, process);
 309                 process.write2(graph);
 310                 if(advisor instanceof IImportAdvisor2) ((IImportAdvisor2)advisor).afterWrite(graph, process);
 311                 return process.getResourceIds(
 312                                 graph.getSession().getService(SerialisationSupport.class));
 313         }
 314
 315         public static long[] applyDelta(WriteGraph graph, long[] oldResources, TransferableGraphDelta1 delta) throws DatabaseException {
 316                 SerialisationSupport serializer =
 317                         graph.getSession().getService(SerialisationSupport.class);
 318
 319                 TGToGraphMap aMap = new TGToGraphMap(delta.a);
 320                 aMap.addOldResources(serializer, oldResources);
 321                 aMap.deny(graph);
 322
 323                 TGToGraphMap bMap = new TGToGraphMap(delta.b);
 324                 bMap.addMappedOldResources(serializer, delta.aToB, aMap.getResources());
 325                 bMap.prepare(graph);
 326                 bMap.claim(graph);
 327
 328                 return bMap.getResources(serializer);
 329         }
 330
 331         public static boolean hasChanges(ReadGraph graph, long[] oldResources, TransferableGraphDelta1 delta) throws DatabaseException {
 332
 333                 SerialisationSupport serializer =
 334                         graph.getSession().getService(SerialisationSupport.class);
 335
 336                 TGToGraphMap aMap = new TGToGraphMap(delta.a);
 337                 aMap.addOldResources(serializer, oldResources);
 338                 if(aMap.checkDeny(graph)) return true;
 339
 340                 TGToGraphMap bMap = new TGToGraphMap(delta.b);
 341                 bMap.addMappedOldResources(serializer, delta.aToB, aMap.getResources());
 342                 bMap.prepare(graph);
 343                 return bMap.checkClaim(graph);
 344
 345         }
 346
 347         public static void uninstallGraph(WriteGraph writeGraph, TransferableGraph1 graph,
 348                         ImportAdvisor advisor) throws TransferableGraphException {
 349                 // TODO HANNU IMPLEMENTS
 350                 throw new UnsupportedOperationException();
 351         }
 352
 353         public static long[] importVirtualGraph(Session session, final VirtualGraph vg, final TransferableGraph1 tg, IImportAdvisor advisor) throws DatabaseException {
 354                 final TransferableGraphImportProcess process = new TransferableGraphImportProcess(tg,
 355                                 advisor == null ? new ImportAdvisor() : advisor);
 356                 session.syncRequest(new ReadRequest() {
 357                         @Override
 358                         public void run(ReadGraph graph) throws DatabaseException {
 359                                 process.prepare(graph);
 360                         }
 361                 });
 362                 session.syncRequest(new WriteOnlyRequest(vg) {
 363                         @Override
 364                         public void perform(WriteOnlyGraph graph) throws DatabaseException {
 365                                 // Needed because process#write does not support virtual WriteOnlyGraph
 366                                 if (vg != null)
 367                                         process.write2(graph);
 368                                 else
 369                                         process.write(graph);
 370                         }
 371
 372                 });
 373                 return process.getResourceIds(session.getService(SerialisationSupport.class));
 374         }
 375
 376         public static long[] importVirtualGraph(WriteGraph graph, final TransferableGraph1 tg, IImportAdvisor advisor) throws DatabaseException {
 377                 final TransferableGraphImportProcess process = new TransferableGraphImportProcess(tg,
 378                                 advisor == null ? new ImportAdvisor() : advisor);
 379                 process.prepare(graph);
 380                 process.write2(graph);
 381                 return process.getResourceIds(graph.getService(SerialisationSupport.class));
 382         }
 383
 384         public static TransferableGraph1 readGraph(File file) throws TransferableGraphException {
 385                 FileVariantAccessor va = null;
 386                 try {
 387                         va = Accessors.openAccessor(file);
 388                         Datatype type = va.getContentType();
 389                         if(type.equals(Datatypes.getDatatype(TransferableGraph1.class)))
 390                                 return  (TransferableGraph1)va.getContentValue(Bindings.getBinding(TransferableGraph1.class));
 391                         else
 392                                 throw new SerializationException("Unknown transferable graph data type.");
 393                 } catch (AccessorException e) {
 394                         throw new TransferableGraphException(e);
 395                 } catch (BindingConstructionException e) {
 396                         throw new TransferableGraphException(e);
 397                 } catch (SerializationException e) {
 398                         throw new TransferableGraphException(e);
 399                 } catch (AccessorConstructionException e) {
 400                         throw new TransferableGraphException(e);
 401                 } catch (DatatypeConstructionException e) {
 402                         throw new TransferableGraphException(e);
 403                 } finally {
 404                         if(va != null) {
 405                                 try {
 406                                         va.close();
 407                                 } catch (AccessorException e) {
 408                                 }
 409                         }
 410                 }
 411         }
 412
 413         public static void importVirtualGraph(Session session, VirtualGraph vg, File file) throws TransferableGraphException {
 414                 try {
 415                         importVirtualGraph(session, vg, readGraph(file), new ImportAdvisor());
 416                 } catch (DatabaseException e) {
 417                         throw new TransferableGraphException(e);
 418                 }
 419         }
 420
 421         public static VirtualGraph importVirtualGraph(Session session, File file) throws TransferableGraphException {
 422                 VirtualGraphSupport support = session.getService(VirtualGraphSupport.class);
 423                 VirtualGraph vg = support.getMemoryPersistent(UUID.randomUUID().toString());
 424                 importVirtualGraph(session, vg, file);
 425                 return vg;
 426         }
 427
 428         public static void writeTransferableGraph(RequestProcessor processor, final String format, final TransferableGraphSource source, File target) throws Exception {
 429                 writeTransferableGraph(processor, format, 1, source, target);
 430         }
 431
 432         public static void writeTransferableGraph(RequestProcessor processor, final String format, final int version, final TransferableGraphSource source, File target) throws Exception {
 433                 writeTransferableGraph(processor, format, version, new TreeMap<String,Variant>(), source, target);
 434         }
 435
 436         public static void writeTransferableGraph(RequestProcessor processor, String format, int version, TreeMap<String, Variant> metadata, TransferableGraphSource source, File target) throws Exception {
 437                 writeTransferableGraph(processor, format, version, metadata, source, target, TGStatusMonitor.NULL_MONITOR);
 438         }
 439
 440         public static void writeTransferableGraph(RequestProcessor processor, String format, int version, TreeMap<String, Variant> metadata, TransferableGraphSource source, File target, TGStatusMonitor monitor) throws Exception {
 441                 final Serializer datatypeSerializer = Bindings.getSerializerUnchecked(Datatype.class);
 442                 try (RandomAccessBinary out = new BinaryFile(target, 128*1024)) {
 443                         DataContainer container = new DataContainer(format, version, metadata, null);
 444                         DataContainers.writeHeader(out, container);
 445                         datatypeSerializer.serialize((DataOutput) out, Datatypes.getDatatypeUnchecked(TransferableGraph1.class));
 446                         writeTransferableGraph(processor, source, out, monitor);
 447                 }
 448         }
 449
 450         private static TGStatusMonitor safeMonitor(TGStatusMonitor mon) {
 451                 return mon == null ? TGStatusMonitor.NULL_MONITOR : mon;
 452         }
 453
 454         private static class CopyingInputStream extends InputStream {
 455                 public DataInput in;
 456                 public DataOutput out;
 457
 458                 @Override
 459                 public int read() throws IOException {
 460                         int value = in.readUnsignedByte();
 461                         out.write(value);
 462                         return value;
 463                 }
 464         }
 465
 466         private static long copy(byte[] buffer, DataInput in, DataOutput out, long bytesToCopy) throws IOException {
 467                 int read = 0;
 468                 long bufferLength = buffer.length;
 469                 while (read < bytesToCopy) {
 470                         int l = (int) Math.min(bufferLength, bytesToCopy-read);
 471                         in.readFully(buffer, 0, l);
 472                         out.write(buffer, 0, l);
 473                         read += l;
 474                 }
 475                 return read;
 476         }
 477
 478         private static final int LITERAL_VALUE_IO_BUFFER_SIZE = 128 * 1024;
 479
 480         private static void writeTransferableGraph(RequestProcessor processor, final TransferableGraphSource source, final RandomAccessBinary out, TGStatusMonitor monitor) throws Exception {
 481                 long start = System.nanoTime();
 482
 483                 final Serializer datatypeSerializer = Bindings.getSerializerUnchecked(Datatype.class);
 484                 final Serializer identitySerializer = Bindings.getSerializerUnchecked(Identity.class);
 485                 final Serializer extensionSerializer = Bindings.getSerializerUnchecked(Extensions.class);
 486
 487                 int resourceCount = source.getResourceCount();
 488                 //System.err.println("resourceCount: " + resourceCount);
 489                 out.writeInt(resourceCount);
 490                 extensionSerializer.serialize(out, new Extensions(source.getExtensions()));
 491
 492 //              System.err.println("resource count: " + source.getResourceCount());
 493 //              System.err.println("identity count: " + source.getIdentityCount());
 494
 495                 byte[] buffer = new byte[LITERAL_VALUE_IO_BUFFER_SIZE];
 496
 497                 processor.syncRequest(new ReadRequest() {
 498                         @Override
 499                         public void run(ReadGraph graph) throws DatabaseException {
 500                                 try {
 501                                         if (monitor.isCanceled())
 502                                                 throw new CancelTransactionException();
 503
 504                                         int identityCount = source.getIdentityCount();
 505                                         TGStatusMonitor.Updater identityProgress = new TGStatusMonitor.Updater(safeMonitor(monitor), 0, 33, identityCount);
 506                                         out.writeInt(identityCount);
 507                                         //System.err.println("identities: " + identityCount);
 508                                         source.forIdentities(graph, value -> {
 509                                                 //System.err.println("id: " + value);
 510                                                 identitySerializer.serialize(out, value);
 511                                                 identityProgress.worked(1);
 512                                         });
 513
 514                                         if (monitor.isCanceled())
 515                                                 throw new CancelTransactionException();
 516
 517                                         long statementCountPos = out.position();
 518                                         int originalStatementCount = source.getStatementCount();
 519                                         TGStatusMonitor.Updater statementProgress = new TGStatusMonitor.Updater(safeMonitor(monitor), 34, 66, originalStatementCount);
 520                                         out.writeInt(originalStatementCount);
 521                                         //System.err.println("original statementCount: " + originalStatementCount);
 522                                         int[] statementCounter = { 0 };
 523                                         source.forStatements(graph, r -> {
 524                                                 for (int i = 0; i < 4; ++i)
 525                                                         out.writeInt(r[i]);
 526                                                 statementCounter[0]++;
 527                                                 //System.err.println("stm " + (statementCounter[0]) + ": " + r[0] + " " + r[1] + " " + r[2] + " " + r[3]);
 528                                                 statementProgress.worked(1);
 529                                         });
 530                                         //System.err.println("wrote " + statementCounter[0] + " statements, " + (statementCounter[0]*4)+ " integers");
 531
 532                                         // Rewrite statement count after knowing exactly how many
 533                                         // statements were written. It is possible that some
 534                                         // statements get filtered out at this stage and the
 535                                         // original statement count does not reflect that.
 536                                         long afterStatementsPos = out.position();
 537                                         out.position(statementCountPos);
 538                                         out.writeInt(statementCounter[0]*4);
 539                                         out.position(afterStatementsPos);
 540
 541                                         if (monitor.isCanceled())
 542                                                 throw new CancelTransactionException();
 543
 544                                         int valueCount = source.getValueCount();
 545                                         TGStatusMonitor.Updater valueProgress = new TGStatusMonitor.Updater(safeMonitor(monitor), 67, 100, valueCount);
 546                                         out.writeInt(valueCount);
 547 //                                      System.err.println("valueCount: " + valueCount);
 548                                         CopyingInputStream cis = new CopyingInputStream();
 549                                         cis.out = out;
 550                                         source.forValues2(graph, new TransferableGraphSourceValueProcedure() {
 551                                                 TObjectIntHashMap<Object> identities = new TObjectIntHashMap<>();
 552
 553                                                 @Override
 554                                                 public void rawCopy(int resource, int length, DataInput input) throws Exception {
 555                                                         out.writeInt(resource);
 556                                                         long copied = copy(buffer, input, out, length);
 557                                                         assert copied == length;
 558                                                         //System.err.println("value " + (num++) + ": raw variant, " + length + " bytes, copied " + copied + " bytes");
 559                                                         valueProgress.worked(1);
 560                                                 }
 561
 562                                                 @Override
 563                                                 public void execute(int resource, Datatype type, DataInput input) throws Exception {
 564                                                         out.writeInt(resource);
 565                                                         identities.clear();
 566                                                         datatypeSerializer.serialize(out, identities, type);
 567                                                         Binding binding = Bindings.getBinding(type);
 568                                                         Serializer serializer = Bindings.getSerializer(binding);
 569                                                         cis.in = input;
 570                                                         serializer.skip(cis);
 571                                                         cis.in = null;
 572                                                         valueProgress.worked(1);
 573                                                 }
 574                                         });
 575
 576                                 } catch (DatabaseException e) {
 577                                         throw e;
 578                                 } catch (Exception e) {
 579                                         throw new DatabaseException(e);
 580                                 }
 581                         }
 582                 });
 583
 584                 long end = System.nanoTime();
 585                 LOGGER.info("Wrote transferable graph in {} seconds.", 1e-9*(end-start));
 586         }
 587
 588         public static TransferableGraph1 create(ReadGraph graph, TransferableGraphSource source) throws DatabaseException {
 589
 590                 final TIntArrayList statements = new TIntArrayList();
 591                 final ArrayList<Value> values = new ArrayList<>();
 592                 final ArrayList<Identity> identities = new ArrayList<>();
 593
 594                 try {
 595
 596                         source.forStatements(graph, r -> statements.addAll(r));
 597                         source.forValues(graph, v -> values.add(v));
 598                         source.forIdentities(graph, i -> identities.add(i));
 599
 600                         return new TransferableGraph1(source.getResourceCount(),
 601                                         identities.toArray(new Identity[identities.size()]),
 602                                         statements.toArray(),
 603                                         values.toArray(new Value[values.size()]),
 604                                         source.getExtensions());
 605
 606                 } catch (Exception e) {
 607
 608                         throw new DatabaseException(e);
 609
 610                 }
 611
 612         }
 613
 614 }