Added more TransferableGraphs.writeTransferableGraph methods
[simantics/platform.git] / bundles / org.simantics.graph.db / src / org / simantics / graph / db / TransferableGraphs.java
1 /*******************************************************************************
2  * Copyright (c) 2007, 2010 Association for Decentralized Information Management
3  * in Industry THTH ry.
4  * All rights reserved. This program and the accompanying materials
5  * are made available under the terms of the Eclipse Public License v1.0
6  * which accompanies this distribution, and is available at
7  * http://www.eclipse.org/legal/epl-v10.html
8  *
9  * Contributors:
10  *     VTT Technical Research Centre of Finland - initial API and implementation
11  *******************************************************************************/
12 package org.simantics.graph.db;
13
14 import java.io.DataInput;
15 import java.io.DataOutput;
16 import java.io.File;
17 import java.io.IOException;
18 import java.io.InputStream;
19 import java.util.ArrayList;
20 import java.util.Collection;
21 import java.util.HashSet;
22 import java.util.TreeMap;
23 import java.util.UUID;
24 import java.util.function.BiFunction;
25
26 import org.simantics.databoard.Accessors;
27 import org.simantics.databoard.Bindings;
28 import org.simantics.databoard.Datatypes;
29 import org.simantics.databoard.accessor.error.AccessorConstructionException;
30 import org.simantics.databoard.accessor.error.AccessorException;
31 import org.simantics.databoard.accessor.file.FileVariantAccessor;
32 import org.simantics.databoard.binding.Binding;
33 import org.simantics.databoard.binding.error.BindingConstructionException;
34 import org.simantics.databoard.binding.error.DatatypeConstructionException;
35 import org.simantics.databoard.binding.mutable.Variant;
36 import org.simantics.databoard.container.DataContainer;
37 import org.simantics.databoard.container.DataContainers;
38 import org.simantics.databoard.serialization.SerializationException;
39 import org.simantics.databoard.serialization.Serializer;
40 import org.simantics.databoard.type.Datatype;
41 import org.simantics.databoard.util.binary.BinaryFile;
42 import org.simantics.databoard.util.binary.RandomAccessBinary;
43 import org.simantics.db.ReadGraph;
44 import org.simantics.db.RequestProcessor;
45 import org.simantics.db.Resource;
46 import org.simantics.db.Session;
47 import org.simantics.db.VirtualGraph;
48 import org.simantics.db.WriteGraph;
49 import org.simantics.db.WriteOnlyGraph;
50 import org.simantics.db.common.CommentMetadata;
51 import org.simantics.db.common.request.ReadRequest;
52 import org.simantics.db.common.request.WriteOnlyRequest;
53 import org.simantics.db.common.request.WriteRequest;
54 import org.simantics.db.exception.CancelTransactionException;
55 import org.simantics.db.exception.DatabaseException;
56 import org.simantics.db.service.SerialisationSupport;
57 import org.simantics.db.service.VirtualGraphSupport;
58 import org.simantics.graph.db.TransferableGraphSource.TransferableGraphSourceValueProcedure;
59 import org.simantics.graph.diff.TransferableGraphDelta1;
60 import org.simantics.graph.representation.Extensions;
61 import org.simantics.graph.representation.External;
62 import org.simantics.graph.representation.Identity;
63 import org.simantics.graph.representation.TransferableGraph1;
64 import org.simantics.graph.representation.Value;
65 import org.slf4j.Logger;
66 import org.slf4j.LoggerFactory;
67
68 import gnu.trove.list.array.TIntArrayList;
69 import gnu.trove.map.hash.TObjectIntHashMap;
70
71 public class TransferableGraphs {
72     final static Logger LOGGER = LoggerFactory.getLogger(TransferableGraphs.class); 
73
74         public static long[] importGraph(Session session, Object tg, IImportAdvisor advisor) throws DatabaseException, TransferableGraphException {
75                 if (tg instanceof TransferableGraph1) 
76                 {
77                         return importGraph1(session, (TransferableGraph1) tg, advisor);
78                 }
79                 throw new TransferableGraphException("Cannot import "+tg.getClass().getName());
80         }
81
82         public static long[] importGraph(WriteGraph g, Object tg, IImportAdvisor advisor) throws DatabaseException, TransferableGraphException {
83                 if (tg instanceof TransferableGraph1) 
84                 {
85                         return importGraph1(g, (TransferableGraph1) tg, advisor);
86                 }
87                 throw new TransferableGraphException("Cannot import "+tg.getClass().getName());
88         }
89         
90         public static long[] importGraph(Session session, Object tg) throws DatabaseException, TransferableGraphException {
91                 if (tg instanceof TransferableGraph1) 
92                 {
93                         return importGraph1(session, (TransferableGraph1) tg);
94                 }
95                 throw new TransferableGraphException("Cannot import "+tg.getClass().getName());
96         }
97
98         public static long[] importGraph(WriteGraph g, Object tg) throws DatabaseException, TransferableGraphException {
99                 if (tg instanceof TransferableGraph1) 
100                 {
101                         return importGraph1(g, (TransferableGraph1) tg);
102                 }
103                 throw new TransferableGraphException("Cannot import "+tg.getClass().getName());
104         }
105         
106         public static Collection<Resource> collectExternals(RequestProcessor processor, final TransferableGraph1 tg) throws DatabaseException, TransferableGraphException {
107                 final TransferableGraphImportProcess process = new TransferableGraphImportProcess(tg, new ImportAdvisor());
108                 processor.syncRequest(new ReadRequest() {
109                         @Override
110                         public void run(ReadGraph graph) throws DatabaseException {
111                                 process.prepare(graph);
112                         }
113                 });
114                 HashSet<Resource> result = new HashSet<Resource>();
115                 for(Identity id : tg.identities) {
116                         if(id.definition instanceof External) {
117                                 result.add(process.resources[id.resource]);
118                         }
119                 }
120                 return result;
121         }
122         
123         /**
124          * Imports transferable graph version 1 to the database. Root advisor is used
125          * to give identities to roots of the transferable graphs. It may be null,
126          * in which case new resources are created for all roots but the root library.
127          * 
128          * @param session
129          * @param tg
130      * @param advisor root advisor or <code>null</code>
131          * @throws DatabaseException
132          */
133         public static long[] importGraph1(Session session, final TransferableGraph1 tg, final IImportAdvisor advisor_) throws DatabaseException, TransferableGraphException {
134                 
135                 final IImportAdvisor2 advisor = (advisor_ instanceof IImportAdvisor2) ? ((IImportAdvisor2)advisor_) : new WrapperAdvisor(advisor_);
136                 
137                 final TransferableGraphImportProcess process = new TransferableGraphImportProcess(tg, 
138                                 advisor == null ? new ImportAdvisor() : advisor);
139                 session.syncRequest(new ReadRequest() {
140                         @Override
141                         public void run(ReadGraph graph) throws DatabaseException {
142                                 process.prepare(graph);
143                         }
144                 });             
145                 session.syncRequest(new WriteOnlyRequest() {
146                         @Override
147                         public void perform(WriteOnlyGraph graph) throws DatabaseException {
148                                 advisor.beforeWrite(graph, process);
149                                 process.write(graph);
150                                 advisor.afterWrite(graph, process);
151                         }
152                 });
153                 return process.getResourceIds(
154                                 session.getService(SerialisationSupport.class));
155         }
156
157         public static void importGraph1(Session session, final TransferableGraph1 tg, IImportAdvisor advisor, final BiFunction<WriteOnlyGraph, TransferableGraphImportProcess, Boolean> callback) throws DatabaseException, TransferableGraphException {
158                 final TransferableGraphImportProcess process = new TransferableGraphImportProcess(tg, 
159                                 advisor == null ? new ImportAdvisor() : advisor);
160                 session.syncRequest(new ReadRequest() {
161                         @Override
162                         public void run(ReadGraph graph) throws DatabaseException {
163                                 process.prepare(graph);
164                         }
165                 });
166                 session.syncRequest(new WriteOnlyRequest() {
167                         @Override
168                         public void perform(WriteOnlyGraph graph) throws DatabaseException {
169                                 process.write(graph);
170                                 if(callback != null)
171                                         callback.apply(graph, process);
172                         }
173                 });
174         }
175
176     public static ImportResult importGraph1(Session session, final TransferableGraphSource tg, IImportAdvisor advisor) throws Exception {
177         return importGraph1(session, tg, advisor, null);
178     }
179
180     public static ImportResult importGraph1(Session session, final TransferableGraphSource tg, IImportAdvisor advisor, TGStatusMonitor monitor) throws DatabaseException {
181         return importGraph1(session, null, tg, advisor, monitor);
182     }
183
184     public static ImportResult importGraph1(Session session, VirtualGraph vg, final TransferableGraphSource tg, IImportAdvisor advisor_, TGStatusMonitor monitor) throws DatabaseException {
185         
186         final IImportAdvisor2 advisor = (advisor_ instanceof IImportAdvisor2) ? ((IImportAdvisor2)advisor_) : new WrapperAdvisor(advisor_);
187
188                 final StreamingTransferableGraphImportProcess process = new StreamingTransferableGraphImportProcess(session, vg, tg, advisor, monitor);
189                 session.syncRequest(new ReadRequest() {
190                         @Override
191                         public void run(ReadGraph graph) throws DatabaseException {
192                                 try {
193                                         process.prepare(graph);
194                                 } catch (DatabaseException e) {
195                                         throw e;
196                                 } catch (Exception e) {
197                                         throw new DatabaseException(e);
198                                 }
199                         }
200                 });
201                 session.syncRequest(new WriteOnlyRequest(vg) {
202                         @Override
203                         public void perform(WriteOnlyGraph graph) throws DatabaseException {
204                                 try {
205                                     advisor.beforeWrite(graph, process);
206                                         process.write(graph);
207                     advisor.afterWrite(graph, process);
208                                 } catch (Exception e) {
209                                         throw new DatabaseException(e);
210                                 }
211                         }
212                 });
213
214                 return new ImportResult(process.missingExternals);
215         }
216
217         public static void importGraph1WithMonitor(Session session, final TransferableGraph1 tg, IImportAdvisor advisor_, TGStatusMonitor monitor) throws DatabaseException {
218                 final IImportAdvisor2 advisor = (advisor_ instanceof IImportAdvisor2) ? ((IImportAdvisor2)advisor_) : new WrapperAdvisor(advisor_);
219                 final TransferableGraphImportProcess process = new TransferableGraphImportProcess(tg, 
220                                 advisor == null ? new ImportAdvisor() : advisor, monitor);
221                 session.syncRequest(new ReadRequest() {
222                         @Override
223                         public void run(ReadGraph graph) throws DatabaseException {
224                                 process.prepare(graph);
225                         }
226                 });
227                 session.syncRequest(new WriteOnlyRequest() {
228                         @Override
229                         public void perform(WriteOnlyGraph graph) throws DatabaseException {
230                                 advisor.beforeWrite(graph, process);
231                                 process.write2(graph);
232                                 advisor.afterWrite(graph, process);
233                                 CommentMetadata comments = graph.getMetadata(CommentMetadata.class);
234                                 comments.add("Imported transferable graph with " + tg.resourceCount + " resources");
235                                 graph.addMetadata(comments);
236                         }
237                 });
238         }
239
240         public static void importGraph1WithChanges(Session session, final TransferableGraph1 tg, IImportAdvisor advisor, final BiFunction<WriteGraph, TransferableGraphImportProcess, Boolean> callback) throws DatabaseException, TransferableGraphException {
241                 final TransferableGraphImportProcess process = new TransferableGraphImportProcess(tg, 
242                                 advisor == null ? new ImportAdvisor() : advisor);
243                 session.syncRequest(new ReadRequest() {
244                         @Override
245                         public void run(ReadGraph graph) throws DatabaseException {
246                                 process.prepare(graph);
247                         }
248                 });
249                 session.syncRequest(new WriteRequest() {
250                         @Override
251                         public void perform(WriteGraph graph) throws DatabaseException {
252                                 process.write2(graph);
253                 CommentMetadata comments = graph.getMetadata(CommentMetadata.class);
254                 comments.add("Imported transferable graph with " + tg.resourceCount + " resources");
255                 graph.addMetadata(comments);
256                                 if(callback != null)
257                                         callback.apply(graph, process);
258                         }
259                 });
260         }
261         
262         public static long[] importGraph1(Session session, final TransferableGraph1 tg) throws DatabaseException, TransferableGraphException {
263                 final TransferableGraphImportProcess process = 
264                         new TransferableGraphImportProcess(tg, 
265                                 null);
266                 session.syncRequest(new ReadRequest() {
267                         @Override
268                         public void run(ReadGraph graph) throws DatabaseException {
269                                 process.prepare(graph);
270                         }
271                 });             
272                 session.syncRequest(new WriteOnlyRequest() {
273                         @Override
274                         public void perform(WriteOnlyGraph graph) throws DatabaseException {
275                                 process.write(graph);
276                         }
277                 });
278                 return process.getResourceIds(
279                                 session.getService(SerialisationSupport.class));
280         }
281         
282         public static long[] importGraph1(WriteGraph graph, final TransferableGraph1 tg) throws DatabaseException, TransferableGraphException {
283                 final TransferableGraphImportProcess process = 
284                         new TransferableGraphImportProcess(tg, 
285                                 null);
286                 process.prepare(graph);
287                 process.write2(graph);
288                 return process.getResourceIds(
289                                 graph.getSession().getService(SerialisationSupport.class));
290         }
291         
292         /**
293          * Import transferable graph version 1 to the database. Root advisor is used
294          * to give identities to roots of the transferable graphs. It may be null,
295          * in which case new resources are created for all roots but the root library.
296          * 
297          * @param session
298          * @param tg
299      * @param advisor root advisor or <code>null</code>
300          * @throws DatabaseException
301          */
302         public static long[] importGraph1(WriteGraph graph, TransferableGraph1 tg, IImportAdvisor advisor) throws DatabaseException {
303             return importGraph1(graph, tg, advisor, null);
304         }
305
306         public static long[] importGraph1(WriteGraph graph, TransferableGraph1 tg, IImportAdvisor advisor, TGStatusMonitor monitor) throws DatabaseException {
307                 TransferableGraphImportProcess process = new TransferableGraphImportProcess(tg, 
308                                 advisor == null ? new ImportAdvisor() : advisor, monitor);
309                 process.prepare(graph);
310                 if(advisor instanceof IImportAdvisor2) ((IImportAdvisor2)advisor).beforeWrite(graph, process);
311                 process.write2(graph);
312                 if(advisor instanceof IImportAdvisor2) ((IImportAdvisor2)advisor).afterWrite(graph, process);
313                 return process.getResourceIds(
314                                 graph.getSession().getService(SerialisationSupport.class));
315         }
316
317         public static long[] applyDelta(WriteGraph graph, long[] oldResources, TransferableGraphDelta1 delta) throws DatabaseException {
318                 SerialisationSupport serializer = 
319                         graph.getSession().getService(SerialisationSupport.class);
320                 
321                 TGToGraphMap aMap = new TGToGraphMap(delta.a);
322                 aMap.addOldResources(serializer, oldResources);
323                 aMap.deny(graph);
324                 
325                 TGToGraphMap bMap = new TGToGraphMap(delta.b);
326                 bMap.addMappedOldResources(serializer, delta.aToB, aMap.getResources());
327                 bMap.prepare(graph);
328                 bMap.claim(graph);
329                 
330                 return bMap.getResources(serializer);
331         }
332         
333         public static boolean hasChanges(ReadGraph graph, long[] oldResources, TransferableGraphDelta1 delta) throws DatabaseException {
334                 
335                 SerialisationSupport serializer = 
336                         graph.getSession().getService(SerialisationSupport.class);
337                 
338                 TGToGraphMap aMap = new TGToGraphMap(delta.a);
339                 aMap.addOldResources(serializer, oldResources);
340                 if(aMap.checkDeny(graph)) return true;
341                 
342                 TGToGraphMap bMap = new TGToGraphMap(delta.b);
343                 bMap.addMappedOldResources(serializer, delta.aToB, aMap.getResources());
344                 bMap.prepare(graph);
345                 return bMap.checkClaim(graph);
346                 
347         }
348
349         public static void uninstallGraph(WriteGraph writeGraph, TransferableGraph1 graph,
350                         ImportAdvisor advisor) throws TransferableGraphException {
351                 // TODO HANNU IMPLEMENTS
352                 throw new UnsupportedOperationException();
353         }       
354         
355         public static long[] importVirtualGraph(Session session, final VirtualGraph vg, final TransferableGraph1 tg, IImportAdvisor advisor) throws DatabaseException {
356                 final TransferableGraphImportProcess process = new TransferableGraphImportProcess(tg, 
357                                 advisor == null ? new ImportAdvisor() : advisor);
358                 session.syncRequest(new ReadRequest() {
359                         @Override
360                         public void run(ReadGraph graph) throws DatabaseException {
361                                 process.prepare(graph);
362                         }
363                 });
364                 session.syncRequest(new WriteOnlyRequest(vg) {
365                         @Override
366                         public void perform(WriteOnlyGraph graph) throws DatabaseException {
367                                 // Needed because process#write does not support virtual WriteOnlyGraph
368                                 if (vg != null)
369                                         process.write2(graph);
370                                 else
371                                         process.write(graph);
372                         }
373                         
374                 });
375                 return process.getResourceIds(session.getService(SerialisationSupport.class));
376         }       
377
378         public static long[] importVirtualGraph(WriteGraph graph, final TransferableGraph1 tg, IImportAdvisor advisor) throws DatabaseException {
379                 final TransferableGraphImportProcess process = new TransferableGraphImportProcess(tg, 
380                                 advisor == null ? new ImportAdvisor() : advisor);
381                 process.prepare(graph);
382                 process.write2(graph);
383                 return process.getResourceIds(graph.getService(SerialisationSupport.class));
384         }       
385         
386         public static TransferableGraph1 readGraph(File file) throws TransferableGraphException {
387                 FileVariantAccessor va = null;
388                 try {
389                         va = Accessors.openAccessor(file);
390                         Datatype type = va.getContentType();
391                         if(type.equals(Datatypes.getDatatype(TransferableGraph1.class))) 
392                                 return  (TransferableGraph1)va.getContentValue(Bindings.getBinding(TransferableGraph1.class));
393                         else
394                                 throw new SerializationException("Unknown transferable graph data type.");
395                 } catch (AccessorException e) {
396                         throw new TransferableGraphException(e);
397                 } catch (BindingConstructionException e) {
398                         throw new TransferableGraphException(e);
399                 } catch (SerializationException e) {
400                         throw new TransferableGraphException(e);
401                 } catch (AccessorConstructionException e) {
402                         throw new TransferableGraphException(e);
403                 } catch (DatatypeConstructionException e) {
404                         throw new TransferableGraphException(e);
405                 } finally {
406                         if(va != null) {
407                                 try {
408                                         va.close();
409                                 } catch (AccessorException e) {
410                                 }
411                         }
412                 } 
413         }
414         
415         public static void importVirtualGraph(Session session, VirtualGraph vg, File file) throws TransferableGraphException {
416                 try {
417                         importVirtualGraph(session, vg, readGraph(file), new ImportAdvisor());
418                 } catch (DatabaseException e) {
419                         throw new TransferableGraphException(e);
420                 }
421         }
422         
423         public static VirtualGraph importVirtualGraph(Session session, File file) throws TransferableGraphException {
424                 VirtualGraphSupport support = session.getService(VirtualGraphSupport.class);
425                 VirtualGraph vg = support.getMemoryPersistent(UUID.randomUUID().toString());
426                 importVirtualGraph(session, vg, file);
427                 return vg;
428         }
429         
430         public static void writeTransferableGraph(RequestProcessor processor, final String format, final TransferableGraphSource source, File target) throws Exception {
431                 writeTransferableGraph(processor, format, 1, source, target);
432         }
433
434         public static void writeTransferableGraph(RequestProcessor processor, final String format, final int version, final TransferableGraphSource source, File target) throws Exception {
435                 writeTransferableGraph(processor, format, version, new TreeMap<String,Variant>(), source, target);
436         }
437
438         public static void writeTransferableGraph(RequestProcessor processor, String format, int version, TreeMap<String, Variant> metadata, TransferableGraphSource source, File target) throws Exception {
439                 writeTransferableGraph(processor, format, version, metadata, source, target, TGStatusMonitor.NULL_MONITOR);
440         }
441
442         public static void writeTransferableGraph(RequestProcessor processor, String format, int version, TreeMap<String, Variant> metadata, TransferableGraphSource source, File target, TGStatusMonitor monitor) throws Exception {
443                 try (RandomAccessBinary out = new BinaryFile(target, 128*1024)) {
444                         DataContainers.writeHeader(out, new DataContainer(format, version, metadata, null));
445                         writeTransferableGraphVariant(processor, source, out, monitor);
446                 }
447         }
448
449         public static void writeTransferableGraph(RequestProcessor processor, TransferableGraphSource source, File target, TGStatusMonitor monitor) throws Exception {
450                 try (RandomAccessBinary out = new BinaryFile(target, 128*1024)) {
451                         writeTransferableGraphVariant(processor, source, out, monitor);
452                 }
453         }
454
455         public static void writeTransferableGraphVariant(RequestProcessor processor, TransferableGraphSource source, RandomAccessBinary out, TGStatusMonitor monitor) throws Exception {
456                 Bindings.getSerializerUnchecked(Datatype.class).serialize(out, Datatypes.getDatatypeUnchecked(TransferableGraph1.class));
457                 writeTransferableGraph(processor, source, out, monitor);
458         }
459
460         private static TGStatusMonitor safeMonitor(TGStatusMonitor mon) {
461                 return mon == null ? TGStatusMonitor.NULL_MONITOR : mon;
462         }
463
464         private static class CopyingInputStream extends InputStream {
465                 public DataInput in;
466                 public DataOutput out;
467
468                 @Override
469                 public int read() throws IOException {
470                         int value = in.readUnsignedByte();
471                         out.write(value);
472                         return value;
473                 }
474         }
475
476         private static long copy(byte[] buffer, DataInput in, DataOutput out, long bytesToCopy) throws IOException {
477                 int read = 0;
478                 long bufferLength = buffer.length;
479                 while (read < bytesToCopy) {
480                         int l = (int) Math.min(bufferLength, bytesToCopy-read);
481                         in.readFully(buffer, 0, l);
482                         out.write(buffer, 0, l);
483                         read += l;
484                 }
485                 return read;
486         }
487
488         private static final int LITERAL_VALUE_IO_BUFFER_SIZE = 128 * 1024;
489
490         private static void writeTransferableGraph(RequestProcessor processor, final TransferableGraphSource source, final RandomAccessBinary out, TGStatusMonitor monitor) throws Exception {
491                 long start = System.nanoTime();
492
493                 final Serializer datatypeSerializer = Bindings.getSerializerUnchecked(Datatype.class);
494                 final Serializer identitySerializer = Bindings.getSerializerUnchecked(Identity.class);
495                 final Serializer extensionSerializer = Bindings.getSerializerUnchecked(Extensions.class);
496
497                 int resourceCount = source.getResourceCount();
498                 //System.err.println("resourceCount: " + resourceCount);
499                 out.writeInt(resourceCount);
500                 extensionSerializer.serialize(out, new Extensions(source.getExtensions()));
501
502 //              System.err.println("resource count: " + source.getResourceCount());
503 //              System.err.println("identity count: " + source.getIdentityCount());
504
505                 byte[] buffer = new byte[LITERAL_VALUE_IO_BUFFER_SIZE];
506
507                 processor.syncRequest(new ReadRequest() {
508                         @Override
509                         public void run(ReadGraph graph) throws DatabaseException {
510                                 try {
511                                         if (monitor.isCanceled())
512                                                 throw new CancelTransactionException();
513
514                                         int identityCount = source.getIdentityCount();
515                                         TGStatusMonitor.Updater identityProgress = new TGStatusMonitor.Updater(safeMonitor(monitor), 0, 33, identityCount);
516                                         out.writeInt(identityCount);
517                                         //System.err.println("identities: " + identityCount);
518                                         source.forIdentities(graph, value -> {
519                                                 //System.err.println("id: " + value);
520                                                 identitySerializer.serialize(out, value);
521                                                 identityProgress.worked(1);
522                                         });
523
524                                         if (monitor.isCanceled())
525                                                 throw new CancelTransactionException();
526
527                                         long statementCountPos = out.position();
528                                         int originalStatementCount = source.getStatementCount();
529                                         TGStatusMonitor.Updater statementProgress = new TGStatusMonitor.Updater(safeMonitor(monitor), 34, 66, originalStatementCount);
530                                         out.writeInt(originalStatementCount);
531                                         //System.err.println("original statementCount: " + originalStatementCount);
532                                         int[] statementCounter = { 0 };
533                                         source.forStatements(graph, r -> {
534                                                 for (int i = 0; i < 4; ++i)
535                                                         out.writeInt(r[i]);
536                                                 statementCounter[0]++;
537                                                 //System.err.println("stm " + (statementCounter[0]) + ": " + r[0] + " " + r[1] + " " + r[2] + " " + r[3]);
538                                                 statementProgress.worked(1);
539                                         });
540                                         //System.err.println("wrote " + statementCounter[0] + " statements, " + (statementCounter[0]*4)+ " integers");
541
542                                         // Rewrite statement count after knowing exactly how many
543                                         // statements were written. It is possible that some
544                                         // statements get filtered out at this stage and the
545                                         // original statement count does not reflect that.
546                                         long afterStatementsPos = out.position();
547                                         out.position(statementCountPos);
548                                         out.writeInt(statementCounter[0]*4);
549                                         out.position(afterStatementsPos);
550
551                                         if (monitor.isCanceled())
552                                                 throw new CancelTransactionException();
553
554                                         int valueCount = source.getValueCount();
555                                         TGStatusMonitor.Updater valueProgress = new TGStatusMonitor.Updater(safeMonitor(monitor), 67, 100, valueCount);
556                                         out.writeInt(valueCount);
557 //                                      System.err.println("valueCount: " + valueCount);
558                                         CopyingInputStream cis = new CopyingInputStream();
559                                         cis.out = out;
560                                         source.forValues2(graph, new TransferableGraphSourceValueProcedure() {
561                                                 TObjectIntHashMap<Object> identities = new TObjectIntHashMap<>();
562
563                                                 @Override
564                                                 public void rawCopy(int resource, int length, DataInput input) throws Exception {
565                                                         out.writeInt(resource);
566                                                         long copied = copy(buffer, input, out, length);
567                                                         assert copied == length;
568                                                         //System.err.println("value " + (num++) + ": raw variant, " + length + " bytes, copied " + copied + " bytes");
569                                                         valueProgress.worked(1);
570                                                 }
571
572                                                 @Override
573                                                 public void execute(int resource, Datatype type, DataInput input) throws Exception {
574                                                         out.writeInt(resource);
575                                                         identities.clear();
576                                                         datatypeSerializer.serialize(out, identities, type);
577                                                         Binding binding = Bindings.getBinding(type);
578                                                         Serializer serializer = Bindings.getSerializer(binding);
579                                                         cis.in = input;
580                                                         serializer.skip(cis);
581                                                         cis.in = null;
582                                                         valueProgress.worked(1);
583                                                 }
584                                         });
585
586                                 } catch (DatabaseException e) {
587                                         throw e;
588                                 } catch (Exception e) {
589                                         throw new DatabaseException(e);
590                                 }
591                         }
592                 });
593
594                 long end = System.nanoTime();
595                 LOGGER.info("Wrote transferable graph in {} seconds.", 1e-9*(end-start));
596         }
597
598         public static TransferableGraph1 create(ReadGraph graph, TransferableGraphSource source) throws DatabaseException {
599                 
600                 final TIntArrayList statements = new TIntArrayList();
601                 final ArrayList<Value> values = new ArrayList<>();
602                 final ArrayList<Identity> identities = new ArrayList<>();
603                 
604                 try {
605
606                         source.forStatements(graph, r -> statements.addAll(r));
607                         source.forValues(graph, v -> values.add(v));
608                         source.forIdentities(graph, i -> identities.add(i));
609
610                         return new TransferableGraph1(source.getResourceCount(), 
611                                         identities.toArray(new Identity[identities.size()]),
612                                         statements.toArray(),
613                                         values.toArray(new Value[values.size()]),
614                                         source.getExtensions());
615                         
616                 } catch (Exception e) {
617                         
618                         throw new DatabaseException(e);
619                         
620                 }
621                 
622         }
623
624 }