X-Git-Url: https://gerrit.simantics.org/r/gitweb?p=simantics%2Fplatform.git;a=blobdiff_plain;f=bundles%2Forg.simantics.graph.db%2Fsrc%2Forg%2Fsimantics%2Fgraph%2Fdb%2FTransferableGraphs.java;h=c2ca6ac7305d26e824a928ffa6b8ee9c6e53b372;hp=bfea63c48d5fc72742c34232463a0ba2b9d207ac;hb=3efd7d44a55ae1fff284a1826e7c9afe27e0815e;hpb=0b471805f017da83d715a0d8409f53bdd009d31e diff --git a/bundles/org.simantics.graph.db/src/org/simantics/graph/db/TransferableGraphs.java b/bundles/org.simantics.graph.db/src/org/simantics/graph/db/TransferableGraphs.java index bfea63c48..c2ca6ac73 100644 --- a/bundles/org.simantics.graph.db/src/org/simantics/graph/db/TransferableGraphs.java +++ b/bundles/org.simantics.graph.db/src/org/simantics/graph/db/TransferableGraphs.java @@ -11,12 +11,9 @@ *******************************************************************************/ package org.simantics.graph.db; -import java.io.BufferedOutputStream; import java.io.DataInput; import java.io.DataOutput; -import java.io.DataOutputStream; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; @@ -40,6 +37,8 @@ import org.simantics.databoard.container.DataContainers; import org.simantics.databoard.serialization.SerializationException; import org.simantics.databoard.serialization.Serializer; import org.simantics.databoard.type.Datatype; +import org.simantics.databoard.util.binary.BinaryFile; +import org.simantics.databoard.util.binary.RandomAccessBinary; import org.simantics.db.ReadGraph; import org.simantics.db.RequestProcessor; import org.simantics.db.Resource; @@ -437,7 +436,7 @@ public class TransferableGraphs { public static void writeTransferableGraph(RequestProcessor processor, String format, int version, TreeMap metadata, TransferableGraphSource source, File target, TGStatusMonitor monitor) throws Exception { final Serializer datatypeSerializer = Bindings.getSerializerUnchecked(Datatype.class); - try (DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(target), 128*1024))) { + try (RandomAccessBinary out = new BinaryFile(target, 128*1024)) { DataContainer container = new DataContainer(format, version, metadata, null); DataContainers.writeHeader(out, container); datatypeSerializer.serialize((DataOutput) out, Datatypes.getDatatypeUnchecked(TransferableGraph1.class)); @@ -475,17 +474,16 @@ public class TransferableGraphs { private static final int LITERAL_VALUE_IO_BUFFER_SIZE = 128 * 1024; - private static void writeTransferableGraph(RequestProcessor processor, final TransferableGraphSource source, final DataOutput out, TGStatusMonitor monitor) throws Exception { + private static void writeTransferableGraph(RequestProcessor processor, final TransferableGraphSource source, final RandomAccessBinary out, TGStatusMonitor monitor) throws Exception { long start = System.nanoTime(); final Serializer datatypeSerializer = Bindings.getSerializerUnchecked(Datatype.class); final Serializer identitySerializer = Bindings.getSerializerUnchecked(Identity.class); final Serializer extensionSerializer = Bindings.getSerializerUnchecked(Extensions.class); - int totalCount = source.getIdentityCount() + source.getStatementCount()/4 + source.getValueCount(); - TGStatusMonitor.Updater progress = new TGStatusMonitor.Updater(safeMonitor(monitor), totalCount); - - out.writeInt(source.getResourceCount()); + int resourceCount = source.getResourceCount(); + //System.err.println("resourceCount: " + resourceCount); + out.writeInt(resourceCount); extensionSerializer.serialize(out, new Extensions(source.getExtensions())); // System.err.println("resource count: " + source.getResourceCount()); @@ -500,31 +498,50 @@ public class TransferableGraphs { if (monitor.isCanceled()) throw new CancelTransactionException(); - out.writeInt(source.getIdentityCount()); + int identityCount = source.getIdentityCount(); + TGStatusMonitor.Updater identityProgress = new TGStatusMonitor.Updater(safeMonitor(monitor), 0, 33, identityCount); + out.writeInt(identityCount); + //System.err.println("identities: " + identityCount); source.forIdentities(graph, value -> { //System.err.println("id: " + value); identitySerializer.serialize(out, value); - progress.worked(1); + identityProgress.worked(1); }); if (monitor.isCanceled()) throw new CancelTransactionException(); - out.writeInt(source.getStatementCount()); - //System.err.println("stms: " + source.getStatementCount()); - //int[] counter = {0}; + long statementCountPos = out.position(); + int originalStatementCount = source.getStatementCount(); + TGStatusMonitor.Updater statementProgress = new TGStatusMonitor.Updater(safeMonitor(monitor), 34, 66, originalStatementCount); + out.writeInt(originalStatementCount); + //System.err.println("original statementCount: " + originalStatementCount); + int[] statementCounter = { 0 }; source.forStatements(graph, r -> { for (int i = 0; i < 4; ++i) out.writeInt(r[i]); - //System.err.println("stm " + (counter[0]++) + ": " + r[0] + " " + r[1] + " " + r[2] + " " + r[3]); - progress.worked(1); + statementCounter[0]++; + //System.err.println("stm " + (statementCounter[0]) + ": " + r[0] + " " + r[1] + " " + r[2] + " " + r[3]); + statementProgress.worked(1); }); + //System.err.println("wrote " + statementCounter[0] + " statements, " + (statementCounter[0]*4)+ " integers"); + + // Rewrite statement count after knowing exactly how many + // statements were written. It is possible that some + // statements get filtered out at this stage and the + // original statement count does not reflect that. + long afterStatementsPos = out.position(); + out.position(statementCountPos); + out.writeInt(statementCounter[0]*4); + out.position(afterStatementsPos); if (monitor.isCanceled()) throw new CancelTransactionException(); - out.writeInt(source.getValueCount()); - //System.err.println("vals: " + source.getValueCount()); + int valueCount = source.getValueCount(); + TGStatusMonitor.Updater valueProgress = new TGStatusMonitor.Updater(safeMonitor(monitor), 67, 100, valueCount); + out.writeInt(valueCount); +// System.err.println("valueCount: " + valueCount); CopyingInputStream cis = new CopyingInputStream(); cis.out = out; source.forValues2(graph, new TransferableGraphSourceValueProcedure() { @@ -536,7 +553,7 @@ public class TransferableGraphs { long copied = copy(buffer, input, out, length); assert copied == length; //System.err.println("value " + (num++) + ": raw variant, " + length + " bytes, copied " + copied + " bytes"); - progress.worked(1); + valueProgress.worked(1); } @Override @@ -549,7 +566,7 @@ public class TransferableGraphs { cis.in = input; serializer.skip(cis); cis.in = null; - progress.worked(1); + valueProgress.worked(1); } });