From 3efd7d44a55ae1fff284a1826e7c9afe27e0815e Mon Sep 17 00:00:00 2001 From: Tuukka Lehtonen Date: Thu, 20 Oct 2016 14:48:00 +0300 Subject: [PATCH] Sync git svn branch with SVN repository r33349. refs #6600 refs #6774 refs #6780 --- .../databoard/util/binary/BinaryFile.java | 50 +++++++++++---- .../simantics/document/server/io/Content.java | 61 ------------------- .../server/io/IAbstractRequestInterface.java | 22 ------- .../simantics/graph/db/TGStatusMonitor.java | 19 +++--- .../graph/db/TransferableGraphs.java | 57 +++++++++++------ 5 files changed, 86 insertions(+), 123 deletions(-) delete mode 100644 bundles/org.simantics.document.server.io/src/org/simantics/document/server/io/Content.java delete mode 100644 bundles/org.simantics.document.server.io/src/org/simantics/document/server/io/IAbstractRequestInterface.java diff --git a/bundles/org.simantics.databoard/src/org/simantics/databoard/util/binary/BinaryFile.java b/bundles/org.simantics.databoard/src/org/simantics/databoard/util/binary/BinaryFile.java index 1e9dfa3fb..e7d8d386e 100644 --- a/bundles/org.simantics.databoard/src/org/simantics/databoard/util/binary/BinaryFile.java +++ b/bundles/org.simantics.databoard/src/org/simantics/databoard/util/binary/BinaryFile.java @@ -578,7 +578,41 @@ public class BinaryFile implements RandomAccessBinary, BinaryReadable, BinaryWri // WRITE - + + void _putLong(long value) throws IOException + { + prepareForWrite(8); + int posInBuf = (int) (pointer - buf_pos); + buf[posInBuf] = (byte) (value >>> 56); + buf[posInBuf+1] = (byte) (value >>> 48); + buf[posInBuf+2] = (byte) (value >>> 40); + buf[posInBuf+3] = (byte) (value >>> 32); + buf[posInBuf+4] = (byte) (value >>> 24); + buf[posInBuf+5] = (byte) (value >>> 16); + buf[posInBuf+6] = (byte) (value >>> 8); + buf[posInBuf+7] = (byte) value; + posInBuf += 8; + pointer += 8; + if (write_buf_count>> 24); + buf[posInBuf+1] = (byte) (value >>> 16); + buf[posInBuf+2] = (byte) (value >>> 8); + buf[posInBuf+3] = (byte) value; + posInBuf += 4; + pointer += 4; + if (write_buf_count>> 24); - _put(value >>> 16); - _put(value >>> 8); - _put(value); + _putInt(value); } @Override public void writeLong(long value) throws IOException { - _put((int) (value >>> 56)); - _put((int) (value >>> 48)); - _put((int) (value >>> 40)); - _put((int) (value >>> 32)); - _put((int) (value >>> 24)); - _put((int) (value >>> 16)); - _put((int) (value >>> 8)); - _put((int) (value)); + _putLong(value); } @Override diff --git a/bundles/org.simantics.document.server.io/src/org/simantics/document/server/io/Content.java b/bundles/org.simantics.document.server.io/src/org/simantics/document/server/io/Content.java deleted file mode 100644 index eb957c8f6..000000000 --- a/bundles/org.simantics.document.server.io/src/org/simantics/document/server/io/Content.java +++ /dev/null @@ -1,61 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2014, 2016 Association for Decentralized - * Information Management in Industry THTH ry. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the THTH Simantics - * Division Member Component License which accompanies this - * distribution, and is available at - * http://www.simantics.org/legal/sdmcl-v10.html - * - * Contributors: - * Semantum Oy - initial API and implementation - *******************************************************************************/ -package org.simantics.document.server.io; - -import java.io.InputStream; - -public class Content { - private InputStream input; - private String mimeType; - private long lastModified; - private int length; - - public Content(InputStream input, String mimeType, long lastModified, int length) { - this.input = input; - this.mimeType = mimeType; - this.lastModified = lastModified; - this.length = length; - } - - public InputStream getInputStream() { - return input; - } - - public void setInputStream(InputStream input) { - this.input = input; - } - - public String getMimeType() { - return mimeType; - } - - public void setMimeType(String mimeType) { - this.mimeType = mimeType; - } - - public long getLastModified() { - return lastModified; - } - - public void setLastModified(long lastModified) { - this.lastModified = lastModified; - } - - public int getLength() { - return length; - } - - public void setLength(int length) { - this.length = length; - } -} diff --git a/bundles/org.simantics.document.server.io/src/org/simantics/document/server/io/IAbstractRequestInterface.java b/bundles/org.simantics.document.server.io/src/org/simantics/document/server/io/IAbstractRequestInterface.java deleted file mode 100644 index 9e736c821..000000000 --- a/bundles/org.simantics.document.server.io/src/org/simantics/document/server/io/IAbstractRequestInterface.java +++ /dev/null @@ -1,22 +0,0 @@ -package org.simantics.document.server.io; - -public interface IAbstractRequestInterface { - - void init(); - void createSession(String sessionGUID, IConsole console); - void deleteSession(String sessionGUID); - void setInputURI(String sessionGUID, String inputURI); - void setSessionParameter(String sessionGUID, String parameterName, Object value); - public IJSONResult requestDocument(String document, int sequenceNumber, int pollingHint, String sessionGUID); - public String requestDocumentURI(String alias); - String requestCSS(String documentURI); - int[] requestLoadingIndicatorDelay(String documentURI); - Double requestPollingInterval(String documentURI); - Content requestContent(String resourceURI); - public CommandResult execute(String id, CommandContext context); - public void subscribeDocument(String document, String sessionGUID, IDocumentListener documentListener); - public IJSONResult readChanges(String uri, String sessionGUID, int sequenceNumber); - public void initializeDocument(String documentURI, String sessionGUID, CommandContext context); - public void onPoll(String sessionGUID); - public String escapeURI(String unicode); -} diff --git a/bundles/org.simantics.graph.db/src/org/simantics/graph/db/TGStatusMonitor.java b/bundles/org.simantics.graph.db/src/org/simantics/graph/db/TGStatusMonitor.java index 06e0adbed..8f93639ea 100644 --- a/bundles/org.simantics.graph.db/src/org/simantics/graph/db/TGStatusMonitor.java +++ b/bundles/org.simantics.graph.db/src/org/simantics/graph/db/TGStatusMonitor.java @@ -20,23 +20,28 @@ public interface TGStatusMonitor { public static class Updater { private final TGStatusMonitor monitor; private final int total; - private int percentage; + private int low; + private int high; + private int current; private int worked; - public Updater(TGStatusMonitor monitor, int total) { + public Updater(TGStatusMonitor monitor, int lowPercentage, int highPercentage, int totalWork) { this.monitor = monitor; - this.total = total; + this.low = lowPercentage; + this.high = highPercentage; + this.total = totalWork; } public int worked(int work) { worked += work; - return percentage = updatePercentage(monitor, percentage, worked, total); + return current = updatePercentage(monitor, low, high, current, worked, total); } } - public static int updatePercentage(TGStatusMonitor monitor, int oldPercentage, int done, int total) { - int current = (100 * done) / total; - if (current > oldPercentage) { + public static int updatePercentage(TGStatusMonitor monitor, int low, int high, int oldPercentage, int done, int total) { + int range = high - low; + int current = low + (range * done) / total; + if (current != oldPercentage) { oldPercentage = current; if (monitor != null) monitor.status(oldPercentage); diff --git a/bundles/org.simantics.graph.db/src/org/simantics/graph/db/TransferableGraphs.java b/bundles/org.simantics.graph.db/src/org/simantics/graph/db/TransferableGraphs.java index bfea63c48..c2ca6ac73 100644 --- a/bundles/org.simantics.graph.db/src/org/simantics/graph/db/TransferableGraphs.java +++ b/bundles/org.simantics.graph.db/src/org/simantics/graph/db/TransferableGraphs.java @@ -11,12 +11,9 @@ *******************************************************************************/ package org.simantics.graph.db; -import java.io.BufferedOutputStream; import java.io.DataInput; import java.io.DataOutput; -import java.io.DataOutputStream; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; @@ -40,6 +37,8 @@ import org.simantics.databoard.container.DataContainers; import org.simantics.databoard.serialization.SerializationException; import org.simantics.databoard.serialization.Serializer; import org.simantics.databoard.type.Datatype; +import org.simantics.databoard.util.binary.BinaryFile; +import org.simantics.databoard.util.binary.RandomAccessBinary; import org.simantics.db.ReadGraph; import org.simantics.db.RequestProcessor; import org.simantics.db.Resource; @@ -437,7 +436,7 @@ public class TransferableGraphs { public static void writeTransferableGraph(RequestProcessor processor, String format, int version, TreeMap metadata, TransferableGraphSource source, File target, TGStatusMonitor monitor) throws Exception { final Serializer datatypeSerializer = Bindings.getSerializerUnchecked(Datatype.class); - try (DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(target), 128*1024))) { + try (RandomAccessBinary out = new BinaryFile(target, 128*1024)) { DataContainer container = new DataContainer(format, version, metadata, null); DataContainers.writeHeader(out, container); datatypeSerializer.serialize((DataOutput) out, Datatypes.getDatatypeUnchecked(TransferableGraph1.class)); @@ -475,17 +474,16 @@ public class TransferableGraphs { private static final int LITERAL_VALUE_IO_BUFFER_SIZE = 128 * 1024; - private static void writeTransferableGraph(RequestProcessor processor, final TransferableGraphSource source, final DataOutput out, TGStatusMonitor monitor) throws Exception { + private static void writeTransferableGraph(RequestProcessor processor, final TransferableGraphSource source, final RandomAccessBinary out, TGStatusMonitor monitor) throws Exception { long start = System.nanoTime(); final Serializer datatypeSerializer = Bindings.getSerializerUnchecked(Datatype.class); final Serializer identitySerializer = Bindings.getSerializerUnchecked(Identity.class); final Serializer extensionSerializer = Bindings.getSerializerUnchecked(Extensions.class); - int totalCount = source.getIdentityCount() + source.getStatementCount()/4 + source.getValueCount(); - TGStatusMonitor.Updater progress = new TGStatusMonitor.Updater(safeMonitor(monitor), totalCount); - - out.writeInt(source.getResourceCount()); + int resourceCount = source.getResourceCount(); + //System.err.println("resourceCount: " + resourceCount); + out.writeInt(resourceCount); extensionSerializer.serialize(out, new Extensions(source.getExtensions())); // System.err.println("resource count: " + source.getResourceCount()); @@ -500,31 +498,50 @@ public class TransferableGraphs { if (monitor.isCanceled()) throw new CancelTransactionException(); - out.writeInt(source.getIdentityCount()); + int identityCount = source.getIdentityCount(); + TGStatusMonitor.Updater identityProgress = new TGStatusMonitor.Updater(safeMonitor(monitor), 0, 33, identityCount); + out.writeInt(identityCount); + //System.err.println("identities: " + identityCount); source.forIdentities(graph, value -> { //System.err.println("id: " + value); identitySerializer.serialize(out, value); - progress.worked(1); + identityProgress.worked(1); }); if (monitor.isCanceled()) throw new CancelTransactionException(); - out.writeInt(source.getStatementCount()); - //System.err.println("stms: " + source.getStatementCount()); - //int[] counter = {0}; + long statementCountPos = out.position(); + int originalStatementCount = source.getStatementCount(); + TGStatusMonitor.Updater statementProgress = new TGStatusMonitor.Updater(safeMonitor(monitor), 34, 66, originalStatementCount); + out.writeInt(originalStatementCount); + //System.err.println("original statementCount: " + originalStatementCount); + int[] statementCounter = { 0 }; source.forStatements(graph, r -> { for (int i = 0; i < 4; ++i) out.writeInt(r[i]); - //System.err.println("stm " + (counter[0]++) + ": " + r[0] + " " + r[1] + " " + r[2] + " " + r[3]); - progress.worked(1); + statementCounter[0]++; + //System.err.println("stm " + (statementCounter[0]) + ": " + r[0] + " " + r[1] + " " + r[2] + " " + r[3]); + statementProgress.worked(1); }); + //System.err.println("wrote " + statementCounter[0] + " statements, " + (statementCounter[0]*4)+ " integers"); + + // Rewrite statement count after knowing exactly how many + // statements were written. It is possible that some + // statements get filtered out at this stage and the + // original statement count does not reflect that. + long afterStatementsPos = out.position(); + out.position(statementCountPos); + out.writeInt(statementCounter[0]*4); + out.position(afterStatementsPos); if (monitor.isCanceled()) throw new CancelTransactionException(); - out.writeInt(source.getValueCount()); - //System.err.println("vals: " + source.getValueCount()); + int valueCount = source.getValueCount(); + TGStatusMonitor.Updater valueProgress = new TGStatusMonitor.Updater(safeMonitor(monitor), 67, 100, valueCount); + out.writeInt(valueCount); +// System.err.println("valueCount: " + valueCount); CopyingInputStream cis = new CopyingInputStream(); cis.out = out; source.forValues2(graph, new TransferableGraphSourceValueProcedure() { @@ -536,7 +553,7 @@ public class TransferableGraphs { long copied = copy(buffer, input, out, length); assert copied == length; //System.err.println("value " + (num++) + ": raw variant, " + length + " bytes, copied " + copied + " bytes"); - progress.worked(1); + valueProgress.worked(1); } @Override @@ -549,7 +566,7 @@ public class TransferableGraphs { cis.in = input; serializer.skip(cis); cis.in = null; - progress.worked(1); + valueProgress.worked(1); } }); -- 2.47.1