From 9df6e0be9b9df41cd66a74e54ac70407a62ff4a1 Mon Sep 17 00:00:00 2001 From: Tuukka Lehtonen Date: Mon, 8 Nov 2021 14:03:22 +0200 Subject: [PATCH] Add more databoard reading utilities to ByteFileReader Also add writeDynamicUInt32(byte[] out, int offset, int length) to Endian class for more direct writing of data to optimize custom serializer implementations. Change MappingBase.browseConfiguration to use THashMap.forEachValue instead of Collection.iterator() for a tighter implementation. gitlab #772 --- .../databoard/util/binary/Endian.java | 44 +++++++++++++ .../graph/representation/ByteFileReader.java | 61 +++++++++++++++++++ .../synchronization/base/MappingBase.java | 10 ++- 3 files changed, 112 insertions(+), 3 deletions(-) diff --git a/bundles/org.simantics.databoard/src/org/simantics/databoard/util/binary/Endian.java b/bundles/org.simantics.databoard/src/org/simantics/databoard/util/binary/Endian.java index 57c98bc4d..8d87aca4b 100644 --- a/bundles/org.simantics.databoard/src/org/simantics/databoard/util/binary/Endian.java +++ b/bundles/org.simantics.databoard/src/org/simantics/databoard/util/binary/Endian.java @@ -71,6 +71,50 @@ public class Endian { } } + /** + * Write UInt32 with dynamic encoding (1-5 bytes). + * + * @param out + * @param length + * @throws IOException + */ + public static void writeDynamicUInt32(byte[] out, int offset, int length) throws IOException { + if(length < 0x80) { + out[offset] = (byte)length; + } + else { + length -= 0x80; + if(length < 0x4000) { + out[offset] = (byte) ((length&0x3f) | 0x80); + out[offset+1] = (byte) ((length>>>6) ); + } + else { + length -= 0x4000; + if(length < 0x200000) { + out[offset] = (byte) ((length&0x1f) | 0xc0); + out[offset+1] = (byte) ((length>>>5)&0xff); + out[offset+2] = (byte) ((length>>>13)&0xff); + } + else { + length -= 0x200000; + if(length < 0x10000000) { + out[offset] = (byte) ((length&0x0f) | 0xe0); + out[offset+1] = (byte) ((length>>>4)&0xff); + out[offset+2] = (byte) ((length>>>12)&0xff); + out[offset+3] = (byte) ((length>>>20)&0xff); + } + else { + length -= 0x10000000; + out[offset] = (byte) ((length&0x07) | 0xf0); + out[offset+1] = (byte) ((length>>>3)&0xff); + out[offset+2] = (byte) ((length>>>11)&0xff); + out[offset+3] = (byte) ((length>>>19)&0xff); + out[offset+4] = (byte) ((length>>>27)&0xff); + } + } + } + } + } public static int readDynamicUInt32(DataInput in) throws IOException { int length = in.readByte()&0xff; diff --git a/bundles/org.simantics.graph/src/org/simantics/graph/representation/ByteFileReader.java b/bundles/org.simantics.graph/src/org/simantics/graph/representation/ByteFileReader.java index 3a76a911a..e55074043 100644 --- a/bundles/org.simantics.graph/src/org/simantics/graph/representation/ByteFileReader.java +++ b/bundles/org.simantics.graph/src/org/simantics/graph/representation/ByteFileReader.java @@ -145,6 +145,43 @@ public class ByteFileReader implements Closeable { } + /** + * @param result the output buffer to read into + * @param off the offset in result to start reading to + * @param len the maximum amount of data to read + * @return the actual amount of bytes read or -1 if EOF was reached + * @throws IOException + */ + protected final int safeBytes(byte[] result, int off, int len) throws IOException { + int has = size-byteIndex; + if(len>= has) { + ReadableByteChannel c = channel; + ByteBuffer bb = byteBuffer; + System.arraycopy(bytes, byteIndex, result, off, has); + ByteBuffer bb2 = ByteBuffer.wrap(result); + bb2.position(off+has); + // For some peculiar reason this seems to avoid OOM with large blocks as compared to c.read(bb2 + while(has < len) { + int todo = Math.min(len-has, 65536); + bb2.limit(off+has+todo); + int got = c.read(bb2); + if(got == -1) throw new IOException("Unexpected end-of-file"); + has += got; + // For some unknown reason this is needed! + // Spec indicates that read would increment position but it does not. + bb2.position(off+has); + } + size = c.read(bb); + bb.position(0); + byteIndex = 0; + return has; + } else { + System.arraycopy(bytes, byteIndex, result, 0, len); + byteIndex += len; + return len; + } + } + final protected int getByte() throws IOException { int has = size-byteIndex; int result; @@ -248,6 +285,30 @@ public class ByteFileReader implements Closeable { } + public long safeLong() throws IOException { + int msi = safeInt(); + int lsi = safeInt(); + return ((long) msi << 32) | ((long) lsi & 0xffffffffL); + } + + public boolean getBoolean() throws IOException { + return getByte() != 0; + } + + public String readString() throws IOException { + int nameLen = getDynamicUInt32(); + if (nameLen == 0) + return ""; + String result; + if (byteIndex+nameLen < size) { + result = utf(bytes, byteIndex, byteIndex + nameLen); + byteIndex += nameLen; + } else { + result = utf(safeBytes(nameLen), 0, nameLen); + } + return result; + } + final protected int getSize() { return size; } diff --git a/bundles/org.simantics.structural.synchronization.client/src/org/simantics/structural/synchronization/base/MappingBase.java b/bundles/org.simantics.structural.synchronization.client/src/org/simantics/structural/synchronization/base/MappingBase.java index 9f557eccf..e9185e639 100644 --- a/bundles/org.simantics.structural.synchronization.client/src/org/simantics/structural/synchronization/base/MappingBase.java +++ b/bundles/org.simantics.structural.synchronization.client/src/org/simantics/structural/synchronization/base/MappingBase.java @@ -104,9 +104,13 @@ abstract public class MappingBase> { THashMap configurationByUid, T configuration) { configurationByUid.put(configuration.uid, configuration); - for(T child : configuration.getChildren()) { - browseConfiguration(configurationByUid, child); - child.parent = configuration; + THashMap children = configuration.getChildMap(); + if (children != null) { + children.forEachValue(child -> { + browseConfiguration(configurationByUid, child); + child.parent = configuration; + return true; + }); } } -- 2.47.1