From: Hannu Niemistö <hannu.niemisto@semantum.fi>
Date: Thu, 5 Apr 2018 08:07:16 +0000 (+0300)
Subject: Streaming serialization of values, debugger for corrupted values
X-Git-Tag: v1.43.0~136^2~506
X-Git-Url: https://gerrit.simantics.org/r/gitweb?p=simantics%2Fplatform.git;a=commitdiff_plain;h=694c553a3c402b9397ff99e904b4f99b77b51df1;ds=sidebyside

Streaming serialization of values, debugger for corrupted values

refs #7856

Change-Id: If9d7bdb84c46c3b3626241f9a63e582fe0a15004
---

diff --git a/bundles/org.simantics.databoard/META-INF/MANIFEST.MF b/bundles/org.simantics.databoard/META-INF/MANIFEST.MF
index 3e292e4bd..9f4b5b81c 100644
--- a/bundles/org.simantics.databoard/META-INF/MANIFEST.MF
+++ b/bundles/org.simantics.databoard/META-INF/MANIFEST.MF
@@ -50,6 +50,7 @@ Export-Package: org.simantics.databoard,
  org.simantics.databoard.primitives,
  org.simantics.databoard.serialization,
  org.simantics.databoard.serialization.impl,
+ org.simantics.databoard.streaming,
  org.simantics.databoard.tests,
  org.simantics.databoard.type,
  org.simantics.databoard.units,
diff --git a/bundles/org.simantics.databoard/src/org/simantics/databoard/serialization/impl/UnionSerializer.java b/bundles/org.simantics.databoard/src/org/simantics/databoard/serialization/impl/UnionSerializer.java
index 42687a48c..a677c86a2 100644
--- a/bundles/org.simantics.databoard/src/org/simantics/databoard/serialization/impl/UnionSerializer.java
+++ b/bundles/org.simantics.databoard/src/org/simantics/databoard/serialization/impl/UnionSerializer.java
@@ -48,7 +48,7 @@ public class UnionSerializer extends CompositeSerializer {
 				fixedSize = null;
 				break;
 			}
-			if (fixedSize!=null && fixedSizeOfComponentSerializer!=fixedSize) {
+			if (fixedSize!=null && !fixedSize.equals(fixedSizeOfComponentSerializer)) {
 				fixedSize = null;
 				break;
 			}
diff --git a/bundles/org.simantics.databoard/src/org/simantics/databoard/streaming/DataReader.java b/bundles/org.simantics.databoard/src/org/simantics/databoard/streaming/DataReader.java
new file mode 100644
index 000000000..f8012244f
--- /dev/null
+++ b/bundles/org.simantics.databoard/src/org/simantics/databoard/streaming/DataReader.java
@@ -0,0 +1,112 @@
+package org.simantics.databoard.streaming;
+
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.simantics.databoard.Bindings;
+import org.simantics.databoard.serialization.Serializer;
+import org.simantics.databoard.type.Datatype;
+import org.simantics.databoard.util.binary.Endian;
+import org.simantics.databoard.util.binary.UTF8;
+
+/**
+ * This is an utility class that encapsulates the databoard value decoding scheme.
+ * It can be used for streaming value reading.
+ */
+public class DataReader {
+    private static final Serializer DATATYPE_SERIALIZER = Bindings.getSerializerUnchecked(Bindings.getBindingUnchecked( Datatype.class ));
+
+    private final DataInput in;
+    
+    public DataReader(DataInput in) {
+        this.in = in;
+    }
+    
+    public DataReader(InputStream stream) {
+        this.in = new DataInputStream(stream);
+    }
+    
+    public boolean readBoolean() throws IOException {
+        return in.readByte() != 0;
+    }
+    
+    public byte readByte() throws IOException {
+        return in.readByte();
+    }
+    
+    public int readInteger() throws IOException {
+        return in.readInt();
+    }
+    
+    public long readLong() throws IOException {
+        return in.readLong();
+    }
+    
+    public float readFloat() throws IOException {
+        return in.readFloat();
+    }
+    
+    public double readDouble() throws IOException {
+        return in.readDouble();
+    }
+    
+    public int readStringLength() throws IOException {
+        return Endian.readDynamicUInt32(in);
+    }
+    
+    public String readStringContent(int length) throws IOException {
+        return UTF8.readModifiedUTF(in, length); 
+    }
+    
+    public String readString() throws IOException {
+        int length = readStringLength();
+        return readStringContent(length);
+    }
+    
+    public Datatype readDatatype() throws IOException {
+        return (Datatype)DATATYPE_SERIALIZER.deserialize(in);
+    }
+    
+    /**
+     * A variable length array is started with the length
+     * followed by the serialization of all elements.
+     */
+    public int beginVariableLengthArray() throws IOException {
+        return in.readInt();
+    }
+    
+    /**
+     * A map is started with its size followed by 
+     * serialization of interleaved keys and values.
+     */
+    public int beginMap() throws IOException {
+        return in.readInt();
+    }
+    
+    /**
+     * Starts reading optional value. False is returned,
+     * if the optional is null.
+     */
+    public boolean beginOptional() throws IOException {
+        return in.readByte() != 0;
+    }
+    
+    /**
+     * Selects the constructor of the union type.
+     * It is written as a variable length integer,
+     * so the total number of tags is required.
+     */
+    public int readUnionTag(int tagCount) throws IOException {
+        return Endian.getUInt(in, tagCount-1);
+    }
+    
+    /**
+     * Reads a record reference. Value 0 indicates that a new record is encountered.
+     * In this case reading the reference again returns the id of the new record.
+     */
+    public int readReferableRecordReference() throws IOException {
+        return in.readInt();
+    }
+}
diff --git a/bundles/org.simantics.databoard/src/org/simantics/databoard/streaming/DataSerializationDebugger.java b/bundles/org.simantics.databoard/src/org/simantics/databoard/streaming/DataSerializationDebugger.java
new file mode 100644
index 000000000..df64db205
--- /dev/null
+++ b/bundles/org.simantics.databoard/src/org/simantics/databoard/streaming/DataSerializationDebugger.java
@@ -0,0 +1,313 @@
+package org.simantics.databoard.streaming;
+
+import java.io.IOException;
+import java.io.PrintStream;
+
+import org.simantics.databoard.Bindings;
+import org.simantics.databoard.serialization.Serializer;
+import org.simantics.databoard.type.ArrayType;
+import org.simantics.databoard.type.BooleanType;
+import org.simantics.databoard.type.ByteType;
+import org.simantics.databoard.type.Component;
+import org.simantics.databoard.type.Datatype;
+import org.simantics.databoard.type.Datatype.Visitor;
+import org.simantics.databoard.type.DoubleType;
+import org.simantics.databoard.type.FloatType;
+import org.simantics.databoard.type.IntegerType;
+import org.simantics.databoard.type.LongType;
+import org.simantics.databoard.type.MapType;
+import org.simantics.databoard.type.OptionalType;
+import org.simantics.databoard.type.RecordType;
+import org.simantics.databoard.type.StringType;
+import org.simantics.databoard.type.UnionType;
+import org.simantics.databoard.type.VariantType;
+import org.simantics.databoard.util.Range;
+import org.simantics.databoard.util.binary.Endian;
+
+/**
+ * Utility for debugging corrupted values.
+ */
+public class DataSerializationDebugger {
+    DataReader in;
+    PrintStream out;
+    int indentation;
+    
+    private void indentation() {
+        for(int i=0;i<indentation;++i)
+            out.print("  ");
+    }
+    
+    /**
+     * Constructs the debugger that reads the given stream and prints the value
+     * content to given print stream.
+     */
+    public DataSerializationDebugger(DataReader in, PrintStream out) {
+        this.in = in;
+        this.out = out;
+    }
+    
+    /**
+     * Expects value of given datatype in the input stream.
+     */
+    public void expect(Datatype datatype) throws IOException {
+        if(datatype instanceof BooleanType)
+            expectBoolean();
+        else if(datatype instanceof ByteType)
+            expectByte();
+        else if(datatype instanceof IntegerType)
+            expectInteger();
+        else if(datatype instanceof LongType)
+            expectLong();
+        else if(datatype instanceof FloatType)
+            expectFloat();
+        else if(datatype instanceof DoubleType)
+            expectDouble();
+        else if(datatype instanceof StringType)
+            expectString();
+        else if(datatype instanceof RecordType)
+            expectRecord((RecordType)datatype);
+        else if(datatype instanceof ArrayType)
+            expectArray((ArrayType)datatype);
+        else if(datatype instanceof MapType)
+            expectMap((MapType)datatype);
+        else if(datatype instanceof OptionalType)
+            expectOptional((OptionalType)datatype);
+        else if(datatype instanceof UnionType)
+            expectUnion((UnionType)datatype);
+        else if(datatype instanceof VariantType)
+            expectVariant();        
+    }
+    
+    private void expectVariant() throws IOException {
+        out.print("Variant:");
+        Datatype datatype = in.readDatatype();
+        out.println(" " + datatype);
+        ++indentation;
+        indentation();
+        expect(datatype);
+        --indentation;        
+    }
+
+    private void expectUnion(UnionType datatype) throws IOException {
+        out.print("Union");
+        int tag = in.readUnionTag(datatype.getComponentCount());
+        Component component = datatype.getComponent(tag);
+        out.print("(" + component.name + "): ");
+        expect(component.type);
+    }
+
+    private void expectOptional(OptionalType datatype) throws IOException {
+        out.print("Optional: ");
+        if(in.beginOptional()) {
+            expect(datatype.componentType);
+        }
+        else
+            out.println("null");
+    }
+
+    private void expectMap(MapType datatype) throws IOException {
+        out.print("Map");
+        int length = in.beginMap();
+        out.println("(" + length + "):");
+        ++indentation;
+        for(int i=0;i<length;++i) {
+            indentation();
+            out.print(i+"-key: ");
+            expect(datatype.keyType);
+            indentation();
+            out.print(i+"-value: ");
+            expect(datatype.valueType);
+        }
+        --indentation;
+    }
+    
+    private static final Integer I1 = 1;
+    private static final Integer I4 = 4;
+    private static final Integer I8 = 8;
+    
+    private static int getFixedLength(ArrayType b) {
+        Range length = b.getLength();
+        return length.getLower().getValue().intValue();
+    }
+    
+    private static Visitor<Integer> FIXED_SIZE = new Visitor<Integer>() {
+
+        @Override
+        public Integer visit(ArrayType b) {
+            Range length = b.getLength();
+            if(length == null
+                    || length.getLower().getValue()==null
+                    || !length.getLower().equals(length.getUpper()))
+                return null;
+            
+            int fixedLength = length.getLower().getValue().intValue();
+            
+            Integer componentLength = b.componentType.accept(this);
+            if(componentLength == null)
+                return null;
+            
+            return fixedLength * componentLength;
+        }
+
+        @Override
+        public Integer visit(BooleanType b) {
+            return I1;
+        }
+
+        @Override
+        public Integer visit(DoubleType b) {
+            return I8;
+        }
+
+        @Override
+        public Integer visit(FloatType b) {
+            return I4;
+        }
+
+        @Override
+        public Integer visit(IntegerType b) {
+            return I4;
+        }
+
+        @Override
+        public Integer visit(ByteType b) {
+            return I1;
+        }
+
+        @Override
+        public Integer visit(LongType b) {
+            return I8;
+        }
+
+        @Override
+        public Integer visit(OptionalType b) {
+            return null;
+        }
+
+        @Override
+        public Integer visit(RecordType b) {
+            if(b.isReferable())
+                return null;
+            int sum = 0;
+            for(Component component : b.getComponents()) {
+                Integer componentSize = component.type.accept(this);
+                if(componentSize == null)
+                    return null;
+                sum += componentSize;
+            }
+            return sum;
+        }
+
+        @Override
+        public Integer visit(StringType b) {
+            return null;
+        }
+
+        @Override
+        public Integer visit(UnionType b) {
+            Integer commonComponentSize = null;
+            for(Component component : b.components) {
+                Integer componentSize = component.type.accept(this);
+                if(componentSize == null)
+                    return null;
+                if(commonComponentSize == null)
+                    commonComponentSize = componentSize;
+                else if(!commonComponentSize.equals(componentSize))
+                    return null;
+            }
+            if(commonComponentSize == null)
+                return 0;
+            return commonComponentSize + Endian.getUIntLength(b.components.length-1);
+        }
+
+        @Override
+        public Integer visit(VariantType b) {
+            return null;
+        }
+
+        @Override
+        public Integer visit(MapType b) {
+            return null;
+        }
+        
+    };
+
+    private void expectArray(ArrayType datatype) throws IOException {
+        out.print("Array");
+        int length;
+        if(datatype.accept(FIXED_SIZE) == null)
+            length = in.beginVariableLengthArray();
+        else
+            length = getFixedLength(datatype);
+        out.println("(" + length + "):");
+        ++indentation;
+        for(int i=0;i<length;++i) {
+            indentation();
+            out.print(i);
+            out.print(": ");
+            expect(datatype.componentType);
+        }
+        --indentation;
+    }
+
+    private void expectRecord(RecordType datatype) throws IOException {
+        out.print("Record:");
+        if(datatype.isReferable()) {
+            int ref = in.readReferableRecordReference();
+            if(ref != 0) {
+                out.println(" old " + ref);
+                return;
+            }
+            else {
+                out.println(" new");
+            }
+        }
+        else
+            out.println();
+        ++indentation;
+        for(Component component : datatype.getComponents()) {
+            indentation();
+            out.print(component.name);
+            out.print(": ");
+            expect(component.type);
+        }
+        --indentation;
+    }
+
+    private void expectString() throws IOException {
+        out.print("String");
+        int length = in.readStringLength();
+        out.print("(" + length + "): ");
+        out.println(in.readStringContent(length));
+    }
+
+    private void expectDouble() throws IOException {
+        out.print("Double: ");
+        out.println(in.readDouble());
+    }
+
+    private void expectFloat() throws IOException {
+        out.print("Float: ");
+        out.println(in.readFloat());
+    }
+
+    private void expectLong() throws IOException {
+        out.print("Long: ");
+        out.println(in.readLong());
+    }
+
+    private void expectInteger() throws IOException {
+        out.print("Integer: ");
+        out.println(in.readInteger());
+    }
+
+    private void expectByte() throws IOException {
+        out.print("Byte: ");
+        out.println(in.readByte());
+    }
+
+    private void expectBoolean() throws IOException {
+        out.print("Boolean: ");
+        out.println(in.readBoolean());
+    }
+}
diff --git a/bundles/org.simantics.databoard/src/org/simantics/databoard/streaming/DataWriter.java b/bundles/org.simantics.databoard/src/org/simantics/databoard/streaming/DataWriter.java
new file mode 100644
index 000000000..188ea32a1
--- /dev/null
+++ b/bundles/org.simantics.databoard/src/org/simantics/databoard/streaming/DataWriter.java
@@ -0,0 +1,113 @@
+package org.simantics.databoard.streaming;
+
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+import org.simantics.databoard.Bindings;
+import org.simantics.databoard.serialization.Serializer;
+import org.simantics.databoard.type.Datatype;
+import org.simantics.databoard.util.binary.Endian;
+import org.simantics.databoard.util.binary.UTF8;
+
+/**
+ * This is an utility class that encapsulates the databoard value encoding scheme.
+ * It can be used for streaming value writing. 
+ */
+public class DataWriter {
+    private static final Serializer DATATYPE_SERIALIZER = Bindings.getSerializerUnchecked(Bindings.getBindingUnchecked( Datatype.class ));
+
+    private final DataOutput out;
+
+    public DataWriter(DataOutput out) {
+        this.out = out;
+    }
+    
+    public DataWriter(OutputStream stream) {
+        this.out = new DataOutputStream(stream);
+    }
+    
+    public void writeBoolean(boolean value) throws IOException {
+        out.write(value ? 1 : 0);
+    }
+    
+    public void writeByte(byte value) throws IOException {
+        out.write(value);
+    }
+    
+    public void writeInteger(int value) throws IOException {
+        out.writeInt(value);
+    }
+    
+    public void writeLong(long value) throws IOException {
+        out.writeLong(value);
+    }
+    
+    public void writeFloat(float value) throws IOException {
+        out.writeFloat(value);
+    }
+    
+    public void writeDouble(double value) throws IOException {
+        out.writeDouble(value);
+    }
+    
+    public void writeString(String value) throws IOException {
+        int utflen = UTF8.getModifiedUTF8EncodingByteLength(value);
+        Endian.writeDynamicUInt32(out, utflen);
+        UTF8.writeModifiedUTF(out, value);
+    }
+    
+    public void writeDatatype(Datatype datatype) throws IOException {
+        DATATYPE_SERIALIZER.serialize(out, datatype);
+    }
+    
+    /**
+     * A variable length array is started with the length
+     * followed by the serialization of all elements.
+     */
+    public void beginVariableLengthArray(int length) throws IOException {
+        out.writeInt(length);
+    }
+    
+    /**
+     * A map is started with its size followed by 
+     * serialization of interleaved keys and values.
+     */
+    public void beginMap(int size) throws IOException {
+        out.writeInt(size);
+    }
+    
+    /**
+     * An optional value that is null is written as byte 0.
+     */
+    public void writeOptionalNull() throws IOException {
+        out.write(0);
+    }
+    
+    /**
+     * An optional value that is not null is started
+     * with byte 1 followed by the actual value.
+     */
+    public void beginOptionalValue() throws IOException {
+        out.write(1);
+    }
+    
+    /**
+     * Selects the constructor of the union type.
+     * It is written as a variable length integer,
+     * so the total number of tags is required.
+     */
+    public void writeUnionTag(int tag, int tagCount) throws IOException {
+        Endian.putUInt(out, tag, tagCount-1);
+    }
+    
+    public void writeReferenceToKnownReferableRecord(int id) throws IOException {
+        out.writeInt(id);
+    }
+    
+    public void beginUnknownReferableRecord(int id) throws IOException {
+        out.writeInt(0);
+        out.writeInt(id);
+    }
+}