X-Git-Url: https://gerrit.simantics.org/r/gitweb?a=blobdiff_plain;f=bundles%2Forg.simantics.graph.db%2Fsrc%2Forg%2Fsimantics%2Fgraph%2Fdb%2FStreamingTransferableGraphFileReader.java;fp=bundles%2Forg.simantics.graph.db%2Fsrc%2Forg%2Fsimantics%2Fgraph%2Fdb%2FStreamingTransferableGraphFileReader.java;h=89c0620d53cd1de2df0ed7f113e162230de9f4d5;hb=969bd23cab98a79ca9101af33334000879fb60c5;hp=0000000000000000000000000000000000000000;hpb=866dba5cd5a3929bbeae85991796acb212338a08;p=simantics%2Fplatform.git diff --git a/bundles/org.simantics.graph.db/src/org/simantics/graph/db/StreamingTransferableGraphFileReader.java b/bundles/org.simantics.graph.db/src/org/simantics/graph/db/StreamingTransferableGraphFileReader.java new file mode 100644 index 000000000..89c0620d5 --- /dev/null +++ b/bundles/org.simantics.graph.db/src/org/simantics/graph/db/StreamingTransferableGraphFileReader.java @@ -0,0 +1,364 @@ +package org.simantics.graph.db; + +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.channels.ReadableByteChannel; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.TreeMap; + +import org.simantics.databoard.Bindings; +import org.simantics.databoard.binding.Binding; +import org.simantics.databoard.binding.mutable.Variant; +import org.simantics.databoard.container.DataContainer; +import org.simantics.databoard.container.DataContainers; +import org.simantics.databoard.serialization.Serializer; +import org.simantics.databoard.type.Datatype; +import org.simantics.db.ReadGraph; +import org.simantics.graph.db.TransferableGraphSource.TransferableGraphSourceValueProcedure; +import org.simantics.graph.representation.ByteFileReader; +import org.simantics.graph.representation.Extensions; +import org.simantics.graph.representation.External; +import org.simantics.graph.representation.Identity; +import org.simantics.graph.representation.Internal; +import org.simantics.graph.representation.Root; +import org.simantics.graph.representation.Value; + + +final public class StreamingTransferableGraphFileReader extends ByteFileReader { + + final static class InputChannel implements ReadableByteChannel { + + final private InputStream stream; + + public InputChannel(InputStream stream) { + this.stream = stream; + } + + @Override + public boolean isOpen() { + return true; + } + + @Override + public void close() throws IOException { + } + + @Override + public int read(ByteBuffer dst) throws IOException { + int pos = dst.position(); + int limit = dst.limit(); + int i=stream.read(dst.array(), pos, limit-pos); + return i; + } + + } + + private static boolean init = true; + + final private static int SIZE = 1<<18; + + public StreamingTransferableGraphFileReader(File file) throws Exception { + super(file, SIZE); + if(init) { + init=false; + @SuppressWarnings("resource") + StreamingTransferableGraphFileReader r = new StreamingTransferableGraphFileReader(file, 0); + for(int i=0;i<40000;i++) r.readTG(); + } + } + + public StreamingTransferableGraphFileReader(InputStream stream) throws Exception { + super(null, new InputChannel(stream), SIZE); + if(init) { + init=false; + @SuppressWarnings("resource") + StreamingTransferableGraphFileReader r = new StreamingTransferableGraphFileReader(stream, 0); + for(int i=0;i<40000;i++) r.readTG(); + } + } + + public StreamingTransferableGraphFileReader(ReadableByteChannel channel) throws Exception { + super(null, channel, SIZE); + if(init) { + init=false; + @SuppressWarnings("resource") + StreamingTransferableGraphFileReader r = new StreamingTransferableGraphFileReader(channel, 0); + for(int i=0;i<40000;i++) r.readTG(); + } + } + + public StreamingTransferableGraphFileReader(ReadableByteChannel channel, int size) throws IOException { + super(null, channel, SIZE); + } + + public StreamingTransferableGraphFileReader(InputStream stream, int size) throws IOException { + super(null, new InputChannel(stream), size); + } + + public StreamingTransferableGraphFileReader(File file, int size) throws IOException { + super(file, size); + } + + class FileTransferableGraphSource implements TransferableGraphSource { + + InputStream in = new InputStream() { + + @Override + public int read() throws IOException { + return getByte(); + } + + @Override + public int read(byte[] b) throws IOException { + // FIXME not correctly implemented + System.arraycopy(safeBytes(b.length), 0, b, 0, b.length); + return b.length; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + for(int i=0;i idcontext = new ArrayList(); + extensions = (Extensions)Bindings.getSerializerUnchecked(Extensions.class).deserialize((DataInput)dis, idcontext); + + } + + @Override + public void reset() throws Exception { + StreamingTransferableGraphFileReader.this.reset(); + throw new UnsupportedOperationException(); + } + + @Override + public DataContainer getHeader() throws Exception { + return header; + } + + @Override + public int getResourceCount() throws Exception { + return resourceCount; + } + + @Override + public int getIdentityCount() throws Exception { + identities = safeInt(); + return identities; + } + + @Override + public int getStatementCount() throws Exception { + stmLength = safeInt(); + return stmLength; + } + + @Override + public int getValueCount() throws Exception { + valueLength = safeInt(); + return valueLength; + } + + @Override + public void forStatements(ReadGraph graph, TransferableGraphSourceProcedure procedure) throws Exception { + + int[] value = new int[4]; + + for(int stmIndex=0;stmIndex> 2; + int allowed = Math.min(stmLength-stmIndex, avail); + for(int index = byteIndex, i=0;i procedure) throws Exception { + + for(int i=0;i procedure) throws Exception { + + Serializer variantSerializer = Bindings.getSerializerUnchecked(Bindings.VARIANT); + + List idcontext = new ArrayList<>(); + + for(int i=0;i idContext = new ArrayList<>(); + + for(int i=0;i getExtensions() { + return extensions.map; + } + + @Override + public void close() { + } + } + + public TransferableGraphSource readTG() throws Exception { + + if(getSize() == 0) return null; + + return new FileTransferableGraphSource(); + + } + + public static void main(String[] args) { + + try { + + File file = new File("c:/work/Model.apros"); + StreamingTransferableGraphFileReader reader = new StreamingTransferableGraphFileReader(file, SIZE); + reader = new StreamingTransferableGraphFileReader(file); + long s = System.nanoTime(); + TransferableGraphSource tgs = reader.readTG(); + int ids = tgs.getIdentityCount(); + System.out.println("identity count " + ids); +// tgs.forIdentities(null, id -> { /*System.out.println("Identity: " + id);*/ }); + tgs.forIdentities(null, id -> { System.out.println("Identity: " + id); }); + int stats = tgs.getStatementCount(); + System.out.println("statement count " + stats/4 + " (" + stats + ")"); +// tgs.forStatements(null, id -> { /*System.out.println(Arrays.toString(id));*/ }); + tgs.forStatements(null, id -> { System.out.println(Arrays.toString(id)); }); + int values = tgs.getValueCount(); + System.out.println("value count " + values); + int[] valueNo = {0}; + tgs.forValues2(null, new TransferableGraphSourceValueProcedure() { + @Override + public void rawCopy(int resource, int length, DataInput input) throws Exception { + System.out.println("value " + (valueNo[0]++) + ": rawCopy("+ resource + ", " + length + ", " + input + ")"); + for (int i = 0; i < length; ++i) + input.readByte(); + } + @Override + public void execute(int resource, Datatype type, DataInput input) throws Exception { + Object value = Bindings.getSerializer(Bindings.getBinding(type)).deserialize(input); + System.out.println("value " + (valueNo[0]++) + ": execute("+ resource + ", " + type.toSingleLineString() + ", " + input + "): " + value); + } + }); + long d = System.nanoTime() - s; + System.err.println("Duration=" + 1e-9*d + "s."); + } catch (Throwable t) { + t.printStackTrace(); + } + } + +}