1 package org.simantics.graph.db;
\r
3 import java.io.DataInput;
\r
4 import java.io.DataInputStream;
\r
6 import java.io.IOException;
\r
7 import java.io.InputStream;
\r
8 import java.nio.ByteBuffer;
\r
9 import java.nio.channels.ReadableByteChannel;
\r
10 import java.util.ArrayList;
\r
11 import java.util.Arrays;
\r
12 import java.util.List;
\r
13 import java.util.TreeMap;
\r
15 import org.simantics.databoard.Bindings;
\r
16 import org.simantics.databoard.binding.Binding;
\r
17 import org.simantics.databoard.binding.mutable.Variant;
\r
18 import org.simantics.databoard.container.DataContainer;
\r
19 import org.simantics.databoard.container.DataContainers;
\r
20 import org.simantics.databoard.serialization.Serializer;
\r
21 import org.simantics.databoard.type.Datatype;
\r
22 import org.simantics.db.ReadGraph;
\r
23 import org.simantics.graph.db.TransferableGraphSource.TransferableGraphSourceValueProcedure;
\r
24 import org.simantics.graph.representation.ByteFileReader;
\r
25 import org.simantics.graph.representation.Extensions;
\r
26 import org.simantics.graph.representation.External;
\r
27 import org.simantics.graph.representation.Identity;
\r
28 import org.simantics.graph.representation.Internal;
\r
29 import org.simantics.graph.representation.Root;
\r
30 import org.simantics.graph.representation.Value;
\r
33 final public class StreamingTransferableGraphFileReader extends ByteFileReader {
\r
35 final static class InputChannel implements ReadableByteChannel {
\r
37 final private InputStream stream;
\r
39 public InputChannel(InputStream stream) {
\r
40 this.stream = stream;
\r
44 public boolean isOpen() {
\r
49 public void close() throws IOException {
\r
53 public int read(ByteBuffer dst) throws IOException {
\r
54 int pos = dst.position();
\r
55 int limit = dst.limit();
\r
56 int i=stream.read(dst.array(), pos, limit-pos);
\r
62 private static boolean init = true;
\r
64 final private static int SIZE = 1<<18;
\r
66 public StreamingTransferableGraphFileReader(File file) throws Exception {
\r
70 @SuppressWarnings("resource")
\r
71 StreamingTransferableGraphFileReader r = new StreamingTransferableGraphFileReader(file, 0);
\r
72 for(int i=0;i<40000;i++) r.readTG();
\r
76 public StreamingTransferableGraphFileReader(InputStream stream) throws Exception {
\r
77 super(null, new InputChannel(stream), SIZE);
\r
80 @SuppressWarnings("resource")
\r
81 StreamingTransferableGraphFileReader r = new StreamingTransferableGraphFileReader(stream, 0);
\r
82 for(int i=0;i<40000;i++) r.readTG();
\r
86 public StreamingTransferableGraphFileReader(ReadableByteChannel channel) throws Exception {
\r
87 super(null, channel, SIZE);
\r
90 @SuppressWarnings("resource")
\r
91 StreamingTransferableGraphFileReader r = new StreamingTransferableGraphFileReader(channel, 0);
\r
92 for(int i=0;i<40000;i++) r.readTG();
\r
96 public StreamingTransferableGraphFileReader(ReadableByteChannel channel, int size) throws IOException {
\r
97 super(null, channel, SIZE);
\r
100 public StreamingTransferableGraphFileReader(InputStream stream, int size) throws IOException {
\r
101 super(null, new InputChannel(stream), size);
\r
104 public StreamingTransferableGraphFileReader(File file, int size) throws IOException {
\r
108 class FileTransferableGraphSource implements TransferableGraphSource {
\r
110 InputStream in = new InputStream() {
\r
113 public int read() throws IOException {
\r
118 public int read(byte[] b) throws IOException {
\r
119 // FIXME not correctly implemented
\r
120 System.arraycopy(safeBytes(b.length), 0, b, 0, b.length);
\r
125 public int read(byte[] b, int off, int len) throws IOException {
\r
126 for(int i=0;i<len;i++)
\r
127 b[off++] = (byte)getByte();
\r
132 DataInputStream dis = new DataInputStream(in);
\r
134 DataContainer header;
\r
135 Extensions extensions;
\r
138 private int identities;
\r
139 private int stmLength;
\r
140 private int valueLength;
\r
142 public FileTransferableGraphSource() throws Exception {
\r
146 private void init() throws Exception {
\r
149 header = DataContainers.readHeader(dis);
\r
151 // Content variant data type
\r
152 Bindings.getSerializerUnchecked(Datatype.class).deserialize((DataInput)dis);
\r
154 resourceCount = safeInt();
\r
156 List<Object> idcontext = new ArrayList<Object>();
\r
157 extensions = (Extensions)Bindings.getSerializerUnchecked(Extensions.class).deserialize((DataInput)dis, idcontext);
\r
162 public void reset() throws Exception {
\r
163 StreamingTransferableGraphFileReader.this.reset();
\r
164 throw new UnsupportedOperationException();
\r
168 public DataContainer getHeader() throws Exception {
\r
173 public int getResourceCount() throws Exception {
\r
174 return resourceCount;
\r
178 public int getIdentityCount() throws Exception {
\r
179 identities = safeInt();
\r
184 public int getStatementCount() throws Exception {
\r
185 stmLength = safeInt();
\r
190 public int getValueCount() throws Exception {
\r
191 valueLength = safeInt();
\r
192 return valueLength;
\r
196 public void forStatements(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
\r
198 int[] value = new int[4];
\r
200 for(int stmIndex=0;stmIndex<stmLength;) {
\r
202 value[stmIndex & 3] = safeInt();
\r
204 if((stmIndex & 3) == 0) procedure.execute(value);
\r
207 int avail = (SIZE-byteIndex) >> 2;
\r
208 int allowed = Math.min(stmLength-stmIndex, avail);
\r
209 for(int index = byteIndex, i=0;i<allowed;i++) {
\r
210 value[stmIndex & 3] = ((bytes[index++]&0xff)<<24) | ((bytes[index++]&0xff)<<16) | ((bytes[index++]&0xff)<<8) | ((bytes[index++]&0xff));
\r
212 if((stmIndex & 3) == 0) procedure.execute(value);
\r
213 // statements[stmIndex++] = ((bytes[index++]&0xff)<<24) | ((bytes[index++]&0xff)<<16) | ((bytes[index++]&0xff)<<8) | ((bytes[index++]&0xff));
\r
215 byteIndex += allowed<<2;
\r
222 public void forIdentities(ReadGraph graph, TransferableGraphSourceProcedure<Identity> procedure) throws Exception {
\r
224 for(int i=0;i<identities;i++) {
\r
226 int rid = safeInt();
\r
227 byte type = bytes[byteIndex++];
\r
231 int parent = safeInt();
\r
232 int nameLen = bytes[byteIndex++]&0xff;
\r
234 if(byteIndex+nameLen < SIZE) {
\r
235 procedure.execute(new Identity(rid, new External(parent, utf(bytes, byteIndex, byteIndex + nameLen))));
\r
236 byteIndex += nameLen;
\r
238 procedure.execute(new Identity(rid, new External(parent, utf(safeBytes(nameLen), 0, nameLen))));
\r
243 else if(type == 3) {
\r
245 int parent = safeInt();
\r
246 int nameLen = bytes[byteIndex++]&0xff;
\r
247 if(byteIndex+nameLen < SIZE) {
\r
248 procedure.execute(new Identity(rid, new Internal(parent, utf(bytes, byteIndex, byteIndex + nameLen))));
\r
249 byteIndex += nameLen;
\r
251 procedure.execute(new Identity(rid, new Internal(parent, utf(safeBytes(nameLen), 0, nameLen))));
\r
256 else if(type == 0) {
\r
258 int nameLen = bytes[byteIndex++]&0xff;
\r
259 String name = utf(safeBytes(nameLen), 0, nameLen);
\r
260 int nameLen2 = bytes[byteIndex++]&0xff;
\r
261 String rType = utf(safeBytes(nameLen2), 0, nameLen2);
\r
262 procedure.execute(new Identity(rid, new Root(name, rType)));
\r
264 } else if(type == 2) {
\r
265 throw new UnsupportedOperationException();
\r
273 public void forValues(ReadGraph graph, TransferableGraphSourceProcedure<Value> procedure) throws Exception {
\r
275 Serializer variantSerializer = Bindings.getSerializerUnchecked(Bindings.VARIANT);
\r
277 List<Object> idcontext = new ArrayList<>();
\r
279 for(int i=0;i<valueLength;i++) {
\r
280 int resource = safeInt();
\r
281 Variant value = (Variant)variantSerializer
\r
282 .deserialize((DataInput)dis, idcontext);
\r
283 procedure.execute(new Value(resource, value));
\r
290 public void forValues2(ReadGraph graph, TransferableGraphSourceValueProcedure procedure) throws Exception {
\r
292 Binding datatypeBinding = Bindings.getBinding(Datatype.class);
\r
293 Serializer datatypeSerializer = Bindings.getSerializerUnchecked(datatypeBinding);
\r
295 List<Object> idContext = new ArrayList<>();
\r
297 for(int i=0;i<valueLength;i++) {
\r
298 int resource = safeInt();
\r
300 Datatype type = (Datatype)datatypeSerializer.deserialize((DataInput)dis, idContext);
\r
301 procedure.execute(resource, type, dis);
\r
307 public TreeMap<String, Variant> getExtensions() {
\r
308 return extensions.map;
\r
312 public void close() {
\r
316 public TransferableGraphSource readTG() throws Exception {
\r
318 if(getSize() == 0) return null;
\r
320 return new FileTransferableGraphSource();
\r
324 public static void main(String[] args) {
\r
328 File file = new File("c:/work/Model.apros");
\r
329 StreamingTransferableGraphFileReader reader = new StreamingTransferableGraphFileReader(file, SIZE);
\r
330 reader = new StreamingTransferableGraphFileReader(file);
\r
331 long s = System.nanoTime();
\r
332 TransferableGraphSource tgs = reader.readTG();
\r
333 int ids = tgs.getIdentityCount();
\r
334 System.out.println("identity count " + ids);
\r
335 // tgs.forIdentities(null, id -> { /*System.out.println("Identity: " + id);*/ });
\r
336 tgs.forIdentities(null, id -> { System.out.println("Identity: " + id); });
\r
337 int stats = tgs.getStatementCount();
\r
338 System.out.println("statement count " + stats/4 + " (" + stats + ")");
\r
339 // tgs.forStatements(null, id -> { /*System.out.println(Arrays.toString(id));*/ });
\r
340 tgs.forStatements(null, id -> { System.out.println(Arrays.toString(id)); });
\r
341 int values = tgs.getValueCount();
\r
342 System.out.println("value count " + values);
\r
343 int[] valueNo = {0};
\r
344 tgs.forValues2(null, new TransferableGraphSourceValueProcedure() {
\r
346 public void rawCopy(int resource, int length, DataInput input) throws Exception {
\r
347 System.out.println("value " + (valueNo[0]++) + ": rawCopy("+ resource + ", " + length + ", " + input + ")");
\r
348 for (int i = 0; i < length; ++i)
\r
352 public void execute(int resource, Datatype type, DataInput input) throws Exception {
\r
353 Object value = Bindings.getSerializer(Bindings.getBinding(type)).deserialize(input);
\r
354 System.out.println("value " + (valueNo[0]++) + ": execute("+ resource + ", " + type.toSingleLineString() + ", " + input + "): " + value);
\r
357 long d = System.nanoTime() - s;
\r
358 System.err.println("Duration=" + 1e-9*d + "s.");
\r
359 } catch (Throwable t) {
\r
360 t.printStackTrace();
\r