1 package org.simantics.graph.db;
3 import java.io.DataInput;
4 import java.io.DataInputStream;
6 import java.io.IOException;
7 import java.io.InputStream;
8 import java.nio.channels.ReadableByteChannel;
9 import java.util.ArrayList;
10 import java.util.Arrays;
11 import java.util.List;
12 import java.util.TreeMap;
14 import org.simantics.databoard.Bindings;
15 import org.simantics.databoard.binding.Binding;
16 import org.simantics.databoard.binding.mutable.Variant;
17 import org.simantics.databoard.container.DataContainer;
18 import org.simantics.databoard.container.DataContainers;
19 import org.simantics.databoard.serialization.Serializer;
20 import org.simantics.databoard.type.Datatype;
21 import org.simantics.db.ReadGraph;
22 import org.simantics.graph.db.TransferableGraphSource.TransferableGraphSourceValueProcedure;
23 import org.simantics.graph.representation.ByteFileReader;
24 import org.simantics.graph.representation.Extensions;
25 import org.simantics.graph.representation.External;
26 import org.simantics.graph.representation.Identity;
27 import org.simantics.graph.representation.InputChannel;
28 import org.simantics.graph.representation.Internal;
29 import org.simantics.graph.representation.Root;
30 import org.simantics.graph.representation.Value;
33 final public class StreamingTransferableGraphFileReader extends ByteFileReader {
35 private static boolean init = true;
37 final private static int SIZE = 1<<18;
39 public StreamingTransferableGraphFileReader(File file) throws Exception {
43 @SuppressWarnings("resource")
44 StreamingTransferableGraphFileReader r = new StreamingTransferableGraphFileReader(file, 0);
45 for(int i=0;i<40000;i++) r.readTG();
49 public StreamingTransferableGraphFileReader(InputStream stream) throws Exception {
50 super(null, new InputChannel(stream), SIZE);
53 @SuppressWarnings("resource")
54 StreamingTransferableGraphFileReader r = new StreamingTransferableGraphFileReader(stream, 0);
55 for(int i=0;i<40000;i++) r.readTG();
59 public StreamingTransferableGraphFileReader(ReadableByteChannel channel) throws Exception {
60 super(null, channel, SIZE);
63 @SuppressWarnings("resource")
64 StreamingTransferableGraphFileReader r = new StreamingTransferableGraphFileReader(channel, 0);
65 for(int i=0;i<40000;i++) r.readTG();
69 public StreamingTransferableGraphFileReader(ReadableByteChannel channel, int size) throws IOException {
70 super(null, channel, SIZE);
73 public StreamingTransferableGraphFileReader(InputStream stream, int size) throws IOException {
74 super(null, new InputChannel(stream), size);
77 public StreamingTransferableGraphFileReader(File file, int size) throws IOException {
81 class FileTransferableGraphSource implements TransferableGraphSource {
83 InputStream in = new InputStream() {
86 public int read() throws IOException {
91 public int read(byte[] b) throws IOException {
92 // FIXME not correctly implemented
93 System.arraycopy(safeBytes(b.length), 0, b, 0, b.length);
98 public int read(byte[] b, int off, int len) throws IOException {
99 for(int i=0;i<len;i++)
100 b[off++] = (byte)getByte();
105 DataInputStream dis = new DataInputStream(in);
107 DataContainer header;
108 Extensions extensions;
111 private int identities = -1;
112 private int stmLength = -1;
113 private int valueLength = -1;
115 public FileTransferableGraphSource() throws Exception {
119 private void init() throws Exception {
122 header = DataContainers.readHeader(dis);
124 // Content variant data type
125 Bindings.getSerializerUnchecked(Datatype.class).deserialize((DataInput)dis);
127 resourceCount = safeInt();
129 List<Object> idcontext = new ArrayList<Object>();
130 extensions = (Extensions)Bindings.getSerializerUnchecked(Extensions.class).deserialize((DataInput)dis, idcontext);
135 public void reset() throws Exception {
136 StreamingTransferableGraphFileReader.this.reset();
137 throw new UnsupportedOperationException();
141 public DataContainer getHeader() throws Exception {
146 public int getResourceCount() throws Exception {
147 return resourceCount;
151 public int getIdentityCount() throws Exception {
152 if(identities == -1) {
153 identities = safeInt();
159 public int getStatementCount() throws Exception {
160 if(stmLength == -1) {
161 stmLength = safeInt();
167 public int getValueCount() throws Exception {
168 if(valueLength == -1) {
169 valueLength = safeInt();
175 public void forStatements(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
177 int[] value = new int[4];
179 int stmLength = getStatementCount();
181 for(int stmIndex=0;stmIndex<stmLength;) {
183 value[stmIndex & 3] = safeInt();
185 if((stmIndex & 3) == 0) procedure.execute(value);
188 int avail = (SIZE-byteIndex) >> 2;
189 int allowed = Math.min(stmLength-stmIndex, avail);
190 for(int index = byteIndex, i=0;i<allowed;i++) {
191 value[stmIndex & 3] = ((bytes[index++]&0xff)<<24) | ((bytes[index++]&0xff)<<16) | ((bytes[index++]&0xff)<<8) | ((bytes[index++]&0xff));
193 if((stmIndex & 3) == 0) procedure.execute(value);
194 // statements[stmIndex++] = ((bytes[index++]&0xff)<<24) | ((bytes[index++]&0xff)<<16) | ((bytes[index++]&0xff)<<8) | ((bytes[index++]&0xff));
196 byteIndex += allowed<<2;
203 public void forIdentities(ReadGraph graph, TransferableGraphSourceProcedure<Identity> procedure) throws Exception {
205 int identities = getIdentityCount();
207 for(int i=0;i<identities;i++) {
210 byte type = bytes[byteIndex++];
214 int parent = safeInt();
215 int nameLen = getDynamicUInt32();
216 if(byteIndex+nameLen < SIZE) {
217 procedure.execute(new Identity(rid, new External(parent, utf(bytes, byteIndex, byteIndex + nameLen))));
218 byteIndex += nameLen;
220 procedure.execute(new Identity(rid, new External(parent, utf(safeBytes(nameLen), 0, nameLen))));
226 int parent = safeInt();
227 int nameLen = getDynamicUInt32();
228 if(byteIndex+nameLen < SIZE) {
229 procedure.execute(new Identity(rid, new Internal(parent, utf(bytes, byteIndex, byteIndex + nameLen))));
230 byteIndex += nameLen;
232 procedure.execute(new Identity(rid, new Internal(parent, utf(safeBytes(nameLen), 0, nameLen))));
239 int nameLen = getDynamicUInt32();
240 String name = utf(safeBytes(nameLen), 0, nameLen);
241 int nameLen2 = getDynamicUInt32();
242 String rType = utf(safeBytes(nameLen2), 0, nameLen2);
243 procedure.execute(new Identity(rid, new Root(name, rType)));
248 throw new UnsupportedOperationException("Optional identities not supported");
250 throw new IllegalStateException("Unsupported identity type " + type);
258 public void forValues(ReadGraph graph, TransferableGraphSourceProcedure<Value> procedure) throws Exception {
260 int valueLength = getValueCount();
262 Serializer variantSerializer = Bindings.getSerializerUnchecked(Bindings.VARIANT);
264 List<Object> idcontext = new ArrayList<>();
266 for(int i=0;i<valueLength;i++) {
267 int resource = safeInt();
269 Variant value = (Variant)variantSerializer
270 .deserialize((DataInput)dis, idcontext);
271 procedure.execute(new Value(resource, value));
277 public void forValues2(ReadGraph graph, TransferableGraphSourceValueProcedure procedure) throws Exception {
279 Binding datatypeBinding = Bindings.getBinding(Datatype.class);
280 Serializer datatypeSerializer = Bindings.getSerializerUnchecked(datatypeBinding);
282 List<Object> idContext = new ArrayList<>();
284 for(int i=0;i<valueLength;i++) {
285 int resource = safeInt();
287 Datatype type = (Datatype)datatypeSerializer.deserialize((DataInput)dis, idContext);
288 procedure.execute(resource, type, dis);
294 public TreeMap<String, Variant> getExtensions() {
295 return extensions.map;
299 public void close() {
303 public TransferableGraphSource readTG() throws Exception {
305 if(getSize() == 0) return null;
307 return new FileTransferableGraphSource();
311 public static void main(String[] args) {
315 File file = new File("c:/work/Model.apros");
316 StreamingTransferableGraphFileReader reader = new StreamingTransferableGraphFileReader(file, SIZE);
317 reader = new StreamingTransferableGraphFileReader(file);
318 long s = System.nanoTime();
319 TransferableGraphSource tgs = reader.readTG();
320 int ids = tgs.getIdentityCount();
321 System.out.println("identity count " + ids);
322 // tgs.forIdentities(null, id -> { /*System.out.println("Identity: " + id);*/ });
323 tgs.forIdentities(null, id -> { System.out.println("Identity: " + id); });
324 int stats = tgs.getStatementCount();
325 System.out.println("statement count " + stats/4 + " (" + stats + ")");
326 // tgs.forStatements(null, id -> { /*System.out.println(Arrays.toString(id));*/ });
327 tgs.forStatements(null, id -> { System.out.println(Arrays.toString(id)); });
328 int values = tgs.getValueCount();
329 System.out.println("value count " + values);
331 tgs.forValues2(null, new TransferableGraphSourceValueProcedure() {
333 public void rawCopy(int resource, int length, DataInput input) throws Exception {
334 System.out.println("value " + (valueNo[0]++) + ": rawCopy("+ resource + ", " + length + ", " + input + ")");
335 for (int i = 0; i < length; ++i)
339 public void execute(int resource, Datatype type, DataInput input) throws Exception {
340 Object value = Bindings.getSerializer(Bindings.getBinding(type)).deserialize(input);
341 System.out.println("value " + (valueNo[0]++) + ": execute("+ resource + ", " + type.toSingleLineString() + ", " + input + "): " + value);
344 long d = System.nanoTime() - s;
345 System.err.println("Duration=" + 1e-9*d + "s.");
346 } catch (Throwable t) {