1 package org.simantics.graph.representation;
3 import java.io.DataInput;
4 import java.io.DataInputStream;
6 import java.io.IOException;
7 import java.io.InputStream;
8 import java.nio.channels.ReadableByteChannel;
9 import java.util.ArrayList;
10 import java.util.List;
12 import org.simantics.databoard.Bindings;
13 import org.simantics.databoard.Datatypes;
14 import org.simantics.databoard.binding.Binding;
15 import org.simantics.databoard.binding.error.RuntimeDatatypeConstructionException;
16 import org.simantics.databoard.binding.mutable.Variant;
17 import org.simantics.databoard.container.DataContainer;
18 import org.simantics.databoard.container.DataContainers;
19 import org.simantics.databoard.serialization.RuntimeSerializerConstructionException;
20 import org.simantics.databoard.serialization.Serializer;
21 import org.simantics.databoard.type.Datatype;
22 import org.slf4j.Logger;
23 import org.slf4j.LoggerFactory;
27 * It is recommended to use {@link #read(File)} and {@link #read(InputStream)}
28 * for reading to ensure proper resource handling.
31 * It is important to use the correct setting for
32 * {@link #sharedValueContext(boolean)} which depends on how the TG was
33 * serialized to begin with. See {@link #DEFAULT_SHARED_VALUE_CONTEXT} for more
34 * information on this.
36 final public class TransferableGraphFileReader extends ByteFileReader {
39 * Serializing TransferableGraph1 structures using the default {@link Binding}
40 * will use shared context for serializing the values Variant array. Thus all TG
41 * files produced by the graph compiler use a shared value context which means
42 * this class must be used with {@link #sharedValueContext(boolean)} set to
43 * true. As an example, <code>true</code> must be used if the corresponding TG
44 * file is written e.g. like this:
46 * DataContainers.writeFile(location
47 * new DataContainer(format,
50 * new Variant(TransferableGraph1.BINDING, tg)));
54 * On the other hand, any TG files serialized using more optimized methods like
55 * <code>ModelTransferableGraphSource</code> do not use a shared value context
56 * when writing the file. This means those files cannot be read safely using
57 * standard {@link Binding} at all, and when using this class,
58 * {@link #sharedValueContext(boolean)} must be set to false to prevent the
59 * import from corrupting datatype values because the referable parts of
60 * datatypes may get bound to the wrong existing types if the data is read using
64 * <code>true</code> is the default setting.
66 public static final boolean DEFAULT_SHARED_VALUE_CONTEXT = true;
68 private static final Logger LOGGER = LoggerFactory.getLogger(TransferableGraphFileReader.class);
70 InputStream in = new InputStream() {
73 public int read() throws IOException {
78 public int read(byte[] b) throws IOException {
79 // FIXME not correctly implemented
80 System.arraycopy(safeBytes(b.length), 0, b, 0, b.length);
85 public int read(byte[] b, int off, int len) throws IOException {
86 // FIXME not correctly implemented
87 System.arraycopy(safeBytes(len), 0, b, off, len);
93 private static boolean init = true;
95 final private static int SIZE = 1<<18;
96 final private static int HEADER = headerSize();
97 final private int header;
98 private boolean shareValueContext = true;
101 * Reads a {@link DataContainer} containing a {@link TransferableGraph1}
102 * structure from the specified {@link File}.
104 * @param file the file to read from
105 * @return the TG contained by the file
106 * @throws IOException
108 public static TransferableGraph1 read(File file) throws IOException {
109 return read(file, DEFAULT_SHARED_VALUE_CONTEXT);
113 * Reads a {@link DataContainer} containing a {@link TransferableGraph1}
114 * structure from the specified InputStream. Note that this implementation does
115 * not close the specified <code>input</code> stream, it is expected to be
116 * closed by the caller.
118 * @param input the input stream to read from
119 * @return the TG contained by the stream
120 * @throws IOException
122 public static TransferableGraph1 read(InputStream input) throws IOException {
123 return read(input, DEFAULT_SHARED_VALUE_CONTEXT);
127 * Reads a {@link DataContainer} containing a {@link TransferableGraph1}
128 * structure from the specified {@link File}.
130 * @param file the file to read from
131 * @return the TG contained by the file
132 * @throws IOException
134 public static TransferableGraph1 read(File file, boolean sharedValueContext) throws IOException {
135 try (TransferableGraphFileReader reader = new TransferableGraphFileReader(file)) {
136 return reader.sharedValueContext(sharedValueContext).readTG();
141 * Reads a {@link DataContainer} containing a {@link TransferableGraph1}
142 * structure from the specified InputStream. Note that this implementation does
143 * not close the specified <code>input</code> stream, it is expected to be
144 * closed by the caller.
146 * @param input the input stream to read from
147 * @return the TG contained by the stream
148 * @throws IOException
150 public static TransferableGraph1 read(InputStream input, boolean sharedValueContext) throws IOException {
151 try (TransferableGraphFileReader reader = new TransferableGraphFileReader(input)) {
152 return reader.sharedValueContext(sharedValueContext).readTG();
156 public TransferableGraphFileReader(File file) throws IOException {
160 TransferableGraphFileReader r = new TransferableGraphFileReader(file, 0);
161 for(int i=0;i<40000;i++) r.readTG();
163 this.header = HEADER;
166 public TransferableGraphFileReader(InputStream stream) throws IOException {
167 super(null, new InputChannel(stream), SIZE);
170 TransferableGraphFileReader r = new TransferableGraphFileReader(stream, 0);
171 for(int i=0;i<40000;i++) r.readTG();
176 public TransferableGraphFileReader(ReadableByteChannel channel) throws IOException {
177 super(null, channel, SIZE);
180 TransferableGraphFileReader r = new TransferableGraphFileReader(channel, 0);
181 for(int i=0;i<40000;i++) r.readTG();
186 public TransferableGraphFileReader(ReadableByteChannel channel, int size) throws IOException {
187 super(null, channel, SIZE);
191 public TransferableGraphFileReader(InputStream stream, int size) throws IOException {
192 super(null, new InputChannel(stream), size);
196 public TransferableGraphFileReader(File file, int size) throws IOException {
198 this.header = HEADER;
201 public TransferableGraphFileReader sharedValueContext(boolean share) {
202 this.shareValueContext = share;
206 private static int headerSize() {
208 return Bindings.getSerializerUnchecked(Datatype.class).serialize(Datatypes.getDatatypeUnchecked(TransferableGraph1.class)).length;
209 } catch (RuntimeSerializerConstructionException e) {
210 throw new Error("Failed to determine TransferableGraph1 header size. ", e);
211 } catch (RuntimeDatatypeConstructionException e) {
212 throw new Error("Failed to determine TransferableGraph1 header size. ", e);
213 } catch (IOException e) {
214 throw new Error("Failed to determine TransferableGraph1 header size. ", e);
218 public TransferableGraph1 readTG() throws IOException {
220 if(getSize() == 0) return null;
222 // long start = System.nanoTime();
224 final byte[] bytes = getBytes();
226 // byteIndex = header;
228 DataInputStream dis = new DataInputStream(in);
231 DataContainers.readHeader(dis);
233 // Content variant data type
234 Bindings.getSerializerUnchecked(Datatype.class).deserialize((DataInput)dis);
236 int resourceCount = safeInt();
238 List<Object> idcontext = new ArrayList<>();
239 dis = new DataInputStream(in);
240 Extensions extensions = (Extensions)Bindings.getSerializerUnchecked(Extensions.class).deserialize((DataInput)dis, idcontext);
242 int identities = safeInt();
243 Identity[] ids = new Identity[identities];
245 // LOGGER.warn("rc: " + resourceCount);
246 // LOGGER.warn("ids: " + identities);
248 // long duration = System.nanoTime() - start;
249 // LOGGER.warn("start in " + 1e-9*duration + "s.");
250 // start = System.nanoTime();
252 for(int i=0;i<identities;i++) {
254 byte type = bytes[byteIndex++];
258 int parent = safeInt();
259 int nameLen = getDynamicUInt32();
261 if(byteIndex+nameLen < SIZE) {
262 ids[i] = new Identity(rid, new External(parent, utf(bytes, byteIndex, byteIndex + nameLen)));
263 byteIndex += nameLen;
265 ids[i] = new Identity(rid, new External(parent, utf(safeBytes(nameLen), 0, nameLen)));
272 int parent = safeInt();
273 int nameLen = getDynamicUInt32();
274 if(byteIndex+nameLen < SIZE) {
275 ids[i] = new Identity(rid, new Internal(parent, utf(bytes, byteIndex, byteIndex + nameLen)));
276 byteIndex += nameLen;
278 ids[i] = new Identity(rid, new Internal(parent, utf(safeBytes(nameLen), 0, nameLen)));
284 int nameLen = getDynamicUInt32();
285 String name = utf(safeBytes(nameLen), 0, nameLen);
286 int nameLen2 = getDynamicUInt32();
287 String rType = utf(safeBytes(nameLen2), 0, nameLen2);
288 ids[i] = new Identity(rid, new Root(name, rType));
290 } else if(type == 2) {
291 throw new UnsupportedOperationException();
293 throw new IllegalStateException();
298 // for(Identity id : ids) LOGGER.warn("id: " + id);
301 // duration = System.nanoTime() - start;
302 // LOGGER.warn("ids in " + 1e-9*duration + "s.");
303 // start = System.nanoTime();
305 int stmLength = safeInt();
306 // LOGGER.warn("statements: " + stmLength + " (" + byteIndex + ")");
308 int[] statements = new int[stmLength];
310 for(int stmIndex=0;stmIndex<stmLength;) {
312 statements[stmIndex++] = safeInt();
315 int avail = (SIZE-byteIndex) >> 2;
316 int allowed = Math.min(stmLength-stmIndex, avail);
317 for(int index = byteIndex, i=0;i<allowed;i++) {
318 statements[stmIndex++] = ((bytes[index++]&0xff)<<24) | ((bytes[index++]&0xff)<<16) | ((bytes[index++]&0xff)<<8) | ((bytes[index++]&0xff));
320 byteIndex += allowed<<2;
324 // duration = System.nanoTime() - start;
325 // LOGGER.warn("stms in " + 1e-9*duration + "s.");
327 // start = System.nanoTime();
329 int valueLength = safeInt();
330 // LOGGER.warn("values: " + valueLength + " (" + byteIndex + ")");
332 Value[] values = new Value[valueLength];
334 Serializer variantSerializer = Bindings.getSerializerUnchecked(Bindings.VARIANT);
336 dis = new DataInputStream(in);
338 for(int i=0;i<valueLength;i++) {
339 int resource = safeInt();
340 if (!shareValueContext)
342 Variant value = (Variant)variantSerializer
343 .deserialize((DataInput)dis, idcontext);
344 values[i] = new Value(resource, value);
346 //LOGGER.warn("read variant[" + resource + "]: " + value.toString().substring(0, Math.min(100, value.toString().length())));
351 // duration = System.nanoTime() - start;
352 // LOGGER.warn("values in " + 1e-9*duration + "s.");
354 return new TransferableGraph1(resourceCount, ids, statements, values, extensions.map);
358 public static void main(String[] args) {
362 File file = new File("c:/users/antti villberg/desktop/test.apros");
363 TransferableGraphFileReader reader = new TransferableGraphFileReader(file, SIZE);
364 reader = new TransferableGraphFileReader(file);
365 long s = System.nanoTime();
367 long d = System.nanoTime() - s;
368 LOGGER.warn("Duration=" + 1e-9*d + "s.");
371 } catch (Throwable t) {