bundles/org.simantics.graph/src/org/simantics/graph/representation/TransferableGraphFileReader.java

   1 package org.simantics.graph.representation;
   2
   3 import java.io.DataInput;
   4 import java.io.DataInputStream;
   5 import java.io.File;
   6 import java.io.IOException;
   7 import java.io.InputStream;
   8 import java.nio.channels.ReadableByteChannel;
   9 import java.util.ArrayList;
  10 import java.util.List;
  11
  12 import org.simantics.databoard.Bindings;
  13 import org.simantics.databoard.Datatypes;
  14 import org.simantics.databoard.binding.Binding;
  15 import org.simantics.databoard.binding.error.RuntimeDatatypeConstructionException;
  16 import org.simantics.databoard.binding.mutable.Variant;
  17 import org.simantics.databoard.container.DataContainer;
  18 import org.simantics.databoard.container.DataContainers;
  19 import org.simantics.databoard.serialization.RuntimeSerializerConstructionException;
  20 import org.simantics.databoard.serialization.Serializer;
  21 import org.simantics.databoard.type.Datatype;
  22 import org.slf4j.Logger;
  23 import org.slf4j.LoggerFactory;
  24
  25
  26 /**
  27  * It is recommended to use {@link #read(File)} and {@link #read(InputStream)}
  28  * for reading to ensure proper resource handling.
  29  *
  30  * <p>
  31  * It is important to use the correct setting for
  32  * {@link #sharedValueContext(boolean)} which depends on how the TG was
  33  * serialized to begin with. See {@link #DEFAULT_SHARED_VALUE_CONTEXT} for more
  34  * information on this.
  35  */
  36 final public class TransferableGraphFileReader extends ByteFileReader {
  37
  38     /**
  39      * Serializing TransferableGraph1 structures using the default {@link Binding}
  40      * will use shared context for serializing the values Variant array. Thus all TG
  41      * files produced by the graph compiler use a shared value context which means
  42      * this class must be used with {@link #sharedValueContext(boolean)} set to
  43      * true. As an example, <code>true</code> must be used if the corresponding TG
  44      * file is written e.g. like this:
  45      * <pre>
  46      * DataContainers.writeFile(location
  47      *     new DataContainer(format,
  48      *                       version,
  49      *                       metadata,
  50      *                       new Variant(TransferableGraph1.BINDING, tg)));
  51      * </pre>
  52      *
  53      * <p>
  54      * On the other hand, any TG files serialized using more optimized methods like
  55      * <code>ModelTransferableGraphSource</code> do not use a shared value context
  56      * when writing the file. This means those files cannot be read safely using
  57      * standard {@link Binding} at all, and when using this class,
  58      * {@link #sharedValueContext(boolean)} must be set to false to prevent the
  59      * import from corrupting datatype values because the referable parts of
  60      * datatypes may get bound to the wrong existing types if the data is read using
  61      * shared context.
  62      *
  63      * <p>
  64      * <code>true</code> is the default setting.
  65      */
  66     public static final boolean DEFAULT_SHARED_VALUE_CONTEXT = true;
  67
  68     private static final Logger LOGGER = LoggerFactory.getLogger(TransferableGraphFileReader.class);
  69
  70         InputStream in = new InputStream() {
  71
  72         @Override
  73         public int read() throws IOException {
  74             return getByte();
  75         }
  76
  77         @Override
  78         public int read(byte[] b) throws IOException {
  79             // FIXME not correctly implemented
  80             System.arraycopy(safeBytes(b.length), 0, b, 0, b.length);
  81             return b.length;
  82         }
  83
  84         @Override
  85         public int read(byte[] b, int off, int len) throws IOException {
  86             // FIXME not correctly implemented
  87             System.arraycopy(safeBytes(len), 0, b, off, len);
  88             return len;
  89         }
  90
  91         };
  92
  93         private static boolean init = true;
  94
  95         final private static int SIZE = 1<<18;
  96         final private static int HEADER = headerSize();
  97         final private int header;
  98         private boolean shareValueContext = true;
  99
 100         /**
 101          * Reads a {@link DataContainer} containing a {@link TransferableGraph1}
 102          * structure from the specified {@link File}.
 103          *
 104          * @param file the file to read from
 105          * @return the TG contained by the file
 106          * @throws IOException
 107          */
 108         public static TransferableGraph1 read(File file) throws IOException {
 109                 return read(file, DEFAULT_SHARED_VALUE_CONTEXT);
 110         }
 111
 112         /**
 113          * Reads a {@link DataContainer} containing a {@link TransferableGraph1}
 114          * structure from the specified InputStream. Note that this implementation does
 115          * not close the specified <code>input</code> stream, it is expected to be
 116          * closed by the caller.
 117          *
 118          * @param input the input stream to read from
 119          * @return the TG contained by the stream
 120          * @throws IOException
 121          */
 122         public static TransferableGraph1 read(InputStream input) throws IOException {
 123                 return read(input, DEFAULT_SHARED_VALUE_CONTEXT);
 124         }
 125
 126         /**
 127          * Reads a {@link DataContainer} containing a {@link TransferableGraph1}
 128          * structure from the specified {@link File}.
 129          *
 130          * @param file the file to read from
 131          * @return the TG contained by the file
 132          * @throws IOException
 133          */
 134         public static TransferableGraph1 read(File file, boolean sharedValueContext) throws IOException {
 135                 try (TransferableGraphFileReader reader = new TransferableGraphFileReader(file)) {
 136                         return reader.sharedValueContext(sharedValueContext).readTG();
 137                 }
 138         }
 139
 140         /**
 141          * Reads a {@link DataContainer} containing a {@link TransferableGraph1}
 142          * structure from the specified InputStream. Note that this implementation does
 143          * not close the specified <code>input</code> stream, it is expected to be
 144          * closed by the caller.
 145          *
 146          * @param input the input stream to read from
 147          * @return the TG contained by the stream
 148          * @throws IOException
 149          */
 150         public static TransferableGraph1 read(InputStream input, boolean sharedValueContext) throws IOException {
 151                 try (TransferableGraphFileReader reader = new TransferableGraphFileReader(input)) {
 152                         return reader.sharedValueContext(sharedValueContext).readTG();
 153                 }
 154         }
 155
 156         public TransferableGraphFileReader(File file) throws IOException {
 157                 super(file, SIZE);
 158                 if(init) {
 159                         init=false;
 160                         TransferableGraphFileReader r = new TransferableGraphFileReader(file, 0);
 161                         for(int i=0;i<40000;i++) r.readTG();
 162                 }
 163                 this.header = HEADER;
 164         }
 165
 166         public TransferableGraphFileReader(InputStream stream) throws IOException {
 167                 super(null, new InputChannel(stream), SIZE);
 168                 if(init) {
 169                         init=false;
 170                         TransferableGraphFileReader r = new TransferableGraphFileReader(stream, 0);
 171                         for(int i=0;i<40000;i++) r.readTG();
 172                 }
 173                 this.header = 0;
 174         }
 175
 176         public TransferableGraphFileReader(ReadableByteChannel channel) throws IOException {
 177                 super(null, channel, SIZE);
 178                 if(init) {
 179                         init=false;
 180                         TransferableGraphFileReader r = new TransferableGraphFileReader(channel, 0);
 181                         for(int i=0;i<40000;i++) r.readTG();
 182                 }
 183                 this.header = 0;
 184         }
 185
 186         public TransferableGraphFileReader(ReadableByteChannel channel, int size) throws IOException {
 187                 super(null, channel, SIZE);
 188                 this.header = 0;
 189         }
 190
 191         public TransferableGraphFileReader(InputStream stream, int size) throws IOException {
 192                 super(null, new InputChannel(stream), size);
 193                 this.header = 0;
 194         }
 195
 196         public TransferableGraphFileReader(File file, int size) throws IOException {
 197                 super(file, size);
 198                 this.header = HEADER;
 199         }
 200
 201         public TransferableGraphFileReader sharedValueContext(boolean share) {
 202                 this.shareValueContext = share;
 203                 return this;
 204         }
 205
 206         private static int headerSize() {
 207                 try {
 208                         return Bindings.getSerializerUnchecked(Datatype.class).serialize(Datatypes.getDatatypeUnchecked(TransferableGraph1.class)).length;
 209                 } catch (RuntimeSerializerConstructionException e) {
 210                         throw new Error("Failed to determine TransferableGraph1 header size. ", e);
 211                 } catch (RuntimeDatatypeConstructionException e) {
 212                         throw new Error("Failed to determine TransferableGraph1 header size. ", e);
 213                 } catch (IOException e) {
 214                         throw new Error("Failed to determine TransferableGraph1 header size. ", e);
 215                 }
 216         }
 217
 218         public TransferableGraph1 readTG() throws IOException {
 219
 220                 if(getSize() == 0) return null;
 221
 222 //              long start = System.nanoTime();
 223
 224                 final byte[] bytes = getBytes();
 225
 226 //              byteIndex = header;
 227
 228                 DataInputStream dis = new DataInputStream(in);
 229
 230                 // Header
 231                 DataContainers.readHeader(dis);
 232
 233                 // Content variant data type
 234                 Bindings.getSerializerUnchecked(Datatype.class).deserialize((DataInput)dis);
 235
 236                 int resourceCount = safeInt();
 237
 238                 List<Object> idcontext = new ArrayList<>();
 239                 dis = new DataInputStream(in);
 240                 Extensions extensions = (Extensions)Bindings.getSerializerUnchecked(Extensions.class).deserialize((DataInput)dis, idcontext);
 241
 242                 int identities = safeInt();
 243                 Identity[] ids = new Identity[identities];
 244
 245 //              LOGGER.warn("rc: " + resourceCount);
 246 //              LOGGER.warn("ids: " + identities);
 247
 248 //              long duration = System.nanoTime() - start;
 249 //              LOGGER.warn("start in " + 1e-9*duration + "s.");
 250 //              start = System.nanoTime();
 251
 252                 for(int i=0;i<identities;i++) {
 253                         int rid = safeInt();
 254                         byte type = bytes[byteIndex++];
 255                         // External
 256                         if(type == 1) {
 257
 258                                 int parent = safeInt();
 259                                 int nameLen = getDynamicUInt32();
 260
 261                                 if(byteIndex+nameLen < SIZE) {
 262                                         ids[i] = new Identity(rid, new External(parent, utf(bytes, byteIndex, byteIndex + nameLen)));
 263                                         byteIndex += nameLen;
 264                                 } else {
 265                                         ids[i] = new Identity(rid, new External(parent, utf(safeBytes(nameLen), 0, nameLen)));
 266                                 }
 267
 268                         }
 269                         // Internal
 270                         else if(type == 3) {
 271
 272                                 int parent = safeInt();
 273                                 int nameLen = getDynamicUInt32();
 274                                 if(byteIndex+nameLen < SIZE) {
 275                                         ids[i] = new Identity(rid, new Internal(parent, utf(bytes, byteIndex, byteIndex + nameLen)));
 276                                         byteIndex += nameLen;
 277                                 } else {
 278                                         ids[i] = new Identity(rid, new Internal(parent, utf(safeBytes(nameLen), 0, nameLen)));
 279                                 }
 280
 281                         }
 282                         // Root
 283                         else if(type == 0) {
 284                                 int nameLen = getDynamicUInt32();
 285                                 String name = utf(safeBytes(nameLen), 0, nameLen);
 286                                 int nameLen2 = getDynamicUInt32();
 287                                 String rType = utf(safeBytes(nameLen2), 0, nameLen2);
 288                                 ids[i] = new Identity(rid, new Root(name, rType));
 289
 290                         } else if(type == 2) {
 291                                 throw new UnsupportedOperationException();
 292                         } else {
 293                                 throw new IllegalStateException();
 294                         }
 295
 296                 }
 297
 298 //              for(Identity id : ids) LOGGER.warn("id: " + id);
 299
 300
 301 //              duration = System.nanoTime() - start;
 302 //              LOGGER.warn("ids in " + 1e-9*duration + "s.");
 303 //              start = System.nanoTime();
 304
 305                 int stmLength = safeInt();
 306 //              LOGGER.warn("statements: " + stmLength + " (" + byteIndex + ")");
 307
 308                 int[] statements = new int[stmLength];
 309
 310                 for(int stmIndex=0;stmIndex<stmLength;) {
 311
 312                         statements[stmIndex++] = safeInt();
 313
 314                         // Cached bytes
 315                         int avail = (SIZE-byteIndex) >> 2;
 316                         int allowed = Math.min(stmLength-stmIndex, avail);
 317                         for(int index = byteIndex, i=0;i<allowed;i++) {
 318                                 statements[stmIndex++] = ((bytes[index++]&0xff)<<24) | ((bytes[index++]&0xff)<<16) | ((bytes[index++]&0xff)<<8) | ((bytes[index++]&0xff));
 319                         }
 320                         byteIndex += allowed<<2;
 321
 322                 }
 323
 324 //              duration = System.nanoTime() - start;
 325 //              LOGGER.warn("stms in " + 1e-9*duration + "s.");
 326 //
 327 //              start = System.nanoTime();
 328
 329                 int valueLength = safeInt();
 330 //              LOGGER.warn("values: " + valueLength + " (" + byteIndex + ")");
 331
 332                 Value[] values = new Value[valueLength];
 333
 334                 Serializer variantSerializer = Bindings.getSerializerUnchecked(Bindings.VARIANT);
 335
 336                 dis = new DataInputStream(in);
 337
 338                 for(int i=0;i<valueLength;i++) {
 339                         int resource = safeInt();
 340                         if (!shareValueContext)
 341                                 idcontext.clear();
 342                         Variant value = (Variant)variantSerializer
 343                                 .deserialize((DataInput)dis, idcontext);
 344                         values[i] = new Value(resource, value);
 345
 346                         //LOGGER.warn("read variant[" + resource + "]: " + value.toString().substring(0, Math.min(100, value.toString().length())));
 347
 348                 }
 349
 350
 351 //              duration = System.nanoTime() - start;
 352 //              LOGGER.warn("values in " + 1e-9*duration + "s.");
 353
 354                 return new TransferableGraph1(resourceCount, ids, statements, values, extensions.map);
 355
 356         }
 357
 358         public static void main(String[] args) {
 359
 360                 try {
 361
 362                         File file = new File("c:/users/antti villberg/desktop/test.apros");
 363                         TransferableGraphFileReader reader = new TransferableGraphFileReader(file, SIZE);
 364                         reader = new TransferableGraphFileReader(file);
 365                         long s = System.nanoTime();
 366                         reader.readTG();
 367                         long d = System.nanoTime() - s;
 368                         LOGGER.warn("Duration=" + 1e-9*d + "s.");
 369
 370
 371                 } catch (Throwable t) {
 372                         t.printStackTrace();
 373                 }
 374
 375         }
 376
 377 }