bundles/org.simantics.graph/src/org/simantics/graph/representation/TransferableGraphFileReader.java

   1 package org.simantics.graph.representation;
   2
   3 import java.io.DataInput;
   4 import java.io.DataInputStream;
   5 import java.io.File;
   6 import java.io.IOException;
   7 import java.io.InputStream;
   8 import java.nio.ByteBuffer;
   9 import java.nio.channels.ReadableByteChannel;
  10 import java.util.ArrayList;
  11 import java.util.List;
  12
  13 import org.simantics.databoard.Bindings;
  14 import org.simantics.databoard.Datatypes;
  15 import org.simantics.databoard.binding.error.RuntimeDatatypeConstructionException;
  16 import org.simantics.databoard.binding.mutable.Variant;
  17 import org.simantics.databoard.container.DataContainer;
  18 import org.simantics.databoard.container.DataContainers;
  19 import org.simantics.databoard.serialization.RuntimeSerializerConstructionException;
  20 import org.simantics.databoard.serialization.Serializer;
  21 import org.simantics.databoard.type.Datatype;
  22 import org.slf4j.Logger;
  23 import org.slf4j.LoggerFactory;
  24
  25
  26 /**
  27  * It is recommended to use {@link #read(File)} and {@link #read(InputStream)}
  28  * for reading to ensure proper resource handling.
  29  */
  30 final public class TransferableGraphFileReader extends ByteFileReader {
  31
  32     private static final Logger LOGGER = LoggerFactory.getLogger(TransferableGraphFileReader.class);
  33
  34         InputStream in = new InputStream() {
  35
  36         @Override
  37         public int read() throws IOException {
  38             return getByte();
  39         }
  40
  41         @Override
  42         public int read(byte[] b) throws IOException {
  43             // FIXME not correctly implemented
  44             System.arraycopy(safeBytes(b.length), 0, b, 0, b.length);
  45             return b.length;
  46         }
  47
  48         @Override
  49         public int read(byte[] b, int off, int len) throws IOException {
  50             // FIXME not correctly implemented
  51             System.arraycopy(safeBytes(len), 0, b, off, len);
  52             return len;
  53         }
  54
  55         };
  56
  57
  58         final static class InputChannel implements ReadableByteChannel {
  59
  60                 final private InputStream stream;
  61
  62                 public InputChannel(InputStream stream) {
  63                         this.stream = stream;
  64                 }
  65
  66                 @Override
  67                 public boolean isOpen() {
  68                         return true;
  69                 }
  70
  71                 @Override
  72                 public void close() throws IOException {
  73                         // NOTE: it is an intentional choice not to close the underlying stream here
  74                         // InputStreams given directly to TransferableGraphFileReader are expected to
  75                         // be closed outside of this implementation.
  76                 }
  77
  78                 @Override
  79                 public int read(ByteBuffer dst) throws IOException {
  80                         int nRead;
  81                         int size = 0;
  82                         int position = dst.position();
  83                         int limit = dst.limit();
  84                         // The users of this channel expect that the data is fully read at this point
  85                         while ((nRead = stream.read(dst.array(), position, limit - position)) != -1 && limit - position > 0) {
  86                                 size += nRead;
  87                                 position += nRead;
  88                         }
  89                         return size;
  90                 }
  91
  92         }
  93
  94         private static boolean init = true;
  95
  96         final private static int SIZE = 1<<18;
  97         final private static int HEADER = headerSize();
  98         final private int header;
  99
 100         /**
 101          * Reads a {@link DataContainer} containing a {@link TransferableGraph1}
 102          * structure from the specified {@link File}.
 103          *
 104          * @param file the file to read from
 105          * @return the TG contained by the file
 106          * @throws IOException
 107          */
 108         public static TransferableGraph1 read(File file) throws IOException {
 109                 try (TransferableGraphFileReader reader = new TransferableGraphFileReader(file)) {
 110                         return reader.readTG();
 111                 }
 112         }
 113
 114         /**
 115          * Reads a {@link DataContainer} containing a {@link TransferableGraph1}
 116          * structure from the specified InputStream. Note that this implementation does
 117          * not close the specified <code>input</code> stream, it is expected to be
 118          * closed by the caller.
 119          *
 120          * @param input the input stream to read from
 121          * @return the TG contained by the stream
 122          * @throws IOException
 123          */
 124         public static TransferableGraph1 read(InputStream input) throws IOException {
 125                 try (TransferableGraphFileReader reader = new TransferableGraphFileReader(input)) {
 126                         return reader.readTG();
 127                 }
 128         }
 129
 130         public TransferableGraphFileReader(File file) throws IOException {
 131                 super(file, SIZE);
 132                 if(init) {
 133                         init=false;
 134                         TransferableGraphFileReader r = new TransferableGraphFileReader(file, 0);
 135                         for(int i=0;i<40000;i++) r.readTG();
 136                 }
 137                 this.header = HEADER;
 138         }
 139
 140         public TransferableGraphFileReader(InputStream stream) throws IOException {
 141                 super(null, new InputChannel(stream), SIZE);
 142                 if(init) {
 143                         init=false;
 144                         TransferableGraphFileReader r = new TransferableGraphFileReader(stream, 0);
 145                         for(int i=0;i<40000;i++) r.readTG();
 146                 }
 147                 this.header = 0;
 148         }
 149
 150         public TransferableGraphFileReader(ReadableByteChannel channel) throws IOException {
 151                 super(null, channel, SIZE);
 152                 if(init) {
 153                         init=false;
 154                         TransferableGraphFileReader r = new TransferableGraphFileReader(channel, 0);
 155                         for(int i=0;i<40000;i++) r.readTG();
 156                 }
 157                 this.header = 0;
 158         }
 159
 160         public TransferableGraphFileReader(ReadableByteChannel channel, int size) throws IOException {
 161                 super(null, channel, SIZE);
 162                 this.header = 0;
 163         }
 164
 165         public TransferableGraphFileReader(InputStream stream, int size) throws IOException {
 166                 super(null, new InputChannel(stream), size);
 167                 this.header = 0;
 168         }
 169
 170         public TransferableGraphFileReader(File file, int size) throws IOException {
 171                 super(file, size);
 172                 this.header = HEADER;
 173         }
 174
 175         private static int headerSize() {
 176                 try {
 177                         return Bindings.getSerializerUnchecked(Datatype.class).serialize(Datatypes.getDatatypeUnchecked(TransferableGraph1.class)).length;
 178                 } catch (RuntimeSerializerConstructionException e) {
 179                         throw new Error("Failed to determine TransferableGraph1 header size. ", e);
 180                 } catch (RuntimeDatatypeConstructionException e) {
 181                         throw new Error("Failed to determine TransferableGraph1 header size. ", e);
 182                 } catch (IOException e) {
 183                         throw new Error("Failed to determine TransferableGraph1 header size. ", e);
 184                 }
 185         }
 186
 187         public TransferableGraph1 readTG() throws IOException {
 188
 189                 if(getSize() == 0) return null;
 190
 191 //              long start = System.nanoTime();
 192
 193                 final byte[] bytes = getBytes();
 194
 195 //              byteIndex = header;
 196
 197                 DataInputStream dis = new DataInputStream(in);
 198
 199                 // Header
 200                 DataContainers.readHeader(dis);
 201
 202                 // Content variant data type
 203                 Bindings.getSerializerUnchecked(Datatype.class).deserialize((DataInput)dis);
 204
 205                 int resourceCount = safeInt();
 206
 207                 List<Object> idcontext = new ArrayList<>();
 208                 dis = new DataInputStream(in);
 209                 Extensions extensions = (Extensions)Bindings.getSerializerUnchecked(Extensions.class).deserialize((DataInput)dis, idcontext);
 210
 211                 int identities = safeInt();
 212                 Identity[] ids = new Identity[identities];
 213
 214 //              LOGGER.warn("rc: " + resourceCount);
 215 //              LOGGER.warn("ids: " + identities);
 216
 217 //              long duration = System.nanoTime() - start;
 218 //              LOGGER.warn("start in " + 1e-9*duration + "s.");
 219 //              start = System.nanoTime();
 220
 221                 for(int i=0;i<identities;i++) {
 222                         int rid = safeInt();
 223                         byte type = bytes[byteIndex++];
 224                         // External
 225                         if(type == 1) {
 226
 227                                 int parent = safeInt();
 228                                 int nameLen = getDynamicUInt32();
 229
 230                                 if(byteIndex+nameLen < SIZE) {
 231                                         ids[i] = new Identity(rid, new External(parent, utf(bytes, byteIndex, byteIndex + nameLen)));
 232                                         byteIndex += nameLen;
 233                                 } else {
 234                                         ids[i] = new Identity(rid, new External(parent, utf(safeBytes(nameLen), 0, nameLen)));
 235                                 }
 236
 237                         }
 238                         // Internal
 239                         else if(type == 3) {
 240
 241                                 int parent = safeInt();
 242                                 int nameLen = getDynamicUInt32();
 243                                 if(byteIndex+nameLen < SIZE) {
 244                                         ids[i] = new Identity(rid, new Internal(parent, utf(bytes, byteIndex, byteIndex + nameLen)));
 245                                         byteIndex += nameLen;
 246                                 } else {
 247                                         ids[i] = new Identity(rid, new Internal(parent, utf(safeBytes(nameLen), 0, nameLen)));
 248                                 }
 249
 250                         }
 251                         // Root
 252                         else if(type == 0) {
 253                                 int nameLen = getDynamicUInt32();
 254                                 String name = utf(safeBytes(nameLen), 0, nameLen);
 255                                 int nameLen2 = getDynamicUInt32();
 256                                 String rType = utf(safeBytes(nameLen2), 0, nameLen2);
 257                                 ids[i] = new Identity(rid, new Root(name, rType));
 258
 259                         } else if(type == 2) {
 260                                 throw new UnsupportedOperationException();
 261                         } else {
 262                                 throw new IllegalStateException();
 263                         }
 264
 265                 }
 266
 267 //              for(Identity id : ids) LOGGER.warn("id: " + id);
 268
 269
 270 //              duration = System.nanoTime() - start;
 271 //              LOGGER.warn("ids in " + 1e-9*duration + "s.");
 272 //              start = System.nanoTime();
 273
 274                 int stmLength = safeInt();
 275 //              LOGGER.warn("statements: " + stmLength + " (" + byteIndex + ")");
 276
 277                 int[] statements = new int[stmLength];
 278
 279                 for(int stmIndex=0;stmIndex<stmLength;) {
 280
 281                         statements[stmIndex++] = safeInt();
 282
 283                         // Cached bytes
 284                         int avail = (SIZE-byteIndex) >> 2;
 285                         int allowed = Math.min(stmLength-stmIndex, avail);
 286                         for(int index = byteIndex, i=0;i<allowed;i++) {
 287                                 statements[stmIndex++] = ((bytes[index++]&0xff)<<24) | ((bytes[index++]&0xff)<<16) | ((bytes[index++]&0xff)<<8) | ((bytes[index++]&0xff));
 288                         }
 289                         byteIndex += allowed<<2;
 290
 291                 }
 292
 293 //              duration = System.nanoTime() - start;
 294 //              LOGGER.warn("stms in " + 1e-9*duration + "s.");
 295 //
 296 //              start = System.nanoTime();
 297
 298                 int valueLength = safeInt();
 299 //              LOGGER.warn("values: " + valueLength + " (" + byteIndex + ")");
 300
 301                 Value[] values = new Value[valueLength];
 302
 303                 Serializer variantSerializer = Bindings.getSerializerUnchecked(Bindings.VARIANT);
 304
 305                 dis = new DataInputStream(in);
 306
 307                 for(int i=0;i<valueLength;i++) {
 308                         int resource = safeInt();
 309                         //idcontext.clear();
 310                         Variant value = (Variant)variantSerializer
 311                                 .deserialize((DataInput)dis, idcontext);
 312                         values[i] = new Value(resource, value);
 313
 314                         //LOGGER.warn("read variant[" + resource + "]: " + value.toString().substring(0, Math.min(100, value.toString().length())));
 315
 316                 }
 317
 318
 319 //              duration = System.nanoTime() - start;
 320 //              LOGGER.warn("values in " + 1e-9*duration + "s.");
 321
 322                 return new TransferableGraph1(resourceCount, ids, statements, values, extensions.map);
 323
 324         }
 325
 326         public static void main(String[] args) {
 327
 328                 try {
 329
 330                         File file = new File("c:/users/antti villberg/desktop/test.apros");
 331                         TransferableGraphFileReader reader = new TransferableGraphFileReader(file, SIZE);
 332                         reader = new TransferableGraphFileReader(file);
 333                         long s = System.nanoTime();
 334                         reader.readTG();
 335                         long d = System.nanoTime() - s;
 336                         LOGGER.warn("Duration=" + 1e-9*d + "s.");
 337
 338
 339                 } catch (Throwable t) {
 340                         t.printStackTrace();
 341                 }
 342
 343         }
 344
 345 }