Finalizing improve startup time for fresh or rollback'd session 29/1629/5
authorTuukka Lehtonen <tuukka.lehtonen@semantum.fi>
Thu, 5 Apr 2018 13:33:58 +0000 (16:33 +0300)
committerTuukka Lehtonen <tuukka.lehtonen@semantum.fi>
Thu, 5 Apr 2018 13:33:58 +0000 (16:33 +0300)
Finally fixed the long-sought InputStream reading bug from
TransferableGraphFileReader.InputChannel.read(ByteBuffer).

The ByteFileReader implementation expected the provided
ReadableByteChannel to always provide a full buffer of data unless EOF
was encountered and the InputChannel.read implementation was not taking
care of that.

Also added simpler utilities for reading TG's from files/InputStreams
directly using static TransferableGraphFileReader.read(File/InputStream)
methods.

PlatformUtil.readTG can now finally use TransferableGraphFileReader for
maximal performance.

refs #7835

Change-Id: I170986bf271a409f9205fad808ae029becc1d3a4

bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/migration/MigrationStateImpl.java
bundles/org.simantics.db.testing/src/org/simantics/db/testing/common/TestingUtils.java
bundles/org.simantics.graph/src/org/simantics/graph/representation/TransferableGraph1Serializer.java
bundles/org.simantics.graph/src/org/simantics/graph/representation/TransferableGraphFileReader.java
bundles/org.simantics.project/src/org/simantics/project/management/PlatformUtil.java

index d7f6b4e95497fe0a1ac0991cf27c38720f316681..f50bae78aa2e8c8ebf702efd86c472c436ce2bea 100644 (file)
@@ -106,12 +106,10 @@ public class MigrationStateImpl implements MigrationState {
                    return (T)tg;
                }
                
-            TransferableGraphFileReader reader = null;
             try {
                 File modelFile = getProperty(MigrationStateKeys.MODEL_FILE);
-                reader = new TransferableGraphFileReader(modelFile);
                 TimeLogger.log(MigrationStateImpl.class, "reading TG into memory from " + modelFile);
-                TransferableGraph1 tg = reader.readTG();
+                TransferableGraph1 tg = TransferableGraphFileReader.read(modelFile);
                 TimeLogger.log(MigrationStateImpl.class, "read TG into memory from " + modelFile);
                 setProperty(MigrationStateKeys.CURRENT_TG, tg);
                 return (T)tg;
@@ -119,8 +117,6 @@ public class MigrationStateImpl implements MigrationState {
                 throw e;
             } catch (Throwable t) {
                 throw new DatabaseException(t);
-            } finally {
-                uncheckedClose(reader);
             }
                
         } else if (MigrationStateKeys.CURRENT_TGS.equals(key)) {
index 819be78076f965bc01575b8b871ad836d1bfdf2d..8f3a8463cd7b74469673d5dde762dd2de0bcbccd 100644 (file)
@@ -25,9 +25,7 @@ public class TestingUtils {
 
        public static Resource importModel(final Resource toLocation, File fromFile, String withName) throws Exception {
                
-        TransferableGraphFileReader importer = new TransferableGraphFileReader(fromFile);
-        TransferableGraph1 tg = importer.readTG();
-        importer.close();
+        TransferableGraph1 tg = TransferableGraphFileReader.read(fromFile);
 
         final DefaultPasteImportAdvisor advisor = new DefaultPasteImportAdvisor(toLocation) {
             @Override
index 770d6c6cab10b5844d7f9485e7476cac1e97cb4e..0baa40231e8a636ffa474196ba4794d9787fc30a 100644 (file)
@@ -108,21 +108,11 @@ public class TransferableGraph1Serializer extends Serializer {
        
        @Override
        public Object deserialize(File file) throws IOException {
-               TransferableGraphFileReader reader = new TransferableGraphFileReader(file);
-               try {
-                       return reader.readTG();
-               } finally {
-                       reader.close();
-               }
+               return TransferableGraphFileReader.read(file);
        }
        
        public Object deserialize(InputStream in) throws IOException {
-               TransferableGraphFileReader reader = new TransferableGraphFileReader(in);
-               try {
-                       return reader.readTG();
-               } finally {
-                       reader.close();
-               }
+               return TransferableGraphFileReader.read(in);
        }
        
        @Override
index 077eb6617a1703f3cbffa913633e2b4c738f95ab..56a112e4d02e7a7e768375618bf687bab437b3ef 100644 (file)
@@ -14,6 +14,7 @@ import org.simantics.databoard.Bindings;
 import org.simantics.databoard.Datatypes;
 import org.simantics.databoard.binding.error.RuntimeDatatypeConstructionException;
 import org.simantics.databoard.binding.mutable.Variant;
+import org.simantics.databoard.container.DataContainer;
 import org.simantics.databoard.container.DataContainers;
 import org.simantics.databoard.serialization.RuntimeSerializerConstructionException;
 import org.simantics.databoard.serialization.Serializer;
@@ -22,6 +23,10 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 
+/**
+ * It is recommended to use {@link #read(File)} and {@link #read(InputStream)}
+ * for reading to ensure proper resource handling.
+ */
 final public class TransferableGraphFileReader extends ByteFileReader {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(TransferableGraphFileReader.class);
@@ -53,11 +58,11 @@ final public class TransferableGraphFileReader extends ByteFileReader {
        final static class InputChannel implements ReadableByteChannel {
 
                final private InputStream stream;
-               
+
                public InputChannel(InputStream stream) {
                        this.stream = stream;
                }
-               
+
                @Override
                public boolean isOpen() {
                        return true;
@@ -65,17 +70,25 @@ final public class TransferableGraphFileReader extends ByteFileReader {
 
                @Override
                public void close() throws IOException {
+                       // NOTE: it is an intentional choice not to close the underlying stream here
+                       // InputStreams given directly to TransferableGraphFileReader are expected to
+                       // be closed outside of this implementation.
                }
 
                @Override
                public int read(ByteBuffer dst) throws IOException {
-                       int pos = dst.position();
+                       int nRead;
+                       int size = 0;
+                       int position = dst.position();
                        int limit = dst.limit();
-                       int i=stream.read(dst.array(), pos, limit-pos);
-                       //LOGGER.warn("Read " + i + " (expected " + dst.array().length + ")");
-                       return i;
+                       // The users of this channel expect that the data is fully read at this point
+                       while ((nRead = stream.read(dst.array(), position, limit - position)) != -1 && limit - position > 0) {
+                               size += nRead;
+                               position += nRead;
+                       }
+                       return size;
                }
-               
+
        }
        
        private static boolean init = true;
@@ -83,7 +96,37 @@ final public class TransferableGraphFileReader extends ByteFileReader {
        final private static int SIZE = 1<<18;
        final private static int HEADER = headerSize();
        final private int header;
-       
+
+       /**
+        * Reads a {@link DataContainer} containing a {@link TransferableGraph1}
+        * structure from the specified {@link File}.
+        * 
+        * @param file the file to read from
+        * @return the TG contained by the file
+        * @throws IOException
+        */
+       public static TransferableGraph1 read(File file) throws IOException {
+               try (TransferableGraphFileReader reader = new TransferableGraphFileReader(file)) {
+                       return reader.readTG();
+               }
+       }
+
+       /**
+        * Reads a {@link DataContainer} containing a {@link TransferableGraph1}
+        * structure from the specified InputStream. Note that this implementation does
+        * not close the specified <code>input</code> stream, it is expected to be
+        * closed by the caller.
+        * 
+        * @param input the input stream to read from
+        * @return the TG contained by the stream
+        * @throws IOException
+        */
+       public static TransferableGraph1 read(InputStream input) throws IOException {
+               try (TransferableGraphFileReader reader = new TransferableGraphFileReader(input)) {
+                       return reader.readTG();
+               }
+       }
+
        public TransferableGraphFileReader(File file) throws IOException {
                super(file, SIZE);
                if(init) {
@@ -263,7 +306,7 @@ final public class TransferableGraphFileReader extends ByteFileReader {
                
                for(int i=0;i<valueLength;i++) {
                        int resource = safeInt();
-                       idcontext.clear();
+                       //idcontext.clear();
                        Variant value = (Variant)variantSerializer
                                .deserialize((DataInput)dis, idcontext);
                        values[i] = new Value(resource, value);
index a8916251e07c738413866bde501a42b3e7c3eec6..344b2d6cf94a6d43c446ea07bb073ed241fd4507 100644 (file)
@@ -27,7 +27,6 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Enumeration;
-import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Objects;
 import java.util.concurrent.atomic.AtomicReference;
@@ -45,22 +44,20 @@ import org.eclipse.equinox.p2.metadata.VersionedId;
 import org.osgi.framework.Bundle;
 import org.simantics.databoard.Bindings;
 import org.simantics.databoard.adapter.AdaptException;
-import org.simantics.databoard.binding.Binding;
 import org.simantics.databoard.binding.mutable.Variant;
 import org.simantics.databoard.container.DataContainer;
 import org.simantics.databoard.container.DataContainers;
-import org.simantics.databoard.container.FormatHandler;
 import org.simantics.graph.compiler.CompilationResult;
 import org.simantics.graph.compiler.GraphCompiler;
 import org.simantics.graph.compiler.GraphCompilerPreferences;
 import org.simantics.graph.compiler.ValidationMode;
 import org.simantics.graph.representation.Extensions;
 import org.simantics.graph.representation.TransferableGraph1;
+import org.simantics.graph.representation.TransferableGraphFileReader;
 import org.simantics.ltk.FileSource;
 import org.simantics.ltk.ISource;
 import org.simantics.ltk.Problem;
 import org.simantics.scl.reflection.OntologyVersions;
-import org.simantics.utils.datastructures.ArrayMap;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -420,40 +417,10 @@ public class PlatformUtil {
                        throw new IOException("Problem loading graph.tg from bundle " + bundle.getSymbolicName(), e);
                }
        }
-       
-       private static FormatHandler<TransferableGraph1> FORMAT_HANDLER = new FormatHandler<TransferableGraph1>() {
-        @Override
-        public Binding getBinding() {
-            return TransferableGraph1.BINDING;
-        }
-        @Override
-        public TransferableGraph1 process(DataContainer container) throws Exception {
-            return (TransferableGraph1) container.content.getValue(TransferableGraph1.BINDING);
-        }
-    };
-
-       @SuppressWarnings("unchecked")
-       private static Map<String, FormatHandler<TransferableGraph1>> handlers = ArrayMap.make(
-                       new String[] {
-                                       "graph:1",
-                                       "sharedLibrary:1"
-                       },
-                       FORMAT_HANDLER,
-                       FORMAT_HANDLER);
-
-       private static TransferableGraph1 readTG(InputStream is) throws Exception {
-               // For an unknown reason this is totally broken when running the TestSCLOsgi
-               // in the SDK Tycho build. It returns incomplete results because the
-               // ReadableByteChannel used by ByteFileReader starts returning 0 unexpectedly.
-//             try (TransferableGraphFileReader reader = new TransferableGraphFileReader(is)) {
-//                     return reader.readTG();
-//             }
-               return DataContainers.readFile(new DataInputStream(is), handlers);
-       }
 
        private static TransferableGraph1 readTG(URL url) throws Exception {
                try (InputStream is = url.openStream()) {
-                       return readTG(is);
+                       return TransferableGraphFileReader.read(is);
                }
        }