HistoryUtil API for exporting subscription history data archive file 32/2532/2
authorTuukka Lehtonen <tuukka.lehtonen@semantum.fi>
Fri, 7 Dec 2018 12:18:49 +0000 (14:18 +0200)
committerTuukka Lehtonen <tuukka.lehtonen@semantum.fi>
Fri, 7 Dec 2018 12:21:46 +0000 (14:21 +0200)
Previously exporting was supported only directly into the database.

gitlab #227

Change-Id: I672cc3f5ca20b3b26e9b35d0c3eaf5415d1a3e5f

bundles/org.simantics.simulation/src/org/simantics/simulation/history/HistoryUtil.java

index 27f2bf895bff382b0ae05cf251cd160425249739..39c084cad4dd5f89c711c480cfeaeef983d542da 100644 (file)
@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 2007, 2014 Association for Decentralized Information Management in
+ * Copyright (c) 2007, 2018 Association for Decentralized Information Management in
  * Industry THTH ry.
  * All rights reserved. This program and the accompanying materials
  * are made available under the terms of the Eclipse Public License v1.0
@@ -8,7 +8,7 @@
  *
  * Contributors:
  *     VTT Technical Research Centre of Finland - initial API and implementation
- *     Semantum Oy - archive implementation
+ *     Semantum Oy - archive implementation, gitlab simantics/platform#227
  *******************************************************************************/
 package org.simantics.simulation.history;
 
@@ -16,6 +16,7 @@ import java.io.EOFException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
@@ -41,6 +42,7 @@ import org.simantics.databoard.container.DataContainer;
 import org.simantics.databoard.container.DataContainers;
 import org.simantics.databoard.serialization.Serializer;
 import org.simantics.databoard.util.Bean;
+import org.simantics.databoard.util.binary.BinaryFile;
 import org.simantics.databoard.util.binary.OutputStreamWriteable;
 import org.simantics.databoard.util.binary.RandomAccessBinary;
 import org.simantics.db.ReadGraph;
@@ -81,18 +83,20 @@ public class HistoryUtil {
        private static final boolean DEBUG = false;
        private static final boolean PROFILE = true;
 
-       private enum Compression {
+       public enum Compression {
                FLZ,
                LZ4,
+               NONE,
        }
 
        private static final Compression USED_COMPRESSION = Compression.FLZ;
 
+       private static final String APPLICATION_X_TAR = "application/x-tar";
        private static final String APPLICATION_X_LZ4 = "application/x-lz4";
        private static final String APPLICATION_X_FLZ = "application/x-flz";
        private static final int ARCHIVE_VERSION_1 = 1; // TarArchiveInput/OutputStream
 
-       private static class ArchivedHistory {
+       public static class ArchivedHistory {
                public long totalSampleSize = 0L;
                public TObjectLongMap<Bean> itemSizes = new TObjectLongHashMap<Bean>();
        }
@@ -323,7 +327,7 @@ public class HistoryUtil {
                                        rab.position(0);
                                        rab.skipBytes(4);
 
-                                       ArchivedHistory archive = exportCompressedArchive(history, collectorState, historyResource, timeShift, from, end, rab);
+                                       ArchivedHistory archive = exportCompressedArchive(history, collectorState, timeShift, from, end, rab);
 
                                        rab.position(0L);
                                        rab.writeInt((int)(rab.length() - 4L));
@@ -344,80 +348,56 @@ public class HistoryUtil {
                };
        }
 
-       private static ArchivedHistory exportCompressedArchive(
-                       final HistoryManager history,
-                       final Bean collectorState,
-                       final Resource historyResource,
-                       final Double timeShift, 
-                       final double from,
-                       final double end,
+       public static ArchivedHistory exportCompressedArchive(
+                       HistoryManager history,
+                       Bean collectorState,
+                       Double timeShift, 
+                       double from,
+                       double end,
                        RandomAccessBinary rab)
                                        throws IOException, HistoryException
        {
-               switch (USED_COMPRESSION) {
-               case FLZ:
-                       return exportArchiveFLZ(history, collectorState, historyResource, timeShift, from, end, rab);
-               case LZ4:
-                       return exportArchiveLZ4(history, collectorState, historyResource, timeShift, from, end, rab);
-               default:
-                       throw new UnsupportedOperationException("Unsupported compression: " + USED_COMPRESSION);
-               }
+               return exportCompressedArchive(history, collectorState, timeShift, from, end, USED_COMPRESSION, rab);
        }
 
-       private static ArchivedHistory exportArchiveLZ4(
-                       final HistoryManager history,
-                       final Bean collectorState,
-                       final Resource historyResource,
-                       final Double timeShift, 
-                       final double from,
-                       final double end,
+       public static ArchivedHistory exportCompressedArchive(
+                       HistoryManager history,
+                       Bean collectorState,
+                       Double timeShift, 
+                       double from,
+                       double end,
+                       Compression compression,
                        RandomAccessBinary rab)
                                        throws IOException, HistoryException
        {
-               OutputStream closeable = null;
-               try {
-                       DataContainers.writeHeader(rab, new DataContainer(APPLICATION_X_LZ4, ARCHIVE_VERSION_1, null));
-                       closeable = new RABOutputStream( rab );
-                       closeable = new LZ4BlockOutputStream( closeable );
-
-                       ArchivedHistory archive = exportArchive( closeable, history, collectorState, timeShift, from, end );
-
-                       closeable.flush();
-                       closeable.close();
-                       closeable = null;
-
-                       return archive;
-               } finally {
-                       FileUtils.uncheckedClose(closeable);
+               switch (compression) {
+                       case FLZ:
+                               DataContainers.writeHeader(rab, new DataContainer(APPLICATION_X_FLZ, ARCHIVE_VERSION_1, null));
+                               return exportArchive(history, collectorState, timeShift, from, end, FastLZ.write(new RABOutputStream(rab)));
+                       case LZ4:
+                               DataContainers.writeHeader(rab, new DataContainer(APPLICATION_X_LZ4, ARCHIVE_VERSION_1, null));
+                               return exportArchive(history, collectorState, timeShift, from, end, new LZ4BlockOutputStream(new RABOutputStream(rab)));
+                       case NONE:
+                               DataContainers.writeHeader(rab, new DataContainer(APPLICATION_X_TAR, ARCHIVE_VERSION_1, null));
+                               return exportArchive(history, collectorState, timeShift, from, end, new RABOutputStream(rab));
+                       default:
+                               throw new UnsupportedOperationException("Unsupported compression: " + compression);
                }
        }
 
-       @SuppressWarnings("resource")
-       private static ArchivedHistory exportArchiveFLZ(
-                       final HistoryManager history,
-                       final Bean collectorState,
-                       final Resource historyResource,
-                       final Double timeShift, 
-                       final double from,
-                       final double end,
-                       RandomAccessBinary rab)
+       private static ArchivedHistory exportArchive(
+                       HistoryManager history,
+                       Bean collectorState,
+                       Double timeShift, 
+                       double from,
+                       double end,
+                       OutputStream out)
                                        throws IOException, HistoryException
        {
-               OutputStream closeable = null;
-               try {
-                       DataContainers.writeHeader(rab, new DataContainer(APPLICATION_X_FLZ, ARCHIVE_VERSION_1, null));
-                       closeable = new RABOutputStream( rab );
-                       closeable = FastLZ.write( closeable );
-
-                       ArchivedHistory archive = exportArchive( closeable, history, collectorState, timeShift, from, end );
-
-                       closeable.flush();
-                       closeable.close();
-                       closeable = null;
-
+               try (OutputStream os = out) {
+                       ArchivedHistory archive = exportArchive( os, history, collectorState, timeShift, from, end );
+                       os.flush();
                        return archive;
-               } finally {
-                       FileUtils.uncheckedClose(closeable);
                }
        }
 
@@ -612,7 +592,7 @@ public class HistoryUtil {
                                        try {
                                                if (PROFILE)
                                                        profile(null, "importing history items from archived format to disk workarea");
-                                               importHistoryArchive(graph, historyResource, archive, history, result);
+                                               importHistoryArchive(graph, archive, history, result);
                                        } catch (IOException e) {
                                                throw new DatabaseException(e);
                                        } catch (HistoryException e) {
@@ -671,6 +651,21 @@ public class HistoryUtil {
                }
        }
 
+       /**
+        * @param history
+        * @param path
+        * @return
+        * @throws HistoryException 
+        * @throws IOException 
+        */
+       public static HistoryImportResult importHistoryArchive(HistoryManager history, Path path) throws IOException, HistoryException {
+               HistoryImportResult result = new HistoryImportResult();
+               try (RandomAccessBinary rab = new BinaryFile(path.toFile())) {
+                       importHistoryArchive(history, rab, result);
+                       return result;
+               }
+       }
+
        /**
         * Import all history items from database archive into a history manager.
         * 
@@ -681,37 +676,51 @@ public class HistoryUtil {
         * @return <code>true</code> if successful
         * @throws DatabaseException 
         */
-       @SuppressWarnings("resource")
-       private static boolean importHistoryArchive(ReadGraph graph, Resource historyResource, Resource archive, HistoryManager history, HistoryImportResult result)
-                       throws DatabaseException, IOException, HistoryException
-       {
+       private static boolean importHistoryArchive(ReadGraph graph, Resource archive, HistoryManager history, HistoryImportResult result) throws DatabaseException, IOException, HistoryException {
                RandomAccessBinary rab = graph.getRandomAccessBinary(archive);
+               // This is here because the data is encoded as an L0.ByteArray
+               // in the database which means the first integer describes the
+               // length of the byte array. We skip that length data here.
                rab.position(4);
+               return importHistoryArchive(history, rab, result);
+       }
+
+       /**
+        * Import all history items from database archive into a history manager.
+        * 
+        * @param graph
+        * @param historyResource
+        * @param archive 
+        * @param history
+        * @return <code>true</code> if successful
+        * @throws DatabaseException 
+        */
+       private static boolean importHistoryArchive(HistoryManager history, RandomAccessBinary rab, HistoryImportResult result)
+                       throws IOException, HistoryException
+       {
                DataContainer dc = DataContainers.readHeader(rab);
 
                if (DEBUG)
                        System.out.println("COMPRESSED HISTORY DATA SIZE: " + (rab.length() - rab.position()));
 
-               if (APPLICATION_X_LZ4.equals(dc.format)) {
-                       if (dc.version == ARCHIVE_VERSION_1) {
-                               InputStream in = new RABInputStream(rab);
-                               //in = new BufferedInputStream(in);
-                               in = new LZ4BlockInputStream(in);
-                               return extractHistoryArchiveTar(graph, history, in, result);
-                       }
-               } else if (APPLICATION_X_FLZ.equals(dc.format)) {
-                       if (dc.version == ARCHIVE_VERSION_1) {
-                               InputStream in = null;
-                               try {
-                                       in = new RABInputStream(rab);
-                                       in = FastLZ.read(in);
-                                       return extractHistoryArchiveTar(graph, history, in, result);
-                               } finally {
-                                       FileUtils.uncheckedClose(in);
-                               }
+               if (dc.version == ARCHIVE_VERSION_1) {
+                       if (APPLICATION_X_LZ4.equals(dc.format)) {
+                               return extractHistoryArchiveInputStream(history, new LZ4BlockInputStream(new RABInputStream(rab)), result);
+                       } else if (APPLICATION_X_FLZ.equals(dc.format)) {
+                               return extractHistoryArchiveInputStream(history, FastLZ.read(new RABInputStream(rab)), result);
+                       } else if (APPLICATION_X_TAR.equals(dc.format)) {
+                               return extractHistoryArchiveInputStream(history, new RABInputStream(rab), result);
                        }
+                       throw new UnsupportedOperationException("Unsupported format " + dc.format + " and version " + dc.version);
+               } else {
+                       throw new UnsupportedOperationException("Unsupported history archive version " + dc.version + " with format " + dc.format);
+               }
+       }
+
+       private static boolean extractHistoryArchiveInputStream(HistoryManager history, InputStream in, HistoryImportResult result) throws IOException, HistoryException {
+               try (InputStream _in = in) {
+                       return extractHistoryArchiveTar(history, in, result);
                }
-               throw new UnsupportedOperationException("Unsupported format " + dc.format + " and version " + dc.version);
        }
 
        /**
@@ -723,8 +732,8 @@ public class HistoryUtil {
         * @return <code>true</code> if successful
         * @throws DatabaseException 
         */
-       private static boolean extractHistoryArchiveTar(ReadGraph graph, HistoryManager history, InputStream in, HistoryImportResult result)
-                       throws DatabaseException, IOException, HistoryException
+       private static boolean extractHistoryArchiveTar(HistoryManager history, InputStream in, HistoryImportResult result)
+                       throws IOException, HistoryException
        {
                // tar is not closed on purpose because we do not want to close RandomAccessBinary rab.
                TarArchiveInputStream tar = new TarArchiveInputStream(in);
@@ -784,9 +793,9 @@ public class HistoryUtil {
                                        //component(3):  = Boolean
                                        //component(4):  = Boolean
                                        //component(5):  = Boolean
-                                       //component(devil):  = Boolean
+                                       //component(6):  = Boolean
                                        //component(7):  = Boolean
-                                       //component(music):  = Boolean
+                                       //component(8):  = Boolean
                                        //component(9):  = Boolean
                                        //component(10):  = Boolean
                                        //