From: Tuukka Lehtonen Date: Fri, 7 Dec 2018 12:18:49 +0000 (+0200) Subject: HistoryUtil API for exporting subscription history data archive file X-Git-Tag: v1.43.0~136^2~241 X-Git-Url: https://gerrit.simantics.org/r/gitweb?p=simantics%2Fplatform.git;a=commitdiff_plain;h=662ee6d0e68c386fdbc22300bdadb58d61de5b17 HistoryUtil API for exporting subscription history data archive file Previously exporting was supported only directly into the database. gitlab #227 Change-Id: I672cc3f5ca20b3b26e9b35d0c3eaf5415d1a3e5f --- diff --git a/bundles/org.simantics.simulation/src/org/simantics/simulation/history/HistoryUtil.java b/bundles/org.simantics.simulation/src/org/simantics/simulation/history/HistoryUtil.java index 27f2bf895..39c084cad 100644 --- a/bundles/org.simantics.simulation/src/org/simantics/simulation/history/HistoryUtil.java +++ b/bundles/org.simantics.simulation/src/org/simantics/simulation/history/HistoryUtil.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2007, 2014 Association for Decentralized Information Management in + * Copyright (c) 2007, 2018 Association for Decentralized Information Management in * Industry THTH ry. * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Public License v1.0 @@ -8,7 +8,7 @@ * * Contributors: * VTT Technical Research Centre of Finland - initial API and implementation - * Semantum Oy - archive implementation + * Semantum Oy - archive implementation, gitlab simantics/platform#227 *******************************************************************************/ package org.simantics.simulation.history; @@ -16,6 +16,7 @@ import java.io.EOFException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.file.Path; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -41,6 +42,7 @@ import org.simantics.databoard.container.DataContainer; import org.simantics.databoard.container.DataContainers; import org.simantics.databoard.serialization.Serializer; import org.simantics.databoard.util.Bean; +import org.simantics.databoard.util.binary.BinaryFile; import org.simantics.databoard.util.binary.OutputStreamWriteable; import org.simantics.databoard.util.binary.RandomAccessBinary; import org.simantics.db.ReadGraph; @@ -81,18 +83,20 @@ public class HistoryUtil { private static final boolean DEBUG = false; private static final boolean PROFILE = true; - private enum Compression { + public enum Compression { FLZ, LZ4, + NONE, } private static final Compression USED_COMPRESSION = Compression.FLZ; + private static final String APPLICATION_X_TAR = "application/x-tar"; private static final String APPLICATION_X_LZ4 = "application/x-lz4"; private static final String APPLICATION_X_FLZ = "application/x-flz"; private static final int ARCHIVE_VERSION_1 = 1; // TarArchiveInput/OutputStream - private static class ArchivedHistory { + public static class ArchivedHistory { public long totalSampleSize = 0L; public TObjectLongMap itemSizes = new TObjectLongHashMap(); } @@ -323,7 +327,7 @@ public class HistoryUtil { rab.position(0); rab.skipBytes(4); - ArchivedHistory archive = exportCompressedArchive(history, collectorState, historyResource, timeShift, from, end, rab); + ArchivedHistory archive = exportCompressedArchive(history, collectorState, timeShift, from, end, rab); rab.position(0L); rab.writeInt((int)(rab.length() - 4L)); @@ -344,80 +348,56 @@ public class HistoryUtil { }; } - private static ArchivedHistory exportCompressedArchive( - final HistoryManager history, - final Bean collectorState, - final Resource historyResource, - final Double timeShift, - final double from, - final double end, + public static ArchivedHistory exportCompressedArchive( + HistoryManager history, + Bean collectorState, + Double timeShift, + double from, + double end, RandomAccessBinary rab) throws IOException, HistoryException { - switch (USED_COMPRESSION) { - case FLZ: - return exportArchiveFLZ(history, collectorState, historyResource, timeShift, from, end, rab); - case LZ4: - return exportArchiveLZ4(history, collectorState, historyResource, timeShift, from, end, rab); - default: - throw new UnsupportedOperationException("Unsupported compression: " + USED_COMPRESSION); - } + return exportCompressedArchive(history, collectorState, timeShift, from, end, USED_COMPRESSION, rab); } - private static ArchivedHistory exportArchiveLZ4( - final HistoryManager history, - final Bean collectorState, - final Resource historyResource, - final Double timeShift, - final double from, - final double end, + public static ArchivedHistory exportCompressedArchive( + HistoryManager history, + Bean collectorState, + Double timeShift, + double from, + double end, + Compression compression, RandomAccessBinary rab) throws IOException, HistoryException { - OutputStream closeable = null; - try { - DataContainers.writeHeader(rab, new DataContainer(APPLICATION_X_LZ4, ARCHIVE_VERSION_1, null)); - closeable = new RABOutputStream( rab ); - closeable = new LZ4BlockOutputStream( closeable ); - - ArchivedHistory archive = exportArchive( closeable, history, collectorState, timeShift, from, end ); - - closeable.flush(); - closeable.close(); - closeable = null; - - return archive; - } finally { - FileUtils.uncheckedClose(closeable); + switch (compression) { + case FLZ: + DataContainers.writeHeader(rab, new DataContainer(APPLICATION_X_FLZ, ARCHIVE_VERSION_1, null)); + return exportArchive(history, collectorState, timeShift, from, end, FastLZ.write(new RABOutputStream(rab))); + case LZ4: + DataContainers.writeHeader(rab, new DataContainer(APPLICATION_X_LZ4, ARCHIVE_VERSION_1, null)); + return exportArchive(history, collectorState, timeShift, from, end, new LZ4BlockOutputStream(new RABOutputStream(rab))); + case NONE: + DataContainers.writeHeader(rab, new DataContainer(APPLICATION_X_TAR, ARCHIVE_VERSION_1, null)); + return exportArchive(history, collectorState, timeShift, from, end, new RABOutputStream(rab)); + default: + throw new UnsupportedOperationException("Unsupported compression: " + compression); } } - @SuppressWarnings("resource") - private static ArchivedHistory exportArchiveFLZ( - final HistoryManager history, - final Bean collectorState, - final Resource historyResource, - final Double timeShift, - final double from, - final double end, - RandomAccessBinary rab) + private static ArchivedHistory exportArchive( + HistoryManager history, + Bean collectorState, + Double timeShift, + double from, + double end, + OutputStream out) throws IOException, HistoryException { - OutputStream closeable = null; - try { - DataContainers.writeHeader(rab, new DataContainer(APPLICATION_X_FLZ, ARCHIVE_VERSION_1, null)); - closeable = new RABOutputStream( rab ); - closeable = FastLZ.write( closeable ); - - ArchivedHistory archive = exportArchive( closeable, history, collectorState, timeShift, from, end ); - - closeable.flush(); - closeable.close(); - closeable = null; - + try (OutputStream os = out) { + ArchivedHistory archive = exportArchive( os, history, collectorState, timeShift, from, end ); + os.flush(); return archive; - } finally { - FileUtils.uncheckedClose(closeable); } } @@ -612,7 +592,7 @@ public class HistoryUtil { try { if (PROFILE) profile(null, "importing history items from archived format to disk workarea"); - importHistoryArchive(graph, historyResource, archive, history, result); + importHistoryArchive(graph, archive, history, result); } catch (IOException e) { throw new DatabaseException(e); } catch (HistoryException e) { @@ -671,6 +651,21 @@ public class HistoryUtil { } } + /** + * @param history + * @param path + * @return + * @throws HistoryException + * @throws IOException + */ + public static HistoryImportResult importHistoryArchive(HistoryManager history, Path path) throws IOException, HistoryException { + HistoryImportResult result = new HistoryImportResult(); + try (RandomAccessBinary rab = new BinaryFile(path.toFile())) { + importHistoryArchive(history, rab, result); + return result; + } + } + /** * Import all history items from database archive into a history manager. * @@ -681,37 +676,51 @@ public class HistoryUtil { * @return true if successful * @throws DatabaseException */ - @SuppressWarnings("resource") - private static boolean importHistoryArchive(ReadGraph graph, Resource historyResource, Resource archive, HistoryManager history, HistoryImportResult result) - throws DatabaseException, IOException, HistoryException - { + private static boolean importHistoryArchive(ReadGraph graph, Resource archive, HistoryManager history, HistoryImportResult result) throws DatabaseException, IOException, HistoryException { RandomAccessBinary rab = graph.getRandomAccessBinary(archive); + // This is here because the data is encoded as an L0.ByteArray + // in the database which means the first integer describes the + // length of the byte array. We skip that length data here. rab.position(4); + return importHistoryArchive(history, rab, result); + } + + /** + * Import all history items from database archive into a history manager. + * + * @param graph + * @param historyResource + * @param archive + * @param history + * @return true if successful + * @throws DatabaseException + */ + private static boolean importHistoryArchive(HistoryManager history, RandomAccessBinary rab, HistoryImportResult result) + throws IOException, HistoryException + { DataContainer dc = DataContainers.readHeader(rab); if (DEBUG) System.out.println("COMPRESSED HISTORY DATA SIZE: " + (rab.length() - rab.position())); - if (APPLICATION_X_LZ4.equals(dc.format)) { - if (dc.version == ARCHIVE_VERSION_1) { - InputStream in = new RABInputStream(rab); - //in = new BufferedInputStream(in); - in = new LZ4BlockInputStream(in); - return extractHistoryArchiveTar(graph, history, in, result); - } - } else if (APPLICATION_X_FLZ.equals(dc.format)) { - if (dc.version == ARCHIVE_VERSION_1) { - InputStream in = null; - try { - in = new RABInputStream(rab); - in = FastLZ.read(in); - return extractHistoryArchiveTar(graph, history, in, result); - } finally { - FileUtils.uncheckedClose(in); - } + if (dc.version == ARCHIVE_VERSION_1) { + if (APPLICATION_X_LZ4.equals(dc.format)) { + return extractHistoryArchiveInputStream(history, new LZ4BlockInputStream(new RABInputStream(rab)), result); + } else if (APPLICATION_X_FLZ.equals(dc.format)) { + return extractHistoryArchiveInputStream(history, FastLZ.read(new RABInputStream(rab)), result); + } else if (APPLICATION_X_TAR.equals(dc.format)) { + return extractHistoryArchiveInputStream(history, new RABInputStream(rab), result); } + throw new UnsupportedOperationException("Unsupported format " + dc.format + " and version " + dc.version); + } else { + throw new UnsupportedOperationException("Unsupported history archive version " + dc.version + " with format " + dc.format); + } + } + + private static boolean extractHistoryArchiveInputStream(HistoryManager history, InputStream in, HistoryImportResult result) throws IOException, HistoryException { + try (InputStream _in = in) { + return extractHistoryArchiveTar(history, in, result); } - throw new UnsupportedOperationException("Unsupported format " + dc.format + " and version " + dc.version); } /** @@ -723,8 +732,8 @@ public class HistoryUtil { * @return true if successful * @throws DatabaseException */ - private static boolean extractHistoryArchiveTar(ReadGraph graph, HistoryManager history, InputStream in, HistoryImportResult result) - throws DatabaseException, IOException, HistoryException + private static boolean extractHistoryArchiveTar(HistoryManager history, InputStream in, HistoryImportResult result) + throws IOException, HistoryException { // tar is not closed on purpose because we do not want to close RandomAccessBinary rab. TarArchiveInputStream tar = new TarArchiveInputStream(in); @@ -784,9 +793,9 @@ public class HistoryUtil { //component(3): = Boolean //component(4): = Boolean //component(5): = Boolean - //component(devil): = Boolean + //component(6): = Boolean //component(7): = Boolean - //component(music): = Boolean + //component(8): = Boolean //component(9): = Boolean //component(10): = Boolean //