From 77921feee3f8331ab54796ff0832921405bea874 Mon Sep 17 00:00:00 2001 From: Tuukka Lehtonen Date: Tue, 13 Mar 2018 11:08:40 +0200 Subject: [PATCH] Added graph.tg hash caching to FixExportedOntology Moved generic database baselining code to DatabaseBaselines from SimanticsPlatform and added new utility functions to DataContainers and cleaned up existing methods therein using resource-try statements. Also note org.simantics.graph.refactoring.TransferableGraphHasher that can be used for adding hash values to graph.tg files. refs #7806 Change-Id: I7f54cfb0db68a8a5602fc2a8d9651cb305a8136d --- .../databoard/container/DataContainers.java | 117 ++++++++++-------- .../refactoring/FixExportedOntology.java | 18 +-- .../refactoring/TransferableGraphHasher.java | 56 +++++++++ .../src/org/simantics/DatabaseBaselines.java | 11 ++ .../src/org/simantics/SimanticsPlatform.java | 11 +- 5 files changed, 137 insertions(+), 76 deletions(-) create mode 100644 bundles/org.simantics.graph/src/org/simantics/graph/refactoring/TransferableGraphHasher.java diff --git a/bundles/org.simantics.databoard/src/org/simantics/databoard/container/DataContainers.java b/bundles/org.simantics.databoard/src/org/simantics/databoard/container/DataContainers.java index 21e54d897..dba475f0e 100644 --- a/bundles/org.simantics.databoard/src/org/simantics/databoard/container/DataContainers.java +++ b/bundles/org.simantics.databoard/src/org/simantics/databoard/container/DataContainers.java @@ -59,24 +59,24 @@ public class DataContainers { String format = (String)STRING_SERIALIZER.deserialize(input); int version = (Integer)INTEGER_SERIALIZER.deserialize(input); @SuppressWarnings("unchecked") - TreeMap metadata = (TreeMap)METADATA_SERIALIZER.deserialize(input); + TreeMap metadata = (TreeMap)METADATA_SERIALIZER.deserialize(input); return new DataContainer(format, version, metadata, null); } - + /** * Consumes a header from a given stream and checks that the header satisfies the given format and version. * Returns the obtained header if the check fails and null on success. * @throws IOException */ public static DataContainer requireHeader(DataInput input, String format, int version) { - try { - DataContainer header = readHeader(input); - if(!format.equals(header.format) || version != header.version) - return header; - else return null; - } catch (Throwable t) { - return new DataContainer("unknown", 0, null, null); - } + try { + DataContainer header = readHeader(input); + if(!format.equals(header.format) || version != header.version) + return header; + else return null; + } catch (Throwable t) { + return new DataContainer("unknown", 0, null, null); + } } /** @@ -104,14 +104,9 @@ public class DataContainers { * @throws IOException */ public static DataContainer validateHeader(File file, String format, int version) throws IOException { - - InputStream stream = new FileInputStream( file ); - try { - return DataContainers.requireHeader(new DataInputStream(stream), format, version); - } finally { - stream.close(); - } - + try (InputStream stream = new FileInputStream( file )) { + return DataContainers.requireHeader(new DataInputStream(stream), format, version); + } } /** @@ -120,14 +115,9 @@ public class DataContainers { * @throws IOException */ public static DataContainer validateHeader(File file, String... allowedFormatStrings) throws IOException { - - InputStream stream = new FileInputStream( file ); - try { + try (InputStream stream = new FileInputStream( file )) { return DataContainers.requireHeader(new DataInputStream(stream), allowedFormatStrings); - } finally { - stream.close(); } - } /** @@ -136,14 +126,11 @@ public class DataContainers { * @throws IOException */ public static DataContainer readHeader(File input) throws IOException { - BinaryFile rf = new BinaryFile( input, "r" ); - try { + try (BinaryFile rf = new BinaryFile( input, "r" )) { return readHeader(rf); - } finally { - rf.close(); - } + } } - + /** * Reads a data container including the content data. * @throws IOException @@ -154,6 +141,23 @@ public class DataContainers { return result; } + /** + * Reads a data container including the content data. + * @throws IOException + */ + public static DataContainer readFile(DataInput input, Binding expectedBinding) throws IOException, DataFormatException { + DataContainer result = readHeader(input); + Datatype contentType = (Datatype) DATATYPE_SERIALIZER.deserialize(input); + if (!expectedBinding.type().equals(contentType)) + throw new DataFormatException( + "Content type didn't match the type expected for the binding " + expectedBinding + + ":\nexpected type: " + expectedBinding.type() + + "\nactual type: " + contentType); + Object value = Bindings.getSerializerUnchecked(expectedBinding).deserialize(input); + result.content = new Variant(expectedBinding, value); + return result; + } + /** * Process a data container using a format handler matching the format and version * of the file. @@ -161,49 +165,54 @@ public class DataContainers { */ public static T readFile(DataInput input, Map> handlers) throws Exception { DataContainer result = readHeader(input); - + FormatHandler handler = handlers.get(result.format + ":" + result.version); if(handler == null) throw new DataFormatException("Unknown data format " + result.format + " version " + result.version + "."); Binding binding = handler.getBinding(); - + Datatype contentType = (Datatype)DATATYPE_SERIALIZER.deserialize(input); if(!binding.type().equals(contentType)) throw new DataFormatException("Content type didn't match the type expected for the format " + result.format + " version " + result.version + "."); - + Object value = Bindings.getSerializerUnchecked(binding).deserialize(input); - + result.content = new Variant(binding, value); return handler.process(result); } - + /** * Reads a data container including the content data. * @throws IOException */ public static DataContainer readFile(File input) throws IOException { - BinaryFile rf = new BinaryFile( input, "r" ); - try { + try (BinaryFile rf = new BinaryFile( input, "r" )) { return readFile(rf); - } finally { - rf.close(); - } + } + } + + /** + * Reads a data container including the content data. + * @throws IOException + * @throws DataFormatException + */ + public static DataContainer readFile(File input, Binding expectedBinding) throws IOException, DataFormatException { + try (BinaryFile rf = new BinaryFile( input, "r" )) { + return readFile(rf, expectedBinding); + } } - + /** * Process a data container using a format handler matching the format and version * of the file. * @param handlers Map of handlers. Keys are strings of form "format:version". */ public static T readFile(File input, Map> handlers) throws Exception { - BinaryFile rf = new BinaryFile( input, "r" ); - try { + try (BinaryFile rf = new BinaryFile( input, "r" )) { return readFile(rf, handlers); - } finally { - rf.close(); - } + } } - + /** * Writes header fields of a container to the given output. Content field is * ignored. @@ -214,7 +223,7 @@ public class DataContainers { INTEGER_SERIALIZER.serialize(output, container.version); METADATA_SERIALIZER.serialize(output, container.metadata); } - + /** * Writes a data container to the given output. * @throws IOException @@ -223,22 +232,20 @@ public class DataContainers { writeHeader(output, container); VARIANT_SERIALIZER.serialize(output, container.content); } - + /** * Writes a data container to the given file * @throws IOException */ public static void writeFile(File output, DataContainer container) throws IOException { - BinaryFile wf = new BinaryFile( output); - try { + try (BinaryFile wf = new BinaryFile(output)) { writeFile(wf, container); wf.setLength(wf.position()); - } finally { - wf.close(); - } + } } - + public static byte[] writeFile(DataContainer container) throws IOException { return DATA_CONTAINER_SERIALIZER.serialize(container); } + } diff --git a/bundles/org.simantics.graph/src/org/simantics/graph/refactoring/FixExportedOntology.java b/bundles/org.simantics.graph/src/org/simantics/graph/refactoring/FixExportedOntology.java index 90738f70d..ffe174eb9 100644 --- a/bundles/org.simantics.graph/src/org/simantics/graph/refactoring/FixExportedOntology.java +++ b/bundles/org.simantics.graph/src/org/simantics/graph/refactoring/FixExportedOntology.java @@ -1,7 +1,6 @@ package org.simantics.graph.refactoring; import java.io.BufferedInputStream; -import java.io.DataInput; import java.io.DataInputStream; import java.io.InputStream; import java.nio.file.Files; @@ -10,7 +9,6 @@ import java.nio.file.Paths; import java.nio.file.StandardOpenOption; import org.simantics.databoard.binding.Binding; -import org.simantics.databoard.binding.mutable.Variant; import org.simantics.databoard.container.DataContainer; import org.simantics.databoard.container.DataContainers; import org.simantics.graph.representation.PrettyPrintTG; @@ -25,19 +23,15 @@ public class FixExportedOntology { static TransferableGraph1 convertExportedSharedOntologyIntoBundleOntology(Path input, Path output) throws Exception { System.out.format("Converting exported shared ontology%n\t" + input.toString() + "%nto bundle-compatible ontology%n\t" + output.toString()); try (InputStream is = new BufferedInputStream(Files.newInputStream(input), 128*1024)) { - DataInput dis = new DataInputStream(is); - org.simantics.databoard.container.DataContainer container = - DataContainers.readFile(dis); - Binding binding = TransferableGraph1.BINDING; - TransferableGraph1 graph = (TransferableGraph1)container.content.getValue(binding); + Binding tgBinding = TransferableGraph1.BINDING; + DataContainer container = DataContainers.readFile(new DataInputStream(is), tgBinding); + TransferableGraph1 graph = (TransferableGraph1) container.content.getValue(tgBinding); + GraphRefactoringUtils.fixOntologyExport(graph); + container = TransferableGraphHasher.addHashToTG(container, graph); - DataContainers.writeFile(output.toFile(), new DataContainer( - container.format, container.version, - container.metadata, new Variant(TransferableGraph1.BINDING, graph))); - + DataContainers.writeFile(output.toFile(), container); return graph; - } } diff --git a/bundles/org.simantics.graph/src/org/simantics/graph/refactoring/TransferableGraphHasher.java b/bundles/org.simantics.graph/src/org/simantics/graph/refactoring/TransferableGraphHasher.java new file mode 100644 index 000000000..e1cf3f6f4 --- /dev/null +++ b/bundles/org.simantics.graph/src/org/simantics/graph/refactoring/TransferableGraphHasher.java @@ -0,0 +1,56 @@ +package org.simantics.graph.refactoring; + +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.simantics.databoard.Bindings; +import org.simantics.databoard.adapter.AdaptException; +import org.simantics.databoard.binding.Binding; +import org.simantics.databoard.binding.error.BindingException; +import org.simantics.databoard.binding.mutable.Variant; +import org.simantics.databoard.container.DataContainer; +import org.simantics.databoard.container.DataContainers; +import org.simantics.graph.representation.Extensions; +import org.simantics.graph.representation.TransferableGraph1; + +/** + * @author Tuukka Lehtonen + * @since 1.34.0 + */ +public class TransferableGraphHasher { + + public static int hashTG(TransferableGraph1 tg) throws BindingException { + return TransferableGraph1.BINDING.hashValue(tg); + } + + public static DataContainer addHashToTG(DataContainer tgContainer, TransferableGraph1 tg) throws BindingException { + Binding tgb = TransferableGraph1.BINDING; + tgContainer.metadata.put(Extensions.CACHED_HASHCODE, new Variant(Bindings.INTEGER, hashTG(tg))); + tgContainer.content = new Variant(tgb, tg); + return tgContainer; + } + + public static DataContainer addHashToTG(DataContainer tgContainer) throws BindingException, AdaptException { + return addHashToTG(tgContainer, (TransferableGraph1) tgContainer.content.getValue(TransferableGraph1.BINDING)); + } + + public static void hashTG(Path input, Path output) throws Exception { + System.out.format("Adding cached hash value to transferable graph%n\t" + input.toString() + "%nto%n\t" + output.toString()); + DataContainers.writeFile(output.toFile(), + addHashToTG( + DataContainers.readFile(input.toFile(), TransferableGraph1.BINDING) )); + } + + public static void main(String[] args) throws Exception { + if (args.length == 0) { + System.out.println("Required arguments: []"); + } else if (args.length == 1) { + // In-place hash + Path p = Paths.get(args[0]); + hashTG(p, p); + } else { + hashTG(Paths.get(args[0]), Paths.get(args[1])); + } + } + +} diff --git a/bundles/org.simantics/src/org/simantics/DatabaseBaselines.java b/bundles/org.simantics/src/org/simantics/DatabaseBaselines.java index 96db5ed72..ca272be6b 100644 --- a/bundles/org.simantics/src/org/simantics/DatabaseBaselines.java +++ b/bundles/org.simantics/src/org/simantics/DatabaseBaselines.java @@ -123,6 +123,17 @@ public class DatabaseBaselines { } } + public static void initializeWorkspaceWithBaseline(Path baseline, Path workspaceLocation, Path indicatorPath) throws PlatformException { + try { + Files.createDirectories(workspaceLocation); + FileUtils.extractZip(baseline.toFile(), workspaceLocation.toFile()); + if (indicatorPath != null) + Files.write(indicatorPath, DatabaseBaselines.baselineIndicatorContents(indicatorPath)); + } catch (IOException e) { + throw new PlatformException(e); + } + } + public static void main(String[] args) throws IOException { packageBaseline(Paths.get("D:/temp/desktop/workspace"), Paths.get("d:/temp/desktop/workspace/baseline.zip")); } diff --git a/bundles/org.simantics/src/org/simantics/SimanticsPlatform.java b/bundles/org.simantics/src/org/simantics/SimanticsPlatform.java index 1cedd4fcb..3638d3407 100644 --- a/bundles/org.simantics/src/org/simantics/SimanticsPlatform.java +++ b/bundles/org.simantics/src/org/simantics/SimanticsPlatform.java @@ -715,15 +715,8 @@ public class SimanticsPlatform implements LifecycleListener { DatabaseBaselines.validateBaselineFile(baseline); DatabaseBaselines.validateWorkspaceForBaselineInitialization(workspaceLocation); - - try { - Files.createDirectories(workspaceLocation); - FileUtils.extractZip(baseline.toFile(), workspaceLocation.toFile()); - Files.write(baselineIndicatorFile, DatabaseBaselines.baselineIndicatorContents(baselineIndicatorFile)); - return true; - } catch (IOException e) { - throw new PlatformException(e); - } + DatabaseBaselines.initializeWorkspaceWithBaseline(baseline, workspaceLocation, baselineIndicatorFile); + return true; } /** -- 2.47.1