From b913419ca9037bf9734c56a5f079024c3a1cd177 Mon Sep 17 00:00:00 2001 From: Tuukka Lehtonen Date: Fri, 9 Mar 2018 23:41:15 +0200 Subject: [PATCH] Platform startup performance improvements 1. DB baselining can now be enabled with ontology update checking due to caching of TG hash values in the DataContainer.metadata map field with key "cached.hashCode". This diminishes the time taken by ontology merge need checking to very little time compared to the old system where all graph.tg files were loaded completely into heap memory to simply check whether they need to be merged or not. 2. BaselineCreator application can be used to construct a baseline with a built product in any OS supported by Simantics. To get the benefit from these changes, you'll also need to use the version 0.0.9 of the graph-builder-maven-plugin. refs #7806 Change-Id: I7fadaf43f3c96d3b989a73e0f1ae6e6fa83e09ce --- .../graph/compiler/GraphCompiler.java | 31 +- .../graph/representation/Extensions.java | 4 +- .../project/management/GraphBundle.java | 96 ++++--- .../project/management/GraphBundleEx.java | 10 + .../project/management/PlatformUtil.java | 267 +++++++++++------- .../src/org/simantics/utils/FileUtils.java | 4 +- bundles/org.simantics/plugin.xml | 13 + .../simantics/BaselineCreatorApplication.java | 113 ++++++++ .../src/org/simantics/DatabaseBaselines.java | 130 +++++++++ .../src/org/simantics/SimanticsPlatform.java | 114 +++----- bundles/pom.xml | 2 +- 11 files changed, 557 insertions(+), 227 deletions(-) create mode 100644 bundles/org.simantics/src/org/simantics/BaselineCreatorApplication.java create mode 100644 bundles/org.simantics/src/org/simantics/DatabaseBaselines.java diff --git a/bundles/org.simantics.graph.compiler/src/org/simantics/graph/compiler/GraphCompiler.java b/bundles/org.simantics.graph.compiler/src/org/simantics/graph/compiler/GraphCompiler.java index 26663b08a..53060b767 100644 --- a/bundles/org.simantics.graph.compiler/src/org/simantics/graph/compiler/GraphCompiler.java +++ b/bundles/org.simantics.graph.compiler/src/org/simantics/graph/compiler/GraphCompiler.java @@ -11,6 +11,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Formatter; import java.util.Locale; +import java.util.TreeMap; import org.simantics.databoard.Bindings; import org.simantics.databoard.Files; @@ -56,23 +57,29 @@ public class GraphCompiler { } public static TransferableGraph1 read(InputStream stream) throws AdaptException, IOException { - DataContainer container = DataContainers.readFile(new DataInputStream(stream)); - stream.close(); - return (TransferableGraph1)container.content.getValue(TransferableGraph1.BINDING); + try (InputStream in = stream) { + DataContainer container = DataContainers.readFile(new DataInputStream(stream)); + return (TransferableGraph1)container.content.getValue(TransferableGraph1.BINDING); + } } public static InputStream write(TransferableGraph1 tg) throws BindingException, IOException { Binding binding = TransferableGraph1.BINDING; int hashCode = binding.hashValue(tg); - tg.extensions.put(Extensions.CACHED_HASHCODE, new Variant(Bindings.INTEGER, hashCode)); - try { - byte[] buffer = DataContainers.writeFile( - new DataContainer("graph", 1, new Variant(TransferableGraph1.BINDING, tg)) - ); - return new ByteArrayInputStream(buffer); - } finally { - tg.extensions.remove(Extensions.CACHED_HASHCODE); - } + TreeMap metadata = new TreeMap<>(); + metadata.put(Extensions.CACHED_HASHCODE, new Variant(Bindings.INTEGER, hashCode)); + byte[] buffer = DataContainers.writeFile( + new DataContainer("graph", 1, metadata, new Variant(binding, tg)) + ); + return new ByteArrayInputStream(buffer); + } + + public static void write(File file, TransferableGraph1 tg) throws BindingException, IOException { + Binding binding = TransferableGraph1.BINDING; + int hashCode = binding.hashValue(tg); + TreeMap metadata = new TreeMap<>(); + metadata.put(Extensions.CACHED_HASHCODE, new Variant(Bindings.INTEGER, hashCode)); + DataContainers.writeFile(file, new DataContainer("graph", 1, metadata, new Variant(binding, tg))); } public static CompilationResult compile( diff --git a/bundles/org.simantics.graph/src/org/simantics/graph/representation/Extensions.java b/bundles/org.simantics.graph/src/org/simantics/graph/representation/Extensions.java index 375fb1416..73a988798 100644 --- a/bundles/org.simantics.graph/src/org/simantics/graph/representation/Extensions.java +++ b/bundles/org.simantics.graph/src/org/simantics/graph/representation/Extensions.java @@ -12,8 +12,8 @@ public class Extensions { /** * Used for storing a cached hash code computed from a - * {@link TransferableGraph1} instance without this cached hashcode - * key,value pair in the extensions map. + * {@link TransferableGraph1} instance in the metadata map of the + * DataContainer containing the TG. */ public final static String CACHED_HASHCODE = "cached.hashCode"; diff --git a/bundles/org.simantics.project/src/org/simantics/project/management/GraphBundle.java b/bundles/org.simantics.project/src/org/simantics/project/management/GraphBundle.java index 952e50cb0..82764ca42 100644 --- a/bundles/org.simantics.project/src/org/simantics/project/management/GraphBundle.java +++ b/bundles/org.simantics.project/src/org/simantics/project/management/GraphBundle.java @@ -11,11 +11,10 @@ *******************************************************************************/ package org.simantics.project.management; +import java.util.function.Supplier; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.simantics.databoard.Bindings; -import org.simantics.databoard.binding.Binding; import org.simantics.databoard.binding.error.BindingException; import org.simantics.databoard.binding.error.RuntimeBindingException; import org.simantics.db.ReadGraph; @@ -26,6 +25,8 @@ import org.simantics.db.common.utils.Transaction; import org.simantics.db.exception.DatabaseException; import org.simantics.graph.representation.TransferableGraph1; import org.simantics.layer0.DatabaseManagementResource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * GraphBundle represents a bundle graph that may exist in memory @@ -58,6 +59,8 @@ import org.simantics.layer0.DatabaseManagementResource; */ public class GraphBundle implements Comparable { + private static final Logger LOGGER = LoggerFactory.getLogger(GraphBundle.class); + /** Versioned Id pattern */ static String ID_PATTERN_STRING = "[a-zA-Z_0-9\\-]+(?:\\.[a-zA-Z_0-9\\-]+)*"; static String VERSION_PATTERN_STRING = "(\\d+).(\\d+).(\\d+).([a-zA-Z_0-9\\-]+)"; @@ -68,6 +71,9 @@ public class GraphBundle implements Comparable { /** User-friendly name */ String name; + /** If {@link #graph} is null then this may be defined to fetch the data on-demand */ + Supplier graphSource; + /** Actual graph */ TransferableGraph1 graph; @@ -93,21 +99,19 @@ public class GraphBundle implements Comparable { boolean immutable = true; GraphBundle() {} - + public GraphBundle(String name, TransferableGraph1 data, String versionedId) - throws RuntimeBindingException { - try { + throws RuntimeBindingException { + try { // Assert version id is correct Matcher m = VERSIONED_ID_PATTERN.matcher(versionedId); if (!m.matches()) { throw new IllegalArgumentException("Illegal VersionId \""+versionedId+"\", / is expected."); } - - Binding binding = Bindings.getBindingUnchecked( TransferableGraph1.class ); - + this.name = name; - this.graph = data; - this.hashcode = data != null ? binding.hashValue( data ) : 0; + this.graph = data; + this.hashcode = hash(data); this.id = m.group(1); this.major = Integer.valueOf( m.group(2) ); this.minor = Integer.valueOf( m.group(3) ); @@ -118,22 +122,21 @@ public class GraphBundle implements Comparable { } catch (BindingException e) { // Unexpected throw new RuntimeBindingException(e); - } + } } - + public GraphBundle(String name, TransferableGraph1 data, String id, String version) - throws RuntimeBindingException { + throws RuntimeBindingException { Matcher m = ID_PATTERN.matcher(id); if (!m.matches()) throw new IllegalArgumentException("Illegal Id, got \""+id+"\""); m = VERSION_PATTERN.matcher(version); if (!m.matches()) throw new IllegalArgumentException("Illegal Version, got \""+id+"\", / is expected."); - try { - Binding binding = Bindings.getBindingUnchecked( TransferableGraph1.class ); + try { this.name = name; - this.graph = data; - this.hashcode = binding.hashValue( data ); + this.graph = data; + this.hashcode = hash(data); this.id = id; this.major = Integer.valueOf( m.group(1) ); this.minor = Integer.valueOf( m.group(2) ); @@ -146,7 +149,30 @@ public class GraphBundle implements Comparable { throw new RuntimeBindingException(e); } } - + + public GraphBundle(String name, Supplier source, int hashCode, String id, String version) { + Matcher m = ID_PATTERN.matcher(id); + if (!m.matches()) + throw new IllegalArgumentException("Illegal Id, got \""+id+"\""); + m = VERSION_PATTERN.matcher(version); + if (!m.matches()) + throw new IllegalArgumentException("Illegal Version, got \""+id+"\", / is expected."); + this.name = name; + this.graphSource = source; + this.hashcode = hashCode; + this.id = id; + this.major = Integer.valueOf( m.group(1) ); + this.minor = Integer.valueOf( m.group(2) ); + this.service = Integer.valueOf( m.group(3) ); + if (m.group(4) != null) { + this.qualifier = m.group(4); + } + } + + private int hash(TransferableGraph1 data) throws BindingException { + return data == null ? 0 : TransferableGraph1.BINDING.hashValue( data ); + } + public String getName() { return name; } @@ -180,15 +206,18 @@ public class GraphBundle implements Comparable { */ public TransferableGraph1 getGraph() { if (graph == null) { - ReadGraph g = Transaction.readGraph(); - if (g == null) - throw new IllegalStateException("No read transaction available"); - try { - Binding tg_binding = Bindings.getBindingUnchecked( TransferableGraph1.class ); - DatabaseManagementResource DatabaseManagement = DatabaseManagementResource.getInstance(g); - graph = g.getRelatedValue(resource, DatabaseManagement.HasFile, tg_binding); - } catch (DatabaseException e) { - e.printStackTrace(); + if (graphSource != null) { + graph = graphSource.get(); + } + if (graph == null) { + ReadGraph g = Transaction.readGraph(); + if (g == null) + throw new IllegalStateException("No read transaction available"); + try { + graph = readTg(g); + } catch (DatabaseException e) { + LOGGER.error("Failed to read transferable graph from " + resource, e); + } } } return graph; @@ -200,18 +229,21 @@ public class GraphBundle implements Comparable { graph = processor.syncRequest(new ResourceRead(resource) { @Override public TransferableGraph1 perform(ReadGraph graph) throws DatabaseException { - Binding tg_binding = Bindings.getBindingUnchecked( TransferableGraph1.class ); - DatabaseManagementResource DatabaseManagement = DatabaseManagementResource.getInstance(graph); - return graph.getRelatedValue(resource, DatabaseManagement.HasFile, tg_binding); + return readTg(graph); } }); } catch (DatabaseException e) { - e.printStackTrace(); + LOGGER.error("Failed to read transferable graph from " + resource, e); } } return graph; } - + + private TransferableGraph1 readTg(ReadGraph graph) throws DatabaseException { + DatabaseManagementResource DatabaseManagement = DatabaseManagementResource.getInstance(graph); + return graph.getRelatedValue(resource, DatabaseManagement.HasFile, TransferableGraph1.BINDING); + } + public int getHashcode() { return hashcode; } diff --git a/bundles/org.simantics.project/src/org/simantics/project/management/GraphBundleEx.java b/bundles/org.simantics.project/src/org/simantics/project/management/GraphBundleEx.java index f90896fa1..450d7808f 100644 --- a/bundles/org.simantics.project/src/org/simantics/project/management/GraphBundleEx.java +++ b/bundles/org.simantics.project/src/org/simantics/project/management/GraphBundleEx.java @@ -13,6 +13,7 @@ package org.simantics.project.management; import java.text.SimpleDateFormat; import java.util.Date; +import java.util.function.Supplier; import org.eclipse.equinox.p2.metadata.IVersionedId; import org.eclipse.equinox.p2.metadata.Version; @@ -60,6 +61,7 @@ public class GraphBundleEx extends GraphBundle implements IVersionedId { VersionedId vid; GraphBundleEx(GraphBundle e) { + this.graphSource = e.graphSource; this.graph = e.graph; this.resource = e.resource; this.hashcode = e.hashcode; @@ -106,6 +108,14 @@ public class GraphBundleEx extends GraphBundle implements IVersionedId { this.immutable = isImmutable; } + public GraphBundleEx(String name, Supplier source, int hashValue, IVersionedId vid, boolean isImmutable) + throws RuntimeBindingException + { + super(name, source, hashValue, vid.getId(), vid.getVersion().getSegment(0).toString()+"."+vid.getVersion().getSegment(1).toString()+"."+vid.getVersion().getSegment(2).toString()+"."+vid.getVersion().getSegment(3).toString()); + this.vid = new VersionedId(id, vid.getVersion()); + this.immutable = isImmutable; + } + public GraphBundleEx(String name, TransferableGraph1 data, IVersionedId vid) throws RuntimeBindingException { diff --git a/bundles/org.simantics.project/src/org/simantics/project/management/PlatformUtil.java b/bundles/org.simantics.project/src/org/simantics/project/management/PlatformUtil.java index ebd987e3f..fde150036 100644 --- a/bundles/org.simantics.project/src/org/simantics/project/management/PlatformUtil.java +++ b/bundles/org.simantics.project/src/org/simantics/project/management/PlatformUtil.java @@ -13,7 +13,6 @@ package org.simantics.project.management; import java.io.BufferedInputStream; import java.io.Closeable; -import java.io.DataInput; import java.io.DataInputStream; import java.io.File; import java.io.FileNotFoundException; @@ -28,10 +27,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Enumeration; +import java.util.Map; import java.util.Map.Entry; import java.util.Objects; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; import java.util.jar.Attributes; import java.util.jar.Manifest; import java.util.stream.Collectors; @@ -43,19 +43,24 @@ import org.eclipse.equinox.p2.metadata.IVersionedId; import org.eclipse.equinox.p2.metadata.Version; import org.eclipse.equinox.p2.metadata.VersionedId; import org.osgi.framework.Bundle; +import org.simantics.databoard.Bindings; +import org.simantics.databoard.adapter.AdaptException; import org.simantics.databoard.binding.Binding; import org.simantics.databoard.binding.mutable.Variant; import org.simantics.databoard.container.DataContainer; import org.simantics.databoard.container.DataContainers; +import org.simantics.databoard.container.FormatHandler; import org.simantics.graph.compiler.CompilationResult; import org.simantics.graph.compiler.GraphCompiler; import org.simantics.graph.compiler.GraphCompilerPreferences; import org.simantics.graph.compiler.ValidationMode; +import org.simantics.graph.representation.Extensions; import org.simantics.graph.representation.TransferableGraph1; import org.simantics.ltk.FileSource; import org.simantics.ltk.ISource; import org.simantics.ltk.Problem; import org.simantics.scl.reflection.OntologyVersions; +import org.simantics.utils.datastructures.ArrayMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -76,7 +81,7 @@ public class PlatformUtil { public static Bundle[] getBundles() { return PlatformActivator.getContext().getBundles(); } - + /** * Get the manifest file of a bundle * @@ -87,14 +92,11 @@ public class PlatformUtil { public static Manifest getManifest(Bundle bundle) throws IOException { URL url = bundle.getEntry("META-INF/MANIFEST.MF"); if (url==null) return null; - InputStream is = url.openStream(); - try { - return new Manifest(is); - } finally { - is.close(); + try (InputStream is = url.openStream()) { + return new Manifest(is); } } - + /** * Get the manifest file of a bundle * @@ -105,11 +107,8 @@ public class PlatformUtil { public static Manifest getSimanticsManifest(Bundle bundle) throws IOException { URL url = bundle.getEntry("META-INF/SIMANTICS.MF"); if (url==null) return null; - InputStream is = url.openStream(); - try { - return new Manifest(is); - } finally { - is.close(); + try (InputStream is = url.openStream()) { + return new Manifest(is); } } @@ -131,13 +130,13 @@ public class PlatformUtil { for (Entry entry2 : attributes.entrySet()) { Object key = entry2.getKey(); if (key.toString().contains("Installable-Unit")) { - String bid = entry2.getValue().toString(); + String bid = entry2.getValue().toString(); list.add( bid ); } } - } + } } - + /** * Get all transferable graphs in the platform * @@ -151,7 +150,7 @@ public class PlatformUtil { org.osgi.framework.Version osgiVersion = bundle.getVersion(); Version p2Version = Version.createOSGi(osgiVersion.getMajor(), osgiVersion.getMinor(), osgiVersion.getMicro(), osgiVersion.getQualifier()); String id = bundle.getSymbolicName(); - + TGInfo info = new TGInfo(); info.location = e.nextElement(); info.bundle = bundle; @@ -169,7 +168,7 @@ public class PlatformUtil { //ignore } } - + private static File copyResource(URL url, File targetFile) throws IOException, FileNotFoundException { FileOutputStream os = null; InputStream is = null; @@ -193,7 +192,7 @@ public class PlatformUtil { uncheckedClose(is); } } - + private static File extractLib(URL libURL, String libName) throws FileNotFoundException, IOException { String tmpDirStr = System.getProperty("java.io.tmpdir"); if (tmpDirStr == null) @@ -202,7 +201,7 @@ public class PlatformUtil { File libFile = new File(tmpDir, libName); return copyResource(libURL, libFile); } - + private static File url2file(URL url, String fileName) { if ("file".equals(url.getProtocol())) { try { @@ -225,12 +224,12 @@ public class PlatformUtil { } return null; } - + public static void compile(Bundle b) throws Exception { - - Collection sources = new ArrayList(); - Collection dependencies = new ArrayList(); - + + Collection sources = new ArrayList<>(); + Collection dependencies = new ArrayList<>(); + for (Bundle b2 : getBundles()) { if(b.equals(b2)) continue; URL url = b2.getEntry("graph.tg"); @@ -238,24 +237,24 @@ public class PlatformUtil { File graphFile = url2file(FileLocator.resolve(b2.getEntry("/graph.tg")), b2.toString()); dependencies.add(GraphCompiler.read(graphFile)); } - + File bundleFile = FileLocator.getBundleFile(b); if(bundleFile.isDirectory()) { File folder = new File(bundleFile, "dynamicGraph"); for(File f : folder.listFiles(new FilenameFilter() { - + @Override public boolean accept(File dir, String name) { return name.endsWith(".pgraph"); } - + })) { sources.add(new FileSource(f)); } - } - + } + // System.out.println("source is " + tmpFile.getAbsolutePath()); - + final StringBuilder errorStringBuilder = new StringBuilder(); GraphCompilerPreferences prefs = new GraphCompilerPreferences(); prefs.validate = true; @@ -263,7 +262,7 @@ public class PlatformUtil { prefs.validateResourceHasType = ValidationMode.ERROR; String currentLayer0Version = OntologyVersions.getInstance().currentOntologyVersion("http://www.simantics.org/Layer0-0.0"); CompilationResult result = GraphCompiler.compile(currentLayer0Version, sources, dependencies, null, prefs); - + for(Problem problem : result.getErrors()) errorStringBuilder.append(problem.getLocation() + ": " + problem.getDescription() + "\n"); for(Problem problem : result.getWarnings()) @@ -272,12 +271,11 @@ public class PlatformUtil { if(errorStringBuilder.length() > 0) { LOGGER.error(errorStringBuilder.toString()); } else { - DataContainers.writeFile(new File(bundleFile, "graph.tg"), - new DataContainer("graph", 1, new Variant(TransferableGraph1.BINDING, result.getGraph()))); + GraphCompiler.write(new File(bundleFile, "graph.tg"), result.getGraph()); } - + } - + /** * Compile all dynamic ontologies in the Platform * @@ -306,37 +304,35 @@ public class PlatformUtil { } } } - + /** * Get all graphs in the Platform * * @param collection * @throws IOException */ - public static Collection getAllGraphs() throws IOException { - CompletableFuture f = new CompletableFuture<>(); - Bundle[] bundles = getBundles(); - Collection gbundles = Arrays.stream(bundles).map(t -> { // this could be done in parallel in the future? - if (f.isCompletedExceptionally()) - return null; - try { - return PlatformUtil.getGraph(t); - } catch (IOException e) { - if (LOGGER.isDebugEnabled()) - LOGGER.debug("Could not get graph {}", t, e); - f.completeExceptionally(e); - return null; - } - }).filter(Objects::nonNull).collect(Collectors.toList()); - if (f.isCompletedExceptionally()) { - try { - f.get(); - } catch (ExecutionException | InterruptedException e) { - throw (IOException) e.getCause(); - } - } - return gbundles; - } + public static Collection getAllGraphs() throws IOException { + AtomicReference problem = new AtomicReference<>(); + + Collection gbundles = Arrays.stream(getBundles()) + .parallel() + .map(b -> { + try { + return problem.get() == null ? getGraph(b) : null; + } catch (IOException e) { + if (LOGGER.isDebugEnabled()) + LOGGER.debug("Could not get graph from bundle {}", b, e); + problem.set(e); + return null; + } + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + if (problem.get() != null) + throw problem.get(); + return gbundles; + } /** * Get bundle @@ -350,7 +346,7 @@ public class PlatformUtil { if (bundle == null) return null; return getGraph( bundle ); } - + /** * Read the graph in a graph bundle. Graph is read from "graph.tg" file in the root. * @@ -360,49 +356,126 @@ public class PlatformUtil { */ public static GraphBundleEx getGraph(Bundle bundle) throws IOException { URL url = bundle.getEntry("graph.tg"); - - if (url==null) return null; - InputStream is = url.openStream(); - // NOTE: this is vital for performance. - is = new BufferedInputStream(is, 128*1024); + if (url == null) + return null; + GraphBundleEx result = tryGetOnDemandGraph(bundle, url); + return result != null ? result : getCompleteGraph(bundle, url); + } + + private static GraphBundleEx getCompleteGraph(Bundle bundle, URL url) throws IOException { try { - DataInput dis = new DataInputStream(is); - // or - // dis = new InputStreamReadable(is, ) to protect from OOM - - org.simantics.databoard.container.DataContainer container = - DataContainers.readFile(dis); - - Binding binding = TransferableGraph1.BINDING; - TransferableGraph1 graph = (TransferableGraph1)container.content.getValue(binding); -// TransferableGraph1 graph = (TransferableGraph1) Files.readFile(is, binding); -// System.out.println("getGraph(" + bundle.getSymbolicName() + "): read transferable graph in " + (System.nanoTime()-start)*1e-6 + "ms"); - org.osgi.framework.Version osgiVersion = bundle.getVersion(); - Version p2Version = Version.createOSGi(osgiVersion.getMajor(), osgiVersion.getMinor(), osgiVersion.getMicro(), osgiVersion.getQualifier()); String id = bundle.getSymbolicName(); - VersionedId vid = new VersionedId(id, p2Version); - String name = (String) bundle.getHeaders().get("Bundle-Name"); - if (name == null) name = id; - String immutable = (String) bundle.getHeaders().get("Immutable"); - boolean isImmutable = - immutable != null ? - "true".equals(immutable) : - true; - -// System.out.println("getGraph(" + bundle.getSymbolicName() + "): before hashcode calculation in " + (System.nanoTime()-start)*1e-6 + "ms"); - GraphBundleEx entry = new GraphBundleEx(name, graph, vid, isImmutable); -// System.out.println("getGraph(" + bundle.getSymbolicName() + "): completed in " + (System.nanoTime()-start)*1e-6 + "ms"); - return entry; + return new GraphBundleEx( + getBundleName(bundle, id), + readTG(url), + new VersionedId(id, toP2Version(bundle)), + isImmutable(bundle)); } catch (Exception e) { throw new IOException("Problem loading graph.tg from bundle " + bundle.getSymbolicName(), e); } catch (Error e) { LOGGER.error("Serious problem loading graph.tg from bundle " + bundle.getSymbolicName(), e); throw e; - } finally { - is.close(); } } + /** + * Read the graph in a graph bundle. Graph is read from "graph.tg" file in the root. + * + * @param bundle + * @return transferable graph, or null if there is no graph in the bundle. + * @throws IOException + */ + private static GraphBundleEx tryGetOnDemandGraph(Bundle bundle, URL url) throws IOException { + try { + Integer cachedHash = readCachedHash(url); + System.out.println("Read cached hashcode from " + bundle + ": " + cachedHash); + if (cachedHash == null) +// if (true) + return null; + + Supplier graphSource = () -> { + try { + return readTG(url); + } catch (Exception e) { + throw new RuntimeException("Problem loading graph.tg from bundle " + bundle.getSymbolicName(), e); + } catch (Error e) { + LOGGER.error("Serious problem loading graph.tg from bundle " + bundle.getSymbolicName(), e); + throw e; + } + }; + + String id = bundle.getSymbolicName(); + + return new GraphBundleEx( + getBundleName(bundle, id), + graphSource, + cachedHash, + new VersionedId(id, toP2Version(bundle)), + isImmutable(bundle)); + } catch (Exception e) { + throw new IOException("Problem loading graph.tg from bundle " + bundle.getSymbolicName(), e); + } + } + + @SuppressWarnings("unchecked") + private static Map> handlers = ArrayMap.make( + new String[] { + "graph:1" + }, + new FormatHandler() { + @Override + public Binding getBinding() { + return TransferableGraph1.BINDING; + } + @Override + public TransferableGraph1 process(DataContainer container) throws Exception { + return (TransferableGraph1) container.content.getValue(TransferableGraph1.BINDING); + } + }); + + private static TransferableGraph1 readTG(InputStream is) throws Exception { + // For an unknown reason this is totally broken when running the TestSCLOsgi + // in the SDK Tycho build. It returns incomplete results because the + // ReadableByteChannel used by ByteFileReader starts returning 0 unexpectedly. +// try (TransferableGraphFileReader reader = new TransferableGraphFileReader(is)) { +// return reader.readTG(); +// } + return DataContainers.readFile(new DataInputStream(is), handlers); + } + + private static TransferableGraph1 readTG(URL url) throws Exception { + try (InputStream is = url.openStream()) { + return readTG(is); + } + } + + private static DataContainer readHeader(URL url) throws IOException { + try (InputStream is = url.openStream()) { + return DataContainers.readHeader(new DataInputStream(new BufferedInputStream(is, 1 << 14))); + } + } + + private static Integer readCachedHash(URL url) throws IOException, AdaptException { + DataContainer header = readHeader(url); + Variant hashCode = header.metadata.get(Extensions.CACHED_HASHCODE); + return hashCode != null ? (Integer) hashCode.getValue(Bindings.INTEGER) : null; + } + + private static Version toP2Version(Bundle bundle) { + org.osgi.framework.Version osgiVersion = bundle.getVersion(); + return Version.createOSGi(osgiVersion.getMajor(), osgiVersion.getMinor(), osgiVersion.getMicro(), osgiVersion.getQualifier()); + } + + private static String getBundleName(Bundle bundle, String id) { + String name = (String) bundle.getHeaders().get("Bundle-Name"); + return name != null ? name : id; + } + + private static boolean isImmutable(Bundle bundle) { + String immutable = (String) bundle.getHeaders().get("Immutable"); + return immutable != null ? "true".equals(immutable) : true; + } + public static class TGInfo { public Bundle bundle; public URL location; diff --git a/bundles/org.simantics.utils/src/org/simantics/utils/FileUtils.java b/bundles/org.simantics.utils/src/org/simantics/utils/FileUtils.java index 3e54e19be..6ae209c9a 100644 --- a/bundles/org.simantics.utils/src/org/simantics/utils/FileUtils.java +++ b/bundles/org.simantics.utils/src/org/simantics/utils/FileUtils.java @@ -828,9 +828,9 @@ public class FileUtils { } } - private static void copy(File file, ZipOutputStream zout) throws IOException { + public static void copy(File file, OutputStream out) throws IOException { try (InputStream in = new FileInputStream(file)) { - copy(in, zout); + copy(in, out); } } diff --git a/bundles/org.simantics/plugin.xml b/bundles/org.simantics/plugin.xml index 3f881e6d0..c526fd8e2 100644 --- a/bundles/org.simantics/plugin.xml +++ b/bundles/org.simantics/plugin.xml @@ -32,5 +32,18 @@ + + + + + + diff --git a/bundles/org.simantics/src/org/simantics/BaselineCreatorApplication.java b/bundles/org.simantics/src/org/simantics/BaselineCreatorApplication.java new file mode 100644 index 000000000..d0b8e6949 --- /dev/null +++ b/bundles/org.simantics/src/org/simantics/BaselineCreatorApplication.java @@ -0,0 +1,113 @@ +package org.simantics; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; + +import org.eclipse.core.runtime.CoreException; +import org.eclipse.core.runtime.IProgressMonitor; +import org.eclipse.core.runtime.IStatus; +import org.eclipse.core.runtime.NullProgressMonitor; +import org.eclipse.core.runtime.Platform; +import org.eclipse.core.runtime.Status; +import org.eclipse.equinox.app.IApplication; +import org.eclipse.equinox.app.IApplicationContext; +import org.eclipse.osgi.service.datalocation.Location; +import org.simantics.application.arguments.Arguments; +import org.simantics.application.arguments.IArgumentFactory; +import org.simantics.application.arguments.IArgumentFactory.StringArgumentFactory; +import org.simantics.application.arguments.IArgumentFactory.NoValueArgumentFactory; +import org.simantics.application.arguments.IArguments; +import org.simantics.application.arguments.SimanticsArguments; +import org.simantics.internal.Activator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author Tuukka Lehtonen + * @since 1.34.0 + */ +public class BaselineCreatorApplication implements IApplication { + + private static final Logger LOGGER = LoggerFactory.getLogger(BaselineCreatorApplication.class); + + private static final IArgumentFactory OUTPUT = new StringArgumentFactory("-o"); + private static final IArgumentFactory VERBOSE = new NoValueArgumentFactory("-v"); + + IArgumentFactory[] accepted = { + SimanticsArguments.RECOVERY_POLICY_FIX_ERRORS, + SimanticsArguments.ONTOLOGY_RECOVERY_POLICY_REINSTALL, + SimanticsArguments.DISABLE_INDEX, + SimanticsArguments.DATABASE_ID, + OUTPUT, + VERBOSE, + }; + + private static String currentLocalDateTimeStamp() { + return LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd_HHmm")); + } + + private static Path constructOutputPath(Path workspace, IArguments parsedArgs) { + if (parsedArgs.contains(OUTPUT)) { + return workspace.resolve(parsedArgs.get(OUTPUT)); + } else { + return workspace.resolve(workspace.getFileName().toString() + "-" + currentLocalDateTimeStamp() + ".zip"); + } + } + + private static Path getInstanceLocation() throws CoreException, IOException { + Location l = Platform.getInstanceLocation(); + if (l == null) + throw new CoreException(new Status(IStatus.ERROR, Activator.PLUGIN_ID, + "Workspace not defined. Use -data argument to define where to place the baselining workspace.")); + + Location instanceLoc = Platform.getInstanceLocation(); + if (instanceLoc == null || instanceLoc.isReadOnly()) + throw new CoreException(new Status(IStatus.ERROR, Activator.PLUGIN_ID, + "Workspace not defined. Use -data argument to define where to place the baselining workspace.")); + + URL workspaceUrl = l.getURL(); + Path workspacePath = new File(workspaceUrl.getPath()).toPath(); + Files.createDirectories(workspacePath); + return workspacePath; + } + + @Override + public Object start(IApplicationContext context) throws Exception { + try { + Path workspace = getInstanceLocation(); + + String[] args = (String[]) context.getArguments().get("application.args"); + IArguments parsedArgs = Arguments.parse(args, accepted); + + Path output = constructOutputPath(workspace, parsedArgs); + + // Create database and indexes + IProgressMonitor progress = parsedArgs.contains(VERBOSE) + ? new TimingProgressMonitor() + : new NullProgressMonitor(); + Simantics.startUpHeadless(parsedArgs, progress); + Simantics.shutdown(progress); + + // Create the baseline package file + Path actualOutput = DatabaseBaselines.packageBaseline(workspace, output); + System.out.println("OK " + actualOutput.toAbsolutePath()); + + return IApplication.EXIT_OK; + } catch (Exception e) { + LOGGER.error("Baseline creation failed.", e); + throw (Exception) e; + } finally { + System.exit(0); + } + } + + @Override + public void stop() { + } + +} diff --git a/bundles/org.simantics/src/org/simantics/DatabaseBaselines.java b/bundles/org.simantics/src/org/simantics/DatabaseBaselines.java new file mode 100644 index 000000000..96db5ed72 --- /dev/null +++ b/bundles/org.simantics/src/org/simantics/DatabaseBaselines.java @@ -0,0 +1,130 @@ +package org.simantics; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.stream.Collectors; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import java.util.zip.ZipOutputStream; + +import org.simantics.utils.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author Tuukka Lehtonen + * @since 1.34.0 + */ +public class DatabaseBaselines { + + private static final Logger LOGGER = LoggerFactory.getLogger(DatabaseBaselines.class); + + private static final boolean REQUIRE_INDEX_IN_BASELINE = false; + + private static final String DB_DIRECTORY = "db"; //$NON-NLS-1$ + private static final String INDEX_DIRECTORY = ".metadata/.plugins/org.simantics.db.indexing"; //$NON-NLS-1$ + + private static final DateTimeFormatter TIMESTAMP_FORMAT = DateTimeFormatter.ofPattern("d. MMM yyyy HH:mm:ss"); + + public static Path packageBaseline(Path fromWorkspace, Path packageFile) throws IOException { + return compressZip(fromWorkspace, collectBaselinePaths(fromWorkspace), packageFile); + } + + private static List collectBaselinePaths(Path workspace) throws IOException { + Path dbPath = workspace.resolve(DB_DIRECTORY); + Path indexPath = workspace.resolve(INDEX_DIRECTORY); + + if (!Files.isDirectory(dbPath)) + throw new IllegalArgumentException("workspace database directory " + dbPath + " does not exist"); + + List paths = Files.walk(dbPath).collect(Collectors.toList()); + if (Files.isDirectory(indexPath)) { + List indexPaths = Files.walk(indexPath).collect(Collectors.toList()); + paths.addAll(indexPaths); + } else { + if (REQUIRE_INDEX_IN_BASELINE) + throw new IllegalArgumentException("workspace database index directory " + indexPath + " does not exist"); + } + return paths; + } + + private static Path compressZip(Path relativeRoot, List paths, Path zipFile) throws IOException { + if (LOGGER.isDebugEnabled()) + LOGGER.debug("Compressing " + paths.size() + " path entries into ZIP file " + zipFile); + try (ZipOutputStream zout = new ZipOutputStream(Files.newOutputStream(zipFile))) { + compressZip(relativeRoot, zout, paths); + return zipFile; + } finally { + if (LOGGER.isDebugEnabled()) + LOGGER.debug("Compressed " + paths.size() + " entries into " + zipFile); + } + } + + private static void compressZip(Path relativeRoot, ZipOutputStream zout, List paths) throws IOException { + for (Path p : paths) { + Path rp = relativeRoot.relativize(p); + String name = rp.toString(); + if (Files.isDirectory(p)) { + name = name.endsWith("/") ? name : name + "/"; + zout.putNextEntry(new ZipEntry(name)); + } else { + zout.putNextEntry(new ZipEntry(name)); + FileUtils.copy(p.toFile(), zout); + zout.closeEntry(); + } + } + } + + public static byte[] baselineIndicatorContents(Path path) throws IOException { + return String.format("%s%n%s%n", + path.toString(), + Instant.now().atZone(ZoneId.systemDefault()).format(TIMESTAMP_FORMAT)) + .getBytes("UTF-8"); + } + + public static void validateWorkspaceForBaselineInitialization(Path workspaceLocation) throws PlatformException { + try { + Path db = workspaceLocation.resolve(DB_DIRECTORY); + if (Files.exists(db)) + throw new PlatformException("Database location " + db + " already exists. Cannot re-initialize workspace from baseline."); + if (REQUIRE_INDEX_IN_BASELINE) { + Path index = workspaceLocation.resolve(INDEX_DIRECTORY); + if (!Files.exists(index) || !isEmptyDirectory(index)) + throw new PlatformException("Index location " + index + " already exists. Cannot re-initialize workspace from baseline."); + } + } catch (IOException e) { + throw new PlatformException("Failed to validate workspace for baseline initialization", e); + } + } + + private static boolean isEmptyDirectory(Path dir) throws IOException { + return Files.walk(dir).count() == 1; + } + + public static void validateBaselineFile(Path baseline) throws PlatformException { + try (ZipFile zip = new ZipFile(baseline.toFile())) { + ZipEntry db = zip.getEntry(DB_DIRECTORY); + if (db == null) + throw new PlatformException("Baseline archive does not contain database directory '" + DB_DIRECTORY + "'"); + + if (REQUIRE_INDEX_IN_BASELINE) { + ZipEntry index = zip.getEntry(INDEX_DIRECTORY); + if (index == null) + throw new PlatformException("Baseline archive does not contain database index directory '" + INDEX_DIRECTORY + "'"); + } + } catch (IOException e) { + throw new PlatformException("Failed to validate baseline archive " + baseline, e); + } + } + + public static void main(String[] args) throws IOException { + packageBaseline(Paths.get("D:/temp/desktop/workspace"), Paths.get("d:/temp/desktop/workspace/baseline.zip")); + } + +} diff --git a/bundles/org.simantics/src/org/simantics/SimanticsPlatform.java b/bundles/org.simantics/src/org/simantics/SimanticsPlatform.java index e529938e2..1cedd4fcb 100644 --- a/bundles/org.simantics/src/org/simantics/SimanticsPlatform.java +++ b/bundles/org.simantics/src/org/simantics/SimanticsPlatform.java @@ -22,9 +22,6 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.time.Instant; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -35,8 +32,6 @@ import java.util.Properties; import java.util.Set; import java.util.TreeMap; import java.util.UUID; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; import org.eclipse.core.runtime.ILog; import org.eclipse.core.runtime.IProduct; @@ -271,23 +266,24 @@ public class SimanticsPlatform implements LifecycleListener { public void synchronizeOntologies(IProgressMonitor progressMonitor, OntologyRecoveryPolicy ontologyPolicy, boolean requireSynchronize) throws PlatformException { - if (progressMonitor == null) progressMonitor = new NullProgressMonitor(); - - final DatabaseManagement mgmt = new DatabaseManagement(); + SubMonitor monitor = SubMonitor.convert(progressMonitor, 100); + monitor.setTaskName("Compile dynamic ontologies"); PlatformUtil.compileAllDynamicOntologies(); String message = "Asserting all ontologies are installed"; LOGGER.info(message); - progressMonitor.setTaskName(message); - final Map platformTGs = new HashMap(); + monitor.setTaskName(message); + + DatabaseManagement mgmt = new DatabaseManagement(); + Map platformTGs = new HashMap<>(); try { // Get a list of bundles installed into the database message = "find installed bundles from database"; - progressMonitor.subTask(message); + monitor.subTask(message); LOGGER.info(message); - Map installedTGs = new HashMap(); + Map installedTGs = new HashMap<>(); for (GraphBundle b : session.syncRequest( mgmt.GraphBundleQuery )) { installedTGs.put(GraphBundleRef.of(b), GraphBundleEx.extend(b)); } @@ -297,11 +293,11 @@ public class SimanticsPlatform implements LifecycleListener { // Get a list of all bundles in the platform (Bundle Context) message = "load all transferable graphs from platform"; - progressMonitor.subTask(message); + monitor.subTask(message); LOGGER.info(message); Collection tgs = PlatformUtil.getAllGraphs(); message = "extend bundles to compile versions"; - progressMonitor.subTask(message); + monitor.subTask(message); LOGGER.info(message); for (GraphBundle b : tgs) { GraphBundleEx gbe = GraphBundleEx.extend(b); @@ -311,11 +307,11 @@ public class SimanticsPlatform implements LifecycleListener { // Compile a list of TGs that need to be installed or reinstalled in the database message = "check bundle reinstallation demand"; - progressMonitor.subTask(message); + monitor.subTask(message); LOGGER.info(message); - List installTGs = new ArrayList(); + List installTGs = new ArrayList<>(); // Create list of TGs to update, - Map reinstallTGs = new TreeMap(); + Map reinstallTGs = new TreeMap<>(); for (Entry e : platformTGs.entrySet()) { GraphBundleRef key = e.getKey(); GraphBundleEx platformBundle = e.getValue(); @@ -386,7 +382,7 @@ public class SimanticsPlatform implements LifecycleListener { if (ontologyPolicy == OntologyRecoveryPolicy.Merge) { message = "Merging ontology changes"; - progressMonitor.subTask(message); + monitor.subTask(message); LOGGER.info(message); // Sort missing TGs into install order GraphDependencyAnalyzer analyzer = new GraphDependencyAnalyzer(); @@ -463,11 +459,9 @@ public class SimanticsPlatform implements LifecycleListener { log.log(new Status(IStatus.INFO, Activator.PLUGIN_ID, "Merging new version of "+tg.toString())); startTransaction(session, true); - + //delta.print(); try { - - long[] resourceArray = TransferableGraphs.applyDelta(writeGraph(), oldResources, delta); tg.setResourceArray(resourceArray); mgmt.setGraphBundleEntry(tg); @@ -489,13 +483,10 @@ public class SimanticsPlatform implements LifecycleListener { if (mergedOntologies) DatabaseIndexing.deleteAllIndexes(); } - - TimeLogger.log("Ontologies synchronized."); - } session.getService(XSupport.class).setServiceMode(false, false); } - progressMonitor.worked(20); + monitor.worked(100); } catch (IOException e) { throw new PlatformException(e); } catch (DatabaseException e) { @@ -722,58 +713,19 @@ public class SimanticsPlatform implements LifecycleListener { if (!Files.isRegularFile(baseline)) throw new PlatformException("Specified database baseline archive " + baseline + " does not exist. Cannot initialize workspace database."); - validateBaselineFile(baseline); - validateWorkspaceForBaselineInitialization(workspaceLocation); + DatabaseBaselines.validateBaselineFile(baseline); + DatabaseBaselines.validateWorkspaceForBaselineInitialization(workspaceLocation); try { Files.createDirectories(workspaceLocation); FileUtils.extractZip(baseline.toFile(), workspaceLocation.toFile()); - Files.write(baselineIndicatorFile, baselineIndicatorContents(baselineIndicatorFile)); + Files.write(baselineIndicatorFile, DatabaseBaselines.baselineIndicatorContents(baselineIndicatorFile)); return true; } catch (IOException e) { throw new PlatformException(e); } } - private static final DateTimeFormatter TIMESTAMP_FORMAT = DateTimeFormatter.ofPattern("d. MMM yyyy HH:mm:ss"); - - private static byte[] baselineIndicatorContents(Path path) throws IOException { - return String.format("%s%n%s%n", - path.toString(), - Instant.now().atZone(ZoneId.systemDefault()).format(TIMESTAMP_FORMAT)) - .getBytes("UTF-8"); - } - - private void validateWorkspaceForBaselineInitialization(Path workspaceLocation) throws PlatformException { - try { - Path db = workspaceLocation.resolve("db"); - if (Files.exists(db)) - throw new PlatformException("Database location " + db + " already exists. Cannot re-initialize workspace from baseline."); - Path index = workspaceLocation.resolve(".metadata/.plugins/org.simantics.db.indexing"); - if (!Files.exists(index) || !isEmptyDirectory(index)) - throw new PlatformException("Index location " + index + " already exists. Cannot re-initialize workspace from baseline."); - } catch (IOException e) { - throw new PlatformException("Failed to validate workspace for baseline initialization", e); - } - } - - private static boolean isEmptyDirectory(Path dir) throws IOException { - return Files.walk(dir).count() == 1; - } - - private void validateBaselineFile(Path baseline) throws PlatformException { - try (ZipFile zip = new ZipFile(baseline.toFile())) { - ZipEntry db = zip.getEntry("db"); - if (db == null) - throw new PlatformException("Baseline archive does not contain database directory 'db'"); - ZipEntry index = zip.getEntry(".metadata/.plugins/org.simantics.db.indexing"); - if (index == null) - throw new PlatformException("Baseline archive does not contain database index directory '.metadata/.plugins/org.simantics.db.indexing'"); - } catch (IOException e) { - throw new PlatformException("Failed to validate baseline archive " + baseline, e); - } - } - /** * Start-up the platform. The procedure consists of 8 steps. Once everything * is up and running, all fields are set property. @@ -826,17 +778,18 @@ public class SimanticsPlatform implements LifecycleListener { // 0.2 Clear VariableRepository.repository static map which holds references to SessionImplDb VariableRepository.clear(); - + // 0.3 Handle baseline database before opening db + @SuppressWarnings("unused") boolean usingBaseline = handleBaselineDatabase(); - + // 1. Assert there is a database at /db SessionDescriptor sessionDescriptor = setupDatabase(databaseDriverId, monitor.newChild(200, SubMonitor.SUPPRESS_NONE), workspacePolicy, userAgent); session = sessionDescriptor.getSession(); TimeLogger.log("Database setup complete"); - // 1.1 Delete all indexes if we cannot be certain they are up-to-date - // A full index rebuild will be done later, before project activation. + // 2. Delete all indexes if we cannot be certain they are up-to-date + // A full index rebuild will be done later, before project activation. XSupport support = session.getService(XSupport.class); if (support.rolledback()) { try { @@ -845,12 +798,10 @@ public class SimanticsPlatform implements LifecycleListener { throw new PlatformException(e); } } - - // 2. Assert all graphs, and correct versions, are installed to the database - if(!usingBaseline) { - synchronizeOntologies(monitor.newChild(400, SubMonitor.SUPPRESS_NONE), ontologyPolicy, requireSynchronize); - TimeLogger.log("Synchronized ontologies"); - } + + // 3. Assert all graphs, and correct versions, are installed to the database + synchronizeOntologies(monitor.newChild(400, SubMonitor.SUPPRESS_NONE), ontologyPolicy, requireSynchronize); + TimeLogger.log("Synchronized ontologies"); // 4. Assert simantics.cfg exists boolean installProject = assertConfiguration(monitor.newChild(25, SubMonitor.SUPPRESS_NONE),workspacePolicy); @@ -916,16 +867,17 @@ public class SimanticsPlatform implements LifecycleListener { } if(loadProject) { - TimeLogger.log("Load projects"); + TimeLogger.log("Load project"); + monitor.setTaskName("Load project"); project = Projects.loadProject(sessionContext.getSession(), SimanticsPlatform.INSTANCE.projectResource); - monitor.worked(100); - sessionContext.setHint(ProjectKeys.KEY_PROJECT, project); + monitor.worked(100); TimeLogger.log("Loading projects complete"); + monitor.setTaskName("Activate project"); project.activate(); - TimeLogger.log("Project activated"); monitor.worked(100); + TimeLogger.log("Project activated"); } } catch (DatabaseException e) { diff --git a/bundles/pom.xml b/bundles/pom.xml index 6bcbd6657..bcefad659 100644 --- a/bundles/pom.xml +++ b/bundles/pom.xml @@ -18,7 +18,7 @@ org.simantics graph-builder-maven-plugin - 0.0.7 + 0.0.9 compile-graphs -- 2.43.2