X-Git-Url: https://gerrit.simantics.org/r/gitweb?p=simantics%2Fplatform.git;a=blobdiff_plain;f=bundles%2Forg.simantics.project%2Fsrc%2Forg%2Fsimantics%2Fproject%2Fmanagement%2FPlatformUtil.java;h=fde15003643668c33548125195e17b199874a6cb;hp=ebd987e3f2b5238e38ba4ace5ce1944a9eea6e55;hb=b913419ca9037bf9734c56a5f079024c3a1cd177;hpb=a8d030f7db1f59a5b51cdc34f18de7c0a0ee8549 diff --git a/bundles/org.simantics.project/src/org/simantics/project/management/PlatformUtil.java b/bundles/org.simantics.project/src/org/simantics/project/management/PlatformUtil.java index ebd987e3f..fde150036 100644 --- a/bundles/org.simantics.project/src/org/simantics/project/management/PlatformUtil.java +++ b/bundles/org.simantics.project/src/org/simantics/project/management/PlatformUtil.java @@ -13,7 +13,6 @@ package org.simantics.project.management; import java.io.BufferedInputStream; import java.io.Closeable; -import java.io.DataInput; import java.io.DataInputStream; import java.io.File; import java.io.FileNotFoundException; @@ -28,10 +27,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Enumeration; +import java.util.Map; import java.util.Map.Entry; import java.util.Objects; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; import java.util.jar.Attributes; import java.util.jar.Manifest; import java.util.stream.Collectors; @@ -43,19 +43,24 @@ import org.eclipse.equinox.p2.metadata.IVersionedId; import org.eclipse.equinox.p2.metadata.Version; import org.eclipse.equinox.p2.metadata.VersionedId; import org.osgi.framework.Bundle; +import org.simantics.databoard.Bindings; +import org.simantics.databoard.adapter.AdaptException; import org.simantics.databoard.binding.Binding; import org.simantics.databoard.binding.mutable.Variant; import org.simantics.databoard.container.DataContainer; import org.simantics.databoard.container.DataContainers; +import org.simantics.databoard.container.FormatHandler; import org.simantics.graph.compiler.CompilationResult; import org.simantics.graph.compiler.GraphCompiler; import org.simantics.graph.compiler.GraphCompilerPreferences; import org.simantics.graph.compiler.ValidationMode; +import org.simantics.graph.representation.Extensions; import org.simantics.graph.representation.TransferableGraph1; import org.simantics.ltk.FileSource; import org.simantics.ltk.ISource; import org.simantics.ltk.Problem; import org.simantics.scl.reflection.OntologyVersions; +import org.simantics.utils.datastructures.ArrayMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -76,7 +81,7 @@ public class PlatformUtil { public static Bundle[] getBundles() { return PlatformActivator.getContext().getBundles(); } - + /** * Get the manifest file of a bundle * @@ -87,14 +92,11 @@ public class PlatformUtil { public static Manifest getManifest(Bundle bundle) throws IOException { URL url = bundle.getEntry("META-INF/MANIFEST.MF"); if (url==null) return null; - InputStream is = url.openStream(); - try { - return new Manifest(is); - } finally { - is.close(); + try (InputStream is = url.openStream()) { + return new Manifest(is); } } - + /** * Get the manifest file of a bundle * @@ -105,11 +107,8 @@ public class PlatformUtil { public static Manifest getSimanticsManifest(Bundle bundle) throws IOException { URL url = bundle.getEntry("META-INF/SIMANTICS.MF"); if (url==null) return null; - InputStream is = url.openStream(); - try { - return new Manifest(is); - } finally { - is.close(); + try (InputStream is = url.openStream()) { + return new Manifest(is); } } @@ -131,13 +130,13 @@ public class PlatformUtil { for (Entry entry2 : attributes.entrySet()) { Object key = entry2.getKey(); if (key.toString().contains("Installable-Unit")) { - String bid = entry2.getValue().toString(); + String bid = entry2.getValue().toString(); list.add( bid ); } } - } + } } - + /** * Get all transferable graphs in the platform * @@ -151,7 +150,7 @@ public class PlatformUtil { org.osgi.framework.Version osgiVersion = bundle.getVersion(); Version p2Version = Version.createOSGi(osgiVersion.getMajor(), osgiVersion.getMinor(), osgiVersion.getMicro(), osgiVersion.getQualifier()); String id = bundle.getSymbolicName(); - + TGInfo info = new TGInfo(); info.location = e.nextElement(); info.bundle = bundle; @@ -169,7 +168,7 @@ public class PlatformUtil { //ignore } } - + private static File copyResource(URL url, File targetFile) throws IOException, FileNotFoundException { FileOutputStream os = null; InputStream is = null; @@ -193,7 +192,7 @@ public class PlatformUtil { uncheckedClose(is); } } - + private static File extractLib(URL libURL, String libName) throws FileNotFoundException, IOException { String tmpDirStr = System.getProperty("java.io.tmpdir"); if (tmpDirStr == null) @@ -202,7 +201,7 @@ public class PlatformUtil { File libFile = new File(tmpDir, libName); return copyResource(libURL, libFile); } - + private static File url2file(URL url, String fileName) { if ("file".equals(url.getProtocol())) { try { @@ -225,12 +224,12 @@ public class PlatformUtil { } return null; } - + public static void compile(Bundle b) throws Exception { - - Collection sources = new ArrayList(); - Collection dependencies = new ArrayList(); - + + Collection sources = new ArrayList<>(); + Collection dependencies = new ArrayList<>(); + for (Bundle b2 : getBundles()) { if(b.equals(b2)) continue; URL url = b2.getEntry("graph.tg"); @@ -238,24 +237,24 @@ public class PlatformUtil { File graphFile = url2file(FileLocator.resolve(b2.getEntry("/graph.tg")), b2.toString()); dependencies.add(GraphCompiler.read(graphFile)); } - + File bundleFile = FileLocator.getBundleFile(b); if(bundleFile.isDirectory()) { File folder = new File(bundleFile, "dynamicGraph"); for(File f : folder.listFiles(new FilenameFilter() { - + @Override public boolean accept(File dir, String name) { return name.endsWith(".pgraph"); } - + })) { sources.add(new FileSource(f)); } - } - + } + // System.out.println("source is " + tmpFile.getAbsolutePath()); - + final StringBuilder errorStringBuilder = new StringBuilder(); GraphCompilerPreferences prefs = new GraphCompilerPreferences(); prefs.validate = true; @@ -263,7 +262,7 @@ public class PlatformUtil { prefs.validateResourceHasType = ValidationMode.ERROR; String currentLayer0Version = OntologyVersions.getInstance().currentOntologyVersion("http://www.simantics.org/Layer0-0.0"); CompilationResult result = GraphCompiler.compile(currentLayer0Version, sources, dependencies, null, prefs); - + for(Problem problem : result.getErrors()) errorStringBuilder.append(problem.getLocation() + ": " + problem.getDescription() + "\n"); for(Problem problem : result.getWarnings()) @@ -272,12 +271,11 @@ public class PlatformUtil { if(errorStringBuilder.length() > 0) { LOGGER.error(errorStringBuilder.toString()); } else { - DataContainers.writeFile(new File(bundleFile, "graph.tg"), - new DataContainer("graph", 1, new Variant(TransferableGraph1.BINDING, result.getGraph()))); + GraphCompiler.write(new File(bundleFile, "graph.tg"), result.getGraph()); } - + } - + /** * Compile all dynamic ontologies in the Platform * @@ -306,37 +304,35 @@ public class PlatformUtil { } } } - + /** * Get all graphs in the Platform * * @param collection * @throws IOException */ - public static Collection getAllGraphs() throws IOException { - CompletableFuture f = new CompletableFuture<>(); - Bundle[] bundles = getBundles(); - Collection gbundles = Arrays.stream(bundles).map(t -> { // this could be done in parallel in the future? - if (f.isCompletedExceptionally()) - return null; - try { - return PlatformUtil.getGraph(t); - } catch (IOException e) { - if (LOGGER.isDebugEnabled()) - LOGGER.debug("Could not get graph {}", t, e); - f.completeExceptionally(e); - return null; - } - }).filter(Objects::nonNull).collect(Collectors.toList()); - if (f.isCompletedExceptionally()) { - try { - f.get(); - } catch (ExecutionException | InterruptedException e) { - throw (IOException) e.getCause(); - } - } - return gbundles; - } + public static Collection getAllGraphs() throws IOException { + AtomicReference problem = new AtomicReference<>(); + + Collection gbundles = Arrays.stream(getBundles()) + .parallel() + .map(b -> { + try { + return problem.get() == null ? getGraph(b) : null; + } catch (IOException e) { + if (LOGGER.isDebugEnabled()) + LOGGER.debug("Could not get graph from bundle {}", b, e); + problem.set(e); + return null; + } + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + if (problem.get() != null) + throw problem.get(); + return gbundles; + } /** * Get bundle @@ -350,7 +346,7 @@ public class PlatformUtil { if (bundle == null) return null; return getGraph( bundle ); } - + /** * Read the graph in a graph bundle. Graph is read from "graph.tg" file in the root. * @@ -360,49 +356,126 @@ public class PlatformUtil { */ public static GraphBundleEx getGraph(Bundle bundle) throws IOException { URL url = bundle.getEntry("graph.tg"); - - if (url==null) return null; - InputStream is = url.openStream(); - // NOTE: this is vital for performance. - is = new BufferedInputStream(is, 128*1024); + if (url == null) + return null; + GraphBundleEx result = tryGetOnDemandGraph(bundle, url); + return result != null ? result : getCompleteGraph(bundle, url); + } + + private static GraphBundleEx getCompleteGraph(Bundle bundle, URL url) throws IOException { try { - DataInput dis = new DataInputStream(is); - // or - // dis = new InputStreamReadable(is, ) to protect from OOM - - org.simantics.databoard.container.DataContainer container = - DataContainers.readFile(dis); - - Binding binding = TransferableGraph1.BINDING; - TransferableGraph1 graph = (TransferableGraph1)container.content.getValue(binding); -// TransferableGraph1 graph = (TransferableGraph1) Files.readFile(is, binding); -// System.out.println("getGraph(" + bundle.getSymbolicName() + "): read transferable graph in " + (System.nanoTime()-start)*1e-6 + "ms"); - org.osgi.framework.Version osgiVersion = bundle.getVersion(); - Version p2Version = Version.createOSGi(osgiVersion.getMajor(), osgiVersion.getMinor(), osgiVersion.getMicro(), osgiVersion.getQualifier()); String id = bundle.getSymbolicName(); - VersionedId vid = new VersionedId(id, p2Version); - String name = (String) bundle.getHeaders().get("Bundle-Name"); - if (name == null) name = id; - String immutable = (String) bundle.getHeaders().get("Immutable"); - boolean isImmutable = - immutable != null ? - "true".equals(immutable) : - true; - -// System.out.println("getGraph(" + bundle.getSymbolicName() + "): before hashcode calculation in " + (System.nanoTime()-start)*1e-6 + "ms"); - GraphBundleEx entry = new GraphBundleEx(name, graph, vid, isImmutable); -// System.out.println("getGraph(" + bundle.getSymbolicName() + "): completed in " + (System.nanoTime()-start)*1e-6 + "ms"); - return entry; + return new GraphBundleEx( + getBundleName(bundle, id), + readTG(url), + new VersionedId(id, toP2Version(bundle)), + isImmutable(bundle)); } catch (Exception e) { throw new IOException("Problem loading graph.tg from bundle " + bundle.getSymbolicName(), e); } catch (Error e) { LOGGER.error("Serious problem loading graph.tg from bundle " + bundle.getSymbolicName(), e); throw e; - } finally { - is.close(); } } + /** + * Read the graph in a graph bundle. Graph is read from "graph.tg" file in the root. + * + * @param bundle + * @return transferable graph, or null if there is no graph in the bundle. + * @throws IOException + */ + private static GraphBundleEx tryGetOnDemandGraph(Bundle bundle, URL url) throws IOException { + try { + Integer cachedHash = readCachedHash(url); + System.out.println("Read cached hashcode from " + bundle + ": " + cachedHash); + if (cachedHash == null) +// if (true) + return null; + + Supplier graphSource = () -> { + try { + return readTG(url); + } catch (Exception e) { + throw new RuntimeException("Problem loading graph.tg from bundle " + bundle.getSymbolicName(), e); + } catch (Error e) { + LOGGER.error("Serious problem loading graph.tg from bundle " + bundle.getSymbolicName(), e); + throw e; + } + }; + + String id = bundle.getSymbolicName(); + + return new GraphBundleEx( + getBundleName(bundle, id), + graphSource, + cachedHash, + new VersionedId(id, toP2Version(bundle)), + isImmutable(bundle)); + } catch (Exception e) { + throw new IOException("Problem loading graph.tg from bundle " + bundle.getSymbolicName(), e); + } + } + + @SuppressWarnings("unchecked") + private static Map> handlers = ArrayMap.make( + new String[] { + "graph:1" + }, + new FormatHandler() { + @Override + public Binding getBinding() { + return TransferableGraph1.BINDING; + } + @Override + public TransferableGraph1 process(DataContainer container) throws Exception { + return (TransferableGraph1) container.content.getValue(TransferableGraph1.BINDING); + } + }); + + private static TransferableGraph1 readTG(InputStream is) throws Exception { + // For an unknown reason this is totally broken when running the TestSCLOsgi + // in the SDK Tycho build. It returns incomplete results because the + // ReadableByteChannel used by ByteFileReader starts returning 0 unexpectedly. +// try (TransferableGraphFileReader reader = new TransferableGraphFileReader(is)) { +// return reader.readTG(); +// } + return DataContainers.readFile(new DataInputStream(is), handlers); + } + + private static TransferableGraph1 readTG(URL url) throws Exception { + try (InputStream is = url.openStream()) { + return readTG(is); + } + } + + private static DataContainer readHeader(URL url) throws IOException { + try (InputStream is = url.openStream()) { + return DataContainers.readHeader(new DataInputStream(new BufferedInputStream(is, 1 << 14))); + } + } + + private static Integer readCachedHash(URL url) throws IOException, AdaptException { + DataContainer header = readHeader(url); + Variant hashCode = header.metadata.get(Extensions.CACHED_HASHCODE); + return hashCode != null ? (Integer) hashCode.getValue(Bindings.INTEGER) : null; + } + + private static Version toP2Version(Bundle bundle) { + org.osgi.framework.Version osgiVersion = bundle.getVersion(); + return Version.createOSGi(osgiVersion.getMajor(), osgiVersion.getMinor(), osgiVersion.getMicro(), osgiVersion.getQualifier()); + } + + private static String getBundleName(Bundle bundle, String id) { + String name = (String) bundle.getHeaders().get("Bundle-Name"); + return name != null ? name : id; + } + + private static boolean isImmutable(Bundle bundle) { + String immutable = (String) bundle.getHeaders().get("Immutable"); + return immutable != null ? "true".equals(immutable) : true; + } + public static class TGInfo { public Bundle bundle; public URL location;