Support ontology install option trueWhenDeployed also during development
[simantics/platform.git] / bundles / org.simantics.project / src / org / simantics / project / management / PlatformUtil.java
1 /*******************************************************************************
2  * Copyright (c) 2007, 2010 Association for Decentralized Information Management
3  * in Industry THTH ry.
4  * All rights reserved. This program and the accompanying materials
5  * are made available under the terms of the Eclipse Public License v1.0
6  * which accompanies this distribution, and is available at
7  * http://www.eclipse.org/legal/epl-v10.html
8  *
9  * Contributors:
10  *     VTT Technical Research Centre of Finland - initial API and implementation
11  *******************************************************************************/
12 package org.simantics.project.management;
13
14 import java.io.BufferedInputStream;
15 import java.io.Closeable;
16 import java.io.DataInputStream;
17 import java.io.File;
18 import java.io.FileNotFoundException;
19 import java.io.FileOutputStream;
20 import java.io.FilenameFilter;
21 import java.io.IOException;
22 import java.io.InputStream;
23 import java.io.UnsupportedEncodingException;
24 import java.net.URL;
25 import java.net.URLDecoder;
26 import java.util.ArrayList;
27 import java.util.Arrays;
28 import java.util.Collection;
29 import java.util.Collections;
30 import java.util.Enumeration;
31 import java.util.Map.Entry;
32 import java.util.Objects;
33 import java.util.concurrent.atomic.AtomicReference;
34 import java.util.function.Supplier;
35 import java.util.jar.Attributes;
36 import java.util.jar.Manifest;
37 import java.util.stream.Collectors;
38
39 import org.eclipse.core.internal.runtime.PlatformActivator;
40 import org.eclipse.core.runtime.FileLocator;
41 import org.eclipse.core.runtime.Platform;
42 import org.eclipse.equinox.p2.metadata.IVersionedId;
43 import org.eclipse.equinox.p2.metadata.Version;
44 import org.eclipse.equinox.p2.metadata.VersionedId;
45 import org.osgi.framework.Bundle;
46 import org.simantics.databoard.Bindings;
47 import org.simantics.databoard.adapter.AdaptException;
48 import org.simantics.databoard.binding.mutable.Variant;
49 import org.simantics.databoard.container.DataContainer;
50 import org.simantics.databoard.container.DataContainers;
51 import org.simantics.graph.compiler.CompilationResult;
52 import org.simantics.graph.compiler.GraphCompiler;
53 import org.simantics.graph.compiler.GraphCompilerPreferences;
54 import org.simantics.graph.compiler.ValidationMode;
55 import org.simantics.graph.compiler.internal.ltk.FileSource;
56 import org.simantics.graph.compiler.internal.ltk.ISource;
57 import org.simantics.graph.compiler.internal.ltk.Problem;
58 import org.simantics.graph.representation.Extensions;
59 import org.simantics.graph.representation.TransferableGraph1;
60 import org.simantics.graph.representation.TransferableGraphFileReader;
61 import org.simantics.scl.reflection.OntologyVersions;
62 import org.slf4j.Logger;
63 import org.slf4j.LoggerFactory;
64
65 /**
66  * This class contains utilities for managing bundles in a active platform. 
67  *
68  */
69 @SuppressWarnings("restriction")
70 public class PlatformUtil {
71
72         private static final Logger LOGGER = LoggerFactory.getLogger(PlatformUtil.class);
73
74         /**
75          * Get all bundles in the platform.
76          * 
77          * @return
78          */
79         public static Bundle[] getBundles() {
80                 return PlatformActivator.getContext().getBundles();
81         }
82
83         /**
84          * Get the manifest file of a bundle
85          * 
86          * @param bundle bundle
87          * @return manifest or <tt>null</tt> if doesn't not exist
88          * @throws IOException 
89          */
90         public static Manifest getManifest(Bundle bundle) throws IOException {
91                 URL url = bundle.getEntry("META-INF/MANIFEST.MF");
92                 if (url==null) return null;
93                 try (InputStream is = url.openStream()) {
94                         return new Manifest(is);
95                 }
96         }
97
98         /**
99          * Get the manifest file of a bundle
100          * 
101          * @param bundle bundle
102          * @return manifest or <tt>null</tt> if doesn't not exist
103          * @throws IOException 
104          */
105         public static Manifest getSimanticsManifest(Bundle bundle) throws IOException {
106                 URL url = bundle.getEntry("META-INF/SIMANTICS.MF");
107                 if (url==null) return null;
108                 try (InputStream is = url.openStream()) {
109                         return new Manifest(is);
110                 }
111         }
112         
113         /**
114          * Get a list (BundleIds) of all user installable units. These are the 
115          * top-level items that are visible for the end-user. 
116          * The list is acquired from the bundles of the current application. 
117          * 
118          * @param list of simantics features URIs
119          * @throws IOException 
120          */
121         public static void getUserInstallableUnits(Collection<String> list) 
122         throws IOException 
123         {
124                 for (Bundle bundle : getBundles()) {
125                         Manifest manifest = getSimanticsManifest(bundle);
126                         if (manifest==null) continue;
127                         Attributes attributes = manifest.getMainAttributes();
128                         for (Entry<Object, Object> entry2 : attributes.entrySet()) {
129                                 Object key = entry2.getKey();
130                         if (key.toString().contains("Installable-Unit")) {
131                                 String bid = entry2.getValue().toString();
132                                 list.add( bid );
133                         }
134                         }
135                 }
136         }
137
138         /**
139          * Get all transferable graphs in the platform
140          * 
141          * @param result collection to be filled with transferable graph info 
142          */
143         public static void getPlatformTGInfos(Collection<TGInfo> result) {
144                 for (Bundle bundle : getBundles()) {
145                         Enumeration<URL> e = bundle.findEntries("graphs/", "*.tg", false);
146                         if (e==null) continue;
147                         while (e.hasMoreElements()) {
148                                 org.osgi.framework.Version osgiVersion = bundle.getVersion();
149                                 Version p2Version = Version.createOSGi(osgiVersion.getMajor(), osgiVersion.getMinor(), osgiVersion.getMicro(), osgiVersion.getQualifier());
150                                 String id = bundle.getSymbolicName();
151
152                                 TGInfo info = new TGInfo();
153                                 info.location = e.nextElement();
154                                 info.bundle = bundle;
155                                 info.vid = new VersionedId(id, p2Version);
156                                 result.add( info );
157                         }
158                 }
159         }
160
161     private static void uncheckedClose(Closeable closeable) {
162         try {
163             if (closeable != null)
164                 closeable.close();
165         } catch (IOException e) {
166             //ignore
167         }
168     }
169
170     private static File copyResource(URL url, File targetFile) throws IOException, FileNotFoundException {
171         FileOutputStream os = null;
172         InputStream is = null;
173         try {
174             if (targetFile.exists())
175                 targetFile.delete();
176
177             is = url.openStream();
178             int read;
179             byte [] buffer = new byte [16384];
180             os = new FileOutputStream (targetFile);
181             while ((read = is.read (buffer)) != -1) {
182                 os.write(buffer, 0, read);
183             }
184             os.close ();
185             is.close ();
186
187             return targetFile;
188         } finally {
189             uncheckedClose(os);
190             uncheckedClose(is);
191         }
192     }
193
194     private static File extractLib(URL libURL, String libName) throws FileNotFoundException, IOException {
195         String tmpDirStr = System.getProperty("java.io.tmpdir");
196         if (tmpDirStr == null)
197             throw new NullPointerException("java.io.tmpdir property is null");
198         File tmpDir = new File(tmpDirStr);
199         File libFile = new File(tmpDir, libName);
200         return copyResource(libURL, libFile);
201     }
202
203     private static File url2file(URL url, String fileName) {
204                 if ("file".equals(url.getProtocol())) {
205                         try {
206                                 File path = new File(URLDecoder.decode(url.getPath(), "UTF-8"));
207                                 return path;
208                         } catch (UnsupportedEncodingException e) {
209                                 LOGGER.error("Failed to decode " + url, e);
210                         }
211                 } else if ("jar".equals(url.getProtocol())) {
212                         try {
213                                 File libFile = extractLib(url, fileName);
214                                 return libFile;
215                         } catch (FileNotFoundException e) {
216                                 LOGGER.error("Extraction to " + fileName + " failed, url not found: " + url, e);
217                         } catch (IOException e) {
218                                 LOGGER.error("Extraction to " + fileName + " failed from url " + url, e);
219                         }
220                 } else {
221                         LOGGER.error("Unsupported URL protocol '" + url + "' for reading as file '" + fileName + "'");
222                 }       
223                 return null;
224         }
225
226         public static Collection<URL> getTGFiles(Bundle b) {
227                 Enumeration<URL> enu = b.findEntries("/", "*.tg", false);
228                 if (enu == null)
229                         return Collections.emptyList();
230                 if (!enu.hasMoreElements())
231                         return Collections.emptyList();
232                 ArrayList<URL> result = new ArrayList<>();
233                 while (enu.hasMoreElements())
234                         result.add(enu.nextElement());
235                 return result;
236         }
237
238         public static void compile(Bundle b) throws Exception {
239
240                 Collection<ISource> sources = new ArrayList<>();
241                 Collection<TransferableGraph1> dependencies = new ArrayList<>();
242
243                 for (Bundle b2 : getBundles()) {
244                         if(b.equals(b2)) continue;
245                         for (URL url : getTGFiles(b2)) {
246                                 File graphFile = url2file(url, b2.toString() + url.toString());
247                                 dependencies.add(GraphCompiler.read(graphFile));
248                         }
249                 }
250
251                 File bundleFile = FileLocator.getBundleFile(b);
252                 if(bundleFile.isDirectory()) {
253                         File folder = new File(bundleFile, "dynamicGraph");
254                         for(File f : folder.listFiles(new FilenameFilter() {
255
256                                 @Override
257                                 public boolean accept(File dir, String name) {
258                                         return name.endsWith(".pgraph");
259                                 }
260
261                         })) {
262                                 sources.add(new FileSource(f));
263                         }
264                 }
265
266 //              System.out.println("source is " + tmpFile.getAbsolutePath());
267
268                 final StringBuilder errorStringBuilder = new StringBuilder();
269                 GraphCompilerPreferences prefs = new GraphCompilerPreferences();
270                 prefs.validate = true;
271                 prefs.validateRelationRestrictions = ValidationMode.ERROR;
272                 prefs.validateResourceHasType = ValidationMode.ERROR;
273                 String currentLayer0Version = OntologyVersions.getInstance().currentOntologyVersion("http://www.simantics.org/Layer0-0.0");
274                 CompilationResult result = GraphCompiler.compile(currentLayer0Version, sources, dependencies, null, prefs);
275
276                 for(Problem problem : result.getErrors())
277                         errorStringBuilder.append(problem.getLocation() + ": " + problem.getDescription() + "\n");
278                 for(Problem problem : result.getWarnings())
279                         errorStringBuilder.append(problem.getLocation() + ": " + problem.getDescription() + "\n");
280
281                 if(errorStringBuilder.length() > 0) {
282                         LOGGER.error(errorStringBuilder.toString());
283                 } else {
284                         GraphCompiler.write(new File(bundleFile, "graph.tg"), result.getGraph());
285                 }
286
287         }
288
289         /**
290          * Compile all dynamic ontologies in the Platform
291          * 
292          * @param collection
293          * @throws IOException
294          */
295         public static void compileAllDynamicOntologies() {
296                 for (Bundle bundle : getBundles()) {
297                         if(bundle.getEntry("dynamicGraph") != null) {
298                                 try {
299                                         File bundleFile = FileLocator.getBundleFile(bundle);
300                                         if(bundleFile.isDirectory()) {
301                                                 File tg = new File(bundleFile, "graph.tg");
302                                                 long tgLastModified = tg.lastModified();
303                                                 File folder = new File(bundleFile, "dynamicGraph");
304                                                 for(File f : folder.listFiles()) {
305                                                         if(f.isFile() && f.getName().endsWith(".pgraph") && f.lastModified() > tgLastModified) {
306                                                                 compile(bundle);
307                                                                 break;
308                                                         }
309                                                 }
310                                         }
311                                 } catch (Throwable e) {
312                                         LOGGER.error("Failed to compile dynamic ontologies in bundle " + bundle.getSymbolicName(), e);
313                                 }
314                         }
315                 }
316         }
317
318         /**
319          * Get all graphs in the Platform
320          * 
321          * @param collection
322          * @throws IOException
323          */
324         public static Collection<GraphBundle> getAllGraphs() throws IOException {
325                 AtomicReference<IOException> problem = new AtomicReference<>();
326
327                 Collection<GraphBundle> gbundles = Arrays.stream(getBundles())
328                                 .parallel()
329                                 .map(b -> {
330                                         try {
331                                                 return problem.get() == null ? getGraphs(b) : Collections.<GraphBundleEx>emptyList();
332                                         } catch (IOException e) {
333                                                 if (LOGGER.isWarnEnabled())
334                                                         LOGGER.debug("Could not get graph from bundle {}", b, e);
335                                                 problem.set(e);
336                                                 return Collections.<GraphBundleEx>emptyList();
337                                         }
338                                 })
339                                 .flatMap(Collection::stream)
340                                 .collect(Collectors.toList());
341
342                 if (problem.get() != null)
343                         throw problem.get();
344                 return gbundles;
345         }
346
347         /**
348          * Get bundle 
349          * 
350          * @param symbolicName
351          * @return bundle or <tt>null</tt> if there is no bundle or graph 
352          * @throws IOException
353          */
354         public static GraphBundle getGraph(String symbolicName) throws IOException {
355                 Bundle bundle = Platform.getBundle(symbolicName);
356                 if (bundle == null) return null;
357                 return getGraph( bundle );
358         }
359
360         public static Collection<GraphBundleEx> getGraphs(Bundle bundle) throws IOException {
361                 return getTGFiles(bundle).stream()
362                         .map(url -> {
363                                 try {
364                                         GraphBundleEx result = tryGetOnDemandGraph(bundle, url);
365                                         return result != null ? result : getCompleteGraph(bundle, url);
366                                 } catch (IOException e) {
367                                         if (LOGGER.isWarnEnabled())
368                                                 LOGGER.warn("Could not get graph from bundle url {}", url, e);
369                                         return null;
370                                 }
371                         })
372                         .filter(Objects::nonNull)
373                         .collect(Collectors.toList());
374         }
375
376         private static String tgFileId(Bundle bundle, URL url) {
377                 String urlString = url.toString();
378                 String file = urlString.substring(urlString.lastIndexOf("/") + 1);
379                 return bundle.getSymbolicName() + "_" + file;
380         }
381
382         /**
383          * Read the graph in a graph bundle. Graph is read from "graph.tg" file in the root.
384          * 
385          * @param bundle
386          * @return transferable graph, or <tt>null</tt> if there is no graph in the bundle. 
387          * @throws IOException 
388          */
389         public static GraphBundleEx getGraph(Bundle bundle) throws IOException {
390                 URL url = bundle.getEntry("graph.tg");
391                 if (url == null)
392                         return null;
393                 GraphBundleEx result = tryGetOnDemandGraph(bundle, url);
394                 return result != null ? result : getCompleteGraph(bundle, url);
395         }
396
397         private static GraphBundleEx getCompleteGraph(Bundle bundle, URL url) throws IOException {
398                 try {
399                         String id = tgFileId(bundle, url);
400                         return new GraphBundleEx(
401                                         getBundleName(bundle, id),
402                                         readTG(url),
403                                         new VersionedId(id, toP2Version(bundle)),
404                                         isImmutable(bundle));
405                 } catch (Exception e) {
406                         throw new IOException("Problem loading graph.tg from bundle " + bundle.getSymbolicName(), e);
407                 } catch (Error e) {
408                         LOGGER.error("Serious problem loading graph.tg from bundle " + bundle.getSymbolicName(), e);
409                         throw e;
410                 }
411         }
412
413         /**
414          * Read the graph in a graph bundle. Graph is read from "graph.tg" file in the root.
415          * 
416          * @param bundle
417          * @return transferable graph, or <tt>null</tt> if there is no graph in the bundle. 
418          * @throws IOException 
419          */
420         private static GraphBundleEx tryGetOnDemandGraph(Bundle bundle, URL url) throws IOException {
421                 try {
422                         Integer cachedHash = readCachedHash(url);
423                         if (cachedHash == null) {
424                                 LOGGER.info("No cached hash for " + bundle);
425                                 return null;
426                         }
427
428                         Supplier<TransferableGraph1> graphSource = () -> {
429                                 try {
430                                         return readTG(url);
431                                 } catch (Exception e) {
432                                         throw new RuntimeException("Problem loading graph.tg from bundle " + bundle.getSymbolicName(), e);
433                                 } catch (Error e) {
434                                         LOGGER.error("Serious problem loading graph.tg from bundle " + bundle.getSymbolicName(), e);
435                                         throw e;
436                                 }
437                         };
438
439                         String id = tgFileId(bundle, url);
440
441                         return new GraphBundleEx(
442                                         getBundleName(bundle, id),
443                                         graphSource,
444                                         cachedHash,
445                                         new VersionedId(id, toP2Version(bundle)),
446                                         isImmutable(bundle));
447                 } catch (Exception e) {
448                         throw new IOException("Problem loading graph.tg from bundle " + bundle.getSymbolicName(), e);
449                 }
450         }
451
452         private static TransferableGraph1 readTG(URL url) throws Exception {
453                 try (InputStream is = url.openStream()) {
454                         return TransferableGraphFileReader.read(is);
455                 }
456         }
457
458         private static DataContainer readHeader(URL url) throws IOException {
459                 try (InputStream is = url.openStream()) {
460                         return DataContainers.readHeader(new DataInputStream(new BufferedInputStream(is, 1 << 14)));
461                 }
462         }
463
464         private static Integer readCachedHash(URL url) throws IOException, AdaptException {
465                 DataContainer header = readHeader(url);
466                 Variant hashCode = header.metadata.get(Extensions.CACHED_HASHCODE);
467                 return hashCode != null ? (Integer) hashCode.getValue(Bindings.INTEGER) : null;
468         }
469
470         private static Version toP2Version(Bundle bundle) {
471                 org.osgi.framework.Version osgiVersion = bundle.getVersion();
472                 return Version.createOSGi(osgiVersion.getMajor(), osgiVersion.getMinor(), osgiVersion.getMicro(), osgiVersion.getQualifier());
473         }
474
475         private static String getBundleName(Bundle bundle, String id) {
476                 String name = (String) bundle.getHeaders().get("Bundle-Name");
477                 return name != null ? name : id;
478         }
479
480         private static boolean isImmutable(Bundle bundle) {
481                 String immutable = (String) bundle.getHeaders().get("Immutable");
482                 if(immutable == null)
483                         return true;
484                 if("false".equals(immutable))
485                         return false;
486                 if("trueWhenDeployed".equals(immutable)) {
487                         String installHint = System.getProperty("installOntologiesAsDeployed");
488                         if("true".equals(installHint))
489                                 return true;
490                         else
491                                 return false;
492                 }
493                 return true;
494         }
495
496         public static class TGInfo {
497                 public Bundle bundle;
498                 public URL location;
499                 public IVersionedId vid;
500         }
501
502 }