Improved shared library structure dump to take more types into account 96/3896/1
authorTuukka Lehtonen <tuukka.lehtonen@semantum.fi>
Thu, 20 Feb 2020 19:31:08 +0000 (21:31 +0200)
committerTuukka Lehtonen <tuukka.lehtonen@semantum.fi>
Thu, 20 Feb 2020 19:31:08 +0000 (21:31 +0200)
Added support for:
* L0.SCLScript
* GF.File

This also removes dumping of the .pgraph file from the shared library
exporter because the contents of the generated .pgraph file have never
been complete, nor stable enough to actually give out any information on
what has changed and how. The system gives no diffs out on what most
usually changes, i.e. SCL modules, scripts, SVG files and other similar
textual data.

gitlab #452

Change-Id: Id35b8b6d339398ba89aca3e10aff7aeb4c610242

bundles/org.simantics.graph/src/org/simantics/graph/refactoring/FixExportedOntology.java
bundles/org.simantics.modeling.ontology/META-INF/MANIFEST.MF
bundles/org.simantics.modeling.ontology/graph/Modeling.pgraph
bundles/org.simantics.modeling.ui/src/org/simantics/modeling/ui/sharedontology/wizard/ExportPlan.java
bundles/org.simantics.modeling.ui/src/org/simantics/modeling/ui/sharedontology/wizard/SharedOntologyExportPage.java
bundles/org.simantics.modeling.ui/src/org/simantics/modeling/ui/sharedontology/wizard/SharedOntologyExporter.java
bundles/org.simantics.modeling/scl/Simantics/Testing.scl
bundles/org.simantics.modeling/src/org/simantics/modeling/ContentDumps.java
bundles/org.simantics.modeling/src/org/simantics/modeling/ModelingUtils.java
bundles/org.simantics.modeling/src/org/simantics/modeling/utils/DumpOntologyStructure.java

index 27c4b9195522d82291c6374586c0db985d1bfe4b..17aa3ed26af30ab5143a2cacf7fb16b01d2a27d3 100644 (file)
@@ -13,15 +13,19 @@ import org.simantics.databoard.container.DataContainer;
 import org.simantics.databoard.container.DataContainers;
 import org.simantics.graph.representation.PrettyPrintTG;
 import org.simantics.graph.representation.TransferableGraph1;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * @author Antti Villberg
  * @since 1.24.0
  */
 public class FixExportedOntology {
-       
+
+       private static final Logger LOGGER = LoggerFactory.getLogger(FixExportedOntology.class);
+
        static TransferableGraph1 convertExportedSharedOntologyIntoBundleOntology(Path input, Path output) throws Exception {
-               System.out.format("Converting exported shared ontology%n\t" + input.toString() + "%nto bundle-compatible ontology%n\t" + output.toString());
+               LOGGER.info("Converting exported shared ontology\n\t{}\nto bundle-compatible ontology\n\t{}", input.toString(), output.toString());
                try (InputStream is = new BufferedInputStream(Files.newInputStream(input), 128*1024)) {
                        Binding tgBinding = TransferableGraph1.BINDING;
                        DataContainer container = DataContainers.readFile(new DataInputStream(is), tgBinding); 
@@ -34,26 +38,32 @@ public class FixExportedOntology {
                        return graph;
                }
        }
-       
-       public static void createTg(Path input) throws Exception {
-           Path output = input.getParent().resolve("graph.tg");
-           createTg(input, output);
+
+       private static Path replaceExtension(Path p, String newExtension) {
+               String newName = p.getFileName().toString();
+               if (newName.contains("."))
+                       newName = newName.split("\\.")[0];
+               return p.resolveSibling(newName + newExtension);
        }
-       
+
        private static void createTg(Path input, Path output) throws Exception {
-           convertExportedSharedOntologyIntoBundleOntology(input, output);
+               convertExportedSharedOntologyIntoBundleOntology(input, output);
+       }
+
+       public static void createTg(Path input) throws Exception {
+               createTg(input, replaceExtension(input, ".tg"));
        }
-       
+
        public static void createTGAndPGraph(Path input) throws Exception {
-        String newName = input.getFileName().toString();
-        if (newName.contains("."))
-            newName = newName.split("\\.")[0];
-        Path parent = input.getParent();
-        Path output1 = parent.resolve(newName + ".tg");
-        TransferableGraph1 tg = convertExportedSharedOntologyIntoBundleOntology(input, output1);
-        String listing = PrettyPrintTG.print(tg, false);
-        Path output2 = parent.resolve(newName + ".pgraph");
-        Files.write(output2, listing.getBytes(),StandardOpenOption.CREATE);
+               createTGAndPGraph(input, true);
+       }
+
+       public static void createTGAndPGraph(Path input, boolean writePGraph) throws Exception {
+               TransferableGraph1 tg = convertExportedSharedOntologyIntoBundleOntology(input, replaceExtension(input, ".tg"));
+               if (writePGraph) {
+                       String listing = PrettyPrintTG.print(tg, false);
+                       Files.write(replaceExtension(input, ".pgraph"), listing.getBytes(),StandardOpenOption.CREATE);
+               }
        }
 
        public static void main(String[] args) throws Exception {
index 3a6001a6dd96f781a55d7762a349bde4a89b4fe8..c814a674d0d0040fb87541418a50d185d2ba30ef 100644 (file)
@@ -20,7 +20,8 @@ Require-Bundle: org.simantics.layer0,
  org.simantics.views.ontology;bundle-version="1.1.0",
  org.simantics.g2d.ontology;bundle-version="1.0.0",
  org.simantics.selectionview.ontology;bundle-version="1.2.0",
- org.simantics.spreadsheet.ontology;bundle-version="1.2.0"
+ org.simantics.spreadsheet.ontology;bundle-version="1.2.0",
+ org.simantics.graphfile.ontology;bundle-version="0.1.0"
 Bundle-RequiredExecutionEnvironment: JavaSE-1.8
 Export-Package: org.simantics.modeling
 Automatic-Module-Name: org.simantics.modeling.ontology
index 52a37f738ca76566ede5fe44d1589a174c042536..82808f837b425f2726f1f9df57f279249997d29e 100644 (file)
@@ -5,6 +5,7 @@ PROJ = <http://www.simantics.org/Project-1.2>
 G2D = <http://www.simantics.org/G2D-1.1>
 DIA = <http://www.simantics.org/Diagram-2.2>
 SIMU = <http://www.simantics.org/Simulation-1.1>
+GF = <http://www.simantics.org/GraphFile-0.1>
 DOC = <http://www.simantics.org/Document-1.2>
 ISSUE = <http://www.simantics.org/Issue-1.2>
 SG = <http://www.simantics.org/Scenegraph-1.1>
@@ -546,18 +547,27 @@ L0.SCLModule
     MOD.SCLModuleContentDumpFunction
       @L0.sclValue "sclModuleContentDump" "Resource -> <ReadGraph> Vector Byte"
 
+L0.SCLScript
+  MOD.contentDumpFunction
+    MOD.SCLScriptContentDumpFunction
+      @L0.sclValue "sclScriptContentDump" "Resource -> <ReadGraph> Vector Byte"
+
 L0.PGraph
   MOD.contentDumpFunction
     MOD.PGraphContentDumpFunction
       @L0.sclValue "pgraphContentDump" "Resource -> <ReadGraph> Vector Byte"
 
+GF.File
+  MOD.contentDumpFunction
+    MOD.GraphFileContentDumpFunction
+      @L0.sclValue "graphFileContentDump" "Resource -> <ReadGraph> Vector Byte"
+
 STR.Component
   MOD.contentDumpFunction
     MOD.StructuralComponentContentDumpFunction
       @L0.sclValue "structuralComponentContentDump" "Resource -> <ReadGraph> Vector Byte"
-      
+
 SEL.GenericParameterType
   MOD.contentDumpFunction
     MOD.GenericParameterTypeContentDumpFunction
       @L0.sclValue "genericParameterTypeContentDump" "Resource -> <ReadGraph> Vector Byte"
-    
\ No newline at end of file
index e70fc426f3c6b6f531ba73e0ebc2940f5ecf691b..2feea0d2ac835c4fd8d1b8d8ff55ff450a79ebdf 100644 (file)
@@ -40,7 +40,8 @@ public class ExportPlan {
      */
     boolean         overwrite;
     boolean         includeDependencies;
-    boolean         tgAndPgraph;
+    boolean         writeTransferableGraph;
+    boolean         dumpStructure;
 
     ExportPlan(ISessionContext sessionContext, Deque<String> recentLocations) {
         this.sessionContext = sessionContext;
index 90a155231e279dce3026c0c30126df579ac28ad1..2b193094e2837dae0fa64993ca6dc36da9a7b403 100644 (file)
@@ -30,6 +30,7 @@ import org.eclipse.swt.events.ModifyEvent;
 import org.eclipse.swt.events.ModifyListener;
 import org.eclipse.swt.events.SelectionAdapter;
 import org.eclipse.swt.events.SelectionEvent;
+import org.eclipse.swt.events.SelectionListener;
 import org.eclipse.swt.layout.GridData;
 import org.eclipse.swt.layout.GridLayout;
 import org.eclipse.swt.widgets.Button;
@@ -162,16 +163,20 @@ public class SharedOntologyExportPage extends WizardPage {
         });
         String prop = System.getProperty("org.simantics.modeling.exportTgAndPgraph");
         if (prop != null || Platform.inDevelopmentMode()) {
-            Button tgAndPgraph = new Button(container, SWT.CHECK);
-            tgAndPgraph.setText("&Generate TG and Pgraph with export");
-            tgAndPgraph.setSelection(exportModel.tgAndPgraph);
-            GridDataFactory.fillDefaults().grab(true, false).span(3, 1).applyTo(tgAndPgraph);
-            tgAndPgraph.addSelectionListener(new SelectionAdapter() {
-                @Override
-                public void widgetSelected(SelectionEvent e) {
-                    exportModel.tgAndPgraph = tgAndPgraph.getSelection();
-                }
-            });
+            Button generateTg = new Button(container, SWT.CHECK);
+            generateTg.setText("&Generate transferable graph");
+            generateTg.setSelection(exportModel.writeTransferableGraph);
+            GridDataFactory.fillDefaults().grab(true, false).span(3, 1).applyTo(generateTg);
+            generateTg.addSelectionListener(SelectionListener.widgetSelectedAdapter(e -> {
+                exportModel.writeTransferableGraph = generateTg.getSelection();
+            }));
+            Button dumpStructure = new Button(container, SWT.CHECK);
+            dumpStructure.setText("&Dump textual shared library structure");
+            dumpStructure.setSelection(exportModel.dumpStructure);
+            GridDataFactory.fillDefaults().grab(true, false).span(3, 1).applyTo(dumpStructure);
+            dumpStructure.addSelectionListener(SelectionListener.widgetSelectedAdapter(e -> {
+                exportModel.dumpStructure = dumpStructure.getSelection();
+            }));
         }
 
         try {
index eae7695747153fdc45ddeb66eda859f49775a7da..5813ff673fd4eb22363d1d2e2ae6674f22f78b85 100644 (file)
@@ -15,7 +15,6 @@ import java.io.File;
 import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
 import java.nio.file.Path;
-import java.nio.file.Paths;
 
 import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.core.runtime.SubMonitor;
@@ -24,8 +23,6 @@ import org.simantics.Simantics;
 import org.simantics.databoard.binding.error.BindingException;
 import org.simantics.databoard.serialization.SerializationException;
 import org.simantics.db.ReadGraph;
-import org.simantics.db.common.request.UniqueRead;
-import org.simantics.db.common.utils.Logger;
 import org.simantics.db.exception.DatabaseException;
 import org.simantics.graph.refactoring.FixExportedOntology;
 import org.simantics.modeling.ModelingUtils;
@@ -40,6 +37,7 @@ import org.slf4j.LoggerFactory;
 public class SharedOntologyExporter implements IRunnableWithProgress {
 
     private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(SharedOntologyExporter.class);
+
     ExportPlan exportModel;
 
     public SharedOntologyExporter(ExportPlan exportModel) {
@@ -64,43 +62,49 @@ public class SharedOntologyExporter implements IRunnableWithProgress {
 
     void exportModel(SubMonitor mon) throws IOException, DatabaseException, SerializationException, BindingException{
         try {
-            doExport(mon, exportModel.exportLocation, exportModel.model, exportModel.tgAndPgraph);
-
+            doExport(mon, exportModel.exportLocation, exportModel.model, exportModel.writeTransferableGraph, exportModel.dumpStructure);
         } catch (DatabaseException e) {
-            e.printStackTrace();
-            Logger.defaultLogError(e);
+            LOGGER.error("Failed to export shared ontology", e);
             mon.setCanceled(true);
             ShowMessage.showError("Export failed.", "Internal application error in export. See log for details.");
         } finally {
             mon.setWorkRemaining(0);
         }
     }
-    
-    public static void doExport(IProgressMonitor monitor, File location, final LibraryInfo info) throws DatabaseException, IOException {
-        doExport(monitor, location, info, false);
+
+    public static void doExport(IProgressMonitor monitor, File location, LibraryInfo info) throws DatabaseException, IOException {
+        doExport(monitor, location, info, false, false);
     }
-    
-    public static void doExport(IProgressMonitor monitor, File location, final LibraryInfo info, boolean pgraphAndTg) throws DatabaseException, IOException {
-       ModelingUtils.exportSharedOntology(monitor, Simantics.getSession(), location,Constants.SHARED_LIBRARY_FORMAT, Constants.SHARED_LIBRARY_CURRENT_VERSION, info);
-       if (pgraphAndTg) {
-            try {
-                Path input = Paths.get(location.toURI());
-                FixExportedOntology.createTGAndPGraph(input);
-                DumpOntologyStructure data = Simantics.sync(new UniqueRead<DumpOntologyStructure>() {
 
-                    @Override
-                    public DumpOntologyStructure perform(ReadGraph graph) throws DatabaseException {
-                        DumpOntologyStructure result = new DumpOntologyStructure();
-                        result.read(graph, info.library.getResource());
-                        return result;
-                    }
-                    
+    public static void doExport(IProgressMonitor monitor, File location, LibraryInfo info, boolean writeTg, boolean dumpStructure) throws DatabaseException, IOException {
+        int work = 1 + (writeTg ? 1 : 0) + (dumpStructure ? 1 : 0);
+        SubMonitor mon = SubMonitor.convert(monitor, work);
+
+        ModelingUtils.exportSharedOntology(mon.split(1, SubMonitor.SUPPRESS_NONE), Simantics.getSession(), location, Constants.SHARED_LIBRARY_FORMAT, Constants.SHARED_LIBRARY_CURRENT_VERSION, info);
+
+        Path input = location.toPath();
+
+        if (writeTg) {
+            try {
+                mon.subTask("Writing transferable graph");
+                FixExportedOntology.createTGAndPGraph(input, false);
+                mon.worked(1);
+            } catch (Exception e) {
+                LOGGER.error("Could not generate Transferable Graph", e);
+            }
+        }
+        if (dumpStructure) {
+            try {
+                monitor.subTask("Dumping library structure");
+                DumpOntologyStructure data = Simantics.getSession().syncRequest((ReadGraph graph) -> {
+                    return new DumpOntologyStructure().read(graph, info.library.getResource());
                 });
                 data.write(new File(new File(location.getParent(), location.getName() + ".dump"), info.library.getName()));
+                mon.worked(1);
             } catch (Exception e) {
-                LOGGER.error("Could not generate TG and Pgraph", e);
+                LOGGER.error("Could not generate shared library structure dump", e);
             }
-       }
+        }
     }
-       
+
 }
index 5742b4f41bd693c9b490abde6a54931f77849e7d..f1ebaf50c3d5117e99f2737ec85cfa0fcb95cbb2 100644 (file)
@@ -7,7 +7,9 @@ importJava "org.simantics.modeling.ModelingUtils" where
   
 importJava "org.simantics.modeling.ContentDumps" where
   sclModuleContentDump :: Resource -> <ReadGraph> Vector Byte
+  sclScriptContentDump :: Resource -> <ReadGraph> Vector Byte
   pgraphContentDump :: Resource -> <ReadGraph> Vector Byte
+  graphFileContentDump :: Resource -> <ReadGraph> Vector Byte
   structuralComponentContentDump :: Resource -> <ReadGraph> Vector Byte
   genericParameterTypeContentDump :: Resource -> <ReadGraph> Vector Byte
-  
+
index bc3b65a036154788dc2e0887eb9e940c7b73377b..687544e78a52d99306436408d0211f23fa157157 100644 (file)
@@ -30,6 +30,12 @@ public class ContentDumps {
         return def.getBytes(UTF8);
     }
 
+    public static byte[] sclScriptContentDump(ReadGraph graph, Resource resource) throws DatabaseException {
+        Layer0 L0 = Layer0.getInstance(graph);
+        String def = graph.getRelatedValue(resource, L0.SCLScript_definition, Bindings.STRING); 
+        return def.getBytes(UTF8);
+    }
+
     public static byte[] pgraphContentDump(ReadGraph graph, Resource resource) throws DatabaseException {
         Layer0 L0 = Layer0.getInstance(graph);
         String def = graph.getRelatedValue(resource, L0.PGraph_definition, Bindings.STRING); 
index 4a4d97a18a2b254f0be58c2a928483dcecd234c6..1edf3cd3121a4e4682e5de0012916fcfd0b74ee5 100644 (file)
@@ -29,6 +29,7 @@ import org.eclipse.core.runtime.CoreException;
 import org.eclipse.core.runtime.IProgressMonitor;
 import org.eclipse.core.runtime.NullProgressMonitor;
 import org.eclipse.core.runtime.Path;
+import org.eclipse.core.runtime.SubMonitor;
 import org.eclipse.jface.dialogs.Dialog;
 import org.eclipse.jface.dialogs.IDialogConstants;
 import org.eclipse.jface.dialogs.MessageDialog;
@@ -1515,53 +1516,46 @@ public class ModelingUtils {
     }
        
     public static TransferableGraph1 exportSharedOntology(IProgressMonitor monitor, RequestProcessor processor, File location, String format, int version, final LibraryInfo info) throws DatabaseException, IOException {
-       
-       if(monitor == null) monitor = new NullProgressMonitor();
-       final IProgressMonitor finalMonitor = monitor;
+        SubMonitor mon = SubMonitor.convert(monitor, "Exporting shared library", 100);
+
         // TODO: figure out a way to make the TG go directly into a file
         // instead of having it all in memory at once.
 
-        monitor.beginTask("Exporting shared library...", 100);
-       SimanticsClipboard clipboard = processor.syncRequest(new Read<SimanticsClipboard>() {
-            @Override
-            public SimanticsClipboard perform(ReadGraph graph) throws DatabaseException {
-                CopyHandler ch = graph.adapt(info.library.getResource(), CopyHandler.class);
-                SimanticsClipboardImpl clipboard = new SimanticsClipboardImpl();
-                ch.copyToClipboard(graph, clipboard, finalMonitor);
-                return clipboard;
-            }
+        SimanticsClipboard clipboard = processor.syncRequest((ReadGraph graph) -> {
+            CopyHandler ch = graph.adapt(info.library.getResource(), CopyHandler.class);
+            SimanticsClipboardImpl result = new SimanticsClipboardImpl();
+            ch.copyToClipboard(graph, result, mon.split(5));
+            return result;
         });
-       
+
         TreeMap<String,Variant> metadata = getExportMetadata();
         DraftStatusBean draft = info.draft;
         if(draft != null) {
-               metadata.put(DraftStatusBean.EXTENSION_KEY, new Variant(DraftStatusBean.BINDING ,draft));
+            metadata.put(DraftStatusBean.EXTENSION_KEY, new Variant(DraftStatusBean.BINDING ,draft));
         }
-        
+
         for (Set<Representation> object : clipboard.getContents()) {
-               
+            mon.subTask("Constructing exported material");
             TransferableGraph1 tg = ClipboardUtils.accept(processor, object, SimanticsKeys.KEY_TRANSFERABLE_GRAPH);
-            monitor.worked(95);
-            
+            mon.worked(90);
+
             Variant edb = tg.extensions.get(ExternalDownloadBean.EXTENSION_KEY);
             if(edb != null) {
-               metadata.put(ExternalDownloadBean.EXTENSION_KEY, edb);
+                metadata.put(ExternalDownloadBean.EXTENSION_KEY, edb);
             }
 
             if(location != null) {
-                   monitor.setTaskName("Writing transferable graph...");
+                   monitor.subTask("Writing transferable graph");
                    DataContainers.writeFile(location, new DataContainer(
                            format, version,
                            metadata, new Variant(TransferableGraph1.BINDING, tg)));
                    monitor.worked(5);
             }
-            
+
             return tg;
-            
         }
-        
-        throw new DatabaseException("Failed to export");
-        
+
+        throw new DatabaseException("Export failed, no contents to export");
     }
 
     public static TreeMap<String, Variant> getExportMetadata() {
index 79622f15e5839bf1cae00c32dc228e75967eb427..5bb3d9ca21c95d30bb5d96200d41caade8bf59df 100644 (file)
@@ -30,7 +30,7 @@ public class DumpOntologyStructure {
     private static final Logger LOGGER = LoggerFactory.getLogger(DumpOntologyStructure.class);
 
     private Resource ontology;
-    
+
     private Map<Resource, String> names = new HashMap<>();
     private Map<Resource,Resource> parents = new HashMap<>();
     private Map<Resource, File> libraryFolders = new HashMap<>();
@@ -40,11 +40,11 @@ public class DumpOntologyStructure {
         parents.put(r, container);
         names.put(r, NameUtils.getSafeName(graph, r));
     }
-    
+
     private Collection<Resource> containers() {
         return parents.values();
     }
-    
+
     private void readHierarchy(ReadGraph graph, Resource container) throws DatabaseException {
         Layer0 L0 = Layer0.getInstance(graph);
         for(Resource r : CommonDBUtils.objectsWithType(graph, container, L0.ConsistsOf, L0.Entity)) {
@@ -94,30 +94,27 @@ public class DumpOntologyStructure {
         }
     }
 
-    public void read(ReadGraph graph, Resource ontology) throws DatabaseException {
+    public DumpOntologyStructure read(ReadGraph graph, Resource ontology) throws DatabaseException {
         this.ontology = ontology;
         readHierarchy(graph, ontology);
         readGeneric(graph);
+        return this;
     }
-    
-    private File escapeFile(File file) {
 
+    private File escapeFile(File file) {
         if(file.exists())
             return file;
-        
         return new File(escapeFile(file.getParentFile()), FileUtils.escapeFileName(file.getName()));
-        
     }
-    
+
     public void write(File unsafeFolder) throws IOException {
         File folder = escapeFile(unsafeFolder);
-        if(folder.exists())
-            FileUtils.deleteAll(folder);
+        FileUtils.delete(folder.toPath());
         folder.mkdirs();
         writeDirectories(folder);
         writeResources(folder);
     }
-    
+
     private File getFolder(File root, Resource library) {
         if(ontology.equals(library))
             return root;
@@ -147,11 +144,11 @@ public class DumpOntologyStructure {
             writeResource(rootFolder, r);
         }
     }
-    
+
     private boolean isParent(Resource r) {
         return parents.values().contains(r);
     }
-    
+
     private void writeResource(File rootFolder, Resource resource) throws IOException {
         byte[] dump = contentDumps.get(resource);
         if(dump == null)
@@ -168,4 +165,4 @@ public class DumpOntologyStructure {
         FileUtils.writeFile(getFile(rootFolder, resource), bytes);
     }
 
-}
+}
\ No newline at end of file