From 9214a600c9401b06057fc2c10ea86a0ce0218d87 Mon Sep 17 00:00:00 2001 From: Tuukka Lehtonen Date: Thu, 20 Feb 2020 21:31:08 +0200 Subject: [PATCH] Improved shared library structure dump to take more types into account Added support for: * L0.SCLScript * GF.File This also removes dumping of the .pgraph file from the shared library exporter because the contents of the generated .pgraph file have never been complete, nor stable enough to actually give out any information on what has changed and how. The system gives no diffs out on what most usually changes, i.e. SCL modules, scripts, SVG files and other similar textual data. gitlab #452 Change-Id: Id35b8b6d339398ba89aca3e10aff7aeb4c610242 --- .../refactoring/FixExportedOntology.java | 46 ++++++++------ .../META-INF/MANIFEST.MF | 3 +- .../graph/Modeling.pgraph | 14 ++++- .../ui/sharedontology/wizard/ExportPlan.java | 3 +- .../wizard/SharedOntologyExportPage.java | 25 ++++---- .../wizard/SharedOntologyExporter.java | 60 ++++++++++--------- .../scl/Simantics/Testing.scl | 4 +- .../org/simantics/modeling/ContentDumps.java | 6 ++ .../org/simantics/modeling/ModelingUtils.java | 44 ++++++-------- .../modeling/utils/DumpOntologyStructure.java | 27 ++++----- 10 files changed, 131 insertions(+), 101 deletions(-) diff --git a/bundles/org.simantics.graph/src/org/simantics/graph/refactoring/FixExportedOntology.java b/bundles/org.simantics.graph/src/org/simantics/graph/refactoring/FixExportedOntology.java index 27c4b9195..17aa3ed26 100644 --- a/bundles/org.simantics.graph/src/org/simantics/graph/refactoring/FixExportedOntology.java +++ b/bundles/org.simantics.graph/src/org/simantics/graph/refactoring/FixExportedOntology.java @@ -13,15 +13,19 @@ import org.simantics.databoard.container.DataContainer; import org.simantics.databoard.container.DataContainers; import org.simantics.graph.representation.PrettyPrintTG; import org.simantics.graph.representation.TransferableGraph1; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * @author Antti Villberg * @since 1.24.0 */ public class FixExportedOntology { - + + private static final Logger LOGGER = LoggerFactory.getLogger(FixExportedOntology.class); + static TransferableGraph1 convertExportedSharedOntologyIntoBundleOntology(Path input, Path output) throws Exception { - System.out.format("Converting exported shared ontology%n\t" + input.toString() + "%nto bundle-compatible ontology%n\t" + output.toString()); + LOGGER.info("Converting exported shared ontology\n\t{}\nto bundle-compatible ontology\n\t{}", input.toString(), output.toString()); try (InputStream is = new BufferedInputStream(Files.newInputStream(input), 128*1024)) { Binding tgBinding = TransferableGraph1.BINDING; DataContainer container = DataContainers.readFile(new DataInputStream(is), tgBinding); @@ -34,26 +38,32 @@ public class FixExportedOntology { return graph; } } - - public static void createTg(Path input) throws Exception { - Path output = input.getParent().resolve("graph.tg"); - createTg(input, output); + + private static Path replaceExtension(Path p, String newExtension) { + String newName = p.getFileName().toString(); + if (newName.contains(".")) + newName = newName.split("\\.")[0]; + return p.resolveSibling(newName + newExtension); } - + private static void createTg(Path input, Path output) throws Exception { - convertExportedSharedOntologyIntoBundleOntology(input, output); + convertExportedSharedOntologyIntoBundleOntology(input, output); + } + + public static void createTg(Path input) throws Exception { + createTg(input, replaceExtension(input, ".tg")); } - + public static void createTGAndPGraph(Path input) throws Exception { - String newName = input.getFileName().toString(); - if (newName.contains(".")) - newName = newName.split("\\.")[0]; - Path parent = input.getParent(); - Path output1 = parent.resolve(newName + ".tg"); - TransferableGraph1 tg = convertExportedSharedOntologyIntoBundleOntology(input, output1); - String listing = PrettyPrintTG.print(tg, false); - Path output2 = parent.resolve(newName + ".pgraph"); - Files.write(output2, listing.getBytes(),StandardOpenOption.CREATE); + createTGAndPGraph(input, true); + } + + public static void createTGAndPGraph(Path input, boolean writePGraph) throws Exception { + TransferableGraph1 tg = convertExportedSharedOntologyIntoBundleOntology(input, replaceExtension(input, ".tg")); + if (writePGraph) { + String listing = PrettyPrintTG.print(tg, false); + Files.write(replaceExtension(input, ".pgraph"), listing.getBytes(),StandardOpenOption.CREATE); + } } public static void main(String[] args) throws Exception { diff --git a/bundles/org.simantics.modeling.ontology/META-INF/MANIFEST.MF b/bundles/org.simantics.modeling.ontology/META-INF/MANIFEST.MF index 3a6001a6d..c814a674d 100644 --- a/bundles/org.simantics.modeling.ontology/META-INF/MANIFEST.MF +++ b/bundles/org.simantics.modeling.ontology/META-INF/MANIFEST.MF @@ -20,7 +20,8 @@ Require-Bundle: org.simantics.layer0, org.simantics.views.ontology;bundle-version="1.1.0", org.simantics.g2d.ontology;bundle-version="1.0.0", org.simantics.selectionview.ontology;bundle-version="1.2.0", - org.simantics.spreadsheet.ontology;bundle-version="1.2.0" + org.simantics.spreadsheet.ontology;bundle-version="1.2.0", + org.simantics.graphfile.ontology;bundle-version="0.1.0" Bundle-RequiredExecutionEnvironment: JavaSE-1.8 Export-Package: org.simantics.modeling Automatic-Module-Name: org.simantics.modeling.ontology diff --git a/bundles/org.simantics.modeling.ontology/graph/Modeling.pgraph b/bundles/org.simantics.modeling.ontology/graph/Modeling.pgraph index 52a37f738..82808f837 100644 --- a/bundles/org.simantics.modeling.ontology/graph/Modeling.pgraph +++ b/bundles/org.simantics.modeling.ontology/graph/Modeling.pgraph @@ -5,6 +5,7 @@ PROJ = G2D = DIA = SIMU = +GF = DOC = ISSUE = SG = @@ -546,18 +547,27 @@ L0.SCLModule MOD.SCLModuleContentDumpFunction @L0.sclValue "sclModuleContentDump" "Resource -> Vector Byte" +L0.SCLScript + MOD.contentDumpFunction + MOD.SCLScriptContentDumpFunction + @L0.sclValue "sclScriptContentDump" "Resource -> Vector Byte" + L0.PGraph MOD.contentDumpFunction MOD.PGraphContentDumpFunction @L0.sclValue "pgraphContentDump" "Resource -> Vector Byte" +GF.File + MOD.contentDumpFunction + MOD.GraphFileContentDumpFunction + @L0.sclValue "graphFileContentDump" "Resource -> Vector Byte" + STR.Component MOD.contentDumpFunction MOD.StructuralComponentContentDumpFunction @L0.sclValue "structuralComponentContentDump" "Resource -> Vector Byte" - + SEL.GenericParameterType MOD.contentDumpFunction MOD.GenericParameterTypeContentDumpFunction @L0.sclValue "genericParameterTypeContentDump" "Resource -> Vector Byte" - \ No newline at end of file diff --git a/bundles/org.simantics.modeling.ui/src/org/simantics/modeling/ui/sharedontology/wizard/ExportPlan.java b/bundles/org.simantics.modeling.ui/src/org/simantics/modeling/ui/sharedontology/wizard/ExportPlan.java index e70fc426f..2feea0d2a 100644 --- a/bundles/org.simantics.modeling.ui/src/org/simantics/modeling/ui/sharedontology/wizard/ExportPlan.java +++ b/bundles/org.simantics.modeling.ui/src/org/simantics/modeling/ui/sharedontology/wizard/ExportPlan.java @@ -40,7 +40,8 @@ public class ExportPlan { */ boolean overwrite; boolean includeDependencies; - boolean tgAndPgraph; + boolean writeTransferableGraph; + boolean dumpStructure; ExportPlan(ISessionContext sessionContext, Deque recentLocations) { this.sessionContext = sessionContext; diff --git a/bundles/org.simantics.modeling.ui/src/org/simantics/modeling/ui/sharedontology/wizard/SharedOntologyExportPage.java b/bundles/org.simantics.modeling.ui/src/org/simantics/modeling/ui/sharedontology/wizard/SharedOntologyExportPage.java index 90a155231..2b193094e 100644 --- a/bundles/org.simantics.modeling.ui/src/org/simantics/modeling/ui/sharedontology/wizard/SharedOntologyExportPage.java +++ b/bundles/org.simantics.modeling.ui/src/org/simantics/modeling/ui/sharedontology/wizard/SharedOntologyExportPage.java @@ -30,6 +30,7 @@ import org.eclipse.swt.events.ModifyEvent; import org.eclipse.swt.events.ModifyListener; import org.eclipse.swt.events.SelectionAdapter; import org.eclipse.swt.events.SelectionEvent; +import org.eclipse.swt.events.SelectionListener; import org.eclipse.swt.layout.GridData; import org.eclipse.swt.layout.GridLayout; import org.eclipse.swt.widgets.Button; @@ -162,16 +163,20 @@ public class SharedOntologyExportPage extends WizardPage { }); String prop = System.getProperty("org.simantics.modeling.exportTgAndPgraph"); if (prop != null || Platform.inDevelopmentMode()) { - Button tgAndPgraph = new Button(container, SWT.CHECK); - tgAndPgraph.setText("&Generate TG and Pgraph with export"); - tgAndPgraph.setSelection(exportModel.tgAndPgraph); - GridDataFactory.fillDefaults().grab(true, false).span(3, 1).applyTo(tgAndPgraph); - tgAndPgraph.addSelectionListener(new SelectionAdapter() { - @Override - public void widgetSelected(SelectionEvent e) { - exportModel.tgAndPgraph = tgAndPgraph.getSelection(); - } - }); + Button generateTg = new Button(container, SWT.CHECK); + generateTg.setText("&Generate transferable graph"); + generateTg.setSelection(exportModel.writeTransferableGraph); + GridDataFactory.fillDefaults().grab(true, false).span(3, 1).applyTo(generateTg); + generateTg.addSelectionListener(SelectionListener.widgetSelectedAdapter(e -> { + exportModel.writeTransferableGraph = generateTg.getSelection(); + })); + Button dumpStructure = new Button(container, SWT.CHECK); + dumpStructure.setText("&Dump textual shared library structure"); + dumpStructure.setSelection(exportModel.dumpStructure); + GridDataFactory.fillDefaults().grab(true, false).span(3, 1).applyTo(dumpStructure); + dumpStructure.addSelectionListener(SelectionListener.widgetSelectedAdapter(e -> { + exportModel.dumpStructure = dumpStructure.getSelection(); + })); } try { diff --git a/bundles/org.simantics.modeling.ui/src/org/simantics/modeling/ui/sharedontology/wizard/SharedOntologyExporter.java b/bundles/org.simantics.modeling.ui/src/org/simantics/modeling/ui/sharedontology/wizard/SharedOntologyExporter.java index eae769574..5813ff673 100644 --- a/bundles/org.simantics.modeling.ui/src/org/simantics/modeling/ui/sharedontology/wizard/SharedOntologyExporter.java +++ b/bundles/org.simantics.modeling.ui/src/org/simantics/modeling/ui/sharedontology/wizard/SharedOntologyExporter.java @@ -15,7 +15,6 @@ import java.io.File; import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.nio.file.Path; -import java.nio.file.Paths; import org.eclipse.core.runtime.IProgressMonitor; import org.eclipse.core.runtime.SubMonitor; @@ -24,8 +23,6 @@ import org.simantics.Simantics; import org.simantics.databoard.binding.error.BindingException; import org.simantics.databoard.serialization.SerializationException; import org.simantics.db.ReadGraph; -import org.simantics.db.common.request.UniqueRead; -import org.simantics.db.common.utils.Logger; import org.simantics.db.exception.DatabaseException; import org.simantics.graph.refactoring.FixExportedOntology; import org.simantics.modeling.ModelingUtils; @@ -40,6 +37,7 @@ import org.slf4j.LoggerFactory; public class SharedOntologyExporter implements IRunnableWithProgress { private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(SharedOntologyExporter.class); + ExportPlan exportModel; public SharedOntologyExporter(ExportPlan exportModel) { @@ -64,43 +62,49 @@ public class SharedOntologyExporter implements IRunnableWithProgress { void exportModel(SubMonitor mon) throws IOException, DatabaseException, SerializationException, BindingException{ try { - doExport(mon, exportModel.exportLocation, exportModel.model, exportModel.tgAndPgraph); - + doExport(mon, exportModel.exportLocation, exportModel.model, exportModel.writeTransferableGraph, exportModel.dumpStructure); } catch (DatabaseException e) { - e.printStackTrace(); - Logger.defaultLogError(e); + LOGGER.error("Failed to export shared ontology", e); mon.setCanceled(true); ShowMessage.showError("Export failed.", "Internal application error in export. See log for details."); } finally { mon.setWorkRemaining(0); } } - - public static void doExport(IProgressMonitor monitor, File location, final LibraryInfo info) throws DatabaseException, IOException { - doExport(monitor, location, info, false); + + public static void doExport(IProgressMonitor monitor, File location, LibraryInfo info) throws DatabaseException, IOException { + doExport(monitor, location, info, false, false); } - - public static void doExport(IProgressMonitor monitor, File location, final LibraryInfo info, boolean pgraphAndTg) throws DatabaseException, IOException { - ModelingUtils.exportSharedOntology(monitor, Simantics.getSession(), location,Constants.SHARED_LIBRARY_FORMAT, Constants.SHARED_LIBRARY_CURRENT_VERSION, info); - if (pgraphAndTg) { - try { - Path input = Paths.get(location.toURI()); - FixExportedOntology.createTGAndPGraph(input); - DumpOntologyStructure data = Simantics.sync(new UniqueRead() { - @Override - public DumpOntologyStructure perform(ReadGraph graph) throws DatabaseException { - DumpOntologyStructure result = new DumpOntologyStructure(); - result.read(graph, info.library.getResource()); - return result; - } - + public static void doExport(IProgressMonitor monitor, File location, LibraryInfo info, boolean writeTg, boolean dumpStructure) throws DatabaseException, IOException { + int work = 1 + (writeTg ? 1 : 0) + (dumpStructure ? 1 : 0); + SubMonitor mon = SubMonitor.convert(monitor, work); + + ModelingUtils.exportSharedOntology(mon.split(1, SubMonitor.SUPPRESS_NONE), Simantics.getSession(), location, Constants.SHARED_LIBRARY_FORMAT, Constants.SHARED_LIBRARY_CURRENT_VERSION, info); + + Path input = location.toPath(); + + if (writeTg) { + try { + mon.subTask("Writing transferable graph"); + FixExportedOntology.createTGAndPGraph(input, false); + mon.worked(1); + } catch (Exception e) { + LOGGER.error("Could not generate Transferable Graph", e); + } + } + if (dumpStructure) { + try { + monitor.subTask("Dumping library structure"); + DumpOntologyStructure data = Simantics.getSession().syncRequest((ReadGraph graph) -> { + return new DumpOntologyStructure().read(graph, info.library.getResource()); }); data.write(new File(new File(location.getParent(), location.getName() + ".dump"), info.library.getName())); + mon.worked(1); } catch (Exception e) { - LOGGER.error("Could not generate TG and Pgraph", e); + LOGGER.error("Could not generate shared library structure dump", e); } - } + } } - + } diff --git a/bundles/org.simantics.modeling/scl/Simantics/Testing.scl b/bundles/org.simantics.modeling/scl/Simantics/Testing.scl index 5742b4f41..f1ebaf50c 100644 --- a/bundles/org.simantics.modeling/scl/Simantics/Testing.scl +++ b/bundles/org.simantics.modeling/scl/Simantics/Testing.scl @@ -7,7 +7,9 @@ importJava "org.simantics.modeling.ModelingUtils" where importJava "org.simantics.modeling.ContentDumps" where sclModuleContentDump :: Resource -> Vector Byte + sclScriptContentDump :: Resource -> Vector Byte pgraphContentDump :: Resource -> Vector Byte + graphFileContentDump :: Resource -> Vector Byte structuralComponentContentDump :: Resource -> Vector Byte genericParameterTypeContentDump :: Resource -> Vector Byte - + diff --git a/bundles/org.simantics.modeling/src/org/simantics/modeling/ContentDumps.java b/bundles/org.simantics.modeling/src/org/simantics/modeling/ContentDumps.java index bc3b65a03..687544e78 100644 --- a/bundles/org.simantics.modeling/src/org/simantics/modeling/ContentDumps.java +++ b/bundles/org.simantics.modeling/src/org/simantics/modeling/ContentDumps.java @@ -30,6 +30,12 @@ public class ContentDumps { return def.getBytes(UTF8); } + public static byte[] sclScriptContentDump(ReadGraph graph, Resource resource) throws DatabaseException { + Layer0 L0 = Layer0.getInstance(graph); + String def = graph.getRelatedValue(resource, L0.SCLScript_definition, Bindings.STRING); + return def.getBytes(UTF8); + } + public static byte[] pgraphContentDump(ReadGraph graph, Resource resource) throws DatabaseException { Layer0 L0 = Layer0.getInstance(graph); String def = graph.getRelatedValue(resource, L0.PGraph_definition, Bindings.STRING); diff --git a/bundles/org.simantics.modeling/src/org/simantics/modeling/ModelingUtils.java b/bundles/org.simantics.modeling/src/org/simantics/modeling/ModelingUtils.java index 4a4d97a18..1edf3cd31 100644 --- a/bundles/org.simantics.modeling/src/org/simantics/modeling/ModelingUtils.java +++ b/bundles/org.simantics.modeling/src/org/simantics/modeling/ModelingUtils.java @@ -29,6 +29,7 @@ import org.eclipse.core.runtime.CoreException; import org.eclipse.core.runtime.IProgressMonitor; import org.eclipse.core.runtime.NullProgressMonitor; import org.eclipse.core.runtime.Path; +import org.eclipse.core.runtime.SubMonitor; import org.eclipse.jface.dialogs.Dialog; import org.eclipse.jface.dialogs.IDialogConstants; import org.eclipse.jface.dialogs.MessageDialog; @@ -1515,53 +1516,46 @@ public class ModelingUtils { } public static TransferableGraph1 exportSharedOntology(IProgressMonitor monitor, RequestProcessor processor, File location, String format, int version, final LibraryInfo info) throws DatabaseException, IOException { - - if(monitor == null) monitor = new NullProgressMonitor(); - final IProgressMonitor finalMonitor = monitor; + SubMonitor mon = SubMonitor.convert(monitor, "Exporting shared library", 100); + // TODO: figure out a way to make the TG go directly into a file // instead of having it all in memory at once. - monitor.beginTask("Exporting shared library...", 100); - SimanticsClipboard clipboard = processor.syncRequest(new Read() { - @Override - public SimanticsClipboard perform(ReadGraph graph) throws DatabaseException { - CopyHandler ch = graph.adapt(info.library.getResource(), CopyHandler.class); - SimanticsClipboardImpl clipboard = new SimanticsClipboardImpl(); - ch.copyToClipboard(graph, clipboard, finalMonitor); - return clipboard; - } + SimanticsClipboard clipboard = processor.syncRequest((ReadGraph graph) -> { + CopyHandler ch = graph.adapt(info.library.getResource(), CopyHandler.class); + SimanticsClipboardImpl result = new SimanticsClipboardImpl(); + ch.copyToClipboard(graph, result, mon.split(5)); + return result; }); - + TreeMap metadata = getExportMetadata(); DraftStatusBean draft = info.draft; if(draft != null) { - metadata.put(DraftStatusBean.EXTENSION_KEY, new Variant(DraftStatusBean.BINDING ,draft)); + metadata.put(DraftStatusBean.EXTENSION_KEY, new Variant(DraftStatusBean.BINDING ,draft)); } - + for (Set object : clipboard.getContents()) { - + mon.subTask("Constructing exported material"); TransferableGraph1 tg = ClipboardUtils.accept(processor, object, SimanticsKeys.KEY_TRANSFERABLE_GRAPH); - monitor.worked(95); - + mon.worked(90); + Variant edb = tg.extensions.get(ExternalDownloadBean.EXTENSION_KEY); if(edb != null) { - metadata.put(ExternalDownloadBean.EXTENSION_KEY, edb); + metadata.put(ExternalDownloadBean.EXTENSION_KEY, edb); } if(location != null) { - monitor.setTaskName("Writing transferable graph..."); + monitor.subTask("Writing transferable graph"); DataContainers.writeFile(location, new DataContainer( format, version, metadata, new Variant(TransferableGraph1.BINDING, tg))); monitor.worked(5); } - + return tg; - } - - throw new DatabaseException("Failed to export"); - + + throw new DatabaseException("Export failed, no contents to export"); } public static TreeMap getExportMetadata() { diff --git a/bundles/org.simantics.modeling/src/org/simantics/modeling/utils/DumpOntologyStructure.java b/bundles/org.simantics.modeling/src/org/simantics/modeling/utils/DumpOntologyStructure.java index 79622f15e..5bb3d9ca2 100644 --- a/bundles/org.simantics.modeling/src/org/simantics/modeling/utils/DumpOntologyStructure.java +++ b/bundles/org.simantics.modeling/src/org/simantics/modeling/utils/DumpOntologyStructure.java @@ -30,7 +30,7 @@ public class DumpOntologyStructure { private static final Logger LOGGER = LoggerFactory.getLogger(DumpOntologyStructure.class); private Resource ontology; - + private Map names = new HashMap<>(); private Map parents = new HashMap<>(); private Map libraryFolders = new HashMap<>(); @@ -40,11 +40,11 @@ public class DumpOntologyStructure { parents.put(r, container); names.put(r, NameUtils.getSafeName(graph, r)); } - + private Collection containers() { return parents.values(); } - + private void readHierarchy(ReadGraph graph, Resource container) throws DatabaseException { Layer0 L0 = Layer0.getInstance(graph); for(Resource r : CommonDBUtils.objectsWithType(graph, container, L0.ConsistsOf, L0.Entity)) { @@ -94,30 +94,27 @@ public class DumpOntologyStructure { } } - public void read(ReadGraph graph, Resource ontology) throws DatabaseException { + public DumpOntologyStructure read(ReadGraph graph, Resource ontology) throws DatabaseException { this.ontology = ontology; readHierarchy(graph, ontology); readGeneric(graph); + return this; } - - private File escapeFile(File file) { + private File escapeFile(File file) { if(file.exists()) return file; - return new File(escapeFile(file.getParentFile()), FileUtils.escapeFileName(file.getName())); - } - + public void write(File unsafeFolder) throws IOException { File folder = escapeFile(unsafeFolder); - if(folder.exists()) - FileUtils.deleteAll(folder); + FileUtils.delete(folder.toPath()); folder.mkdirs(); writeDirectories(folder); writeResources(folder); } - + private File getFolder(File root, Resource library) { if(ontology.equals(library)) return root; @@ -147,11 +144,11 @@ public class DumpOntologyStructure { writeResource(rootFolder, r); } } - + private boolean isParent(Resource r) { return parents.values().contains(r); } - + private void writeResource(File rootFolder, Resource resource) throws IOException { byte[] dump = contentDumps.get(resource); if(dump == null) @@ -168,4 +165,4 @@ public class DumpOntologyStructure { FileUtils.writeFile(getFile(rootFolder, resource), bytes); } -} +} \ No newline at end of file -- 2.43.2