From: jsimomaa Date: Thu, 12 Jan 2017 11:21:57 +0000 (+0200) Subject: Add SCL support for exporting subscription data as CSV X-Git-Tag: v1.27.0~25 X-Git-Url: https://gerrit.simantics.org/r/gitweb?p=simantics%2Fplatform.git;a=commitdiff_plain;h=bf39e5c2544ec6e60d4e68996a6a7d4a84b30950 Add SCL support for exporting subscription data as CSV * Providing more user friendly SCL API and also documentation for the functions and required parameters * Renaming models -> items refs #6952 Change-Id: I9a20ac30403a93824683baa73747731009db2e5d --- diff --git a/bundles/org.simantics.charts/scl/Simantics/Chart.md b/bundles/org.simantics.charts/scl/Simantics/Chart.md index 50d055446..3f877fcfe 100644 --- a/bundles/org.simantics.charts/scl/Simantics/Chart.md +++ b/bundles/org.simantics.charts/scl/Simantics/Chart.md @@ -18,6 +18,20 @@ ChartGroup represents a ChartGroup instance in Simantics ontology (http://www.si ::value[createChartGroup] +## Export subscription data as CSV + +::data[CSVExportPlan] + +::data[SubscriptionCSVExportPlan] + +::value[exportSubscriptionsCSV] + +::value[columnSeparatorFromString] + +::value[decimalSeparatorFromString ] + +::value[exportInterpolationFromString ] + # Undocumented entities ::undocumented[] \ No newline at end of file diff --git a/bundles/org.simantics.charts/scl/Simantics/Chart.scl b/bundles/org.simantics.charts/scl/Simantics/Chart.scl index efbe3143f..3972fd95c 100644 --- a/bundles/org.simantics.charts/scl/Simantics/Chart.scl +++ b/bundles/org.simantics.charts/scl/Simantics/Chart.scl @@ -54,3 +54,150 @@ chartsOf model = recurse (toResource model) charts + concatMap recurse chartGrp isChart r = isInstanceOf r CHART.TimeSeriesChart isChartGroup r = isInstanceOf r CHART.ChartGroup + +//----------------------------------------------------------------------------- +// Support for exporting subscription data in CSV format + +import "UI/Progress" as Progress +import "File" + +importJava "org.simantics.history.csv.DecimalSeparator" where + data DecimalSeparator + + @JavaName fromPreference + """ + Possible arguments are: + + Dot (.) + Comma (,) + """ + decimalSeparatorFromString :: String -> DecimalSeparator + +importJava "org.simantics.history.csv.ColumnSeparator" where + data ColumnSeparator + + @JavaName fromPreference + """ + Possible arguments are: + + Comma (,) + Tabulator (\\t) + Semicolon (;) + Colon (:) + Space ( ) + """ + columnSeparatorFromString :: String -> ColumnSeparator + +importJava "org.simantics.history.csv.ExportInterpolation" where + data ExportInterpolation + + @JavaName fromPreference + """ + Possible arguments are: + + Linear Interpolation (lerp) + Previous Sample (previous) + """ + exportInterpolationFromString :: String -> ExportInterpolation + +importJava "org.simantics.charts.ui.CSVExportPlan" where + @FieldNames [startTime, timeStep, decimalSeparator, columnSeparator, resample, samplingMode, timeDigits, floatDigits, doubleDigits] + + """ + Example of construction: + + plan = CSVExportPlan { + startTime = 0.0, + timeStep = 1.0, + decimalSeparator = decimalSeparatorFromString ".", + columnSeparator = columnSeparatorFromString ",", + resample = True, + samplingMode = exportInterpolationFromString "lerp", + timeDigits = 7, + floatDigits = 9, + doubleDigits = 15 + } + """ + data CSVExportPlan = CSVExportPlan { + startTime :: Double, + timeStep :: Double, + decimalSeparator :: DecimalSeparator, + columnSeparator :: ColumnSeparator, + resample :: Boolean, + samplingMode :: ExportInterpolation , + timeDigits :: Integer, + floatDigits :: Integer, + doubleDigits :: Integer + } + + @JavaName setItems + @private + setCSVExportPlanItems :: CSVExportPlan -> [Resource] -> () + +""" +Example of construction: + + plan = SubscriptionCSVExportPlan { + modelName = "Model", + filePath = "D:/folder/output.csv", + subscriptionNames = ["Default"], + exportPlan = CSVExportPlan {} + } +""" +data SubscriptionCSVExportPlan = SubscriptionCSVExportPlan { + modelName :: String, + subscriptionNames :: [String], + filePath :: String, + exportPlan :: CSVExportPlan +} + +@private +modelNameOf SubscriptionCSVExportPlan { modelName } = modelName +@private +subscriptionNamesOf SubscriptionCSVExportPlan { subscriptionNames } = subscriptionNames +@private +filePathOf SubscriptionCSVExportPlan { filePath } = filePath +@private +exportPlanOf SubscriptionCSVExportPlan { exportPlan } = exportPlan + +importJava "org.simantics.charts.ui.CSVExporter" where + @JavaName doExport + @private + exportSubscriptionsCSVInternal :: Progress.ProgressMonitor -> File -> CSVExportPlan -> () + +""" +Exports subscription data as CSV values in a similar manner as the CSV Exporter provided by the user interface + +Example of usage: + + exportSubscriptionsCSV SubscriptionCSVExportPlan { + modelName = "Model", + filePath = "D:/folder/output.csv", + subscriptionNames = ["Default"], + exportPlan = CSVExportPlan { + startTime = 0.0, + timeStep = 1.0, + decimalSeparator = decimalSeparatorFromString ".", + columnSeparator = columnSeparatorFromString ",", + resample = True, + samplingMode = exportInterpolationFromString "lerp", + timeDigits = 7, + floatDigits = 9, + doubleDigits = 15 + } + } +""" +exportSubscriptionsCSV :: SubscriptionCSVExportPlan -> () +exportSubscriptionsCSV subscriptionExportPlan = do + items = syncRead (\_ -> resolveSubscriptionItems (modelNameOf subscriptionExportPlan) (subscriptionNamesOf subscriptionExportPlan)) + csvExportPlan = exportPlanOf subscriptionExportPlan + setCSVExportPlanItems csvExportPlan items + exportSubscriptionsCSVInternal (Progress.createNullProgressMonitor ()) (file (filePathOf subscriptionExportPlan)) csvExportPlan + +@private +resolveSubscriptionItems :: String -> [String] -> [Resource] +resolveSubscriptionItems modelName subscriptionNames = concatMap itemsOf (filter nameFilter subscriptions) + where + nameFilter sub = elem (relatedValue2 sub L0.HasLabel) subscriptionNames + subscriptions = (objectsWithType (model modelName) L0.ConsistsOf MOD.Subscription) + itemsOf subscription = objectsWithType subscription L0.ConsistsOf MOD.Subscription.Item diff --git a/bundles/org.simantics.charts/src/org/simantics/charts/ui/CSVExportPage.java b/bundles/org.simantics.charts/src/org/simantics/charts/ui/CSVExportPage.java index 002884d98..047376dbb 100644 --- a/bundles/org.simantics.charts/src/org/simantics/charts/ui/CSVExportPage.java +++ b/bundles/org.simantics.charts/src/org/simantics/charts/ui/CSVExportPage.java @@ -410,7 +410,7 @@ public class CSVExportPage extends WizardPage { if(hasInitial) { model.select(i); item.removeAll(); - exportModel.models.clear(); + exportModel.items.clear(); int index = 0; int firstIndex = 0; for(NamedResource nr : p.second) { @@ -418,7 +418,7 @@ public class CSVExportPage extends WizardPage { ti.setText(nr.getName()); ti.setData(nr); if(selected.contains(nr)) { - exportModel.models.add(nr); + exportModel.items.add(nr.getResource()); ti.setChecked(true); if(firstIndex == 0) firstIndex = index; } @@ -498,7 +498,7 @@ public class CSVExportPage extends WizardPage { Pair> p = getModel(selectedModel); if(p != null) { - HashSet checked = new HashSet<>(); + Set checked = new HashSet<>(); NamedResource existing = (NamedResource)item.getData(); if(!p.first.equals(existing)) { @@ -514,10 +514,10 @@ public class CSVExportPage extends WizardPage { } for(TableItem ti : item.getItems()) { - if(ti.getChecked()) checked.add((NamedResource)ti.getData()); + if(ti.getChecked()) checked.add(((NamedResource)ti.getData()).getResource()); } - exportModel.models = checked; + exportModel.items = checked; } @@ -545,7 +545,7 @@ public class CSVExportPage extends WizardPage { } else { - if(exportModel.models.size() > 1) { + if(exportModel.items.size() > 1) { setErrorMessage("Recorded samples can only be exported for a single subscription item."); setPageComplete(false); return; @@ -559,7 +559,7 @@ public class CSVExportPage extends WizardPage { return; } - if (exportModel.models.isEmpty()) { + if (exportModel.items.isEmpty()) { setErrorMessage("No items selected for export."); setPageComplete(false); return; diff --git a/bundles/org.simantics.charts/src/org/simantics/charts/ui/CSVExportPlan.java b/bundles/org.simantics.charts/src/org/simantics/charts/ui/CSVExportPlan.java index 7902cee37..762fcd95a 100644 --- a/bundles/org.simantics.charts/src/org/simantics/charts/ui/CSVExportPlan.java +++ b/bundles/org.simantics.charts/src/org/simantics/charts/ui/CSVExportPlan.java @@ -12,11 +12,14 @@ package org.simantics.charts.ui; import java.io.File; +import java.util.ArrayDeque; import java.util.Collection; import java.util.Deque; import java.util.HashSet; +import java.util.List; -import org.simantics.db.common.NamedResource; +import org.simantics.Simantics; +import org.simantics.db.Resource; import org.simantics.db.management.ISessionContext; import org.simantics.history.csv.ColumnSeparator; import org.simantics.history.csv.DecimalSeparator; @@ -35,19 +38,19 @@ public class CSVExportPlan { Deque recentLocations; // Output - Collection models = new HashSet(); - File exportLocation; + Collection items = new HashSet<>(); + File exportLocation; - double startTime; - double timeStep; - DecimalSeparator decimalSeparator; - ColumnSeparator columnSeparator; - boolean resample; - ExportInterpolation samplingMode; - int timeDigits; - int floatDigits; - int doubleDigits; + public double startTime; + public double timeStep; + public DecimalSeparator decimalSeparator; + public ColumnSeparator columnSeparator; + public boolean resample; + public ExportInterpolation samplingMode; + public int timeDigits; + public int floatDigits; + public int doubleDigits; /** @@ -55,6 +58,28 @@ public class CSVExportPlan { */ boolean overwrite; + /** + * Constructor for SCL support + */ + public CSVExportPlan(double startTime, double timeStep, DecimalSeparator dsep, ColumnSeparator csep, boolean resample, ExportInterpolation mode, int timeDigits, int floatDigits, int doubleDigits) { + this.sessionContext = Simantics.getSessionContext(); + this.recentLocations = new ArrayDeque<>(); + + this.startTime = startTime; + this.timeStep = timeStep; + this.decimalSeparator = dsep; + this.columnSeparator = csep; + this.resample = resample; + this.samplingMode = mode; + this.timeDigits = timeDigits; + this.floatDigits = floatDigits; + this.doubleDigits = doubleDigits; + } + + public void setItems(List items) { + this.items = items; + } + CSVExportPlan(ISessionContext sessionContext, Deque recentLocations) { this.sessionContext = sessionContext; this.recentLocations = recentLocations; diff --git a/bundles/org.simantics.charts/src/org/simantics/charts/ui/CSVExporter.java b/bundles/org.simantics.charts/src/org/simantics/charts/ui/CSVExporter.java index d4d480601..9798b57f2 100644 --- a/bundles/org.simantics.charts/src/org/simantics/charts/ui/CSVExporter.java +++ b/bundles/org.simantics.charts/src/org/simantics/charts/ui/CSVExporter.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2012 Association for Decentralized Information Management in + * Copyright (c) 2012,2017 Association for Decentralized Information Management in * Industry THTH ry. * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Public License v1.0 @@ -21,7 +21,6 @@ import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; -import java.util.List; import java.util.Set; import org.eclipse.core.commands.ExecutionException; @@ -42,8 +41,6 @@ import org.simantics.databoard.binding.error.BindingException; import org.simantics.databoard.serialization.SerializationException; import org.simantics.db.ReadGraph; import org.simantics.db.Resource; -import org.simantics.db.Session; -import org.simantics.db.common.NamedResource; import org.simantics.db.common.request.UniqueRead; import org.simantics.db.common.utils.Logger; import org.simantics.db.exception.DatabaseException; @@ -83,7 +80,6 @@ public class CSVExporter implements IRunnableWithProgress { } void exportModel(SubMonitor mon) throws IOException, DatabaseException, SerializationException, BindingException{ - try { doExport(mon, exportModel.exportLocation, exportModel); } catch (ExecutionException e) { @@ -94,16 +90,15 @@ public class CSVExporter implements IRunnableWithProgress { } finally { mon.setWorkRemaining(0); } - } - - private static Set resolveContainingModels(final Collection res) throws DatabaseException { + + private static Set resolveContainingModels(Collection res) throws DatabaseException { return Simantics.getSession().syncRequest(new UniqueRead>() { @Override public Set perform(ReadGraph graph) throws DatabaseException { - Set models = new HashSet(); - for (NamedResource r : res) { - Resource m = graph.syncRequest(new PossibleModel(r.getResource())); + Set models = new HashSet<>(); + for (Resource r : res) { + Resource m = graph.syncRequest(new PossibleModel(r)); if (m != null) models.add(m); } @@ -111,9 +106,8 @@ public class CSVExporter implements IRunnableWithProgress { } }); } - + public static void doExport(IProgressMonitor monitor, final File f, final CSVExportPlan plan) throws ExecutionException, IOException { - IScopeContext context = InstanceScope.INSTANCE; Preferences node = context.getNode(CSVPreferences.P_NODE); @@ -138,7 +132,7 @@ public class CSVExporter implements IRunnableWithProgress { Set models; try { - models = resolveContainingModels(plan.models); + models = resolveContainingModels(plan.items); } catch (DatabaseException e3) { throw new ExecutionException("Containing model resolution failed.", e3); } @@ -148,7 +142,7 @@ public class CSVExporter implements IRunnableWithProgress { throw new ExecutionException("Selected resources are part of several models, only subscriptions from a single model can be selected"); Resource model = models.iterator().next(); Key chartDataKey = ChartKeys.chartSourceKey(model); - + final ChartData data = Simantics.getProject().getHint(chartDataKey); if ( data == null ) { throw new ExecutionException("There is no "+chartDataKey); @@ -169,50 +163,38 @@ public class CSVExporter implements IRunnableWithProgress { csv.setFloatFormat( FormattingUtils.significantDigitFormat( plan.floatDigits ) ); csv.setNumberFormat( FormattingUtils.significantDigitFormat( plan.doubleDigits ) ); - try { - Session session = Simantics.getSession(); - List list = new ArrayList(); - for(NamedResource nr : plan.models) list.add(nr.getResource()); - session.sync( new CSVParamsQuery(data.history, csv, list) ); - csv.sort(); - } catch (DatabaseException e2) { - throw new ExecutionException(e2.getMessage(), e2); - } catch (HistoryException e) { - throw new ExecutionException(e.getMessage(), e); - } - - try { - // Ensure all views are built. - monitor.beginTask("Exporting Time Series as CSV...", IProgressMonitor.UNKNOWN); - try { - data.collector.flush(); - if ( !f.exists() ) { - f.createNewFile(); - } else { - RandomAccessFile raf = new RandomAccessFile(f, "rw"); - raf.setLength(0); - raf.close(); - } - - FileOutputStream fos = new FileOutputStream(f, true); - BufferedOutputStream bos = new BufferedOutputStream( fos ); - try { - PrintStream ps = new PrintStream( bos ); - csv.formulate2( new CSVProgressMonitor( monitor ), ps ); - bos.flush(); - } finally { - fos.close(); - } - } catch (HistoryException e) { - throw new ExecutionException(e.getMessage(), e); - } catch (IOException e1) { - throw new ExecutionException(e1.getMessage(), e1); - } - monitor.setTaskName("Done"); - } finally { - monitor.done(); - } - + try { + Simantics.getSession().syncRequest( + new CSVParamsQuery(data.history, csv, + new ArrayList<>(plan.items)) ); + csv.sort(); + + // Ensure all views are built. + monitor.beginTask("Exporting Time Series as CSV...", IProgressMonitor.UNKNOWN); + data.collector.flush(); + + // Truncate existing file it if happens to exist. + try (RandomAccessFile raf = new RandomAccessFile(f, "rw")) { + raf.setLength(0); + } + + // Write CSV + try (PrintStream ps = new PrintStream( + new BufferedOutputStream( + new FileOutputStream(f, true) ))) + { + csv.formulate2( new CSVProgressMonitor( monitor ), ps ); + ps.flush(); + } + + monitor.setTaskName("Done"); + } catch (DatabaseException e2) { + throw new ExecutionException(e2.getMessage(), e2); + } catch (HistoryException e) { + throw new ExecutionException(e.getMessage(), e); + } finally { + monitor.done(); + } } - + }