--- /dev/null
+/*******************************************************************************\r
+ * Copyright (c) 2011 Association for Decentralized Information Management in\r
+ * Industry THTH ry.\r
+ * All rights reserved. This program and the accompanying materials\r
+ * are made available under the terms of the Eclipse Public License v1.0\r
+ * which accompanies this distribution, and is available at\r
+ * http://www.eclipse.org/legal/epl-v10.html\r
+ *\r
+ * Contributors:\r
+ * VTT Technical Research Centre of Finland - initial API and implementation\r
+ *******************************************************************************/\r
+package org.simantics.history.csv;\r
+\r
+import java.io.IOException;\r
+import java.io.UnsupportedEncodingException;\r
+import java.math.BigDecimal;\r
+import java.net.URLDecoder;\r
+import java.text.DecimalFormat;\r
+import java.text.DecimalFormatSymbols;\r
+import java.text.Format;\r
+import java.util.ArrayList;\r
+import java.util.Collections;\r
+import java.util.List;\r
+import java.util.Locale;\r
+\r
+import org.simantics.databoard.accessor.StreamAccessor;\r
+import org.simantics.databoard.accessor.error.AccessorException;\r
+import org.simantics.history.HistoryException;\r
+import org.simantics.history.HistoryManager;\r
+import org.simantics.history.util.HistoryExportUtil;\r
+import org.simantics.history.util.ProgressMonitor;\r
+import org.simantics.history.util.StreamIterator;\r
+import org.simantics.history.util.ValueBand;\r
+\r
+/**\r
+ * CSV writer for history items.\r
+ * \r
+ * @author Toni Kalajainen\r
+ * @author Tuukka Lehtonen\r
+ */\r
+public class CSVFormatter {\r
+\r
+ List<Item> items = new ArrayList<Item>();\r
+ double from = -Double.MAX_VALUE;\r
+ double end = Double.MAX_VALUE;\r
+ double startTime = 0.0;\r
+ double timeStep = 0.0;\r
+ ColumnSeparator columnSeparator;\r
+ DecimalSeparator decimalSeparator;\r
+ boolean resample;\r
+ String lineFeed;\r
+ Locale locale;\r
+\r
+ Format timeFormat;\r
+ Format floatFormat;\r
+ Format numberFormat;\r
+\r
+ Formatter timeFormatter;\r
+ Formatter floatFormatter;\r
+ Formatter numberFormatter;\r
+\r
+ ExportInterpolation numberInterpolation = ExportInterpolation.LINEAR_INTERPOLATION;\r
+\r
+ public CSVFormatter() {\r
+ this.lineFeed = resolvePlatformLineFeed();\r
+ this.locale = Locale.getDefault(Locale.Category.FORMAT);\r
+\r
+ DecimalFormat defaultFormat = new DecimalFormat();\r
+ defaultFormat.setGroupingUsed(false);\r
+ setTimeFormat(defaultFormat);\r
+ setFloatFormat(defaultFormat);\r
+ setNumberFormat(defaultFormat);\r
+ }\r
+\r
+ /**\r
+ * Add item to formatter\r
+ * \r
+ * @param historyItemId\r
+ * @param label\r
+ * @param variableReference\r
+ * @param unit\r
+ */\r
+ public void addItem( HistoryManager history, String historyItemId, String label, String variableReference, String unit ) {\r
+ Item i = new Item();\r
+ i.history = history;\r
+ i.label = label!=null?label:"";\r
+ i.variableReference = variableReference!=null?variableReference:"";\r
+ i.variableReference = unescape(i.variableReference);\r
+ i.historyItemId = historyItemId;\r
+ i.unit = unit;\r
+ if ( !items.contains(i) ) items.add( i );\r
+ }\r
+\r
+ private static String unescape(String url) {\r
+ try {\r
+ return URLDecoder.decode(url, "UTF-8");\r
+ } catch (UnsupportedEncodingException e) {\r
+ return url;\r
+ }\r
+ }\r
+\r
+ /**\r
+ * Sort items by variableId, label1, label2\r
+ */\r
+ public void sort() {\r
+ Collections.sort(items);\r
+ }\r
+ \r
+ public void setTimeRange( double from, double end ) {\r
+ this.from = from;\r
+ this.end = end;\r
+ }\r
+ \r
+ public void setStartTime( double startTime ) {\r
+ this.startTime = startTime; \r
+ }\r
+\r
+ public void setTimeStep( double timeStep ) {\r
+ this.timeStep = timeStep; \r
+ }\r
+ \r
+ void openHistory() throws HistoryException {\r
+ try {\r
+ for (Item item : items) item.open();\r
+ } catch (HistoryException e) {\r
+ for (Item item : items) item.close();\r
+ throw e;\r
+ }\r
+ }\r
+ \r
+ void closeHistory() {\r
+ for (Item item : items) item.close();\r
+ }\r
+ \r
+ /**\r
+ * Reads visible data of all variables and formats as CSV lines (Comma \r
+ * Separated Values). Line Feed is \n, variable separator is \t, and\r
+ * decimal separator locale dependent.\r
+ * \r
+ * ReadData1 outputs separate time and value columns for each variable\r
+ * \r
+ * Variable1 Time | Variable1 Value | Variable2 Time | Variable2 Value\r
+ * 0.0 | 1.0 | 0.1 | 23423.0\r
+ * \r
+ * @param monitor\r
+ * @param sb\r
+ * @throws HistoryException \r
+ * @throws IOException \r
+ */\r
+ /*\r
+ public void formulate1( ProgressMonitor monitor, Appendable sb ) throws HistoryException, IOException {\r
+ if (items.isEmpty()) return;\r
+ boolean adaptComma = decimalSeparatorInLocale != decimalSeparator;\r
+ openHistory();\r
+ try {\r
+ // Prepare columns: First time\r
+ for (Item item : items)\r
+ {\r
+ if (monitor.isCanceled())\r
+ return;\r
+ if (item.stream.isEmpty()) continue;\r
+ Double firstTime = (Double) item.stream.getFirstTime( Bindings.DOUBLE );\r
+ if (from <= firstTime) {\r
+ item.time = firstTime;\r
+ } else {\r
+ item.time = (Double) item.stream.getFloorTime(Bindings.DOUBLE, from);\r
+ }\r
+ } \r
+ \r
+ // Write Headers\r
+ for (Item i : items)\r
+ {\r
+ if (monitor.isCanceled())\r
+ return;\r
+ boolean lastColumn = i == items.get( items.size()-1 );\r
+ sb.append(i.label + " Time");\r
+ sb.append( columnSeparator );\r
+ sb.append(i.label + " Value");\r
+ sb.append(lastColumn ? lineFeed : columnSeparator );\r
+ }\r
+ \r
+ // Iterate until endTime is met for all variables\r
+ int readyColumns;\r
+ do {\r
+ if (monitor.isCanceled())\r
+ return;\r
+\r
+ readyColumns = 0;\r
+ for (Item i : items)\r
+ {\r
+ boolean lastColumn = i == items.get( items.size()-1 );\r
+ \r
+ if (i.time == null || i.time > end) {\r
+ readyColumns++;\r
+ sb.append( lastColumn ? columnSeparator+lineFeed : columnSeparator+columnSeparator);\r
+ continue;\r
+ }\r
+ \r
+ sb.append("");\r
+ \r
+ // Write time\r
+ String timeStr = format.format( i.time );\r
+ if ( adaptComma ) timeStr = timeStr.replace(decimalSeparatorInLocale, decimalSeparator);\r
+ sb.append( timeStr ); \r
+ sb.append( columnSeparator );\r
+ \r
+ // Write value \r
+ i.value = i.stream.getValue(Bindings.DOUBLE, i.time);\r
+ if (i.value instanceof Number) {\r
+ String str = format.format( i.value );\r
+ if ( adaptComma ) str = str.replace(decimalSeparatorInLocale, decimalSeparator);\r
+ sb.append( str );\r
+ } else if (i.value instanceof Boolean) {\r
+ sb.append( (Boolean)i.value ? "1": "0");\r
+ } else {\r
+ sb.append( i.value.toString() );\r
+ }\r
+ sb.append(lastColumn ? lineFeed : columnSeparator);\r
+ \r
+ // Get next time\r
+ i.time = (Double) i.stream.getHigherTime(Bindings.DOUBLE, i.time);\r
+ }\r
+ \r
+ } while (readyColumns < items.size());\r
+ } finally {\r
+ closeHistory();\r
+ }\r
+ }*/\r
+ \r
+ /**\r
+ * Reads visible data of all variables and formats as CSV lines (Comma \r
+ * Separated Values). Line Feed is \n, variable separator is \t, and\r
+ * decimal separator locale dependent.\r
+ * \r
+ * ReadData2 outputs one shared time and one value column for each variable\r
+ * \r
+ * Time | Variable1 Label | Variable3 Label\r
+ * | Variable1 Id | Variable3 Id\r
+ * | Variable1 Unit | Variable3 Unit\r
+ * 0.0 | 1.0 | 0.1\r
+ * \r
+ * @param monitor\r
+ * @param sb\r
+ * @throws HistoryException \r
+ * @throws IOException \r
+ */\r
+ public void formulate2( ProgressMonitor monitor, Appendable sb ) throws HistoryException, IOException\r
+ {\r
+ if ( items.isEmpty() ) return;\r
+\r
+ timeFormatter = evaluateFormatter(timeFormat, decimalSeparator);\r
+ floatFormatter = evaluateFormatter(floatFormat, decimalSeparator);\r
+ numberFormatter = evaluateFormatter(numberFormat, decimalSeparator);\r
+\r
+ openHistory();\r
+ try {\r
+ // What is the time range of all items combined\r
+ double allFrom = Double.MAX_VALUE;\r
+ double allEnd = -Double.MAX_VALUE;\r
+ for (Item i : items) {\r
+ if (i.iter.isEmpty()) continue;\r
+ allFrom = Math.min(allFrom, i.iter.getFirstTime());\r
+ allEnd = Math.max(allEnd, i.iter.getLastTime());\r
+ }\r
+ \r
+ // Write Headers\r
+ for (int hl = 0; hl < 3; ++hl) {\r
+ switch(hl) {\r
+ case 0: sb.append("Time"); break;\r
+ case 1: sb.append("----"); break;\r
+ case 2: sb.append("Unit"); break;\r
+ }\r
+ sb.append( columnSeparator.preference );\r
+ for (Item i : items)\r
+ {\r
+ boolean lastColumn = i == items.get( items.size()-1 );\r
+ switch (hl) {\r
+ case 0:\r
+ sb.append(i.label != null ? i.label : "");\r
+ break;\r
+ case 1:\r
+ sb.append(i.variableReference != null ? i.variableReference : "");\r
+ break;\r
+ case 2:\r
+ sb.append(i.unit==null?"no unit":i.unit);\r
+ break;\r
+ }\r
+ if (!lastColumn) sb.append( columnSeparator.preference );\r
+ }\r
+ sb.append( lineFeed );\r
+ }\r
+ \r
+ // Prepare time \r
+ boolean hasAnyValues = allFrom != Double.MAX_VALUE && allEnd != -Double.MAX_VALUE;\r
+ \r
+ // Make intersection of actual data range (allFrom, allEnd) and requested data (from, end)\r
+ double _from = Double.MAX_VALUE, _end = -Double.MAX_VALUE; \r
+ if (hasAnyValues) {\r
+ _from = Math.max(allFrom, from);\r
+ _end = Math.min(allEnd, end);\r
+ }\r
+\r
+ if (!hasAnyValues) return;\r
+\r
+ // Iterate until endTime is met for all variables\r
+ double time = _from;\r
+\r
+ if(!resample) {\r
+ \r
+ // If resample is false then all samples are reported as is. The code achieves this by setting startTime to _from and timeStep to 0.0 \r
+ time = _from;\r
+ timeStep = 0.0;\r
+ \r
+ } else {\r
+\r
+ // time = startTime + n*timeStep \r
+ \r
+ // Sampling based on given startTime and timeStep\r
+ if(timeStep > 0) {\r
+\r
+ // Find the first sample time that contains data \r
+ double n = Math.max(0, Math.ceil((_from-startTime) / timeStep));\r
+ time = startTime + n*timeStep;\r
+\r
+ } else {\r
+ \r
+ // Start sampling from startTime but make sure that it is not less than _from\r
+ if(startTime > _from) time = startTime;\r
+ \r
+ }\r
+ \r
+\r
+ }\r
+ \r
+ // Must convert double times to String when initializing BigDecimal.\r
+ // Otherwise BigDecimal will pick up inaccuracies from beyond 15 precise digits\r
+ // thus making a mess of the time step calculations.\r
+\r
+ BigDecimal bigTime = new BigDecimal(String.valueOf(time));\r
+ BigDecimal bigTimeStep = new BigDecimal(String.valueOf(timeStep));\r
+\r
+ for (Item i : items) i.iter.gotoTime(time);\r
+ do {\r
+ if ( monitor!=null && monitor.isCanceled() ) return;\r
+\r
+ // Write time\r
+ String timeStr = timeFormatter.format( time );\r
+ //System.out.println("SAMPLING TIME: " + time);\r
+ sb.append( timeStr );\r
+\r
+ // Write values\r
+ for (Item i : items)\r
+ {\r
+ sb.append( columnSeparator.preference );\r
+ \r
+ // Write value\r
+ if ( i.iter.hasValidValue() ) {\r
+ Object value = i.iter.getValueBand().getValue();\r
+ if (value instanceof Number) {\r
+ if (value instanceof Float || value instanceof Double) {\r
+ switch (numberInterpolation) {\r
+ case PREVIOUS_SAMPLE:\r
+ sb.append( formatNumber(value) );\r
+ break;\r
+\r
+ case LINEAR_INTERPOLATION:\r
+ if (time != i.iter.getValueBand().getTimeDouble() && i.iter.hasNext()) {\r
+ \r
+ // Interpolate\r
+ int currentIndex = i.iter.getIndex();\r
+ ValueBand band = i.iter.getValueBand();\r
+ //double t1 = band.getTimeDouble();\r
+ Number v1 = (Number) value;\r
+ double t12 = band.getEndTimeDouble();\r
+ i.iter.next();\r
+ double t2 = i.iter.getValueBand().getTimeDouble();\r
+ Number v2 = (Number) i.iter.getValueBand().getValue();\r
+ i.iter.gotoIndex(currentIndex);\r
+\r
+ double vs = v1.doubleValue();\r
+ if(time > t12)\r
+ vs = HistoryExportUtil.biglerp(t12, v1.doubleValue(), t2, v2.doubleValue(), time);\r
+\r
+ sb.append( formatDouble(vs) );\r
+ } else {\r
+ // Exact timestamp match, or last sample.\r
+ // Don't interpolate nor extrapolate.\r
+ sb.append( formatNumber(value) );\r
+ }\r
+ break;\r
+ default:\r
+ throw new UnsupportedOperationException("Unsupported interpolation: " + numberInterpolation);\r
+ }\r
+ } else {\r
+ sb.append( value.toString() );\r
+ }\r
+ } else if (value instanceof Boolean) {\r
+ sb.append( (Boolean)value ? "1": "0");\r
+ } else {\r
+ sb.append( value.toString() );\r
+ }\r
+ }\r
+ }\r
+\r
+ sb.append( lineFeed );\r
+\r
+ // Read next values, and the following times\r
+ if ( timeStep>0.0 ) {\r
+ bigTime = bigTime.add(bigTimeStep);\r
+ time = bigTime.doubleValue();\r
+ } else {\r
+ // Get smallest end time that is larger than current time\r
+ Double nextTime = null;\r
+// System.out.println("time = "+time);\r
+ for (Item i : items) {\r
+ Double itemNextTime = i.iter.getNextTime( time );\r
+// System.err.println(" "+i.label+" nextTime="+itemNextTime);\r
+ if ( itemNextTime == null ) continue;\r
+ if ( itemNextTime < time ) continue;\r
+ if ( nextTime == null || ( nextTime > itemNextTime && !itemNextTime.equals( time ) ) ) nextTime = itemNextTime; \r
+ }\r
+ if ( nextTime == null || nextTime.equals( time ) ) break;\r
+ time = nextTime;\r
+ }\r
+\r
+ boolean hasMore = false;\r
+ \r
+ for (Item i : items) {\r
+ i.iter.proceedToTime(time);\r
+ if(contains(i, time)) hasMore = true;\r
+ }\r
+ \r
+ if(!hasMore) break;\r
+ \r
+ } while (time<=_end);\r
+ } finally {\r
+ closeHistory();\r
+ }\r
+ }\r
+\r
+ private boolean contains(Item item, double time) {\r
+ double start = item.iter.getStartTime();\r
+ double end = item.iter.getEndTime();\r
+ // A special case, where start == end => accept\r
+ if(time == start) return true;\r
+ else if(time < start) return false;\r
+ else if(time >= end) return false;\r
+ else return true;\r
+ }\r
+\r
+ private CharSequence formatNumber(Object value) {\r
+ return value instanceof Float\r
+ ? floatFormatter.format( value )\r
+ : numberFormatter.format( value );\r
+ }\r
+\r
+ private CharSequence formatDouble(double value) {\r
+ return numberFormatter.format( value );\r
+ }\r
+\r
+ public void setDecimalSeparator(DecimalSeparator separator) {\r
+ this.decimalSeparator = separator;\r
+ }\r
+\r
+ public void setColumnSeparator(ColumnSeparator separator) {\r
+ this.columnSeparator = separator;\r
+ }\r
+ \r
+ public void setResample(boolean resample) {\r
+ this.resample = resample;\r
+ }\r
+ \r
+ public void setLineFeed( String lf ) {\r
+ this.lineFeed = lf;\r
+ }\r
+\r
+ public void setTimeFormat(Format format) {\r
+ this.timeFormat = format;\r
+ }\r
+\r
+ public void setFloatFormat(Format format) {\r
+ this.floatFormat = format;\r
+ }\r
+\r
+ public void setNumberFormat(Format format) {\r
+ this.numberFormat = format;\r
+ }\r
+\r
+ public void setLocale(Locale locale) {\r
+ this.locale = locale;\r
+ }\r
+\r
+ public void setNumberInterpolation(ExportInterpolation interpolation) {\r
+ this.numberInterpolation = interpolation;\r
+ }\r
+\r
+ private static String resolvePlatformLineFeed() {\r
+ String osName = System.getProperty("os.name", "");\r
+ osName = osName.toLowerCase();\r
+ if (osName.contains("windows"))\r
+ return "\r\n";\r
+ return "\n";\r
+ }\r
+\r
+ private class Item implements Comparable<Item> {\r
+ // Static data\r
+ String label; // Label\r
+ String variableReference; // Label\r
+ HistoryManager history; // History source for this item\r
+ String historyItemId;\r
+ String unit;\r
+\r
+ // State data\r
+ StreamAccessor accessor; // Stream accessor\r
+ StreamIterator iter;\r
+ \r
+ public void open() throws HistoryException {\r
+ accessor = history.openStream(historyItemId, "r");\r
+ iter = new StreamIterator( accessor );\r
+ }\r
+ \r
+ public void close() {\r
+ if (accessor!=null) {\r
+ try {\r
+ accessor.close();\r
+ } catch (AccessorException e) {\r
+ }\r
+ }\r
+ accessor = null;\r
+ iter = null;\r
+ }\r
+\r
+ @Override\r
+ public int compareTo(Item o) {\r
+ int i;\r
+ i = label.compareTo(o.label);\r
+ if (i!=0) return i;\r
+ i = variableReference.compareTo(o.variableReference);\r
+ if (i!=0) return i;\r
+ i = historyItemId.compareTo(o.historyItemId); \r
+ if (i!=0) return i;\r
+ return 0;\r
+ }\r
+ \r
+ @Override\r
+ public int hashCode() {\r
+ int code = 0x2304;\r
+ code = 13*code + variableReference.hashCode();\r
+ code = 13*code + label.hashCode();\r
+ code = 13*code + historyItemId.hashCode();\r
+ code = 13*code + history.hashCode(); \r
+ return code;\r
+ }\r
+ \r
+ @Override\r
+ public boolean equals(Object obj) {\r
+ if ( obj == null ) return false;\r
+ if ( obj instanceof Item == false ) return false;\r
+ Item other = (Item) obj; \r
+ if ( !other.label.equals(label) ) return false;\r
+ if ( !other.variableReference.equals(variableReference) ) return false;\r
+ if ( !other.history.equals(history) ) return false;\r
+ if ( !other.historyItemId.equals(historyItemId) ) return false;\r
+ return true;\r
+ }\r
+ \r
+ }\r
+\r
+ static interface Formatter {\r
+ String format(Object number);\r
+ }\r
+\r
+ static class NopFormatter implements Formatter {\r
+ private final Format format;\r
+ public NopFormatter(Format format) {\r
+ this.format = format;\r
+ }\r
+ public String format(Object number) {\r
+ return format.format(number);\r
+ }\r
+ }\r
+\r
+ static class ReplacingFormatter implements Formatter {\r
+ private final Format format;\r
+ private final char from;\r
+ private final char to;\r
+ public ReplacingFormatter(Format format, char from, char to) {\r
+ this.format = format;\r
+ this.from = from;\r
+ this.to = to;\r
+ }\r
+ public String format(Object number) {\r
+ return format.format(number).replace(from, to);\r
+ }\r
+ }\r
+\r
+ private Formatter evaluateFormatter(Format format, DecimalSeparator target) {\r
+ // Probe decimal separator\r
+ String onePointTwo = format.format(1.2);\r
+ System.out.println("formatted zeroPointOne: " + onePointTwo);\r
+\r
+ DecimalSeparator formatSeparator;\r
+ if (onePointTwo.indexOf('.') != -1) {\r
+ formatSeparator = DecimalSeparator.DOT;\r
+ } else if (onePointTwo.indexOf(',') != -1) {\r
+ formatSeparator = DecimalSeparator.COMMA;\r
+ } else {\r
+ DecimalFormatSymbols symbols = DecimalFormatSymbols.getInstance(locale);\r
+ formatSeparator = DecimalSeparator.fromChar(symbols.getDecimalSeparator());\r
+ }\r
+\r
+ switch (formatSeparator) {\r
+ case COMMA:\r
+ switch (target) {\r
+ case COMMA:\r
+ return new NopFormatter(format);\r
+ case DOT:\r
+ return new ReplacingFormatter(format, ',', '.');\r
+ }\r
+ case DOT:\r
+ switch (target) {\r
+ case COMMA:\r
+ return new ReplacingFormatter(format, '.', ',');\r
+ case DOT:\r
+ return new NopFormatter(format);\r
+ }\r
+ }\r
+ return new NopFormatter(format);\r
+ }\r
+\r
+}\r
Code Review / simantics / platform.git / blobdiff