X-Git-Url: https://gerrit.simantics.org/r/gitweb?a=blobdiff_plain;f=bundles%2Forg.simantics.history%2Fsrc%2Forg%2Fsimantics%2Fhistory%2Fcsv%2FCSVFormatter.java;h=b04e0dd37b37e34d0398855a451cd93cb534d6ff;hb=HEAD;hp=5ff6db54ad4ca56f17d93819dbdc6835e70af3d8;hpb=969bd23cab98a79ca9101af33334000879fb60c5;p=simantics%2Fplatform.git diff --git a/bundles/org.simantics.history/src/org/simantics/history/csv/CSVFormatter.java b/bundles/org.simantics.history/src/org/simantics/history/csv/CSVFormatter.java index 5ff6db54a..b04e0dd37 100644 --- a/bundles/org.simantics.history/src/org/simantics/history/csv/CSVFormatter.java +++ b/bundles/org.simantics.history/src/org/simantics/history/csv/CSVFormatter.java @@ -1,631 +1,662 @@ -/******************************************************************************* - * Copyright (c) 2011 Association for Decentralized Information Management in - * Industry THTH ry. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Eclipse Public License v1.0 - * which accompanies this distribution, and is available at - * http://www.eclipse.org/legal/epl-v10.html - * - * Contributors: - * VTT Technical Research Centre of Finland - initial API and implementation - *******************************************************************************/ -package org.simantics.history.csv; - -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.math.BigDecimal; -import java.net.URLDecoder; -import java.text.DecimalFormat; -import java.text.DecimalFormatSymbols; -import java.text.Format; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Locale; - -import org.simantics.databoard.accessor.StreamAccessor; -import org.simantics.databoard.accessor.error.AccessorException; -import org.simantics.history.HistoryException; -import org.simantics.history.HistoryManager; -import org.simantics.history.util.HistoryExportUtil; -import org.simantics.history.util.ProgressMonitor; -import org.simantics.history.util.StreamIterator; -import org.simantics.history.util.ValueBand; - -/** - * CSV writer for history items. - * - * @author Toni Kalajainen - * @author Tuukka Lehtonen - */ -public class CSVFormatter { - - List items = new ArrayList(); - double from = -Double.MAX_VALUE; - double end = Double.MAX_VALUE; - double startTime = 0.0; - double timeStep = 0.0; - ColumnSeparator columnSeparator; - DecimalSeparator decimalSeparator; - boolean resample; - String lineFeed; - Locale locale; - - Format timeFormat; - Format floatFormat; - Format numberFormat; - - Formatter timeFormatter; - Formatter floatFormatter; - Formatter numberFormatter; - - ExportInterpolation numberInterpolation = ExportInterpolation.LINEAR_INTERPOLATION; - - public CSVFormatter() { - this.lineFeed = resolvePlatformLineFeed(); - this.locale = Locale.getDefault(Locale.Category.FORMAT); - - DecimalFormat defaultFormat = new DecimalFormat(); - defaultFormat.setGroupingUsed(false); - setTimeFormat(defaultFormat); - setFloatFormat(defaultFormat); - setNumberFormat(defaultFormat); - } - - /** - * Add item to formatter - * - * @param historyItemId - * @param label - * @param variableReference - * @param unit - */ - public void addItem( HistoryManager history, String historyItemId, String label, String variableReference, String unit ) { - Item i = new Item(); - i.history = history; - i.label = label!=null?label:""; - i.variableReference = variableReference!=null?variableReference:""; - i.variableReference = unescape(i.variableReference); - i.historyItemId = historyItemId; - i.unit = unit; - if ( !items.contains(i) ) items.add( i ); - } - - private static String unescape(String url) { - try { - return URLDecoder.decode(url, "UTF-8"); - } catch (UnsupportedEncodingException e) { - return url; - } - } - - /** - * Sort items by variableId, label1, label2 - */ - public void sort() { - Collections.sort(items); - } - - public void setTimeRange( double from, double end ) { - this.from = from; - this.end = end; - } - - public void setStartTime( double startTime ) { - this.startTime = startTime; - } - - public void setTimeStep( double timeStep ) { - this.timeStep = timeStep; - } - - void openHistory() throws HistoryException { - try { - for (Item item : items) item.open(); - } catch (HistoryException e) { - for (Item item : items) item.close(); - throw e; - } - } - - void closeHistory() { - for (Item item : items) item.close(); - } - - /** - * Reads visible data of all variables and formats as CSV lines (Comma - * Separated Values). Line Feed is \n, variable separator is \t, and - * decimal separator locale dependent. - * - * ReadData1 outputs separate time and value columns for each variable - * - * Variable1 Time | Variable1 Value | Variable2 Time | Variable2 Value - * 0.0 | 1.0 | 0.1 | 23423.0 - * - * @param monitor - * @param sb - * @throws HistoryException - * @throws IOException - */ - /* - public void formulate1( ProgressMonitor monitor, Appendable sb ) throws HistoryException, IOException { - if (items.isEmpty()) return; - boolean adaptComma = decimalSeparatorInLocale != decimalSeparator; - openHistory(); - try { - // Prepare columns: First time - for (Item item : items) - { - if (monitor.isCanceled()) - return; - if (item.stream.isEmpty()) continue; - Double firstTime = (Double) item.stream.getFirstTime( Bindings.DOUBLE ); - if (from <= firstTime) { - item.time = firstTime; - } else { - item.time = (Double) item.stream.getFloorTime(Bindings.DOUBLE, from); - } - } - - // Write Headers - for (Item i : items) - { - if (monitor.isCanceled()) - return; - boolean lastColumn = i == items.get( items.size()-1 ); - sb.append(i.label + " Time"); - sb.append( columnSeparator ); - sb.append(i.label + " Value"); - sb.append(lastColumn ? lineFeed : columnSeparator ); - } - - // Iterate until endTime is met for all variables - int readyColumns; - do { - if (monitor.isCanceled()) - return; - - readyColumns = 0; - for (Item i : items) - { - boolean lastColumn = i == items.get( items.size()-1 ); - - if (i.time == null || i.time > end) { - readyColumns++; - sb.append( lastColumn ? columnSeparator+lineFeed : columnSeparator+columnSeparator); - continue; - } - - sb.append(""); - - // Write time - String timeStr = format.format( i.time ); - if ( adaptComma ) timeStr = timeStr.replace(decimalSeparatorInLocale, decimalSeparator); - sb.append( timeStr ); - sb.append( columnSeparator ); - - // Write value - i.value = i.stream.getValue(Bindings.DOUBLE, i.time); - if (i.value instanceof Number) { - String str = format.format( i.value ); - if ( adaptComma ) str = str.replace(decimalSeparatorInLocale, decimalSeparator); - sb.append( str ); - } else if (i.value instanceof Boolean) { - sb.append( (Boolean)i.value ? "1": "0"); - } else { - sb.append( i.value.toString() ); - } - sb.append(lastColumn ? lineFeed : columnSeparator); - - // Get next time - i.time = (Double) i.stream.getHigherTime(Bindings.DOUBLE, i.time); - } - - } while (readyColumns < items.size()); - } finally { - closeHistory(); - } - }*/ - - /** - * Reads visible data of all variables and formats as CSV lines (Comma - * Separated Values). Line Feed is \n, variable separator is \t, and - * decimal separator locale dependent. - * - * ReadData2 outputs one shared time and one value column for each variable - * - * Time | Variable1 Label | Variable3 Label - * | Variable1 Id | Variable3 Id - * | Variable1 Unit | Variable3 Unit - * 0.0 | 1.0 | 0.1 - * - * @param monitor - * @param sb - * @throws HistoryException - * @throws IOException - */ - public void formulate2( ProgressMonitor monitor, Appendable sb ) throws HistoryException, IOException - { - if ( items.isEmpty() ) return; - - timeFormatter = evaluateFormatter(timeFormat, decimalSeparator); - floatFormatter = evaluateFormatter(floatFormat, decimalSeparator); - numberFormatter = evaluateFormatter(numberFormat, decimalSeparator); - - openHistory(); - try { - // What is the time range of all items combined - double allFrom = Double.MAX_VALUE; - double allEnd = -Double.MAX_VALUE; - for (Item i : items) { - if (i.iter.isEmpty()) continue; - allFrom = Math.min(allFrom, i.iter.getFirstTime()); - allEnd = Math.max(allEnd, i.iter.getLastTime()); - } - - // Write Headers - for (int hl = 0; hl < 3; ++hl) { - switch(hl) { - case 0: sb.append("Time"); break; - case 1: sb.append("----"); break; - case 2: sb.append("Unit"); break; - } - sb.append( columnSeparator.preference ); - for (Item i : items) - { - boolean lastColumn = i == items.get( items.size()-1 ); - switch (hl) { - case 0: - sb.append(i.label != null ? i.label : ""); - break; - case 1: - sb.append(i.variableReference != null ? i.variableReference : ""); - break; - case 2: - sb.append(i.unit==null?"no unit":i.unit); - break; - } - if (!lastColumn) sb.append( columnSeparator.preference ); - } - sb.append( lineFeed ); - } - - // Prepare time - boolean hasAnyValues = allFrom != Double.MAX_VALUE && allEnd != -Double.MAX_VALUE; - - // Make intersection of actual data range (allFrom, allEnd) and requested data (from, end) - double _from = Double.MAX_VALUE, _end = -Double.MAX_VALUE; - if (hasAnyValues) { - _from = Math.max(allFrom, from); - _end = Math.min(allEnd, end); - } - - if (!hasAnyValues) return; - - // Iterate until endTime is met for all variables - double time = _from; - - if(!resample) { - - // If resample is false then all samples are reported as is. The code achieves this by setting startTime to _from and timeStep to 0.0 - time = _from; - timeStep = 0.0; - - } else { - - // time = startTime + n*timeStep - - // Sampling based on given startTime and timeStep - if(timeStep > 0) { - - // Find the first sample time that contains data - double n = Math.max(0, Math.ceil((_from-startTime) / timeStep)); - time = startTime + n*timeStep; - - } else { - - // Start sampling from startTime but make sure that it is not less than _from - if(startTime > _from) time = startTime; - - } - - - } - - // Must convert double times to String when initializing BigDecimal. - // Otherwise BigDecimal will pick up inaccuracies from beyond 15 precise digits - // thus making a mess of the time step calculations. - - BigDecimal bigTime = new BigDecimal(String.valueOf(time)); - BigDecimal bigTimeStep = new BigDecimal(String.valueOf(timeStep)); - - for (Item i : items) i.iter.gotoTime(time); - do { - if ( monitor!=null && monitor.isCanceled() ) return; - - // Write time - String timeStr = timeFormatter.format( time ); - //System.out.println("SAMPLING TIME: " + time); - sb.append( timeStr ); - - // Write values - for (Item i : items) - { - sb.append( columnSeparator.preference ); - - // Write value - if ( i.iter.hasValidValue() ) { - Object value = i.iter.getValueBand().getValue(); - if (value instanceof Number) { - if (value instanceof Float || value instanceof Double) { - switch (numberInterpolation) { - case PREVIOUS_SAMPLE: - sb.append( formatNumber(value) ); - break; - - case LINEAR_INTERPOLATION: - if (time != i.iter.getValueBand().getTimeDouble() && i.iter.hasNext()) { - - // Interpolate - int currentIndex = i.iter.getIndex(); - ValueBand band = i.iter.getValueBand(); - //double t1 = band.getTimeDouble(); - Number v1 = (Number) value; - double t12 = band.getEndTimeDouble(); - i.iter.next(); - double t2 = i.iter.getValueBand().getTimeDouble(); - Number v2 = (Number) i.iter.getValueBand().getValue(); - i.iter.gotoIndex(currentIndex); - - double vs = v1.doubleValue(); - if(time > t12) - vs = HistoryExportUtil.biglerp(t12, v1.doubleValue(), t2, v2.doubleValue(), time); - - sb.append( formatDouble(vs) ); - } else { - // Exact timestamp match, or last sample. - // Don't interpolate nor extrapolate. - sb.append( formatNumber(value) ); - } - break; - default: - throw new UnsupportedOperationException("Unsupported interpolation: " + numberInterpolation); - } - } else { - sb.append( value.toString() ); - } - } else if (value instanceof Boolean) { - sb.append( (Boolean)value ? "1": "0"); - } else { - sb.append( value.toString() ); - } - } - } - - sb.append( lineFeed ); - - // Read next values, and the following times - if ( timeStep>0.0 ) { - bigTime = bigTime.add(bigTimeStep); - time = bigTime.doubleValue(); - } else { - // Get smallest end time that is larger than current time - Double nextTime = null; -// System.out.println("time = "+time); - for (Item i : items) { - Double itemNextTime = i.iter.getNextTime( time ); -// System.err.println(" "+i.label+" nextTime="+itemNextTime); - if ( itemNextTime == null ) continue; - if ( itemNextTime < time ) continue; - if ( nextTime == null || ( nextTime > itemNextTime && !itemNextTime.equals( time ) ) ) nextTime = itemNextTime; - } - if ( nextTime == null || nextTime.equals( time ) ) break; - time = nextTime; - } - - boolean hasMore = false; - - for (Item i : items) { - i.iter.proceedToTime(time); - if(contains(i, time)) hasMore = true; - } - - if(!hasMore) break; - - } while (time<=_end); - } finally { - closeHistory(); - } - } - - private boolean contains(Item item, double time) { - double start = item.iter.getStartTime(); - double end = item.iter.getEndTime(); - // A special case, where start == end => accept - if(time == start) return true; - else if(time < start) return false; - else if(time >= end) return false; - else return true; - } - - private CharSequence formatNumber(Object value) { - return value instanceof Float - ? floatFormatter.format( value ) - : numberFormatter.format( value ); - } - - private CharSequence formatDouble(double value) { - return numberFormatter.format( value ); - } - - public void setDecimalSeparator(DecimalSeparator separator) { - this.decimalSeparator = separator; - } - - public void setColumnSeparator(ColumnSeparator separator) { - this.columnSeparator = separator; - } - - public void setResample(boolean resample) { - this.resample = resample; - } - - public void setLineFeed( String lf ) { - this.lineFeed = lf; - } - - public void setTimeFormat(Format format) { - this.timeFormat = format; - } - - public void setFloatFormat(Format format) { - this.floatFormat = format; - } - - public void setNumberFormat(Format format) { - this.numberFormat = format; - } - - public void setLocale(Locale locale) { - this.locale = locale; - } - - public void setNumberInterpolation(ExportInterpolation interpolation) { - this.numberInterpolation = interpolation; - } - - private static String resolvePlatformLineFeed() { - String osName = System.getProperty("os.name", ""); - osName = osName.toLowerCase(); - if (osName.contains("windows")) - return "\r\n"; - return "\n"; - } - - private class Item implements Comparable { - // Static data - String label; // Label - String variableReference; // Label - HistoryManager history; // History source for this item - String historyItemId; - String unit; - - // State data - StreamAccessor accessor; // Stream accessor - StreamIterator iter; - - public void open() throws HistoryException { - accessor = history.openStream(historyItemId, "r"); - iter = new StreamIterator( accessor ); - } - - public void close() { - if (accessor!=null) { - try { - accessor.close(); - } catch (AccessorException e) { - } - } - accessor = null; - iter = null; - } - - @Override - public int compareTo(Item o) { - int i; - i = label.compareTo(o.label); - if (i!=0) return i; - i = variableReference.compareTo(o.variableReference); - if (i!=0) return i; - i = historyItemId.compareTo(o.historyItemId); - if (i!=0) return i; - return 0; - } - - @Override - public int hashCode() { - int code = 0x2304; - code = 13*code + variableReference.hashCode(); - code = 13*code + label.hashCode(); - code = 13*code + historyItemId.hashCode(); - code = 13*code + history.hashCode(); - return code; - } - - @Override - public boolean equals(Object obj) { - if ( obj == null ) return false; - if ( obj instanceof Item == false ) return false; - Item other = (Item) obj; - if ( !other.label.equals(label) ) return false; - if ( !other.variableReference.equals(variableReference) ) return false; - if ( !other.history.equals(history) ) return false; - if ( !other.historyItemId.equals(historyItemId) ) return false; - return true; - } - - } - - static interface Formatter { - String format(Object number); - } - - static class NopFormatter implements Formatter { - private final Format format; - public NopFormatter(Format format) { - this.format = format; - } - public String format(Object number) { - return format.format(number); - } - } - - static class ReplacingFormatter implements Formatter { - private final Format format; - private final char from; - private final char to; - public ReplacingFormatter(Format format, char from, char to) { - this.format = format; - this.from = from; - this.to = to; - } - public String format(Object number) { - return format.format(number).replace(from, to); - } - } - - private Formatter evaluateFormatter(Format format, DecimalSeparator target) { - // Probe decimal separator - String onePointTwo = format.format(1.2); - System.out.println("formatted zeroPointOne: " + onePointTwo); - - DecimalSeparator formatSeparator; - if (onePointTwo.indexOf('.') != -1) { - formatSeparator = DecimalSeparator.DOT; - } else if (onePointTwo.indexOf(',') != -1) { - formatSeparator = DecimalSeparator.COMMA; - } else { - DecimalFormatSymbols symbols = DecimalFormatSymbols.getInstance(locale); - formatSeparator = DecimalSeparator.fromChar(symbols.getDecimalSeparator()); - } - - switch (formatSeparator) { - case COMMA: - switch (target) { - case COMMA: - return new NopFormatter(format); - case DOT: - return new ReplacingFormatter(format, ',', '.'); - } - case DOT: - switch (target) { - case COMMA: - return new ReplacingFormatter(format, '.', ','); - case DOT: - return new NopFormatter(format); - } - } - return new NopFormatter(format); - } - -} +/******************************************************************************* + * Copyright (c) 2011 Association for Decentralized Information Management in + * Industry THTH ry. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * VTT Technical Research Centre of Finland - initial API and implementation + *******************************************************************************/ +package org.simantics.history.csv; + +import java.io.IOException; +import java.math.BigDecimal; +import java.text.DecimalFormat; +import java.text.DecimalFormatSymbols; +import java.text.Format; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; + +import org.simantics.databoard.accessor.StreamAccessor; +import org.simantics.databoard.accessor.error.AccessorException; +import org.simantics.history.HistoryException; +import org.simantics.history.HistoryManager; +import org.simantics.history.util.HistoryExportUtil; +import org.simantics.history.util.ProgressMonitor; +import org.simantics.history.util.StreamIterator; +import org.simantics.history.util.ValueBand; + +/** + * CSV writer for history items. + * + * @author Toni Kalajainen + * @author Tuukka Lehtonen + */ +public class CSVFormatter { + + /** + * This is the tolerance used to decide whether or not the last data point of + * the exported items is included in the exported material or not. If + * 0 <= (t - t(lastDataPoint) < {@value #RESAMPLING_END_TIMESTAMP_INCLUSION_TOLERANCE} + * is true, then the last exported data point will be + * lastDataPoint, with timestamp t(lastDataPoint) even + * if t > t(lastDataPoint). + * + *

+ * This works around problems where floating point inaccuracy causes a data + * point to be left out from the the export when it would be fair for the user + * to expect the data to be exported would contain a point with time stamp + * 9.999999999999996 when sampling with time-step 1.0 + * starting from time 0.0. + */ + private static final double RESAMPLING_END_TIMESTAMP_INCLUSION_TOLERANCE = 1e-13; + + List items = new ArrayList(); + double from = -Double.MAX_VALUE; + double end = Double.MAX_VALUE; + double startTime = 0.0; + double timeStep = 0.0; + ColumnSeparator columnSeparator; + DecimalSeparator decimalSeparator; + boolean resample; + String lineFeed; + Locale locale; + + Format timeFormat; + Format floatFormat; + Format numberFormat; + + Formatter timeFormatter; + Formatter floatFormatter; + Formatter numberFormatter; + + ExportInterpolation numberInterpolation = ExportInterpolation.LINEAR_INTERPOLATION; + + public CSVFormatter() { + this.lineFeed = resolvePlatformLineFeed(); + this.locale = Locale.getDefault(Locale.Category.FORMAT); + + DecimalFormat defaultFormat = new DecimalFormat(); + defaultFormat.setGroupingUsed(false); + setTimeFormat(defaultFormat); + setFloatFormat(defaultFormat); + setNumberFormat(defaultFormat); + } + + /** + * Add item to formatter + * + * @param historyItemId + * @param label + * @param variableReference + * @param unit + */ + public void addItem( HistoryManager history, String historyItemId, String label, String variableReference, String unit ) { + Item i = new Item(); + i.history = history; + i.label = label!=null?label:""; + i.variableReference = variableReference!=null?variableReference:""; + i.variableReference = URIs.safeUnescape(i.variableReference); + i.historyItemId = historyItemId; + i.unit = unit; + if ( !items.contains(i) ) items.add( i ); + } + + /** + * Sort items by variableId, label1, label2 + */ + public void sort() { + Collections.sort(items); + } + + public void setTimeRange( double from, double end ) { + this.from = from; + this.end = end; + } + + public void setStartTime( double startTime ) { + this.startTime = startTime; + } + + public void setTimeStep( double timeStep ) { + this.timeStep = timeStep; + } + + void openHistory() throws HistoryException { + try { + for (Item item : items) item.open(); + } catch (HistoryException e) { + for (Item item : items) item.close(); + throw e; + } + } + + void closeHistory() { + for (Item item : items) item.close(); + } + + /** + * Reads visible data of all variables and formats as CSV lines (Comma + * Separated Values). Line Feed is \n, variable separator is \t, and + * decimal separator locale dependent. + * + * ReadData1 outputs separate time and value columns for each variable + * + * Variable1 Time | Variable1 Value | Variable2 Time | Variable2 Value + * 0.0 | 1.0 | 0.1 | 23423.0 + * + * @param monitor + * @param sb + * @throws HistoryException + * @throws IOException + */ + /* + public void formulate1( ProgressMonitor monitor, Appendable sb ) throws HistoryException, IOException { + if (items.isEmpty()) return; + boolean adaptComma = decimalSeparatorInLocale != decimalSeparator; + openHistory(); + try { + // Prepare columns: First time + for (Item item : items) + { + if (monitor.isCanceled()) + return; + if (item.stream.isEmpty()) continue; + Double firstTime = (Double) item.stream.getFirstTime( Bindings.DOUBLE ); + if (from <= firstTime) { + item.time = firstTime; + } else { + item.time = (Double) item.stream.getFloorTime(Bindings.DOUBLE, from); + } + } + + // Write Headers + for (Item i : items) + { + if (monitor.isCanceled()) + return; + boolean lastColumn = i == items.get( items.size()-1 ); + sb.append(i.label + " Time"); + sb.append( columnSeparator ); + sb.append(i.label + " Value"); + sb.append(lastColumn ? lineFeed : columnSeparator ); + } + + // Iterate until endTime is met for all variables + int readyColumns; + do { + if (monitor.isCanceled()) + return; + + readyColumns = 0; + for (Item i : items) + { + boolean lastColumn = i == items.get( items.size()-1 ); + + if (i.time == null || i.time > end) { + readyColumns++; + sb.append( lastColumn ? columnSeparator+lineFeed : columnSeparator+columnSeparator); + continue; + } + + sb.append(""); + + // Write time + String timeStr = format.format( i.time ); + if ( adaptComma ) timeStr = timeStr.replace(decimalSeparatorInLocale, decimalSeparator); + sb.append( timeStr ); + sb.append( columnSeparator ); + + // Write value + i.value = i.stream.getValue(Bindings.DOUBLE, i.time); + if (i.value instanceof Number) { + String str = format.format( i.value ); + if ( adaptComma ) str = str.replace(decimalSeparatorInLocale, decimalSeparator); + sb.append( str ); + } else if (i.value instanceof Boolean) { + sb.append( (Boolean)i.value ? "1": "0"); + } else { + sb.append( i.value.toString() ); + } + sb.append(lastColumn ? lineFeed : columnSeparator); + + // Get next time + i.time = (Double) i.stream.getHigherTime(Bindings.DOUBLE, i.time); + } + + } while (readyColumns < items.size()); + } finally { + closeHistory(); + } + }*/ + + /** + * Reads visible data of all variables and formats as CSV lines (Comma + * Separated Values). Line Feed is \n, variable separator is \t, and + * decimal separator locale dependent. + * + * ReadData2 outputs one shared time and one value column for each variable + * + * Time | Variable1 Label | Variable3 Label + * | Variable1 Id | Variable3 Id + * | Variable1 Unit | Variable3 Unit + * 0.0 | 1.0 | 0.1 + * + * @param monitor + * @param sb + * @throws HistoryException + * @throws IOException + */ + public void formulate2( ProgressMonitor monitor, Appendable sb ) throws HistoryException, IOException + { + if ( items.isEmpty() ) return; + + timeFormatter = evaluateFormatter(timeFormat, decimalSeparator); + floatFormatter = evaluateFormatter(floatFormat, decimalSeparator); + numberFormatter = evaluateFormatter(numberFormat, decimalSeparator); + + openHistory(); + try { + // What is the time range of all items combined + double allFrom = Double.MAX_VALUE; + double allEnd = -Double.MAX_VALUE; + for (Item i : items) { + if (i.iter.isEmpty()) continue; + allFrom = Math.min(allFrom, i.iter.getFirstTime()); + allEnd = Math.max(allEnd, i.iter.getLastTime()); + } + + // Write Headers + for (int hl = 0; hl < 3; ++hl) { + switch(hl) { + case 0: sb.append("Time"); break; + case 1: sb.append("----"); break; + case 2: sb.append("Unit"); break; + } + sb.append( columnSeparator.preference ); + for (Item i : items) + { + boolean lastColumn = i == items.get( items.size()-1 ); + switch (hl) { + case 0: + sb.append(i.label != null ? i.label : ""); + break; + case 1: + sb.append(i.variableReference != null ? i.variableReference : ""); + break; + case 2: + sb.append(i.unit==null?"no unit":i.unit); + break; + } + if (!lastColumn) sb.append( columnSeparator.preference ); + } + sb.append( lineFeed ); + } + + // Prepare time + boolean hasAnyValues = allFrom != Double.MAX_VALUE && allEnd != -Double.MAX_VALUE; + + // Make intersection of actual data range (allFrom, allEnd) and requested data (from, end) + double _from = Double.MAX_VALUE, _end = -Double.MAX_VALUE; + if (hasAnyValues) { + _from = Math.max(allFrom, from); + _end = Math.min(allEnd, end); + } + + if (!hasAnyValues) return; + + // Iterate until endTime is met for all variables + double time = _from; + + if(!resample) { + + // If resample is false then all samples are reported as is. The code achieves this by setting startTime to _from and timeStep to 0.0 + time = _from; + timeStep = 0.0; + + } else { + + // time = startTime + n*timeStep + + // Sampling based on given startTime and timeStep + if(timeStep > 0) { + + // Find the first sample time that contains data if startTime < _from + double n = Math.max(0, Math.ceil((_from-startTime) / timeStep)); + time = startTime + n*timeStep; + + } else { + + // Start sampling from startTime but make sure that it is not less than _from + if(startTime > _from) time = startTime; + + } + + + } + + // Must convert double times to String when initializing BigDecimal. + // Otherwise BigDecimal will pick up inaccuracies from beyond 15 precise digits + // thus making a mess of the time step calculations. + + BigDecimal bigTime = new BigDecimal(String.valueOf(time)); + BigDecimal bigTimeStep = new BigDecimal(String.valueOf(timeStep)); + + // Loop kill-switch for the case where timeStep > 0 + boolean breakAfterNextWrite = false; + +// System.out.println("time: " + time); +// System.out.println("timeStep: " + timeStep); +// System.out.println("_end: " + Double.toString(_end)); + + for (Item i : items) i.iter.gotoTime(time); + do { + if ( monitor!=null && monitor.isCanceled() ) return; + + // Write time + String timeStr = timeFormatter.format( time ); + //System.out.println("SAMPLING TIME: " + time); + sb.append( timeStr ); + + // Write values + for (Item i : items) + { + sb.append( columnSeparator.preference ); + + // Write value + if ( i.iter.hasValidValue() ) { + Object value = i.iter.getValueBand().getValue(); + if (value instanceof Number) { + if (value instanceof Float || value instanceof Double) { + switch (numberInterpolation) { + case PREVIOUS_SAMPLE: + sb.append( formatNumber(value) ); + break; + + case LINEAR_INTERPOLATION: + if (time != i.iter.getValueBand().getTimeDouble() && i.iter.hasNext()) { + + // Interpolate + int currentIndex = i.iter.getIndex(); + ValueBand band = i.iter.getValueBand(); + //double t1 = band.getTimeDouble(); + Number v1 = (Number) value; + double t12 = band.getEndTimeDouble(); + i.iter.next(); + double t2 = i.iter.getValueBand().getTimeDouble(); + Number v2 = (Number) i.iter.getValueBand().getValue(); + i.iter.gotoIndex(currentIndex); + + double vs = v1.doubleValue(); + if(time > t12) + vs = HistoryExportUtil.biglerp(t12, v1.doubleValue(), t2, v2.doubleValue(), time); + + sb.append( formatDouble(vs) ); + } else { + // Exact timestamp match, or last sample. + // Don't interpolate nor extrapolate. + sb.append( formatNumber(value) ); + } + break; + default: + throw new UnsupportedOperationException("Unsupported interpolation: " + numberInterpolation); + } + } else { + sb.append( value.toString() ); + } + } else if (value instanceof Boolean) { + sb.append( (Boolean)value ? "1": "0"); + } else { + sb.append( value.toString() ); + } + } + } + + sb.append( lineFeed ); + + if (breakAfterNextWrite) + break; + + // Read next values, and the following times + if ( timeStep>0.0 ) { + bigTime = bigTime.add(bigTimeStep); + time = bigTime.doubleValue(); + + // gitlab #529: prevent last data point from getting dropped + // due to small imprecisions in re-sampling mode. + double diff = time - _end; + if (diff > 0 && diff <= RESAMPLING_END_TIMESTAMP_INCLUSION_TOLERANCE) { + time = _end; + breakAfterNextWrite = true; + // Take floating point inaccuracy into account when re-sampling + // to prevent the last data point from being left out if there + // is small-enough imprecision in the last data point time stamp + // to be considered negligible compared to expected stepped time. + } + + } else { + // Get smallest end time that is larger than current time + Double nextTime = null; +// System.out.println("time = "+time); + for (Item i : items) { + Double itemNextTime = i.iter.getNextTime( time ); +// System.err.println(" "+i.label+" nextTime="+itemNextTime); + if ( itemNextTime == null ) continue; + if ( itemNextTime < time ) continue; + if ( nextTime == null || ( nextTime > itemNextTime && !itemNextTime.equals( time ) ) ) nextTime = itemNextTime; + } + if ( nextTime == null || nextTime.equals( time ) ) break; + time = nextTime; + } + + boolean hasMore = false; + + for (Item i : items) { + i.iter.proceedToTime(time); + if(contains(i, time)) hasMore = true; + } + + //System.out.println("hasMore @ " + time + " (" + bigTime + ") = " + hasMore); + if(!hasMore) break; + + } while (time<=_end); + } finally { + closeHistory(); + } + } + + private boolean contains(Item item, double time) { + double start = item.iter.getStartTime(); + double end = item.iter.getEndTime(); + // A special case, where start == end => accept + if(time == start) return true; + else if(time < start) return false; + else if(time >= end) return false; + else return true; + } + + private CharSequence formatNumber(Object value) { + return value instanceof Float + ? floatFormatter.format( value ) + : numberFormatter.format( value ); + } + + private CharSequence formatDouble(double value) { + return numberFormatter.format( value ); + } + + public void setDecimalSeparator(DecimalSeparator separator) { + this.decimalSeparator = separator; + } + + public void setColumnSeparator(ColumnSeparator separator) { + this.columnSeparator = separator; + } + + public void setResample(boolean resample) { + this.resample = resample; + } + + public void setLineFeed( String lf ) { + this.lineFeed = lf; + } + + public void setTimeFormat(Format format) { + this.timeFormat = format; + } + + public void setFloatFormat(Format format) { + this.floatFormat = format; + } + + public void setNumberFormat(Format format) { + this.numberFormat = format; + } + + public void setLocale(Locale locale) { + this.locale = locale; + } + + public void setNumberInterpolation(ExportInterpolation interpolation) { + this.numberInterpolation = interpolation; + } + + private static String resolvePlatformLineFeed() { + String osName = System.getProperty("os.name", ""); + osName = osName.toLowerCase(); + if (osName.contains("windows")) + return "\r\n"; + return "\n"; + } + + private class Item implements Comparable { + // Static data + String label; // Label + String variableReference; // Label + HistoryManager history; // History source for this item + String historyItemId; + String unit; + + // State data + StreamAccessor accessor; // Stream accessor + StreamIterator iter; + + public void open() throws HistoryException { + accessor = history.openStream(historyItemId, "r"); + iter = new StreamIterator( accessor ); + } + + public void close() { + if (accessor!=null) { + try { + accessor.close(); + } catch (AccessorException e) { + } + } + accessor = null; + iter = null; + } + + @Override + public int compareTo(Item o) { + int i; + i = label.compareTo(o.label); + if (i!=0) return i; + i = variableReference.compareTo(o.variableReference); + if (i!=0) return i; + i = historyItemId.compareTo(o.historyItemId); + if (i!=0) return i; + return 0; + } + + @Override + public int hashCode() { + int code = 0x2304; + code = 13*code + variableReference.hashCode(); + code = 13*code + label.hashCode(); + code = 13*code + historyItemId.hashCode(); + code = 13*code + history.hashCode(); + return code; + } + + @Override + public boolean equals(Object obj) { + if ( obj == null ) return false; + if ( obj instanceof Item == false ) return false; + Item other = (Item) obj; + if ( !other.label.equals(label) ) return false; + if ( !other.variableReference.equals(variableReference) ) return false; + if ( !other.history.equals(history) ) return false; + if ( !other.historyItemId.equals(historyItemId) ) return false; + return true; + } + + } + + static interface Formatter { + String format(Object number); + } + + static class NopFormatter implements Formatter { + private final Format format; + public NopFormatter(Format format) { + this.format = format; + } + public String format(Object number) { + return format.format(number); + } + } + + static class ReplacingFormatter implements Formatter { + private final Format format; + private final char from; + private final char to; + public ReplacingFormatter(Format format, char from, char to) { + this.format = format; + this.from = from; + this.to = to; + } + public String format(Object number) { + return format.format(number).replace(from, to); + } + } + + private Formatter evaluateFormatter(Format format, DecimalSeparator target) { + // Probe decimal separator + String onePointTwo = format.format(1.2); + //System.out.println("formatted zeroPointOne: " + onePointTwo); + + DecimalSeparator formatSeparator; + if (onePointTwo.indexOf('.') != -1) { + formatSeparator = DecimalSeparator.DOT; + } else if (onePointTwo.indexOf(',') != -1) { + formatSeparator = DecimalSeparator.COMMA; + } else { + DecimalFormatSymbols symbols = DecimalFormatSymbols.getInstance(locale); + formatSeparator = DecimalSeparator.fromChar(symbols.getDecimalSeparator()); + } + + switch (formatSeparator) { + case COMMA: + switch (target) { + case COMMA: + return new NopFormatter(format); + case DOT: + return new ReplacingFormatter(format, ',', '.'); + } + case DOT: + switch (target) { + case COMMA: + return new ReplacingFormatter(format, '.', ','); + case DOT: + return new NopFormatter(format); + } + } + return new NopFormatter(format); + } + +}