/*******************************************************************************
* Copyright (c) 2011 Association for Decentralized Information Management in
* Industry THTH ry.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* VTT Technical Research Centre of Finland - initial API and implementation
*******************************************************************************/
package org.simantics.history.csv;
import java.io.IOException;
import java.math.BigDecimal;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.text.Format;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import org.simantics.databoard.accessor.StreamAccessor;
import org.simantics.databoard.accessor.error.AccessorException;
import org.simantics.history.HistoryException;
import org.simantics.history.HistoryManager;
import org.simantics.history.util.HistoryExportUtil;
import org.simantics.history.util.ProgressMonitor;
import org.simantics.history.util.StreamIterator;
import org.simantics.history.util.ValueBand;
/**
* CSV writer for history items.
*
* @author Toni Kalajainen
* @author Tuukka Lehtonen
*/
public class CSVFormatter {
/**
* This is the tolerance used to decide whether or not the last data point of
* the exported items is included in the exported material or not. If
* 0 <= (t - t(lastDataPoint) < {@value #RESAMPLING_END_TIMESTAMP_INCLUSION_TOLERANCE}
* is true, then the last exported data point will be
* lastDataPoint
, with timestamp t(lastDataPoint)
even
* if t > t(lastDataPoint)
.
*
*
* This works around problems where floating point inaccuracy causes a data
* point to be left out from the the export when it would be fair for the user
* to expect the data to be exported would contain a point with time stamp
* 9.999999999999996
when sampling with time-step 1.0
* starting from time 0.0
.
*/
private static final double RESAMPLING_END_TIMESTAMP_INCLUSION_TOLERANCE = 1e-13;
List- items = new ArrayList
- ();
double from = -Double.MAX_VALUE;
double end = Double.MAX_VALUE;
double startTime = 0.0;
double timeStep = 0.0;
ColumnSeparator columnSeparator;
DecimalSeparator decimalSeparator;
boolean resample;
String lineFeed;
Locale locale;
Format timeFormat;
Format floatFormat;
Format numberFormat;
Formatter timeFormatter;
Formatter floatFormatter;
Formatter numberFormatter;
ExportInterpolation numberInterpolation = ExportInterpolation.LINEAR_INTERPOLATION;
public CSVFormatter() {
this.lineFeed = resolvePlatformLineFeed();
this.locale = Locale.getDefault(Locale.Category.FORMAT);
DecimalFormat defaultFormat = new DecimalFormat();
defaultFormat.setGroupingUsed(false);
setTimeFormat(defaultFormat);
setFloatFormat(defaultFormat);
setNumberFormat(defaultFormat);
}
/**
* Add item to formatter
*
* @param historyItemId
* @param label
* @param variableReference
* @param unit
*/
public void addItem( HistoryManager history, String historyItemId, String label, String variableReference, String unit ) {
Item i = new Item();
i.history = history;
i.label = label!=null?label:"";
i.variableReference = variableReference!=null?variableReference:"";
i.variableReference = URIs.safeUnescape(i.variableReference);
i.historyItemId = historyItemId;
i.unit = unit;
if ( !items.contains(i) ) items.add( i );
}
/**
* Sort items by variableId, label1, label2
*/
public void sort() {
Collections.sort(items);
}
public void setTimeRange( double from, double end ) {
this.from = from;
this.end = end;
}
public void setStartTime( double startTime ) {
this.startTime = startTime;
}
public void setTimeStep( double timeStep ) {
this.timeStep = timeStep;
}
void openHistory() throws HistoryException {
try {
for (Item item : items) item.open();
} catch (HistoryException e) {
for (Item item : items) item.close();
throw e;
}
}
void closeHistory() {
for (Item item : items) item.close();
}
/**
* Reads visible data of all variables and formats as CSV lines (Comma
* Separated Values). Line Feed is \n, variable separator is \t, and
* decimal separator locale dependent.
*
* ReadData1 outputs separate time and value columns for each variable
*
* Variable1 Time | Variable1 Value | Variable2 Time | Variable2 Value
* 0.0 | 1.0 | 0.1 | 23423.0
*
* @param monitor
* @param sb
* @throws HistoryException
* @throws IOException
*/
/*
public void formulate1( ProgressMonitor monitor, Appendable sb ) throws HistoryException, IOException {
if (items.isEmpty()) return;
boolean adaptComma = decimalSeparatorInLocale != decimalSeparator;
openHistory();
try {
// Prepare columns: First time
for (Item item : items)
{
if (monitor.isCanceled())
return;
if (item.stream.isEmpty()) continue;
Double firstTime = (Double) item.stream.getFirstTime( Bindings.DOUBLE );
if (from <= firstTime) {
item.time = firstTime;
} else {
item.time = (Double) item.stream.getFloorTime(Bindings.DOUBLE, from);
}
}
// Write Headers
for (Item i : items)
{
if (monitor.isCanceled())
return;
boolean lastColumn = i == items.get( items.size()-1 );
sb.append(i.label + " Time");
sb.append( columnSeparator );
sb.append(i.label + " Value");
sb.append(lastColumn ? lineFeed : columnSeparator );
}
// Iterate until endTime is met for all variables
int readyColumns;
do {
if (monitor.isCanceled())
return;
readyColumns = 0;
for (Item i : items)
{
boolean lastColumn = i == items.get( items.size()-1 );
if (i.time == null || i.time > end) {
readyColumns++;
sb.append( lastColumn ? columnSeparator+lineFeed : columnSeparator+columnSeparator);
continue;
}
sb.append("");
// Write time
String timeStr = format.format( i.time );
if ( adaptComma ) timeStr = timeStr.replace(decimalSeparatorInLocale, decimalSeparator);
sb.append( timeStr );
sb.append( columnSeparator );
// Write value
i.value = i.stream.getValue(Bindings.DOUBLE, i.time);
if (i.value instanceof Number) {
String str = format.format( i.value );
if ( adaptComma ) str = str.replace(decimalSeparatorInLocale, decimalSeparator);
sb.append( str );
} else if (i.value instanceof Boolean) {
sb.append( (Boolean)i.value ? "1": "0");
} else {
sb.append( i.value.toString() );
}
sb.append(lastColumn ? lineFeed : columnSeparator);
// Get next time
i.time = (Double) i.stream.getHigherTime(Bindings.DOUBLE, i.time);
}
} while (readyColumns < items.size());
} finally {
closeHistory();
}
}*/
/**
* Reads visible data of all variables and formats as CSV lines (Comma
* Separated Values). Line Feed is \n, variable separator is \t, and
* decimal separator locale dependent.
*
* ReadData2 outputs one shared time and one value column for each variable
*
* Time | Variable1 Label | Variable3 Label
* | Variable1 Id | Variable3 Id
* | Variable1 Unit | Variable3 Unit
* 0.0 | 1.0 | 0.1
*
* @param monitor
* @param sb
* @throws HistoryException
* @throws IOException
*/
public void formulate2( ProgressMonitor monitor, Appendable sb ) throws HistoryException, IOException
{
if ( items.isEmpty() ) return;
timeFormatter = evaluateFormatter(timeFormat, decimalSeparator);
floatFormatter = evaluateFormatter(floatFormat, decimalSeparator);
numberFormatter = evaluateFormatter(numberFormat, decimalSeparator);
openHistory();
try {
// What is the time range of all items combined
double allFrom = Double.MAX_VALUE;
double allEnd = -Double.MAX_VALUE;
for (Item i : items) {
if (i.iter.isEmpty()) continue;
allFrom = Math.min(allFrom, i.iter.getFirstTime());
allEnd = Math.max(allEnd, i.iter.getLastTime());
}
// Write Headers
for (int hl = 0; hl < 3; ++hl) {
switch(hl) {
case 0: sb.append("Time"); break;
case 1: sb.append("----"); break;
case 2: sb.append("Unit"); break;
}
sb.append( columnSeparator.preference );
for (Item i : items)
{
boolean lastColumn = i == items.get( items.size()-1 );
switch (hl) {
case 0:
sb.append(i.label != null ? i.label : "");
break;
case 1:
sb.append(i.variableReference != null ? i.variableReference : "");
break;
case 2:
sb.append(i.unit==null?"no unit":i.unit);
break;
}
if (!lastColumn) sb.append( columnSeparator.preference );
}
sb.append( lineFeed );
}
// Prepare time
boolean hasAnyValues = allFrom != Double.MAX_VALUE && allEnd != -Double.MAX_VALUE;
// Make intersection of actual data range (allFrom, allEnd) and requested data (from, end)
double _from = Double.MAX_VALUE, _end = -Double.MAX_VALUE;
if (hasAnyValues) {
_from = Math.max(allFrom, from);
_end = Math.min(allEnd, end);
}
if (!hasAnyValues) return;
// Iterate until endTime is met for all variables
double time = _from;
if(!resample) {
// If resample is false then all samples are reported as is. The code achieves this by setting startTime to _from and timeStep to 0.0
time = _from;
timeStep = 0.0;
} else {
// time = startTime + n*timeStep
// Sampling based on given startTime and timeStep
if(timeStep > 0) {
// Find the first sample time that contains data if startTime < _from
double n = Math.max(0, Math.ceil((_from-startTime) / timeStep));
time = startTime + n*timeStep;
} else {
// Start sampling from startTime but make sure that it is not less than _from
if(startTime > _from) time = startTime;
}
}
// Must convert double times to String when initializing BigDecimal.
// Otherwise BigDecimal will pick up inaccuracies from beyond 15 precise digits
// thus making a mess of the time step calculations.
BigDecimal bigTime = new BigDecimal(String.valueOf(time));
BigDecimal bigTimeStep = new BigDecimal(String.valueOf(timeStep));
// Loop kill-switch for the case where timeStep > 0
boolean breakAfterNextWrite = false;
// System.out.println("time: " + time);
// System.out.println("timeStep: " + timeStep);
// System.out.println("_end: " + Double.toString(_end));
for (Item i : items) i.iter.gotoTime(time);
do {
if ( monitor!=null && monitor.isCanceled() ) return;
// Write time
String timeStr = timeFormatter.format( time );
//System.out.println("SAMPLING TIME: " + time);
sb.append( timeStr );
// Write values
for (Item i : items)
{
sb.append( columnSeparator.preference );
// Write value
if ( i.iter.hasValidValue() ) {
Object value = i.iter.getValueBand().getValue();
if (value instanceof Number) {
if (value instanceof Float || value instanceof Double) {
switch (numberInterpolation) {
case PREVIOUS_SAMPLE:
sb.append( formatNumber(value) );
break;
case LINEAR_INTERPOLATION:
if (time != i.iter.getValueBand().getTimeDouble() && i.iter.hasNext()) {
// Interpolate
int currentIndex = i.iter.getIndex();
ValueBand band = i.iter.getValueBand();
//double t1 = band.getTimeDouble();
Number v1 = (Number) value;
double t12 = band.getEndTimeDouble();
i.iter.next();
double t2 = i.iter.getValueBand().getTimeDouble();
Number v2 = (Number) i.iter.getValueBand().getValue();
i.iter.gotoIndex(currentIndex);
double vs = v1.doubleValue();
if(time > t12)
vs = HistoryExportUtil.biglerp(t12, v1.doubleValue(), t2, v2.doubleValue(), time);
sb.append( formatDouble(vs) );
} else {
// Exact timestamp match, or last sample.
// Don't interpolate nor extrapolate.
sb.append( formatNumber(value) );
}
break;
default:
throw new UnsupportedOperationException("Unsupported interpolation: " + numberInterpolation);
}
} else {
sb.append( value.toString() );
}
} else if (value instanceof Boolean) {
sb.append( (Boolean)value ? "1": "0");
} else {
sb.append( value.toString() );
}
}
}
sb.append( lineFeed );
if (breakAfterNextWrite)
break;
// Read next values, and the following times
if ( timeStep>0.0 ) {
bigTime = bigTime.add(bigTimeStep);
time = bigTime.doubleValue();
// gitlab #529: prevent last data point from getting dropped
// due to small imprecisions in re-sampling mode.
double diff = time - _end;
if (diff > 0 && diff <= RESAMPLING_END_TIMESTAMP_INCLUSION_TOLERANCE) {
time = _end;
breakAfterNextWrite = true;
// Take floating point inaccuracy into account when re-sampling
// to prevent the last data point from being left out if there
// is small-enough imprecision in the last data point time stamp
// to be considered negligible compared to expected stepped time.
}
} else {
// Get smallest end time that is larger than current time
Double nextTime = null;
// System.out.println("time = "+time);
for (Item i : items) {
Double itemNextTime = i.iter.getNextTime( time );
// System.err.println(" "+i.label+" nextTime="+itemNextTime);
if ( itemNextTime == null ) continue;
if ( itemNextTime < time ) continue;
if ( nextTime == null || ( nextTime > itemNextTime && !itemNextTime.equals( time ) ) ) nextTime = itemNextTime;
}
if ( nextTime == null || nextTime.equals( time ) ) break;
time = nextTime;
}
boolean hasMore = false;
for (Item i : items) {
i.iter.proceedToTime(time);
if(contains(i, time)) hasMore = true;
}
//System.out.println("hasMore @ " + time + " (" + bigTime + ") = " + hasMore);
if(!hasMore) break;
} while (time<=_end);
} finally {
closeHistory();
}
}
private boolean contains(Item item, double time) {
double start = item.iter.getStartTime();
double end = item.iter.getEndTime();
// A special case, where start == end => accept
if(time == start) return true;
else if(time < start) return false;
else if(time >= end) return false;
else return true;
}
private CharSequence formatNumber(Object value) {
return value instanceof Float
? floatFormatter.format( value )
: numberFormatter.format( value );
}
private CharSequence formatDouble(double value) {
return numberFormatter.format( value );
}
public void setDecimalSeparator(DecimalSeparator separator) {
this.decimalSeparator = separator;
}
public void setColumnSeparator(ColumnSeparator separator) {
this.columnSeparator = separator;
}
public void setResample(boolean resample) {
this.resample = resample;
}
public void setLineFeed( String lf ) {
this.lineFeed = lf;
}
public void setTimeFormat(Format format) {
this.timeFormat = format;
}
public void setFloatFormat(Format format) {
this.floatFormat = format;
}
public void setNumberFormat(Format format) {
this.numberFormat = format;
}
public void setLocale(Locale locale) {
this.locale = locale;
}
public void setNumberInterpolation(ExportInterpolation interpolation) {
this.numberInterpolation = interpolation;
}
private static String resolvePlatformLineFeed() {
String osName = System.getProperty("os.name", "");
osName = osName.toLowerCase();
if (osName.contains("windows"))
return "\r\n";
return "\n";
}
private class Item implements Comparable
- {
// Static data
String label; // Label
String variableReference; // Label
HistoryManager history; // History source for this item
String historyItemId;
String unit;
// State data
StreamAccessor accessor; // Stream accessor
StreamIterator iter;
public void open() throws HistoryException {
accessor = history.openStream(historyItemId, "r");
iter = new StreamIterator( accessor );
}
public void close() {
if (accessor!=null) {
try {
accessor.close();
} catch (AccessorException e) {
}
}
accessor = null;
iter = null;
}
@Override
public int compareTo(Item o) {
int i;
i = label.compareTo(o.label);
if (i!=0) return i;
i = variableReference.compareTo(o.variableReference);
if (i!=0) return i;
i = historyItemId.compareTo(o.historyItemId);
if (i!=0) return i;
return 0;
}
@Override
public int hashCode() {
int code = 0x2304;
code = 13*code + variableReference.hashCode();
code = 13*code + label.hashCode();
code = 13*code + historyItemId.hashCode();
code = 13*code + history.hashCode();
return code;
}
@Override
public boolean equals(Object obj) {
if ( obj == null ) return false;
if ( obj instanceof Item == false ) return false;
Item other = (Item) obj;
if ( !other.label.equals(label) ) return false;
if ( !other.variableReference.equals(variableReference) ) return false;
if ( !other.history.equals(history) ) return false;
if ( !other.historyItemId.equals(historyItemId) ) return false;
return true;
}
}
static interface Formatter {
String format(Object number);
}
static class NopFormatter implements Formatter {
private final Format format;
public NopFormatter(Format format) {
this.format = format;
}
public String format(Object number) {
return format.format(number);
}
}
static class ReplacingFormatter implements Formatter {
private final Format format;
private final char from;
private final char to;
public ReplacingFormatter(Format format, char from, char to) {
this.format = format;
this.from = from;
this.to = to;
}
public String format(Object number) {
return format.format(number).replace(from, to);
}
}
private Formatter evaluateFormatter(Format format, DecimalSeparator target) {
// Probe decimal separator
String onePointTwo = format.format(1.2);
//System.out.println("formatted zeroPointOne: " + onePointTwo);
DecimalSeparator formatSeparator;
if (onePointTwo.indexOf('.') != -1) {
formatSeparator = DecimalSeparator.DOT;
} else if (onePointTwo.indexOf(',') != -1) {
formatSeparator = DecimalSeparator.COMMA;
} else {
DecimalFormatSymbols symbols = DecimalFormatSymbols.getInstance(locale);
formatSeparator = DecimalSeparator.fromChar(symbols.getDecimalSeparator());
}
switch (formatSeparator) {
case COMMA:
switch (target) {
case COMMA:
return new NopFormatter(format);
case DOT:
return new ReplacingFormatter(format, ',', '.');
}
case DOT:
switch (target) {
case COMMA:
return new ReplacingFormatter(format, '.', ',');
case DOT:
return new NopFormatter(format);
}
}
return new NopFormatter(format);
}
}