1 /*******************************************************************************
2 * Copyright (c) 2011 Association for Decentralized Information Management in
4 * All rights reserved. This program and the accompanying materials
5 * are made available under the terms of the Eclipse Public License v1.0
6 * which accompanies this distribution, and is available at
7 * http://www.eclipse.org/legal/epl-v10.html
10 * VTT Technical Research Centre of Finland - initial API and implementation
11 *******************************************************************************/
12 package org.simantics.history.csv;
14 import java.io.IOException;
15 import java.math.BigDecimal;
16 import java.text.DecimalFormat;
17 import java.text.DecimalFormatSymbols;
18 import java.text.Format;
19 import java.util.ArrayList;
20 import java.util.Collections;
21 import java.util.List;
22 import java.util.Locale;
24 import org.simantics.databoard.accessor.StreamAccessor;
25 import org.simantics.databoard.accessor.error.AccessorException;
26 import org.simantics.history.HistoryException;
27 import org.simantics.history.HistoryManager;
28 import org.simantics.history.util.HistoryExportUtil;
29 import org.simantics.history.util.ProgressMonitor;
30 import org.simantics.history.util.StreamIterator;
31 import org.simantics.history.util.ValueBand;
34 * CSV writer for history items.
36 * @author Toni Kalajainen
37 * @author Tuukka Lehtonen
39 public class CSVFormatter {
41 List<Item> items = new ArrayList<Item>();
42 double from = -Double.MAX_VALUE;
43 double end = Double.MAX_VALUE;
44 double startTime = 0.0;
45 double timeStep = 0.0;
46 ColumnSeparator columnSeparator;
47 DecimalSeparator decimalSeparator;
56 Formatter timeFormatter;
57 Formatter floatFormatter;
58 Formatter numberFormatter;
60 ExportInterpolation numberInterpolation = ExportInterpolation.LINEAR_INTERPOLATION;
62 public CSVFormatter() {
63 this.lineFeed = resolvePlatformLineFeed();
64 this.locale = Locale.getDefault(Locale.Category.FORMAT);
66 DecimalFormat defaultFormat = new DecimalFormat();
67 defaultFormat.setGroupingUsed(false);
68 setTimeFormat(defaultFormat);
69 setFloatFormat(defaultFormat);
70 setNumberFormat(defaultFormat);
74 * Add item to formatter
76 * @param historyItemId
78 * @param variableReference
81 public void addItem( HistoryManager history, String historyItemId, String label, String variableReference, String unit ) {
84 i.label = label!=null?label:"";
85 i.variableReference = variableReference!=null?variableReference:"";
86 i.variableReference = URIs.safeUnescape(i.variableReference);
87 i.historyItemId = historyItemId;
89 if ( !items.contains(i) ) items.add( i );
93 * Sort items by variableId, label1, label2
96 Collections.sort(items);
99 public void setTimeRange( double from, double end ) {
104 public void setStartTime( double startTime ) {
105 this.startTime = startTime;
108 public void setTimeStep( double timeStep ) {
109 this.timeStep = timeStep;
112 void openHistory() throws HistoryException {
114 for (Item item : items) item.open();
115 } catch (HistoryException e) {
116 for (Item item : items) item.close();
121 void closeHistory() {
122 for (Item item : items) item.close();
126 * Reads visible data of all variables and formats as CSV lines (Comma
127 * Separated Values). Line Feed is \n, variable separator is \t, and
128 * decimal separator locale dependent.
130 * ReadData1 outputs separate time and value columns for each variable
132 * Variable1 Time | Variable1 Value | Variable2 Time | Variable2 Value
133 * 0.0 | 1.0 | 0.1 | 23423.0
137 * @throws HistoryException
138 * @throws IOException
141 public void formulate1( ProgressMonitor monitor, Appendable sb ) throws HistoryException, IOException {
142 if (items.isEmpty()) return;
143 boolean adaptComma = decimalSeparatorInLocale != decimalSeparator;
146 // Prepare columns: First time
147 for (Item item : items)
149 if (monitor.isCanceled())
151 if (item.stream.isEmpty()) continue;
152 Double firstTime = (Double) item.stream.getFirstTime( Bindings.DOUBLE );
153 if (from <= firstTime) {
154 item.time = firstTime;
156 item.time = (Double) item.stream.getFloorTime(Bindings.DOUBLE, from);
163 if (monitor.isCanceled())
165 boolean lastColumn = i == items.get( items.size()-1 );
166 sb.append(i.label + " Time");
167 sb.append( columnSeparator );
168 sb.append(i.label + " Value");
169 sb.append(lastColumn ? lineFeed : columnSeparator );
172 // Iterate until endTime is met for all variables
175 if (monitor.isCanceled())
181 boolean lastColumn = i == items.get( items.size()-1 );
183 if (i.time == null || i.time > end) {
185 sb.append( lastColumn ? columnSeparator+lineFeed : columnSeparator+columnSeparator);
192 String timeStr = format.format( i.time );
193 if ( adaptComma ) timeStr = timeStr.replace(decimalSeparatorInLocale, decimalSeparator);
194 sb.append( timeStr );
195 sb.append( columnSeparator );
198 i.value = i.stream.getValue(Bindings.DOUBLE, i.time);
199 if (i.value instanceof Number) {
200 String str = format.format( i.value );
201 if ( adaptComma ) str = str.replace(decimalSeparatorInLocale, decimalSeparator);
203 } else if (i.value instanceof Boolean) {
204 sb.append( (Boolean)i.value ? "1": "0");
206 sb.append( i.value.toString() );
208 sb.append(lastColumn ? lineFeed : columnSeparator);
211 i.time = (Double) i.stream.getHigherTime(Bindings.DOUBLE, i.time);
214 } while (readyColumns < items.size());
221 * Reads visible data of all variables and formats as CSV lines (Comma
222 * Separated Values). Line Feed is \n, variable separator is \t, and
223 * decimal separator locale dependent.
225 * ReadData2 outputs one shared time and one value column for each variable
227 * Time | Variable1 Label | Variable3 Label
228 * | Variable1 Id | Variable3 Id
229 * | Variable1 Unit | Variable3 Unit
234 * @throws HistoryException
235 * @throws IOException
237 public void formulate2( ProgressMonitor monitor, Appendable sb ) throws HistoryException, IOException
239 if ( items.isEmpty() ) return;
241 timeFormatter = evaluateFormatter(timeFormat, decimalSeparator);
242 floatFormatter = evaluateFormatter(floatFormat, decimalSeparator);
243 numberFormatter = evaluateFormatter(numberFormat, decimalSeparator);
247 // What is the time range of all items combined
248 double allFrom = Double.MAX_VALUE;
249 double allEnd = -Double.MAX_VALUE;
250 for (Item i : items) {
251 if (i.iter.isEmpty()) continue;
252 allFrom = Math.min(allFrom, i.iter.getFirstTime());
253 allEnd = Math.max(allEnd, i.iter.getLastTime());
257 for (int hl = 0; hl < 3; ++hl) {
259 case 0: sb.append("Time"); break;
260 case 1: sb.append("----"); break;
261 case 2: sb.append("Unit"); break;
263 sb.append( columnSeparator.preference );
266 boolean lastColumn = i == items.get( items.size()-1 );
269 sb.append(i.label != null ? i.label : "");
272 sb.append(i.variableReference != null ? i.variableReference : "");
275 sb.append(i.unit==null?"no unit":i.unit);
278 if (!lastColumn) sb.append( columnSeparator.preference );
280 sb.append( lineFeed );
284 boolean hasAnyValues = allFrom != Double.MAX_VALUE && allEnd != -Double.MAX_VALUE;
286 // Make intersection of actual data range (allFrom, allEnd) and requested data (from, end)
287 double _from = Double.MAX_VALUE, _end = -Double.MAX_VALUE;
289 _from = Math.max(allFrom, from);
290 _end = Math.min(allEnd, end);
293 if (!hasAnyValues) return;
295 // Iterate until endTime is met for all variables
300 // If resample is false then all samples are reported as is. The code achieves this by setting startTime to _from and timeStep to 0.0
306 // time = startTime + n*timeStep
308 // Sampling based on given startTime and timeStep
311 // Find the first sample time that contains data if startTime < _from
312 double n = Math.max(0, Math.ceil((_from-startTime) / timeStep));
313 time = startTime + n*timeStep;
317 // Start sampling from startTime but make sure that it is not less than _from
318 if(startTime > _from) time = startTime;
325 // Must convert double times to String when initializing BigDecimal.
326 // Otherwise BigDecimal will pick up inaccuracies from beyond 15 precise digits
327 // thus making a mess of the time step calculations.
329 BigDecimal bigTime = new BigDecimal(String.valueOf(time));
330 BigDecimal bigTimeStep = new BigDecimal(String.valueOf(timeStep));
332 for (Item i : items) i.iter.gotoTime(time);
334 if ( monitor!=null && monitor.isCanceled() ) return;
337 String timeStr = timeFormatter.format( time );
338 //System.out.println("SAMPLING TIME: " + time);
339 sb.append( timeStr );
344 sb.append( columnSeparator.preference );
347 if ( i.iter.hasValidValue() ) {
348 Object value = i.iter.getValueBand().getValue();
349 if (value instanceof Number) {
350 if (value instanceof Float || value instanceof Double) {
351 switch (numberInterpolation) {
352 case PREVIOUS_SAMPLE:
353 sb.append( formatNumber(value) );
356 case LINEAR_INTERPOLATION:
357 if (time != i.iter.getValueBand().getTimeDouble() && i.iter.hasNext()) {
360 int currentIndex = i.iter.getIndex();
361 ValueBand band = i.iter.getValueBand();
362 //double t1 = band.getTimeDouble();
363 Number v1 = (Number) value;
364 double t12 = band.getEndTimeDouble();
366 double t2 = i.iter.getValueBand().getTimeDouble();
367 Number v2 = (Number) i.iter.getValueBand().getValue();
368 i.iter.gotoIndex(currentIndex);
370 double vs = v1.doubleValue();
372 vs = HistoryExportUtil.biglerp(t12, v1.doubleValue(), t2, v2.doubleValue(), time);
374 sb.append( formatDouble(vs) );
376 // Exact timestamp match, or last sample.
377 // Don't interpolate nor extrapolate.
378 sb.append( formatNumber(value) );
382 throw new UnsupportedOperationException("Unsupported interpolation: " + numberInterpolation);
385 sb.append( value.toString() );
387 } else if (value instanceof Boolean) {
388 sb.append( (Boolean)value ? "1": "0");
390 sb.append( value.toString() );
395 sb.append( lineFeed );
397 // Read next values, and the following times
398 if ( timeStep>0.0 ) {
399 bigTime = bigTime.add(bigTimeStep);
400 time = bigTime.doubleValue();
402 // Get smallest end time that is larger than current time
403 Double nextTime = null;
404 // System.out.println("time = "+time);
405 for (Item i : items) {
406 Double itemNextTime = i.iter.getNextTime( time );
407 // System.err.println(" "+i.label+" nextTime="+itemNextTime);
408 if ( itemNextTime == null ) continue;
409 if ( itemNextTime < time ) continue;
410 if ( nextTime == null || ( nextTime > itemNextTime && !itemNextTime.equals( time ) ) ) nextTime = itemNextTime;
412 if ( nextTime == null || nextTime.equals( time ) ) break;
416 boolean hasMore = false;
418 for (Item i : items) {
419 i.iter.proceedToTime(time);
420 if(contains(i, time)) hasMore = true;
425 } while (time<=_end);
431 private boolean contains(Item item, double time) {
432 double start = item.iter.getStartTime();
433 double end = item.iter.getEndTime();
434 // A special case, where start == end => accept
435 if(time == start) return true;
436 else if(time < start) return false;
437 else if(time >= end) return false;
441 private CharSequence formatNumber(Object value) {
442 return value instanceof Float
443 ? floatFormatter.format( value )
444 : numberFormatter.format( value );
447 private CharSequence formatDouble(double value) {
448 return numberFormatter.format( value );
451 public void setDecimalSeparator(DecimalSeparator separator) {
452 this.decimalSeparator = separator;
455 public void setColumnSeparator(ColumnSeparator separator) {
456 this.columnSeparator = separator;
459 public void setResample(boolean resample) {
460 this.resample = resample;
463 public void setLineFeed( String lf ) {
467 public void setTimeFormat(Format format) {
468 this.timeFormat = format;
471 public void setFloatFormat(Format format) {
472 this.floatFormat = format;
475 public void setNumberFormat(Format format) {
476 this.numberFormat = format;
479 public void setLocale(Locale locale) {
480 this.locale = locale;
483 public void setNumberInterpolation(ExportInterpolation interpolation) {
484 this.numberInterpolation = interpolation;
487 private static String resolvePlatformLineFeed() {
488 String osName = System.getProperty("os.name", "");
489 osName = osName.toLowerCase();
490 if (osName.contains("windows"))
495 private class Item implements Comparable<Item> {
497 String label; // Label
498 String variableReference; // Label
499 HistoryManager history; // History source for this item
500 String historyItemId;
504 StreamAccessor accessor; // Stream accessor
507 public void open() throws HistoryException {
508 accessor = history.openStream(historyItemId, "r");
509 iter = new StreamIterator( accessor );
512 public void close() {
513 if (accessor!=null) {
516 } catch (AccessorException e) {
524 public int compareTo(Item o) {
526 i = label.compareTo(o.label);
528 i = variableReference.compareTo(o.variableReference);
530 i = historyItemId.compareTo(o.historyItemId);
536 public int hashCode() {
538 code = 13*code + variableReference.hashCode();
539 code = 13*code + label.hashCode();
540 code = 13*code + historyItemId.hashCode();
541 code = 13*code + history.hashCode();
546 public boolean equals(Object obj) {
547 if ( obj == null ) return false;
548 if ( obj instanceof Item == false ) return false;
549 Item other = (Item) obj;
550 if ( !other.label.equals(label) ) return false;
551 if ( !other.variableReference.equals(variableReference) ) return false;
552 if ( !other.history.equals(history) ) return false;
553 if ( !other.historyItemId.equals(historyItemId) ) return false;
559 static interface Formatter {
560 String format(Object number);
563 static class NopFormatter implements Formatter {
564 private final Format format;
565 public NopFormatter(Format format) {
566 this.format = format;
568 public String format(Object number) {
569 return format.format(number);
573 static class ReplacingFormatter implements Formatter {
574 private final Format format;
575 private final char from;
576 private final char to;
577 public ReplacingFormatter(Format format, char from, char to) {
578 this.format = format;
582 public String format(Object number) {
583 return format.format(number).replace(from, to);
587 private Formatter evaluateFormatter(Format format, DecimalSeparator target) {
588 // Probe decimal separator
589 String onePointTwo = format.format(1.2);
590 //System.out.println("formatted zeroPointOne: " + onePointTwo);
592 DecimalSeparator formatSeparator;
593 if (onePointTwo.indexOf('.') != -1) {
594 formatSeparator = DecimalSeparator.DOT;
595 } else if (onePointTwo.indexOf(',') != -1) {
596 formatSeparator = DecimalSeparator.COMMA;
598 DecimalFormatSymbols symbols = DecimalFormatSymbols.getInstance(locale);
599 formatSeparator = DecimalSeparator.fromChar(symbols.getDecimalSeparator());
602 switch (formatSeparator) {
606 return new NopFormatter(format);
608 return new ReplacingFormatter(format, ',', '.');
613 return new ReplacingFormatter(format, '.', ',');
615 return new NopFormatter(format);
618 return new NopFormatter(format);