1 /*******************************************************************************
\r
2 * Copyright (c) 2011 Association for Decentralized Information Management in
\r
4 * All rights reserved. This program and the accompanying materials
\r
5 * are made available under the terms of the Eclipse Public License v1.0
\r
6 * which accompanies this distribution, and is available at
\r
7 * http://www.eclipse.org/legal/epl-v10.html
\r
10 * VTT Technical Research Centre of Finland - initial API and implementation
\r
11 *******************************************************************************/
\r
12 package org.simantics.history.csv;
\r
14 import java.io.IOException;
\r
15 import java.io.UnsupportedEncodingException;
\r
16 import java.math.BigDecimal;
\r
17 import java.net.URLDecoder;
\r
18 import java.text.DecimalFormat;
\r
19 import java.text.DecimalFormatSymbols;
\r
20 import java.text.Format;
\r
21 import java.util.ArrayList;
\r
22 import java.util.Collections;
\r
23 import java.util.List;
\r
24 import java.util.Locale;
\r
26 import org.simantics.databoard.accessor.StreamAccessor;
\r
27 import org.simantics.databoard.accessor.error.AccessorException;
\r
28 import org.simantics.history.HistoryException;
\r
29 import org.simantics.history.HistoryManager;
\r
30 import org.simantics.history.util.HistoryExportUtil;
\r
31 import org.simantics.history.util.ProgressMonitor;
\r
32 import org.simantics.history.util.StreamIterator;
\r
33 import org.simantics.history.util.ValueBand;
\r
36 * CSV writer for history items.
\r
38 * @author Toni Kalajainen
\r
39 * @author Tuukka Lehtonen
\r
41 public class CSVFormatter {
\r
43 List<Item> items = new ArrayList<Item>();
\r
44 double from = -Double.MAX_VALUE;
\r
45 double end = Double.MAX_VALUE;
\r
46 double startTime = 0.0;
\r
47 double timeStep = 0.0;
\r
48 ColumnSeparator columnSeparator;
\r
49 DecimalSeparator decimalSeparator;
\r
56 Format numberFormat;
\r
58 Formatter timeFormatter;
\r
59 Formatter floatFormatter;
\r
60 Formatter numberFormatter;
\r
62 ExportInterpolation numberInterpolation = ExportInterpolation.LINEAR_INTERPOLATION;
\r
64 public CSVFormatter() {
\r
65 this.lineFeed = resolvePlatformLineFeed();
\r
66 this.locale = Locale.getDefault(Locale.Category.FORMAT);
\r
68 DecimalFormat defaultFormat = new DecimalFormat();
\r
69 defaultFormat.setGroupingUsed(false);
\r
70 setTimeFormat(defaultFormat);
\r
71 setFloatFormat(defaultFormat);
\r
72 setNumberFormat(defaultFormat);
\r
76 * Add item to formatter
\r
78 * @param historyItemId
\r
80 * @param variableReference
\r
83 public void addItem( HistoryManager history, String historyItemId, String label, String variableReference, String unit ) {
\r
84 Item i = new Item();
\r
85 i.history = history;
\r
86 i.label = label!=null?label:"";
\r
87 i.variableReference = variableReference!=null?variableReference:"";
\r
88 i.variableReference = unescape(i.variableReference);
\r
89 i.historyItemId = historyItemId;
\r
91 if ( !items.contains(i) ) items.add( i );
\r
94 private static String unescape(String url) {
\r
96 return URLDecoder.decode(url, "UTF-8");
\r
97 } catch (UnsupportedEncodingException e) {
\r
103 * Sort items by variableId, label1, label2
\r
105 public void sort() {
\r
106 Collections.sort(items);
\r
109 public void setTimeRange( double from, double end ) {
\r
114 public void setStartTime( double startTime ) {
\r
115 this.startTime = startTime;
\r
118 public void setTimeStep( double timeStep ) {
\r
119 this.timeStep = timeStep;
\r
122 void openHistory() throws HistoryException {
\r
124 for (Item item : items) item.open();
\r
125 } catch (HistoryException e) {
\r
126 for (Item item : items) item.close();
\r
131 void closeHistory() {
\r
132 for (Item item : items) item.close();
\r
136 * Reads visible data of all variables and formats as CSV lines (Comma
\r
137 * Separated Values). Line Feed is \n, variable separator is \t, and
\r
138 * decimal separator locale dependent.
\r
140 * ReadData1 outputs separate time and value columns for each variable
\r
142 * Variable1 Time | Variable1 Value | Variable2 Time | Variable2 Value
\r
143 * 0.0 | 1.0 | 0.1 | 23423.0
\r
147 * @throws HistoryException
\r
148 * @throws IOException
\r
151 public void formulate1( ProgressMonitor monitor, Appendable sb ) throws HistoryException, IOException {
\r
152 if (items.isEmpty()) return;
\r
153 boolean adaptComma = decimalSeparatorInLocale != decimalSeparator;
\r
156 // Prepare columns: First time
\r
157 for (Item item : items)
\r
159 if (monitor.isCanceled())
\r
161 if (item.stream.isEmpty()) continue;
\r
162 Double firstTime = (Double) item.stream.getFirstTime( Bindings.DOUBLE );
\r
163 if (from <= firstTime) {
\r
164 item.time = firstTime;
\r
166 item.time = (Double) item.stream.getFloorTime(Bindings.DOUBLE, from);
\r
171 for (Item i : items)
\r
173 if (monitor.isCanceled())
\r
175 boolean lastColumn = i == items.get( items.size()-1 );
\r
176 sb.append(i.label + " Time");
\r
177 sb.append( columnSeparator );
\r
178 sb.append(i.label + " Value");
\r
179 sb.append(lastColumn ? lineFeed : columnSeparator );
\r
182 // Iterate until endTime is met for all variables
\r
185 if (monitor.isCanceled())
\r
189 for (Item i : items)
\r
191 boolean lastColumn = i == items.get( items.size()-1 );
\r
193 if (i.time == null || i.time > end) {
\r
195 sb.append( lastColumn ? columnSeparator+lineFeed : columnSeparator+columnSeparator);
\r
202 String timeStr = format.format( i.time );
\r
203 if ( adaptComma ) timeStr = timeStr.replace(decimalSeparatorInLocale, decimalSeparator);
\r
204 sb.append( timeStr );
\r
205 sb.append( columnSeparator );
\r
208 i.value = i.stream.getValue(Bindings.DOUBLE, i.time);
\r
209 if (i.value instanceof Number) {
\r
210 String str = format.format( i.value );
\r
211 if ( adaptComma ) str = str.replace(decimalSeparatorInLocale, decimalSeparator);
\r
213 } else if (i.value instanceof Boolean) {
\r
214 sb.append( (Boolean)i.value ? "1": "0");
\r
216 sb.append( i.value.toString() );
\r
218 sb.append(lastColumn ? lineFeed : columnSeparator);
\r
221 i.time = (Double) i.stream.getHigherTime(Bindings.DOUBLE, i.time);
\r
224 } while (readyColumns < items.size());
\r
231 * Reads visible data of all variables and formats as CSV lines (Comma
\r
232 * Separated Values). Line Feed is \n, variable separator is \t, and
\r
233 * decimal separator locale dependent.
\r
235 * ReadData2 outputs one shared time and one value column for each variable
\r
237 * Time | Variable1 Label | Variable3 Label
\r
238 * | Variable1 Id | Variable3 Id
\r
239 * | Variable1 Unit | Variable3 Unit
\r
244 * @throws HistoryException
\r
245 * @throws IOException
\r
247 public void formulate2( ProgressMonitor monitor, Appendable sb ) throws HistoryException, IOException
\r
249 if ( items.isEmpty() ) return;
\r
251 timeFormatter = evaluateFormatter(timeFormat, decimalSeparator);
\r
252 floatFormatter = evaluateFormatter(floatFormat, decimalSeparator);
\r
253 numberFormatter = evaluateFormatter(numberFormat, decimalSeparator);
\r
257 // What is the time range of all items combined
\r
258 double allFrom = Double.MAX_VALUE;
\r
259 double allEnd = -Double.MAX_VALUE;
\r
260 for (Item i : items) {
\r
261 if (i.iter.isEmpty()) continue;
\r
262 allFrom = Math.min(allFrom, i.iter.getFirstTime());
\r
263 allEnd = Math.max(allEnd, i.iter.getLastTime());
\r
267 for (int hl = 0; hl < 3; ++hl) {
\r
269 case 0: sb.append("Time"); break;
\r
270 case 1: sb.append("----"); break;
\r
271 case 2: sb.append("Unit"); break;
\r
273 sb.append( columnSeparator.preference );
\r
274 for (Item i : items)
\r
276 boolean lastColumn = i == items.get( items.size()-1 );
\r
279 sb.append(i.label != null ? i.label : "");
\r
282 sb.append(i.variableReference != null ? i.variableReference : "");
\r
285 sb.append(i.unit==null?"no unit":i.unit);
\r
288 if (!lastColumn) sb.append( columnSeparator.preference );
\r
290 sb.append( lineFeed );
\r
294 boolean hasAnyValues = allFrom != Double.MAX_VALUE && allEnd != -Double.MAX_VALUE;
\r
296 // Make intersection of actual data range (allFrom, allEnd) and requested data (from, end)
\r
297 double _from = Double.MAX_VALUE, _end = -Double.MAX_VALUE;
\r
298 if (hasAnyValues) {
\r
299 _from = Math.max(allFrom, from);
\r
300 _end = Math.min(allEnd, end);
\r
303 if (!hasAnyValues) return;
\r
305 // Iterate until endTime is met for all variables
\r
306 double time = _from;
\r
310 // If resample is false then all samples are reported as is. The code achieves this by setting startTime to _from and timeStep to 0.0
\r
316 // time = startTime + n*timeStep
\r
318 // Sampling based on given startTime and timeStep
\r
321 // Find the first sample time that contains data
\r
322 double n = Math.max(0, Math.ceil((_from-startTime) / timeStep));
\r
323 time = startTime + n*timeStep;
\r
327 // Start sampling from startTime but make sure that it is not less than _from
\r
328 if(startTime > _from) time = startTime;
\r
335 // Must convert double times to String when initializing BigDecimal.
\r
336 // Otherwise BigDecimal will pick up inaccuracies from beyond 15 precise digits
\r
337 // thus making a mess of the time step calculations.
\r
339 BigDecimal bigTime = new BigDecimal(String.valueOf(time));
\r
340 BigDecimal bigTimeStep = new BigDecimal(String.valueOf(timeStep));
\r
342 for (Item i : items) i.iter.gotoTime(time);
\r
344 if ( monitor!=null && monitor.isCanceled() ) return;
\r
347 String timeStr = timeFormatter.format( time );
\r
348 //System.out.println("SAMPLING TIME: " + time);
\r
349 sb.append( timeStr );
\r
352 for (Item i : items)
\r
354 sb.append( columnSeparator.preference );
\r
357 if ( i.iter.hasValidValue() ) {
\r
358 Object value = i.iter.getValueBand().getValue();
\r
359 if (value instanceof Number) {
\r
360 if (value instanceof Float || value instanceof Double) {
\r
361 switch (numberInterpolation) {
\r
362 case PREVIOUS_SAMPLE:
\r
363 sb.append( formatNumber(value) );
\r
366 case LINEAR_INTERPOLATION:
\r
367 if (time != i.iter.getValueBand().getTimeDouble() && i.iter.hasNext()) {
\r
370 int currentIndex = i.iter.getIndex();
\r
371 ValueBand band = i.iter.getValueBand();
\r
372 //double t1 = band.getTimeDouble();
\r
373 Number v1 = (Number) value;
\r
374 double t12 = band.getEndTimeDouble();
\r
376 double t2 = i.iter.getValueBand().getTimeDouble();
\r
377 Number v2 = (Number) i.iter.getValueBand().getValue();
\r
378 i.iter.gotoIndex(currentIndex);
\r
380 double vs = v1.doubleValue();
\r
382 vs = HistoryExportUtil.biglerp(t12, v1.doubleValue(), t2, v2.doubleValue(), time);
\r
384 sb.append( formatDouble(vs) );
\r
386 // Exact timestamp match, or last sample.
\r
387 // Don't interpolate nor extrapolate.
\r
388 sb.append( formatNumber(value) );
\r
392 throw new UnsupportedOperationException("Unsupported interpolation: " + numberInterpolation);
\r
395 sb.append( value.toString() );
\r
397 } else if (value instanceof Boolean) {
\r
398 sb.append( (Boolean)value ? "1": "0");
\r
400 sb.append( value.toString() );
\r
405 sb.append( lineFeed );
\r
407 // Read next values, and the following times
\r
408 if ( timeStep>0.0 ) {
\r
409 bigTime = bigTime.add(bigTimeStep);
\r
410 time = bigTime.doubleValue();
\r
412 // Get smallest end time that is larger than current time
\r
413 Double nextTime = null;
\r
414 // System.out.println("time = "+time);
\r
415 for (Item i : items) {
\r
416 Double itemNextTime = i.iter.getNextTime( time );
\r
417 // System.err.println(" "+i.label+" nextTime="+itemNextTime);
\r
418 if ( itemNextTime == null ) continue;
\r
419 if ( itemNextTime < time ) continue;
\r
420 if ( nextTime == null || ( nextTime > itemNextTime && !itemNextTime.equals( time ) ) ) nextTime = itemNextTime;
\r
422 if ( nextTime == null || nextTime.equals( time ) ) break;
\r
426 boolean hasMore = false;
\r
428 for (Item i : items) {
\r
429 i.iter.proceedToTime(time);
\r
430 if(contains(i, time)) hasMore = true;
\r
433 if(!hasMore) break;
\r
435 } while (time<=_end);
\r
441 private boolean contains(Item item, double time) {
\r
442 double start = item.iter.getStartTime();
\r
443 double end = item.iter.getEndTime();
\r
444 // A special case, where start == end => accept
\r
445 if(time == start) return true;
\r
446 else if(time < start) return false;
\r
447 else if(time >= end) return false;
\r
451 private CharSequence formatNumber(Object value) {
\r
452 return value instanceof Float
\r
453 ? floatFormatter.format( value )
\r
454 : numberFormatter.format( value );
\r
457 private CharSequence formatDouble(double value) {
\r
458 return numberFormatter.format( value );
\r
461 public void setDecimalSeparator(DecimalSeparator separator) {
\r
462 this.decimalSeparator = separator;
\r
465 public void setColumnSeparator(ColumnSeparator separator) {
\r
466 this.columnSeparator = separator;
\r
469 public void setResample(boolean resample) {
\r
470 this.resample = resample;
\r
473 public void setLineFeed( String lf ) {
\r
474 this.lineFeed = lf;
\r
477 public void setTimeFormat(Format format) {
\r
478 this.timeFormat = format;
\r
481 public void setFloatFormat(Format format) {
\r
482 this.floatFormat = format;
\r
485 public void setNumberFormat(Format format) {
\r
486 this.numberFormat = format;
\r
489 public void setLocale(Locale locale) {
\r
490 this.locale = locale;
\r
493 public void setNumberInterpolation(ExportInterpolation interpolation) {
\r
494 this.numberInterpolation = interpolation;
\r
497 private static String resolvePlatformLineFeed() {
\r
498 String osName = System.getProperty("os.name", "");
\r
499 osName = osName.toLowerCase();
\r
500 if (osName.contains("windows"))
\r
505 private class Item implements Comparable<Item> {
\r
507 String label; // Label
\r
508 String variableReference; // Label
\r
509 HistoryManager history; // History source for this item
\r
510 String historyItemId;
\r
514 StreamAccessor accessor; // Stream accessor
\r
515 StreamIterator iter;
\r
517 public void open() throws HistoryException {
\r
518 accessor = history.openStream(historyItemId, "r");
\r
519 iter = new StreamIterator( accessor );
\r
522 public void close() {
\r
523 if (accessor!=null) {
\r
526 } catch (AccessorException e) {
\r
534 public int compareTo(Item o) {
\r
536 i = label.compareTo(o.label);
\r
537 if (i!=0) return i;
\r
538 i = variableReference.compareTo(o.variableReference);
\r
539 if (i!=0) return i;
\r
540 i = historyItemId.compareTo(o.historyItemId);
\r
541 if (i!=0) return i;
\r
546 public int hashCode() {
\r
548 code = 13*code + variableReference.hashCode();
\r
549 code = 13*code + label.hashCode();
\r
550 code = 13*code + historyItemId.hashCode();
\r
551 code = 13*code + history.hashCode();
\r
556 public boolean equals(Object obj) {
\r
557 if ( obj == null ) return false;
\r
558 if ( obj instanceof Item == false ) return false;
\r
559 Item other = (Item) obj;
\r
560 if ( !other.label.equals(label) ) return false;
\r
561 if ( !other.variableReference.equals(variableReference) ) return false;
\r
562 if ( !other.history.equals(history) ) return false;
\r
563 if ( !other.historyItemId.equals(historyItemId) ) return false;
\r
569 static interface Formatter {
\r
570 String format(Object number);
\r
573 static class NopFormatter implements Formatter {
\r
574 private final Format format;
\r
575 public NopFormatter(Format format) {
\r
576 this.format = format;
\r
578 public String format(Object number) {
\r
579 return format.format(number);
\r
583 static class ReplacingFormatter implements Formatter {
\r
584 private final Format format;
\r
585 private final char from;
\r
586 private final char to;
\r
587 public ReplacingFormatter(Format format, char from, char to) {
\r
588 this.format = format;
\r
592 public String format(Object number) {
\r
593 return format.format(number).replace(from, to);
\r
597 private Formatter evaluateFormatter(Format format, DecimalSeparator target) {
\r
598 // Probe decimal separator
\r
599 String onePointTwo = format.format(1.2);
\r
600 System.out.println("formatted zeroPointOne: " + onePointTwo);
\r
602 DecimalSeparator formatSeparator;
\r
603 if (onePointTwo.indexOf('.') != -1) {
\r
604 formatSeparator = DecimalSeparator.DOT;
\r
605 } else if (onePointTwo.indexOf(',') != -1) {
\r
606 formatSeparator = DecimalSeparator.COMMA;
\r
608 DecimalFormatSymbols symbols = DecimalFormatSymbols.getInstance(locale);
\r
609 formatSeparator = DecimalSeparator.fromChar(symbols.getDecimalSeparator());
\r
612 switch (formatSeparator) {
\r
616 return new NopFormatter(format);
\r
618 return new ReplacingFormatter(format, ',', '.');
\r
623 return new ReplacingFormatter(format, '.', ',');
\r
625 return new NopFormatter(format);
\r
628 return new NopFormatter(format);
\r