]> gerrit.simantics Code Review - simantics/platform.git/blob - bundles/org.simantics.history/src/org/simantics/history/csv/CSVFormatter.java
Migrated source code from Simantics SVN
[simantics/platform.git] / bundles / org.simantics.history / src / org / simantics / history / csv / CSVFormatter.java
1 /*******************************************************************************\r
2  * Copyright (c) 2011 Association for Decentralized Information Management in\r
3  * Industry THTH ry.\r
4  * All rights reserved. This program and the accompanying materials\r
5  * are made available under the terms of the Eclipse Public License v1.0\r
6  * which accompanies this distribution, and is available at\r
7  * http://www.eclipse.org/legal/epl-v10.html\r
8  *\r
9  * Contributors:\r
10  *     VTT Technical Research Centre of Finland - initial API and implementation\r
11  *******************************************************************************/\r
12 package org.simantics.history.csv;\r
13 \r
14 import java.io.IOException;\r
15 import java.io.UnsupportedEncodingException;\r
16 import java.math.BigDecimal;\r
17 import java.net.URLDecoder;\r
18 import java.text.DecimalFormat;\r
19 import java.text.DecimalFormatSymbols;\r
20 import java.text.Format;\r
21 import java.util.ArrayList;\r
22 import java.util.Collections;\r
23 import java.util.List;\r
24 import java.util.Locale;\r
25 \r
26 import org.simantics.databoard.accessor.StreamAccessor;\r
27 import org.simantics.databoard.accessor.error.AccessorException;\r
28 import org.simantics.history.HistoryException;\r
29 import org.simantics.history.HistoryManager;\r
30 import org.simantics.history.util.HistoryExportUtil;\r
31 import org.simantics.history.util.ProgressMonitor;\r
32 import org.simantics.history.util.StreamIterator;\r
33 import org.simantics.history.util.ValueBand;\r
34 \r
35 /**\r
36  * CSV writer for history items.\r
37  * \r
38  * @author Toni Kalajainen\r
39  * @author Tuukka Lehtonen\r
40  */\r
41 public class CSVFormatter {\r
42 \r
43         List<Item> items = new ArrayList<Item>();\r
44         double from = -Double.MAX_VALUE;\r
45         double end  =  Double.MAX_VALUE;\r
46         double startTime = 0.0;\r
47         double timeStep = 0.0;\r
48         ColumnSeparator columnSeparator;\r
49         DecimalSeparator decimalSeparator;\r
50         boolean resample;\r
51         String lineFeed;\r
52         Locale locale;\r
53 \r
54         Format timeFormat;\r
55         Format floatFormat;\r
56         Format numberFormat;\r
57 \r
58         Formatter timeFormatter;\r
59         Formatter floatFormatter;\r
60         Formatter numberFormatter;\r
61 \r
62         ExportInterpolation numberInterpolation = ExportInterpolation.LINEAR_INTERPOLATION;\r
63 \r
64         public CSVFormatter() {\r
65                 this.lineFeed = resolvePlatformLineFeed();\r
66                 this.locale = Locale.getDefault(Locale.Category.FORMAT);\r
67 \r
68                 DecimalFormat defaultFormat = new DecimalFormat();\r
69                 defaultFormat.setGroupingUsed(false);\r
70                 setTimeFormat(defaultFormat);\r
71                 setFloatFormat(defaultFormat);\r
72                 setNumberFormat(defaultFormat);\r
73         }\r
74 \r
75         /**\r
76          * Add item to formatter\r
77          * \r
78          * @param historyItemId\r
79          * @param label\r
80          * @param variableReference\r
81          * @param unit\r
82          */\r
83         public void addItem( HistoryManager history, String historyItemId, String label, String variableReference, String unit ) {\r
84                 Item i = new Item();\r
85                 i.history = history;\r
86                 i.label = label!=null?label:"";\r
87                 i.variableReference = variableReference!=null?variableReference:"";\r
88                 i.variableReference = unescape(i.variableReference);\r
89                 i.historyItemId = historyItemId;\r
90                 i.unit = unit;\r
91                 if ( !items.contains(i) ) items.add( i );\r
92         }\r
93 \r
94         private static String unescape(String url) {\r
95                 try {\r
96                         return URLDecoder.decode(url, "UTF-8");\r
97                 } catch (UnsupportedEncodingException e) {\r
98                         return url;\r
99                 }\r
100         }\r
101 \r
102         /**\r
103          * Sort items by variableId, label1, label2\r
104          */\r
105         public void sort() {\r
106                 Collections.sort(items);\r
107         }\r
108         \r
109         public void setTimeRange( double from, double end ) {\r
110                 this.from = from;\r
111                 this.end = end;\r
112         }\r
113         \r
114         public void setStartTime( double startTime ) {\r
115                 this.startTime = startTime;             \r
116         }\r
117 \r
118         public void setTimeStep( double timeStep ) {\r
119                 this.timeStep = timeStep;               \r
120         }\r
121         \r
122         void openHistory() throws HistoryException {\r
123                 try {\r
124                         for (Item item : items) item.open();\r
125                 } catch (HistoryException e) {\r
126                         for (Item item : items) item.close();\r
127                         throw e;\r
128                 }\r
129         }\r
130         \r
131         void closeHistory() {\r
132                 for (Item item : items) item.close();\r
133         }\r
134         \r
135         /**\r
136      * Reads visible data of all variables and formats as CSV lines (Comma \r
137      * Separated Values). Line Feed is \n, variable separator is \t, and\r
138      * decimal separator locale dependent.\r
139      * \r
140      * ReadData1 outputs separate time and value columns for each variable\r
141      * \r
142      * Variable1 Time | Variable1 Value | Variable2 Time | Variable2 Value\r
143      * 0.0            | 1.0             | 0.1            | 23423.0\r
144          * \r
145          * @param monitor\r
146          * @param sb\r
147          * @throws HistoryException \r
148          * @throws IOException \r
149          */\r
150         /*\r
151         public void formulate1( ProgressMonitor monitor, Appendable sb ) throws HistoryException, IOException {\r
152                 if (items.isEmpty()) return;\r
153         boolean adaptComma = decimalSeparatorInLocale != decimalSeparator;\r
154                 openHistory();\r
155                 try {\r
156                 // Prepare columns: First time\r
157                 for (Item item : items)\r
158                 {\r
159                 if (monitor.isCanceled())\r
160                     return;\r
161                         if (item.stream.isEmpty()) continue;\r
162                         Double firstTime = (Double) item.stream.getFirstTime( Bindings.DOUBLE );\r
163                         if (from <= firstTime) {\r
164                                 item.time = firstTime;\r
165                         } else {\r
166                                 item.time = (Double) item.stream.getFloorTime(Bindings.DOUBLE, from);\r
167                         }\r
168                 }               \r
169                 \r
170                 // Write Headers\r
171                 for (Item i : items)\r
172                 {\r
173                     if (monitor.isCanceled())\r
174                         return;\r
175                     boolean lastColumn = i == items.get( items.size()-1 );\r
176                     sb.append(i.label + " Time");\r
177                     sb.append( columnSeparator );\r
178                     sb.append(i.label + " Value");\r
179                     sb.append(lastColumn ? lineFeed : columnSeparator );\r
180                 }\r
181                 \r
182                 // Iterate until endTime is met for all variables\r
183                 int readyColumns;\r
184                 do {\r
185                 if (monitor.isCanceled())\r
186                     return;\r
187 \r
188                 readyColumns = 0;\r
189                     for (Item i : items)\r
190                     {\r
191                             boolean lastColumn = i == items.get( items.size()-1 );\r
192         \r
193                         if (i.time == null || i.time > end) {\r
194                             readyColumns++;\r
195                             sb.append( lastColumn ? columnSeparator+lineFeed : columnSeparator+columnSeparator);\r
196                             continue;\r
197                         }\r
198         \r
199                         sb.append("");\r
200         \r
201                         // Write time\r
202                         String timeStr = format.format( i.time );\r
203                         if ( adaptComma ) timeStr = timeStr.replace(decimalSeparatorInLocale, decimalSeparator);\r
204                         sb.append( timeStr );           \r
205                         sb.append( columnSeparator );\r
206         \r
207                         // Write value                  \r
208                         i.value = i.stream.getValue(Bindings.DOUBLE, i.time);\r
209                         if (i.value instanceof Number) {\r
210                                 String str = format.format( i.value );\r
211                                 if ( adaptComma ) str = str.replace(decimalSeparatorInLocale, decimalSeparator);\r
212                             sb.append( str );\r
213                         } else if (i.value instanceof Boolean) {\r
214                                 sb.append( (Boolean)i.value ? "1": "0");\r
215                         } else {\r
216                             sb.append( i.value.toString() );\r
217                         }\r
218                         sb.append(lastColumn ? lineFeed : columnSeparator);\r
219         \r
220                         // Get next time\r
221                         i.time = (Double) i.stream.getHigherTime(Bindings.DOUBLE, i.time);\r
222                     }\r
223         \r
224                 } while (readyColumns < items.size());\r
225                 } finally {\r
226                         closeHistory();\r
227                 }\r
228         }*/\r
229         \r
230         /**\r
231      * Reads visible data of all variables and formats as CSV lines (Comma \r
232      * Separated Values). Line Feed is \n, variable separator is \t, and\r
233      * decimal separator locale dependent.\r
234      * \r
235      * ReadData2 outputs one shared time and one value column for each variable\r
236      * \r
237      * Time | Variable1 Label | Variable3 Label\r
238      *      | Variable1 Id    | Variable3 Id\r
239      *      | Variable1 Unit  | Variable3 Unit\r
240      * 0.0  | 1.0             | 0.1\r
241      * \r
242      * @param monitor\r
243      * @param sb\r
244      * @throws HistoryException \r
245          * @throws IOException \r
246      */\r
247     public void formulate2( ProgressMonitor monitor, Appendable sb ) throws HistoryException, IOException\r
248     {\r
249         if ( items.isEmpty() ) return;\r
250 \r
251         timeFormatter = evaluateFormatter(timeFormat, decimalSeparator);\r
252         floatFormatter = evaluateFormatter(floatFormat, decimalSeparator);\r
253         numberFormatter = evaluateFormatter(numberFormat, decimalSeparator);\r
254 \r
255         openHistory();\r
256         try {\r
257                 // What is the time range of all items combined\r
258                 double allFrom = Double.MAX_VALUE;\r
259                 double allEnd = -Double.MAX_VALUE;\r
260                 for (Item i : items) {\r
261                         if (i.iter.isEmpty()) continue;\r
262                         allFrom = Math.min(allFrom, i.iter.getFirstTime());\r
263                         allEnd = Math.max(allEnd, i.iter.getLastTime());\r
264                 }\r
265                 \r
266                 // Write Headers\r
267                 for (int hl = 0; hl < 3; ++hl) {\r
268                         switch(hl) {\r
269                         case 0: sb.append("Time"); break;\r
270                         case 1: sb.append("----"); break;\r
271                         case 2: sb.append("Unit"); break;\r
272                         }\r
273                     sb.append( columnSeparator.preference );\r
274                     for (Item i : items)\r
275                     {\r
276                            boolean lastColumn = i == items.get( items.size()-1 );\r
277                        switch (hl) {\r
278                            case 0:\r
279                                sb.append(i.label != null ? i.label : "");\r
280                                break;\r
281                            case 1:\r
282                                sb.append(i.variableReference != null ? i.variableReference : "");\r
283                                break;\r
284                            case 2:\r
285                                sb.append(i.unit==null?"no unit":i.unit);\r
286                                break;\r
287                        }\r
288                        if (!lastColumn) sb.append( columnSeparator.preference );\r
289                     }\r
290                     sb.append( lineFeed );\r
291                 }\r
292                 \r
293                 // Prepare time         \r
294                 boolean hasAnyValues = allFrom != Double.MAX_VALUE && allEnd != -Double.MAX_VALUE;\r
295                 \r
296                 // Make intersection of actual data range (allFrom, allEnd) and requested data (from, end)\r
297                 double _from = Double.MAX_VALUE, _end = -Double.MAX_VALUE;              \r
298                 if (hasAnyValues) {\r
299                         _from = Math.max(allFrom, from);\r
300                         _end = Math.min(allEnd, end);\r
301                 }\r
302 \r
303                 if (!hasAnyValues) return;\r
304 \r
305                         // Iterate until endTime is met for all variables\r
306                         double time = _from;\r
307 \r
308                         if(!resample) {\r
309                                 \r
310                                 // If resample is false then all samples are reported as is. The code achieves this by setting startTime to _from and timeStep to 0.0 \r
311                                 time = _from;\r
312                                 timeStep = 0.0;\r
313                                 \r
314                         } else {\r
315 \r
316                                 // time = startTime + n*timeStep \r
317                                 \r
318                                 // Sampling based on given startTime and timeStep\r
319                                 if(timeStep > 0) {\r
320 \r
321                                         // Find the first sample time that contains data \r
322                                         double n = Math.max(0, Math.ceil((_from-startTime) / timeStep));\r
323                                         time = startTime + n*timeStep;\r
324 \r
325                                 } else {\r
326                                         \r
327                                         // Start sampling from startTime but make sure that it is not less than _from\r
328                                         if(startTime > _from) time = startTime;\r
329                                         \r
330                                 }\r
331                                 \r
332 \r
333                         }\r
334                         \r
335                         // Must convert double times to String when initializing BigDecimal.\r
336                         // Otherwise BigDecimal will pick up inaccuracies from beyond 15 precise digits\r
337                         // thus making a mess of the time step calculations.\r
338 \r
339                         BigDecimal bigTime = new BigDecimal(String.valueOf(time));\r
340                         BigDecimal bigTimeStep = new BigDecimal(String.valueOf(timeStep));\r
341 \r
342                         for (Item i : items) i.iter.gotoTime(time);\r
343                         do {\r
344                                 if ( monitor!=null && monitor.isCanceled() ) return;\r
345 \r
346                                 // Write time\r
347                                 String timeStr = timeFormatter.format( time );\r
348                                 //System.out.println("SAMPLING TIME: " + time);\r
349                                 sb.append( timeStr );\r
350 \r
351                                 // Write values\r
352                                 for (Item i : items)\r
353                                 {\r
354                                         sb.append( columnSeparator.preference );\r
355                                         \r
356                                         // Write value\r
357                                         if ( i.iter.hasValidValue() ) {\r
358                                                 Object value = i.iter.getValueBand().getValue();\r
359                                                 if (value instanceof Number) {\r
360                                                         if (value instanceof Float || value instanceof Double) {\r
361                                                                 switch (numberInterpolation) {\r
362                                                                 case PREVIOUS_SAMPLE:\r
363                                                                         sb.append( formatNumber(value) );\r
364                                                                         break;\r
365 \r
366                                                                 case LINEAR_INTERPOLATION:\r
367                                                                         if (time != i.iter.getValueBand().getTimeDouble() && i.iter.hasNext()) {\r
368                                                                                 \r
369                                                                                 // Interpolate\r
370                                                                                 int currentIndex = i.iter.getIndex();\r
371                                                                                 ValueBand band = i.iter.getValueBand();\r
372                                                                                 //double t1 = band.getTimeDouble();\r
373                                                                                 Number v1 = (Number) value;\r
374                                                                                 double t12 = band.getEndTimeDouble();\r
375                                                                                 i.iter.next();\r
376                                                                                 double t2 = i.iter.getValueBand().getTimeDouble();\r
377                                                                                 Number v2 = (Number) i.iter.getValueBand().getValue();\r
378                                                                                 i.iter.gotoIndex(currentIndex);\r
379 \r
380                                                                                 double vs = v1.doubleValue();\r
381                                                                                 if(time > t12)\r
382                                                                                         vs = HistoryExportUtil.biglerp(t12, v1.doubleValue(), t2, v2.doubleValue(), time);\r
383 \r
384                                                                                 sb.append( formatDouble(vs) );\r
385                                                                         } else {\r
386                                                                                 // Exact timestamp match, or last sample.\r
387                                                                                 // Don't interpolate nor extrapolate.\r
388                                                                                 sb.append( formatNumber(value) );\r
389                                                                         }\r
390                                                                         break;\r
391                                                                 default:\r
392                                                                         throw new UnsupportedOperationException("Unsupported interpolation: " + numberInterpolation);\r
393                                                                 }\r
394                                                         } else {\r
395                                                                 sb.append( value.toString() );\r
396                                                         }\r
397                                                 } else if (value instanceof Boolean) {\r
398                                                         sb.append( (Boolean)value ? "1": "0");\r
399                                                 } else {\r
400                                                         sb.append( value.toString() );\r
401                                                 }\r
402                                         }\r
403                                 }\r
404 \r
405                                 sb.append( lineFeed );\r
406 \r
407                     // Read next values, and the following times\r
408                     if ( timeStep>0.0 ) {\r
409                         bigTime = bigTime.add(bigTimeStep);\r
410                         time = bigTime.doubleValue();\r
411                     } else {\r
412                         // Get smallest end time that is larger than current time\r
413                         Double nextTime = null;\r
414 //                      System.out.println("time = "+time);\r
415                         for (Item i : items) {\r
416                                 Double itemNextTime = i.iter.getNextTime( time );\r
417 //                              System.err.println("  "+i.label+" nextTime="+itemNextTime);\r
418                                 if ( itemNextTime == null ) continue;\r
419                                 if ( itemNextTime < time ) continue;\r
420                                 if ( nextTime == null || ( nextTime > itemNextTime && !itemNextTime.equals( time ) ) ) nextTime = itemNextTime; \r
421                         }\r
422                         if ( nextTime == null || nextTime.equals( time ) ) break;\r
423                         time = nextTime;\r
424                     }\r
425 \r
426                     boolean hasMore = false;\r
427                     \r
428                 for (Item i : items) {\r
429                         i.iter.proceedToTime(time);\r
430                         if(contains(i, time)) hasMore = true;\r
431                 }\r
432                 \r
433                 if(!hasMore) break;\r
434                 \r
435                 } while (time<=_end);\r
436         } finally {\r
437                 closeHistory();\r
438         }\r
439     }\r
440 \r
441     private boolean contains(Item item, double time) {\r
442         double start = item.iter.getStartTime();\r
443         double end = item.iter.getEndTime();\r
444         // A special case, where start == end => accept\r
445         if(time == start) return true;\r
446         else if(time < start) return false;\r
447         else if(time >= end) return false;\r
448         else return true;\r
449     }\r
450 \r
451         private CharSequence formatNumber(Object value) {\r
452                 return value instanceof Float\r
453                                 ? floatFormatter.format( value )\r
454                                 : numberFormatter.format( value );\r
455         }\r
456 \r
457         private CharSequence formatDouble(double value) {\r
458                 return numberFormatter.format( value );\r
459         }\r
460 \r
461         public void setDecimalSeparator(DecimalSeparator separator) {\r
462                 this.decimalSeparator = separator;\r
463         }\r
464 \r
465         public void setColumnSeparator(ColumnSeparator separator) {\r
466                 this.columnSeparator = separator;\r
467         }\r
468         \r
469         public void setResample(boolean resample) {\r
470                 this.resample = resample;\r
471         }\r
472         \r
473         public void setLineFeed( String lf ) {\r
474                 this.lineFeed = lf;\r
475         }\r
476 \r
477         public void setTimeFormat(Format format) {\r
478                 this.timeFormat = format;\r
479         }\r
480 \r
481         public void setFloatFormat(Format format) {\r
482                 this.floatFormat = format;\r
483         }\r
484 \r
485         public void setNumberFormat(Format format) {\r
486                 this.numberFormat = format;\r
487         }\r
488 \r
489         public void setLocale(Locale locale) {\r
490                 this.locale = locale;\r
491         }\r
492 \r
493         public void setNumberInterpolation(ExportInterpolation interpolation) {\r
494                 this.numberInterpolation = interpolation;\r
495         }\r
496 \r
497         private static String resolvePlatformLineFeed() {\r
498                 String osName = System.getProperty("os.name", "");\r
499                 osName = osName.toLowerCase();\r
500                 if (osName.contains("windows"))\r
501                         return "\r\n";\r
502                 return "\n";\r
503         }\r
504 \r
505     private class Item implements Comparable<Item> {\r
506         // Static data\r
507         String label;                                   // Label\r
508         String variableReference;               // Label\r
509         HistoryManager history;         // History source for this item\r
510         String historyItemId;\r
511         String unit;\r
512 \r
513         // State data\r
514         StreamAccessor accessor;                        // Stream accessor\r
515         StreamIterator iter;\r
516         \r
517         public void open() throws HistoryException {\r
518                 accessor = history.openStream(historyItemId, "r");\r
519                 iter = new StreamIterator( accessor );\r
520         }\r
521         \r
522         public void close() {\r
523                 if (accessor!=null) {\r
524                         try {\r
525                                         accessor.close();\r
526                                 } catch (AccessorException e) {\r
527                                 }\r
528                 }\r
529                 accessor = null;\r
530                 iter = null;\r
531         }\r
532 \r
533                 @Override\r
534                 public int compareTo(Item o) {\r
535                         int i;\r
536                         i = label.compareTo(o.label);\r
537                         if (i!=0) return i;\r
538                         i = variableReference.compareTo(o.variableReference);\r
539                         if (i!=0) return i;\r
540                         i = historyItemId.compareTo(o.historyItemId);                   \r
541                         if (i!=0) return i;\r
542                         return 0;\r
543                 }\r
544                 \r
545                 @Override\r
546                 public int hashCode() {\r
547                         int code = 0x2304;\r
548                         code = 13*code + variableReference.hashCode();\r
549                         code = 13*code + label.hashCode();\r
550                         code = 13*code + historyItemId.hashCode();\r
551                         code = 13*code + history.hashCode();                    \r
552                         return code;\r
553                 }\r
554                 \r
555                 @Override\r
556                 public boolean equals(Object obj) {\r
557                         if ( obj == null ) return false;\r
558                         if ( obj instanceof Item == false ) return false;\r
559                         Item other = (Item) obj;                        \r
560                         if ( !other.label.equals(label) ) return false;\r
561                         if ( !other.variableReference.equals(variableReference) ) return false;\r
562                         if ( !other.history.equals(history) ) return false;\r
563                         if ( !other.historyItemId.equals(historyItemId) ) return false;\r
564                         return true;\r
565                 }\r
566                 \r
567     }\r
568 \r
569     static interface Formatter {\r
570         String format(Object number);\r
571     }\r
572 \r
573     static class NopFormatter implements Formatter {\r
574         private final Format format;\r
575         public NopFormatter(Format format) {\r
576             this.format = format;\r
577         }\r
578         public String format(Object number) {\r
579             return format.format(number);\r
580         }\r
581     }\r
582 \r
583     static class ReplacingFormatter implements Formatter {\r
584         private final Format format;\r
585         private final char from;\r
586         private final char to;\r
587         public ReplacingFormatter(Format format, char from, char to) {\r
588             this.format = format;\r
589             this.from = from;\r
590             this.to = to;\r
591         }\r
592         public String format(Object number) {\r
593             return format.format(number).replace(from, to);\r
594         }\r
595     }\r
596 \r
597     private Formatter evaluateFormatter(Format format, DecimalSeparator target) {\r
598         // Probe decimal separator\r
599         String onePointTwo = format.format(1.2);\r
600         System.out.println("formatted zeroPointOne: " + onePointTwo);\r
601 \r
602         DecimalSeparator formatSeparator;\r
603         if (onePointTwo.indexOf('.') != -1) {\r
604             formatSeparator = DecimalSeparator.DOT;\r
605         } else if (onePointTwo.indexOf(',') != -1) {\r
606             formatSeparator = DecimalSeparator.COMMA;\r
607         } else {\r
608             DecimalFormatSymbols symbols = DecimalFormatSymbols.getInstance(locale);\r
609             formatSeparator = DecimalSeparator.fromChar(symbols.getDecimalSeparator());\r
610         }\r
611 \r
612         switch (formatSeparator) {\r
613         case COMMA:\r
614             switch (target) {\r
615             case COMMA:\r
616                 return new NopFormatter(format);\r
617             case DOT:\r
618                 return new ReplacingFormatter(format, ',', '.');\r
619             }\r
620         case DOT:\r
621             switch (target) {\r
622             case COMMA:\r
623                 return new ReplacingFormatter(format, '.', ',');\r
624             case DOT:\r
625                 return new NopFormatter(format);\r
626             }\r
627         }\r
628         return new NopFormatter(format);\r
629     }\r
630 \r
631 }\r