bundles/org.simantics.history/src/org/simantics/history/csv/CSVFormatter.java

   1 /*******************************************************************************
   2  * Copyright (c) 2011 Association for Decentralized Information Management in
   3  * Industry THTH ry.
   4  * All rights reserved. This program and the accompanying materials
   5  * are made available under the terms of the Eclipse Public License v1.0
   6  * which accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *     VTT Technical Research Centre of Finland - initial API and implementation
  11  *******************************************************************************/
  12 package org.simantics.history.csv;
  13
  14 import java.io.IOException;
  15 import java.math.BigDecimal;
  16 import java.text.DecimalFormat;
  17 import java.text.DecimalFormatSymbols;
  18 import java.text.Format;
  19 import java.util.ArrayList;
  20 import java.util.Collections;
  21 import java.util.List;
  22 import java.util.Locale;
  23
  24 import org.simantics.databoard.accessor.StreamAccessor;
  25 import org.simantics.databoard.accessor.error.AccessorException;
  26 import org.simantics.history.HistoryException;
  27 import org.simantics.history.HistoryManager;
  28 import org.simantics.history.util.HistoryExportUtil;
  29 import org.simantics.history.util.ProgressMonitor;
  30 import org.simantics.history.util.StreamIterator;
  31 import org.simantics.history.util.ValueBand;
  32
  33 /**
  34  * CSV writer for history items.
  35  *
  36  * @author Toni Kalajainen
  37  * @author Tuukka Lehtonen
  38  */
  39 public class CSVFormatter {
  40
  41         List<Item> items = new ArrayList<Item>();
  42         double from = -Double.MAX_VALUE;
  43         double end  =  Double.MAX_VALUE;
  44         double startTime = 0.0;
  45         double timeStep = 0.0;
  46         ColumnSeparator columnSeparator;
  47         DecimalSeparator decimalSeparator;
  48         boolean resample;
  49         String lineFeed;
  50         Locale locale;
  51
  52         Format timeFormat;
  53         Format floatFormat;
  54         Format numberFormat;
  55
  56         Formatter timeFormatter;
  57         Formatter floatFormatter;
  58         Formatter numberFormatter;
  59
  60         ExportInterpolation numberInterpolation = ExportInterpolation.LINEAR_INTERPOLATION;
  61
  62         public CSVFormatter() {
  63                 this.lineFeed = resolvePlatformLineFeed();
  64                 this.locale = Locale.getDefault(Locale.Category.FORMAT);
  65
  66                 DecimalFormat defaultFormat = new DecimalFormat();
  67                 defaultFormat.setGroupingUsed(false);
  68                 setTimeFormat(defaultFormat);
  69                 setFloatFormat(defaultFormat);
  70                 setNumberFormat(defaultFormat);
  71         }
  72
  73         /**
  74          * Add item to formatter
  75          *
  76          * @param historyItemId
  77          * @param label
  78          * @param variableReference
  79          * @param unit
  80          */
  81         public void addItem( HistoryManager history, String historyItemId, String label, String variableReference, String unit ) {
  82                 Item i = new Item();
  83                 i.history = history;
  84                 i.label = label!=null?label:"";
  85                 i.variableReference = variableReference!=null?variableReference:"";
  86                 i.variableReference = URIs.safeUnescape(i.variableReference);
  87                 i.historyItemId = historyItemId;
  88                 i.unit = unit;
  89                 if ( !items.contains(i) ) items.add( i );
  90         }
  91
  92         /**
  93          * Sort items by variableId, label1, label2
  94          */
  95         public void sort() {
  96                 Collections.sort(items);
  97         }
  98
  99         public void setTimeRange( double from, double end ) {
 100                 this.from = from;
 101                 this.end = end;
 102         }
 103
 104         public void setStartTime( double startTime ) {
 105                 this.startTime = startTime;
 106         }
 107
 108         public void setTimeStep( double timeStep ) {
 109                 this.timeStep = timeStep;
 110         }
 111
 112         void openHistory() throws HistoryException {
 113                 try {
 114                         for (Item item : items) item.open();
 115                 } catch (HistoryException e) {
 116                         for (Item item : items) item.close();
 117                         throw e;
 118                 }
 119         }
 120
 121         void closeHistory() {
 122                 for (Item item : items) item.close();
 123         }
 124
 125         /**
 126      * Reads visible data of all variables and formats as CSV lines (Comma
 127      * Separated Values). Line Feed is \n, variable separator is \t, and
 128      * decimal separator locale dependent.
 129      *
 130      * ReadData1 outputs separate time and value columns for each variable
 131      *
 132      * Variable1 Time | Variable1 Value | Variable2 Time | Variable2 Value
 133      * 0.0            | 1.0             | 0.1            | 23423.0
 134          *
 135          * @param monitor
 136          * @param sb
 137          * @throws HistoryException
 138          * @throws IOException
 139          */
 140         /*
 141         public void formulate1( ProgressMonitor monitor, Appendable sb ) throws HistoryException, IOException {
 142                 if (items.isEmpty()) return;
 143         boolean adaptComma = decimalSeparatorInLocale != decimalSeparator;
 144                 openHistory();
 145                 try {
 146                 // Prepare columns: First time
 147                 for (Item item : items)
 148                 {
 149                 if (monitor.isCanceled())
 150                     return;
 151                         if (item.stream.isEmpty()) continue;
 152                         Double firstTime = (Double) item.stream.getFirstTime( Bindings.DOUBLE );
 153                         if (from <= firstTime) {
 154                                 item.time = firstTime;
 155                         } else {
 156                                 item.time = (Double) item.stream.getFloorTime(Bindings.DOUBLE, from);
 157                         }
 158                 }
 159
 160                 // Write Headers
 161                 for (Item i : items)
 162                 {
 163                     if (monitor.isCanceled())
 164                         return;
 165                     boolean lastColumn = i == items.get( items.size()-1 );
 166                     sb.append(i.label + " Time");
 167                     sb.append( columnSeparator );
 168                     sb.append(i.label + " Value");
 169                     sb.append(lastColumn ? lineFeed : columnSeparator );
 170                 }
 171
 172                 // Iterate until endTime is met for all variables
 173                 int readyColumns;
 174                 do {
 175                 if (monitor.isCanceled())
 176                     return;
 177
 178                 readyColumns = 0;
 179                     for (Item i : items)
 180                     {
 181                             boolean lastColumn = i == items.get( items.size()-1 );
 182
 183                         if (i.time == null || i.time > end) {
 184                             readyColumns++;
 185                             sb.append( lastColumn ? columnSeparator+lineFeed : columnSeparator+columnSeparator);
 186                             continue;
 187                         }
 188
 189                         sb.append("");
 190
 191                         // Write time
 192                         String timeStr = format.format( i.time );
 193                         if ( adaptComma ) timeStr = timeStr.replace(decimalSeparatorInLocale, decimalSeparator);
 194                         sb.append( timeStr );
 195                         sb.append( columnSeparator );
 196
 197                         // Write value
 198                         i.value = i.stream.getValue(Bindings.DOUBLE, i.time);
 199                         if (i.value instanceof Number) {
 200                                 String str = format.format( i.value );
 201                                 if ( adaptComma ) str = str.replace(decimalSeparatorInLocale, decimalSeparator);
 202                             sb.append( str );
 203                         } else if (i.value instanceof Boolean) {
 204                                 sb.append( (Boolean)i.value ? "1": "0");
 205                         } else {
 206                             sb.append( i.value.toString() );
 207                         }
 208                         sb.append(lastColumn ? lineFeed : columnSeparator);
 209
 210                         // Get next time
 211                         i.time = (Double) i.stream.getHigherTime(Bindings.DOUBLE, i.time);
 212                     }
 213
 214                 } while (readyColumns < items.size());
 215                 } finally {
 216                         closeHistory();
 217                 }
 218         }*/
 219
 220         /**
 221      * Reads visible data of all variables and formats as CSV lines (Comma
 222      * Separated Values). Line Feed is \n, variable separator is \t, and
 223      * decimal separator locale dependent.
 224      *
 225      * ReadData2 outputs one shared time and one value column for each variable
 226      *
 227      * Time | Variable1 Label | Variable3 Label
 228      *      | Variable1 Id    | Variable3 Id
 229      *      | Variable1 Unit  | Variable3 Unit
 230      * 0.0  | 1.0             | 0.1
 231      *
 232      * @param monitor
 233      * @param sb
 234      * @throws HistoryException
 235          * @throws IOException
 236      */
 237     public void formulate2( ProgressMonitor monitor, Appendable sb ) throws HistoryException, IOException
 238     {
 239         if ( items.isEmpty() ) return;
 240
 241         timeFormatter = evaluateFormatter(timeFormat, decimalSeparator);
 242         floatFormatter = evaluateFormatter(floatFormat, decimalSeparator);
 243         numberFormatter = evaluateFormatter(numberFormat, decimalSeparator);
 244
 245         openHistory();
 246         try {
 247                 // What is the time range of all items combined
 248                 double allFrom = Double.MAX_VALUE;
 249                 double allEnd = -Double.MAX_VALUE;
 250                 for (Item i : items) {
 251                         if (i.iter.isEmpty()) continue;
 252                         allFrom = Math.min(allFrom, i.iter.getFirstTime());
 253                         allEnd = Math.max(allEnd, i.iter.getLastTime());
 254                 }
 255
 256                 // Write Headers
 257                 for (int hl = 0; hl < 3; ++hl) {
 258                         switch(hl) {
 259                         case 0: sb.append("Time"); break;
 260                         case 1: sb.append("----"); break;
 261                         case 2: sb.append("Unit"); break;
 262                         }
 263                     sb.append( columnSeparator.preference );
 264                     for (Item i : items)
 265                     {
 266                            boolean lastColumn = i == items.get( items.size()-1 );
 267                        switch (hl) {
 268                            case 0:
 269                                sb.append(i.label != null ? i.label : "");
 270                                break;
 271                            case 1:
 272                                sb.append(i.variableReference != null ? i.variableReference : "");
 273                                break;
 274                            case 2:
 275                                sb.append(i.unit==null?"no unit":i.unit);
 276                                break;
 277                        }
 278                        if (!lastColumn) sb.append( columnSeparator.preference );
 279                     }
 280                     sb.append( lineFeed );
 281                 }
 282
 283                 // Prepare time
 284                 boolean hasAnyValues = allFrom != Double.MAX_VALUE && allEnd != -Double.MAX_VALUE;
 285
 286                 // Make intersection of actual data range (allFrom, allEnd) and requested data (from, end)
 287                 double _from = Double.MAX_VALUE, _end = -Double.MAX_VALUE;
 288                 if (hasAnyValues) {
 289                         _from = Math.max(allFrom, from);
 290                         _end = Math.min(allEnd, end);
 291                 }
 292
 293                 if (!hasAnyValues) return;
 294
 295                         // Iterate until endTime is met for all variables
 296                         double time = _from;
 297
 298                         if(!resample) {
 299
 300                                 // If resample is false then all samples are reported as is. The code achieves this by setting startTime to _from and timeStep to 0.0
 301                                 time = _from;
 302                                 timeStep = 0.0;
 303
 304                         } else {
 305
 306                                 // time = startTime + n*timeStep
 307
 308                                 // Sampling based on given startTime and timeStep
 309                                 if(timeStep > 0) {
 310
 311                                         // Find the first sample time that contains data
 312                                         double n = Math.max(0, Math.ceil((_from-startTime) / timeStep));
 313                                         time = startTime + n*timeStep;
 314
 315                                 } else {
 316
 317                                         // Start sampling from startTime but make sure that it is not less than _from
 318                                         if(startTime > _from) time = startTime;
 319
 320                                 }
 321
 322
 323                         }
 324
 325                         // Must convert double times to String when initializing BigDecimal.
 326                         // Otherwise BigDecimal will pick up inaccuracies from beyond 15 precise digits
 327                         // thus making a mess of the time step calculations.
 328
 329                         BigDecimal bigTime = new BigDecimal(String.valueOf(time));
 330                         BigDecimal bigTimeStep = new BigDecimal(String.valueOf(timeStep));
 331
 332                         for (Item i : items) i.iter.gotoTime(time);
 333                         do {
 334                                 if ( monitor!=null && monitor.isCanceled() ) return;
 335
 336                                 // Write time
 337                                 String timeStr = timeFormatter.format( time );
 338                                 //System.out.println("SAMPLING TIME: " + time);
 339                                 sb.append( timeStr );
 340
 341                                 // Write values
 342                                 for (Item i : items)
 343                                 {
 344                                         sb.append( columnSeparator.preference );
 345
 346                                         // Write value
 347                                         if ( i.iter.hasValidValue() ) {
 348                                                 Object value = i.iter.getValueBand().getValue();
 349                                                 if (value instanceof Number) {
 350                                                         if (value instanceof Float || value instanceof Double) {
 351                                                                 switch (numberInterpolation) {
 352                                                                 case PREVIOUS_SAMPLE:
 353                                                                         sb.append( formatNumber(value) );
 354                                                                         break;
 355
 356                                                                 case LINEAR_INTERPOLATION:
 357                                                                         if (time != i.iter.getValueBand().getTimeDouble() && i.iter.hasNext()) {
 358
 359                                                                                 // Interpolate
 360                                                                                 int currentIndex = i.iter.getIndex();
 361                                                                                 ValueBand band = i.iter.getValueBand();
 362                                                                                 //double t1 = band.getTimeDouble();
 363                                                                                 Number v1 = (Number) value;
 364                                                                                 double t12 = band.getEndTimeDouble();
 365                                                                                 i.iter.next();
 366                                                                                 double t2 = i.iter.getValueBand().getTimeDouble();
 367                                                                                 Number v2 = (Number) i.iter.getValueBand().getValue();
 368                                                                                 i.iter.gotoIndex(currentIndex);
 369
 370                                                                                 double vs = v1.doubleValue();
 371                                                                                 if(time > t12)
 372                                                                                         vs = HistoryExportUtil.biglerp(t12, v1.doubleValue(), t2, v2.doubleValue(), time);
 373
 374                                                                                 sb.append( formatDouble(vs) );
 375                                                                         } else {
 376                                                                                 // Exact timestamp match, or last sample.
 377                                                                                 // Don't interpolate nor extrapolate.
 378                                                                                 sb.append( formatNumber(value) );
 379                                                                         }
 380                                                                         break;
 381                                                                 default:
 382                                                                         throw new UnsupportedOperationException("Unsupported interpolation: " + numberInterpolation);
 383                                                                 }
 384                                                         } else {
 385                                                                 sb.append( value.toString() );
 386                                                         }
 387                                                 } else if (value instanceof Boolean) {
 388                                                         sb.append( (Boolean)value ? "1": "0");
 389                                                 } else {
 390                                                         sb.append( value.toString() );
 391                                                 }
 392                                         }
 393                                 }
 394
 395                                 sb.append( lineFeed );
 396
 397                     // Read next values, and the following times
 398                     if ( timeStep>0.0 ) {
 399                         bigTime = bigTime.add(bigTimeStep);
 400                         time = bigTime.doubleValue();
 401                     } else {
 402                         // Get smallest end time that is larger than current time
 403                         Double nextTime = null;
 404 //                      System.out.println("time = "+time);
 405                         for (Item i : items) {
 406                                 Double itemNextTime = i.iter.getNextTime( time );
 407 //                              System.err.println("  "+i.label+" nextTime="+itemNextTime);
 408                                 if ( itemNextTime == null ) continue;
 409                                 if ( itemNextTime < time ) continue;
 410                                 if ( nextTime == null || ( nextTime > itemNextTime && !itemNextTime.equals( time ) ) ) nextTime = itemNextTime;
 411                         }
 412                         if ( nextTime == null || nextTime.equals( time ) ) break;
 413                         time = nextTime;
 414                     }
 415
 416                     boolean hasMore = false;
 417
 418                 for (Item i : items) {
 419                         i.iter.proceedToTime(time);
 420                         if(contains(i, time)) hasMore = true;
 421                 }
 422
 423                 if(!hasMore) break;
 424
 425                 } while (time<=_end);
 426         } finally {
 427                 closeHistory();
 428         }
 429     }
 430
 431     private boolean contains(Item item, double time) {
 432         double start = item.iter.getStartTime();
 433         double end = item.iter.getEndTime();
 434         // A special case, where start == end => accept
 435         if(time == start) return true;
 436         else if(time < start) return false;
 437         else if(time >= end) return false;
 438         else return true;
 439     }
 440
 441         private CharSequence formatNumber(Object value) {
 442                 return value instanceof Float
 443                                 ? floatFormatter.format( value )
 444                                 : numberFormatter.format( value );
 445         }
 446
 447         private CharSequence formatDouble(double value) {
 448                 return numberFormatter.format( value );
 449         }
 450
 451         public void setDecimalSeparator(DecimalSeparator separator) {
 452                 this.decimalSeparator = separator;
 453         }
 454
 455         public void setColumnSeparator(ColumnSeparator separator) {
 456                 this.columnSeparator = separator;
 457         }
 458
 459         public void setResample(boolean resample) {
 460                 this.resample = resample;
 461         }
 462
 463         public void setLineFeed( String lf ) {
 464                 this.lineFeed = lf;
 465         }
 466
 467         public void setTimeFormat(Format format) {
 468                 this.timeFormat = format;
 469         }
 470
 471         public void setFloatFormat(Format format) {
 472                 this.floatFormat = format;
 473         }
 474
 475         public void setNumberFormat(Format format) {
 476                 this.numberFormat = format;
 477         }
 478
 479         public void setLocale(Locale locale) {
 480                 this.locale = locale;
 481         }
 482
 483         public void setNumberInterpolation(ExportInterpolation interpolation) {
 484                 this.numberInterpolation = interpolation;
 485         }
 486
 487         private static String resolvePlatformLineFeed() {
 488                 String osName = System.getProperty("os.name", "");
 489                 osName = osName.toLowerCase();
 490                 if (osName.contains("windows"))
 491                         return "\r\n";
 492                 return "\n";
 493         }
 494
 495     private class Item implements Comparable<Item> {
 496         // Static data
 497         String label;                                   // Label
 498         String variableReference;               // Label
 499         HistoryManager history;         // History source for this item
 500         String historyItemId;
 501         String unit;
 502
 503         // State data
 504         StreamAccessor accessor;                        // Stream accessor
 505         StreamIterator iter;
 506
 507         public void open() throws HistoryException {
 508                 accessor = history.openStream(historyItemId, "r");
 509                 iter = new StreamIterator( accessor );
 510         }
 511
 512         public void close() {
 513                 if (accessor!=null) {
 514                         try {
 515                                         accessor.close();
 516                                 } catch (AccessorException e) {
 517                                 }
 518                 }
 519                 accessor = null;
 520                 iter = null;
 521         }
 522
 523                 @Override
 524                 public int compareTo(Item o) {
 525                         int i;
 526                         i = label.compareTo(o.label);
 527                         if (i!=0) return i;
 528                         i = variableReference.compareTo(o.variableReference);
 529                         if (i!=0) return i;
 530                         i = historyItemId.compareTo(o.historyItemId);
 531                         if (i!=0) return i;
 532                         return 0;
 533                 }
 534
 535                 @Override
 536                 public int hashCode() {
 537                         int code = 0x2304;
 538                         code = 13*code + variableReference.hashCode();
 539                         code = 13*code + label.hashCode();
 540                         code = 13*code + historyItemId.hashCode();
 541                         code = 13*code + history.hashCode();
 542                         return code;
 543                 }
 544
 545                 @Override
 546                 public boolean equals(Object obj) {
 547                         if ( obj == null ) return false;
 548                         if ( obj instanceof Item == false ) return false;
 549                         Item other = (Item) obj;
 550                         if ( !other.label.equals(label) ) return false;
 551                         if ( !other.variableReference.equals(variableReference) ) return false;
 552                         if ( !other.history.equals(history) ) return false;
 553                         if ( !other.historyItemId.equals(historyItemId) ) return false;
 554                         return true;
 555                 }
 556
 557     }
 558
 559     static interface Formatter {
 560         String format(Object number);
 561     }
 562
 563     static class NopFormatter implements Formatter {
 564         private final Format format;
 565         public NopFormatter(Format format) {
 566             this.format = format;
 567         }
 568         public String format(Object number) {
 569             return format.format(number);
 570         }
 571     }
 572
 573     static class ReplacingFormatter implements Formatter {
 574         private final Format format;
 575         private final char from;
 576         private final char to;
 577         public ReplacingFormatter(Format format, char from, char to) {
 578             this.format = format;
 579             this.from = from;
 580             this.to = to;
 581         }
 582         public String format(Object number) {
 583             return format.format(number).replace(from, to);
 584         }
 585     }
 586
 587     private Formatter evaluateFormatter(Format format, DecimalSeparator target) {
 588         // Probe decimal separator
 589         String onePointTwo = format.format(1.2);
 590         System.out.println("formatted zeroPointOne: " + onePointTwo);
 591
 592         DecimalSeparator formatSeparator;
 593         if (onePointTwo.indexOf('.') != -1) {
 594             formatSeparator = DecimalSeparator.DOT;
 595         } else if (onePointTwo.indexOf(',') != -1) {
 596             formatSeparator = DecimalSeparator.COMMA;
 597         } else {
 598             DecimalFormatSymbols symbols = DecimalFormatSymbols.getInstance(locale);
 599             formatSeparator = DecimalSeparator.fromChar(symbols.getDecimalSeparator());
 600         }
 601
 602         switch (formatSeparator) {
 603         case COMMA:
 604             switch (target) {
 605             case COMMA:
 606                 return new NopFormatter(format);
 607             case DOT:
 608                 return new ReplacingFormatter(format, ',', '.');
 609             }
 610         case DOT:
 611             switch (target) {
 612             case COMMA:
 613                 return new ReplacingFormatter(format, '.', ',');
 614             case DOT:
 615                 return new NopFormatter(format);
 616             }
 617         }
 618         return new NopFormatter(format);
 619     }
 620
 621 }