Fixed CSVFormatter mind floating point inaccuracy when resampling

[simantics/platform.git] / bundles / org.simantics.history / src / org / simantics / history / csv / CSVFormatter.java

diff --git a/bundles/org.simantics.history/src/org/simantics/history/csv/CSVFormatter.java b/bundles/org.simantics.history/src/org/simantics/history/csv/CSVFormatter.java
index 6610d17f2456081d73031797f44599f43cf01ffb..b04e0dd37b37e34d0398855a451cd93cb534d6ff 100644 (file)
--- a/bundles/org.simantics.history/src/org/simantics/history/csv/CSVFormatter.java
+++ b/bundles/org.simantics.history/src/org/simantics/history/csv/CSVFormatter.java
@@ -12,9 +12,7 @@
 package org.simantics.history.csv;
 
 import java.io.IOException;
-import java.io.UnsupportedEncodingException;
 import java.math.BigDecimal;
-import java.net.URLDecoder;
 import java.text.DecimalFormat;
 import java.text.DecimalFormatSymbols;
 import java.text.Format;
@@ -40,6 +38,23 @@ import org.simantics.history.util.ValueBand;
  */
 public class CSVFormatter {
 
+       /**
+        * This is the tolerance used to decide whether or not the last data point of
+        * the exported items is included in the exported material or not. If
+        * <code>0 <= (t - t(lastDataPoint) < {@value #RESAMPLING_END_TIMESTAMP_INCLUSION_TOLERANCE}</code>
+        * is true, then the last exported data point will be
+        * <code>lastDataPoint</code>, with timestamp <code>t(lastDataPoint)</code> even
+        * if <code>t > t(lastDataPoint)</code>.
+        * 
+        * <p>
+        * This works around problems where floating point inaccuracy causes a data
+        * point to be left out from the the export when it would be fair for the user
+        * to expect the data to be exported would contain a point with time stamp
+        * <code>9.999999999999996</code> when sampling with time-step <code>1.0</code>
+        * starting from time <code>0.0</code>.
+        */
+       private static final double RESAMPLING_END_TIMESTAMP_INCLUSION_TOLERANCE = 1e-13;
+
        List<Item> items = new ArrayList<Item>();
        double from = -Double.MAX_VALUE;
        double end  =  Double.MAX_VALUE;
@@ -85,20 +100,12 @@ public class CSVFormatter {
                i.history = history;
                i.label = label!=null?label:"";
                i.variableReference = variableReference!=null?variableReference:"";
-               i.variableReference = unescape(i.variableReference);
+               i.variableReference = URIs.safeUnescape(i.variableReference);
                i.historyItemId = historyItemId;
                i.unit = unit;
                if ( !items.contains(i) ) items.add( i );
        }
 
-       private static String unescape(String url) {
-               try {
-                       return URLDecoder.decode(url, "UTF-8");
-               } catch (UnsupportedEncodingException e) {
-                       return url;
-               }
-       }
-
        /**
         * Sort items by variableId, label1, label2
         */
@@ -318,7 +325,7 @@ public class CSVFormatter {
                                // Sampling based on given startTime and timeStep
                                if(timeStep > 0) {
 
-                                       // Find the first sample time that contains data 
+                                       // Find the first sample time that contains data if startTime < _from 
                                        double n = Math.max(0, Math.ceil((_from-startTime) / timeStep));
                                        time = startTime + n*timeStep;
 
@@ -339,6 +346,13 @@ public class CSVFormatter {
                        BigDecimal bigTime = new BigDecimal(String.valueOf(time));
                        BigDecimal bigTimeStep = new BigDecimal(String.valueOf(timeStep));
 
+                       // Loop kill-switch for the case where timeStep > 0
+                       boolean breakAfterNextWrite = false;
+
+//                     System.out.println("time:     " + time);
+//                     System.out.println("timeStep: " + timeStep);
+//                     System.out.println("_end:     " + Double.toString(_end));
+
                        for (Item i : items) i.iter.gotoTime(time);
                        do {
                                if ( monitor!=null && monitor.isCanceled() ) return;
@@ -404,10 +418,26 @@ public class CSVFormatter {
 
                                sb.append( lineFeed );
 
-                   // Read next values, and the following times
-                   if ( timeStep>0.0 ) {
-                       bigTime = bigTime.add(bigTimeStep);
-                       time = bigTime.doubleValue();
+                               if (breakAfterNextWrite)
+                                       break;
+
+                               // Read next values, and the following times
+                               if ( timeStep>0.0 ) {
+                                       bigTime = bigTime.add(bigTimeStep);
+                                       time = bigTime.doubleValue();
+
+                                       // gitlab #529: prevent last data point from getting dropped
+                                       // due to small imprecisions in re-sampling mode.
+                                       double diff = time - _end;
+                                       if (diff > 0 && diff <= RESAMPLING_END_TIMESTAMP_INCLUSION_TOLERANCE) {
+                                               time = _end;
+                                               breakAfterNextWrite = true;
+                                               // Take floating point inaccuracy into account when re-sampling
+                                               // to prevent the last data point from being left out if there
+                                               // is small-enough imprecision in the last data point time stamp
+                                               // to be considered negligible compared to expected stepped time.
+                                       }
+
                    } else {
                        // Get smallest end time that is larger than current time
                        Double nextTime = null;
@@ -430,6 +460,7 @@ public class CSVFormatter {
                        if(contains(i, time)) hasMore = true;
                }
                
+               //System.out.println("hasMore @ " + time + " (" + bigTime + ") = " + hasMore);
                if(!hasMore) break;
                
                } while (time<=_end);
@@ -597,7 +628,7 @@ public class CSVFormatter {
     private Formatter evaluateFormatter(Format format, DecimalSeparator target) {
         // Probe decimal separator
         String onePointTwo = format.format(1.2);
-        System.out.println("formatted zeroPointOne: " + onePointTwo);
+        //System.out.println("formatted zeroPointOne: " + onePointTwo);
 
         DecimalSeparator formatSeparator;
         if (onePointTwo.indexOf('.') != -1) {