HistorySampler.sample(HistorySamplerItem2, ...) now uses mipmapped data 57/1857/1
authorTuukka Lehtonen <tuukka.lehtonen@semantum.fi>
Mon, 18 Jun 2018 10:58:54 +0000 (13:58 +0300)
committerTuukka Lehtonen <tuukka.lehtonen@semantum.fi>
Mon, 18 Jun 2018 11:01:05 +0000 (14:01 +0300)
Implemented selection of sampled history data source based on the
seconds/pixel (= timeWindow / maxSamples) ratio provided to the sampler.
This is one way to attempt not sampling too much data for trend views
that cannot inherently handle large amounts of data.

gitlab #9

Change-Id: I8c67068d6e832d33d4ac3adef270d31f1cb0e722

bundles/org.simantics.history/src/org/simantics/history/HistorySampler.java
bundles/org.simantics.history/src/org/simantics/history/HistorySamplerItem2.java

index 61f1f8bef7dfbe3864f0c7539688fecef758c015..7320f02499e552b576aad73df1c65bc9650536f7 100644 (file)
@@ -1,7 +1,5 @@
 package org.simantics.history;
 
-import gnu.trove.list.array.TDoubleArrayList;
-
 import java.io.IOException;
 import java.math.BigDecimal;
 
@@ -10,6 +8,8 @@ import org.simantics.history.util.HistoryExportUtil;
 import org.simantics.history.util.StreamIterator;
 import org.simantics.history.util.ValueBand;
 
+import gnu.trove.list.array.TDoubleArrayList;
+
 public class HistorySampler {
     
     public synchronized static TDoubleArrayList sample( HistorySamplerItem item, double from, double end, double timeWindow, double timeStep, boolean resample ) throws HistoryException, IOException {
@@ -226,67 +226,64 @@ public class HistorySampler {
 
     }
 
+    // ------------------------------------------------------------------------
+    // New history sampling routine, supports mip-mapped subscriptions
+
        public synchronized static TDoubleArrayList sample(HistorySamplerItem2 item, double end, double timeWindow, int maxSamples, boolean resample) throws HistoryException, IOException {
                try {
+                       // Avoid div / 0
+                       if (maxSamples <= 0)
+                               return new TDoubleArrayList(0);
+
                        // If there is something pending at this point, flush before opening for read
                        if (item.collector != null)
                                item.collector.flush();
-                       double pixelsPerSecond = (double) maxSamples / timeWindow;
-                       item.open(pixelsPerSecond);
+
+                       // Open data source with most suitable sampling interval
+                       double secondsPerPixel = timeWindow / (double) maxSamples;
+                       //System.out.println("SECONDS / PIXEL: " + secondsPerPixel);
+                       item.open(secondsPerPixel);
+
                        return sample(item.iter, end, timeWindow, maxSamples, resample);
                } finally {
                        item.close();
                }
        }
 
-       private static TDoubleArrayList sample(StreamIterator iter, double end, double timeWindow, int maxSamples, boolean resample) throws HistoryException, IOException {
-               //System.out.println("sample: " + end + ", " + timeWindow + ", " + maxSamples + ", " + resample);
-               ExportInterpolation numberInterpolation = ExportInterpolation.LINEAR_INTERPOLATION;
+       private static TDoubleArrayList sample(StreamIterator iter, double endTime, double timeWindow, int maxSamples, boolean resample) throws HistoryException, IOException {
+               double fromTime = endTime - timeWindow;
+               //System.out.println("sample: [" + fromTime + " .. " + endTime + "] window = " + timeWindow + " s, max samples = " + maxSamples + ", resample = " + resample);
+               ExportInterpolation interpolation = ExportInterpolation.LINEAR_INTERPOLATION;
 
-               double timeStep = 0;
-               double startTime = -1e10;
-
-               TDoubleArrayList result = new TDoubleArrayList();
-               if (iter.isEmpty())
-                       return result;
+               if (iter.isEmpty() || (resample && maxSamples <= 0))
+                       return new TDoubleArrayList(0);
 
-               double allFrom = iter.getFirstTime();
-               double allEnd = iter.getLastTime();
-               //System.out.println("data available [" + allFrom + " .. " + allEnd + "]");
-
-               double from = allEnd-timeWindow;
-               end = allEnd;
-               //System.out.println("sample between [" + from + " .. " + end + "]");
+               double dataFrom = iter.getFirstTime();
+               double dataEnd = iter.getLastTime();
 
                // Prepare time
-               boolean hasAnyValues = allFrom != Double.MAX_VALUE && allEnd != -Double.MAX_VALUE;
+               boolean hasAnyValues = dataFrom != Double.MAX_VALUE && dataEnd != -Double.MAX_VALUE;
                if (!hasAnyValues) {
-                       //System.err.println("=> no values");
-                       return result;
+                       //System.out.println("=> no values");
+                       return new TDoubleArrayList(0);
                }
 
                // Make intersection of actual data range (allFrom, allEnd) and requested data (from, end)
-               double _from = Math.max(allFrom, from);
-               double _end = Math.min(allEnd, end);
+               double from = Math.max(fromTime, dataFrom);
+               double end = Math.min(endTime, dataEnd);
 
-               // Iterate until endTime is met for all variables
-               double time = _from;
+               //System.out.println("data available [" + dataFrom + " .. " + dataEnd + "]");
+               //System.out.println("will sample between [" + from + " .. " + end + "]");
+
+               // Iterate until endTime is met
+               double time = from;
+               double timeStep = 0;
 
-               if (!resample) {
-                       // If resample is false then all samples are reported as is. The code achieves this by setting startTime to _from and timeStep to 0.0 
-                       time = _from;
-                       timeStep = 0.0;
+               if (resample) {
+                       timeStep = timeWindow / maxSamples;
                } else {
-                       // time = startTime + n*timeStep 
-                       // Sampling based on given startTime and timeStep
-                       if (timeStep > 0) {
-                               // Find the first sample time that contains data 
-                               double n = Math.max(0, Math.ceil((_from-startTime) / timeStep));
-                               time = startTime + n*timeStep;
-                       } else {
-                               // Start sampling from startTime but make sure that it is not less than _from
-                               if(startTime > _from) time = startTime;
-                       }
+                       // If resample is false then all samples are reported as is.
+                       // The code achieves this by setting startTime to from and timeStep to 0.0. 
                }
 
                // Must convert double times to String when initializing BigDecimal.
@@ -295,49 +292,46 @@ public class HistorySampler {
                BigDecimal bigTime = new BigDecimal(String.valueOf(time));
                BigDecimal bigTimeStep = new BigDecimal(String.valueOf(timeStep));
 
-               //System.err.println("=> goto " + time);
+               //System.out.println("=> goto " + time);
 
-               if(!iter.gotoTime(time)) {
-                       //System.err.println("=> no sample found at " + time);
+               TDoubleArrayList result = new TDoubleArrayList();
+               if (!iter.gotoTime(time)) {
+                       //System.out.println("=> no sample found at " + time);
                        return result;
                }
 
                //time = iter.getValueBand().getTimeDouble();
 
-               boolean ignore = Math.abs(time-from) < 1e-6; 
+               //System.out.println("=> ignore first item?: " + time + " - " + from + " = " + (time-from) + " => " + (Math.abs(time-from) < 1e-6));
+               //boolean ignore = Math.abs(time-from) < 1e-6;
+               boolean ignore = false;
 
                do {
-
-                       //System.err.println("process " + time + " " + iter.getValueBand());
-
-
+                       //System.out.println("process " + time + " " + iter.getValueBand() + " (ignore = " + ignore + ")");
 
                        // Check for valid value
                        if ( iter.hasValidValue() ) {
 
                                // Write time
-                               if(!ignore) {
-                                       //System.err.println("Add time : " + time);
+                               if (!ignore) {
+                                       //System.out.println("Add time : " + time);
                                        result.add(time);
                                }
                                // Write value
                                Object value = iter.getValueBand().getValue();
-                               //System.err.print("Add value : " + value);
+                               //System.out.print("Add value : " + value);
                                if (value instanceof Number) {
                                        if (value instanceof Float || value instanceof Double) {
-                                               switch (numberInterpolation) {
+                                               switch (interpolation) {
                                                case PREVIOUS_SAMPLE:
-
-                                                       if(!ignore) {
-                                                               //                                          System.err.println(" previous .. done!");
+                                                       if (!ignore) {
+                                                               //System.out.println(" previous .. done!");
                                                                result.add(((Number) value).doubleValue());
                                                        }
-
                                                        break;
 
                                                case LINEAR_INTERPOLATION:
                                                        if (time != iter.getValueBand().getTimeDouble() && iter.hasNext()) {
-
                                                                // Interpolate
                                                                int currentIndex = iter.getIndex();
                                                                ValueBand band = iter.getValueBand();
@@ -350,25 +344,26 @@ public class HistorySampler {
                                                                iter.gotoIndex(currentIndex);
 
                                                                double vs = v1.doubleValue();
-                                                               if(time > t12)
+                                                               if (time > t12)
                                                                        vs = HistoryExportUtil.biglerp(t12, v1.doubleValue(), t2, v2.doubleValue(), time);
 
-                                                               if(!ignore) {
-                                                                       //System.err.println(" linear .. done!");
+                                                               if (!ignore) {
+                                                                       //System.out.println(" linear .. done!");
                                                                        result.add(vs);
                                                                }
 
                                                        } else {
                                                                // Exact timestamp match, or last sample.
                                                                // Don't interpolate nor extrapolate.
-                                                               if(!ignore) {
-                                                                       //System.err.println(" else .. done!");
+                                                               if (!ignore) {
+                                                                       //System.out.println(" else .. done!");
                                                                        result.add(((Number) value).doubleValue());
                                                                }
                                                        }
                                                        break;
+
                                                default:
-                                                       throw new UnsupportedOperationException("Unsupported interpolation: " + numberInterpolation);
+                                                       throw new UnsupportedOperationException("Unsupported interpolation: " + interpolation);
                                                }
                                        } else {
                                                throw new IllegalStateException("Value is not a number " + value);
@@ -390,10 +385,13 @@ public class HistorySampler {
                        } else {
                                // Get smallest end time that is larger than current time
                                Double nextTime = null;
-                               //                      System.out.println("time = "+time);
-                               if(!iter.hasNext()) break;
+                               //System.out.println(" time = "+time);
+                               if(!iter.hasNext()) {
+                                       duplicateLastDataPoint(result, end);
+                                       break;
+                               }
                                Double itemNextTime = iter.getNextTime( time );
-                               //                              System.err.println("  "+i.label+" nextTime="+itemNextTime);
+                               //System.out.println("  "+iter.toString()+" nextTime="+itemNextTime);
                                if ( nextTime == null || ( nextTime > itemNextTime && !itemNextTime.equals( time ) ) ) nextTime = itemNextTime; 
                                if ( nextTime == null || nextTime.equals( time ) ) break;
                                time = nextTime;
@@ -402,16 +400,27 @@ public class HistorySampler {
                        boolean hasMore = false;
 
                        iter.proceedToTime(time);
-                       if(HistoryExportUtil.contains(iter, time)) hasMore = true;
-
-                       if(!hasMore) break;
+                       if (HistoryExportUtil.contains(iter, time)) hasMore = true;
 
-               } while (time<=_end);
+                       if (!hasMore) {
+                               if (time <= end) {
+                                       duplicateLastDataPoint(result, end);
+                               }
+                               break;
+                       }
 
-               //System.err.println("=> " + Arrays.toString(result.toArray()));
+               } while (time <= end);
 
+               //System.out.println("=> [" + result.size() + "]" + Arrays.toString(result.toArray()));
+               //System.out.println("=> [" + result.size() + "]");
                return result;
+       }
 
+       private static void duplicateLastDataPoint(TDoubleArrayList data, double timestamp) {
+               double lastValue = data.get(data.size() - 1);
+               //System.out.println("Duplicating last sample value " + lastValue + " @ " + timestamp);
+               data.add(timestamp);
+               data.add(lastValue);
        }
 
 }
index 73d89fa56a1aed8d73bed10724f12fe4417f4d40..be6940333a34486af59d3d1c16379daa23043254 100644 (file)
@@ -93,11 +93,11 @@ public class HistorySamplerItem2 implements Comparable<HistorySamplerItem2> {
                return true;
        }
 
-       private LevelItem getFormat(double pixelsPerSecond) throws HistoryException {
+       private LevelItem getFormat(double secondsPerPixel) throws HistoryException {
                LevelItem result = null;
                for (LevelItem format : items) {
                        double interval = format.samplingInterval;
-                       if (Double.isNaN( interval ) || interval <= pixelsPerSecond) {
+                       if (Double.isNaN( interval ) || interval <= secondsPerPixel) {
                                result = format;
                        } else {
                                break;