From e3940e67813aa0988bdd0fe6161500034e6e7518 Mon Sep 17 00:00:00 2001 From: Tuukka Lehtonen Date: Mon, 18 Jun 2018 13:58:54 +0300 Subject: [PATCH] HistorySampler.sample(HistorySamplerItem2, ...) now uses mipmapped data Implemented selection of sampled history data source based on the seconds/pixel (= timeWindow / maxSamples) ratio provided to the sampler. This is one way to attempt not sampling too much data for trend views that cannot inherently handle large amounts of data. gitlab #9 Change-Id: I8c67068d6e832d33d4ac3adef270d31f1cb0e722 --- .../org/simantics/history/HistorySampler.java | 153 +++++++++--------- .../history/HistorySamplerItem2.java | 4 +- 2 files changed, 83 insertions(+), 74 deletions(-) diff --git a/bundles/org.simantics.history/src/org/simantics/history/HistorySampler.java b/bundles/org.simantics.history/src/org/simantics/history/HistorySampler.java index 61f1f8bef..7320f0249 100644 --- a/bundles/org.simantics.history/src/org/simantics/history/HistorySampler.java +++ b/bundles/org.simantics.history/src/org/simantics/history/HistorySampler.java @@ -1,7 +1,5 @@ package org.simantics.history; -import gnu.trove.list.array.TDoubleArrayList; - import java.io.IOException; import java.math.BigDecimal; @@ -10,6 +8,8 @@ import org.simantics.history.util.HistoryExportUtil; import org.simantics.history.util.StreamIterator; import org.simantics.history.util.ValueBand; +import gnu.trove.list.array.TDoubleArrayList; + public class HistorySampler { public synchronized static TDoubleArrayList sample( HistorySamplerItem item, double from, double end, double timeWindow, double timeStep, boolean resample ) throws HistoryException, IOException { @@ -226,67 +226,64 @@ public class HistorySampler { } + // ------------------------------------------------------------------------ + // New history sampling routine, supports mip-mapped subscriptions + public synchronized static TDoubleArrayList sample(HistorySamplerItem2 item, double end, double timeWindow, int maxSamples, boolean resample) throws HistoryException, IOException { try { + // Avoid div / 0 + if (maxSamples <= 0) + return new TDoubleArrayList(0); + // If there is something pending at this point, flush before opening for read if (item.collector != null) item.collector.flush(); - double pixelsPerSecond = (double) maxSamples / timeWindow; - item.open(pixelsPerSecond); + + // Open data source with most suitable sampling interval + double secondsPerPixel = timeWindow / (double) maxSamples; + //System.out.println("SECONDS / PIXEL: " + secondsPerPixel); + item.open(secondsPerPixel); + return sample(item.iter, end, timeWindow, maxSamples, resample); } finally { item.close(); } } - private static TDoubleArrayList sample(StreamIterator iter, double end, double timeWindow, int maxSamples, boolean resample) throws HistoryException, IOException { - //System.out.println("sample: " + end + ", " + timeWindow + ", " + maxSamples + ", " + resample); - ExportInterpolation numberInterpolation = ExportInterpolation.LINEAR_INTERPOLATION; + private static TDoubleArrayList sample(StreamIterator iter, double endTime, double timeWindow, int maxSamples, boolean resample) throws HistoryException, IOException { + double fromTime = endTime - timeWindow; + //System.out.println("sample: [" + fromTime + " .. " + endTime + "] window = " + timeWindow + " s, max samples = " + maxSamples + ", resample = " + resample); + ExportInterpolation interpolation = ExportInterpolation.LINEAR_INTERPOLATION; - double timeStep = 0; - double startTime = -1e10; - - TDoubleArrayList result = new TDoubleArrayList(); - if (iter.isEmpty()) - return result; + if (iter.isEmpty() || (resample && maxSamples <= 0)) + return new TDoubleArrayList(0); - double allFrom = iter.getFirstTime(); - double allEnd = iter.getLastTime(); - //System.out.println("data available [" + allFrom + " .. " + allEnd + "]"); - - double from = allEnd-timeWindow; - end = allEnd; - //System.out.println("sample between [" + from + " .. " + end + "]"); + double dataFrom = iter.getFirstTime(); + double dataEnd = iter.getLastTime(); // Prepare time - boolean hasAnyValues = allFrom != Double.MAX_VALUE && allEnd != -Double.MAX_VALUE; + boolean hasAnyValues = dataFrom != Double.MAX_VALUE && dataEnd != -Double.MAX_VALUE; if (!hasAnyValues) { - //System.err.println("=> no values"); - return result; + //System.out.println("=> no values"); + return new TDoubleArrayList(0); } // Make intersection of actual data range (allFrom, allEnd) and requested data (from, end) - double _from = Math.max(allFrom, from); - double _end = Math.min(allEnd, end); + double from = Math.max(fromTime, dataFrom); + double end = Math.min(endTime, dataEnd); - // Iterate until endTime is met for all variables - double time = _from; + //System.out.println("data available [" + dataFrom + " .. " + dataEnd + "]"); + //System.out.println("will sample between [" + from + " .. " + end + "]"); + + // Iterate until endTime is met + double time = from; + double timeStep = 0; - if (!resample) { - // If resample is false then all samples are reported as is. The code achieves this by setting startTime to _from and timeStep to 0.0 - time = _from; - timeStep = 0.0; + if (resample) { + timeStep = timeWindow / maxSamples; } else { - // time = startTime + n*timeStep - // Sampling based on given startTime and timeStep - if (timeStep > 0) { - // Find the first sample time that contains data - double n = Math.max(0, Math.ceil((_from-startTime) / timeStep)); - time = startTime + n*timeStep; - } else { - // Start sampling from startTime but make sure that it is not less than _from - if(startTime > _from) time = startTime; - } + // If resample is false then all samples are reported as is. + // The code achieves this by setting startTime to from and timeStep to 0.0. } // Must convert double times to String when initializing BigDecimal. @@ -295,49 +292,46 @@ public class HistorySampler { BigDecimal bigTime = new BigDecimal(String.valueOf(time)); BigDecimal bigTimeStep = new BigDecimal(String.valueOf(timeStep)); - //System.err.println("=> goto " + time); + //System.out.println("=> goto " + time); - if(!iter.gotoTime(time)) { - //System.err.println("=> no sample found at " + time); + TDoubleArrayList result = new TDoubleArrayList(); + if (!iter.gotoTime(time)) { + //System.out.println("=> no sample found at " + time); return result; } //time = iter.getValueBand().getTimeDouble(); - boolean ignore = Math.abs(time-from) < 1e-6; + //System.out.println("=> ignore first item?: " + time + " - " + from + " = " + (time-from) + " => " + (Math.abs(time-from) < 1e-6)); + //boolean ignore = Math.abs(time-from) < 1e-6; + boolean ignore = false; do { - - //System.err.println("process " + time + " " + iter.getValueBand()); - - + //System.out.println("process " + time + " " + iter.getValueBand() + " (ignore = " + ignore + ")"); // Check for valid value if ( iter.hasValidValue() ) { // Write time - if(!ignore) { - //System.err.println("Add time : " + time); + if (!ignore) { + //System.out.println("Add time : " + time); result.add(time); } // Write value Object value = iter.getValueBand().getValue(); - //System.err.print("Add value : " + value); + //System.out.print("Add value : " + value); if (value instanceof Number) { if (value instanceof Float || value instanceof Double) { - switch (numberInterpolation) { + switch (interpolation) { case PREVIOUS_SAMPLE: - - if(!ignore) { - // System.err.println(" previous .. done!"); + if (!ignore) { + //System.out.println(" previous .. done!"); result.add(((Number) value).doubleValue()); } - break; case LINEAR_INTERPOLATION: if (time != iter.getValueBand().getTimeDouble() && iter.hasNext()) { - // Interpolate int currentIndex = iter.getIndex(); ValueBand band = iter.getValueBand(); @@ -350,25 +344,26 @@ public class HistorySampler { iter.gotoIndex(currentIndex); double vs = v1.doubleValue(); - if(time > t12) + if (time > t12) vs = HistoryExportUtil.biglerp(t12, v1.doubleValue(), t2, v2.doubleValue(), time); - if(!ignore) { - //System.err.println(" linear .. done!"); + if (!ignore) { + //System.out.println(" linear .. done!"); result.add(vs); } } else { // Exact timestamp match, or last sample. // Don't interpolate nor extrapolate. - if(!ignore) { - //System.err.println(" else .. done!"); + if (!ignore) { + //System.out.println(" else .. done!"); result.add(((Number) value).doubleValue()); } } break; + default: - throw new UnsupportedOperationException("Unsupported interpolation: " + numberInterpolation); + throw new UnsupportedOperationException("Unsupported interpolation: " + interpolation); } } else { throw new IllegalStateException("Value is not a number " + value); @@ -390,10 +385,13 @@ public class HistorySampler { } else { // Get smallest end time that is larger than current time Double nextTime = null; - // System.out.println("time = "+time); - if(!iter.hasNext()) break; + //System.out.println(" time = "+time); + if(!iter.hasNext()) { + duplicateLastDataPoint(result, end); + break; + } Double itemNextTime = iter.getNextTime( time ); - // System.err.println(" "+i.label+" nextTime="+itemNextTime); + //System.out.println(" "+iter.toString()+" nextTime="+itemNextTime); if ( nextTime == null || ( nextTime > itemNextTime && !itemNextTime.equals( time ) ) ) nextTime = itemNextTime; if ( nextTime == null || nextTime.equals( time ) ) break; time = nextTime; @@ -402,16 +400,27 @@ public class HistorySampler { boolean hasMore = false; iter.proceedToTime(time); - if(HistoryExportUtil.contains(iter, time)) hasMore = true; - - if(!hasMore) break; + if (HistoryExportUtil.contains(iter, time)) hasMore = true; - } while (time<=_end); + if (!hasMore) { + if (time <= end) { + duplicateLastDataPoint(result, end); + } + break; + } - //System.err.println("=> " + Arrays.toString(result.toArray())); + } while (time <= end); + //System.out.println("=> [" + result.size() + "]" + Arrays.toString(result.toArray())); + //System.out.println("=> [" + result.size() + "]"); return result; + } + private static void duplicateLastDataPoint(TDoubleArrayList data, double timestamp) { + double lastValue = data.get(data.size() - 1); + //System.out.println("Duplicating last sample value " + lastValue + " @ " + timestamp); + data.add(timestamp); + data.add(lastValue); } } diff --git a/bundles/org.simantics.history/src/org/simantics/history/HistorySamplerItem2.java b/bundles/org.simantics.history/src/org/simantics/history/HistorySamplerItem2.java index 73d89fa56..be6940333 100644 --- a/bundles/org.simantics.history/src/org/simantics/history/HistorySamplerItem2.java +++ b/bundles/org.simantics.history/src/org/simantics/history/HistorySamplerItem2.java @@ -93,11 +93,11 @@ public class HistorySamplerItem2 implements Comparable { return true; } - private LevelItem getFormat(double pixelsPerSecond) throws HistoryException { + private LevelItem getFormat(double secondsPerPixel) throws HistoryException { LevelItem result = null; for (LevelItem format : items) { double interval = format.samplingInterval; - if (Double.isNaN( interval ) || interval <= pixelsPerSecond) { + if (Double.isNaN( interval ) || interval <= secondsPerPixel) { result = format; } else { break; -- 2.43.2