From f3617565a8ffe49d63016f0c1d78e0d4282214b0 Mon Sep 17 00:00:00 2001 From: Tuukka Lehtonen Date: Fri, 18 May 2018 21:27:28 +0300 Subject: [PATCH] Added another sampling method to HistorySampler This intentionally doesn't replace the original history sampling function since it might be used elsewhere. gitlab #9 Change-Id: Ie7ea8f37f29a5a277a6678e7716e340884d30e69 --- .../src/org/simantics/charts/Charts.java | 48 ++++- .../org/simantics/history/HistorySampler.java | 189 +++++++++++++++++- .../history/HistorySamplerItem2.java | 109 ++++++++++ .../util/subscription/SamplingFormat.java | 20 +- 4 files changed, 356 insertions(+), 10 deletions(-) create mode 100644 bundles/org.simantics.history/src/org/simantics/history/HistorySamplerItem2.java diff --git a/bundles/org.simantics.charts/src/org/simantics/charts/Charts.java b/bundles/org.simantics.charts/src/org/simantics/charts/Charts.java index 8ca1ca1ea..c7dee69b0 100644 --- a/bundles/org.simantics.charts/src/org/simantics/charts/Charts.java +++ b/bundles/org.simantics.charts/src/org/simantics/charts/Charts.java @@ -1,6 +1,7 @@ package org.simantics.charts; import java.util.Collections; +import java.util.Comparator; import java.util.List; import org.simantics.charts.editor.ChartData; @@ -14,7 +15,9 @@ import org.simantics.db.exception.DatabaseException; import org.simantics.db.layer0.variable.Variable; import org.simantics.history.HistoryException; import org.simantics.history.HistorySamplerItem; +import org.simantics.history.HistorySamplerItem2; import org.simantics.history.ItemManager; +import org.simantics.history.HistorySamplerItem2.LevelItem; import org.simantics.history.util.subscription.SamplingFormat; import org.simantics.modeling.subscription.SubscriptionItem; import org.simantics.modeling.subscription.SubscriptionItemQuery; @@ -64,7 +67,7 @@ public final class Charts { List items = im.search("variableId", i.variableId); Collections.sort(items, SamplingFormat.INTERVAL_COMPARATOR); if (items.isEmpty()) - new DatabaseException("There is history item for " + i.variableId); + throw new DatabaseException("There is history item for " + i.variableId); Bean config = items.get(0); String historyId = (String) config.getFieldUnchecked("id"); @@ -76,4 +79,47 @@ public final class Charts { } } + public static HistorySamplerItem2 createHistorySamplerItem2(ReadGraph graph, Variable run, Resource subscriptionItem) throws DatabaseException { + IExperiment exp = (IExperiment) run.getPropertyValue(graph, "iExperiment"); + ITrendSupport support = exp.getService(ITrendSupport.class); + ChartData data = support.getChartData(); + return createHistorySamplerItem2(graph, subscriptionItem, data); + } + + public static HistorySamplerItem2 createHistorySamplerItem2(ReadGraph graph, Resource subscriptionItem, ChartData data) throws DatabaseException { + try { + Resource indexRoot = graph.syncRequest(new PossibleIndexRoot(subscriptionItem)); + if (indexRoot == null) + throw new DatabaseException("There is no index root for " + subscriptionItem); + if (data == null) + throw new DatabaseException("There is no chart data for " + subscriptionItem); + + SubscriptionItem i = graph.syncRequest(new SubscriptionItemQuery(subscriptionItem)); + + // TODO: this is super-inefficient! + ItemManager im = new ItemManager(data.history.getItems()); + + List items = im.search("variableId", i.variableId); + if (items.isEmpty()) + throw new DatabaseException("There is history item for " + i.variableId); + + LevelItem[] historyItems = items.stream().map(Charts::toLevelItem).sorted(ASCENDING_INTERVAL_ORDER).toArray(LevelItem[]::new); + + return new HistorySamplerItem2(data.collector, data.history, historyItems, System.identityHashCode(data)); + } catch (HistoryException e) { + throw new DatabaseException(e); + } catch (BindingException e) { + throw new DatabaseException(e); + } + } + + private static LevelItem toLevelItem(Bean bean) { + String id = (String) bean.getFieldUnchecked("id"); + double interval = (Double) bean.getFieldUnchecked("interval"); + return new LevelItem(id, interval); + } + + public final static Comparator ASCENDING_INTERVAL_ORDER = + (o1, o2) -> SamplingFormat.compareSamplingInterval(o1.samplingInterval, o2.samplingInterval); + } diff --git a/bundles/org.simantics.history/src/org/simantics/history/HistorySampler.java b/bundles/org.simantics.history/src/org/simantics/history/HistorySampler.java index 1c45a220f..61f1f8bef 100644 --- a/bundles/org.simantics.history/src/org/simantics/history/HistorySampler.java +++ b/bundles/org.simantics.history/src/org/simantics/history/HistorySampler.java @@ -226,5 +226,192 @@ public class HistorySampler { } - + public synchronized static TDoubleArrayList sample(HistorySamplerItem2 item, double end, double timeWindow, int maxSamples, boolean resample) throws HistoryException, IOException { + try { + // If there is something pending at this point, flush before opening for read + if (item.collector != null) + item.collector.flush(); + double pixelsPerSecond = (double) maxSamples / timeWindow; + item.open(pixelsPerSecond); + return sample(item.iter, end, timeWindow, maxSamples, resample); + } finally { + item.close(); + } + } + + private static TDoubleArrayList sample(StreamIterator iter, double end, double timeWindow, int maxSamples, boolean resample) throws HistoryException, IOException { + //System.out.println("sample: " + end + ", " + timeWindow + ", " + maxSamples + ", " + resample); + ExportInterpolation numberInterpolation = ExportInterpolation.LINEAR_INTERPOLATION; + + double timeStep = 0; + double startTime = -1e10; + + TDoubleArrayList result = new TDoubleArrayList(); + if (iter.isEmpty()) + return result; + + double allFrom = iter.getFirstTime(); + double allEnd = iter.getLastTime(); + //System.out.println("data available [" + allFrom + " .. " + allEnd + "]"); + + double from = allEnd-timeWindow; + end = allEnd; + //System.out.println("sample between [" + from + " .. " + end + "]"); + + // Prepare time + boolean hasAnyValues = allFrom != Double.MAX_VALUE && allEnd != -Double.MAX_VALUE; + if (!hasAnyValues) { + //System.err.println("=> no values"); + return result; + } + + // Make intersection of actual data range (allFrom, allEnd) and requested data (from, end) + double _from = Math.max(allFrom, from); + double _end = Math.min(allEnd, end); + + // Iterate until endTime is met for all variables + double time = _from; + + if (!resample) { + // If resample is false then all samples are reported as is. The code achieves this by setting startTime to _from and timeStep to 0.0 + time = _from; + timeStep = 0.0; + } else { + // time = startTime + n*timeStep + // Sampling based on given startTime and timeStep + if (timeStep > 0) { + // Find the first sample time that contains data + double n = Math.max(0, Math.ceil((_from-startTime) / timeStep)); + time = startTime + n*timeStep; + } else { + // Start sampling from startTime but make sure that it is not less than _from + if(startTime > _from) time = startTime; + } + } + + // Must convert double times to String when initializing BigDecimal. + // Otherwise BigDecimal will pick up inaccuracies from beyond 15 precise digits + // thus making a mess of the time step calculations. + BigDecimal bigTime = new BigDecimal(String.valueOf(time)); + BigDecimal bigTimeStep = new BigDecimal(String.valueOf(timeStep)); + + //System.err.println("=> goto " + time); + + if(!iter.gotoTime(time)) { + //System.err.println("=> no sample found at " + time); + return result; + } + + //time = iter.getValueBand().getTimeDouble(); + + boolean ignore = Math.abs(time-from) < 1e-6; + + do { + + //System.err.println("process " + time + " " + iter.getValueBand()); + + + + // Check for valid value + if ( iter.hasValidValue() ) { + + // Write time + if(!ignore) { + //System.err.println("Add time : " + time); + result.add(time); + } + // Write value + Object value = iter.getValueBand().getValue(); + //System.err.print("Add value : " + value); + if (value instanceof Number) { + if (value instanceof Float || value instanceof Double) { + switch (numberInterpolation) { + case PREVIOUS_SAMPLE: + + if(!ignore) { + // System.err.println(" previous .. done!"); + result.add(((Number) value).doubleValue()); + } + + break; + + case LINEAR_INTERPOLATION: + if (time != iter.getValueBand().getTimeDouble() && iter.hasNext()) { + + // Interpolate + int currentIndex = iter.getIndex(); + ValueBand band = iter.getValueBand(); + //double t1 = band.getTimeDouble(); + Number v1 = (Number) value; + double t12 = band.getEndTimeDouble(); + iter.next(); + double t2 = iter.getValueBand().getTimeDouble(); + Number v2 = (Number) iter.getValueBand().getValue(); + iter.gotoIndex(currentIndex); + + double vs = v1.doubleValue(); + if(time > t12) + vs = HistoryExportUtil.biglerp(t12, v1.doubleValue(), t2, v2.doubleValue(), time); + + if(!ignore) { + //System.err.println(" linear .. done!"); + result.add(vs); + } + + } else { + // Exact timestamp match, or last sample. + // Don't interpolate nor extrapolate. + if(!ignore) { + //System.err.println(" else .. done!"); + result.add(((Number) value).doubleValue()); + } + } + break; + default: + throw new UnsupportedOperationException("Unsupported interpolation: " + numberInterpolation); + } + } else { + throw new IllegalStateException("Value is not a number " + value); + } + } else if (value instanceof Boolean) { + if(!ignore) + result.add( (Boolean)value ? 1.0: 0.0); + } else { + throw new IllegalStateException("Value is not a number " + value); + } + } + + ignore = false; + + // Read next values, and the following times + if ( timeStep>0.0 ) { + bigTime = bigTime.add(bigTimeStep); + time = bigTime.doubleValue(); + } else { + // Get smallest end time that is larger than current time + Double nextTime = null; + // System.out.println("time = "+time); + if(!iter.hasNext()) break; + Double itemNextTime = iter.getNextTime( time ); + // System.err.println(" "+i.label+" nextTime="+itemNextTime); + if ( nextTime == null || ( nextTime > itemNextTime && !itemNextTime.equals( time ) ) ) nextTime = itemNextTime; + if ( nextTime == null || nextTime.equals( time ) ) break; + time = nextTime; + } + + boolean hasMore = false; + + iter.proceedToTime(time); + if(HistoryExportUtil.contains(iter, time)) hasMore = true; + + if(!hasMore) break; + + } while (time<=_end); + + //System.err.println("=> " + Arrays.toString(result.toArray())); + + return result; + + } + } diff --git a/bundles/org.simantics.history/src/org/simantics/history/HistorySamplerItem2.java b/bundles/org.simantics.history/src/org/simantics/history/HistorySamplerItem2.java new file mode 100644 index 000000000..73d89fa56 --- /dev/null +++ b/bundles/org.simantics.history/src/org/simantics/history/HistorySamplerItem2.java @@ -0,0 +1,109 @@ +package org.simantics.history; + +import java.util.Arrays; + +import org.simantics.databoard.accessor.StreamAccessor; +import org.simantics.databoard.accessor.error.AccessorException; +import org.simantics.history.util.StreamIterator; + +public class HistorySamplerItem2 implements Comparable { + + public static class LevelItem implements Comparable { + public final String id; + public final double samplingInterval; + + public LevelItem(String id, double samplingInterval) { + this.id = id; + this.samplingInterval = samplingInterval; + } + + @Override + public int compareTo(LevelItem o) { + int i = id.compareTo(o.id); + return i != 0 ? i : Double.compare(samplingInterval, o.samplingInterval); + } + } + + Collector collector; + HistoryManager history; // History source for this item + LevelItem[] items; + + // State data + StreamAccessor accessor; // Stream accessor + public StreamIterator iter; + public int chartDataId; + + public HistorySamplerItem2(Collector collector, HistoryManager history, LevelItem[] items, int identityHashCode) { + if (items.length == 0) + throw new IllegalArgumentException("Must have at least one existing history item to sample, zero provided"); + this.collector = collector; + this.history = history; + this.items = items; + this.chartDataId = identityHashCode; + } + + public void open() throws HistoryException { + accessor = history.openStream(items[0].id, "r"); + iter = new StreamIterator( accessor ); + } + + public void open(double pixelsPerSecond) throws HistoryException { + LevelItem f = getFormat(pixelsPerSecond); + accessor = history.openStream(f.id, "r"); + iter = new StreamIterator( accessor ); + } + + public void close() { + if (accessor!=null) { + try { + accessor.close(); + } catch (AccessorException e) { + } + } + accessor = null; + iter = null; + } + + @Override + public int compareTo(HistorySamplerItem2 o) { + int m = Math.min(items.length, o.items.length); + for (int j = 0; j < m; ++j) { + int i = items[j].compareTo(o.items[j]); + if (i != 0) + return i; + } + return 0; + } + + @Override + public int hashCode() { + int code = 0x2304; + code = 13*code + Arrays.hashCode(items); + code = 13*code + history.hashCode(); + return code; + } + + @Override + public boolean equals(Object obj) { + if ( obj == null ) return false; + if ( obj instanceof HistorySamplerItem2 == false ) return false; + HistorySamplerItem2 other = (HistorySamplerItem2) obj; + if ( !other.history.equals(history) ) return false; + if ( !Arrays.equals(other.items, items) ) return false; + return true; + } + + private LevelItem getFormat(double pixelsPerSecond) throws HistoryException { + LevelItem result = null; + for (LevelItem format : items) { + double interval = format.samplingInterval; + if (Double.isNaN( interval ) || interval <= pixelsPerSecond) { + result = format; + } else { + break; + } + } + return result; + } + +} diff --git a/bundles/org.simantics.history/src/org/simantics/history/util/subscription/SamplingFormat.java b/bundles/org.simantics.history/src/org/simantics/history/util/subscription/SamplingFormat.java index 2db7ed3a6..1334cae53 100644 --- a/bundles/org.simantics.history/src/org/simantics/history/util/subscription/SamplingFormat.java +++ b/bundles/org.simantics.history/src/org/simantics/history/util/subscription/SamplingFormat.java @@ -38,19 +38,23 @@ import org.simantics.databoard.util.Range; public class SamplingFormat extends Bean { public static final SamplingFormat[] EMPTY = new SamplingFormat[0]; - + + public static int compareSamplingInterval(double i1, double i2) { + boolean nan1 = Double.isNaN( i1 ); + boolean nan2 = Double.isNaN( i2 ); + + if ( nan1 && nan2 ) return 0; + if ( nan1 && !nan2) return -1; + if ( !nan1 && nan2) return 1; + return i1 == i2 ? 0 : ( i1 < i2 ? -1 : 1 ); + } + public final static Comparator INTERVAL_COMPARATOR = new Comparator() { @Override public int compare(Bean o1, Bean o2) { double i1 = (Double) o1.getFieldUnchecked("interval"); double i2 = (Double) o2.getFieldUnchecked("interval"); - boolean nan1 = Double.isNaN( i1 ); - boolean nan2 = Double.isNaN( i2 ); - - if ( nan1 && nan2 ) return 0; - if ( nan1 && !nan2) return -1; - if ( !nan1 && nan2) return 1; - return i1 == i2 ? 0 : ( i1 < i2 ? -1 : 1 ); + return compareSamplingInterval(i1, i2); } }; -- 2.43.2