From: Marko Luukkainen Date: Fri, 18 Feb 2022 16:10:30 +0000 (+0200) Subject: Progress Monitor support X-Git-Url: https://gerrit.simantics.org/r/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F94%2F4894%2F1;p=simantics%2Finterop.git Progress Monitor support For GraphComparator and ModelUpdate. Added iteration limit for GraphComparator to avoid long read transactions (disabled by default). gitlab #35 Change-Id: If39e74a1c8ac95d288d5100602c9b1f79c28cbf6 --- diff --git a/org.simantics.interop.update/src/org/simantics/interop/update/model/ModelUpdate.java b/org.simantics.interop.update/src/org/simantics/interop/update/model/ModelUpdate.java index f3fbdda..32da62b 100644 --- a/org.simantics.interop.update/src/org/simantics/interop/update/model/ModelUpdate.java +++ b/org.simantics.interop.update/src/org/simantics/interop/update/model/ModelUpdate.java @@ -7,6 +7,8 @@ import java.util.Deque; import java.util.List; import java.util.Map.Entry; +import org.eclipse.core.runtime.IProgressMonitor; +import org.eclipse.core.runtime.NullProgressMonitor; import org.simantics.Simantics; import org.simantics.db.ReadGraph; import org.simantics.db.Resource; @@ -60,9 +62,16 @@ public abstract class ModelUpdate { * @throws DatabaseException */ public void setInput(Resource oldModel, Resource newModel, Resource originalModel, boolean newDistinct) throws DatabaseException{ + this.setInput(oldModel, newModel, originalModel, newDistinct, null); + } + public void setInput(Resource oldModel, Resource newModel, Resource originalModel, boolean newDistinct, IProgressMonitor monitor) throws DatabaseException{ + this.oldModel = oldModel; this.newModel = newModel; this.originalModel = originalModel; + + if (monitor == null) + monitor = new NullProgressMonitor(); // addFilters(filters); if (originalModel != null) { // tree way comparison @@ -71,10 +80,12 @@ public abstract class ModelUpdate { GraphComparator comparator2 = result2.first; if (result2.second != null) showWarning(result2.second); - comparator2.test(getSession()); + comparator2.test(getSession(), monitor); changes2 = comparator2.getChanges(); + if (monitor.isCanceled()) + return; changes2 = getSession().syncRequest(createFilterRead(changes2, filters)); - Pair chg2 = createChangeObjects(changes2); + Pair chg2 = createChangeObjects(changes2, null); updateTree2 = chg2.first; updateList2 = chg2.second; @@ -83,10 +94,12 @@ public abstract class ModelUpdate { GraphComparator comparator3 = result3.first; if (result3.second != null) showWarning(result3.second); - comparator3.test(getSession()); + comparator3.test(getSession(), monitor); changes3 = comparator3.getChanges(); changes3 = getSession().syncRequest(createFilterRead(changes3, filters)); } + if (monitor.isCanceled()) + return; Pair result = getChanges(oldModel,newModel); GraphComparator comparator = result.first; @@ -125,10 +138,14 @@ public abstract class ModelUpdate { } } } - comparator.test(getSession()); + if (monitor.isCanceled()) + return; + comparator.test(getSession(), monitor); + monitor.setTaskName("Processing changes..."); + monitor.subTask(""); changes = comparator.getChanges(); changes = getSession().syncRequest(createFilterRead(changes, filters)); - Pair chg = createChangeObjects(changes); + Pair chg = createChangeObjects(changes, monitor); updateTree = chg.first; updateList = chg.second; if (userFilters.size() != 0) { @@ -139,7 +156,6 @@ public abstract class ModelUpdate { if (originalModel != null) { defaultSelections(); } - init = true; } @@ -253,9 +269,15 @@ public abstract class ModelUpdate { protected abstract Pair getChanges(Resource r1, Resource r2) throws DatabaseException; - protected Pair createChangeObjects(GraphChanges changes) throws DatabaseException{ - UpdateTree updateTree = getUpdateTree(changes); + protected Pair createChangeObjects(GraphChanges changes, IProgressMonitor monitor) throws DatabaseException{ + if (monitor != null) + monitor.subTask("Processing structural changes"); + UpdateTree updateTree = getUpdateTree(changes); + if (monitor != null) + monitor.subTask("Processing property changes"); UpdateList updateList = getUpdateList(changes); + if (monitor != null) + monitor.subTask("Postprocessing changes"); postProcess(updateTree, updateList); return new Pair(updateTree, updateList); } @@ -312,7 +334,7 @@ public abstract class ModelUpdate { public UpdateTree getUpdateTree3() throws DatabaseException{ if (updateTree3 == null && changes3 != null) { - Pair chg3 = createChangeObjects(changes3); + Pair chg3 = createChangeObjects(changes3, null); updateTree3 = chg3.first; updateList3 = chg3.second; } @@ -320,7 +342,7 @@ public abstract class ModelUpdate { } public UpdateList getUpdateList3() throws DatabaseException { if (updateList3 == null && changes3 != null) { - Pair chg3 = createChangeObjects(changes3); + Pair chg3 = createChangeObjects(changes3, null); updateTree3 = chg3.first; updateList3 = chg3.second; } diff --git a/org.simantics.interop/src/org/simantics/interop/test/GraphComparator.java b/org.simantics.interop/src/org/simantics/interop/test/GraphComparator.java index 52a8bab..9ad2061 100644 --- a/org.simantics.interop/src/org/simantics/interop/test/GraphComparator.java +++ b/org.simantics.interop/src/org/simantics/interop/test/GraphComparator.java @@ -26,6 +26,8 @@ import java.util.Map.Entry; import java.util.Set; import java.util.Stack; +import org.eclipse.core.runtime.IProgressMonitor; +import org.eclipse.core.runtime.NullProgressMonitor; import org.simantics.databoard.Bindings; import org.simantics.db.ReadGraph; import org.simantics.db.Resource; @@ -85,6 +87,9 @@ public class GraphComparator { private Set nonMatchedLeft = new HashSet(); private Set nonMatchedRight = new HashSet(); + private long iter = 0; + private long maxIter = -1; + // runtime attributes private ReadGraph g; @@ -184,12 +189,46 @@ public class GraphComparator { nonMatchedRight.add(r); } + public long getMaxIter() { + return maxIter; + } + + /** + * Sets maximum iteration done on a single DB transaction. Affects only test(Session) methods. + * Default value is -1, which disables iteration limit. + * Note that using iteration limit causes comparison to repeat some work, thus total execution time will increase. + * + * @param maxIter + */ + public void setMaxIter(long maxIter) { + this.maxIter = maxIter; + } + + public void clear() { + changes1.clear(); + changes1Set.clear(); + changes2.clear(); + changes2Set.clear(); + comparableResources.clear(); + comparableStatements.clear(); + modifications.clear(); + modificationsSet.clear(); + processedResources.clear(); + } + public void test(ReadGraph g) throws DatabaseException { + test(g,null); + } + + public void test(ReadGraph g, IProgressMonitor monitor) throws DatabaseException { this.g = g; this.b = Layer0.getInstance(g); comparator.setComparator(this); comparator.initialize(g, r1, r2); + if (monitor == null) + monitor = new NullProgressMonitor(); + Stack objectsLeft = new Stack(); Stack objectsRight = new Stack(); objectsLeft.push(r1); @@ -202,22 +241,16 @@ public class GraphComparator { while (true) { if (objectsLeft.isEmpty() && !changed) break; + if (monitor.isCanceled()) { + clear(); + return; + } + monitor.subTask(monitorTaskName(objectsLeft, objectsRight, unreliableLeft, unreliableRight)); changed = false; // process compares objects that are identified and searches for more resources to process. - changed |= process(objectsLeft, objectsRight, unreliableLeft, unreliableRight); - // process unreliable handles cases where unidentified statements subject and object have been identified - changed |= processUnreliable(unreliableLeft, unreliableRight); - // process unreliable handles cases where unidentified resources have path of length one to identified resource - changed |= processUnreliable(unreliableLeft, unreliableRight,objectsLeft,objectsRight); - if (objectsLeft.isEmpty() && unreliableLeft.size() > 0 && unreliableRight.size() > 0) { - changed |= processUnreliable2(unreliableLeft, unreliableRight,objectsLeft,objectsRight); - } - if (objectsLeft.isEmpty() && unreliableLeft.size() > 0 && unreliableRight.size() > 0) { - // comparison is ending, but we have still unprocessed unidentified resources left. - // These cases have longer path than one to identified objects. - changed |= processUnreliableDeep(unreliableLeft, unreliableRight, objectsLeft, objectsRight); - } + changed = compareIter(g, objectsLeft, objectsRight, unreliableLeft, unreliableRight); + monitor.worked(1); } for (Statement s : unreliableLeft) { @@ -232,16 +265,29 @@ public class GraphComparator { addAddition(s); } } + monitor.subTask(monitorTaskName(objectsLeft, objectsRight, unreliableLeft, unreliableRight) + " done."); + //printStats(); } public void test(Session session) throws DatabaseException { - test(session, r1, r2); + test(session, r1, r2, null); + } + + public void test(Session session, IProgressMonitor monitor) throws DatabaseException { + test(session, r1, r2, monitor); } public void test(Session session, Resource r1, Resource r2) throws DatabaseException { + test(session, r1, r2, null); + } + + public void test(Session session, Resource r1, Resource r2, IProgressMonitor monitor) throws DatabaseException { comparator.setComparator(this); + if (monitor == null) + monitor = new NullProgressMonitor(); + session.syncRequest(new ReadRequest() { @Override @@ -264,35 +310,22 @@ public class GraphComparator { while (true) { if (objectsLeft.isEmpty() && !changed) break; + if (monitor.isCanceled()) { + clear(); + return; + } + monitor.subTask(monitorTaskName(objectsLeft, objectsRight, unreliableLeft, unreliableRight)); changed = session.syncRequest(new Read() { @Override public Boolean perform(ReadGraph graph) throws DatabaseException { + //System.out.println("Iter " + monitorTaskName(objectsLeft, objectsRight, unreliableLeft, unreliableRight)); g = graph; b = Layer0.getInstance(graph); - // process compares objects that are identified and searches for more resources to process. - boolean c = process(objectsLeft, objectsRight, unreliableLeft, unreliableRight); - // process unreliable handles cases where unidentified statements subject and object have been identified - c |= processUnreliable(unreliableLeft, unreliableRight); - // process unreliable handles cases where unidentified resources have path of length one to identified resource - c |= processUnreliable(unreliableLeft, unreliableRight,objectsLeft,objectsRight); - if (objectsLeft.isEmpty() && unreliableLeft.size() > 0 && unreliableRight.size() > 0) { - c |= processUnreliable2(unreliableLeft, unreliableRight,objectsLeft,objectsRight); - } - if (objectsLeft.isEmpty() && unreliableLeft.size() > 0 && unreliableRight.size() > 0) { - // comparison is ending, but we have still unprocessed unidentified resources left. - // These cases have longer path than one to identified objects. - c |= processUnreliableDeep(unreliableLeft, unreliableRight, objectsLeft, objectsRight); - } - if (objectsLeft.isEmpty() && unreliableLeft.size() > 0 && unreliableRight.size() > 0) { - // comparison is ending, but we have still unprocessed unidentified resources left. - // These cases have longer path than one to identified objects. - c |= processUnreliableDeep(unreliableLeft, unreliableRight, objectsLeft, objectsRight); - } - return c; + return compareIter(graph, objectsLeft, objectsRight, unreliableLeft, unreliableRight); } }); - + monitor.worked(1); } @@ -304,15 +337,53 @@ public class GraphComparator { if (!comparableStatements.containsRight(s)) addAddition(s); } + unreliableLeft.clear(); + unreliableRight.clear(); - + monitor.subTask(monitorTaskName(objectsLeft, objectsRight, unreliableLeft, unreliableRight) + " done."); + //printStats(); } + private boolean compareIter(ReadGraph graph, Stack objectsLeft, Stack objectsRight, Set unreliableLeft, Set unreliableRight) throws DatabaseException { + // process compares objects that are identified and searches for more resources to process. + iter = 0; + boolean c = process(objectsLeft, objectsRight, unreliableLeft, unreliableRight); + if (objectsLeft.isEmpty()) { + // process unreliable handles cases where unidentified statements subject and object have been identified + c |= processUnreliable(unreliableLeft, unreliableRight); + // process unreliable handles cases where unidentified resources have path of length one to identified resource + if (!c) { + c |= processUnreliable(unreliableLeft, unreliableRight,objectsLeft,objectsRight); + if (!c && objectsLeft.isEmpty() && unreliableLeft.size() > 0 && unreliableRight.size() > 0) { + c |= processUnreliable2(unreliableLeft, unreliableRight,objectsLeft,objectsRight); + } + if (!c && objectsLeft.isEmpty() && unreliableLeft.size() > 0 && unreliableRight.size() > 0) { + // comparison is ending, but we have still unprocessed unidentified resources left. + // These cases have longer path than one to identified objects. + c |= processUnreliableDeep(unreliableLeft, unreliableRight, objectsLeft, objectsRight); + } + if (!c && objectsLeft.isEmpty() && unreliableLeft.size() > 0 && unreliableRight.size() > 0) { + // comparison is ending, but we have still unprocessed unidentified resources left. + // These cases have longer path than one to identified objects. + c |= processUnreliableDeep(unreliableLeft, unreliableRight, objectsLeft, objectsRight); + } + } + } + return c; + + } + + private String monitorTaskName(Stack objectsLeft, Stack objectsRight, Set unreliableLeft, Set unreliableRight) { + int cr = comparableResources.size(); + int ch = Math.max(changes1.size(), changes2.size()); + return "Graph compare " + (cr + ch) + " / " + (cr+ch+(Math.max(objectsLeft.size(), objectsRight.size())+Math.max(unreliableLeft.size(), unreliableRight.size()))); + } + private boolean process(Stack objectsLeft, Stack objectsRight, Set unreliableLeft, Set unreliableRight) throws DatabaseException { List ss1 = new ArrayList(); List ss2 = new ArrayList(); boolean didSomething = false; - while (!objectsLeft.isEmpty()) { + while (!objectsLeft.isEmpty()&& (maxIter < 0 || iter < maxIter)) { Resource r1 = objectsLeft.pop(); Resource r2 = objectsRight.pop(); @@ -360,6 +431,7 @@ public class GraphComparator { ss2.clear(); } + iter++; } return didSomething; } @@ -380,6 +452,8 @@ public class GraphComparator { } for (Resource left : subjectLeft.getKeys()) { + if (maxIter > 0 && iter > maxIter) + break; Resource right = comparableResources.getRight(left); if (right == null) continue; @@ -401,6 +475,7 @@ public class GraphComparator { unreliableRight.remove(rightS); addComparable(leftS, rightS); didSomething = true; + iter++; } } } @@ -422,8 +497,9 @@ public class GraphComparator { subjectRight.add(s.getSubject(),s); objectRight.add(s.getObject(),s); } - for (Resource ol : objectLeft.getKeys()) { + if (maxIter > 0 && iter > maxIter) + break; // all statements to the left side object List left = objectLeft.getValues(ol); // all subjects that have statements to the left side object (ol) @@ -581,6 +657,7 @@ public class GraphComparator { unreliableLeft.remove(sl); unreliableRight.remove(sr); } + iter++; } else { @@ -606,8 +683,9 @@ public class GraphComparator { subjectRight.add(s.getSubject(),s); objectRight.add(s.getObject(),s); } - for (Resource ol : objectLeft.getKeys()) { + if (maxIter > 0 && iter > maxIter) + break; // all statements to the left side object List left = objectLeft.getValues(ol); // all subjects that have statements to the left side object (ol) @@ -630,6 +708,7 @@ public class GraphComparator { for (Entry entry : comparableStatements.getEntries()) { unreliableLeft.remove(entry.getKey()); unreliableRight.remove(entry.getValue()); + iter++; } } } @@ -652,6 +731,8 @@ public class GraphComparator { objectRight.add(s.getObject(),s); } for (Resource ol : objectLeft.getKeys()) { + if (maxIter > 0 && iter > maxIter) + break; Set pathsLeft = new HashSet(); for (Resource rel : traversed) { pathsLeft.addAll(Path.create(g.getStatements(ol, rel))); @@ -706,8 +787,10 @@ public class GraphComparator { Path right = map.getRight(left); for (int i = 0; i < left.getLength(); i++) { addComparable(left.getStatements().get(i),right.getStatements().get(i)); + iter++; } } + iter++; } } } @@ -836,11 +919,18 @@ public class GraphComparator { } else { if (DEBUG) System.out.println(left + " = " + right); comparableResources.map(left, right); + +// if (comparableResources.size() % 1000 == 0) +// printStats(); } } } + public void printStats() { + System.out.println("Comp " + comparableResources.size() + " L " + changes1.size() + " R " + changes2.size() + " M " + modifications.size() + " P " + processedResources.size()); + } + public List filterAsserted(Resource r, Collection in) throws DatabaseException { List out = new ArrayList(); for (Statement s : in) {