From: Marko Luukkainen Date: Fri, 25 Feb 2022 16:22:04 +0000 (+0200) Subject: Process changes in smaller chunks X-Git-Url: https://gerrit.simantics.org/r/gitweb?a=commitdiff_plain;h=88fcf43b9eb2e217b50bf67cee58edaef4637a59;p=simantics%2Finterop.git Process changes in smaller chunks UpdateTree and UpdateOperation processing in chunks. Applying changes in chunks. Fixed bookkeeping problems in one of GraphComparator subroutines, which processed same statements several times, and could cause conflicting results. gitlab #39 Change-Id: Ic897802ddd8c2906e48a0e7d9ba6d1ec35ced13d --- diff --git a/org.simantics.interop.update/src/org/simantics/interop/update/model/AddDeleteUpdateOp.java b/org.simantics.interop.update/src/org/simantics/interop/update/model/AddDeleteUpdateOp.java index a505864..0ddbcb7 100644 --- a/org.simantics.interop.update/src/org/simantics/interop/update/model/AddDeleteUpdateOp.java +++ b/org.simantics.interop.update/src/org/simantics/interop/update/model/AddDeleteUpdateOp.java @@ -67,7 +67,7 @@ public abstract class AddDeleteUpdateOp extends UpdateOp { continue; if (g.isInstanceOf(s.getObject(), l0.Literal)) { Object value = g.getPossibleValue(s.getObject()); - System.out.println(NameUtils.getSafeName(g, s.getSubject()) + " " + NameUtils.getSafeName(g, s.getPredicate()) + " " + NameUtils.getSafeName(g, s.getObject()) + " " + value); + //System.out.println(NameUtils.getSafeName(g, s.getSubject()) + " " + NameUtils.getSafeName(g, s.getPredicate()) + " " + NameUtils.getSafeName(g, s.getObject()) + " " + value); if (value != null) { Statement valueStm = g.getPossibleStatement(destination, s.getPredicate()); Resource valueResource = null; diff --git a/org.simantics.interop.update/src/org/simantics/interop/update/model/ModelUpdate.java b/org.simantics.interop.update/src/org/simantics/interop/update/model/ModelUpdate.java index 6e51757..17eee72 100644 --- a/org.simantics.interop.update/src/org/simantics/interop/update/model/ModelUpdate.java +++ b/org.simantics.interop.update/src/org/simantics/interop/update/model/ModelUpdate.java @@ -14,8 +14,10 @@ import org.simantics.db.ReadGraph; import org.simantics.db.Resource; import org.simantics.db.Session; import org.simantics.db.Statement; +import org.simantics.db.VirtualGraph; import org.simantics.db.WriteGraph; import org.simantics.db.common.request.ReadRequest; +import org.simantics.db.common.request.WriteRequest; import org.simantics.db.exception.DatabaseException; import org.simantics.db.layer0.util.Layer0Utils; import org.simantics.db.request.Read; @@ -48,6 +50,7 @@ public abstract class ModelUpdate { private List userFilters = new ArrayList(); boolean init = false; + private int chunkSize = -1; public void setInput(Resource oldModel, Resource newModel) throws DatabaseException { setInput(oldModel, newModel, null, false); @@ -82,6 +85,7 @@ public abstract class ModelUpdate { showWarning(result2.second); comparator2.test(getSession(), monitor); changes2 = comparator2.getChanges(); + comparator2.dispose(); if (monitor.isCanceled()) { dispose(); return; @@ -98,6 +102,7 @@ public abstract class ModelUpdate { showWarning(result3.second); comparator3.test(getSession(), monitor); changes3 = comparator3.getChanges(); + comparator2.dispose(); changes3 = getSession().syncRequest(createFilterRead(changes3, filters)); } if (monitor.isCanceled()) { @@ -150,6 +155,7 @@ public abstract class ModelUpdate { monitor.setTaskName("Processing changes..."); monitor.subTask(""); changes = comparator.getChanges(); + comparator.dispose(); changes = getSession().syncRequest(createFilterRead(changes, filters)); Pair chg = createChangeObjects(changes, monitor); if (chg == null) { @@ -231,6 +237,14 @@ public abstract class ModelUpdate { return userFilters; } + public int getChunkSize() { + return chunkSize; + } + + public void setChunkSize(int chunkSize) { + this.chunkSize = chunkSize; + } + public void refreshUserFilters() throws DatabaseException{ // use user filters to set visible flags of changes. // First, set all changes visible. @@ -309,13 +323,13 @@ public abstract class ModelUpdate { return null; monitor.subTask("Processing structural changes"); } - UpdateTree updateTree = getUpdateTree(changes); + UpdateTree updateTree = getUpdateTree(changes, monitor); if (monitor != null) { if (monitor.isCanceled()) return null; monitor.subTask("Processing property changes"); } - UpdateList updateList = getUpdateList(changes); + UpdateList updateList = getUpdateList(changes, monitor); if (monitor != null) { if (monitor.isCanceled()) return null; @@ -325,8 +339,8 @@ public abstract class ModelUpdate { return new Pair(updateTree, updateList); } - protected abstract UpdateTree getUpdateTree(GraphChanges changes) throws DatabaseException; - protected UpdateList getUpdateList(GraphChanges changes) throws DatabaseException { + protected abstract UpdateTree getUpdateTree(GraphChanges changes, IProgressMonitor monitor) throws DatabaseException; + protected UpdateList getUpdateList(GraphChanges changes, IProgressMonitor monitor) throws DatabaseException { return new UpdateList(changes, changes.getModifications()); } @@ -399,30 +413,91 @@ public abstract class ModelUpdate { return updateNode3; } - public void applyAll(WriteGraph graph) throws DatabaseException { - Layer0Utils.addCommentMetadata(graph, "Apply all model updates"); - graph.markUndoPoint(); - for (PropertyChange mod : updateList.getChanges()) { - mod.apply(graph); - } + List list = updateList.getChanges(); + apply(graph, list, "Apply all model updates"); updateTree.getUpdateOps().applyAll(graph); } public void applySelected(WriteGraph graph) throws DatabaseException { - Layer0Utils.addCommentMetadata(graph, "Apply selected model updates"); - graph.markUndoPoint(); + List list = new ArrayList(); + for (PropertyChange mod : updateList.getChanges()) { + if (mod.selected()) + list.add(mod); + } + apply(graph, list, "Apply selected model updates"); updateTree.getUpdateOps().applySelected(graph); + } + + public void applyAll(Session session, VirtualGraph vg) throws DatabaseException { + List list = updateList.getChanges(); + apply(session, list, "Apply all model updates", vg); + updateTree.getUpdateOps().applyAll(session, vg); + } + + public void applySelected(Session session, VirtualGraph vg) throws DatabaseException { + List list = new ArrayList(); for (PropertyChange mod : updateList.getChanges()) { if (mod.selected()) - mod.apply(graph); + list.add(mod); } + apply(session, list, "Apply selected model updates", vg); + + updateTree.getUpdateOps().applySelected(session, vg); + } + protected void apply(WriteGraph graph, List list, String message) throws DatabaseException { + Layer0Utils.addCommentMetadata(graph, message); + graph.markUndoPoint(); + for (PropertyChange mod : list) { + mod.apply(graph); + } + } + protected void apply(Session session, List list, String message, VirtualGraph vg) throws DatabaseException { + if (getChunkSize() > 0) { + for (int s = 0; s < list.size(); ) { + int e = s + getChunkSize(); + if (e > list.size()) + e = list.size(); + List subList = updateList.getChanges().subList(e, e); + if (s == 0) { + session.syncRequest(new WriteRequest(vg) { + public void perform(WriteGraph graph) throws DatabaseException { + Layer0Utils.addCommentMetadata(graph, message); + graph.markUndoPoint(); + } + }); + } + session.syncRequest(new WriteRequest(vg) { + + @Override + public void perform(WriteGraph graph) throws DatabaseException { + for (PropertyChange mod : subList) { + mod.apply(graph); + } + } + }); + s = e; + } + } else { + session.syncRequest(new WriteRequest(vg) { + + @Override + public void perform(WriteGraph graph) throws DatabaseException { + Layer0Utils.addCommentMetadata(graph, message); + graph.markUndoPoint(); + for (PropertyChange mod : list) { + mod.apply(graph); + } + } + }); + } + } protected Session getSession() { diff --git a/org.simantics.interop.update/src/org/simantics/interop/update/model/UpdateList.java b/org.simantics.interop.update/src/org/simantics/interop/update/model/UpdateList.java index 2267ebe..8679a3b 100644 --- a/org.simantics.interop.update/src/org/simantics/interop/update/model/UpdateList.java +++ b/org.simantics.interop.update/src/org/simantics/interop/update/model/UpdateList.java @@ -35,7 +35,7 @@ public class UpdateList { return new PropertyChange(changes, left,right, change); } - public Collection getChanges() { + public List getChanges() { return changes; } diff --git a/org.simantics.interop.update/src/org/simantics/interop/update/model/UpdateNode.java b/org.simantics.interop.update/src/org/simantics/interop/update/model/UpdateNode.java index 54d6b2c..f68f887 100644 --- a/org.simantics.interop.update/src/org/simantics/interop/update/model/UpdateNode.java +++ b/org.simantics.interop.update/src/org/simantics/interop/update/model/UpdateNode.java @@ -1,7 +1,9 @@ package org.simantics.interop.update.model; import java.util.ArrayList; -import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; import org.eclipse.jface.resource.ImageDescriptor; import org.simantics.db.ReadGraph; @@ -20,7 +22,7 @@ public class UpdateNode { private boolean visible = true; - private Collection children = new ArrayList(); + private List children; /** * * @param resource old Resource if status is DELETED or EXISTS. @@ -69,11 +71,16 @@ public class UpdateNode { return status; } - public Collection getChildren() { + @SuppressWarnings("unchecked") + public List getChildren() { + if (children == null) + return Collections.EMPTY_LIST; return children; } public void addChild(UpdateNode node) { + if (children == null) + children = new ArrayList(2); children.add(node); node.parent = this; if (op != null && node.op != null) { @@ -83,6 +90,17 @@ public class UpdateNode { } } } + + public void sort() { + if (children == null) + return; + Collections.sort(this.children, new Comparator() { + @Override + public int compare(UpdateNode o1, UpdateNode o2) { + return o1.getLabel().compareTo(o2.getLabel()); + } + }); + } public ImageDescriptor getImage(ReadGraph graph) throws DatabaseException { return null; @@ -122,17 +140,19 @@ public class UpdateNode { if (parent != null && !parent.visible) parent.setVisible(true); } else { - for (UpdateNode n : children) - n.setVisible(false); + if (children != null) + for (UpdateNode n : children) + n.setVisible(false); } } public void setAllVisible(boolean visible) { - this.visible = visible; - if (op != null) - op.visible = visible; - for (UpdateNode n : children) - n.setAllVisible(visible); - } + this.visible = visible; + if (op != null) + op.visible = visible; + if (children != null) + for (UpdateNode n : children) + n.setAllVisible(visible); + } } diff --git a/org.simantics.interop.update/src/org/simantics/interop/update/model/UpdateOperations.java b/org.simantics.interop.update/src/org/simantics/interop/update/model/UpdateOperations.java index 825ccf1..c8f4f1d 100644 --- a/org.simantics.interop.update/src/org/simantics/interop/update/model/UpdateOperations.java +++ b/org.simantics.interop.update/src/org/simantics/interop/update/model/UpdateOperations.java @@ -6,12 +6,15 @@ import java.util.List; import java.util.Map; import java.util.Stack; +import org.eclipse.core.runtime.IProgressMonitor; import org.simantics.db.ReadGraph; import org.simantics.db.Resource; import org.simantics.db.Session; import org.simantics.db.Statement; +import org.simantics.db.VirtualGraph; import org.simantics.db.WriteGraph; import org.simantics.db.common.request.ReadRequest; +import org.simantics.db.common.request.WriteRequest; import org.simantics.db.exception.DatabaseException; import org.simantics.interop.test.GraphChanges; @@ -26,6 +29,7 @@ public abstract class UpdateOperations { private Map resourceMap = new HashMap(); private Map statementMap = new HashMap(); private GraphChanges changes; + private int chunkSize = -1; public UpdateOperations(GraphChanges changes) { this.changes = changes; @@ -39,16 +43,101 @@ public abstract class UpdateOperations { return statementMap.get(s); } - public void applyAll(WriteGraph g) throws DatabaseException { + public int getChunkSize() { + return chunkSize; + } + + public void setChunkSize(int chunkSize) { + this.chunkSize = chunkSize; + } + + /** + * Applies all changes. + * + * @param graph + * @throws DatabaseException + */ + public void applyAll(WriteGraph graph) throws DatabaseException { + List list = operations; + apply(graph, list); + } + + /** + * Applies selected changes. + * @param graph + * @throws DatabaseException + */ + public void applySelected(WriteGraph graph) throws DatabaseException { + List list = new ArrayList(); for (UpdateOp op : operations) { - apply(g, op); + if (op.selected()) + list.add(op); } + apply(graph, list); + } + + /** + * Applies all changes with chunked DB writes. + * + * @param session + * @throws DatabaseException + */ + public void applyAll(Session session, VirtualGraph vg) throws DatabaseException { + List list = operations; + apply(session, list, vg); } - public void applySelected(WriteGraph g) throws DatabaseException { + /** + * Applies selected changes with chunked DB writes. + * + * @param session + * @throws DatabaseException + */ + public void applySelected(Session session, VirtualGraph vg) throws DatabaseException { + List list = new ArrayList(); for (UpdateOp op : operations) { if (op.selected()) - apply(g, op); + list.add(op); + } + apply(session, list, vg); + } + + protected void apply(WriteGraph graph, List list) throws DatabaseException { + for (UpdateOp op : list) { + apply(graph, op); + } + } + + protected void apply(Session session, List list, VirtualGraph vg) throws DatabaseException { + if (getChunkSize() > 0) { + for (int s = 0; s < list.size(); ) { + int e = s + getChunkSize(); + if (e > list.size()) + e = list.size(); + List subList = list.subList(s, e); + session.syncRequest(new WriteRequest(vg) { + + @Override + public void perform(WriteGraph graph) throws DatabaseException { + + for (UpdateOp op : subList) { + apply(graph, op); + } + } + }); + s = e; + } + } else { + session.syncRequest(new WriteRequest(vg) { + + @Override + public void perform(WriteGraph graph) throws DatabaseException { + + for (UpdateOp op : list) { + apply(graph, op); + } + } + }); } } @@ -135,7 +224,7 @@ public abstract class UpdateOperations { * @param session * @throws DatabaseException */ - public void populate(Session session) throws DatabaseException { + public void populate(Session session, IProgressMonitor monitor) throws DatabaseException { session.syncRequest(new ReadRequest() { @Override diff --git a/org.simantics.interop.update/src/org/simantics/interop/update/model/UpdateTree.java b/org.simantics.interop.update/src/org/simantics/interop/update/model/UpdateTree.java index a558f84..bf7e068 100644 --- a/org.simantics.interop.update/src/org/simantics/interop/update/model/UpdateTree.java +++ b/org.simantics.interop.update/src/org/simantics/interop/update/model/UpdateTree.java @@ -3,6 +3,8 @@ package org.simantics.interop.update.model; import java.util.HashMap; import java.util.Map; +import org.eclipse.core.runtime.IProgressMonitor; +import org.eclipse.core.runtime.NullProgressMonitor; import org.simantics.db.ReadGraph; import org.simantics.db.Resource; import org.simantics.db.Session; @@ -29,17 +31,30 @@ public class UpdateTree { this.updateOps = updateOps; this.updateOps.populate(g); populate(g); + this.rootNode.sort(); } public UpdateTree(Session session, GraphChanges changes, UpdateOperations updateOps) throws DatabaseException { + this(session, changes, updateOps, new NullProgressMonitor()); + } + + public UpdateTree(Session session, GraphChanges changes, UpdateOperations updateOps, IProgressMonitor monitor) throws DatabaseException { this.changes = changes; this.nodes = new HashMap(); this.rootNode = session.syncRequest(new NodeRequest(UpdateStatus.EXIST, changes.getResource1())); nodes.put(changes.getResource1(), rootNode); nodes.put(changes.getResource2(), rootNode); this.updateOps = updateOps; - this.updateOps.populate(session); - populate(session); + this.updateOps.populate(session, monitor); + populate(session, monitor); + if (monitor.isCanceled()) { + this.changes = null; + this.nodes.clear(); + this.updateOps = null; + this.rootNode = null; + } else { + this.rootNode.sort(); + } } private class NodeRequest extends ResourceRead { @@ -176,9 +191,11 @@ public class UpdateTree { } } - protected void populate(Session session) throws DatabaseException{ + protected void populate(Session session, IProgressMonitor monitor) throws DatabaseException{ int i = 0; while (i < updateOps.getOperations().size()) { + if (monitor.isCanceled()) + return; i = session.syncRequest(new PopulateRead(i)); } } diff --git a/org.simantics.interop/src/org/simantics/interop/test/GraphChanges.java b/org.simantics.interop/src/org/simantics/interop/test/GraphChanges.java index 3fe7417..e52ad97 100644 --- a/org.simantics.interop/src/org/simantics/interop/test/GraphChanges.java +++ b/org.simantics.interop/src/org/simantics/interop/test/GraphChanges.java @@ -39,9 +39,6 @@ public class GraphChanges { this.leftStm = leftStm; this.rightStm = rightStm; - if (leftSub.getResourceId() < 0) - System.out.println(); - hashCode = leftSub.hashCode() + rightSub.hashCode(); if (leftStm != null) hashCode += leftStm.hashCode(); diff --git a/org.simantics.interop/src/org/simantics/interop/test/GraphComparator.java b/org.simantics.interop/src/org/simantics/interop/test/GraphComparator.java index d8750b2..7ea8f4d 100644 --- a/org.simantics.interop/src/org/simantics/interop/test/GraphComparator.java +++ b/org.simantics.interop/src/org/simantics/interop/test/GraphComparator.java @@ -95,6 +95,8 @@ public class GraphComparator { private ReadGraph g; private Layer0 b; + private boolean mapLiterals = true; + public GraphComparator(Resource r1, Resource r2) { this.r1 = r1; this.r2 = r2; @@ -204,7 +206,15 @@ public class GraphComparator { this.maxIter = maxIter; } - public void clear() { + public boolean isMapLiterals() { + return mapLiterals; + } + + public void setMapLiterals(boolean mapLiterals) { + this.mapLiterals = mapLiterals; + } + + private void clear() { changes1.clear(); changes1Set.clear(); changes2.clear(); @@ -216,6 +226,28 @@ public class GraphComparator { processedResources.clear(); } + public void dispose() { + changes1 = null; + changes1Set = null; + changes2 = null; + changes2Set = null; + comparableResources = null; + comparableStatements = null; + comparator = null; + modifications = null; + modificationsSet = null; + processedResources = null; + nonMatchedLeft = null; + nonMatchedRight = null; + nonTested = null; + nonTraversed = null; + ss1 = null; + ss2 = null; + strong = null; + tested = null; + traversed = null; + } + public void test(ReadGraph g) throws DatabaseException { test(g,null); } @@ -683,6 +715,7 @@ public class GraphComparator { subjectRight.add(s.getSubject(),s); objectRight.add(s.getObject(),s); } + Set> processed = new HashSet>(); for (Resource ol : objectLeft.getKeys()) { if (maxIter > 0 && iter > maxIter) break; @@ -698,12 +731,29 @@ public class GraphComparator { } if (sLeft.size() == 1 && sRight.size() == 1) { - List ss1 = new ArrayList(subjectLeft.getValues(sLeft.iterator().next())); - List ss2 = new ArrayList(subjectRight.getValues(sRight.iterator().next())); + Resource sl = sLeft.iterator().next(); + Resource sr = sRight.iterator().next(); + Pair p = new Pair(sl, sr); + if (processed.contains(p)) + continue; + processed.add(p); + List ss1 = new ArrayList(subjectLeft.getValuesSnapshot(sl)); + List ss2 = new ArrayList(subjectRight.getValuesSnapshot(sr)); + for (int i = ss1.size() -1; i >= 0; i--) { + if (!unreliableLeft.contains(ss1.get(i))) + ss1.remove(i); + } + for (int i = ss2.size() -1; i >= 0; i--) { + if (!unreliableRight.contains(ss2.get(i))) + ss2.remove(i); + } + + int ccount = comparableStatements.size(); + int lcount = changes1.size(); + int rcount = changes2.size(); - int count = comparableStatements.size(); compareStatements(ss1, ss2, objectsLeft, objectsRight,unreliableLeft,unreliableRight); - if (comparableStatements.size() > count) { + if (comparableStatements.size() > ccount) { didSomething = true; for (Entry entry : comparableStatements.getEntries()) { unreliableLeft.remove(entry.getKey()); @@ -711,6 +761,20 @@ public class GraphComparator { iter++; } } + if (changes1.size() > lcount) { + didSomething = true; + for (Statement stm : changes1) { + unreliableLeft.remove(stm); + iter++; + } + } + if (changes2.size() > rcount) { + didSomething = true; + for (Statement stm : changes2) { + unreliableRight.remove(stm); + iter++; + } + } } } return didSomething; @@ -1451,7 +1515,7 @@ public class GraphComparator { if (!eq) { addModification(r1,s1,r2,s2); } - if (!a1 && !a2) + if (mapLiterals && !a1 && !a2) addComparable(s1, s2); } else { // Non literal properties.