]> gerrit.simantics Code Review - simantics/interop.git/commitdiff
Process changes in smaller chunks 00/4900/1
authorMarko Luukkainen <marko.luukkainen@semantum.fi>
Fri, 25 Feb 2022 16:22:04 +0000 (18:22 +0200)
committerMarko Luukkainen <marko.luukkainen@semantum.fi>
Fri, 25 Feb 2022 16:22:04 +0000 (18:22 +0200)
UpdateTree and UpdateOperation processing in chunks.
Applying changes in chunks.
Fixed bookkeeping problems in one of GraphComparator subroutines, which
processed same statements several times, and could cause conflicting
results.

gitlab #39

Change-Id: Ic897802ddd8c2906e48a0e7d9ba6d1ec35ced13d

org.simantics.interop.update/src/org/simantics/interop/update/model/AddDeleteUpdateOp.java
org.simantics.interop.update/src/org/simantics/interop/update/model/ModelUpdate.java
org.simantics.interop.update/src/org/simantics/interop/update/model/UpdateList.java
org.simantics.interop.update/src/org/simantics/interop/update/model/UpdateNode.java
org.simantics.interop.update/src/org/simantics/interop/update/model/UpdateOperations.java
org.simantics.interop.update/src/org/simantics/interop/update/model/UpdateTree.java
org.simantics.interop/src/org/simantics/interop/test/GraphChanges.java
org.simantics.interop/src/org/simantics/interop/test/GraphComparator.java

index a5058647379f65c182f7a97272a1bd27e3ad1ac4..0ddbcb76c1c36e46ad4b91883bda75b1cdcda9f6 100644 (file)
@@ -67,7 +67,7 @@ public abstract class AddDeleteUpdateOp extends UpdateOp {
                                continue;
                        if (g.isInstanceOf(s.getObject(), l0.Literal)) {
                                Object value = g.getPossibleValue(s.getObject());
-                               System.out.println(NameUtils.getSafeName(g, s.getSubject()) + " " + NameUtils.getSafeName(g, s.getPredicate()) + " " + NameUtils.getSafeName(g, s.getObject()) + " " + value);
+                               //System.out.println(NameUtils.getSafeName(g, s.getSubject()) + " " + NameUtils.getSafeName(g, s.getPredicate()) + " " + NameUtils.getSafeName(g, s.getObject()) + " " + value);
                                if (value != null) {
                                        Statement valueStm = g.getPossibleStatement(destination, s.getPredicate());
                                        Resource valueResource = null;
index 6e5175707080e7f5587c7b9ea7d52b5ab4b43261..17eee72ee6a40861c6ebb716bbfa8199fe62c384 100644 (file)
@@ -14,8 +14,10 @@ import org.simantics.db.ReadGraph;
 import org.simantics.db.Resource;
 import org.simantics.db.Session;
 import org.simantics.db.Statement;
+import org.simantics.db.VirtualGraph;
 import org.simantics.db.WriteGraph;
 import org.simantics.db.common.request.ReadRequest;
+import org.simantics.db.common.request.WriteRequest;
 import org.simantics.db.exception.DatabaseException;
 import org.simantics.db.layer0.util.Layer0Utils;
 import org.simantics.db.request.Read;
@@ -48,6 +50,7 @@ public abstract class ModelUpdate {
        private List<ChangeFilter2> userFilters = new ArrayList<ChangeFilter2>();
        
        boolean init = false;
+       private int chunkSize = -1;
        
        public void setInput(Resource oldModel, Resource newModel) throws DatabaseException {
                setInput(oldModel, newModel, null, false);
@@ -82,6 +85,7 @@ public abstract class ModelUpdate {
                                showWarning(result2.second);
                        comparator2.test(getSession(), monitor);
                        changes2 = comparator2.getChanges();
+                       comparator2.dispose();
                        if (monitor.isCanceled()) {
                                dispose();
                                return;
@@ -98,6 +102,7 @@ public abstract class ModelUpdate {
                                showWarning(result3.second);
                        comparator3.test(getSession(), monitor);
                        changes3 = comparator3.getChanges();
+                       comparator2.dispose();
                        changes3 = getSession().syncRequest(createFilterRead(changes3, filters));
                }
                if (monitor.isCanceled()) {
@@ -150,6 +155,7 @@ public abstract class ModelUpdate {
                monitor.setTaskName("Processing changes...");
                monitor.subTask("");
                changes = comparator.getChanges();
+               comparator.dispose();
                changes = getSession().syncRequest(createFilterRead(changes, filters));
                Pair<UpdateTree, UpdateList> chg = createChangeObjects(changes, monitor);
                if (chg == null) {
@@ -231,6 +237,14 @@ public abstract class ModelUpdate {
                return userFilters;
        }
        
+       public int getChunkSize() {
+               return chunkSize;
+       }
+       
+       public void setChunkSize(int chunkSize) {
+               this.chunkSize = chunkSize;
+       }
+       
        public void refreshUserFilters() throws DatabaseException{
                // use user filters to set visible flags of changes.
                // First, set all changes visible.
@@ -309,13 +323,13 @@ public abstract class ModelUpdate {
                                return null;
                        monitor.subTask("Processing structural changes");
                }
-               UpdateTree updateTree = getUpdateTree(changes);
+               UpdateTree updateTree = getUpdateTree(changes, monitor);
                if (monitor != null) {
                        if (monitor.isCanceled())
                                return null;
                        monitor.subTask("Processing property changes");
                }
-        UpdateList updateList = getUpdateList(changes);
+        UpdateList updateList = getUpdateList(changes, monitor);
         if (monitor != null) {
                if (monitor.isCanceled())
                                return null;
@@ -325,8 +339,8 @@ public abstract class ModelUpdate {
         return new Pair<UpdateTree, UpdateList>(updateTree, updateList);
        }
        
-       protected abstract UpdateTree getUpdateTree(GraphChanges changes) throws DatabaseException;
-       protected UpdateList getUpdateList(GraphChanges changes) throws DatabaseException {
+       protected abstract UpdateTree getUpdateTree(GraphChanges changes, IProgressMonitor monitor) throws DatabaseException;
+       protected UpdateList getUpdateList(GraphChanges changes, IProgressMonitor monitor) throws DatabaseException {
                return new UpdateList(changes, changes.getModifications());
        }
        
@@ -399,30 +413,91 @@ public abstract class ModelUpdate {
            return updateNode3;
        }
 
-       
        public void applyAll(WriteGraph graph) throws DatabaseException {
-               Layer0Utils.addCommentMetadata(graph, "Apply all model updates");
-               graph.markUndoPoint();
-               for (PropertyChange mod : updateList.getChanges()) {
-                       mod.apply(graph);
-               }
+               List<PropertyChange> list = updateList.getChanges();
+               apply(graph, list, "Apply all model updates");
                
                updateTree.getUpdateOps().applyAll(graph);
        }
        
        public void applySelected(WriteGraph graph) throws DatabaseException {
-               Layer0Utils.addCommentMetadata(graph, "Apply selected model updates");
-               graph.markUndoPoint();
+               List<PropertyChange> list = new ArrayList<PropertyChange>();
+               for (PropertyChange mod : updateList.getChanges()) {
+                       if (mod.selected())
+                               list.add(mod);
+               }
+               apply(graph, list, "Apply selected model updates");
                
                updateTree.getUpdateOps().applySelected(graph);
+       }
+       
+       public void applyAll(Session session, VirtualGraph vg) throws DatabaseException {
+               List<PropertyChange> list = updateList.getChanges();
+               apply(session, list, "Apply all model updates", vg);
                
+               updateTree.getUpdateOps().applyAll(session, vg);
+       }
+       
+       public void applySelected(Session session, VirtualGraph vg) throws DatabaseException {
+               List<PropertyChange> list = new ArrayList<PropertyChange>();
                for (PropertyChange mod : updateList.getChanges()) {
                        if (mod.selected())
-                               mod.apply(graph);
+                               list.add(mod);
                }
+               apply(session, list, "Apply selected model updates", vg);
+               
+               updateTree.getUpdateOps().applySelected(session, vg);
+               
        }
        
+       protected void apply(WriteGraph graph, List<PropertyChange> list, String message) throws DatabaseException {
+               Layer0Utils.addCommentMetadata(graph, message);
+               graph.markUndoPoint();
+               for (PropertyChange mod : list) {
+                       mod.apply(graph);
+               }
+       }
        
+       protected void apply(Session session, List<PropertyChange> list, String message, VirtualGraph vg) throws DatabaseException {
+               if (getChunkSize() > 0) {
+                       for (int s = 0; s < list.size(); ) {
+                               int e = s + getChunkSize();
+                               if (e > list.size())
+                                       e = list.size();
+                               List<PropertyChange> subList = updateList.getChanges().subList(e, e);
+                               if (s == 0) {
+                                       session.syncRequest(new WriteRequest(vg) {
+                                               public void perform(WriteGraph graph) throws DatabaseException {
+                                                       Layer0Utils.addCommentMetadata(graph, message);
+                                                       graph.markUndoPoint();
+                                               }
+                                       });
+                               }
+                               session.syncRequest(new WriteRequest(vg) {
+                                       
+                                       @Override
+                                       public void perform(WriteGraph graph) throws DatabaseException {
+                                               for (PropertyChange mod : subList) {
+                                                       mod.apply(graph);
+                                               }
+                                       }
+                               });
+                               s = e;
+                       }
+               } else {
+                       session.syncRequest(new WriteRequest(vg) {
+                               
+                               @Override
+                               public void perform(WriteGraph graph) throws DatabaseException {
+                                       Layer0Utils.addCommentMetadata(graph, message);
+                                       graph.markUndoPoint();
+                                       for (PropertyChange mod : list) {
+                                               mod.apply(graph);
+                                       }
+                               }
+                       });
+               }
+       }
        
        
        protected Session getSession() {
index 2267ebec6f47e8ec101a39d1945e19d99aa1325a..8679a3b3a399bf5be50f3f660554515c4690358b 100644 (file)
@@ -35,7 +35,7 @@ public class UpdateList {
                return new PropertyChange(changes, left,right, change);
        }
        
-       public Collection<PropertyChange> getChanges() {
+       public List<PropertyChange> getChanges() {
                return changes;
        }
 
index 54d6b2ce670bc3f28d4e3fa75e73dc1adaf34522..f68f887dca33c98e925862c203dab27926ae5480 100644 (file)
@@ -1,7 +1,9 @@
 package org.simantics.interop.update.model;
 
 import java.util.ArrayList;
-import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
 
 import org.eclipse.jface.resource.ImageDescriptor;
 import org.simantics.db.ReadGraph;
@@ -20,7 +22,7 @@ public class UpdateNode {
        private boolean visible = true;
        
        
-       private Collection<UpdateNode> children = new ArrayList<UpdateNode>();
+       private List<UpdateNode> children;
        /**
         * 
         * @param resource old Resource if status is DELETED or EXISTS.
@@ -69,11 +71,16 @@ public class UpdateNode {
                return status;
        }
        
-       public Collection<UpdateNode> getChildren() {
+       @SuppressWarnings("unchecked")
+       public List<UpdateNode> getChildren() {
+               if (children == null)
+                       return Collections.EMPTY_LIST;
                return children;
        }
        
        public void addChild(UpdateNode node) {
+               if (children == null)
+                       children = new ArrayList<UpdateNode>(2);
                children.add(node);
                node.parent = this;
                if (op != null && node.op != null) {
@@ -83,6 +90,17 @@ public class UpdateNode {
                        }
                }
        }
+       
+       public void sort() {
+               if (children == null)
+                       return;
+               Collections.sort(this.children, new Comparator<UpdateNode>() {
+                       @Override
+                       public int compare(UpdateNode o1, UpdateNode o2) {
+                               return o1.getLabel().compareTo(o2.getLabel());
+                       }
+               });
+       }
 
        public ImageDescriptor getImage(ReadGraph graph) throws DatabaseException {
                return null;
@@ -122,17 +140,19 @@ public class UpdateNode {
                    if (parent != null && !parent.visible)
                        parent.setVisible(true);
                } else {
-                   for (UpdateNode n : children)
-                       n.setVisible(false);
+                       if (children != null)
+                               for (UpdateNode n : children)
+                                       n.setVisible(false);
                }
        }
        
        public void setAllVisible(boolean visible) {
-        this.visible = visible;
-        if (op != null)
-            op.visible = visible;
-        for (UpdateNode n : children)
-             n.setAllVisible(visible);
-    }
+               this.visible = visible;
+               if (op != null)
+                       op.visible = visible;
+               if (children != null)
+                       for (UpdateNode n : children)
+                               n.setAllVisible(visible);
+       }
 
 }
index 825ccf1b252ccf59c523e070cbb089d3d0120154..c8f4f1d6727fcc2e46f3a9ea6c97a52245d9cb8a 100644 (file)
@@ -6,12 +6,15 @@ import java.util.List;
 import java.util.Map;
 import java.util.Stack;
 
+import org.eclipse.core.runtime.IProgressMonitor;
 import org.simantics.db.ReadGraph;
 import org.simantics.db.Resource;
 import org.simantics.db.Session;
 import org.simantics.db.Statement;
+import org.simantics.db.VirtualGraph;
 import org.simantics.db.WriteGraph;
 import org.simantics.db.common.request.ReadRequest;
+import org.simantics.db.common.request.WriteRequest;
 import org.simantics.db.exception.DatabaseException;
 import org.simantics.interop.test.GraphChanges;
 
@@ -26,6 +29,7 @@ public abstract  class UpdateOperations {
        private Map<Resource, UpdateOp> resourceMap = new HashMap<Resource, UpdateOp>();
        private Map<Statement, UpdateOp> statementMap = new HashMap<Statement, UpdateOp>();
        private GraphChanges changes;
+       private int chunkSize = -1;
        
        public UpdateOperations(GraphChanges changes) {
                this.changes = changes; 
@@ -39,16 +43,101 @@ public abstract  class UpdateOperations {
                return statementMap.get(s);
        }
        
-       public void applyAll(WriteGraph g) throws DatabaseException {
+       public int getChunkSize() {
+               return chunkSize;
+       }
+       
+       public void setChunkSize(int chunkSize) {
+               this.chunkSize = chunkSize;
+       }
+       
+       /**
+        * Applies all changes.
+        * 
+        * @param graph
+        * @throws DatabaseException
+        */
+       public void applyAll(WriteGraph graph) throws DatabaseException {
+               List<UpdateOp> list = operations;
+               apply(graph, list);
+       }
+
+       /**
+        * Applies selected changes.
+        * @param graph
+        * @throws DatabaseException
+        */
+       public void applySelected(WriteGraph graph) throws DatabaseException {
+               List<UpdateOp> list = new ArrayList<UpdateOp>();
                for (UpdateOp op : operations) {
-                       apply(g, op);
+                       if (op.selected())
+                               list.add(op);
                }
+               apply(graph, list);
+       }
+       
+       /**
+        * Applies all changes with chunked DB writes.
+        * 
+        * @param session
+        * @throws DatabaseException
+        */
+       public void applyAll(Session session, VirtualGraph vg) throws DatabaseException {
+               List<UpdateOp> list = operations;
+               apply(session, list, vg);
        }
 
-       public void applySelected(WriteGraph g) throws DatabaseException {
+       /**
+        * Applies selected changes with chunked DB writes.
+        * 
+        * @param session
+        * @throws DatabaseException
+        */
+       public void applySelected(Session session, VirtualGraph vg) throws DatabaseException {
+               List<UpdateOp> list = new ArrayList<UpdateOp>();
                for (UpdateOp op : operations) {
                        if (op.selected())
-                               apply(g, op);
+                               list.add(op);
+               }
+               apply(session, list, vg);
+       }
+       
+       protected void apply(WriteGraph graph, List<UpdateOp> list) throws DatabaseException {
+               for (UpdateOp op : list) {
+                       apply(graph, op);
+               }
+       }
+       
+       protected void apply(Session session, List<UpdateOp> list, VirtualGraph vg) throws DatabaseException {
+               if (getChunkSize() > 0) {
+                       for (int s = 0; s < list.size(); ) {
+                               int e = s + getChunkSize();
+                               if (e > list.size())
+                                       e = list.size();
+                               List<UpdateOp> subList = list.subList(s, e);
+                               session.syncRequest(new WriteRequest(vg) {
+                                       
+                                       @Override
+                                       public void perform(WriteGraph graph) throws DatabaseException {
+                                               
+                                               for (UpdateOp op : subList) {
+                                                       apply(graph, op);
+                                               }
+                                       }
+                               });
+                               s = e;
+                       }
+               } else {
+                       session.syncRequest(new WriteRequest(vg) {
+                               
+                               @Override
+                               public void perform(WriteGraph graph) throws DatabaseException {
+                                       
+                                       for (UpdateOp op : list) {
+                                               apply(graph, op);
+                                       }
+                               }
+                       });
                }
        }
        
@@ -135,7 +224,7 @@ public abstract  class UpdateOperations {
         * @param session
         * @throws DatabaseException
         */
-       public void populate(Session session) throws DatabaseException {
+       public void populate(Session session, IProgressMonitor monitor) throws DatabaseException {
                session.syncRequest(new ReadRequest() {
                        
                        @Override
index a558f84462cf5e8cfe6548f761afa0c85cdb00fe..bf7e0688546beb2e053529ee30e54b41350b738d 100644 (file)
@@ -3,6 +3,8 @@ package  org.simantics.interop.update.model;
 import java.util.HashMap;
 import java.util.Map;
 
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.core.runtime.NullProgressMonitor;
 import org.simantics.db.ReadGraph;
 import org.simantics.db.Resource;
 import org.simantics.db.Session;
@@ -29,17 +31,30 @@ public class UpdateTree {
                this.updateOps = updateOps;
                this.updateOps.populate(g);
                populate(g);
+               this.rootNode.sort();
        }
        
        public UpdateTree(Session session, GraphChanges changes, UpdateOperations updateOps) throws DatabaseException {
+               this(session, changes, updateOps, new NullProgressMonitor());
+       }
+       
+       public UpdateTree(Session session, GraphChanges changes, UpdateOperations updateOps, IProgressMonitor monitor) throws DatabaseException {
                this.changes = changes;
                this.nodes = new HashMap<Resource, UpdateNode>();
                this.rootNode = session.syncRequest(new NodeRequest(UpdateStatus.EXIST, changes.getResource1()));
                nodes.put(changes.getResource1(), rootNode);
                nodes.put(changes.getResource2(), rootNode);
                this.updateOps = updateOps;
-               this.updateOps.populate(session);
-               populate(session);
+               this.updateOps.populate(session, monitor);
+               populate(session, monitor);
+               if (monitor.isCanceled()) {
+                       this.changes = null;
+                       this.nodes.clear();
+                       this.updateOps = null;
+                       this.rootNode = null;
+               } else {
+                       this.rootNode.sort();
+               }
        }
        
        private class NodeRequest extends ResourceRead<UpdateNode> {
@@ -176,9 +191,11 @@ public class UpdateTree {
                }
        }
        
-       protected void populate(Session session) throws DatabaseException{
+       protected void populate(Session session, IProgressMonitor monitor) throws DatabaseException{
                int i = 0;
                while (i < updateOps.getOperations().size()) {
+                       if (monitor.isCanceled())
+                               return;
                        i = session.syncRequest(new PopulateRead(i));
                }
        }
index 3fe74177a2b93bcc71f09e90656b8ffc09858d74..e52ad97924aff74d121ad27d6453fca44393f73d 100644 (file)
@@ -39,9 +39,6 @@ public class GraphChanges {
                        this.leftStm = leftStm;
                        this.rightStm = rightStm;
                        
-                       if (leftSub.getResourceId() < 0)
-                               System.out.println();
-                       
                        hashCode = leftSub.hashCode() + rightSub.hashCode();
                        if (leftStm != null)
                                hashCode += leftStm.hashCode();
index d8750b2c5f2c5a4ed9b8a833e9b1cc03fcfc4151..7ea8f4db055c02fa00d8c685c47e9621113e2294 100644 (file)
@@ -95,6 +95,8 @@ public class GraphComparator {
        private ReadGraph g;
        private Layer0 b;
        
+       private boolean mapLiterals = true;
+       
        public GraphComparator(Resource r1, Resource r2) {
                this.r1 = r1;
                this.r2 = r2;
@@ -204,7 +206,15 @@ public class GraphComparator {
                this.maxIter = maxIter;
        }
        
-       public void clear() {
+       public boolean isMapLiterals() {
+               return mapLiterals;
+       }
+       
+       public void setMapLiterals(boolean mapLiterals) {
+               this.mapLiterals = mapLiterals;
+       }
+       
+       private void clear() {
                changes1.clear();
                changes1Set.clear();
                changes2.clear();
@@ -216,6 +226,28 @@ public class GraphComparator {
                processedResources.clear();
        }
        
+       public void dispose() {
+               changes1 = null;
+               changes1Set = null;
+               changes2 = null;
+               changes2Set = null;
+               comparableResources = null;
+               comparableStatements = null;
+               comparator = null;
+               modifications = null;
+               modificationsSet = null;
+               processedResources = null;
+               nonMatchedLeft = null;
+               nonMatchedRight = null;
+               nonTested = null;
+               nonTraversed = null;
+               ss1 = null;
+               ss2 = null;
+               strong = null;
+               tested = null;
+               traversed = null;
+       }
+       
        public void test(ReadGraph g) throws DatabaseException {
                test(g,null);
        }
@@ -683,6 +715,7 @@ public class GraphComparator {
                        subjectRight.add(s.getSubject(),s);
                        objectRight.add(s.getObject(),s);
                }
+               Set<Pair<Resource, Resource>> processed = new HashSet<Pair<Resource,Resource>>();
                for (Resource ol : objectLeft.getKeys()) {
                        if (maxIter > 0 && iter > maxIter)
                                break;
@@ -698,12 +731,29 @@ public class GraphComparator {
                        }
                        
                        if (sLeft.size() == 1 && sRight.size() == 1) {
-                               List<Statement> ss1 = new ArrayList<Statement>(subjectLeft.getValues(sLeft.iterator().next()));
-                               List<Statement> ss2 = new ArrayList<Statement>(subjectRight.getValues(sRight.iterator().next()));
+                               Resource sl = sLeft.iterator().next();
+                               Resource sr = sRight.iterator().next();
+                               Pair<Resource, Resource> p = new Pair<Resource, Resource>(sl, sr);
+                               if (processed.contains(p))
+                                       continue;
+                               processed.add(p);
+                               List<Statement> ss1 = new ArrayList<Statement>(subjectLeft.getValuesSnapshot(sl));
+                               List<Statement> ss2 = new ArrayList<Statement>(subjectRight.getValuesSnapshot(sr));
+                               for (int i = ss1.size() -1; i >= 0; i--) {
+                                       if (!unreliableLeft.contains(ss1.get(i)))
+                                               ss1.remove(i);
+                               }
+                               for (int i = ss2.size() -1; i >= 0; i--) {
+                                       if (!unreliableRight.contains(ss2.get(i)))
+                                               ss2.remove(i);
+                               }
+                               
+                               int ccount = comparableStatements.size();
+                               int lcount = changes1.size();
+                               int rcount = changes2.size();
                                
-                               int count = comparableStatements.size();
                                compareStatements(ss1, ss2, objectsLeft, objectsRight,unreliableLeft,unreliableRight);
-                               if (comparableStatements.size() > count) {
+                               if (comparableStatements.size() > ccount) {
                                        didSomething = true;
                                        for (Entry<Statement, Statement> entry : comparableStatements.getEntries()) {
                                                unreliableLeft.remove(entry.getKey());
@@ -711,6 +761,20 @@ public class GraphComparator {
                                                iter++;
                                        }
                                }
+                               if (changes1.size() > lcount) {
+                                       didSomething = true;
+                                       for (Statement stm : changes1) {
+                                               unreliableLeft.remove(stm);
+                                               iter++;
+                                       }
+                               }
+                               if (changes2.size() > rcount) {
+                                       didSomething = true;
+                                       for (Statement stm : changes2) {
+                                               unreliableRight.remove(stm);
+                                               iter++;
+                                       }
+                               }
                        }
                }
                return didSomething;
@@ -1451,7 +1515,7 @@ public class GraphComparator {
                                                        if (!eq) {
                                                                addModification(r1,s1,r2,s2);
                                                        }
-                                                       if (!a1 && !a2)
+                                                       if (mapLiterals && !a1 && !a2)
                                                                addComparable(s1, s2); 
                                                } else {
                                                        // Non literal properties.