]> gerrit.simantics Code Review - simantics/platform.git/commitdiff
Generate parts of db client query code
authorAntti Villberg <antti.villberg@semantum.fi>
Thu, 29 Mar 2018 11:25:01 +0000 (14:25 +0300)
committerAntti Villberg <antti.villberg@semantum.fi>
Fri, 27 Jul 2018 16:27:27 +0000 (19:27 +0300)
refs #6961

Change-Id: I444c5b7e1357fb80e52f46b8668153a19af7c587

Multiple simultaneous readers

refs #6961

Change-Id: If2374874965c23d07a2fbe1f0075c515e1e12d5c

Multiple readers and variable optimization

gitlab #5
gitlab #6
(refs #6961)

Change-Id: Iabc5ff369cb42e8a3813519b6e5d981f538ebdff
Some enhancements made by Antti for multiple readers

gitlab #5
gitlab #6

Change-Id: Ic185e28ff85cb72783ebadf908f8b58bf523e2b1

Still working for multiple readers

gitlab #5
gitlab #6

Change-Id: Ica299d5dd091a641a7a1a18e1093602a9027dd57

Write model activation information in activations VG

gitlab #5
Work in progress

gitlab #5
gitlab #6

Change-Id: I37da9d60fbdb476fe62dfe0f360064e924738d6f

Adding Logger to DiagramContentRequest

Change-Id: I904c7ed209198d897370003b0676df6831892927

Fixed asynchronous recompute problems

gitlab #5

Change-Id: Ia80fec89736dbef1f6bd44e730d6c4186921836d
(cherry picked from commit 5d26dc0201bd7fafbaafa4f74ff1a10929a14e5b)

Fixing URIToResource

Change-Id: I67d0145fe48235eeef79212269f5d561f4899180
(cherry picked from commit 4af843556485409e1c018e58eb2409f116e4d40c)

Added regression suite library for DB client

gitlab #5

Change-Id: I142376af2f4e663622dd77b18ba51ae0cc9914f6

Trying to remove synchronization problems

Change-Id: I52a00b8405aa1ca969fe339befa787f79c9ad10a

Thread count was wrong

Change-Id: If1b97dda9811140a4aa0c5faf5b2f5514141cfc0

Working towards multiple readers.

Async is no longer a subset of sync but the other way around. Sync
invocation of async request waits until all computatations
initiated by the request are done.

Conflicts:
bundles/org.simantics.db.common/src/org/simantics/db/common/utils/CommonDBUtils.java
bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/adapter/impl/TGRemover.java

Change-Id: I347acba9289cb2629dc1d21d8666b37e49f6d936

To work with Balas

Change-Id: Ie54c022ffa319c86b9d4468428a100de76145cd3

bundles/org.simantics.db.common/src/org/simantics/db/common/utils/CommonDBUtils.java
bundles/org.simantics.db.impl/src/org/simantics/db/impl/BlockingAsyncProcedure.java
bundles/org.simantics.db.impl/src/org/simantics/db/impl/graph/ReadGraphImpl.java
bundles/org.simantics.db.impl/src/org/simantics/db/impl/query/CodeGen.java
bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/adapter/impl/TGRemover.java
bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/SessionImplSocket.java
bundles/org.simantics.document.server/META-INF/MANIFEST.MF
bundles/org.simantics.document.server/src/org/simantics/document/server/request/DocumentRequest.java

index 2e16b284903997a204f1c229f3a1adc83df10a96..77eacf2055e9002c86aa99da5fed5c96a3f15906 100644 (file)
@@ -234,12 +234,10 @@ public class CommonDBUtils {
                DirectStatementProcedure proc = new DirectStatementProcedure();
 
                if (ignoreVirtual) {
-                       dqs.forEachDirectPersistentStatement(graph, resource, proc);
+                       return dqs.getDirectPersistentStatements(graph, resource);
                } else {
-                       dqs.forEachDirectStatement(graph, resource, proc);
+                       return dqs.getDirectStatements(graph, resource);
                }
-               
-               return proc.getOrThrow();
 
        }
        
index 943322cd9dc8f6af1aadf710aac387312de652ea..b751e8f70d5771f8bd0994e58dbaa1d890b53b93 100644 (file)
@@ -22,6 +22,7 @@ import org.simantics.db.impl.graph.ReadGraphImpl;
 import org.simantics.db.procedure.AsyncProcedure;
 
 public class BlockingAsyncProcedure<Result> implements AsyncProcedure<Result> {
+<<<<<<< Upstream, based on branch 'private/antti_threads' of ssh://villberg@gerrit.simantics.org:29418/simantics/platform.git
 
        final private Object key;
     final private ReadGraphImpl graph;
@@ -75,6 +76,63 @@ public class BlockingAsyncProcedure<Result> implements AsyncProcedure<Result> {
     
     public Result getResult() {
         return result;
+=======
+       
+       final private static Object NO_RESULT = new Object();
+
+       final private Object key;
+    final private ReadGraphImpl graph;
+       final private AsyncProcedure<Result> procedure;
+       
+    private Object result = NO_RESULT;
+    private Throwable exception = null;
+    
+    public BlockingAsyncProcedure(ReadGraphImpl graph, AsyncProcedure<Result> procedure, Object key) {
+       this.procedure = procedure;
+       this.key = key;
+       this.graph = ReadGraphImpl.newAsync(graph);
+        this.graph.asyncBarrier.inc();
+    }
+    
+    @Override
+    public void execute(AsyncReadGraph graph, Result result) {
+        this.result = result;
+        this.graph.asyncBarrier.dec();
+        try {
+               if(procedure != null) procedure.execute(graph, result);
+        } catch (Throwable throwable) {
+               Logger.defaultLogError("AsyncProcedure.execute threw for " + procedure, throwable);
+        }
+    }
+
+    @Override
+    public void exception(AsyncReadGraph graph, Throwable t) {
+        this.exception = t;
+        try {
+               if(procedure != null) procedure.exception(graph, t);
+        } catch (Throwable throwable) {
+               Logger.defaultLogError("AsyncProcedure.exception threw for " + procedure, throwable);
+        } finally {
+        }
+        this.graph.asyncBarrier.dec();
+    }
+    
+    public Result get() throws DatabaseException {
+       
+       graph.asyncBarrier.waitBarrier(key, graph);
+       
+       if(exception != null) {
+               if(exception instanceof DatabaseException) throw (DatabaseException)exception;
+               throw new DatabaseException(exception);
+       } else {
+               return (Result)result;
+       }
+       
+    }
+    
+    public Result getResult() {
+        return (Result)result;
+>>>>>>> 82fa68e Generate parts of db client query code
     }
     
     public Throwable getException() {
index 7070ee5b8b9552ed0ab9317124b2861b23833bfc..61fa8a3aa008655b3541ccf6d1fc4a3c43300a44 100644 (file)
@@ -116,6 +116,7 @@ import org.simantics.db.common.procedure.wrapper.SyncToAsyncProcedure;
 import org.simantics.db.common.procedure.wrapper.SyncToAsyncSetProcedure;
 import org.simantics.db.common.request.AdaptValue;
 import org.simantics.db.common.request.ResourceRead;
+import org.simantics.db.common.utils.Functions;
 import org.simantics.db.common.utils.Logger;
 import org.simantics.db.common.utils.NameUtils;
 import org.simantics.db.common.validation.L0Validations;
index 3d5f14ac493603d800102438e65ff5751786e8ec..16af00b30ba0250725ae9d2107bb132d10f56e8f 100644 (file)
@@ -25,6 +25,7 @@ public class CodeGen {
        String[] signatureChildMap = { "int r", "r", "keyR", "long", "InternalProcedure<ObjectResourceIdMap<String>>", "entry.id" };
        String[] signatureRead = { "Read<?> r", "r", "id", "long", "AsyncProcedure", "entry.request" };
        String[] signatureAsyncRead = { "AsyncRead<?> r", "r", "id", "long", "AsyncProcedure", "entry.request" };
+<<<<<<< Upstream, based on branch 'private/antti_threads' of ssh://villberg@gerrit.simantics.org:29418/simantics/platform.git
        String[] signatureMultiRead = { "MultiRead<?> r", "r", "id", "long", "AsyncMultiProcedure", "entry.request" };
        String[] signatureAsyncMultiRead = { "AsyncMultiRead<?> r", "r", "id", "long", "AsyncMultiProcedure", "entry.request" };
        String[] signatureExternalRead = { "ExternalRead<?> r", "r", "id", "long", "AsyncProcedure", "entry.request" };
@@ -164,6 +165,146 @@ public class CodeGen {
                     content.append("import org.simantics.db.request.Read;\n");
                     content.append("\n");
                     content.append("import gnu.trove.map.hash.TObjectIntHashMap;\n");
+=======
+       String[] signatureMultiRead = { "MultiRead<?> r", "r", "id", "long", "SyncMultiProcedure", "entry.request" };
+       String[] signatureAsyncMultiRead = { "AsyncMultiRead<?> r", "r", "id", "long", "AsyncMultiProcedure", "entry.request" };
+       String[] signatureExternalRead = { "ExternalRead<?> r", "r", "id", "long", "AsyncProcedure", "entry.request" };
+
+       private void line(StringBuilder content, String line) {
+               for(int i=0;i<indent;i++)
+                       content.append(" ");
+               content.append(line);
+               content.append("\n");
+       }
+       
+       public void generateQuery(StringBuilder content, String clazz, String[] signature, boolean runnerShortcut, boolean genAsync) {
+               generateGetOrCreate(content, clazz, signature, genAsync);
+               generateRemove(content, clazz, signature);
+               generateRunner(content, clazz, signature, runnerShortcut, genAsync);
+       }
+       
+       public void generateRunner(StringBuilder content, String clazz, String[] signature, boolean shortcut, boolean genAsync) {
+               
+               line(content, "public static void runner" + clazz + "(ReadGraphImpl graph, " + signature[0] + ", CacheEntry parent, ListenerBase listener, final " + signature[4] + " procedure" + (genAsync ? ", boolean isSync" : "") + ") throws DatabaseException {");
+               line(content, "    QueryCache cache  = graph.processor.cache;");
+               if(shortcut) {
+                       line(content, "    if(parent == null && listener == null && !cache.shouldCache(graph.processor, " + signature[1] + ")) {");
+                       line(content, "        " + clazz + ".computeForEach(graph, " + signature[1] + ", null, procedure);");
+                       line(content, "        return;");
+                       line(content, "    }");
+               }
+               line(content, "    " + clazz + " entry = (" + clazz + ")cache.getOrCreate" + clazz + "(graph.processor, " + signature[1] + (genAsync ? ", isSync" : "") + ");");
+               if(genAsync) {
+                       line(content, "    if(entry == null) {");
+                       line(content, "        graph.processor.schedule(new SessionTask(false) {");
+                       line(content, "            @Override");
+                       line(content, "            public void run(int thread) {");
+                       line(content, "                try {");
+                       line(content, "                    assert(!isSync);");
+                       line(content, "                    runner" + clazz + "(graph, r, parent, listener, procedure, isSync);");
+                       line(content, "                } catch (DatabaseException e) {");
+                       line(content, "                    Logger.defaultLogError(e);");
+                       line(content, "                }");
+                       line(content, "            }");
+                       line(content, "        });");
+                       line(content, "        return;");
+                       line(content, "    }");
+               }
+               line(content, "    " + signature[4] + " procedure_ = procedure != null ? procedure : emptyProcedure" + clazz + ";");
+               line(content, "    ListenerEntry listenerEntry = cache.registerDependencies(graph, entry, parent, listener, procedure_, false);");
+               line(content, "    if(entry.isReady()) entry.performFromCache(graph, procedure_);");
+               line(content, "    else {");
+               line(content, "      assert(entry.isPending());");
+               if(shortcut) line(content, "        " + clazz + ".computeForEach(graph, " + signature[1] + ", entry, procedure_);");
+               else line(content, "        entry.compute(graph, procedure_);"); 
+               line(content, "        if(listenerEntry != null) cache.primeListenerEntry(listenerEntry, entry.getResult());");
+               line(content, "    }");
+               line(content, "}");
+               line(content, "");
+
+       }
+       
+       public void generateRemove(StringBuilder content, String clazz, String[] signature) {
+
+               String lower = Character.toLowerCase(clazz.charAt(0)) + clazz.substring(1);
+
+               line(content, "void remove(" + clazz + " entry) {");
+               line(content, "    synchronized(" + lower + "Map) {");
+               line(content, "        " + lower + "Map.remove(" + signature[5] + ");");
+               line(content, "    }");
+               line(content, "}");
+               line(content, "");
+
+       }
+
+       public void generateGetOrCreate(StringBuilder content, String clazz, String[] signature, boolean genAsync) {
+               
+               String lower = Character.toLowerCase(clazz.charAt(0)) + clazz.substring(1);
+               
+               line(content, "" + clazz + " getOrCreate" + clazz + "(QueryProcessor processor, " + signature[0] + (genAsync ? ", boolean isSync" : "") + ") throws DatabaseException {");
+               line(content, "    " + clazz + " existing = null;");
+               line(content, "    synchronized(" + lower + "Map) {");
+               line(content, "        existing = (" + clazz + ")" + lower + "Map.get(" + signature[1] + ");");
+               line(content, "        if(existing == null) {");
+               line(content, "            existing = new " + clazz + "(" + signature[1] + ");");
+               line(content, "            existing.clearResult(querySupport);");
+               line(content, "            existing.setPending();");
+               line(content, "            " + lower + "Map.put(" + signature[2] + "(" + signature[1] + "), existing);");
+               line(content, "            size++;");
+               line(content, "            return existing;");
+               line(content, "        }");
+               line(content, "        if(existing.requiresComputation()) {");
+               line(content, "            existing.setPending();");
+               line(content, "            return existing;");
+               line(content, "        }");
+               line(content, "    }");
+               if(genAsync) {
+                       line(content, "    if(existing.isPending()) {");
+                       line(content, "      if(isSync) waitPending(processor, existing);");
+                       line(content, "      else return null;");
+                       line(content, "    }");
+               } else {
+                       line(content, "    if(existing.isPending()) waitPending(processor, existing);");
+               }
+               line(content, "    return existing;");
+               line(content, "}");
+               line(content, "");
+               
+       }
+       
+       public void generate() {
+
+        URL classLocation = CodeGen.class.getResource(".");
+        if (classLocation != null) {
+            if (classLocation.getProtocol().equals("file")) {
+                try {
+                    URL resource = new URL(classLocation, ".");
+                    File path = new File(URLDecoder.decode(resource.getPath(), "UTF-8"));
+                    String target = path.getAbsolutePath().replace("\\", "/");
+                    target = target.replace("/bin/", "/src/") + "/QueryCache.java";
+                    System.err.println("target=" + target);
+                    File source = new File(target);
+                    StringBuilder content = new StringBuilder();
+                    content.append("package org.simantics.db.impl.query;\n");
+                    content.append("\n");
+                    
+                    content.append("import org.simantics.db.ObjectResourceIdMap;\n");
+                    content.append("import org.simantics.db.RelationInfo;\n");
+                    content.append("import org.simantics.db.common.utils.Logger;\n");
+                    content.append("import org.simantics.db.exception.DatabaseException;\n");
+                    content.append("import org.simantics.db.impl.graph.ReadGraphImpl;\n");
+                    content.append("import org.simantics.db.impl.procedure.InternalProcedure;\n");
+                    content.append("import org.simantics.db.impl.query.QueryProcessor.SessionTask;\n");
+                    content.append("import org.simantics.db.procedure.AsyncMultiProcedure;\n");
+                    content.append("import org.simantics.db.procedure.AsyncProcedure;\n");
+                    content.append("import org.simantics.db.procedure.ListenerBase;\n");
+                    content.append("import org.simantics.db.procedure.SyncMultiProcedure;\n");
+                    content.append("import org.simantics.db.request.AsyncMultiRead;\n");
+                    content.append("import org.simantics.db.request.AsyncRead;\n");
+                    content.append("import org.simantics.db.request.ExternalRead;\n");
+                    content.append("import org.simantics.db.request.MultiRead;\n");
+                    content.append("import org.simantics.db.request.Read;\n");
+>>>>>>> 82fa68e Generate parts of db client query code
                     content.append("\n");
                     
                     content.append("public class QueryCache extends QueryCacheBase {\n");                    
index be34bb9ba5165c9714e3e8573caed16077ccbcd6..8b29a9349b1a969c064ddf218b4837956790f872 100644 (file)
@@ -79,9 +79,7 @@ public class TGRemover extends AbstractRemover {
                         if (name != null) {
                             graph.deny(r, L0.HasName, L0.NameOf, name);
                             graph.denyValue(name);
-                            DirectStatementProcedure proc = new DirectStatementProcedure();
-                            dqs.forEachDirectPersistentStatement(graph, name, proc);
-                            for (Statement stm : proc.getOrThrow()) {
+                            for (Statement stm : dqs.getDirectPersistentStatements(graph, name)) {
                                 graph.deny(name, stm.getPredicate(), stm.getObject());
                             }
                         }
index 6d34d9341c425bbdd465616af54a12d6e3e62cf4..91faecfb752986a6eb1694f4d70e2b671271b5f8 100644 (file)
@@ -1653,7 +1653,19 @@ public abstract class SessionImplSocket implements Session, WriteRequestSchedule
 
                             request.perform(newGraph, wrap);
                             wrap.get();
+<<<<<<< Upstream, based on branch 'private/antti_threads' of ssh://villberg@gerrit.simantics.org:29418/simantics/platform.git
+=======
 
+                        } catch (DatabaseException e) {
+
+                                                       Logger.defaultLogError(e);
+
+//                             wrap.exception(newGraph, t);
+//                            wrapper.exception(newGraph, t);
+//                            newGraph.waitAsync(request);
+>>>>>>> 82fa68e Generate parts of db client query code
+
+<<<<<<< Upstream, based on branch 'private/antti_threads' of ssh://villberg@gerrit.simantics.org:29418/simantics/platform.git
                         } catch (Throwable t) {
 
                                wrap.exception(newGraph, t);
@@ -1662,6 +1674,8 @@ public abstract class SessionImplSocket implements Session, WriteRequestSchedule
 //                            newGraph.waitAsync(request);
 
 
+=======
+>>>>>>> 82fa68e Generate parts of db client query code
                         }
 
                     }
index 875ff601fa3b244b464bfae6afaf817f0c6a47d2..d7b561f8f7f653e4359904f40782d16578d71ac9 100644 (file)
@@ -24,7 +24,13 @@ Require-Bundle:
  org.simantics.document.server.io;visibility:=reexport,
  org.simantics.scl.db;bundle-version="0.1.3",
  org.slf4j.api,
+<<<<<<< Upstream, based on branch 'private/antti_threads' of ssh://villberg@gerrit.simantics.org:29418/simantics/platform.git
  org.simantics.threadlog
+=======
+ org.simantics.threadlog,
+ org.simantics.simulator.toolkit,
+ org.simantics.simulator.toolkit.db;bundle-version="1.0.0";visibility:=reexport
+>>>>>>> 82fa68e Generate parts of db client query code
 Bundle-ActivationPolicy: lazy
 Bundle-Activator: org.simantics.document.server.Activator
 Export-Package: org.simantics.document.server,
index 1726887cb359d8f92acafb3bb34be41dbd693165..779cf4fe1381acb5979e541d54fce12047115103 100644 (file)
@@ -9,9 +9,11 @@ import java.util.Set;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 
+import org.simantics.db.AsyncReadGraph;
 import org.simantics.db.ReadGraph;
 import org.simantics.db.common.GraphSemaphore;
 import org.simantics.db.common.procedure.adapter.TransientCacheAsyncListener;
+import org.simantics.db.common.request.AsyncReadRequest;
 import org.simantics.db.exception.DatabaseException;
 import org.simantics.db.layer0.request.VariableRead;
 import org.simantics.db.layer0.variable.Variable;
@@ -80,6 +82,7 @@ public class DocumentRequest extends VariableRead<List<JSONObject>> {
                        System.out.println("               " + node.getURI(graph));
                }*/
 
+<<<<<<< Upstream, based on branch 'private/antti_threads' of ssh://villberg@gerrit.simantics.org:29418/simantics/platform.git
                GraphSemaphore done = new GraphSemaphore(graph, 0);
                
                for(Variable node : nodes) {
@@ -116,6 +119,41 @@ public class DocumentRequest extends VariableRead<List<JSONObject>> {
                        } catch (InterruptedException e) {
                                e.printStackTrace();
                        }
+=======
+               graph.syncRequest(new AsyncReadRequest() {
+                               
+                               @Override
+                               public void run(AsyncReadGraph graph) {
+                                       
+                               for(Variable node : nodes) {
+                                       
+                                       graph.asyncRequest(new NodeRequestE(node), new Listener<JSONObject>() {
+
+                                                       @Override
+                                                       public void execute(JSONObject result) {
+                                                               synchronized(rs) {
+                                                                       rs.add(result);
+                                                               }
+                                                       }
+
+                                                       @Override
+                                                       public void exception(Throwable t) {
+                                                               t.printStackTrace();
+                                                       }
+
+                                                       @Override
+                                                       public boolean isDisposed() {
+                                                               return true;
+                                                       }
+                                                       
+                                               });
+                                       
+                               }
+                                       
+                               }
+                               
+                       });
+>>>>>>> 82fa68e Generate parts of db client query code
        
                        ArrayList<JSONObject> result = new ArrayList<JSONObject>(rs);
                        Collections.sort(result, new Comparator<JSONObject>() {