From 25b0e1909c94977871479b469da722ed38e980c0 Mon Sep 17 00:00:00 2001 From: Antti Villberg Date: Thu, 29 Mar 2018 14:25:01 +0300 Subject: [PATCH] Generate parts of db client query code refs #6961 Change-Id: I444c5b7e1357fb80e52f46b8668153a19af7c587 Multiple simultaneous readers refs #6961 Change-Id: If2374874965c23d07a2fbe1f0075c515e1e12d5c Multiple readers and variable optimization gitlab #5 gitlab #6 (refs #6961) Change-Id: Iabc5ff369cb42e8a3813519b6e5d981f538ebdff Some enhancements made by Antti for multiple readers gitlab #5 gitlab #6 Change-Id: Ic185e28ff85cb72783ebadf908f8b58bf523e2b1 Still working for multiple readers gitlab #5 gitlab #6 Change-Id: Ica299d5dd091a641a7a1a18e1093602a9027dd57 Write model activation information in activations VG gitlab #5 Work in progress gitlab #5 gitlab #6 Change-Id: I37da9d60fbdb476fe62dfe0f360064e924738d6f Adding Logger to DiagramContentRequest Change-Id: I904c7ed209198d897370003b0676df6831892927 Fixed asynchronous recompute problems gitlab #5 Change-Id: Ia80fec89736dbef1f6bd44e730d6c4186921836d (cherry picked from commit 5d26dc0201bd7fafbaafa4f74ff1a10929a14e5b) Fixing URIToResource Change-Id: I67d0145fe48235eeef79212269f5d561f4899180 (cherry picked from commit 4af843556485409e1c018e58eb2409f116e4d40c) Added regression suite library for DB client gitlab #5 Change-Id: I142376af2f4e663622dd77b18ba51ae0cc9914f6 Trying to remove synchronization problems Change-Id: I52a00b8405aa1ca969fe339befa787f79c9ad10a Thread count was wrong Change-Id: If1b97dda9811140a4aa0c5faf5b2f5514141cfc0 Working towards multiple readers. Async is no longer a subset of sync but the other way around. Sync invocation of async request waits until all computatations initiated by the request are done. Conflicts: bundles/org.simantics.db.common/src/org/simantics/db/common/utils/CommonDBUtils.java bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/adapter/impl/TGRemover.java Change-Id: I347acba9289cb2629dc1d21d8666b37e49f6d936 To work with Balas Change-Id: Ie54c022ffa319c86b9d4468428a100de76145cd3 --- .../db/common/utils/CommonDBUtils.java | 6 +- .../db/impl/BlockingAsyncProcedure.java | 58 +++++++ .../db/impl/graph/ReadGraphImpl.java | 1 + .../org/simantics/db/impl/query/CodeGen.java | 141 ++++++++++++++++++ .../db/layer0/adapter/impl/TGRemover.java | 4 +- .../procore/internal/SessionImplSocket.java | 14 ++ .../META-INF/MANIFEST.MF | 6 + .../server/request/DocumentRequest.java | 38 +++++ 8 files changed, 261 insertions(+), 7 deletions(-) diff --git a/bundles/org.simantics.db.common/src/org/simantics/db/common/utils/CommonDBUtils.java b/bundles/org.simantics.db.common/src/org/simantics/db/common/utils/CommonDBUtils.java index 2e16b2849..77eacf205 100644 --- a/bundles/org.simantics.db.common/src/org/simantics/db/common/utils/CommonDBUtils.java +++ b/bundles/org.simantics.db.common/src/org/simantics/db/common/utils/CommonDBUtils.java @@ -234,12 +234,10 @@ public class CommonDBUtils { DirectStatementProcedure proc = new DirectStatementProcedure(); if (ignoreVirtual) { - dqs.forEachDirectPersistentStatement(graph, resource, proc); + return dqs.getDirectPersistentStatements(graph, resource); } else { - dqs.forEachDirectStatement(graph, resource, proc); + return dqs.getDirectStatements(graph, resource); } - - return proc.getOrThrow(); } diff --git a/bundles/org.simantics.db.impl/src/org/simantics/db/impl/BlockingAsyncProcedure.java b/bundles/org.simantics.db.impl/src/org/simantics/db/impl/BlockingAsyncProcedure.java index 943322cd9..b751e8f70 100644 --- a/bundles/org.simantics.db.impl/src/org/simantics/db/impl/BlockingAsyncProcedure.java +++ b/bundles/org.simantics.db.impl/src/org/simantics/db/impl/BlockingAsyncProcedure.java @@ -22,6 +22,7 @@ import org.simantics.db.impl.graph.ReadGraphImpl; import org.simantics.db.procedure.AsyncProcedure; public class BlockingAsyncProcedure implements AsyncProcedure { +<<<<<<< Upstream, based on branch 'private/antti_threads' of ssh://villberg@gerrit.simantics.org:29418/simantics/platform.git final private Object key; final private ReadGraphImpl graph; @@ -75,6 +76,63 @@ public class BlockingAsyncProcedure implements AsyncProcedure { public Result getResult() { return result; +======= + + final private static Object NO_RESULT = new Object(); + + final private Object key; + final private ReadGraphImpl graph; + final private AsyncProcedure procedure; + + private Object result = NO_RESULT; + private Throwable exception = null; + + public BlockingAsyncProcedure(ReadGraphImpl graph, AsyncProcedure procedure, Object key) { + this.procedure = procedure; + this.key = key; + this.graph = ReadGraphImpl.newAsync(graph); + this.graph.asyncBarrier.inc(); + } + + @Override + public void execute(AsyncReadGraph graph, Result result) { + this.result = result; + this.graph.asyncBarrier.dec(); + try { + if(procedure != null) procedure.execute(graph, result); + } catch (Throwable throwable) { + Logger.defaultLogError("AsyncProcedure.execute threw for " + procedure, throwable); + } + } + + @Override + public void exception(AsyncReadGraph graph, Throwable t) { + this.exception = t; + try { + if(procedure != null) procedure.exception(graph, t); + } catch (Throwable throwable) { + Logger.defaultLogError("AsyncProcedure.exception threw for " + procedure, throwable); + } finally { + } + this.graph.asyncBarrier.dec(); + } + + public Result get() throws DatabaseException { + + graph.asyncBarrier.waitBarrier(key, graph); + + if(exception != null) { + if(exception instanceof DatabaseException) throw (DatabaseException)exception; + throw new DatabaseException(exception); + } else { + return (Result)result; + } + + } + + public Result getResult() { + return (Result)result; +>>>>>>> 82fa68e Generate parts of db client query code } public Throwable getException() { diff --git a/bundles/org.simantics.db.impl/src/org/simantics/db/impl/graph/ReadGraphImpl.java b/bundles/org.simantics.db.impl/src/org/simantics/db/impl/graph/ReadGraphImpl.java index 7070ee5b8..61fa8a3aa 100644 --- a/bundles/org.simantics.db.impl/src/org/simantics/db/impl/graph/ReadGraphImpl.java +++ b/bundles/org.simantics.db.impl/src/org/simantics/db/impl/graph/ReadGraphImpl.java @@ -116,6 +116,7 @@ import org.simantics.db.common.procedure.wrapper.SyncToAsyncProcedure; import org.simantics.db.common.procedure.wrapper.SyncToAsyncSetProcedure; import org.simantics.db.common.request.AdaptValue; import org.simantics.db.common.request.ResourceRead; +import org.simantics.db.common.utils.Functions; import org.simantics.db.common.utils.Logger; import org.simantics.db.common.utils.NameUtils; import org.simantics.db.common.validation.L0Validations; diff --git a/bundles/org.simantics.db.impl/src/org/simantics/db/impl/query/CodeGen.java b/bundles/org.simantics.db.impl/src/org/simantics/db/impl/query/CodeGen.java index 3d5f14ac4..16af00b30 100644 --- a/bundles/org.simantics.db.impl/src/org/simantics/db/impl/query/CodeGen.java +++ b/bundles/org.simantics.db.impl/src/org/simantics/db/impl/query/CodeGen.java @@ -25,6 +25,7 @@ public class CodeGen { String[] signatureChildMap = { "int r", "r", "keyR", "long", "InternalProcedure>", "entry.id" }; String[] signatureRead = { "Read r", "r", "id", "long", "AsyncProcedure", "entry.request" }; String[] signatureAsyncRead = { "AsyncRead r", "r", "id", "long", "AsyncProcedure", "entry.request" }; +<<<<<<< Upstream, based on branch 'private/antti_threads' of ssh://villberg@gerrit.simantics.org:29418/simantics/platform.git String[] signatureMultiRead = { "MultiRead r", "r", "id", "long", "AsyncMultiProcedure", "entry.request" }; String[] signatureAsyncMultiRead = { "AsyncMultiRead r", "r", "id", "long", "AsyncMultiProcedure", "entry.request" }; String[] signatureExternalRead = { "ExternalRead r", "r", "id", "long", "AsyncProcedure", "entry.request" }; @@ -164,6 +165,146 @@ public class CodeGen { content.append("import org.simantics.db.request.Read;\n"); content.append("\n"); content.append("import gnu.trove.map.hash.TObjectIntHashMap;\n"); +======= + String[] signatureMultiRead = { "MultiRead r", "r", "id", "long", "SyncMultiProcedure", "entry.request" }; + String[] signatureAsyncMultiRead = { "AsyncMultiRead r", "r", "id", "long", "AsyncMultiProcedure", "entry.request" }; + String[] signatureExternalRead = { "ExternalRead r", "r", "id", "long", "AsyncProcedure", "entry.request" }; + + private void line(StringBuilder content, String line) { + for(int i=0;i>>>>>> 82fa68e Generate parts of db client query code content.append("\n"); content.append("public class QueryCache extends QueryCacheBase {\n"); diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/adapter/impl/TGRemover.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/adapter/impl/TGRemover.java index be34bb9ba..8b29a9349 100644 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/adapter/impl/TGRemover.java +++ b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/adapter/impl/TGRemover.java @@ -79,9 +79,7 @@ public class TGRemover extends AbstractRemover { if (name != null) { graph.deny(r, L0.HasName, L0.NameOf, name); graph.denyValue(name); - DirectStatementProcedure proc = new DirectStatementProcedure(); - dqs.forEachDirectPersistentStatement(graph, name, proc); - for (Statement stm : proc.getOrThrow()) { + for (Statement stm : dqs.getDirectPersistentStatements(graph, name)) { graph.deny(name, stm.getPredicate(), stm.getObject()); } } diff --git a/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/SessionImplSocket.java b/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/SessionImplSocket.java index 6d34d9341..91faecfb7 100644 --- a/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/SessionImplSocket.java +++ b/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/SessionImplSocket.java @@ -1653,7 +1653,19 @@ public abstract class SessionImplSocket implements Session, WriteRequestSchedule request.perform(newGraph, wrap); wrap.get(); +<<<<<<< Upstream, based on branch 'private/antti_threads' of ssh://villberg@gerrit.simantics.org:29418/simantics/platform.git +======= + } catch (DatabaseException e) { + + Logger.defaultLogError(e); + +// wrap.exception(newGraph, t); +// wrapper.exception(newGraph, t); +// newGraph.waitAsync(request); +>>>>>>> 82fa68e Generate parts of db client query code + +<<<<<<< Upstream, based on branch 'private/antti_threads' of ssh://villberg@gerrit.simantics.org:29418/simantics/platform.git } catch (Throwable t) { wrap.exception(newGraph, t); @@ -1662,6 +1674,8 @@ public abstract class SessionImplSocket implements Session, WriteRequestSchedule // newGraph.waitAsync(request); +======= +>>>>>>> 82fa68e Generate parts of db client query code } } diff --git a/bundles/org.simantics.document.server/META-INF/MANIFEST.MF b/bundles/org.simantics.document.server/META-INF/MANIFEST.MF index 875ff601f..d7b561f8f 100644 --- a/bundles/org.simantics.document.server/META-INF/MANIFEST.MF +++ b/bundles/org.simantics.document.server/META-INF/MANIFEST.MF @@ -24,7 +24,13 @@ Require-Bundle: org.simantics.document.server.io;visibility:=reexport, org.simantics.scl.db;bundle-version="0.1.3", org.slf4j.api, +<<<<<<< Upstream, based on branch 'private/antti_threads' of ssh://villberg@gerrit.simantics.org:29418/simantics/platform.git org.simantics.threadlog +======= + org.simantics.threadlog, + org.simantics.simulator.toolkit, + org.simantics.simulator.toolkit.db;bundle-version="1.0.0";visibility:=reexport +>>>>>>> 82fa68e Generate parts of db client query code Bundle-ActivationPolicy: lazy Bundle-Activator: org.simantics.document.server.Activator Export-Package: org.simantics.document.server, diff --git a/bundles/org.simantics.document.server/src/org/simantics/document/server/request/DocumentRequest.java b/bundles/org.simantics.document.server/src/org/simantics/document/server/request/DocumentRequest.java index 1726887cb..779cf4fe1 100644 --- a/bundles/org.simantics.document.server/src/org/simantics/document/server/request/DocumentRequest.java +++ b/bundles/org.simantics.document.server/src/org/simantics/document/server/request/DocumentRequest.java @@ -9,9 +9,11 @@ import java.util.Set; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; +import org.simantics.db.AsyncReadGraph; import org.simantics.db.ReadGraph; import org.simantics.db.common.GraphSemaphore; import org.simantics.db.common.procedure.adapter.TransientCacheAsyncListener; +import org.simantics.db.common.request.AsyncReadRequest; import org.simantics.db.exception.DatabaseException; import org.simantics.db.layer0.request.VariableRead; import org.simantics.db.layer0.variable.Variable; @@ -80,6 +82,7 @@ public class DocumentRequest extends VariableRead> { System.out.println(" " + node.getURI(graph)); }*/ +<<<<<<< Upstream, based on branch 'private/antti_threads' of ssh://villberg@gerrit.simantics.org:29418/simantics/platform.git GraphSemaphore done = new GraphSemaphore(graph, 0); for(Variable node : nodes) { @@ -116,6 +119,41 @@ public class DocumentRequest extends VariableRead> { } catch (InterruptedException e) { e.printStackTrace(); } +======= + graph.syncRequest(new AsyncReadRequest() { + + @Override + public void run(AsyncReadGraph graph) { + + for(Variable node : nodes) { + + graph.asyncRequest(new NodeRequestE(node), new Listener() { + + @Override + public void execute(JSONObject result) { + synchronized(rs) { + rs.add(result); + } + } + + @Override + public void exception(Throwable t) { + t.printStackTrace(); + } + + @Override + public boolean isDisposed() { + return true; + } + + }); + + } + + } + + }); +>>>>>>> 82fa68e Generate parts of db client query code ArrayList result = new ArrayList(rs); Collections.sort(result, new Comparator() { -- 2.47.1