From: Tuukka Lehtonen Date: Wed, 17 Jun 2020 20:13:57 +0000 (+0000) Subject: Merge "Optimize ReadGraph.hasStatements(s,p) to not use getObjects(s,p)" into release... X-Git-Url: https://gerrit.simantics.org/r/gitweb?a=commitdiff_plain;h=2f79fa2a6d636373340bffb6d09b2f849b722f88;hp=ac990d10ff2d1d1042a68f0594489aed23a27453;p=simantics%2Fplatform.git Merge "Optimize ReadGraph.hasStatements(s,p) to not use getObjects(s,p)" into release/1.35.3-sulca --- diff --git a/bundles/org.simantics.browsing.ui.model/src/org/simantics/browsing/ui/model/browsecontexts/BrowseContext.java b/bundles/org.simantics.browsing.ui.model/src/org/simantics/browsing/ui/model/browsecontexts/BrowseContext.java index 42ac58420..f94ffb3e4 100644 --- a/bundles/org.simantics.browsing.ui.model/src/org/simantics/browsing/ui/model/browsecontexts/BrowseContext.java +++ b/bundles/org.simantics.browsing.ui.model/src/org/simantics/browsing/ui/model/browsecontexts/BrowseContext.java @@ -57,10 +57,12 @@ import org.simantics.browsing.ui.model.visuals.VisualsContribution; import org.simantics.db.ReadGraph; import org.simantics.db.RequestProcessor; import org.simantics.db.Resource; +import org.simantics.db.common.procedure.adapter.TransientCacheAsyncListener; +import org.simantics.db.common.request.ResourceRead; +import org.simantics.db.common.request.UnaryRead; import org.simantics.db.exception.DatabaseException; import org.simantics.db.exception.ResourceNotFoundException; import org.simantics.db.layer0.variable.Variable; -import org.simantics.db.request.Read; import org.simantics.scl.reflection.OntologyVersions; import org.simantics.viewpoint.ontology.ViewpointResource; import org.slf4j.Logger; @@ -76,21 +78,21 @@ public class BrowseContext { private static final Logger LOGGER = LoggerFactory.getLogger(BrowseContext.class); public static final boolean DEBUG = false; - NodeTypeMultiMap childContributions = new NodeTypeMultiMap(); - NodeTypeMultiMap parentContributions = new NodeTypeMultiMap(); - OrderedNodeTypeMultiMap labelContributions = new OrderedNodeTypeMultiMap(); - OrderedNodeTypeMultiMap imageContributions = new OrderedNodeTypeMultiMap(); - OrderedNodeTypeMultiMap checkedStateContributions = new OrderedNodeTypeMultiMap(); - OrderedNodeTypeMultiMap labelDecorationContributions = new OrderedNodeTypeMultiMap(); - OrderedNodeTypeMultiMap imageDecorationContributions = new OrderedNodeTypeMultiMap(); - OrderedNodeTypeMultiMap modifierContributions = new OrderedNodeTypeMultiMap(); - OrderedNodeTypeMultiMap sorterContributions = new OrderedNodeTypeMultiMap(); - OrderedNodeTypeMultiMap flatNodeContributions = new OrderedNodeTypeMultiMap(); + NodeTypeMultiMap childContributions = new NodeTypeMultiMap<>(); + NodeTypeMultiMap parentContributions = new NodeTypeMultiMap<>(); + OrderedNodeTypeMultiMap labelContributions = new OrderedNodeTypeMultiMap<>(); + OrderedNodeTypeMultiMap imageContributions = new OrderedNodeTypeMultiMap<>(); + OrderedNodeTypeMultiMap checkedStateContributions = new OrderedNodeTypeMultiMap<>(); + OrderedNodeTypeMultiMap labelDecorationContributions = new OrderedNodeTypeMultiMap<>(); + OrderedNodeTypeMultiMap imageDecorationContributions = new OrderedNodeTypeMultiMap<>(); + OrderedNodeTypeMultiMap modifierContributions = new OrderedNodeTypeMultiMap<>(); + OrderedNodeTypeMultiMap sorterContributions = new OrderedNodeTypeMultiMap<>(); + OrderedNodeTypeMultiMap flatNodeContributions = new OrderedNodeTypeMultiMap<>(); OrderedNodeTypeMultiMap tooltipContributions = new OrderedNodeTypeMultiMap<>(); private final String[] uris; - private BrowseContext(String[] uris) { + private BrowseContext(String... uris) { if (uris == null) throw new NullPointerException("null URIs"); this.uris = uris; @@ -109,6 +111,38 @@ public class BrowseContext { return defaultContext; } + private static BrowseContext loadCachedVisuals(ReadGraph g, Resource visualsContributionResource) throws DatabaseException, InvalidContribution { + try { + return g.syncRequest(new ResourceRead(visualsContributionResource) { + @Override + public BrowseContext perform(ReadGraph graph) throws DatabaseException { + try { + BrowseContext bc = new BrowseContext(); + VisualsContribution.load(g, visualsContributionResource, + bc.labelContributions, + bc.imageContributions, + bc.checkedStateContributions, + bc.labelDecorationContributions, + bc.imageDecorationContributions, + bc.modifierContributions, + bc.sorterContributions, + bc.flatNodeContributions, + bc.tooltipContributions + ); + return bc; + } catch (InvalidContribution e) { + throw new DatabaseException(e); + } + } + }, TransientCacheAsyncListener.instance()); + } catch (DatabaseException e) { + Throwable c = e.getCause(); + if (c instanceof InvalidContribution) + throw (InvalidContribution) c; + throw e; + } + } + /** * Creates a new BrowseContext for the given Collection of {@link Resource}s. * @@ -125,24 +159,23 @@ public class BrowseContext { for(Resource childContributionResource : g.getObjects(browseContextResource, vr.BrowseContext_HasChildContribution)) { - ChildContribution contribution = ChildContribution.create(g, childContributionResource); + ChildContribution contribution = ChildContribution.createCached(g, childContributionResource); browseContext.childContributions.put(contribution.getParentNodeType(), contribution); browseContext.parentContributions.put(contribution.getChildNodeType(), contribution); } for(Resource visualsContributionResource : g.getObjects(browseContextResource, vr.BrowseContext_HasVisualsContribution)) { - VisualsContribution.load(g, visualsContributionResource, - browseContext.labelContributions, - browseContext.imageContributions, - browseContext.checkedStateContributions, - browseContext.labelDecorationContributions, - browseContext.imageDecorationContributions, - browseContext.modifierContributions, - browseContext.sorterContributions, - browseContext.flatNodeContributions, - browseContext.tooltipContributions - ); + BrowseContext visuals = loadCachedVisuals(g, visualsContributionResource); + visuals.labelContributions.appendTo(browseContext.labelContributions); + visuals.imageContributions.appendTo(browseContext.imageContributions); + visuals.checkedStateContributions.appendTo(browseContext.checkedStateContributions); + visuals.labelDecorationContributions.appendTo(browseContext.labelDecorationContributions); + visuals.imageDecorationContributions.appendTo(browseContext.imageDecorationContributions); + visuals.modifierContributions.appendTo(browseContext.modifierContributions); + visuals.sorterContributions.appendTo(browseContext.sorterContributions); + visuals.flatNodeContributions.appendTo(browseContext.flatNodeContributions); + visuals.tooltipContributions.appendTo(browseContext.tooltipContributions); } } //browseContext.visualize(); @@ -150,11 +183,11 @@ public class BrowseContext { } public static Set getBrowseContextClosure(RequestProcessor processor, final Set browseContexts) throws DatabaseException { - return processor.syncRequest(new Read>() { + return processor.syncRequest(new UnaryRead, Set>(browseContexts) { @Override public Set perform(ReadGraph graph) throws DatabaseException { - Collection browseContextResources = new ArrayList(browseContexts.size()); - for (String browseContext : browseContexts) { + Collection browseContextResources = new ArrayList<>(parameter.size()); + for (String browseContext : parameter) { try { browseContextResources.add(graph.getResource(browseContext)); } catch (ResourceNotFoundException e) { @@ -162,19 +195,29 @@ public class BrowseContext { } } Collection allBrowseContextResources = BrowseContext.findSubcontexts(graph, browseContextResources); - Set result = new HashSet(); + Set result = new HashSet<>(); for (Resource r : allBrowseContextResources) result.add(graph.getURI(r)); - return result; + return result; } - }); + }, TransientCacheAsyncListener.instance()); } - public static Collection findSubcontexts(ReadGraph g, + public static Collection findSubcontexts(ReadGraph g, Collection browseContexts) + throws DatabaseException { + return g.syncRequest(new UnaryRead, Collection>(browseContexts) { + @Override + public Collection perform(ReadGraph graph) throws DatabaseException { + return findSubcontexts0(graph, parameter); + } + }, TransientCacheAsyncListener.instance()); + } + + private static Collection findSubcontexts0(ReadGraph g, Collection browseContexts) throws DatabaseException { ViewpointResource vr = ViewpointResource.getInstance(g); - HashSet result = new HashSet(browseContexts); - ArrayList stack = new ArrayList(browseContexts); + HashSet result = new HashSet<>(browseContexts); + ArrayList stack = new ArrayList<>(browseContexts); while(!stack.isEmpty()) { Resource cur = stack.remove(stack.size()-1); for(Resource sc : g.getObjects(cur, vr.BrowseContext_Includes)) @@ -183,7 +226,7 @@ public class BrowseContext { } return result; } - + /** * Finds the possible children of the given {@link NodeContext} parameter. * diff --git a/bundles/org.simantics.browsing.ui.model/src/org/simantics/browsing/ui/model/children/ChildContribution.java b/bundles/org.simantics.browsing.ui.model/src/org/simantics/browsing/ui/model/children/ChildContribution.java index ca9f59da0..09a15fefc 100644 --- a/bundles/org.simantics.browsing.ui.model/src/org/simantics/browsing/ui/model/children/ChildContribution.java +++ b/bundles/org.simantics.browsing.ui.model/src/org/simantics/browsing/ui/model/children/ChildContribution.java @@ -22,6 +22,8 @@ import org.simantics.browsing.ui.model.nodetypes.NodeType; import org.simantics.databoard.Bindings; import org.simantics.db.ReadGraph; import org.simantics.db.Resource; +import org.simantics.db.common.procedure.adapter.TransientCacheAsyncListener; +import org.simantics.db.common.request.ResourceRead; import org.simantics.db.common.utils.NameUtils; import org.simantics.db.exception.DatabaseException; import org.simantics.db.layer0.exception.PendingVariableException; @@ -52,6 +54,26 @@ public class ChildContribution { this.priority = priority; } + public static ChildContribution createCached(ReadGraph g, Resource childContributionResource) throws DatabaseException, InvalidContribution { + try { + return g.syncRequest(new ResourceRead(childContributionResource) { + @Override + public ChildContribution perform(ReadGraph graph) throws DatabaseException { + try { + return create(g, resource); + } catch (InvalidContribution e) { + throw new DatabaseException(e); + } + } + }, TransientCacheAsyncListener.instance()); + } catch (DatabaseException e) { + Throwable c = e.getCause(); + if (c instanceof InvalidContribution) + throw (InvalidContribution) c; + throw e; + } + } + public static ChildContribution create(ReadGraph g, Resource childContributionResource) throws DatabaseException, InvalidContribution { ViewpointResource vr = ViewpointResource.getInstance(g); diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/HierarchyMultiMap.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/HierarchyMultiMap.java index a4ac87b50..b12d5ad2b 100644 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/HierarchyMultiMap.java +++ b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/HierarchyMultiMap.java @@ -37,7 +37,28 @@ public abstract class HierarchyMultiMap { bs.add(b); cache = null; } - + + /** + * Appends the contents of the specified map to this map. + * + * @param from the map to append contents from + */ + public void append(HierarchyMultiMap from) { + from.appendTo(this); + } + + /** + * Appends the contents of this map to the specified map. + * + * @param to the map to append to + */ + public void appendTo(HierarchyMultiMap to) { + map.forEachEntry((a, bl) -> { + bl.forEach(b -> to.put(a, b)); + return true; + }); + } + /** * Gets the values stored into the map for the key {@code a} or * its superelements. diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/SessionGarbageCollection.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/SessionGarbageCollection.java index c168e2286..c48ec3630 100644 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/SessionGarbageCollection.java +++ b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/SessionGarbageCollection.java @@ -76,7 +76,18 @@ public class SessionGarbageCollection { QueryControl qc = graph.getService(QueryControl.class); ClusterControl cc = graph.getService(ClusterControl.class); _monitor.beginTask("Collect clusters", IProgressMonitor.UNKNOWN); - cc.gc(graph, clusterTarget); + //cc.gc(graph, clusterTarget); + if(clusterTarget == -1) { + int used = cc.used(); + //System.err.println("session gc, cluster use = " + used); + int baseline = 32*(1<<20); + if(used > baseline) { + int dynamicTarget = (int)(0.95 * (used-baseline)) + baseline; + cc.gc(graph, dynamicTarget); + } + } else { + cc.gc(graph, clusterTarget); + } _monitor.beginTask("Collect queries", IProgressMonitor.UNKNOWN); qc.gc(graph, allowedTimeInMs); } diff --git a/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/ClusterTable.java b/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/ClusterTable.java index 7df13315f..a52870a15 100644 --- a/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/ClusterTable.java +++ b/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/ClusterTable.java @@ -524,10 +524,11 @@ public final class ClusterTable implements IClusterTable { ArrayList toRelease = new ArrayList(); for (CollectorCluster cluster : support.getResidentClusters()) { - target -= support.getClusterSize(cluster); - if (target > 0) { - toRelease.add(cluster); - } else { + toRelease.add(cluster); + long clusterSize = support.getClusterSize(cluster); + //System.err.println("release cluster with " + (clusterSize/1024) + " kiB - " + cluster); + target -= clusterSize; + if (target <= 0) { break; } } diff --git a/bundles/org.simantics.db.services/src/org/simantics/db/services/adaption/AdapterRegistry2.java b/bundles/org.simantics.db.services/src/org/simantics/db/services/adaption/AdapterRegistry2.java index 0e52b8a41..478d3ebee 100644 --- a/bundles/org.simantics.db.services/src/org/simantics/db/services/adaption/AdapterRegistry2.java +++ b/bundles/org.simantics.db.services/src/org/simantics/db/services/adaption/AdapterRegistry2.java @@ -237,9 +237,9 @@ public class AdapterRegistry2 { NamedNodeMap attr = n.getAttributes(); IDynamicAdapter2 da = null; if(n.getNodeName().equals("this")) - da = new ThisResource2(); + da = ThisResource2.INSTANCE; else if(n.getNodeName().equals("graph")) - da = new GraphObject2(); + da = GraphObject2.INSTANCE; else if(n.getNodeName().equals("bundle")) { String bundleId = null; Node fc = n.getFirstChild(); diff --git a/bundles/org.simantics.db.services/src/org/simantics/db/services/adaption/reflection/GraphObject2.java b/bundles/org.simantics.db.services/src/org/simantics/db/services/adaption/reflection/GraphObject2.java index e8f171b8c..8a2db578f 100644 --- a/bundles/org.simantics.db.services/src/org/simantics/db/services/adaption/reflection/GraphObject2.java +++ b/bundles/org.simantics.db.services/src/org/simantics/db/services/adaption/reflection/GraphObject2.java @@ -15,6 +15,8 @@ import org.simantics.db.ReadGraph; public class GraphObject2 implements IDynamicAdapter2 { + public static final IDynamicAdapter2 INSTANCE = new GraphObject2(); + @Override public Class getType() { return ReadGraph.class; diff --git a/bundles/org.simantics.db.services/src/org/simantics/db/services/adaption/reflection/ThisResource2.java b/bundles/org.simantics.db.services/src/org/simantics/db/services/adaption/reflection/ThisResource2.java index 24001ee0a..bacf3adee 100644 --- a/bundles/org.simantics.db.services/src/org/simantics/db/services/adaption/reflection/ThisResource2.java +++ b/bundles/org.simantics.db.services/src/org/simantics/db/services/adaption/reflection/ThisResource2.java @@ -16,6 +16,8 @@ import org.simantics.db.ReadGraph; public class ThisResource2 implements IDynamicAdapter2 { + public static IDynamicAdapter2 INSTANCE = new ThisResource2(); + @Override public Class getType() { return Resource.class;