From bfdceb3b5b0e4967358277cd506f237c9ee364c3 Mon Sep 17 00:00:00 2001 From: Antti Villberg Date: Tue, 17 Jul 2018 12:49:35 +0300 Subject: [PATCH] Utilities for listing of statements in DB gitlab #42 Change-Id: I5d79c2542cffaff56cb6e6405c466e030115c7c3 --- .../adapter/DirectStatementProcedure.java | 32 +++++ .../db/common/utils/CommonDBUtils.java | 135 ++++++++++++++++++ .../internal/DirectStatementsImpl.java | 90 +++++++++--- .../internal/SerialisationSupportImpl.java | 25 +++- .../procore/internal/XSupportImpl.java | 7 + .../db/procore/cluster/ClusterSmall.java | 36 ++++- .../org/simantics/db/DirectStatements.java | 4 +- .../org/simantics/db/service/XSupport.java | 2 + .../org.simantics.scl.db/scl/Simantics/DB.scl | 2 + .../scl/Simantics/DBSupport.scl | 12 ++ 10 files changed, 318 insertions(+), 27 deletions(-) create mode 100644 bundles/org.simantics.db.common/src/org/simantics/db/common/procedure/adapter/DirectStatementProcedure.java diff --git a/bundles/org.simantics.db.common/src/org/simantics/db/common/procedure/adapter/DirectStatementProcedure.java b/bundles/org.simantics.db.common/src/org/simantics/db/common/procedure/adapter/DirectStatementProcedure.java new file mode 100644 index 000000000..398b1851a --- /dev/null +++ b/bundles/org.simantics.db.common/src/org/simantics/db/common/procedure/adapter/DirectStatementProcedure.java @@ -0,0 +1,32 @@ +package org.simantics.db.common.procedure.adapter; + +import org.simantics.db.AsyncReadGraph; +import org.simantics.db.DirectStatements; +import org.simantics.db.exception.DatabaseException; +import org.simantics.db.procedure.AsyncProcedure; + +public class DirectStatementProcedure implements AsyncProcedure { + + DirectStatements result = null; + DatabaseException exception = null; + + @Override + public void execute(AsyncReadGraph graph, final DirectStatements ds) { + result = ds; + } + + @Override + public void exception(AsyncReadGraph graph, Throwable throwable) { + if(throwable instanceof DatabaseException) { + exception = (DatabaseException)throwable; + } else { + exception = new DatabaseException(throwable); + } + } + + public DirectStatements getOrThrow() throws DatabaseException { + if(exception != null) throw exception; + return result; + } + +} \ No newline at end of file diff --git a/bundles/org.simantics.db.common/src/org/simantics/db/common/utils/CommonDBUtils.java b/bundles/org.simantics.db.common/src/org/simantics/db/common/utils/CommonDBUtils.java index ed09abb54..2e16b2849 100644 --- a/bundles/org.simantics.db.common/src/org/simantics/db/common/utils/CommonDBUtils.java +++ b/bundles/org.simantics.db.common/src/org/simantics/db/common/utils/CommonDBUtils.java @@ -13,17 +13,31 @@ import org.simantics.db.ReadGraph; import org.simantics.db.Resource; import org.simantics.db.Statement; import org.simantics.db.WriteGraph; +import org.simantics.db.common.procedure.adapter.DirectStatementProcedure; import org.simantics.db.common.request.IsParent; import org.simantics.db.common.request.ObjectsWithType; import org.simantics.db.common.request.PossibleObjectWithType; import org.simantics.db.common.request.PossibleOwner; import org.simantics.db.exception.DatabaseException; +import org.simantics.db.exception.InvalidResourceReferenceException; +import org.simantics.db.service.ClusterUID; import org.simantics.db.service.ClusteringSupport; +import org.simantics.db.service.DirectQuerySupport; +import org.simantics.db.service.SerialisationSupport; +import org.simantics.db.service.XSupport; import org.simantics.layer0.Layer0; import org.simantics.utils.datastructures.collections.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import gnu.trove.list.array.TIntArrayList; +import gnu.trove.procedure.TIntProcedure; +import gnu.trove.set.hash.TIntHashSet; public class CommonDBUtils { + private static final Logger LOGGER = LoggerFactory.getLogger(CommonDBUtils.class); + public static boolean isParent(ReadGraph graph, Resource possibleParent, Resource possibleChild) throws DatabaseException { return graph.sync(new IsParent(possibleParent, possibleChild)); } @@ -193,4 +207,125 @@ public class CommonDBUtils { return graph.syncRequest(new PossibleObjectWithType(subject, relation, type)); } + public static List listClusters(ReadGraph graph) throws DatabaseException { + XSupport xs = graph.getService(XSupport.class); + ClusterUID uids[] = xs.listClusters(); + ArrayList result = new ArrayList<>(uids.length); + for(ClusterUID uid : uids) result.add(uid); + return result; + } + + public static List resourcesByCluster(ReadGraph graph, ClusterUID uid) throws DatabaseException { + SerialisationSupport ss = graph.getService(SerialisationSupport.class); + ArrayList result = new ArrayList(); + // Index 0 is illegal + for(int i=1;i<1<<12;i++) { + try { + result.add(ss.getResource(uid.toRID(i))); + } catch (InvalidResourceReferenceException e) { + } + } + return result; + } + + public static List directStatements(ReadGraph graph, Resource resource, boolean ignoreVirtual) throws DatabaseException { + + DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class); + DirectStatementProcedure proc = new DirectStatementProcedure(); + + if (ignoreVirtual) { + dqs.forEachDirectPersistentStatement(graph, resource, proc); + } else { + dqs.forEachDirectStatement(graph, resource, proc); + } + + return proc.getOrThrow(); + + } + + public static List garbageResources(ReadGraph graph) throws DatabaseException { + + SerialisationSupport ss = graph.getService(SerialisationSupport.class); + + TIntArrayList refs = new TIntArrayList(); + TIntArrayList res = new TIntArrayList(); + + // Find all statements in the database + for(ClusterUID uid : listClusters(graph)) { + for(Resource r : resourcesByCluster(graph, uid)) { + int sid = ss.getTransientId(r); + for(Statement stm : directStatements(graph, r, true)) { + int oid = ss.getTransientId(stm.getObject()); + refs.add(sid); + refs.add(oid); + } + res.add(sid); + } + } + + TIntHashSet reached = new TIntHashSet(); + + // Initialize root + int root = ss.getTransientId(graph.getRootLibrary()); + reached.add(root); + + int[] refArray = refs.toArray(); + + boolean changes = true; + + while(changes) { + changes = false; + for(int i=0;i() { int index = 0; - - int max = statements.size(); - + int max = statements.size(); + @Override public boolean hasNext() { - return index < max; + return index < max; } @Override public Statement next() { - return new DirectStatementImpl(support, subject, statements.getQuick(index++), - statements.getQuick(index++)); + return new DirectStatementImpl(support, subject, statements.getQuick(index++), statements.getQuick(index++)); } @Override public void remove() { throw new Error("Not supported"); } - + }; } @@ -141,4 +138,55 @@ final public class DirectStatementsImpl implements DirectStatements { throw new Error("Not supported"); } + @Override + public void add(int index, Statement element) { + throw new Error("Not supported"); + } + + @Override + public boolean addAll(int index, Collection c) { + throw new Error("Not supported"); + } + + @Override + public Statement get(int index_) { + int index = 2*index_; + return new DirectStatementImpl(support, subject, statements.getQuick(index), statements.getQuick(index+1)); + } + + @Override + public int indexOf(Object o) { + throw new Error("Not supported"); + } + + @Override + public int lastIndexOf(Object o) { + throw new Error("Not supported"); + } + + @Override + public ListIterator listIterator() { + throw new Error("Not supported"); + } + + @Override + public ListIterator listIterator(int index) { + throw new Error("Not supported"); + } + + @Override + public Statement remove(int index) { + throw new Error("Not supported"); + } + + @Override + public Statement set(int index, Statement element) { + throw new Error("Not supported"); + } + + @Override + public List subList(int fromIndex, int toIndex) { + throw new Error("Not supported"); + } + } diff --git a/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/SerialisationSupportImpl.java b/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/SerialisationSupportImpl.java index e0e871bb9..0eaf0ecf0 100644 --- a/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/SerialisationSupportImpl.java +++ b/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/SerialisationSupportImpl.java @@ -4,7 +4,9 @@ import org.simantics.db.Resource; import org.simantics.db.ResourceSerializer; import org.simantics.db.common.utils.Logger; import org.simantics.db.exception.DatabaseException; +import org.simantics.db.exception.InvalidResourceReferenceException; import org.simantics.db.impl.ClusterI; +import org.simantics.db.impl.ClusterI.PredicateProcedure; import org.simantics.db.impl.ResourceImpl; import org.simantics.db.procore.cluster.ClusterImpl; import org.simantics.db.procore.cluster.ClusterTraits; @@ -73,11 +75,32 @@ public class SerialisationSupportImpl implements SerialisationSupport { return cluster.clusterUID.toRID(ClusterTraits.getResourceIndexFromResourceKey(resourceKey)); } + static class ExistsPredicateProcedure implements PredicateProcedure { + + boolean exists = false; + + @Override + public boolean execute(Integer c, final int predicateKey, int objectIndex) { + exists = true; + return false; + } + + } + @Override public Resource getResource(ResourceUID uid) throws DatabaseException { ClusterI cluster = session.clusterTable.getClusterByClusterUIDOrMakeProxy(uid.asCID()); int key = ClusterTraits.createResourceKey(cluster.getClusterKey(), (int) uid.getIndex()); - return new ResourceImpl(session.resourceSupport, key); + if (cluster.hasResource(key, session.clusterTranslator)) { + ExistsPredicateProcedure pp = new ExistsPredicateProcedure(); + cluster.forPredicates(key, pp, 0, session.clusterTranslator); + if(pp.exists) { + return new ResourceImpl(session.resourceSupport, key); + } else if (cluster.hasValue(key, session.clusterTranslator)) { + return new ResourceImpl(session.resourceSupport, key); + } + } + throw new InvalidResourceReferenceException("Resource with uid = " + uid + " does not exist."); } } diff --git a/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/XSupportImpl.java b/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/XSupportImpl.java index 4ac08fc0b..058706686 100644 --- a/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/XSupportImpl.java +++ b/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/XSupportImpl.java @@ -169,4 +169,11 @@ public class XSupportImpl implements XSupport { public boolean rolledback() { return session.graphSession.rolledback(); } + + @Override + public boolean isClusterLoaded(ClusterUID clusterUID) throws DatabaseException { + ClusterImpl clusterImpl = session.clusterTable.getClusterByClusterUIDOrMakeProxy(clusterUID); + return clusterImpl.isLoaded(); + } + } diff --git a/bundles/org.simantics.db.procore/src/org/simantics/db/procore/cluster/ClusterSmall.java b/bundles/org.simantics.db.procore/src/org/simantics/db/procore/cluster/ClusterSmall.java index ead009984..c8309c6bf 100644 --- a/bundles/org.simantics.db.procore/src/org/simantics/db/procore/cluster/ClusterSmall.java +++ b/bundles/org.simantics.db.procore/src/org/simantics/db/procore/cluster/ClusterSmall.java @@ -33,6 +33,8 @@ import org.simantics.db.procedure.AsyncContextMultiProcedure; import org.simantics.db.procedure.AsyncMultiProcedure; import org.simantics.db.service.ClusterUID; import org.simantics.db.service.ResourceUID; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import fi.vtt.simantics.procore.DebugPolicy; import fi.vtt.simantics.procore.internal.ClusterChange; @@ -43,6 +45,8 @@ import gnu.trove.map.hash.TIntShortHashMap; import gnu.trove.set.hash.TIntHashSet; final public class ClusterSmall extends ClusterImpl { + + private static final Logger LOGGER = LoggerFactory.getLogger(ClusterSmall.class); private static final int TABLE_HEADER_SIZE = TableHeader.HEADER_SIZE + TableHeader.EXTRA_SIZE; private static final int RESOURCE_TABLE_OFFSET = 0; @@ -522,6 +526,11 @@ final public class ClusterSmall extends ClusterImpl { if (DEBUG) System.out.println("ClusterSmall.forPredicates: rk=" + resourceKey ); if (deleted) return false; + if(proxy) { + ClusterImpl cluster = clusterTable.load2(clusterId, clusterKey); + return cluster.forPredicates(resourceKey, procedure, context, support); + } + final int resourceIndex = getLocalReference(resourceKey); final int predicateIndex = resourceTable.getPredicateIndex(resourceIndex); if (0 == predicateIndex) @@ -652,6 +661,10 @@ final public class ClusterSmall extends ClusterImpl { public boolean hasValue(int resourceKey, ClusterSupport support) throws DatabaseException { if (deleted) return false; + if(proxy) { + ClusterImpl cluster = clusterTable.load2(clusterId, clusterKey); + return cluster.hasValue(resourceKey, support); + } int resourceIndex = getLocalReference(resourceKey); return resourceTable.hasValue(resourceIndex); } @@ -739,6 +752,15 @@ final public class ClusterSmall extends ClusterImpl { } @Override public boolean hasResource(int resourceKey, ClusterSupport support) { + if(proxy) { + try { + ClusterImpl cluster = clusterTable.load2(clusterId, clusterKey); + return cluster.hasResource(resourceKey, support); + } catch (DatabaseException e) { + LOGGER.error("Failed to load cluster with clusterId " + clusterId); + return false; + } + } if (deleted) return false; int clusterKey = ClusterTraitsBase.getClusterKeyFromResourceKeyNoThrow(resourceKey); if (this.clusterKey != clusterKey) // foreign resource @@ -935,13 +957,21 @@ final public class ClusterSmall extends ClusterImpl { @Override public void load() { if (deleted) return; - throw new Error("Not supported."); + try { + clusterTable.load2(clusterId, clusterKey); + } catch (DatabaseException e) { + LOGGER.error("Failed to load cluster with clusterId " + clusterId, e); + } } @Override public void load(Consumer r) { if (deleted) return; - throw new Error("Not supported."); + try { + clusterTable.load2(clusterId, clusterKey); + } catch (DatabaseException e) { + r.accept(e); + } } public boolean contains(int resourceKey) { @@ -955,7 +985,7 @@ final public class ClusterSmall extends ClusterImpl { clusterTable.load2(clusterId, clusterKey); callback.run(); } catch (DatabaseException e) { - e.printStackTrace(); + LOGGER.error("Failed to load cluster with clusterId " + clusterId, e); } } diff --git a/bundles/org.simantics.db/src/org/simantics/db/DirectStatements.java b/bundles/org.simantics.db/src/org/simantics/db/DirectStatements.java index d4b4db363..4f4b6c5d8 100644 --- a/bundles/org.simantics.db/src/org/simantics/db/DirectStatements.java +++ b/bundles/org.simantics.db/src/org/simantics/db/DirectStatements.java @@ -11,12 +11,12 @@ *******************************************************************************/ package org.simantics.db; -import java.util.Collection; +import java.util.List; /** * @author Antti Villberg */ -public interface DirectStatements extends Collection { +public interface DirectStatements extends List { Resource getSubject(); diff --git a/bundles/org.simantics.db/src/org/simantics/db/service/XSupport.java b/bundles/org.simantics.db/src/org/simantics/db/service/XSupport.java index 8e08147c5..cf9e96e78 100644 --- a/bundles/org.simantics.db/src/org/simantics/db/service/XSupport.java +++ b/bundles/org.simantics.db/src/org/simantics/db/service/XSupport.java @@ -161,4 +161,6 @@ public interface XSupport { public boolean rolledback(); + public boolean isClusterLoaded(ClusterUID clusterUID) throws DatabaseException; + } diff --git a/bundles/org.simantics.scl.db/scl/Simantics/DB.scl b/bundles/org.simantics.scl.db/scl/Simantics/DB.scl index a097dc8ae..a400659ee 100644 --- a/bundles/org.simantics.scl.db/scl/Simantics/DB.scl +++ b/bundles/org.simantics.scl.db/scl/Simantics/DB.scl @@ -151,6 +151,8 @@ importJava "org.simantics.db.ReadGraph" where @JavaName getPredicates predicatesOf :: Resource -> [Resource] + + isImmutable :: Resource -> Boolean importJava "org.simantics.db.layer0.util.ExtendedUris" where "Converts an absolute URI to a resource" diff --git a/bundles/org.simantics.scl.db/scl/Simantics/DBSupport.scl b/bundles/org.simantics.scl.db/scl/Simantics/DBSupport.scl index ee61b88b1..f6d1453f0 100644 --- a/bundles/org.simantics.scl.db/scl/Simantics/DBSupport.scl +++ b/bundles/org.simantics.scl.db/scl/Simantics/DBSupport.scl @@ -1,6 +1,15 @@ import "UI/Progress" import "Simantics/DB" +importJava "org.simantics.db.common.utils.CommonDBUtils" where + + listClusters :: [ClusterUID] + resourcesByCluster :: ClusterUID -> [Resource] + directStatements :: Resource -> Boolean -> [Statement] + garbageResources :: [Resource] + clusterUIDOfResource :: Resource -> ClusterUID + isClusterLoaded :: ClusterUID -> Boolean + importJava "org.simantics.scl.db.SCLFunctions" where collectClusters :: () countQueries :: Integer @@ -18,3 +27,6 @@ importJava "org.simantics.db.layer0.util.Layer0Utils" where //emptyTrashBin :: () emptyTrashBin = emptyTrashBinWithMonitor $ createNullProgressMonitor () + +importJava "org.simantics.db.service.ClusterUID" where + data ClusterUID -- 2.43.2