Utilities for listing of statements in DB 25/1925/4
authorAntti Villberg <antti.villberg@semantum.fi>
Tue, 17 Jul 2018 09:49:35 +0000 (12:49 +0300)
committerTuukka Lehtonen <tuukka.lehtonen@semantum.fi>
Fri, 20 Jul 2018 06:21:02 +0000 (06:21 +0000)
gitlab #42

Change-Id: I5d79c2542cffaff56cb6e6405c466e030115c7c3

bundles/org.simantics.db.common/src/org/simantics/db/common/procedure/adapter/DirectStatementProcedure.java [new file with mode: 0644]
bundles/org.simantics.db.common/src/org/simantics/db/common/utils/CommonDBUtils.java
bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/DirectStatementsImpl.java
bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/SerialisationSupportImpl.java
bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/XSupportImpl.java
bundles/org.simantics.db.procore/src/org/simantics/db/procore/cluster/ClusterSmall.java
bundles/org.simantics.db/src/org/simantics/db/DirectStatements.java
bundles/org.simantics.db/src/org/simantics/db/service/XSupport.java
bundles/org.simantics.scl.db/scl/Simantics/DB.scl
bundles/org.simantics.scl.db/scl/Simantics/DBSupport.scl

diff --git a/bundles/org.simantics.db.common/src/org/simantics/db/common/procedure/adapter/DirectStatementProcedure.java b/bundles/org.simantics.db.common/src/org/simantics/db/common/procedure/adapter/DirectStatementProcedure.java
new file mode 100644 (file)
index 0000000..398b185
--- /dev/null
@@ -0,0 +1,32 @@
+package org.simantics.db.common.procedure.adapter;
+
+import org.simantics.db.AsyncReadGraph;
+import org.simantics.db.DirectStatements;
+import org.simantics.db.exception.DatabaseException;
+import org.simantics.db.procedure.AsyncProcedure;
+
+public class DirectStatementProcedure implements AsyncProcedure<DirectStatements> {
+
+       DirectStatements result = null;
+       DatabaseException exception = null;
+
+       @Override
+       public void execute(AsyncReadGraph graph, final DirectStatements ds) {
+               result = ds;
+       }
+
+       @Override
+       public void exception(AsyncReadGraph graph, Throwable throwable) {
+               if(throwable instanceof DatabaseException) {
+                       exception = (DatabaseException)throwable;
+               } else {
+                       exception = new DatabaseException(throwable);
+               }
+       }
+       
+       public DirectStatements getOrThrow() throws DatabaseException {
+               if(exception != null) throw exception;
+               return result;
+       }
+
+}
\ No newline at end of file
index ed09abb54cf7266afd95d6c1dfd158164bf80b24..2e16b284903997a204f1c229f3a1adc83df10a96 100644 (file)
@@ -13,17 +13,31 @@ import org.simantics.db.ReadGraph;
 import org.simantics.db.Resource;
 import org.simantics.db.Statement;
 import org.simantics.db.WriteGraph;
+import org.simantics.db.common.procedure.adapter.DirectStatementProcedure;
 import org.simantics.db.common.request.IsParent;
 import org.simantics.db.common.request.ObjectsWithType;
 import org.simantics.db.common.request.PossibleObjectWithType;
 import org.simantics.db.common.request.PossibleOwner;
 import org.simantics.db.exception.DatabaseException;
+import org.simantics.db.exception.InvalidResourceReferenceException;
+import org.simantics.db.service.ClusterUID;
 import org.simantics.db.service.ClusteringSupport;
+import org.simantics.db.service.DirectQuerySupport;
+import org.simantics.db.service.SerialisationSupport;
+import org.simantics.db.service.XSupport;
 import org.simantics.layer0.Layer0;
 import org.simantics.utils.datastructures.collections.CollectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import gnu.trove.list.array.TIntArrayList;
+import gnu.trove.procedure.TIntProcedure;
+import gnu.trove.set.hash.TIntHashSet;
 
 public class CommonDBUtils {
 
+       private static final Logger LOGGER = LoggerFactory.getLogger(CommonDBUtils.class);
+
        public static boolean isParent(ReadGraph graph, Resource possibleParent, Resource possibleChild) throws DatabaseException {
                return graph.sync(new IsParent(possibleParent, possibleChild));
        }
@@ -193,4 +207,125 @@ public class CommonDBUtils {
        return graph.syncRequest(new PossibleObjectWithType(subject, relation, type));
     }
 
+       public static List<ClusterUID> listClusters(ReadGraph graph) throws DatabaseException {
+       XSupport xs = graph.getService(XSupport.class);
+       ClusterUID uids[] = xs.listClusters();
+       ArrayList<ClusterUID> result = new ArrayList<>(uids.length);
+       for(ClusterUID uid : uids) result.add(uid);
+       return result;
+    }
+       
+       public static List<Resource> resourcesByCluster(ReadGraph graph, ClusterUID uid) throws DatabaseException {
+       SerialisationSupport ss = graph.getService(SerialisationSupport.class);
+       ArrayList<Resource> result = new ArrayList<Resource>();
+       // Index 0 is illegal
+               for(int i=1;i<1<<12;i++) {
+                       try {
+                               result.add(ss.getResource(uid.toRID(i)));
+                       } catch (InvalidResourceReferenceException e) {
+                       }
+               }
+               return result;
+       }
+       
+       public static List<Statement> directStatements(ReadGraph graph, Resource resource, boolean ignoreVirtual) throws DatabaseException {
+               
+               DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class);
+               DirectStatementProcedure proc = new DirectStatementProcedure();
+
+               if (ignoreVirtual) {
+                       dqs.forEachDirectPersistentStatement(graph, resource, proc);
+               } else {
+                       dqs.forEachDirectStatement(graph, resource, proc);
+               }
+               
+               return proc.getOrThrow();
+
+       }
+       
+    public static List<Resource> garbageResources(ReadGraph graph) throws DatabaseException {
+       
+       SerialisationSupport ss = graph.getService(SerialisationSupport.class);
+       
+       TIntArrayList refs = new TIntArrayList();
+       TIntArrayList res = new TIntArrayList();
+       
+       // Find all statements in the database
+       for(ClusterUID uid : listClusters(graph)) {
+               for(Resource r : resourcesByCluster(graph, uid)) {
+                       int sid = ss.getTransientId(r);
+                       for(Statement stm : directStatements(graph, r, true)) {
+                               int oid = ss.getTransientId(stm.getObject());
+                               refs.add(sid);
+                               refs.add(oid);
+                       }
+                       res.add(sid);
+               }
+       }
+
+       TIntHashSet reached = new TIntHashSet();
+       
+       // Initialize root
+       int root = ss.getTransientId(graph.getRootLibrary());
+       reached.add(root);
+       
+       int[] refArray = refs.toArray();
+       
+       boolean changes = true;
+       
+       while(changes) {
+               changes = false;
+               for(int i=0;i<refArray.length;i+=2) {
+                       int s = refArray[i];
+                       int o = refArray[i+1];
+                       if(reached.contains(s)) {
+                               if(reached.add(o)) {
+                                       changes = true;
+                               }
+                       }
+               }
+               
+               System.err.println("Reachability iteration, changes = " + changes);
+       }
+       
+       ArrayList<Resource> result = new ArrayList<>();
+               for(int i=0;i<refArray.length;i+=2) {
+                       int s = refArray[i];
+                       if(reached.contains(s)) {
+                               if(reached.add(refArray[i+1]))
+                                       changes = true;
+                       }
+               }
+
+               res.forEach(new TIntProcedure() {
+                       
+                       @Override
+                       public boolean execute(int r) {
+                               if(!reached.contains(r)) {
+                                       try {
+                                               result.add(ss.getResource(r));
+                                       } catch (DatabaseException e) {
+                                               LOGGER.error("Unexpected error while resolving garbage resources.", e);
+                                       }
+                               }
+                               return true;
+                       }
+                       
+               });
+               
+       return result;
+       
+    }
+
+    public static ClusterUID clusterUIDOfResource(ReadGraph graph, Resource resource) throws DatabaseException {
+       SerialisationSupport ss = graph.getService(SerialisationSupport.class);
+       return ss.getUID(resource).asCID();
+    }
+    
+    public static boolean isClusterLoaded(ReadGraph graph, ClusterUID clusterUID) throws DatabaseException {
+       XSupport xs = graph.getService(XSupport.class);
+       return xs.isClusterLoaded(clusterUID);
+    }
+    
+    
 }
index 10511979176a64c21dbd90c1c2ab69f9176fba0a..18031603f20ee8464f7829e4d6f92ebd274dbf18 100644 (file)
@@ -15,6 +15,8 @@ import gnu.trove.list.array.TIntArrayList;
 
 import java.util.Collection;
 import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
 
 import org.simantics.db.DirectStatements;
 import org.simantics.db.Resource;
@@ -25,21 +27,19 @@ import org.simantics.db.impl.support.ResourceSupport;
 final public class DirectStatementsImpl implements DirectStatements {
 
     final ResourceSupport support;
-
-    final int             subject;
-
-    final TIntArrayList   statements = new TIntArrayList();
-
+    final int subject;
+    final TIntArrayList statements = new TIntArrayList();
+    
     DirectStatementsImpl(ResourceSupport support, int subject) {
         this.support = support;
         this.subject = subject;
     }
-
+    
     void addStatement(int p, int o) {
         statements.add(p);
         statements.add(o);
     }
-
+    
     @Override
     public Resource getSubject() {
         return new ResourceImpl(support, subject);
@@ -62,14 +62,13 @@ final public class DirectStatementsImpl implements DirectStatements {
 
     @Override
     public boolean contains(Object arg0) {
-        Statement stm = (Statement) arg0;
-        ResourceImpl p = (ResourceImpl) stm.getPredicate();
-        ResourceImpl o = (ResourceImpl) stm.getObject();
-        for (int i = 0; i < statements.size(); i += 2) {
+        Statement stm = (Statement)arg0;
+        ResourceImpl p = (ResourceImpl)stm.getPredicate();
+        ResourceImpl o = (ResourceImpl)stm.getObject();
+        for(int i=0;i<statements.size();i+=2) {
             int pi = statements.getQuick(i);
-            int oi = statements.getQuick(i + 1);
-            if (p.id == pi && o.id == oi)
-                return true;
+            int oi = statements.getQuick(i+1);
+            if(p.id == pi && o.id == oi) return true;
         }
         return false;
     }
@@ -89,25 +88,23 @@ final public class DirectStatementsImpl implements DirectStatements {
         return new Iterator<Statement>() {
 
             int index = 0;
-
-            int max   = statements.size();
-
+            int max = statements.size();
+            
             @Override
             public boolean hasNext() {
-                return index < max;
+                return index < max; 
             }
 
             @Override
             public Statement next() {
-                return new DirectStatementImpl(support, subject, statements.getQuick(index++),
-                        statements.getQuick(index++));
+                return new DirectStatementImpl(support, subject, statements.getQuick(index++), statements.getQuick(index++));
             }
 
             @Override
             public void remove() {
                 throw new Error("Not supported");
             }
-
+            
         };
     }
 
@@ -141,4 +138,55 @@ final public class DirectStatementsImpl implements DirectStatements {
         throw new Error("Not supported");
     }
 
+    @Override
+    public void add(int index, Statement element) {
+        throw new Error("Not supported");
+    }
+
+    @Override
+    public boolean addAll(int index, Collection<? extends Statement> c) {
+        throw new Error("Not supported");
+    }
+
+    @Override
+    public Statement get(int index_) {
+        int index = 2*index_;
+        return new DirectStatementImpl(support, subject, statements.getQuick(index), statements.getQuick(index+1));
+    }
+
+    @Override
+    public int indexOf(Object o) {
+        throw new Error("Not supported");
+    }
+
+    @Override
+    public int lastIndexOf(Object o) {
+        throw new Error("Not supported");
+    }
+
+    @Override
+    public ListIterator<Statement> listIterator() {
+        throw new Error("Not supported");
+    }
+
+    @Override
+    public ListIterator<Statement> listIterator(int index) {
+        throw new Error("Not supported");
+    }
+
+    @Override
+    public Statement remove(int index) {
+        throw new Error("Not supported");
+    }
+
+    @Override
+    public Statement set(int index, Statement element) {
+        throw new Error("Not supported");
+    }
+
+    @Override
+    public List<Statement> subList(int fromIndex, int toIndex) {
+        throw new Error("Not supported");
+    }
+    
 }
index e0e871bb9ad154ae3a6ba93a274b3c86dd9e319b..0eaf0ecf0ef7bff56f9851b0fbca774f24c9bcb4 100644 (file)
@@ -4,7 +4,9 @@ import org.simantics.db.Resource;
 import org.simantics.db.ResourceSerializer;
 import org.simantics.db.common.utils.Logger;
 import org.simantics.db.exception.DatabaseException;
+import org.simantics.db.exception.InvalidResourceReferenceException;
 import org.simantics.db.impl.ClusterI;
+import org.simantics.db.impl.ClusterI.PredicateProcedure;
 import org.simantics.db.impl.ResourceImpl;
 import org.simantics.db.procore.cluster.ClusterImpl;
 import org.simantics.db.procore.cluster.ClusterTraits;
@@ -73,11 +75,32 @@ public class SerialisationSupportImpl implements SerialisationSupport {
                return cluster.clusterUID.toRID(ClusterTraits.getResourceIndexFromResourceKey(resourceKey));
        }
 
+       static class ExistsPredicateProcedure implements PredicateProcedure<Integer> {
+               
+               boolean exists = false;
+
+               @Override
+               public boolean execute(Integer c, final int predicateKey, int objectIndex) {
+                       exists = true;
+                       return false;
+               }
+
+       }
+       
        @Override
        public Resource getResource(ResourceUID uid) throws DatabaseException {
         ClusterI cluster = session.clusterTable.getClusterByClusterUIDOrMakeProxy(uid.asCID());
         int key = ClusterTraits.createResourceKey(cluster.getClusterKey(), (int) uid.getIndex());
-        return new ResourceImpl(session.resourceSupport, key);
+        if (cluster.hasResource(key, session.clusterTranslator)) {
+               ExistsPredicateProcedure pp = new ExistsPredicateProcedure();
+                       cluster.forPredicates(key, pp, 0, session.clusterTranslator);
+                       if(pp.exists) {
+                               return new ResourceImpl(session.resourceSupport, key);
+                       } else if (cluster.hasValue(key, session.clusterTranslator)) {
+                               return new ResourceImpl(session.resourceSupport, key);
+                       }
+        }
+        throw new InvalidResourceReferenceException("Resource with uid = " + uid + " does not exist.");
        }
     
 }
index 4ac08fc0b8a25ff02f1596f402e8c669d1634726..05870668616b258e648634e5ac4383a88f8277f1 100644 (file)
@@ -169,4 +169,11 @@ public class XSupportImpl implements XSupport {
     public boolean rolledback() {
         return session.graphSession.rolledback();
     }
+
+       @Override
+       public boolean isClusterLoaded(ClusterUID clusterUID) throws DatabaseException {
+               ClusterImpl clusterImpl = session.clusterTable.getClusterByClusterUIDOrMakeProxy(clusterUID);
+               return clusterImpl.isLoaded();
+       }
+       
 }
index ead0099847e70aacf5914535b74600c4d84dbb06..c8309c6bf5303a2909564de1a2f29637a8a34534 100644 (file)
@@ -33,6 +33,8 @@ import org.simantics.db.procedure.AsyncContextMultiProcedure;
 import org.simantics.db.procedure.AsyncMultiProcedure;
 import org.simantics.db.service.ClusterUID;
 import org.simantics.db.service.ResourceUID;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import fi.vtt.simantics.procore.DebugPolicy;
 import fi.vtt.simantics.procore.internal.ClusterChange;
@@ -43,6 +45,8 @@ import gnu.trove.map.hash.TIntShortHashMap;
 import gnu.trove.set.hash.TIntHashSet;
 
 final public class ClusterSmall extends ClusterImpl {
+       
+       private static final Logger LOGGER = LoggerFactory.getLogger(ClusterSmall.class);
     
     private static final int TABLE_HEADER_SIZE = TableHeader.HEADER_SIZE + TableHeader.EXTRA_SIZE;
     private static final int RESOURCE_TABLE_OFFSET = 0;
@@ -522,6 +526,11 @@ final public class ClusterSmall extends ClusterImpl {
         if (DEBUG)
             System.out.println("ClusterSmall.forPredicates: rk=" + resourceKey );
         if (deleted) return false;
+        if(proxy) {
+            ClusterImpl cluster = clusterTable.load2(clusterId, clusterKey);
+            return cluster.forPredicates(resourceKey, procedure, context, support);
+        }
+
         final int resourceIndex = getLocalReference(resourceKey);
         final int predicateIndex = resourceTable.getPredicateIndex(resourceIndex);
         if (0 == predicateIndex)
@@ -652,6 +661,10 @@ final public class ClusterSmall extends ClusterImpl {
     public boolean hasValue(int resourceKey, ClusterSupport support)
     throws DatabaseException {
         if (deleted) return false;
+        if(proxy) {
+            ClusterImpl cluster = clusterTable.load2(clusterId, clusterKey);
+            return cluster.hasValue(resourceKey, support);
+        }
         int resourceIndex = getLocalReference(resourceKey);
         return resourceTable.hasValue(resourceIndex);
     }
@@ -739,6 +752,15 @@ final public class ClusterSmall extends ClusterImpl {
     }
     @Override
     public boolean hasResource(int resourceKey, ClusterSupport support) {
+        if(proxy) {
+                       try {
+                               ClusterImpl cluster = clusterTable.load2(clusterId, clusterKey);
+                   return cluster.hasResource(resourceKey, support);
+                       } catch (DatabaseException e) {
+                               LOGGER.error("Failed to load cluster with clusterId " + clusterId);
+                               return false;
+                       }
+        }
         if (deleted) return false;
         int clusterKey = ClusterTraitsBase.getClusterKeyFromResourceKeyNoThrow(resourceKey);
         if (this.clusterKey != clusterKey) // foreign resource
@@ -935,13 +957,21 @@ final public class ClusterSmall extends ClusterImpl {
     @Override
     public void load() {
         if (deleted) return;
-        throw new Error("Not supported.");
+        try {
+               clusterTable.load2(clusterId, clusterKey);
+        } catch (DatabaseException e) {
+               LOGGER.error("Failed to load cluster with clusterId " + clusterId, e);
+        }
     }
 
     @Override
     public void load(Consumer<DatabaseException> r) {
         if (deleted) return;
-        throw new Error("Not supported.");
+        try {
+               clusterTable.load2(clusterId, clusterKey);
+        } catch (DatabaseException e) {
+               r.accept(e);
+        }
     }
 
     public boolean contains(int resourceKey) {
@@ -955,7 +985,7 @@ final public class ClusterSmall extends ClusterImpl {
             clusterTable.load2(clusterId, clusterKey);
             callback.run();
         } catch (DatabaseException e) {
-            e.printStackTrace();
+               LOGGER.error("Failed to load cluster with clusterId " + clusterId, e);
         }
 
     }
index d4b4db363b6e8445d5f0227bfcb94c9d8df59b87..4f4b6c5d841df975e4e6a43f20ec3fc72c217ce6 100644 (file)
  *******************************************************************************/
 package org.simantics.db;
 
-import java.util.Collection;
+import java.util.List;
 
 /**
  * @author Antti Villberg
  */
-public interface DirectStatements extends Collection<Statement> {
+public interface DirectStatements extends List<Statement> {
 
     Resource getSubject();
 
index 8e08147c5b34a2c61af386c2154a0d5238f20707..cf9e96e7833421fb7617cc9ebeaef422a9dd3947 100644 (file)
@@ -161,4 +161,6 @@ public interface XSupport {
 
     public boolean rolledback();
     
+    public boolean isClusterLoaded(ClusterUID clusterUID) throws DatabaseException;
+    
 }
index a097dc8ae64320f27fd169cf32e2c9b557610a74..a400659ee0c940b18b129fadf0f9a572e1ff643d 100644 (file)
@@ -151,6 +151,8 @@ importJava "org.simantics.db.ReadGraph" where
     
     @JavaName getPredicates
     predicatesOf :: Resource -> <ReadGraph> [Resource]
+    
+    isImmutable :: Resource -> <ReadGraph> Boolean
 
 importJava "org.simantics.db.layer0.util.ExtendedUris" where
     "Converts an absolute URI to a resource"
index ee61b88b1e39a9be685b19731a12141161b9f47a..f6d1453f08f484c6fbde645cd56177e4cd5ace25 100644 (file)
@@ -1,6 +1,15 @@
 import "UI/Progress"
 import "Simantics/DB"
 
+importJava "org.simantics.db.common.utils.CommonDBUtils" where
+
+    listClusters :: <ReadGraph> [ClusterUID]
+    resourcesByCluster :: ClusterUID -> <ReadGraph> [Resource]
+    directStatements :: Resource -> Boolean -> <ReadGraph> [Statement]
+    garbageResources :: <ReadGraph> [Resource]
+    clusterUIDOfResource :: Resource -> <ReadGraph> ClusterUID
+    isClusterLoaded :: ClusterUID -> <ReadGraph> Boolean
+
 importJava "org.simantics.scl.db.SCLFunctions" where
     collectClusters :: <Proc> ()
     countQueries :: <ReadGraph> Integer
@@ -18,3 +27,6 @@ importJava "org.simantics.db.layer0.util.Layer0Utils" where
 
 //emptyTrashBin :: <Proc> ()
 emptyTrashBin = emptyTrashBinWithMonitor $ createNullProgressMonitor ()
+
+importJava "org.simantics.db.service.ClusterUID" where
+    data ClusterUID