X-Git-Url: https://gerrit.simantics.org/r/gitweb?p=simantics%2Fplatform.git;a=blobdiff_plain;f=bundles%2Forg.simantics.db.procore%2Fsrc%2Ffi%2Fvtt%2Fsimantics%2Fprocore%2Finternal%2FClusterTable.java;h=2c65ee8e85088f6f1f051de7b5b851733a345c54;hp=40b2530f2a569a4dedfdb7295095895871074f1d;hb=1dd02a0d0e604a82e67f04559e213db76c9ebac5;hpb=fe1a2f532761669e67da4db4ae15096ced8a04db diff --git a/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/ClusterTable.java b/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/ClusterTable.java index 40b2530f2..2c65ee8e8 100644 --- a/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/ClusterTable.java +++ b/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/ClusterTable.java @@ -18,13 +18,14 @@ import java.util.HashMap; import java.util.Map; import java.util.TreeMap; import java.util.concurrent.Semaphore; +import java.util.function.Consumer; import org.simantics.databoard.Bindings; import org.simantics.db.ClusterCreator; import org.simantics.db.Database; +import org.simantics.db.Database.Session.ClusterChanges; import org.simantics.db.DevelopmentKeys; import org.simantics.db.SessionVariables; -import org.simantics.db.Database.Session.ClusterChanges; import org.simantics.db.common.utils.Logger; import org.simantics.db.exception.ClusterDoesNotExistException; import org.simantics.db.exception.DatabaseException; @@ -46,7 +47,6 @@ import org.simantics.db.service.ClusterCollectorPolicy; import org.simantics.db.service.ClusterCollectorPolicy.CollectorCluster; import org.simantics.db.service.ClusterUID; import org.simantics.utils.Development; -import org.simantics.utils.datastructures.Callback; import fi.vtt.simantics.procore.DebugPolicy; import fi.vtt.simantics.procore.internal.ClusterControlImpl.ClusterStateImpl; @@ -60,6 +60,8 @@ import gnu.trove.set.hash.TIntHashSet; public final class ClusterTable implements IClusterTable { + private static final boolean VALIDATE_SIZE = false; + int maximumBytes = 128 * 1024 * 1024; int limit = (int)(0.8*(double)maximumBytes); @@ -297,10 +299,14 @@ public final class ClusterTable implements IClusterTable { importanceMap.put(cluster.getImportance(), new ImportanceEntry((ClusterImpl)cluster)); if(collectorPolicy != null) collectorPolicy.added((ClusterImpl)cluster); - if (!dirtySizeInBytes && !existing.isLoaded()) { - sizeInBytes += cluster.getCachedSize(); + if (!dirtySizeInBytes) { + if (existing != cluster) { + adjustCachedSize(-existing.getCachedSize(), existing); + } + // This will update sizeInBytes through adjustCachedSize + cluster.getCachedSize(); } - + validateSize(); } synchronized void release(CollectorCluster cluster) { @@ -309,6 +315,8 @@ public final class ClusterTable implements IClusterTable { } synchronized void release(long clusterId) { + //System.out.println("ClusterTable.release(" + clusterId + "): " + sizeInBytes); + //validateSize(); ClusterImpl clusterImpl = clusters.getClusterByClusterId(clusterId); if (null == clusterImpl) return; @@ -321,7 +329,7 @@ public final class ClusterTable implements IClusterTable { if (sessionImpl.writeState != null) sessionImpl.clusterStream.flush(clusterImpl.clusterUID); if (!dirtySizeInBytes) { - sizeInBytes -= clusterImpl.getCachedSize(); + adjustCachedSize(-clusterImpl.getCachedSize(), clusterImpl); } } @@ -529,17 +537,19 @@ public final class ClusterTable implements IClusterTable { ClusterImpl getNewResourceCluster(ClusterSupport cs, GraphSession graphSession, boolean writeOnly) throws DatabaseException { + ClusterImpl result = null; if (Constants.NewClusterId == newResourceClusterId) { newResourceClusterId = graphSession.newClusterId(); - return getClusterByClusterIdOrMake(newResourceClusterId, writeOnly); + result = getClusterByClusterIdOrMake(newResourceClusterId, writeOnly); } else { ClusterImpl cluster = getClusterByClusterIdOrThrow(newResourceClusterId); if (cluster.getNumberOfResources(cs) >= CLUSTER_FILL_SIZE) { newResourceClusterId = graphSession.newClusterId(); cluster = getClusterByClusterIdOrMake(newResourceClusterId, writeOnly); } - return cluster; + result = cluster; } + return ensureLoaded(result); } void flushCluster(GraphSession graphSession) { @@ -746,6 +756,7 @@ public final class ClusterTable implements IClusterTable { } finally { if (null != th) session.writeState = null; + cs.dispose(); } } } @@ -818,25 +829,9 @@ public final class ClusterTable implements IClusterTable { TLongIntHashMap clusterLoadHistogram = new TLongIntHashMap(); int clusterLoadCounter = 0; - @SuppressWarnings("unchecked") - public final T getClusterByResourceKey(final int resourceKey) { - int clusterKey = ClusterTraitsBase.getClusterKeyFromResourceKeyNoThrow(resourceKey); - if (ClusterTraitsBase.isVirtualClusterKey(clusterKey)) - throw new RuntimeException("Tried to get a persistent cluster for a virtual resource."); - ClusterI c = clusterArray[clusterKey]; - if (c == null) - return null; - if (c.isLoaded()) { - if ((counter++ & 4095) == 0) - refreshImportance((ClusterImpl) c); - return (T) c; - } - if (!(c instanceof ClusterSmall)) { - Logger.defaultLogError("Proxy must be instance of ClusterSmall"); - return null; - } + private T ensureLoaded(T c) { ClusterI cluster; - ClusterSmall cs = (ClusterSmall) c; + ClusterImpl cs = (ClusterImpl) c; try { if(DebugPolicy.REPORT_CLUSTER_LOADING) { long start = System.nanoTime(); @@ -861,13 +856,32 @@ public final class ClusterTable implements IClusterTable { Logger.defaultLogError(e); if (DebugPolicy.REPORT_CLUSTER_EVENTS) e.printStackTrace(); - String msg = "Failed to load cluster " + cs.getClusterUID() + " for resource key " + resourceKey - + " resourceId=" + (((cs.getClusterId() << 16 + (resourceKey & 65535)))); + String msg = "Failed to load cluster " + cs.getClusterUID();// + " resourceId=" + (((cs.getClusterId() << 16 + (resourceKey & 65535)))); // TODO: this jams the system => needs refactoring. throw new RuntimeDatabaseException(msg, e); } return (T) cluster; } + + @SuppressWarnings("unchecked") + public final T getClusterByResourceKey(final int resourceKey) { + int clusterKey = ClusterTraitsBase.getClusterKeyFromResourceKeyNoThrow(resourceKey); + if (ClusterTraitsBase.isVirtualClusterKey(clusterKey)) + throw new RuntimeException("Tried to get a persistent cluster for a virtual resource."); + ClusterI c = clusterArray[clusterKey]; + if (c == null) + return null; + if (c.isLoaded()) { + if ((counter++ & 4095) == 0) + refreshImportance((ClusterImpl) c); + return (T) c; + } + if (!(c instanceof ClusterSmall)) { + Logger.defaultLogError("Proxy must be instance of ClusterSmall"); + return null; + } + return ensureLoaded((T)c); + } @SuppressWarnings("unchecked") final T checkedGetClusterByResourceKey(final int resourceKey) { @@ -936,12 +950,9 @@ public final class ClusterTable implements IClusterTable { final Semaphore s = new Semaphore(0); final DatabaseException[] ex = new DatabaseException[1]; - load2(clusterId, clusterKey, new Callback() { - @Override - public void run(DatabaseException e) { - ex[0] = e; - s.release(); - } + load2(clusterId, clusterKey, e -> { + ex[0] = e; + s.release(); }); try { s.acquire(); @@ -1014,7 +1025,7 @@ public final class ClusterTable implements IClusterTable { } } - public synchronized void load2(long clusterId, int clusterKey, final Callback runnable) { + public synchronized void load2(long clusterId, int clusterKey, final Consumer runnable) { assert (Constants.ReservedClusterId != clusterId); @@ -1056,7 +1067,7 @@ public final class ClusterTable implements IClusterTable { e = new DatabaseException("Load cluster failed.", t); } if (null == cluster) { - runnable.run(e); + runnable.accept(e); return; } @@ -1065,7 +1076,7 @@ public final class ClusterTable implements IClusterTable { // Can not be called with null argument. replaceCluster(cluster); sessionImpl.onClusterLoaded(clusterId); - runnable.run(null); + runnable.accept(null); } @@ -1157,4 +1168,47 @@ public final class ClusterTable implements IClusterTable { return getClusterKeyByClusterUIDOrMakeProxy(ClusterUID.make(first, second)); } + public void adjustCachedSize(long l, ClusterI cluster) { + if (l != 0) { + //System.out.println("ClusterTable: adjusting cluster table cached size by " + l + ": " + sizeInBytes + " -> " + // + (sizeInBytes + l) + ", for cluster " + cluster.getClusterId() + " (" + cluster + ")"); + sizeInBytes += l; + } + } + + private void validateSize() { + if (!VALIDATE_SIZE) + return; + +// System.out.println("validating cached cluster sizes: " + sizeInBytes + ", hashMap.size=" +// + clusters.hashMap.size() + ", importanceMap.size=" + importanceMap.size()); + + int i = clusterArray.length; + long size = 0; + for (int j = 0; j < i; ++j) { + ClusterI c = clusterArray[j]; + if (c == null) + continue; + size += c.getCachedSize(); + } + if (sizeInBytes != size) { + if (!dirtySizeInBytes) + System.out.println("BUG: CACHED CLUSTER SIZE DIFFERS FROM CALCULATED: " + sizeInBytes + " != " + size + ", delta = " + (sizeInBytes - size)); + //else System.out.println("\"BUG?\": SIZES DIFFER: " + sizeInBytes + " != " + size + ", delta = " + (sizeInBytes - size)); + } + + int ims = importanceMap.size(); + int[] hms = {0}; + clusters.hashMap.forEachEntry((cid, c) -> { + if (c != null && !c.isWriteOnly() && !c.isEmpty() && c.isLoaded()) { + //System.out.println(cid + ": " + c); + hms[0]++; + } + return true; + }); + if (Math.abs(ims-hms[0]) > 0) { + System.out.println("BUG2: hashmap and importanceMap sizes differ: " + hms[0] + " != " + ims + ", delta=" + (hms[0] - ims)); + } + } + }