From: Tuukka Lehtonen Date: Thu, 5 Apr 2018 06:47:03 +0000 (+0300) Subject: Fixing problems in the database unit testing environment with Acorn X-Git-Tag: v1.43.0~136^2~503 X-Git-Url: https://gerrit.simantics.org/r/gitweb?p=simantics%2Fplatform.git;a=commitdiff_plain;h=e5c006a3e29dcb1f29ae5bcc21ac28573bd37648 Fixing problems in the database unit testing environment with Acorn * Removed static map from org.simantics.acorn.FileIO. Moved the map into a session service as org.simantics.acorn.FileCache * Changed org.simantics.db.testing.common.Tests utilities to use Acorn instead of ProCore (AcornTestHandler). Did not introduce abstraction for this for now. refs #7855 Conflicts: bundles/org.simantics.acorn/src/org/simantics/acorn/FileIO.java Change-Id: Iec42395cc71b30a4c376a44778b358e3807e6ec9 (cherry picked from commit 03fac449ec81e88669169d671bb386eb93ac060d) --- diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/ClusterManager.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/ClusterManager.java index a2cb2fa86..cb9ac578e 100644 --- a/bundles/org.simantics.acorn/src/org/simantics/acorn/ClusterManager.java +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/ClusterManager.java @@ -47,8 +47,9 @@ public class ClusterManager { private ArrayList currentChanges = new ArrayList(); public final Path dbFolder; - public Path lastSessionDirectory; - public Path workingDirectory; + private FileCache fileCache; + Path lastSessionDirectory; + Path workingDirectory; public LRU streamLRU; public LRU csLRU; @@ -67,8 +68,9 @@ public class ClusterManager { * */ - public ClusterManager(Path dbFolder) { + public ClusterManager(Path dbFolder, FileCache fileCache) { this.dbFolder = dbFolder; + this.fileCache = fileCache; } public ArrayList getChanges(long changeSetId) throws AcornAccessVerificationException, IllegalAcornStateException { @@ -467,7 +469,7 @@ public class ClusterManager { Path readDir = dbFolder.resolve(parts[1]); int offset = Integer.parseInt(parts[2]); int length = Integer.parseInt(parts[3]); - FileInfo info = new FileInfo(fileLRU, readDir, parts[0], offset, length); + FileInfo info = new FileInfo(fileLRU, fileCache, readDir, parts[0], offset, length); fileLRU.map(info); } // Update chunks @@ -488,7 +490,7 @@ public class ClusterManager { Long revisionId = Long.parseLong(parts[0]); int offset = Integer.parseInt(parts[2]); int length = Integer.parseInt(parts[3]); - ChangeSetInfo info = new ChangeSetInfo(csLRU, readDir, revisionId, offset, length); + ChangeSetInfo info = new ChangeSetInfo(csLRU, fileCache, readDir, revisionId, offset, length); csLRU.map(info); } @@ -602,7 +604,7 @@ public class ClusterManager { try { ArrayList csids = new ArrayList(currentChanges); currentChanges = new ArrayList(); - new ChangeSetInfo(csLRU, changeSetId, data, csids); + new ChangeSetInfo(csLRU, fileCache, changeSetId, data, csids); } catch (Throwable t) { throw new IllegalAcornStateException(t); } finally { @@ -669,7 +671,7 @@ public class ClusterManager { try { info = fileLRU.get(key); if (info == null) { - info = new FileInfo(fileLRU, key, (int) (offset + size)); + info = new FileInfo(fileLRU, fileCache, key, (int) (offset + size)); } } catch (Throwable t) { throw new IllegalAcornStateException(t); @@ -705,5 +707,9 @@ public class ClusterManager { public long getTailChangeSetId() { return state.tailChangeSetId; } - + + public FileCache getFileCache() { + return fileCache; + } + } diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/FileCache.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/FileCache.java new file mode 100644 index 000000000..b0f9d3740 --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/FileCache.java @@ -0,0 +1,28 @@ +package org.simantics.acorn; + +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; + +import org.simantics.db.Disposable; + +/** + * @author Tuukka Lehtonen + * @since 1.32.0 + */ +public class FileCache implements Disposable { + + private Map map = new HashMap<>(); + + public FileIO get(Path path) { + synchronized (map) { + return map.computeIfAbsent(path, FileIO::new); + } + } + + @Override + public void dispose() { + map = new HashMap<>(); + } + +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/FileIO.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/FileIO.java index 8f3009afa..305094f83 100644 --- a/bundles/org.simantics.acorn/src/org/simantics/acorn/FileIO.java +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/FileIO.java @@ -25,38 +25,26 @@ import org.slf4j.LoggerFactory; public class FileIO { private static final Logger LOGGER = LoggerFactory.getLogger(FileIO.class); - private static final FileAttribute[] NO_ATTRIBUTES = new FileAttribute[0]; - + + private static final FileAttribute[] NO_ATTRIBUTES = {}; + private static final Set CREATE_OPTIONS = new HashSet<>(2); private static final Set APPEND_OPTIONS = new HashSet<>(1); - + static { CREATE_OPTIONS.add(StandardOpenOption.WRITE); CREATE_OPTIONS.add(StandardOpenOption.CREATE); - + APPEND_OPTIONS.add(StandardOpenOption.APPEND); } - + private Path path; private int writePosition = 0; - private FileIO(Path path) { + public FileIO(Path path) { this.path = path; } - - private static Map map = new HashMap(); - - public static FileIO get(Path path) { - synchronized(map) { - FileIO existing = map.get(path); - if(existing == null) { - existing = new FileIO(path); - map.put(path, existing); - } - return existing; - } - } - + //private static final boolean TRACE_SWAP = false; private static final boolean TRACE_PERF = false; diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/GraphClientImpl2.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/GraphClientImpl2.java index 7a57053bc..2ffdc715f 100644 --- a/bundles/org.simantics.acorn/src/org/simantics/acorn/GraphClientImpl2.java +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/GraphClientImpl2.java @@ -68,6 +68,7 @@ public class GraphClientImpl2 implements Database.Session { private Path dbFolder; private final Database database; private ServiceLocator locator; + private FileCache fileCache; private MainProgram mainProgram; private static class ClientThreadFactory implements ThreadFactory { @@ -92,7 +93,10 @@ public class GraphClientImpl2 implements Database.Session { this.database = database; this.dbFolder = dbFolder; this.locator = locator; - this.clusters = new ClusterManager(dbFolder); + this.fileCache = new FileCache(); + // This disposes the cache when the session is shut down + locator.registerService(FileCache.class, fileCache); + this.clusters = new ClusterManager(dbFolder, fileCache); load(); ClusterSetsSupport cssi = locator.getService(ClusterSetsSupport.class); cssi.setReadDirectory(clusters.lastSessionDirectory); diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/AcornDatabase.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/AcornDatabase.java index a423fbdbe..8109df6cd 100644 --- a/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/AcornDatabase.java +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/AcornDatabase.java @@ -13,6 +13,7 @@ import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; import java.util.EnumSet; import java.util.Properties; +import java.util.stream.Stream; import org.simantics.acorn.GraphClientImpl2; import org.simantics.db.Database; @@ -23,6 +24,8 @@ import org.simantics.db.server.ProCoreException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import fi.vtt.simantics.procore.internal.StaticSessionProperties; + /** * @author Tuukka Lehtonen */ @@ -112,6 +115,15 @@ public class AcornDatabase implements Database { @Override public void deleteFiles() throws ProCoreException { deleteTree(folder); + File vgPath = StaticSessionProperties.virtualGraphStoragePath; + if (vgPath != null) { + try (Stream vgs = Files.list(vgPath.toPath())) { + for (Path p : vgs.toArray(Path[]::new)) + deleteTree(p); + } catch (IOException e) { + throw new ProCoreException(e); + } + } } @Override @@ -147,6 +159,8 @@ public class AcornDatabase implements Database { raLockFile = null; Files.deleteIfExists(lockFile); isRunning = false; + safeLoggingClose(currentClient, currentClient.getDbFolder()); + currentClient = null; } catch (IOException e) { LOGGER.error("Failed to start database at " + folder.toAbsolutePath(), e); } @@ -224,39 +238,37 @@ public class AcornDatabase implements Database { throw new UnsupportedOperationException(); } - private static void deleteTree(Path path) throws ProCoreException { - if (!Files.exists(path)) - return; - - class Visitor extends SimpleFileVisitor { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + static class Visitor extends SimpleFileVisitor { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + try { + Files.delete(file); + } catch (IOException ioe) { + LOGGER.error("Failed to delete file {}", file, ioe); + throw ioe; + } + return FileVisitResult.CONTINUE; + } + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException e) throws IOException { + if (e == null) { try { - Files.delete(file); + Files.delete(dir); } catch (IOException ioe) { - ioe.printStackTrace(); + LOGGER.error("Failed to delete directory {}", dir, ioe); throw ioe; } return FileVisitResult.CONTINUE; } - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException e) throws IOException { - if (e == null) { - try { - Files.delete(dir); - } catch (IOException ioe) { - ioe.printStackTrace(); - throw ioe; - } - return FileVisitResult.CONTINUE; - } - throw e; - } + throw e; } + } + + private static void deleteTree(Path path) throws ProCoreException { + if (!Files.exists(path)) + return; try { - Visitor v = new Visitor(); - EnumSet opts = EnumSet.noneOf(FileVisitOption.class); - Files.walkFileTree(path, opts, Integer.MAX_VALUE, v); + Files.walkFileTree(path, EnumSet.noneOf(FileVisitOption.class), Integer.MAX_VALUE, new Visitor()); } catch (IOException e) { throw new ProCoreException("Could not delete " + path, e); } @@ -272,7 +284,17 @@ public class AcornDatabase implements Database { return; try (AutoCloseable c = closeable) { } catch (Exception e) { - LOGGER.error("Failed to close " + closeable.getClass() + " of " + file.toAbsolutePath()); + LOGGER.error("Failed to close " + closeable.getClass() + " of " + file.toAbsolutePath(), e); + } + } + + private static void safeLoggingClose(Database.Session session, Path file) { + if (session == null) + return; + try { + session.close(); + } catch (Exception e) { + LOGGER.error("Failed to close " + session.getClass() + " of " + file.toAbsolutePath(), e); } } diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/ChangeSetInfo.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/ChangeSetInfo.java index e1915d32b..235b26418 100644 --- a/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/ChangeSetInfo.java +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/ChangeSetInfo.java @@ -4,6 +4,7 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; +import org.simantics.acorn.FileCache; import org.simantics.acorn.exception.AcornAccessVerificationException; import org.simantics.acorn.exception.IllegalAcornStateException; import org.simantics.db.service.Bytes; @@ -20,14 +21,14 @@ public class ChangeSetInfo extends LRUObject { private ArrayList clusterChangeSetIds; // Stub - public ChangeSetInfo(LRU LRU, Path readDir, Long revision, int offset, int length) throws AcornAccessVerificationException { - super(LRU, revision, readDir, "clusterStream", offset, length, false, false); + public ChangeSetInfo(LRU LRU, FileCache fileCache, Path readDir, Long revision, int offset, int length) throws AcornAccessVerificationException { + super(LRU, fileCache, revision, readDir, "clusterStream", offset, length, false, false); LRU.map(this); } // New - public ChangeSetInfo(LRU LRU, Long revision, byte[] bytes, ArrayList clusterChangeSetIds) throws AcornAccessVerificationException { - super(LRU, revision, LRU.getDirectory(), "clusterStream", true, true); + public ChangeSetInfo(LRU LRU, FileCache fileCache, Long revision, byte[] bytes, ArrayList clusterChangeSetIds) throws AcornAccessVerificationException { + super(LRU, fileCache, revision, LRU.getDirectory(), "clusterStream", true, true); this.metadataBytes = bytes; this.metadataBytes = bytes; this.clusterChangeSetIds = clusterChangeSetIds; diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/ClusterInfo.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/ClusterInfo.java index 54689394d..f058a559d 100644 --- a/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/ClusterInfo.java +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/ClusterInfo.java @@ -35,7 +35,7 @@ public class ClusterInfo extends LRUObject implements P // Stub public ClusterInfo(ClusterManager manager, LRU LRU, Path readDirectory, ClusterUID uid, int offset, int length) throws AcornAccessVerificationException { - super(LRU, uid, readDirectory, uid.toString() + ".cluster", offset, length, false, false); + super(LRU, manager.getFileCache(), uid, readDirectory, uid.toString() + ".cluster", offset, length, false, false); this.manager = manager; this.cluster = null; LRU.map(this); @@ -43,7 +43,7 @@ public class ClusterInfo extends LRUObject implements P // New public ClusterInfo(ClusterManager manager, LRU LRU, ClusterImpl cluster) throws AcornAccessVerificationException, IllegalAcornStateException { - super(LRU, cluster.getClusterUID(), LRU.getDirectory(), cluster.getClusterUID().toString() + ".cluster", true, true); + super(LRU, manager.getFileCache(), cluster.getClusterUID(), LRU.getDirectory(), cluster.getClusterUID().toString() + ".cluster", true, true); this.manager = manager; this.cluster = cluster; LRU.insert(this, accessTime); diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/ClusterStreamChunk.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/ClusterStreamChunk.java index 767746eb0..3fa5e327f 100644 --- a/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/ClusterStreamChunk.java +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/ClusterStreamChunk.java @@ -36,14 +36,14 @@ public class ClusterStreamChunk extends LRUObject im // Stub public ClusterStreamChunk(ClusterManager manager, LRU LRU, Path readDir, String id, int offset, int length) throws AcornAccessVerificationException { - super(LRU, id, readDir, "clusterStream", offset, length, false, false); + super(LRU, manager.getFileCache(), id, readDir, "clusterStream", offset, length, false, false); this.manager = manager; LRU.map(this); } // Creation public ClusterStreamChunk(ClusterManager manager, LRU LRU, String id) throws AcornAccessVerificationException { - super(LRU, id, LRU.getDirectory(), "clusterStream", true, true); + super(LRU, manager.getFileCache(), id, LRU.getDirectory(), "clusterStream", true, true); this.manager = manager; LRU.insert(this, accessTime); } diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/FileInfo.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/FileInfo.java index 0e1367f99..d65d3dc86 100644 --- a/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/FileInfo.java +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/FileInfo.java @@ -2,6 +2,7 @@ package org.simantics.acorn.lru; import java.nio.file.Path; +import org.simantics.acorn.FileCache; import org.simantics.acorn.exception.AcornAccessVerificationException; import org.simantics.acorn.exception.IllegalAcornStateException; import org.simantics.db.Database.Session.ResourceSegment; @@ -17,14 +18,14 @@ public class FileInfo extends LRUObject { private TByteArrayList bytes; // Stub - public FileInfo(LRU LRU, Path readDir, String id, int offset, int length) throws AcornAccessVerificationException { - super(LRU, id, readDir, id.toString() + ".extFile", offset, length, false, false); + public FileInfo(LRU LRU, FileCache fileCache, Path readDir, String id, int offset, int length) throws AcornAccessVerificationException { + super(LRU, fileCache, id, readDir, id.toString() + ".extFile", offset, length, false, false); LRU.map(this); } // New - public FileInfo(LRU LRU, String id, int size) throws AcornAccessVerificationException { - super(LRU, id, LRU.getDirectory(), id.toString() + ".extFile", true, true); + public FileInfo(LRU LRU, FileCache fileCache, String id, int size) throws AcornAccessVerificationException { + super(LRU, fileCache, id, LRU.getDirectory(), id.toString() + ".extFile", true, true); this.bytes = new TByteArrayList(size); LRU.insert(this, accessTime); } diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/LRUObject.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/LRUObject.java index a84a28108..5230723b7 100644 --- a/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/LRUObject.java +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/LRUObject.java @@ -5,6 +5,7 @@ import java.nio.file.Path; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; +import org.simantics.acorn.FileCache; import org.simantics.acorn.FileIO; import org.simantics.acorn.Persistable; import org.simantics.acorn.exception.AcornAccessVerificationException; @@ -19,6 +20,7 @@ public abstract class LRUObject LRU; + final protected FileCache fileCache; final private Semaphore mutex = new Semaphore(1); final private MapKey key; final private String fileName; @@ -39,8 +41,9 @@ public abstract class LRUObject LRU, MapKey key, Path readDirectory, String fileName, int offset, int length, boolean dirty, boolean resident) { + public LRUObject(LRU LRU, FileCache fileCache, MapKey key, Path readDirectory, String fileName, int offset, int length, boolean dirty, boolean resident) { this.LRU = LRU; + this.fileCache = fileCache; this.key = key; this.fileName = fileName; this.offset = offset; @@ -51,8 +54,8 @@ public abstract class LRUObject LRU, MapKey key, Path readDirectory, String fileName, boolean dirty, boolean resident) { - this(LRU, key, readDirectory, fileName, -1, -1, dirty, resident); + public LRUObject(LRU LRU, FileCache fileCache, MapKey key, Path readDirectory, String fileName, boolean dirty, boolean resident) { + this(LRU, fileCache, key, readDirectory, fileName, -1, -1, dirty, resident); } /* @@ -98,7 +101,7 @@ public abstract class LRUObject pair = toBytes(); byte[] data = pair.first; int length = pair.second; - FileIO fio = FileIO.get(bytes); + FileIO fio = fileCache.get(bytes); int offset = fio.saveBytes(data, length, overwrite()); setPosition(offset, length); } catch (AcornAccessVerificationException | IllegalAcornStateException e) { @@ -191,7 +194,7 @@ public abstract class LRUObject stms, Resource predicate, Resource object) { for(Statement stm : stms) { if(stm.getPredicate().equals(predicate) && stm.getObject().equals(object)) return true; @@ -48,17 +49,21 @@ public class Tests { } return false; } + public static DatabaseState freshWorkspace(String CORE_DIR, ArrayList fileFilter) throws Exception { SimanticsPlatform.INSTANCE.resetWorkspace(null, fileFilter); return newSimanticsWorkspace(null, CORE_DIR); } + public static void freshDatabase() throws Exception { DatabaseState state = freshWorkspace(Configuration.get().coreDir, null); - Tests.shutdown(state); + shutdown(state); } + public static DatabaseState existingDatabase() throws Exception { return oldSimanticsWorkspace(null, Configuration.get().coreDir); } + private static void initSimanticsStuff() { // Set session context provider. final ISessionContextProvider provider = new SessionContextProvider(null); @@ -66,76 +71,28 @@ public class Tests { Simantics.setSessionContextProviderSource(source); org.simantics.db.layer0.internal.SimanticsInternal.setSessionContextProviderSource(source); } + public static DatabaseState newSimanticsWorkspace(TestSettings testSettings, String address) throws Exception { - TestHandler testHandler = Tests.getTestHandler(testSettings, address); + AcornTestHandler testHandler = getTestHandler(testSettings, address); testHandler.initNew(); initSimanticsStuff(); SessionContext sessionContext = SimanticsPlatform.INSTANCE.startUp(Simantics.getDefaultDatabaseDriver(), null, RecoveryPolicy.FixError, OntologyRecoveryPolicy.Merge, true, new DefaultChoiceUserAgent()); - return new DatabaseState(address, sessionContext); + return new DatabaseState(address, sessionContext); } + public static DatabaseState oldSimanticsWorkspace(TestSettings testSettings, String address) throws Exception { - Tests.getTestHandler(testSettings, address); + getTestHandler(testSettings, address); initSimanticsStuff(); SessionContext sessionContext = SimanticsPlatform.INSTANCE.startUp(Simantics.getDefaultDatabaseDriver(), null, RecoveryPolicy.FixError, OntologyRecoveryPolicy.Merge, false, null); return new DatabaseState(address, sessionContext); } -// for (String ontology : testSettings.getOntologies()) { -// initOntology(session, ontology); -// } -// String[] adapters = testSettings.getAdapters(); -// if(adapters != null) { -// File[] files = new File[adapters.length]; -// for(int i=0;i