X-Git-Url: https://gerrit.simantics.org/r/gitweb?a=blobdiff_plain;f=bundles%2Forg.simantics.db.server%2Fsrc%2Forg%2Fsimantics%2Fdb%2Fserver%2Finternal%2FDatabaseI.java;h=2f3187d5ae2acb2c1b919ac44be392f484781c67;hb=refs%2Fchanges%2F76%2F2276%2F1;hp=9d03ee4ae0df604cbfe5f46f7e2db42e6f1ebed8;hpb=969bd23cab98a79ca9101af33334000879fb60c5;p=simantics%2Fplatform.git diff --git a/bundles/org.simantics.db.server/src/org/simantics/db/server/internal/DatabaseI.java b/bundles/org.simantics.db.server/src/org/simantics/db/server/internal/DatabaseI.java index 9d03ee4ae..2f3187d5a 100644 --- a/bundles/org.simantics.db.server/src/org/simantics/db/server/internal/DatabaseI.java +++ b/bundles/org.simantics.db.server/src/org/simantics/db/server/internal/DatabaseI.java @@ -1,1366 +1,1370 @@ -package org.simantics.db.server.internal; - -import static java.nio.file.StandardCopyOption.COPY_ATTRIBUTES; -import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; - -import java.io.EOFException; -import java.io.File; -import java.io.FilenameFilter; -import java.io.IOException; -import java.io.OutputStream; -import java.io.RandomAccessFile; -import java.nio.file.DirectoryStream; -import java.nio.file.FileSystems; -import java.nio.file.FileVisitOption; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.PathMatcher; -import java.nio.file.Paths; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.ArrayList; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.List; -import java.util.Properties; -import java.util.TreeMap; -import java.util.UUID; - -import org.simantics.databoard.Bindings; -import org.simantics.databoard.binding.impl.TreeMapBinding; -import org.simantics.databoard.serialization.Serializer; -import org.simantics.db.Database; -import org.simantics.db.DatabaseUserAgent; -import org.simantics.db.ServiceLocator; -import org.simantics.db.common.CommentMetadata; -import org.simantics.db.common.CommitMetadata; -import org.simantics.db.common.utils.Logger; -import org.simantics.db.common.utils.MetadataUtil; -import org.simantics.db.exception.SDBException; -import org.simantics.db.server.DatabaseLastExitException; -import org.simantics.db.server.ProCoreException; -import org.simantics.db.server.internal.ProCoreServer.TailFile; -import org.simantics.db.server.protocol.MessageNumber; -import org.simantics.db.server.protocol.MessageText; - -public class DatabaseI implements Database { - private static String NL = System.getProperty("line.separator"); - private static String TEMP_PREFIX = "db.temp."; - public static DatabaseI newDatabaseI(File dbFolder) { - return new DatabaseI(dbFolder); - } - private final SessionManager sessionManager = new SessionManager(); - private final Server server; - private final JournalI journal; - private DatabaseUserAgent dbUserAgent = null; - private DatabaseI(File dbFolder) { - server = new Server(dbFolder); - journal = new JournalI(dbFolder); - } - public boolean isSubFolder(Path base, Path sub) - throws IOException { - if (null == base || null == sub) - return false; - return isSubFolder(base.toFile(), sub.toFile()); - } - public boolean isSubFolder(File base, File sub) - throws IOException { - if (null == base || null == sub) - return false; - Path basePath = base.getCanonicalFile().toPath(); - Path subPath = sub.getCanonicalFile().toPath(); - if (subPath.startsWith(basePath)) - return true; - else - return false; - } - public Path saveDatabase() - throws ProCoreException { - Path dbFolder = getFolder().toPath(); - Path parent = getFolder().getParentFile().toPath(); - Path temp = createTempFolder(parent, "Could not create temporary directory for saving database."); - copyTree(dbFolder, temp, FileOption.IF_NEW); - return temp; - } - public Path saveDatabase(Path parent) - throws ProCoreException { - Path dbFolder = getFolder().toPath(); - try { - boolean yes = isSubFolder(dbFolder, parent); - if (yes) - throw new ProCoreException("Parent must not be subdirectory of database directory." - + NL + "database=" + dbFolder - + NL + "parent=" + parent); - } catch (IOException e) { - throw new ProCoreException("Failed to save database to parent." - + NL + "database=" + dbFolder - + NL + "parent=" + parent); - } - Path temp = createTempFolder(parent, "Could not create temporary directory for saving database."); - copyTree(dbFolder, temp, FileOption.IF_NEW); - return temp; - } - public void deleteDatabaseFiles() throws ProCoreException { - if (server.isAlive()) { - server.tryToStop(); - if (server.isAlive()) - throw new ProCoreException("Server must be dead to delete database."); - } - Path db = getFolder().toPath(); - deleteDatabaseFiles(db); // Redundant, but tests the method. - deleteTree(db); - } - Path copyDatabaseFiles(Path from, Path to) throws ProCoreException { - copyTree(from, to, ProCoreServer.BRANCH_DIR, FileOption.IF_EXIST); - if (Files.exists(from.resolve(ProCoreServer.TAIL_DIR))) - copyTree(from, to, ProCoreServer.TAIL_DIR, FileOption.IF_EXIST); - copyPath(from, to, ProCoreServer.CONFIG_FILE, FileOption.IF_EXIST); - copyPath(from, to, ProCoreServer.GUARD_FILE, FileOption.IF_EXIST); - copyPath(from, to, ProCoreServer.JOURNAL_FILE, FileOption.IF_EXIST); - copyPath(from, to, ProCoreServer.LOG_FILE, FileOption.IF_EXIST); - copyFiles(from, to, ProCoreServer.PAGE_FILE_PATTERN, FileOption.IF_EXIST); - return to; - } - public static void deleteDatabaseFiles(Path from) throws ProCoreException { - deleteFile(from, ProCoreServer.DCS_FILE, FileOption.IF_EXIST); - deleteFile(from, ProCoreServer.RECOVERY_NEEDED_FILE, FileOption.IF_EXIST); - deleteFile(from, ProCoreServer.RECOVERY_IGNORED_FILE, FileOption.IF_EXIST); - deleteFile(from, ProCoreServer.PROTOCOL_IGNORED_FILE, FileOption.IF_EXIST); - deleteFile(from, ProCoreServer.LOG_FILE, FileOption.IF_EXIST); - deleteFile(from, ProCoreServer.GUARD_FILE, FileOption.IF_EXIST); - deleteFile(from, ProCoreServer.CONFIG_FILE, FileOption.IF_EXIST); - deleteFiles(from, ProCoreServer.PAGE_FILE_PATTERN); - deleteTree(from.resolve(ProCoreServer.BRANCH_DIR)); - deleteFile(from, ProCoreServer.JOURNAL_FILE, FileOption.IF_EXIST); - deleteTree(from.resolve(ProCoreServer.TAIL_DIR)); - } - Path moveDatabaseFiles(Path from, Path to) throws ProCoreException { - moveFolder(from, to, ProCoreServer.BRANCH_DIR, FileOption.IF_NEW); - if (Files.exists(from.resolve(ProCoreServer.TAIL_DIR))) - moveFolder(from, to, ProCoreServer.TAIL_DIR, FileOption.IF_NEW); - movePath(from, to, ProCoreServer.CONFIG_FILE, FileOption.IF_NEW); - movePath(from, to, ProCoreServer.GUARD_FILE, FileOption.IF_NEW); - movePath(from, to, ProCoreServer.JOURNAL_FILE, FileOption.IF_NEW); - movePath(from, to, ProCoreServer.LOG_FILE, FileOption.IF_NEW); - moveFiles(from, to, ProCoreServer.PAGE_FILE_PATTERN, FileOption.IF_NEW); - movePath(from, to, ProCoreServer.DCS_FILE, FileOption.IF_NEW); - moveFile(from, to, ProCoreServer.RECOVERY_NEEDED_FILE, FileOption.IF_NEW); - moveFile(from, to, ProCoreServer.RECOVERY_IGNORED_FILE, FileOption.IF_NEW); - moveFile(from, to, ProCoreServer.PROTOCOL_IGNORED_FILE, FileOption.IF_NEW); - return to; - } - private void moveFile(Path from, Path to, String file, FileOption fileOption) throws ProCoreException { - if (Files.exists(from.resolve(file))) - movePath(from, to, file, fileOption); - } - public void serverCreateConfiguration() throws ProCoreException { - server.createConfiguration(); - } - public boolean ignoreExit() throws ProCoreException { - if (!isFolderOk()) - throw new ProCoreException("Folder must be valid database folder to ignore exit." ); - server.deleteGuard(); - File folder = server.getFolder(); - Path db = Paths.get(folder.getAbsolutePath()); - Path ri = db.resolve(ProCoreServer.RECOVERY_IGNORED_FILE); - if (!Files.exists(ri)) - try (OutputStream os = Files.newOutputStream(ri)) { - } catch (IOException e) { - throw new ProCoreException("Could not create file: " + ri, e); - } - boolean ok = false; - DatabaseUserAgent dbu = dbUserAgent; // Save current user agent. - try { - dbUserAgent = null; // Recursion not supported. Disable user agent. - server.start(); - ok = server.isActive(); - } finally { - try { - dbUserAgent = dbu; // Restore used user agent. - server.stop(); - } finally { - try { - Files.deleteIfExists(ri); - } catch (IOException e) { - Logger.defaultLogError("Could not delete file: " + ri, e); - } - Path rn = db.resolve(ProCoreServer.RECOVERY_NEEDED_FILE); - try { - Files.deleteIfExists(rn); - } catch (IOException e) { - Logger.defaultLogError("Could not delete file: " + rn, e); - } - } - } - return ok; - } - public boolean ignoreProtocol() throws ProCoreException { - if (!isFolderOk()) - throw new ProCoreException("Folder must be valid database folder to ignore exit." ); - File folder = server.getFolder(); - Path db = Paths.get(folder.getAbsolutePath()); - Path ri = db.resolve(ProCoreServer.PROTOCOL_IGNORED_FILE); - if (!Files.exists(ri)) - try (OutputStream os = Files.newOutputStream(ri)) { - } catch (IOException e) { - throw new ProCoreException("Could not create file: " + ri, e); - } - boolean ok = false; - try { - server.start(); - ok = server.isActive(); - } finally { - server.stop(); - } - return ok; - } - private long preJournalCheck() throws ProCoreException { - File folder = server.getFolder(); - if (!folder.isDirectory()) - throw new ProCoreException("Database folder does not exist." + NL + "folder=" + folder); - File file = new File(folder, ProCoreServer.JOURNAL_FILE); - if (!file.isFile()) - throw new ProCoreException("Journal file does not exist." + NL + "file=" + file); - else if (!file.canRead()) - throw new ProCoreException("Journal file must be readale to create database from journal." + NL + "file=" + file); - else if (server.isAlive()) - throw new ProCoreException("Server must be dead to create database from journal file." + NL + "file=" + file); - return getNextClusterId(folder.toPath()); - } - private void postJournalFix(long nextFreeId) throws SDBException { - Session s = newSession(null); - long current = s.reserveIds(0); - if (current < nextFreeId) - s.reserveIds((int)(nextFreeId - current)); - s.isClosed(); - } - private Path createTempFolder(Path parent, String fail) throws ProCoreException { - try { - return Files.createTempDirectory(parent, TEMP_PREFIX); - } catch (IOException e) { - throw new ProCoreException(fail, e); - } - } - public void replaceFromJournal() throws SDBException { - long nextFreeId = preJournalCheck(); - Path db = Paths.get(server.getFolder().getAbsolutePath()); - Path temp = createTempFolder(db, "Could not create temporary directory for database to be created from journal."); - movePath(db, temp, ProCoreServer.CONFIG_FILE, FileOption.IF_NEW); - movePath(db, temp, ProCoreServer.JOURNAL_FILE, FileOption.IF_NEW); - deleteFile(db, ProCoreServer.GUARD_FILE, FileOption.IF_EXIST); - deleteFiles(db, ProCoreServer.PAGE_FILE_PATTERN); - Path dbHead = db.resolve(ProCoreServer.BRANCH_DIR); - deleteFiles(dbHead, ProCoreServer.VALUE_PATTERN); - deleteFiles(dbHead, ProCoreServer.DATA_PATTERN); - deleteFiles(dbHead, ProCoreServer.INDEX_PATTERN); - Path dbHeadTailFile = dbHead.resolve(ProCoreServer.TAIL_FILE); - boolean headTailFileExisted = Files.exists(dbHeadTailFile); - final long NEXT_REVISION = 1; - if (!headTailFileExisted) // Number of change sets and database id not known at this point. Fortunately they are not used by recovery. - TailFile.createTailFile(dbHeadTailFile.toFile(), NEXT_REVISION, nextFreeId, UUID.randomUUID().toString()); - movePath(dbHead, temp, ProCoreServer.TAIL_FILE, FileOption.IF_NEW); - Path dbTail = db.resolve(ProCoreServer.TAIL_DIR); - boolean tailExisted = Files.isDirectory(dbTail); - if (tailExisted) - copyPath(dbTail, dbHead, ProCoreServer.TAIL_FILE, FileOption.IF_NEW); - else { - try { - Files.createDirectory(dbTail); - } catch (IOException e) { - throw new ProCoreException("Failed to create directory: " + dbTail, e); - } - copyPath(temp, dbTail, ProCoreServer.TAIL_FILE, FileOption.IF_NEW); - } - server.createConfiguration(); - server.start(); - try { - String t = server.execute("journalRead " + temp.getFileName()); - if (t.length() > 0) - throw new ProCoreException("Could not read journal. reply=" + t); - postJournalFix(nextFreeId); // Not the right way for implementing this, but better than incorrect recovery. - } finally { - server.tryToStop(); - } - movePath(temp, db, ProCoreServer.CONFIG_FILE, FileOption.OVERWRITE); - deleteTree(temp); - if (!tailExisted) - deleteTree(dbTail); - } - public boolean canPurge() throws ProCoreException { - File folder = server.getFolder(); - Path db = Paths.get(folder.getAbsolutePath()); - Path dbHead = db.resolve(ProCoreServer.BRANCH_DIR); - if (!Files.exists(dbHead)) - return false; // Already clean. - boolean empty = isFolderEmpty(dbHead); - if (empty) - return false; // Already clean. - final boolean[] found = new boolean[1]; - found[0] = false; - dbHead.toFile().listFiles(new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - if (found[0]) - return false; - else if (!name.equals(ProCoreServer.TAIL_FILE)) { - found[0] = true; - return false; - } - return true; - } - }); - return found[0]; - } - /** - * Removes old history i.e. change sets. - * Note that also cleans last exit status and removes guard file. - * This means that all information about last exit status is also lost - * . - * @throws ProCoreException - */ - public void purge() throws SDBException { - synchronized (server.proCoreServer.getProCoreClient()) { - boolean wasAlive = server.isAlive(); - if (wasAlive) - server.stop(); - Path db = Paths.get(server.getFolder().getAbsolutePath()); - Path dbHead = db.resolve(ProCoreServer.BRANCH_DIR); - Path dbTail = db.resolve(ProCoreServer.TAIL_DIR); - if (!Files.isDirectory(dbTail)) { - try { - Files.createDirectory(dbTail); - } catch (IOException e) { - throw new ProCoreException("Failed to create directory: " + dbTail, e); - } - } - long nextFreeId = getNextClusterId(db); - boolean cleanHead = Files.isDirectory(dbHead) && !isFolderEmpty(dbHead); - Logger.defaultLog("Purging old history and exit information. folder=" + db); - if (cleanHead) { - deleteClusters(dbHead, dbTail); - movePath(dbHead, dbTail, ProCoreServer.TAIL_FILE, FileOption.OVERWRITE); - moveFiles(dbHead, dbTail, ProCoreServer.VALUE_PATTERN, FileOption.IF_NEW); - moveFiles(dbHead, dbTail, ProCoreServer.DATA_PATTERN, FileOption.OVERWRITE); - deleteFiles(dbHead, ProCoreServer.INDEX_PATTERN); - } - deleteFiles(db, ProCoreServer.PAGE_FILE_PATTERN); - deleteFile(db, ProCoreServer.JOURNAL_FILE, FileOption.IF_EXIST); - deleteFile(db, ProCoreServer.LOG_FILE, FileOption.IF_EXIST); - deleteFile(db, ProCoreServer.DCS_FILE, FileOption.IF_EXIST); - deleteFile(db, ProCoreServer.RECOVERY_NEEDED_FILE, FileOption.IF_EXIST); - deleteFile(db, ProCoreServer.RECOVERY_IGNORED_FILE, FileOption.IF_EXIST); - deleteFile(db, ProCoreServer.PROTOCOL_IGNORED_FILE, FileOption.IF_EXIST); - deleteFile(db, ProCoreServer.GUARD_FILE, FileOption.IF_EXIST); - purgeValues(dbTail); - server.start(); - Session s = newSession(null); - long current = s.reserveIds(0); - if (current < nextFreeId) - s.reserveIds((int)(nextFreeId - current)); - s.isClosed(); - if (!wasAlive) - server.tryToStop(); - } - } - private void copyPath(Path fromFolder, Path toFolder, String file, FileOption fileOption) throws ProCoreException { - Path from = fromFolder.resolve(file); - Path to = toFolder.resolve(file); - try { - if (FileOption.IF_EXIST.equals(fileOption)) { - if (!Files.exists(from)) - return; - } - if (FileOption.OVERWRITE.equals(fileOption)) - Files.copy(from, to, COPY_ATTRIBUTES, REPLACE_EXISTING); - else - Files.copy(from, to, COPY_ATTRIBUTES); - } catch (IOException e) { - throw new ProCoreException("Could not copy " + from + " to " + to, e); - } - } - private static void deleteFile(Path from, String file, FileOption fileOption) throws ProCoreException { - Path path = from.resolve(file); - deletePath(path, fileOption); - } - private static void deletePath(Path path, FileOption fileOption) throws ProCoreException { - try { - if (FileOption.IF_EXIST.equals(fileOption)) - Files.deleteIfExists(path); - else - Files.delete(path); - } catch (IOException e) { - throw new ProCoreException("Could not delete " + path, e); - } - } - private boolean isFolderEmpty(final Path folder) { // True if folder exists and is empty. - if (!Files.isDirectory(folder)) - return false; - try(DirectoryStream folderStream = Files.newDirectoryStream(folder)) { - return !folderStream.iterator().hasNext(); - } catch (IOException e) { - Logger.defaultLogError("Failed to open folder stream. folder=" + folder, e); - return false; - } - } - private void movePath(Path fromPath, Path toPath, String file, FileOption fileOption) throws ProCoreException { - Path from = fromPath.resolve(file); - Path to = toPath.resolve(file); - try { - if (FileOption.OVERWRITE.equals(fileOption)) - Files.move(from, to, REPLACE_EXISTING); - else - Files.move(from, to); - } catch (IOException e) { - throw new ProCoreException("Could not move " + from + " to " + to, e); - } - } - private static void copyTree(Path from, Path to, String path, final FileOption fileOption) throws ProCoreException { - copyTree(from.resolve(path), to.resolve(path), fileOption); - } - private static void copyTree(Path from, Path to, final FileOption fileOption) throws ProCoreException { - class Visitor extends SimpleFileVisitor { - private Path fromPath; - private Path toPath; - Visitor(Path from, Path to) { - fromPath = from; - toPath = to; - } - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - Path targetPath = toPath.resolve(fromPath.relativize(dir)); - if (!Files.exists(targetPath)) { - Files.createDirectory(targetPath); - } - return FileVisitResult.CONTINUE; - } - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - if (FileOption.OVERWRITE.equals(fileOption)) - Files.copy(file, toPath.resolve(fromPath.relativize(file)), COPY_ATTRIBUTES, REPLACE_EXISTING); - else - Files.copy(file, toPath.resolve(fromPath.relativize(file)), COPY_ATTRIBUTES); - return FileVisitResult.CONTINUE; - } - } - try { - Visitor v = new Visitor(from, to); - EnumSet opts = EnumSet.noneOf(FileVisitOption.class); - Files.walkFileTree(from, opts, Integer.MAX_VALUE, v); - } catch (IOException e) { - throw new ProCoreException("Could not copy " + from + " to " + to, e); - } - } - private static void deleteTree(Path path) throws ProCoreException { - if (!Files.exists(path)) - return; - class Visitor extends SimpleFileVisitor { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - try { - Files.delete(file); - } catch (IOException ioe) { - ioe.printStackTrace(); - throw ioe; - } - return FileVisitResult.CONTINUE; - } - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException e) throws IOException { - if (e == null) { - try { - Files.delete(dir); - } catch (IOException ioe) { - ioe.printStackTrace(); - throw ioe; - } - return FileVisitResult.CONTINUE; - } - throw e; - } - } - try { - Visitor v = new Visitor(); - EnumSet opts = EnumSet.noneOf(FileVisitOption.class); - Files.walkFileTree(path, opts, Integer.MAX_VALUE, v); - } catch (IOException e) { - throw new ProCoreException("Could not delete " + path, e); - } - } - private static void moveFolder(Path fromPath, Path toPath, String folder, FileOption fileOption) throws ProCoreException { - Path from = fromPath.resolve(folder); - Path to = toPath.resolve(folder); - copyTree(from, to, fileOption); - deleteTree(from); - } - enum FileOption { - IF_EXIST, - IF_NEW, - OVERWRITE - } - private void copyFiles(final Path from, final Path to, String pattern, final FileOption fileOption) throws ProCoreException { - class Visitor extends SimpleFileVisitor { - private final PathMatcher matcher; - Visitor(String pattern) { - matcher = FileSystems.getDefault().getPathMatcher("glob:" + pattern); - } - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - Path name = file.getFileName(); - if (null == name) - return FileVisitResult.CONTINUE; - else if (!matcher.matches(name)) - return FileVisitResult.CONTINUE; - if (FileOption.OVERWRITE.equals(fileOption)) - Files.copy(file, to.resolve(name), REPLACE_EXISTING); - else - Files.copy(file, to.resolve(name)); - return FileVisitResult.CONTINUE; - } - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - if (dir.equals(from)) - return FileVisitResult.CONTINUE; - else - return FileVisitResult.SKIP_SUBTREE; - } - } - try { - Visitor v = new Visitor(pattern); - EnumSet opts = EnumSet.noneOf(FileVisitOption.class); - Files.walkFileTree(from, opts, Integer.MAX_VALUE, v); - } catch (IOException e) { - throw new ProCoreException("Could not copy " + from + " to " + to, e); - } - } - private void moveFiles(final Path from, final Path to, String pattern, final FileOption fileOption) throws ProCoreException { - class Visitor extends SimpleFileVisitor { - private final PathMatcher matcher; - Visitor(String pattern) { - matcher = FileSystems.getDefault().getPathMatcher("glob:" + pattern); - } - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - Path name = file.getFileName(); - if (null == name) - return FileVisitResult.CONTINUE; - else if (!matcher.matches(name)) - return FileVisitResult.CONTINUE; - if (FileOption.OVERWRITE.equals(fileOption)) - Files.move(file, to.resolve(name), REPLACE_EXISTING); - else - Files.move(file, to.resolve(name)); - return FileVisitResult.CONTINUE; - } - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - if (dir.equals(from)) - return FileVisitResult.CONTINUE; - else - return FileVisitResult.SKIP_SUBTREE; - } - } - try { - Visitor v = new Visitor(pattern); - EnumSet opts = EnumSet.noneOf(FileVisitOption.class); - Files.walkFileTree(from, opts, Integer.MAX_VALUE, v); - } catch (IOException e) { - throw new ProCoreException("Could not move " + from + " to " + to, e); - } - } - private void deleteClusters(final Path head, final Path tail) throws ProCoreException { - class Visitor extends SimpleFileVisitor { - private final PathMatcher matcher; - Visitor(String pattern) { - matcher = FileSystems.getDefault().getPathMatcher("glob:" + ProCoreServer.DELETED_PATTERN); - } - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - Path name = file.getFileName(); - if (null == name) - return FileVisitResult.CONTINUE; - else if (!matcher.matches(name)) - return FileVisitResult.CONTINUE; - String deletedStr = name.toString(); - String indexName = deletedStr.replaceFirst(ProCoreServer.DELETED_PREFIX, ProCoreServer.INDEX_PREFIX); - String dataName = deletedStr.replaceFirst(ProCoreServer.DELETED_PREFIX, ProCoreServer.DATA_PREFIX); - String[] ss = deletedStr.split("\\."); - String valuePattern = ProCoreServer.VALUE_PREFIX + ss[1] + "." + ss[2] + "*"; - Files.delete(file); - Files.delete(head.resolve(indexName)); - Files.delete(head.resolve(dataName)); - try { - deleteFiles(head, valuePattern); - if (Files.isDirectory(tail)) { - Files.deleteIfExists(tail.resolve(dataName)); - deleteFiles(tail, valuePattern); - } - } catch (ProCoreException e) { - throw new IOException("Delete values failed.", e); - } - return FileVisitResult.CONTINUE; - } - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - if (dir.equals(head)) - return FileVisitResult.CONTINUE; - else - return FileVisitResult.SKIP_SUBTREE; - } - } - try { - Visitor v = new Visitor(ProCoreServer.DELETED_PREFIX); - EnumSet opts = EnumSet.noneOf(FileVisitOption.class); - Files.walkFileTree(head, opts, Integer.MAX_VALUE, v); - } catch (IOException e) { - throw new ProCoreException("Could not delete cluster(s).\nhead=" + head + "\ntail=" + tail); - } - } - private static long convertHexStringToLong(String hex) { - if (hex.length() < 16) - return Long.parseLong(hex, 16); - long f = Long.parseLong(hex.substring(0,1), 16) << 60; - long l = Long.parseLong(hex.substring(1,16), 16); - return f | l; - } - private void purgeValues(final Path folder) throws ProCoreException { - final class Value { - long first; // First part of cluster id. - long second; // Secon part of cluster id. - int index; // Resource index of the value. - transient long cs; // Change set of the value. - public Value(long first, long second, int index, long cs) { - this.first = first; - this.second = second; - this.index = index; - this.cs = cs; - } - String getName() { - return ProCoreServer.VALUE_PREFIX + toString() + ProCoreServer.VALUE_SUFFIX; - } - @Override - public String toString() { - return String.format("%x.%x.%d.%d", first, second, index, cs); - } - @Override - public boolean equals(Object o) { - if (this == o) - return true; - else if (!(o instanceof Value)) - return false; - Value x = (Value)o; - return first == x.first && second == x.second && index == x.index; - } - @Override - public int hashCode() { - int result = 17; - int f = (int)(first ^ (first >>> 32)); - result = 31 * result + f; - int s = (int)(second ^ (second >>> 32)); - result = 31 * result + s; - return result + index; - } - } - File[] files = folder.toFile().listFiles(new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return name.startsWith(ProCoreServer.VALUE_PREFIX); - } - }); - HashMap values = new HashMap(); - for (int i = 0; i < files.length; ++i) { - String s = files[i].getName(); - String[] ss = s.split("\\."); - if (ss.length != 6) { - Logger.defaultLogError("Illegal external value file name. name=" + s); - continue; - } - long first = convertHexStringToLong(ss[1]); - long second = convertHexStringToLong(ss[2]); - int ri = Integer.parseInt(ss[3]); - long cs = Long.parseLong(ss[4]); - Value nv = new Value(first, second, ri, cs); - Value ov = values.get(nv); - if (null == ov) - values.put(nv, nv); - else if (ov.cs < nv.cs) { - deleteFile(folder, ov.getName(), FileOption.IF_EXIST); - ov.cs = nv.cs;; - } else - deleteFile(folder, nv.getName(), FileOption.IF_EXIST); - } - } - private long getNextClusterId(final Path db) throws ProCoreException { - long clusterId = 0; - Path dbHead = db.resolve(ProCoreServer.BRANCH_DIR); - File[] files = dbHead.toFile().listFiles(new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return name.startsWith(ProCoreServer.DATA_PREFIX); - } - }); - for (int i = 0; i < files.length; ++i) { - String s = files[i].getName(); - String[] ss = s.split("\\.", 4); - if (ss.length != 4) { - Logger.defaultLogError("Illegal cluster file name. name=" + s); - continue; - } - long id = convertHexStringToLong(ss[2]); - if (id > clusterId) - clusterId = id; - } - final Path dbTail = db.resolve(ProCoreServer.TAIL_DIR); - if (!Files.exists(dbTail)) - return clusterId + 1; - class Visitor extends SimpleFileVisitor { - private final PathMatcher matcher; - long clusterId; - Visitor(String pattern, long clusterId) { - this.clusterId = clusterId; - matcher = FileSystems.getDefault().getPathMatcher("glob:" + pattern); - } - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - Path name = file.getFileName(); - if (null == name) - return FileVisitResult.CONTINUE; - else if (!matcher.matches(name)) - return FileVisitResult.CONTINUE; - String s = name.toString(); - String[] ss = s.split("\\.", 4); - if (ss.length != 4) { - Logger.defaultLogError("Illegal cluster file name. name=" + s); - return FileVisitResult.CONTINUE; - } - long id = convertHexStringToLong(ss[2]); - if (id > clusterId) - clusterId = id; - return FileVisitResult.CONTINUE; - } - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - if (dir.equals(dbTail)) - return FileVisitResult.CONTINUE; - else - return FileVisitResult.SKIP_SUBTREE; - } - } - try { - Visitor v = new Visitor(ProCoreServer.DATA_PATTERN, clusterId); - EnumSet opts = EnumSet.noneOf(FileVisitOption.class); - Files.walkFileTree(dbTail, opts, Integer.MAX_VALUE, v); - return v.clusterId + 1; - } catch (IOException e) { - throw new ProCoreException("Could not get next free cluster id for " + db, e); - } - } - private static void deleteFiles(final Path folder, String pattern) throws ProCoreException { - if (!Files.exists(folder)) - return; - class Visitor extends SimpleFileVisitor { - private final PathMatcher matcher; - Visitor(String pattern) { - matcher = FileSystems.getDefault().getPathMatcher("glob:" + pattern); - } - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - Path name = file.getFileName(); - if (null == name) - return FileVisitResult.CONTINUE; - else if (!matcher.matches(name)) - return FileVisitResult.CONTINUE; - Files.delete(file); - return FileVisitResult.CONTINUE; - } - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - if (dir.equals(folder)) - return FileVisitResult.CONTINUE; - else - return FileVisitResult.SKIP_SUBTREE; - } - } - try { - Visitor v = new Visitor(pattern); - EnumSet opts = EnumSet.noneOf(FileVisitOption.class); - Files.walkFileTree(folder, opts, Integer.MAX_VALUE, v); - } catch (IOException e) { - throw new ProCoreException("Could not delete " + folder + " with " + pattern, e); - } - } - public void copy(File to) throws ProCoreException { - copyDatabaseFiles(server.getFolder().toPath(), to.toPath()); - } - public boolean isServerAlive() { - return server.isAlive(); - } - Client newClient() throws ProCoreException { - return server.newClient(); - } - public void createFolder() throws ProCoreException { - if (server.folder.exists()) - return; - boolean created = server.folder.mkdirs(); - if (!created) - throw new ProCoreException("Could not create folder=" + server.folder); - } - /** - * @throws ProCoreException if could not stop server. - */ - public boolean serverTryToStop() throws ProCoreException { - return server.tryToStop(); - } - class Server { - class Command { - void Do(ProCoreServer proCoreServer) { - } - } - class Event { - } - private final File folder; // Always non null. - private final ProCoreServer proCoreServer; - void start() throws ProCoreException { - try { - proCoreServer.start(); - } catch (DatabaseLastExitException e) { - if (null == dbUserAgent) - return; - if (!dbUserAgent.handleStart(e)) - throw new ProCoreException(folder, "Failed to handle start exception.", e); - proCoreServer.start(); - } - } - void stop() throws ProCoreException { - try { - proCoreServer.stop(); - } catch (InterruptedException e) { - if (proCoreServer.isAlive()) - throw new ProCoreException("ProCoreServer stop was interrupted.", e); - } - } - boolean isAlive() { - try { - return proCoreServer.isAlive(); - } catch (ProCoreException e) { - Logger.defaultLogError(e); - return false; - } - } - Server(File aFolder) { - if (null == aFolder) - throw new RuntimeException("Database folder can not be null."); - try { - folder = aFolder.getCanonicalFile(); - } catch (IOException e) { - String t = "Could not get canonical path. file=" + aFolder; - Logger.defaultLogError(t); - throw new RuntimeException(t); - } - try { - File serverFolder = org.simantics.db.server.internal.Activator.getServerFolder(); - if (!folder.isDirectory()) - folder.mkdirs(); - proCoreServer = ProCoreServer.getProCoreServer(serverFolder, folder); - return; - } catch (Throwable t) { - Logger.defaultLogError(t); - throw new RuntimeException(t); - } - } - void createConfiguration() throws ProCoreException { - if (!folder.isDirectory()) - throw new ProCoreException("Can't create configuration because folder is not ok. folder=" + folder.getAbsolutePath()); - File file = new File(folder, ProCoreServer.CONFIG_FILE); - try { - file.createNewFile(); - } catch (IOException e) { - throw new ProCoreException("Can't create configuration file. file=" + file.getAbsolutePath()); - } - } - void deleteGuard() throws ProCoreException { - if (server.isActive()) - throw new ProCoreException("Will not delete guard file when server is alive."); - if (!server.isFolderOk(server.getFolder())) - throw new ProCoreException("Will not delete guard file when server folder is not ok. folder=" + server.getFolder()); - File file = new File(folder, ProCoreServer.GUARD_FILE); - if (!file.exists()) - throw new ProCoreException("Guard file does not exist. file=" + file.getAbsolutePath()); - boolean deleted = file.delete(); - if (!deleted) - throw new ProCoreException("Failed to delete file=" + file.getAbsolutePath()); - } - String execute(String aCommand) throws ProCoreException { - try { - return proCoreServer.execute(aCommand); - } catch (InterruptedException e) { - throw new ProCoreException("Execute call was interrupted.", e); - } - } - Status getStatus() { - Status status = Status.NoDatabase; - String path; - try { - path = folder.getCanonicalPath(); - } catch (IOException e) { - Util.logError("Could not get canonical path for folder. folder=" + folder.getAbsolutePath(), e); - path = ""; - } - if (!isFolderOk(folder)) - status = Status.NoDatabase; - else if (!isAlive()) - status = Status.NotRunning; - else if (isConnected()) { - if (isLocal()) - status = Status.Local; - else - status = Status.Remote; - } else - status = Status.Standalone; - status.message = status.getString() + "@" + path; - return status; - } - File getFolder() { - return folder; - } - boolean isActive() { - try { - return proCoreServer.isActive(); - } catch (ProCoreException e) { - Logger.defaultLogError("IsActive failed.", e); - return false; - } - } - boolean isConnected() { - try { - return proCoreServer.isConnected(); - } catch (ProCoreException e) { - Logger.defaultLogError("IsConnected failed.", e); - return false; - } - } - boolean isLocal() { - try { - return proCoreServer.isLocal(); - } catch (ProCoreException e) { - Logger.defaultLogError("IsLocal faailed.", e); - return false; - } - } - void connect() throws ProCoreException, InterruptedException { - proCoreServer.connect(); - } - void disconnect() { - try { - proCoreServer.disconnect(); - } catch (ProCoreException e) { - Logger.defaultLogError("Could not disconnect.", e); - } - } - /** - * @param aFolder - * @return true if given folder contains database journal file or - * configuration file. - */ - boolean isFolderOk(File aFolder) { - if (!aFolder.isDirectory()) - return false; - File config = new File(aFolder, ProCoreServer.CONFIG_FILE); - if (config.exists()) - return true; - File journal = new File(aFolder, ProCoreServer.JOURNAL_FILE); - if (journal.exists()) - return true; - return false; - } - /** - * @throws ProCoreException if could not stop server. - */ - boolean tryToStop() throws ProCoreException { - return proCoreServer.tryToStop(); - } - Client newClient() throws ProCoreException { - return proCoreServer.newClient(); - } - } -// From interface Database - @Override - public void initFolder(Properties properties) throws ProCoreException { - createFolder(); - serverCreateConfiguration(); - } - @Override - public void deleteFiles() throws ProCoreException { - deleteDatabaseFiles(); - } - @Override - public void start() throws ProCoreException { - connect(); - } - @Override - public boolean isRunning() throws ProCoreException { - return isServerAlive(); - } - @Override - public boolean tryToStop() throws ProCoreException { - return serverTryToStop(); - } - @Override - public String execute(String aCommand) throws ProCoreException { - return server.execute(aCommand); - } - @Override - public void purgeDatabase() throws SDBException { - synchronized (server.proCoreServer.getProCoreClient()) { - if (!server.isLocal()) - throw new ProCoreException("Purge is allowed only for local server."); - List clients = sessionManager.disconnect(this); - purge(); - try { - sessionManager.connect(this, clients); - } catch (InterruptedException e) { - throw new ProCoreException("Failed to connect after purge.", e); - } - } - } - @Override - public Session newSession(ServiceLocator locator) throws ProCoreException { - return sessionManager.newSession(this); - } - @Override - public Path createFromChangeSets(int revision) throws ProCoreException { - if (!isFolderOk()) - throw new ProCoreException("Folder must be valid database folder to create database from journal." ); - File folder = server.getFolder(); - File file = new File(folder, ProCoreServer.DCS_FILE); - if (!file.isFile() && !file.canRead()) - throw new ProCoreException("Dump file must be readable. file=" + file.getAbsolutePath()); - Path db = Paths.get(folder.getAbsolutePath()); - Path temp = createTempFolder(db, "Could not create temporary directory for database to be created from change sets."); - Server s = new Server(temp.toFile()); - s.createConfiguration(); - s.start(); - String t = s.execute("loadChangeSets .. " + revision); - if (t.length() < 1) - throw new ProCoreException("Could not read journal. reply=" + t); - s.tryToStop(); - try { - int cs = Integer.parseInt(t); - if (cs == revision) - return temp; - throw new ProCoreException("Could not load change sets. reply=" + t); - } catch (NumberFormatException e) { - throw new ProCoreException("Could not load change sets. reply=" + t); - } - } - @Override - public void clone(File to, int revision, boolean saveHistory) { - String history = saveHistory ? "with history." : "without history."; - String message = "Clone to " + to.getAbsolutePath() + "@" + revision + " " + history; - Util.trace(message); - } - @Override - public DatabaseUserAgent getUserAgent() { - return dbUserAgent; - } - @Override - public void setUserAgent(DatabaseUserAgent dbUserAgent) { - this.dbUserAgent = dbUserAgent; - } - @Override - public Status getStatus() { - return server.getStatus(); - } - @Override - public File getFolder() { - return server.getFolder(); - } - @Override - public boolean isFolderOk() { - return server.isFolderOk(server.getFolder()); - } - @Override - public boolean isFolderOk(File folder) { - return server.isFolderOk(folder); - } - public boolean isFolderEmpty() { - return isFolderEmpty(server.getFolder()); - } - @Override - public boolean isFolderEmpty(File folder) { - return isFolderEmpty(folder.toPath()); - } - @Override - public void deleteGuard() throws ProCoreException { - server.deleteGuard(); - } - @Override - public boolean isConnected() throws ProCoreException { - return server.isConnected(); - } - @Override - public void connect() throws ProCoreException { - if (!isFolderOk()) - throw new ProCoreException("Could not connect to " + getFolder()); - if (!server.isAlive()) - server.start(); - if (isConnected()) - return; - try { - server.connect(); - } catch (InterruptedException e) { - Util.logError("Server connect was interrupted.", e); - } - if (server.isActive()) - return; - throw new ProCoreException("Could not connect to " + getFolder()); - } - @Override - public void disconnect() throws ProCoreException { - server.disconnect(); - } - @Override - public Path dumpChangeSets() throws ProCoreException { - if (!isFolderOk()) - throw new ProCoreException("Folder must be set to dump change sets."); - if (!server.isActive()) - throw new ProCoreException("Server must be responsive to dump change sets."); - String t = server.execute("dumpChangeSets").replaceAll("\n", ""); - try { - int ncs = Integer.parseInt(t); - if (ncs < 1) - return null; - File file = new File(getFolder(), ProCoreServer.DCS_FILE); - return file.toPath(); - } catch (NumberFormatException e) { - throw new ProCoreException("Could not dump change sets.", e); - } - } - @Override - public long serverGetTailChangeSetId() throws ProCoreException { - try { - return server.proCoreServer.getTailData().nextChangeSetId; - } catch (TailReadException e) { - return 1; - } - } - @Override - public JournalI getJournal() throws ProCoreException { - return journal; - } - class JournalI implements Database.Journal { - class Analyzed { - private final File journalFile; - private long lastModified; - private RandomAccessFile file; // Journal file. - private ArrayList offsets = new ArrayList(); // Offsets to commands in journal file. - private Line lines[] = new Line[256]; - int firstLine = 0; // Index of the first element in lines table. - Analyzed(File dbFolder) { - journalFile = new File(dbFolder, ProCoreServer.JOURNAL_FILE); - for (int i=0; i metadata = (TreeMap) METADATA_SERIALIZER.deserialize(bytes); - CommitMetadata commit = MetadataUtil.getMetadata(metadata, CommitMetadata.class); - String date = ""; - if (null != commit && null != commit.date) - date = commit.date.toString(); - CommentMetadata comment = MetadataUtil.getMetadata(metadata, CommentMetadata.class); - String comments = ""; - if (null != comment) - comments = comment.toString(); - return date + " " + comments; - } catch (IOException e) { - throw new ProCoreException("Failed to interpret metadata.", e); - } - } - private Line getRow(int index, Line line) throws ProCoreException { - if (index < 0 || index >= offsets.size()) - return null; // Index out of range. - if (index < firstLine || index >= firstLine + lines.length) { - readLines(index); - if (index < firstLine) - return null; // Index out of range. - } - int offset = index - firstLine; - line.status = lines[offset].status; - line.request = lines[offset].request; - line.comment = lines[offset].comment; - return line; - } - private void readLines(int first) throws ProCoreException { - open(); - try { - for (int i=0, index = first; i= offsets.size()) { - line.status = false; - line.request = ""; - line.comment = ""; - return; - } - long offset = offsets.get(index); - try { - file.seek(offset); - int i = file.readInt(); - int length = Integer.reverseBytes(i); - byte b = file.readByte(); - boolean ok = b != 0; - b = file.readByte(); - // boolean littleEndian = b != 0; - i = file.readInt(); - int type = Integer.reverseBytes(i); - String comment = ""; - if (length < 6) - throw new ProCoreException("Journal file corrupted at" + file.getFilePointer() + "."); - else if (length > 6) { - if (type != 3) - file.skipBytes(length - 6); - else if (length > 22){ - file.skipBytes(16); - i = file.readInt(); - int size = Integer.reverseBytes(i); - if (size != length - 26) - throw new ProCoreException("Metadata corrupted at" + file.getFilePointer() + "."); - byte[] bytes = new byte[size]; - file.readFully(bytes); - comment = getComment(bytes); - if (null == comment) - comment = ""; - } - } - i = file.readInt(); - int length2 = Integer.reverseBytes(i); - if (length != length2) - throw new ProCoreException("Journal file corrupted at" + file.getFilePointer() + "."); - String command = MessageText.get(type); - line.status = ok; - line.request = command; - line.comment = comment; - } catch (IOException e) { - throw new ProCoreException("Journal file corrupted."); - } - } - private ArrayList readOffsets() throws ProCoreException { - if (!canRead()) { - lastModified = 0; - offsets.clear(); - firstLine = 0; - return offsets; - } - long modified = journalFile.lastModified(); - if (lastModified != 0 && lastModified == modified) - return offsets; // Offsets already up to date. - lastModified = 0; - offsets.clear(); - firstLine = 0; - open(); - try { - file.seek(0); - int i = file.readInt(); - int version = Integer.reverseBytes(i); - if (version != 1) - throw new ProCoreException("Unsupported journal file version. expected=1 got=" + version); - i = file.readInt(); - int major = Integer.reverseBytes(i); - if (major != MessageNumber.ProtocolVersionMajor) - throw new ProCoreException("Unsupported journal request major version. expected=" + MessageNumber.ProtocolVersionMajor + " got=" + major); - i = file.readInt(); - int minor = Integer.reverseBytes(i); - if (minor > MessageNumber.ProtocolVersionMinor) - throw new ProCoreException("Unsupported journal request minor version. expected=" + MessageNumber.ProtocolVersionMinor + " got=" + minor); - i = file.readInt(); - int length = Integer.reverseBytes(i); - while (length > 0) { // Not supporting unsigned integers. - if (length < 6) - throw new ProCoreException("Journal file corrupted at" + file.getFilePointer() + "."); - file.skipBytes(length); - i = file.readInt(); - int length2 = Integer.reverseBytes(i); - if (length != length2) - throw new ProCoreException("Journal file corrupted at " + file.getFilePointer() + "."); - long offset = file.getFilePointer() - length - 8; - offsets.add(offset); - i = file.readInt(); - length = Integer.reverseBytes(i); - } - } catch (EOFException e) { - } catch (IOException e) { - throw new ProCoreException("Failed to get command count.", e); - } finally { - close(); - } - lastModified = modified; - readLines(0); - return offsets; - } - } - private final Analyzed analyzed; - JournalI(File dbFolder) { - this.analyzed = new Analyzed(dbFolder); - } - @Override - public boolean canRead() { - return analyzed.canRead() && !isServerAlive(); - } - @Override - public int count() { - return analyzed.count(); - } - @Override - public int read(int index, Line line) throws ProCoreException { - int count = analyzed.count(); - analyzed.getRow(index, line); - return count; - } - } - @Override - public String getCompression() { - return "FLZ"; - } -} +package org.simantics.db.server.internal; + +import static java.nio.file.StandardCopyOption.COPY_ATTRIBUTES; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; + +import java.io.EOFException; +import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; +import java.io.OutputStream; +import java.io.RandomAccessFile; +import java.nio.file.DirectoryStream; +import java.nio.file.FileSystems; +import java.nio.file.FileVisitOption; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Properties; +import java.util.TreeMap; +import java.util.UUID; + +import org.simantics.databoard.Bindings; +import org.simantics.databoard.binding.impl.TreeMapBinding; +import org.simantics.databoard.serialization.Serializer; +import org.simantics.db.Database; +import org.simantics.db.DatabaseUserAgent; +import org.simantics.db.ServiceLocator; +import org.simantics.db.common.CommentMetadata; +import org.simantics.db.common.CommitMetadata; +import org.simantics.db.common.utils.Logger; +import org.simantics.db.common.utils.MetadataUtil; +import org.simantics.db.exception.SDBException; +import org.simantics.db.server.DatabaseLastExitException; +import org.simantics.db.server.ProCoreException; +import org.simantics.db.server.internal.ProCoreServer.TailFile; +import org.simantics.db.server.protocol.MessageNumber; +import org.simantics.db.server.protocol.MessageText; +import org.slf4j.LoggerFactory; + +public class DatabaseI implements Database { + + private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(DatabaseI.class); + + private static String NL = System.getProperty("line.separator"); + private static String TEMP_PREFIX = "db.temp."; + public static DatabaseI newDatabaseI(File dbFolder) { + return new DatabaseI(dbFolder); + } + private final SessionManager sessionManager = new SessionManager(); + private final Server server; + private final JournalI journal; + private DatabaseUserAgent dbUserAgent = null; + private DatabaseI(File dbFolder) { + server = new Server(dbFolder); + journal = new JournalI(dbFolder); + } + public boolean isSubFolder(Path base, Path sub) + throws IOException { + if (null == base || null == sub) + return false; + return isSubFolder(base.toFile(), sub.toFile()); + } + public boolean isSubFolder(File base, File sub) + throws IOException { + if (null == base || null == sub) + return false; + Path basePath = base.getCanonicalFile().toPath(); + Path subPath = sub.getCanonicalFile().toPath(); + if (subPath.startsWith(basePath)) + return true; + else + return false; + } + public Path saveDatabase() + throws ProCoreException { + Path dbFolder = getFolder().toPath(); + Path parent = getFolder().getParentFile().toPath(); + Path temp = createTempFolder(parent, "Could not create temporary directory for saving database."); + copyTree(dbFolder, temp, FileOption.IF_NEW); + return temp; + } + public Path saveDatabase(Path parent) + throws ProCoreException { + Path dbFolder = getFolder().toPath(); + try { + boolean yes = isSubFolder(dbFolder, parent); + if (yes) + throw new ProCoreException("Parent must not be subdirectory of database directory." + + NL + "database=" + dbFolder + + NL + "parent=" + parent); + } catch (IOException e) { + throw new ProCoreException("Failed to save database to parent." + + NL + "database=" + dbFolder + + NL + "parent=" + parent); + } + Path temp = createTempFolder(parent, "Could not create temporary directory for saving database."); + copyTree(dbFolder, temp, FileOption.IF_NEW); + return temp; + } + public void deleteDatabaseFiles() throws ProCoreException { + if (server.isAlive()) { + server.tryToStop(); + if (server.isAlive()) + throw new ProCoreException("Server must be dead to delete database."); + } + Path db = getFolder().toPath(); + deleteDatabaseFiles(db); // Redundant, but tests the method. + deleteTree(db); + } + Path copyDatabaseFiles(Path from, Path to) throws ProCoreException { + copyTree(from, to, ProCoreServer.BRANCH_DIR, FileOption.IF_EXIST); + if (Files.exists(from.resolve(ProCoreServer.TAIL_DIR))) + copyTree(from, to, ProCoreServer.TAIL_DIR, FileOption.IF_EXIST); + copyPath(from, to, ProCoreServer.CONFIG_FILE, FileOption.IF_EXIST); + copyPath(from, to, ProCoreServer.GUARD_FILE, FileOption.IF_EXIST); + copyPath(from, to, ProCoreServer.JOURNAL_FILE, FileOption.IF_EXIST); + copyPath(from, to, ProCoreServer.LOG_FILE, FileOption.IF_EXIST); + copyFiles(from, to, ProCoreServer.PAGE_FILE_PATTERN, FileOption.IF_EXIST); + return to; + } + public static void deleteDatabaseFiles(Path from) throws ProCoreException { + deleteFile(from, ProCoreServer.DCS_FILE, FileOption.IF_EXIST); + deleteFile(from, ProCoreServer.RECOVERY_NEEDED_FILE, FileOption.IF_EXIST); + deleteFile(from, ProCoreServer.RECOVERY_IGNORED_FILE, FileOption.IF_EXIST); + deleteFile(from, ProCoreServer.PROTOCOL_IGNORED_FILE, FileOption.IF_EXIST); + deleteFile(from, ProCoreServer.LOG_FILE, FileOption.IF_EXIST); + deleteFile(from, ProCoreServer.GUARD_FILE, FileOption.IF_EXIST); + deleteFile(from, ProCoreServer.CONFIG_FILE, FileOption.IF_EXIST); + deleteFiles(from, ProCoreServer.PAGE_FILE_PATTERN); + deleteTree(from.resolve(ProCoreServer.BRANCH_DIR)); + deleteFile(from, ProCoreServer.JOURNAL_FILE, FileOption.IF_EXIST); + deleteTree(from.resolve(ProCoreServer.TAIL_DIR)); + } + Path moveDatabaseFiles(Path from, Path to) throws ProCoreException { + moveFolder(from, to, ProCoreServer.BRANCH_DIR, FileOption.IF_NEW); + if (Files.exists(from.resolve(ProCoreServer.TAIL_DIR))) + moveFolder(from, to, ProCoreServer.TAIL_DIR, FileOption.IF_NEW); + movePath(from, to, ProCoreServer.CONFIG_FILE, FileOption.IF_NEW); + movePath(from, to, ProCoreServer.GUARD_FILE, FileOption.IF_NEW); + movePath(from, to, ProCoreServer.JOURNAL_FILE, FileOption.IF_NEW); + movePath(from, to, ProCoreServer.LOG_FILE, FileOption.IF_NEW); + moveFiles(from, to, ProCoreServer.PAGE_FILE_PATTERN, FileOption.IF_NEW); + movePath(from, to, ProCoreServer.DCS_FILE, FileOption.IF_NEW); + moveFile(from, to, ProCoreServer.RECOVERY_NEEDED_FILE, FileOption.IF_NEW); + moveFile(from, to, ProCoreServer.RECOVERY_IGNORED_FILE, FileOption.IF_NEW); + moveFile(from, to, ProCoreServer.PROTOCOL_IGNORED_FILE, FileOption.IF_NEW); + return to; + } + private void moveFile(Path from, Path to, String file, FileOption fileOption) throws ProCoreException { + if (Files.exists(from.resolve(file))) + movePath(from, to, file, fileOption); + } + public void serverCreateConfiguration() throws ProCoreException { + server.createConfiguration(); + } + public boolean ignoreExit() throws ProCoreException { + if (!isFolderOk()) + throw new ProCoreException("Folder must be valid database folder to ignore exit." ); + server.deleteGuard(); + File folder = server.getFolder(); + Path db = Paths.get(folder.getAbsolutePath()); + Path ri = db.resolve(ProCoreServer.RECOVERY_IGNORED_FILE); + if (!Files.exists(ri)) + try (OutputStream os = Files.newOutputStream(ri)) { + } catch (IOException e) { + throw new ProCoreException("Could not create file: " + ri, e); + } + boolean ok = false; + DatabaseUserAgent dbu = dbUserAgent; // Save current user agent. + try { + dbUserAgent = null; // Recursion not supported. Disable user agent. + server.start(); + ok = server.isActive(); + } finally { + try { + dbUserAgent = dbu; // Restore used user agent. + server.stop(); + } finally { + try { + Files.deleteIfExists(ri); + } catch (IOException e) { + Logger.defaultLogError("Could not delete file: " + ri, e); + } + Path rn = db.resolve(ProCoreServer.RECOVERY_NEEDED_FILE); + try { + Files.deleteIfExists(rn); + } catch (IOException e) { + Logger.defaultLogError("Could not delete file: " + rn, e); + } + } + } + return ok; + } + public boolean ignoreProtocol() throws ProCoreException { + if (!isFolderOk()) + throw new ProCoreException("Folder must be valid database folder to ignore exit." ); + File folder = server.getFolder(); + Path db = Paths.get(folder.getAbsolutePath()); + Path ri = db.resolve(ProCoreServer.PROTOCOL_IGNORED_FILE); + if (!Files.exists(ri)) + try (OutputStream os = Files.newOutputStream(ri)) { + } catch (IOException e) { + throw new ProCoreException("Could not create file: " + ri, e); + } + boolean ok = false; + try { + server.start(); + ok = server.isActive(); + } finally { + server.stop(); + } + return ok; + } + private long preJournalCheck() throws ProCoreException { + File folder = server.getFolder(); + if (!folder.isDirectory()) + throw new ProCoreException("Database folder does not exist." + NL + "folder=" + folder); + File file = new File(folder, ProCoreServer.JOURNAL_FILE); + if (!file.isFile()) + throw new ProCoreException("Journal file does not exist." + NL + "file=" + file); + else if (!file.canRead()) + throw new ProCoreException("Journal file must be readale to create database from journal." + NL + "file=" + file); + else if (server.isAlive()) + throw new ProCoreException("Server must be dead to create database from journal file." + NL + "file=" + file); + return getNextClusterId(folder.toPath()); + } + private void postJournalFix(long nextFreeId) throws SDBException { + Session s = newSession(null); + long current = s.reserveIds(0); + if (current < nextFreeId) + s.reserveIds((int)(nextFreeId - current)); + s.isClosed(); + } + private Path createTempFolder(Path parent, String fail) throws ProCoreException { + try { + return Files.createTempDirectory(parent, TEMP_PREFIX); + } catch (IOException e) { + throw new ProCoreException(fail, e); + } + } + public void replaceFromJournal() throws SDBException { + long nextFreeId = preJournalCheck(); + Path db = Paths.get(server.getFolder().getAbsolutePath()); + Path temp = createTempFolder(db, "Could not create temporary directory for database to be created from journal."); + movePath(db, temp, ProCoreServer.CONFIG_FILE, FileOption.IF_NEW); + movePath(db, temp, ProCoreServer.JOURNAL_FILE, FileOption.IF_NEW); + deleteFile(db, ProCoreServer.GUARD_FILE, FileOption.IF_EXIST); + deleteFiles(db, ProCoreServer.PAGE_FILE_PATTERN); + Path dbHead = db.resolve(ProCoreServer.BRANCH_DIR); + deleteFiles(dbHead, ProCoreServer.VALUE_PATTERN); + deleteFiles(dbHead, ProCoreServer.DATA_PATTERN); + deleteFiles(dbHead, ProCoreServer.INDEX_PATTERN); + Path dbHeadTailFile = dbHead.resolve(ProCoreServer.TAIL_FILE); + boolean headTailFileExisted = Files.exists(dbHeadTailFile); + final long NEXT_REVISION = 1; + if (!headTailFileExisted) // Number of change sets and database id not known at this point. Fortunately they are not used by recovery. + TailFile.createTailFile(dbHeadTailFile.toFile(), NEXT_REVISION, nextFreeId, UUID.randomUUID().toString()); + movePath(dbHead, temp, ProCoreServer.TAIL_FILE, FileOption.IF_NEW); + Path dbTail = db.resolve(ProCoreServer.TAIL_DIR); + boolean tailExisted = Files.isDirectory(dbTail); + if (tailExisted) + copyPath(dbTail, dbHead, ProCoreServer.TAIL_FILE, FileOption.IF_NEW); + else { + try { + Files.createDirectory(dbTail); + } catch (IOException e) { + throw new ProCoreException("Failed to create directory: " + dbTail, e); + } + copyPath(temp, dbTail, ProCoreServer.TAIL_FILE, FileOption.IF_NEW); + } + server.createConfiguration(); + server.start(); + try { + String t = server.execute("journalRead " + temp.getFileName()); + if (t.length() > 0) + throw new ProCoreException("Could not read journal. reply=" + t); + postJournalFix(nextFreeId); // Not the right way for implementing this, but better than incorrect recovery. + } finally { + server.tryToStop(); + } + movePath(temp, db, ProCoreServer.CONFIG_FILE, FileOption.OVERWRITE); + deleteTree(temp); + if (!tailExisted) + deleteTree(dbTail); + } + public boolean canPurge() throws ProCoreException { + File folder = server.getFolder(); + Path db = Paths.get(folder.getAbsolutePath()); + Path dbHead = db.resolve(ProCoreServer.BRANCH_DIR); + if (!Files.exists(dbHead)) + return false; // Already clean. + boolean empty = isFolderEmpty(dbHead); + if (empty) + return false; // Already clean. + final boolean[] found = new boolean[1]; + found[0] = false; + dbHead.toFile().listFiles(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + if (found[0]) + return false; + else if (!name.equals(ProCoreServer.TAIL_FILE)) { + found[0] = true; + return false; + } + return true; + } + }); + return found[0]; + } + /** + * Removes old history i.e. change sets. + * Note that also cleans last exit status and removes guard file. + * This means that all information about last exit status is also lost + * . + * @throws ProCoreException + */ + public void purge() throws SDBException { + synchronized (server.proCoreServer.getProCoreClient()) { + boolean wasAlive = server.isAlive(); + if (wasAlive) + server.stop(); + Path db = Paths.get(server.getFolder().getAbsolutePath()); + Path dbHead = db.resolve(ProCoreServer.BRANCH_DIR); + Path dbTail = db.resolve(ProCoreServer.TAIL_DIR); + if (!Files.isDirectory(dbTail)) { + try { + Files.createDirectory(dbTail); + } catch (IOException e) { + throw new ProCoreException("Failed to create directory: " + dbTail, e); + } + } + long nextFreeId = getNextClusterId(db); + boolean cleanHead = Files.isDirectory(dbHead) && !isFolderEmpty(dbHead); + LOGGER.info("Purging old history and exit information. folder=" + db); + if (cleanHead) { + deleteClusters(dbHead, dbTail); + movePath(dbHead, dbTail, ProCoreServer.TAIL_FILE, FileOption.OVERWRITE); + moveFiles(dbHead, dbTail, ProCoreServer.VALUE_PATTERN, FileOption.IF_NEW); + moveFiles(dbHead, dbTail, ProCoreServer.DATA_PATTERN, FileOption.OVERWRITE); + deleteFiles(dbHead, ProCoreServer.INDEX_PATTERN); + } + deleteFiles(db, ProCoreServer.PAGE_FILE_PATTERN); + deleteFile(db, ProCoreServer.JOURNAL_FILE, FileOption.IF_EXIST); + deleteFile(db, ProCoreServer.LOG_FILE, FileOption.IF_EXIST); + deleteFile(db, ProCoreServer.DCS_FILE, FileOption.IF_EXIST); + deleteFile(db, ProCoreServer.RECOVERY_NEEDED_FILE, FileOption.IF_EXIST); + deleteFile(db, ProCoreServer.RECOVERY_IGNORED_FILE, FileOption.IF_EXIST); + deleteFile(db, ProCoreServer.PROTOCOL_IGNORED_FILE, FileOption.IF_EXIST); + deleteFile(db, ProCoreServer.GUARD_FILE, FileOption.IF_EXIST); + purgeValues(dbTail); + server.start(); + Session s = newSession(null); + long current = s.reserveIds(0); + if (current < nextFreeId) + s.reserveIds((int)(nextFreeId - current)); + s.isClosed(); + if (!wasAlive) + server.tryToStop(); + } + } + private void copyPath(Path fromFolder, Path toFolder, String file, FileOption fileOption) throws ProCoreException { + Path from = fromFolder.resolve(file); + Path to = toFolder.resolve(file); + try { + if (FileOption.IF_EXIST.equals(fileOption)) { + if (!Files.exists(from)) + return; + } + if (FileOption.OVERWRITE.equals(fileOption)) + Files.copy(from, to, COPY_ATTRIBUTES, REPLACE_EXISTING); + else + Files.copy(from, to, COPY_ATTRIBUTES); + } catch (IOException e) { + throw new ProCoreException("Could not copy " + from + " to " + to, e); + } + } + private static void deleteFile(Path from, String file, FileOption fileOption) throws ProCoreException { + Path path = from.resolve(file); + deletePath(path, fileOption); + } + private static void deletePath(Path path, FileOption fileOption) throws ProCoreException { + try { + if (FileOption.IF_EXIST.equals(fileOption)) + Files.deleteIfExists(path); + else + Files.delete(path); + } catch (IOException e) { + throw new ProCoreException("Could not delete " + path, e); + } + } + private boolean isFolderEmpty(final Path folder) { // True if folder exists and is empty. + if (!Files.isDirectory(folder)) + return false; + try(DirectoryStream folderStream = Files.newDirectoryStream(folder)) { + return !folderStream.iterator().hasNext(); + } catch (IOException e) { + Logger.defaultLogError("Failed to open folder stream. folder=" + folder, e); + return false; + } + } + private void movePath(Path fromPath, Path toPath, String file, FileOption fileOption) throws ProCoreException { + Path from = fromPath.resolve(file); + Path to = toPath.resolve(file); + try { + if (FileOption.OVERWRITE.equals(fileOption)) + Files.move(from, to, REPLACE_EXISTING); + else + Files.move(from, to); + } catch (IOException e) { + throw new ProCoreException("Could not move " + from + " to " + to, e); + } + } + private static void copyTree(Path from, Path to, String path, final FileOption fileOption) throws ProCoreException { + copyTree(from.resolve(path), to.resolve(path), fileOption); + } + private static void copyTree(Path from, Path to, final FileOption fileOption) throws ProCoreException { + class Visitor extends SimpleFileVisitor { + private Path fromPath; + private Path toPath; + Visitor(Path from, Path to) { + fromPath = from; + toPath = to; + } + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + Path targetPath = toPath.resolve(fromPath.relativize(dir)); + if (!Files.exists(targetPath)) { + Files.createDirectory(targetPath); + } + return FileVisitResult.CONTINUE; + } + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + if (FileOption.OVERWRITE.equals(fileOption)) + Files.copy(file, toPath.resolve(fromPath.relativize(file)), COPY_ATTRIBUTES, REPLACE_EXISTING); + else + Files.copy(file, toPath.resolve(fromPath.relativize(file)), COPY_ATTRIBUTES); + return FileVisitResult.CONTINUE; + } + } + try { + Visitor v = new Visitor(from, to); + EnumSet opts = EnumSet.noneOf(FileVisitOption.class); + Files.walkFileTree(from, opts, Integer.MAX_VALUE, v); + } catch (IOException e) { + throw new ProCoreException("Could not copy " + from + " to " + to, e); + } + } + private static void deleteTree(Path path) throws ProCoreException { + if (!Files.exists(path)) + return; + class Visitor extends SimpleFileVisitor { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + try { + Files.delete(file); + } catch (IOException ioe) { + ioe.printStackTrace(); + throw ioe; + } + return FileVisitResult.CONTINUE; + } + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException e) throws IOException { + if (e == null) { + try { + Files.delete(dir); + } catch (IOException ioe) { + ioe.printStackTrace(); + throw ioe; + } + return FileVisitResult.CONTINUE; + } + throw e; + } + } + try { + Visitor v = new Visitor(); + EnumSet opts = EnumSet.noneOf(FileVisitOption.class); + Files.walkFileTree(path, opts, Integer.MAX_VALUE, v); + } catch (IOException e) { + throw new ProCoreException("Could not delete " + path, e); + } + } + private static void moveFolder(Path fromPath, Path toPath, String folder, FileOption fileOption) throws ProCoreException { + Path from = fromPath.resolve(folder); + Path to = toPath.resolve(folder); + copyTree(from, to, fileOption); + deleteTree(from); + } + enum FileOption { + IF_EXIST, + IF_NEW, + OVERWRITE + } + private void copyFiles(final Path from, final Path to, String pattern, final FileOption fileOption) throws ProCoreException { + class Visitor extends SimpleFileVisitor { + private final PathMatcher matcher; + Visitor(String pattern) { + matcher = FileSystems.getDefault().getPathMatcher("glob:" + pattern); + } + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Path name = file.getFileName(); + if (null == name) + return FileVisitResult.CONTINUE; + else if (!matcher.matches(name)) + return FileVisitResult.CONTINUE; + if (FileOption.OVERWRITE.equals(fileOption)) + Files.copy(file, to.resolve(name), REPLACE_EXISTING); + else + Files.copy(file, to.resolve(name)); + return FileVisitResult.CONTINUE; + } + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + if (dir.equals(from)) + return FileVisitResult.CONTINUE; + else + return FileVisitResult.SKIP_SUBTREE; + } + } + try { + Visitor v = new Visitor(pattern); + EnumSet opts = EnumSet.noneOf(FileVisitOption.class); + Files.walkFileTree(from, opts, Integer.MAX_VALUE, v); + } catch (IOException e) { + throw new ProCoreException("Could not copy " + from + " to " + to, e); + } + } + private void moveFiles(final Path from, final Path to, String pattern, final FileOption fileOption) throws ProCoreException { + class Visitor extends SimpleFileVisitor { + private final PathMatcher matcher; + Visitor(String pattern) { + matcher = FileSystems.getDefault().getPathMatcher("glob:" + pattern); + } + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Path name = file.getFileName(); + if (null == name) + return FileVisitResult.CONTINUE; + else if (!matcher.matches(name)) + return FileVisitResult.CONTINUE; + if (FileOption.OVERWRITE.equals(fileOption)) + Files.move(file, to.resolve(name), REPLACE_EXISTING); + else + Files.move(file, to.resolve(name)); + return FileVisitResult.CONTINUE; + } + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + if (dir.equals(from)) + return FileVisitResult.CONTINUE; + else + return FileVisitResult.SKIP_SUBTREE; + } + } + try { + Visitor v = new Visitor(pattern); + EnumSet opts = EnumSet.noneOf(FileVisitOption.class); + Files.walkFileTree(from, opts, Integer.MAX_VALUE, v); + } catch (IOException e) { + throw new ProCoreException("Could not move " + from + " to " + to, e); + } + } + private void deleteClusters(final Path head, final Path tail) throws ProCoreException { + class Visitor extends SimpleFileVisitor { + private final PathMatcher matcher; + Visitor(String pattern) { + matcher = FileSystems.getDefault().getPathMatcher("glob:" + ProCoreServer.DELETED_PATTERN); + } + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Path name = file.getFileName(); + if (null == name) + return FileVisitResult.CONTINUE; + else if (!matcher.matches(name)) + return FileVisitResult.CONTINUE; + String deletedStr = name.toString(); + String indexName = deletedStr.replaceFirst(ProCoreServer.DELETED_PREFIX, ProCoreServer.INDEX_PREFIX); + String dataName = deletedStr.replaceFirst(ProCoreServer.DELETED_PREFIX, ProCoreServer.DATA_PREFIX); + String[] ss = deletedStr.split("\\."); + String valuePattern = ProCoreServer.VALUE_PREFIX + ss[1] + "." + ss[2] + "*"; + Files.delete(file); + Files.delete(head.resolve(indexName)); + Files.delete(head.resolve(dataName)); + try { + deleteFiles(head, valuePattern); + if (Files.isDirectory(tail)) { + Files.deleteIfExists(tail.resolve(dataName)); + deleteFiles(tail, valuePattern); + } + } catch (ProCoreException e) { + throw new IOException("Delete values failed.", e); + } + return FileVisitResult.CONTINUE; + } + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + if (dir.equals(head)) + return FileVisitResult.CONTINUE; + else + return FileVisitResult.SKIP_SUBTREE; + } + } + try { + Visitor v = new Visitor(ProCoreServer.DELETED_PREFIX); + EnumSet opts = EnumSet.noneOf(FileVisitOption.class); + Files.walkFileTree(head, opts, Integer.MAX_VALUE, v); + } catch (IOException e) { + throw new ProCoreException("Could not delete cluster(s).\nhead=" + head + "\ntail=" + tail); + } + } + private static long convertHexStringToLong(String hex) { + if (hex.length() < 16) + return Long.parseLong(hex, 16); + long f = Long.parseLong(hex.substring(0,1), 16) << 60; + long l = Long.parseLong(hex.substring(1,16), 16); + return f | l; + } + private void purgeValues(final Path folder) throws ProCoreException { + final class Value { + long first; // First part of cluster id. + long second; // Secon part of cluster id. + int index; // Resource index of the value. + transient long cs; // Change set of the value. + public Value(long first, long second, int index, long cs) { + this.first = first; + this.second = second; + this.index = index; + this.cs = cs; + } + String getName() { + return ProCoreServer.VALUE_PREFIX + toString() + ProCoreServer.VALUE_SUFFIX; + } + @Override + public String toString() { + return String.format("%x.%x.%d.%d", first, second, index, cs); + } + @Override + public boolean equals(Object o) { + if (this == o) + return true; + else if (!(o instanceof Value)) + return false; + Value x = (Value)o; + return first == x.first && second == x.second && index == x.index; + } + @Override + public int hashCode() { + int result = 17; + int f = (int)(first ^ (first >>> 32)); + result = 31 * result + f; + int s = (int)(second ^ (second >>> 32)); + result = 31 * result + s; + return result + index; + } + } + File[] files = folder.toFile().listFiles(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.startsWith(ProCoreServer.VALUE_PREFIX); + } + }); + HashMap values = new HashMap(); + for (int i = 0; i < files.length; ++i) { + String s = files[i].getName(); + String[] ss = s.split("\\."); + if (ss.length != 6) { + Logger.defaultLogError("Illegal external value file name. name=" + s); + continue; + } + long first = convertHexStringToLong(ss[1]); + long second = convertHexStringToLong(ss[2]); + int ri = Integer.parseInt(ss[3]); + long cs = Long.parseLong(ss[4]); + Value nv = new Value(first, second, ri, cs); + Value ov = values.get(nv); + if (null == ov) + values.put(nv, nv); + else if (ov.cs < nv.cs) { + deleteFile(folder, ov.getName(), FileOption.IF_EXIST); + ov.cs = nv.cs;; + } else + deleteFile(folder, nv.getName(), FileOption.IF_EXIST); + } + } + private long getNextClusterId(final Path db) throws ProCoreException { + long clusterId = 0; + Path dbHead = db.resolve(ProCoreServer.BRANCH_DIR); + File[] files = dbHead.toFile().listFiles(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.startsWith(ProCoreServer.DATA_PREFIX); + } + }); + for (int i = 0; i < files.length; ++i) { + String s = files[i].getName(); + String[] ss = s.split("\\.", 4); + if (ss.length != 4) { + Logger.defaultLogError("Illegal cluster file name. name=" + s); + continue; + } + long id = convertHexStringToLong(ss[2]); + if (id > clusterId) + clusterId = id; + } + final Path dbTail = db.resolve(ProCoreServer.TAIL_DIR); + if (!Files.exists(dbTail)) + return clusterId + 1; + class Visitor extends SimpleFileVisitor { + private final PathMatcher matcher; + long clusterId; + Visitor(String pattern, long clusterId) { + this.clusterId = clusterId; + matcher = FileSystems.getDefault().getPathMatcher("glob:" + pattern); + } + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Path name = file.getFileName(); + if (null == name) + return FileVisitResult.CONTINUE; + else if (!matcher.matches(name)) + return FileVisitResult.CONTINUE; + String s = name.toString(); + String[] ss = s.split("\\.", 4); + if (ss.length != 4) { + Logger.defaultLogError("Illegal cluster file name. name=" + s); + return FileVisitResult.CONTINUE; + } + long id = convertHexStringToLong(ss[2]); + if (id > clusterId) + clusterId = id; + return FileVisitResult.CONTINUE; + } + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + if (dir.equals(dbTail)) + return FileVisitResult.CONTINUE; + else + return FileVisitResult.SKIP_SUBTREE; + } + } + try { + Visitor v = new Visitor(ProCoreServer.DATA_PATTERN, clusterId); + EnumSet opts = EnumSet.noneOf(FileVisitOption.class); + Files.walkFileTree(dbTail, opts, Integer.MAX_VALUE, v); + return v.clusterId + 1; + } catch (IOException e) { + throw new ProCoreException("Could not get next free cluster id for " + db, e); + } + } + private static void deleteFiles(final Path folder, String pattern) throws ProCoreException { + if (!Files.exists(folder)) + return; + class Visitor extends SimpleFileVisitor { + private final PathMatcher matcher; + Visitor(String pattern) { + matcher = FileSystems.getDefault().getPathMatcher("glob:" + pattern); + } + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Path name = file.getFileName(); + if (null == name) + return FileVisitResult.CONTINUE; + else if (!matcher.matches(name)) + return FileVisitResult.CONTINUE; + Files.delete(file); + return FileVisitResult.CONTINUE; + } + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + if (dir.equals(folder)) + return FileVisitResult.CONTINUE; + else + return FileVisitResult.SKIP_SUBTREE; + } + } + try { + Visitor v = new Visitor(pattern); + EnumSet opts = EnumSet.noneOf(FileVisitOption.class); + Files.walkFileTree(folder, opts, Integer.MAX_VALUE, v); + } catch (IOException e) { + throw new ProCoreException("Could not delete " + folder + " with " + pattern, e); + } + } + public void copy(File to) throws ProCoreException { + copyDatabaseFiles(server.getFolder().toPath(), to.toPath()); + } + public boolean isServerAlive() { + return server.isAlive(); + } + Client newClient() throws ProCoreException { + return server.newClient(); + } + public void createFolder() throws ProCoreException { + if (server.folder.exists()) + return; + boolean created = server.folder.mkdirs(); + if (!created) + throw new ProCoreException("Could not create folder=" + server.folder); + } + /** + * @throws ProCoreException if could not stop server. + */ + public boolean serverTryToStop() throws ProCoreException { + return server.tryToStop(); + } + class Server { + class Command { + void Do(ProCoreServer proCoreServer) { + } + } + class Event { + } + private final File folder; // Always non null. + private final ProCoreServer proCoreServer; + void start() throws ProCoreException { + try { + proCoreServer.start(); + } catch (DatabaseLastExitException e) { + if (null == dbUserAgent) + return; + if (!dbUserAgent.handleStart(e)) + throw new ProCoreException(folder, "Failed to handle start exception.", e); + proCoreServer.start(); + } + } + void stop() throws ProCoreException { + try { + proCoreServer.stop(); + } catch (InterruptedException e) { + if (proCoreServer.isAlive()) + throw new ProCoreException("ProCoreServer stop was interrupted.", e); + } + } + boolean isAlive() { + try { + return proCoreServer.isAlive(); + } catch (ProCoreException e) { + Logger.defaultLogError(e); + return false; + } + } + Server(File aFolder) { + if (null == aFolder) + throw new RuntimeException("Database folder can not be null."); + try { + folder = aFolder.getCanonicalFile(); + } catch (IOException e) { + String t = "Could not get canonical path. file=" + aFolder; + Logger.defaultLogError(t); + throw new RuntimeException(t); + } + try { + File serverFolder = org.simantics.db.server.internal.Activator.getServerFolder(); + if (!folder.isDirectory()) + folder.mkdirs(); + proCoreServer = ProCoreServer.getProCoreServer(serverFolder, folder); + return; + } catch (Throwable t) { + Logger.defaultLogError(t); + throw new RuntimeException(t); + } + } + void createConfiguration() throws ProCoreException { + if (!folder.isDirectory()) + throw new ProCoreException("Can't create configuration because folder is not ok. folder=" + folder.getAbsolutePath()); + File file = new File(folder, ProCoreServer.CONFIG_FILE); + try { + file.createNewFile(); + } catch (IOException e) { + throw new ProCoreException("Can't create configuration file. file=" + file.getAbsolutePath()); + } + } + void deleteGuard() throws ProCoreException { + if (server.isActive()) + throw new ProCoreException("Will not delete guard file when server is alive."); + if (!server.isFolderOk(server.getFolder())) + throw new ProCoreException("Will not delete guard file when server folder is not ok. folder=" + server.getFolder()); + File file = new File(folder, ProCoreServer.GUARD_FILE); + if (!file.exists()) + throw new ProCoreException("Guard file does not exist. file=" + file.getAbsolutePath()); + boolean deleted = file.delete(); + if (!deleted) + throw new ProCoreException("Failed to delete file=" + file.getAbsolutePath()); + } + String execute(String aCommand) throws ProCoreException { + try { + return proCoreServer.execute(aCommand); + } catch (InterruptedException e) { + throw new ProCoreException("Execute call was interrupted.", e); + } + } + Status getStatus() { + Status status = Status.NoDatabase; + String path; + try { + path = folder.getCanonicalPath(); + } catch (IOException e) { + Util.logError("Could not get canonical path for folder. folder=" + folder.getAbsolutePath(), e); + path = ""; + } + if (!isFolderOk(folder)) + status = Status.NoDatabase; + else if (!isAlive()) + status = Status.NotRunning; + else if (isConnected()) { + if (isLocal()) + status = Status.Local; + else + status = Status.Remote; + } else + status = Status.Standalone; + status.message = status.getString() + "@" + path; + return status; + } + File getFolder() { + return folder; + } + boolean isActive() { + try { + return proCoreServer.isActive(); + } catch (ProCoreException e) { + Logger.defaultLogError("IsActive failed.", e); + return false; + } + } + boolean isConnected() { + try { + return proCoreServer.isConnected(); + } catch (ProCoreException e) { + Logger.defaultLogError("IsConnected failed.", e); + return false; + } + } + boolean isLocal() { + try { + return proCoreServer.isLocal(); + } catch (ProCoreException e) { + Logger.defaultLogError("IsLocal faailed.", e); + return false; + } + } + void connect() throws ProCoreException, InterruptedException { + proCoreServer.connect(); + } + void disconnect() { + try { + proCoreServer.disconnect(); + } catch (ProCoreException e) { + Logger.defaultLogError("Could not disconnect.", e); + } + } + /** + * @param aFolder + * @return true if given folder contains database journal file or + * configuration file. + */ + boolean isFolderOk(File aFolder) { + if (!aFolder.isDirectory()) + return false; + File config = new File(aFolder, ProCoreServer.CONFIG_FILE); + if (config.exists()) + return true; + File journal = new File(aFolder, ProCoreServer.JOURNAL_FILE); + if (journal.exists()) + return true; + return false; + } + /** + * @throws ProCoreException if could not stop server. + */ + boolean tryToStop() throws ProCoreException { + return proCoreServer.tryToStop(); + } + Client newClient() throws ProCoreException { + return proCoreServer.newClient(); + } + } +// From interface Database + @Override + public void initFolder(Properties properties) throws ProCoreException { + createFolder(); + serverCreateConfiguration(); + } + @Override + public void deleteFiles() throws ProCoreException { + deleteDatabaseFiles(); + } + @Override + public void start() throws ProCoreException { + connect(); + } + @Override + public boolean isRunning() throws ProCoreException { + return isServerAlive(); + } + @Override + public boolean tryToStop() throws ProCoreException { + return serverTryToStop(); + } + @Override + public String execute(String aCommand) throws ProCoreException { + return server.execute(aCommand); + } + @Override + public void purgeDatabase() throws SDBException { + synchronized (server.proCoreServer.getProCoreClient()) { + if (!server.isLocal()) + throw new ProCoreException("Purge is allowed only for local server."); + List clients = sessionManager.disconnect(this); + purge(); + try { + sessionManager.connect(this, clients); + } catch (InterruptedException e) { + throw new ProCoreException("Failed to connect after purge.", e); + } + } + } + @Override + public Session newSession(ServiceLocator locator) throws ProCoreException { + return sessionManager.newSession(this); + } + @Override + public Path createFromChangeSets(int revision) throws ProCoreException { + if (!isFolderOk()) + throw new ProCoreException("Folder must be valid database folder to create database from journal." ); + File folder = server.getFolder(); + File file = new File(folder, ProCoreServer.DCS_FILE); + if (!file.isFile() && !file.canRead()) + throw new ProCoreException("Dump file must be readable. file=" + file.getAbsolutePath()); + Path db = Paths.get(folder.getAbsolutePath()); + Path temp = createTempFolder(db, "Could not create temporary directory for database to be created from change sets."); + Server s = new Server(temp.toFile()); + s.createConfiguration(); + s.start(); + String t = s.execute("loadChangeSets .. " + revision); + if (t.length() < 1) + throw new ProCoreException("Could not read journal. reply=" + t); + s.tryToStop(); + try { + int cs = Integer.parseInt(t); + if (cs == revision) + return temp; + throw new ProCoreException("Could not load change sets. reply=" + t); + } catch (NumberFormatException e) { + throw new ProCoreException("Could not load change sets. reply=" + t); + } + } + @Override + public void clone(File to, int revision, boolean saveHistory) { + String history = saveHistory ? "with history." : "without history."; + String message = "Clone to " + to.getAbsolutePath() + "@" + revision + " " + history; + Util.trace(message); + } + @Override + public DatabaseUserAgent getUserAgent() { + return dbUserAgent; + } + @Override + public void setUserAgent(DatabaseUserAgent dbUserAgent) { + this.dbUserAgent = dbUserAgent; + } + @Override + public Status getStatus() { + return server.getStatus(); + } + @Override + public File getFolder() { + return server.getFolder(); + } + @Override + public boolean isFolderOk() { + return server.isFolderOk(server.getFolder()); + } + @Override + public boolean isFolderOk(File folder) { + return server.isFolderOk(folder); + } + public boolean isFolderEmpty() { + return isFolderEmpty(server.getFolder()); + } + @Override + public boolean isFolderEmpty(File folder) { + return isFolderEmpty(folder.toPath()); + } + @Override + public void deleteGuard() throws ProCoreException { + server.deleteGuard(); + } + @Override + public boolean isConnected() throws ProCoreException { + return server.isConnected(); + } + @Override + public void connect() throws ProCoreException { + if (!isFolderOk()) + throw new ProCoreException("Could not connect to " + getFolder()); + if (!server.isAlive()) + server.start(); + if (isConnected()) + return; + try { + server.connect(); + } catch (InterruptedException e) { + Util.logError("Server connect was interrupted.", e); + } + if (server.isActive()) + return; + throw new ProCoreException("Could not connect to " + getFolder()); + } + @Override + public void disconnect() throws ProCoreException { + server.disconnect(); + } + @Override + public Path dumpChangeSets() throws ProCoreException { + if (!isFolderOk()) + throw new ProCoreException("Folder must be set to dump change sets."); + if (!server.isActive()) + throw new ProCoreException("Server must be responsive to dump change sets."); + String t = server.execute("dumpChangeSets").replaceAll("\n", ""); + try { + int ncs = Integer.parseInt(t); + if (ncs < 1) + return null; + File file = new File(getFolder(), ProCoreServer.DCS_FILE); + return file.toPath(); + } catch (NumberFormatException e) { + throw new ProCoreException("Could not dump change sets.", e); + } + } + @Override + public long serverGetTailChangeSetId() throws ProCoreException { + try { + return server.proCoreServer.getTailData().nextChangeSetId; + } catch (TailReadException e) { + return 1; + } + } + @Override + public JournalI getJournal() throws ProCoreException { + return journal; + } + class JournalI implements Database.Journal { + class Analyzed { + private final File journalFile; + private long lastModified; + private RandomAccessFile file; // Journal file. + private ArrayList offsets = new ArrayList(); // Offsets to commands in journal file. + private Line lines[] = new Line[256]; + int firstLine = 0; // Index of the first element in lines table. + Analyzed(File dbFolder) { + journalFile = new File(dbFolder, ProCoreServer.JOURNAL_FILE); + for (int i=0; i metadata = (TreeMap) METADATA_SERIALIZER.deserialize(bytes); + CommitMetadata commit = MetadataUtil.getMetadata(metadata, CommitMetadata.class); + String date = ""; + if (null != commit && null != commit.date) + date = commit.date.toString(); + CommentMetadata comment = MetadataUtil.getMetadata(metadata, CommentMetadata.class); + String comments = ""; + if (null != comment) + comments = comment.toString(); + return date + " " + comments; + } catch (IOException e) { + throw new ProCoreException("Failed to interpret metadata.", e); + } + } + private Line getRow(int index, Line line) throws ProCoreException { + if (index < 0 || index >= offsets.size()) + return null; // Index out of range. + if (index < firstLine || index >= firstLine + lines.length) { + readLines(index); + if (index < firstLine) + return null; // Index out of range. + } + int offset = index - firstLine; + line.status = lines[offset].status; + line.request = lines[offset].request; + line.comment = lines[offset].comment; + return line; + } + private void readLines(int first) throws ProCoreException { + open(); + try { + for (int i=0, index = first; i= offsets.size()) { + line.status = false; + line.request = ""; + line.comment = ""; + return; + } + long offset = offsets.get(index); + try { + file.seek(offset); + int i = file.readInt(); + int length = Integer.reverseBytes(i); + byte b = file.readByte(); + boolean ok = b != 0; + b = file.readByte(); + // boolean littleEndian = b != 0; + i = file.readInt(); + int type = Integer.reverseBytes(i); + String comment = ""; + if (length < 6) + throw new ProCoreException("Journal file corrupted at" + file.getFilePointer() + "."); + else if (length > 6) { + if (type != 3) + file.skipBytes(length - 6); + else if (length > 22){ + file.skipBytes(16); + i = file.readInt(); + int size = Integer.reverseBytes(i); + if (size != length - 26) + throw new ProCoreException("Metadata corrupted at" + file.getFilePointer() + "."); + byte[] bytes = new byte[size]; + file.readFully(bytes); + comment = getComment(bytes); + if (null == comment) + comment = ""; + } + } + i = file.readInt(); + int length2 = Integer.reverseBytes(i); + if (length != length2) + throw new ProCoreException("Journal file corrupted at" + file.getFilePointer() + "."); + String command = MessageText.get(type); + line.status = ok; + line.request = command; + line.comment = comment; + } catch (IOException e) { + throw new ProCoreException("Journal file corrupted."); + } + } + private ArrayList readOffsets() throws ProCoreException { + if (!canRead()) { + lastModified = 0; + offsets.clear(); + firstLine = 0; + return offsets; + } + long modified = journalFile.lastModified(); + if (lastModified != 0 && lastModified == modified) + return offsets; // Offsets already up to date. + lastModified = 0; + offsets.clear(); + firstLine = 0; + open(); + try { + file.seek(0); + int i = file.readInt(); + int version = Integer.reverseBytes(i); + if (version != 1) + throw new ProCoreException("Unsupported journal file version. expected=1 got=" + version); + i = file.readInt(); + int major = Integer.reverseBytes(i); + if (major != MessageNumber.ProtocolVersionMajor) + throw new ProCoreException("Unsupported journal request major version. expected=" + MessageNumber.ProtocolVersionMajor + " got=" + major); + i = file.readInt(); + int minor = Integer.reverseBytes(i); + if (minor > MessageNumber.ProtocolVersionMinor) + throw new ProCoreException("Unsupported journal request minor version. expected=" + MessageNumber.ProtocolVersionMinor + " got=" + minor); + i = file.readInt(); + int length = Integer.reverseBytes(i); + while (length > 0) { // Not supporting unsigned integers. + if (length < 6) + throw new ProCoreException("Journal file corrupted at" + file.getFilePointer() + "."); + file.skipBytes(length); + i = file.readInt(); + int length2 = Integer.reverseBytes(i); + if (length != length2) + throw new ProCoreException("Journal file corrupted at " + file.getFilePointer() + "."); + long offset = file.getFilePointer() - length - 8; + offsets.add(offset); + i = file.readInt(); + length = Integer.reverseBytes(i); + } + } catch (EOFException e) { + } catch (IOException e) { + throw new ProCoreException("Failed to get command count.", e); + } finally { + close(); + } + lastModified = modified; + readLines(0); + return offsets; + } + } + private final Analyzed analyzed; + JournalI(File dbFolder) { + this.analyzed = new Analyzed(dbFolder); + } + @Override + public boolean canRead() { + return analyzed.canRead() && !isServerAlive(); + } + @Override + public int count() { + return analyzed.count(); + } + @Override + public int read(int index, Line line) throws ProCoreException { + int count = analyzed.count(); + analyzed.getRow(index, line); + return count; + } + } + @Override + public String getCompression() { + return "FLZ"; + } +}