X-Git-Url: https://gerrit.simantics.org/r/gitweb?a=blobdiff_plain;f=bundles%2Forg.simantics.db.server%2Fsrc%2Forg%2Fsimantics%2Fdb%2Fserver%2Finternal%2FDatabaseI.java;fp=bundles%2Forg.simantics.db.server%2Fsrc%2Forg%2Fsimantics%2Fdb%2Fserver%2Finternal%2FDatabaseI.java;h=9d03ee4ae0df604cbfe5f46f7e2db42e6f1ebed8;hb=969bd23cab98a79ca9101af33334000879fb60c5;hp=0000000000000000000000000000000000000000;hpb=866dba5cd5a3929bbeae85991796acb212338a08;p=simantics%2Fplatform.git diff --git a/bundles/org.simantics.db.server/src/org/simantics/db/server/internal/DatabaseI.java b/bundles/org.simantics.db.server/src/org/simantics/db/server/internal/DatabaseI.java new file mode 100644 index 000000000..9d03ee4ae --- /dev/null +++ b/bundles/org.simantics.db.server/src/org/simantics/db/server/internal/DatabaseI.java @@ -0,0 +1,1366 @@ +package org.simantics.db.server.internal; + +import static java.nio.file.StandardCopyOption.COPY_ATTRIBUTES; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; + +import java.io.EOFException; +import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; +import java.io.OutputStream; +import java.io.RandomAccessFile; +import java.nio.file.DirectoryStream; +import java.nio.file.FileSystems; +import java.nio.file.FileVisitOption; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Properties; +import java.util.TreeMap; +import java.util.UUID; + +import org.simantics.databoard.Bindings; +import org.simantics.databoard.binding.impl.TreeMapBinding; +import org.simantics.databoard.serialization.Serializer; +import org.simantics.db.Database; +import org.simantics.db.DatabaseUserAgent; +import org.simantics.db.ServiceLocator; +import org.simantics.db.common.CommentMetadata; +import org.simantics.db.common.CommitMetadata; +import org.simantics.db.common.utils.Logger; +import org.simantics.db.common.utils.MetadataUtil; +import org.simantics.db.exception.SDBException; +import org.simantics.db.server.DatabaseLastExitException; +import org.simantics.db.server.ProCoreException; +import org.simantics.db.server.internal.ProCoreServer.TailFile; +import org.simantics.db.server.protocol.MessageNumber; +import org.simantics.db.server.protocol.MessageText; + +public class DatabaseI implements Database { + private static String NL = System.getProperty("line.separator"); + private static String TEMP_PREFIX = "db.temp."; + public static DatabaseI newDatabaseI(File dbFolder) { + return new DatabaseI(dbFolder); + } + private final SessionManager sessionManager = new SessionManager(); + private final Server server; + private final JournalI journal; + private DatabaseUserAgent dbUserAgent = null; + private DatabaseI(File dbFolder) { + server = new Server(dbFolder); + journal = new JournalI(dbFolder); + } + public boolean isSubFolder(Path base, Path sub) + throws IOException { + if (null == base || null == sub) + return false; + return isSubFolder(base.toFile(), sub.toFile()); + } + public boolean isSubFolder(File base, File sub) + throws IOException { + if (null == base || null == sub) + return false; + Path basePath = base.getCanonicalFile().toPath(); + Path subPath = sub.getCanonicalFile().toPath(); + if (subPath.startsWith(basePath)) + return true; + else + return false; + } + public Path saveDatabase() + throws ProCoreException { + Path dbFolder = getFolder().toPath(); + Path parent = getFolder().getParentFile().toPath(); + Path temp = createTempFolder(parent, "Could not create temporary directory for saving database."); + copyTree(dbFolder, temp, FileOption.IF_NEW); + return temp; + } + public Path saveDatabase(Path parent) + throws ProCoreException { + Path dbFolder = getFolder().toPath(); + try { + boolean yes = isSubFolder(dbFolder, parent); + if (yes) + throw new ProCoreException("Parent must not be subdirectory of database directory." + + NL + "database=" + dbFolder + + NL + "parent=" + parent); + } catch (IOException e) { + throw new ProCoreException("Failed to save database to parent." + + NL + "database=" + dbFolder + + NL + "parent=" + parent); + } + Path temp = createTempFolder(parent, "Could not create temporary directory for saving database."); + copyTree(dbFolder, temp, FileOption.IF_NEW); + return temp; + } + public void deleteDatabaseFiles() throws ProCoreException { + if (server.isAlive()) { + server.tryToStop(); + if (server.isAlive()) + throw new ProCoreException("Server must be dead to delete database."); + } + Path db = getFolder().toPath(); + deleteDatabaseFiles(db); // Redundant, but tests the method. + deleteTree(db); + } + Path copyDatabaseFiles(Path from, Path to) throws ProCoreException { + copyTree(from, to, ProCoreServer.BRANCH_DIR, FileOption.IF_EXIST); + if (Files.exists(from.resolve(ProCoreServer.TAIL_DIR))) + copyTree(from, to, ProCoreServer.TAIL_DIR, FileOption.IF_EXIST); + copyPath(from, to, ProCoreServer.CONFIG_FILE, FileOption.IF_EXIST); + copyPath(from, to, ProCoreServer.GUARD_FILE, FileOption.IF_EXIST); + copyPath(from, to, ProCoreServer.JOURNAL_FILE, FileOption.IF_EXIST); + copyPath(from, to, ProCoreServer.LOG_FILE, FileOption.IF_EXIST); + copyFiles(from, to, ProCoreServer.PAGE_FILE_PATTERN, FileOption.IF_EXIST); + return to; + } + public static void deleteDatabaseFiles(Path from) throws ProCoreException { + deleteFile(from, ProCoreServer.DCS_FILE, FileOption.IF_EXIST); + deleteFile(from, ProCoreServer.RECOVERY_NEEDED_FILE, FileOption.IF_EXIST); + deleteFile(from, ProCoreServer.RECOVERY_IGNORED_FILE, FileOption.IF_EXIST); + deleteFile(from, ProCoreServer.PROTOCOL_IGNORED_FILE, FileOption.IF_EXIST); + deleteFile(from, ProCoreServer.LOG_FILE, FileOption.IF_EXIST); + deleteFile(from, ProCoreServer.GUARD_FILE, FileOption.IF_EXIST); + deleteFile(from, ProCoreServer.CONFIG_FILE, FileOption.IF_EXIST); + deleteFiles(from, ProCoreServer.PAGE_FILE_PATTERN); + deleteTree(from.resolve(ProCoreServer.BRANCH_DIR)); + deleteFile(from, ProCoreServer.JOURNAL_FILE, FileOption.IF_EXIST); + deleteTree(from.resolve(ProCoreServer.TAIL_DIR)); + } + Path moveDatabaseFiles(Path from, Path to) throws ProCoreException { + moveFolder(from, to, ProCoreServer.BRANCH_DIR, FileOption.IF_NEW); + if (Files.exists(from.resolve(ProCoreServer.TAIL_DIR))) + moveFolder(from, to, ProCoreServer.TAIL_DIR, FileOption.IF_NEW); + movePath(from, to, ProCoreServer.CONFIG_FILE, FileOption.IF_NEW); + movePath(from, to, ProCoreServer.GUARD_FILE, FileOption.IF_NEW); + movePath(from, to, ProCoreServer.JOURNAL_FILE, FileOption.IF_NEW); + movePath(from, to, ProCoreServer.LOG_FILE, FileOption.IF_NEW); + moveFiles(from, to, ProCoreServer.PAGE_FILE_PATTERN, FileOption.IF_NEW); + movePath(from, to, ProCoreServer.DCS_FILE, FileOption.IF_NEW); + moveFile(from, to, ProCoreServer.RECOVERY_NEEDED_FILE, FileOption.IF_NEW); + moveFile(from, to, ProCoreServer.RECOVERY_IGNORED_FILE, FileOption.IF_NEW); + moveFile(from, to, ProCoreServer.PROTOCOL_IGNORED_FILE, FileOption.IF_NEW); + return to; + } + private void moveFile(Path from, Path to, String file, FileOption fileOption) throws ProCoreException { + if (Files.exists(from.resolve(file))) + movePath(from, to, file, fileOption); + } + public void serverCreateConfiguration() throws ProCoreException { + server.createConfiguration(); + } + public boolean ignoreExit() throws ProCoreException { + if (!isFolderOk()) + throw new ProCoreException("Folder must be valid database folder to ignore exit." ); + server.deleteGuard(); + File folder = server.getFolder(); + Path db = Paths.get(folder.getAbsolutePath()); + Path ri = db.resolve(ProCoreServer.RECOVERY_IGNORED_FILE); + if (!Files.exists(ri)) + try (OutputStream os = Files.newOutputStream(ri)) { + } catch (IOException e) { + throw new ProCoreException("Could not create file: " + ri, e); + } + boolean ok = false; + DatabaseUserAgent dbu = dbUserAgent; // Save current user agent. + try { + dbUserAgent = null; // Recursion not supported. Disable user agent. + server.start(); + ok = server.isActive(); + } finally { + try { + dbUserAgent = dbu; // Restore used user agent. + server.stop(); + } finally { + try { + Files.deleteIfExists(ri); + } catch (IOException e) { + Logger.defaultLogError("Could not delete file: " + ri, e); + } + Path rn = db.resolve(ProCoreServer.RECOVERY_NEEDED_FILE); + try { + Files.deleteIfExists(rn); + } catch (IOException e) { + Logger.defaultLogError("Could not delete file: " + rn, e); + } + } + } + return ok; + } + public boolean ignoreProtocol() throws ProCoreException { + if (!isFolderOk()) + throw new ProCoreException("Folder must be valid database folder to ignore exit." ); + File folder = server.getFolder(); + Path db = Paths.get(folder.getAbsolutePath()); + Path ri = db.resolve(ProCoreServer.PROTOCOL_IGNORED_FILE); + if (!Files.exists(ri)) + try (OutputStream os = Files.newOutputStream(ri)) { + } catch (IOException e) { + throw new ProCoreException("Could not create file: " + ri, e); + } + boolean ok = false; + try { + server.start(); + ok = server.isActive(); + } finally { + server.stop(); + } + return ok; + } + private long preJournalCheck() throws ProCoreException { + File folder = server.getFolder(); + if (!folder.isDirectory()) + throw new ProCoreException("Database folder does not exist." + NL + "folder=" + folder); + File file = new File(folder, ProCoreServer.JOURNAL_FILE); + if (!file.isFile()) + throw new ProCoreException("Journal file does not exist." + NL + "file=" + file); + else if (!file.canRead()) + throw new ProCoreException("Journal file must be readale to create database from journal." + NL + "file=" + file); + else if (server.isAlive()) + throw new ProCoreException("Server must be dead to create database from journal file." + NL + "file=" + file); + return getNextClusterId(folder.toPath()); + } + private void postJournalFix(long nextFreeId) throws SDBException { + Session s = newSession(null); + long current = s.reserveIds(0); + if (current < nextFreeId) + s.reserveIds((int)(nextFreeId - current)); + s.isClosed(); + } + private Path createTempFolder(Path parent, String fail) throws ProCoreException { + try { + return Files.createTempDirectory(parent, TEMP_PREFIX); + } catch (IOException e) { + throw new ProCoreException(fail, e); + } + } + public void replaceFromJournal() throws SDBException { + long nextFreeId = preJournalCheck(); + Path db = Paths.get(server.getFolder().getAbsolutePath()); + Path temp = createTempFolder(db, "Could not create temporary directory for database to be created from journal."); + movePath(db, temp, ProCoreServer.CONFIG_FILE, FileOption.IF_NEW); + movePath(db, temp, ProCoreServer.JOURNAL_FILE, FileOption.IF_NEW); + deleteFile(db, ProCoreServer.GUARD_FILE, FileOption.IF_EXIST); + deleteFiles(db, ProCoreServer.PAGE_FILE_PATTERN); + Path dbHead = db.resolve(ProCoreServer.BRANCH_DIR); + deleteFiles(dbHead, ProCoreServer.VALUE_PATTERN); + deleteFiles(dbHead, ProCoreServer.DATA_PATTERN); + deleteFiles(dbHead, ProCoreServer.INDEX_PATTERN); + Path dbHeadTailFile = dbHead.resolve(ProCoreServer.TAIL_FILE); + boolean headTailFileExisted = Files.exists(dbHeadTailFile); + final long NEXT_REVISION = 1; + if (!headTailFileExisted) // Number of change sets and database id not known at this point. Fortunately they are not used by recovery. + TailFile.createTailFile(dbHeadTailFile.toFile(), NEXT_REVISION, nextFreeId, UUID.randomUUID().toString()); + movePath(dbHead, temp, ProCoreServer.TAIL_FILE, FileOption.IF_NEW); + Path dbTail = db.resolve(ProCoreServer.TAIL_DIR); + boolean tailExisted = Files.isDirectory(dbTail); + if (tailExisted) + copyPath(dbTail, dbHead, ProCoreServer.TAIL_FILE, FileOption.IF_NEW); + else { + try { + Files.createDirectory(dbTail); + } catch (IOException e) { + throw new ProCoreException("Failed to create directory: " + dbTail, e); + } + copyPath(temp, dbTail, ProCoreServer.TAIL_FILE, FileOption.IF_NEW); + } + server.createConfiguration(); + server.start(); + try { + String t = server.execute("journalRead " + temp.getFileName()); + if (t.length() > 0) + throw new ProCoreException("Could not read journal. reply=" + t); + postJournalFix(nextFreeId); // Not the right way for implementing this, but better than incorrect recovery. + } finally { + server.tryToStop(); + } + movePath(temp, db, ProCoreServer.CONFIG_FILE, FileOption.OVERWRITE); + deleteTree(temp); + if (!tailExisted) + deleteTree(dbTail); + } + public boolean canPurge() throws ProCoreException { + File folder = server.getFolder(); + Path db = Paths.get(folder.getAbsolutePath()); + Path dbHead = db.resolve(ProCoreServer.BRANCH_DIR); + if (!Files.exists(dbHead)) + return false; // Already clean. + boolean empty = isFolderEmpty(dbHead); + if (empty) + return false; // Already clean. + final boolean[] found = new boolean[1]; + found[0] = false; + dbHead.toFile().listFiles(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + if (found[0]) + return false; + else if (!name.equals(ProCoreServer.TAIL_FILE)) { + found[0] = true; + return false; + } + return true; + } + }); + return found[0]; + } + /** + * Removes old history i.e. change sets. + * Note that also cleans last exit status and removes guard file. + * This means that all information about last exit status is also lost + * . + * @throws ProCoreException + */ + public void purge() throws SDBException { + synchronized (server.proCoreServer.getProCoreClient()) { + boolean wasAlive = server.isAlive(); + if (wasAlive) + server.stop(); + Path db = Paths.get(server.getFolder().getAbsolutePath()); + Path dbHead = db.resolve(ProCoreServer.BRANCH_DIR); + Path dbTail = db.resolve(ProCoreServer.TAIL_DIR); + if (!Files.isDirectory(dbTail)) { + try { + Files.createDirectory(dbTail); + } catch (IOException e) { + throw new ProCoreException("Failed to create directory: " + dbTail, e); + } + } + long nextFreeId = getNextClusterId(db); + boolean cleanHead = Files.isDirectory(dbHead) && !isFolderEmpty(dbHead); + Logger.defaultLog("Purging old history and exit information. folder=" + db); + if (cleanHead) { + deleteClusters(dbHead, dbTail); + movePath(dbHead, dbTail, ProCoreServer.TAIL_FILE, FileOption.OVERWRITE); + moveFiles(dbHead, dbTail, ProCoreServer.VALUE_PATTERN, FileOption.IF_NEW); + moveFiles(dbHead, dbTail, ProCoreServer.DATA_PATTERN, FileOption.OVERWRITE); + deleteFiles(dbHead, ProCoreServer.INDEX_PATTERN); + } + deleteFiles(db, ProCoreServer.PAGE_FILE_PATTERN); + deleteFile(db, ProCoreServer.JOURNAL_FILE, FileOption.IF_EXIST); + deleteFile(db, ProCoreServer.LOG_FILE, FileOption.IF_EXIST); + deleteFile(db, ProCoreServer.DCS_FILE, FileOption.IF_EXIST); + deleteFile(db, ProCoreServer.RECOVERY_NEEDED_FILE, FileOption.IF_EXIST); + deleteFile(db, ProCoreServer.RECOVERY_IGNORED_FILE, FileOption.IF_EXIST); + deleteFile(db, ProCoreServer.PROTOCOL_IGNORED_FILE, FileOption.IF_EXIST); + deleteFile(db, ProCoreServer.GUARD_FILE, FileOption.IF_EXIST); + purgeValues(dbTail); + server.start(); + Session s = newSession(null); + long current = s.reserveIds(0); + if (current < nextFreeId) + s.reserveIds((int)(nextFreeId - current)); + s.isClosed(); + if (!wasAlive) + server.tryToStop(); + } + } + private void copyPath(Path fromFolder, Path toFolder, String file, FileOption fileOption) throws ProCoreException { + Path from = fromFolder.resolve(file); + Path to = toFolder.resolve(file); + try { + if (FileOption.IF_EXIST.equals(fileOption)) { + if (!Files.exists(from)) + return; + } + if (FileOption.OVERWRITE.equals(fileOption)) + Files.copy(from, to, COPY_ATTRIBUTES, REPLACE_EXISTING); + else + Files.copy(from, to, COPY_ATTRIBUTES); + } catch (IOException e) { + throw new ProCoreException("Could not copy " + from + " to " + to, e); + } + } + private static void deleteFile(Path from, String file, FileOption fileOption) throws ProCoreException { + Path path = from.resolve(file); + deletePath(path, fileOption); + } + private static void deletePath(Path path, FileOption fileOption) throws ProCoreException { + try { + if (FileOption.IF_EXIST.equals(fileOption)) + Files.deleteIfExists(path); + else + Files.delete(path); + } catch (IOException e) { + throw new ProCoreException("Could not delete " + path, e); + } + } + private boolean isFolderEmpty(final Path folder) { // True if folder exists and is empty. + if (!Files.isDirectory(folder)) + return false; + try(DirectoryStream folderStream = Files.newDirectoryStream(folder)) { + return !folderStream.iterator().hasNext(); + } catch (IOException e) { + Logger.defaultLogError("Failed to open folder stream. folder=" + folder, e); + return false; + } + } + private void movePath(Path fromPath, Path toPath, String file, FileOption fileOption) throws ProCoreException { + Path from = fromPath.resolve(file); + Path to = toPath.resolve(file); + try { + if (FileOption.OVERWRITE.equals(fileOption)) + Files.move(from, to, REPLACE_EXISTING); + else + Files.move(from, to); + } catch (IOException e) { + throw new ProCoreException("Could not move " + from + " to " + to, e); + } + } + private static void copyTree(Path from, Path to, String path, final FileOption fileOption) throws ProCoreException { + copyTree(from.resolve(path), to.resolve(path), fileOption); + } + private static void copyTree(Path from, Path to, final FileOption fileOption) throws ProCoreException { + class Visitor extends SimpleFileVisitor { + private Path fromPath; + private Path toPath; + Visitor(Path from, Path to) { + fromPath = from; + toPath = to; + } + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + Path targetPath = toPath.resolve(fromPath.relativize(dir)); + if (!Files.exists(targetPath)) { + Files.createDirectory(targetPath); + } + return FileVisitResult.CONTINUE; + } + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + if (FileOption.OVERWRITE.equals(fileOption)) + Files.copy(file, toPath.resolve(fromPath.relativize(file)), COPY_ATTRIBUTES, REPLACE_EXISTING); + else + Files.copy(file, toPath.resolve(fromPath.relativize(file)), COPY_ATTRIBUTES); + return FileVisitResult.CONTINUE; + } + } + try { + Visitor v = new Visitor(from, to); + EnumSet opts = EnumSet.noneOf(FileVisitOption.class); + Files.walkFileTree(from, opts, Integer.MAX_VALUE, v); + } catch (IOException e) { + throw new ProCoreException("Could not copy " + from + " to " + to, e); + } + } + private static void deleteTree(Path path) throws ProCoreException { + if (!Files.exists(path)) + return; + class Visitor extends SimpleFileVisitor { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + try { + Files.delete(file); + } catch (IOException ioe) { + ioe.printStackTrace(); + throw ioe; + } + return FileVisitResult.CONTINUE; + } + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException e) throws IOException { + if (e == null) { + try { + Files.delete(dir); + } catch (IOException ioe) { + ioe.printStackTrace(); + throw ioe; + } + return FileVisitResult.CONTINUE; + } + throw e; + } + } + try { + Visitor v = new Visitor(); + EnumSet opts = EnumSet.noneOf(FileVisitOption.class); + Files.walkFileTree(path, opts, Integer.MAX_VALUE, v); + } catch (IOException e) { + throw new ProCoreException("Could not delete " + path, e); + } + } + private static void moveFolder(Path fromPath, Path toPath, String folder, FileOption fileOption) throws ProCoreException { + Path from = fromPath.resolve(folder); + Path to = toPath.resolve(folder); + copyTree(from, to, fileOption); + deleteTree(from); + } + enum FileOption { + IF_EXIST, + IF_NEW, + OVERWRITE + } + private void copyFiles(final Path from, final Path to, String pattern, final FileOption fileOption) throws ProCoreException { + class Visitor extends SimpleFileVisitor { + private final PathMatcher matcher; + Visitor(String pattern) { + matcher = FileSystems.getDefault().getPathMatcher("glob:" + pattern); + } + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Path name = file.getFileName(); + if (null == name) + return FileVisitResult.CONTINUE; + else if (!matcher.matches(name)) + return FileVisitResult.CONTINUE; + if (FileOption.OVERWRITE.equals(fileOption)) + Files.copy(file, to.resolve(name), REPLACE_EXISTING); + else + Files.copy(file, to.resolve(name)); + return FileVisitResult.CONTINUE; + } + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + if (dir.equals(from)) + return FileVisitResult.CONTINUE; + else + return FileVisitResult.SKIP_SUBTREE; + } + } + try { + Visitor v = new Visitor(pattern); + EnumSet opts = EnumSet.noneOf(FileVisitOption.class); + Files.walkFileTree(from, opts, Integer.MAX_VALUE, v); + } catch (IOException e) { + throw new ProCoreException("Could not copy " + from + " to " + to, e); + } + } + private void moveFiles(final Path from, final Path to, String pattern, final FileOption fileOption) throws ProCoreException { + class Visitor extends SimpleFileVisitor { + private final PathMatcher matcher; + Visitor(String pattern) { + matcher = FileSystems.getDefault().getPathMatcher("glob:" + pattern); + } + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Path name = file.getFileName(); + if (null == name) + return FileVisitResult.CONTINUE; + else if (!matcher.matches(name)) + return FileVisitResult.CONTINUE; + if (FileOption.OVERWRITE.equals(fileOption)) + Files.move(file, to.resolve(name), REPLACE_EXISTING); + else + Files.move(file, to.resolve(name)); + return FileVisitResult.CONTINUE; + } + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + if (dir.equals(from)) + return FileVisitResult.CONTINUE; + else + return FileVisitResult.SKIP_SUBTREE; + } + } + try { + Visitor v = new Visitor(pattern); + EnumSet opts = EnumSet.noneOf(FileVisitOption.class); + Files.walkFileTree(from, opts, Integer.MAX_VALUE, v); + } catch (IOException e) { + throw new ProCoreException("Could not move " + from + " to " + to, e); + } + } + private void deleteClusters(final Path head, final Path tail) throws ProCoreException { + class Visitor extends SimpleFileVisitor { + private final PathMatcher matcher; + Visitor(String pattern) { + matcher = FileSystems.getDefault().getPathMatcher("glob:" + ProCoreServer.DELETED_PATTERN); + } + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Path name = file.getFileName(); + if (null == name) + return FileVisitResult.CONTINUE; + else if (!matcher.matches(name)) + return FileVisitResult.CONTINUE; + String deletedStr = name.toString(); + String indexName = deletedStr.replaceFirst(ProCoreServer.DELETED_PREFIX, ProCoreServer.INDEX_PREFIX); + String dataName = deletedStr.replaceFirst(ProCoreServer.DELETED_PREFIX, ProCoreServer.DATA_PREFIX); + String[] ss = deletedStr.split("\\."); + String valuePattern = ProCoreServer.VALUE_PREFIX + ss[1] + "." + ss[2] + "*"; + Files.delete(file); + Files.delete(head.resolve(indexName)); + Files.delete(head.resolve(dataName)); + try { + deleteFiles(head, valuePattern); + if (Files.isDirectory(tail)) { + Files.deleteIfExists(tail.resolve(dataName)); + deleteFiles(tail, valuePattern); + } + } catch (ProCoreException e) { + throw new IOException("Delete values failed.", e); + } + return FileVisitResult.CONTINUE; + } + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + if (dir.equals(head)) + return FileVisitResult.CONTINUE; + else + return FileVisitResult.SKIP_SUBTREE; + } + } + try { + Visitor v = new Visitor(ProCoreServer.DELETED_PREFIX); + EnumSet opts = EnumSet.noneOf(FileVisitOption.class); + Files.walkFileTree(head, opts, Integer.MAX_VALUE, v); + } catch (IOException e) { + throw new ProCoreException("Could not delete cluster(s).\nhead=" + head + "\ntail=" + tail); + } + } + private static long convertHexStringToLong(String hex) { + if (hex.length() < 16) + return Long.parseLong(hex, 16); + long f = Long.parseLong(hex.substring(0,1), 16) << 60; + long l = Long.parseLong(hex.substring(1,16), 16); + return f | l; + } + private void purgeValues(final Path folder) throws ProCoreException { + final class Value { + long first; // First part of cluster id. + long second; // Secon part of cluster id. + int index; // Resource index of the value. + transient long cs; // Change set of the value. + public Value(long first, long second, int index, long cs) { + this.first = first; + this.second = second; + this.index = index; + this.cs = cs; + } + String getName() { + return ProCoreServer.VALUE_PREFIX + toString() + ProCoreServer.VALUE_SUFFIX; + } + @Override + public String toString() { + return String.format("%x.%x.%d.%d", first, second, index, cs); + } + @Override + public boolean equals(Object o) { + if (this == o) + return true; + else if (!(o instanceof Value)) + return false; + Value x = (Value)o; + return first == x.first && second == x.second && index == x.index; + } + @Override + public int hashCode() { + int result = 17; + int f = (int)(first ^ (first >>> 32)); + result = 31 * result + f; + int s = (int)(second ^ (second >>> 32)); + result = 31 * result + s; + return result + index; + } + } + File[] files = folder.toFile().listFiles(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.startsWith(ProCoreServer.VALUE_PREFIX); + } + }); + HashMap values = new HashMap(); + for (int i = 0; i < files.length; ++i) { + String s = files[i].getName(); + String[] ss = s.split("\\."); + if (ss.length != 6) { + Logger.defaultLogError("Illegal external value file name. name=" + s); + continue; + } + long first = convertHexStringToLong(ss[1]); + long second = convertHexStringToLong(ss[2]); + int ri = Integer.parseInt(ss[3]); + long cs = Long.parseLong(ss[4]); + Value nv = new Value(first, second, ri, cs); + Value ov = values.get(nv); + if (null == ov) + values.put(nv, nv); + else if (ov.cs < nv.cs) { + deleteFile(folder, ov.getName(), FileOption.IF_EXIST); + ov.cs = nv.cs;; + } else + deleteFile(folder, nv.getName(), FileOption.IF_EXIST); + } + } + private long getNextClusterId(final Path db) throws ProCoreException { + long clusterId = 0; + Path dbHead = db.resolve(ProCoreServer.BRANCH_DIR); + File[] files = dbHead.toFile().listFiles(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.startsWith(ProCoreServer.DATA_PREFIX); + } + }); + for (int i = 0; i < files.length; ++i) { + String s = files[i].getName(); + String[] ss = s.split("\\.", 4); + if (ss.length != 4) { + Logger.defaultLogError("Illegal cluster file name. name=" + s); + continue; + } + long id = convertHexStringToLong(ss[2]); + if (id > clusterId) + clusterId = id; + } + final Path dbTail = db.resolve(ProCoreServer.TAIL_DIR); + if (!Files.exists(dbTail)) + return clusterId + 1; + class Visitor extends SimpleFileVisitor { + private final PathMatcher matcher; + long clusterId; + Visitor(String pattern, long clusterId) { + this.clusterId = clusterId; + matcher = FileSystems.getDefault().getPathMatcher("glob:" + pattern); + } + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Path name = file.getFileName(); + if (null == name) + return FileVisitResult.CONTINUE; + else if (!matcher.matches(name)) + return FileVisitResult.CONTINUE; + String s = name.toString(); + String[] ss = s.split("\\.", 4); + if (ss.length != 4) { + Logger.defaultLogError("Illegal cluster file name. name=" + s); + return FileVisitResult.CONTINUE; + } + long id = convertHexStringToLong(ss[2]); + if (id > clusterId) + clusterId = id; + return FileVisitResult.CONTINUE; + } + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + if (dir.equals(dbTail)) + return FileVisitResult.CONTINUE; + else + return FileVisitResult.SKIP_SUBTREE; + } + } + try { + Visitor v = new Visitor(ProCoreServer.DATA_PATTERN, clusterId); + EnumSet opts = EnumSet.noneOf(FileVisitOption.class); + Files.walkFileTree(dbTail, opts, Integer.MAX_VALUE, v); + return v.clusterId + 1; + } catch (IOException e) { + throw new ProCoreException("Could not get next free cluster id for " + db, e); + } + } + private static void deleteFiles(final Path folder, String pattern) throws ProCoreException { + if (!Files.exists(folder)) + return; + class Visitor extends SimpleFileVisitor { + private final PathMatcher matcher; + Visitor(String pattern) { + matcher = FileSystems.getDefault().getPathMatcher("glob:" + pattern); + } + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Path name = file.getFileName(); + if (null == name) + return FileVisitResult.CONTINUE; + else if (!matcher.matches(name)) + return FileVisitResult.CONTINUE; + Files.delete(file); + return FileVisitResult.CONTINUE; + } + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + if (dir.equals(folder)) + return FileVisitResult.CONTINUE; + else + return FileVisitResult.SKIP_SUBTREE; + } + } + try { + Visitor v = new Visitor(pattern); + EnumSet opts = EnumSet.noneOf(FileVisitOption.class); + Files.walkFileTree(folder, opts, Integer.MAX_VALUE, v); + } catch (IOException e) { + throw new ProCoreException("Could not delete " + folder + " with " + pattern, e); + } + } + public void copy(File to) throws ProCoreException { + copyDatabaseFiles(server.getFolder().toPath(), to.toPath()); + } + public boolean isServerAlive() { + return server.isAlive(); + } + Client newClient() throws ProCoreException { + return server.newClient(); + } + public void createFolder() throws ProCoreException { + if (server.folder.exists()) + return; + boolean created = server.folder.mkdirs(); + if (!created) + throw new ProCoreException("Could not create folder=" + server.folder); + } + /** + * @throws ProCoreException if could not stop server. + */ + public boolean serverTryToStop() throws ProCoreException { + return server.tryToStop(); + } + class Server { + class Command { + void Do(ProCoreServer proCoreServer) { + } + } + class Event { + } + private final File folder; // Always non null. + private final ProCoreServer proCoreServer; + void start() throws ProCoreException { + try { + proCoreServer.start(); + } catch (DatabaseLastExitException e) { + if (null == dbUserAgent) + return; + if (!dbUserAgent.handleStart(e)) + throw new ProCoreException(folder, "Failed to handle start exception.", e); + proCoreServer.start(); + } + } + void stop() throws ProCoreException { + try { + proCoreServer.stop(); + } catch (InterruptedException e) { + if (proCoreServer.isAlive()) + throw new ProCoreException("ProCoreServer stop was interrupted.", e); + } + } + boolean isAlive() { + try { + return proCoreServer.isAlive(); + } catch (ProCoreException e) { + Logger.defaultLogError(e); + return false; + } + } + Server(File aFolder) { + if (null == aFolder) + throw new RuntimeException("Database folder can not be null."); + try { + folder = aFolder.getCanonicalFile(); + } catch (IOException e) { + String t = "Could not get canonical path. file=" + aFolder; + Logger.defaultLogError(t); + throw new RuntimeException(t); + } + try { + File serverFolder = org.simantics.db.server.internal.Activator.getServerFolder(); + if (!folder.isDirectory()) + folder.mkdirs(); + proCoreServer = ProCoreServer.getProCoreServer(serverFolder, folder); + return; + } catch (Throwable t) { + Logger.defaultLogError(t); + throw new RuntimeException(t); + } + } + void createConfiguration() throws ProCoreException { + if (!folder.isDirectory()) + throw new ProCoreException("Can't create configuration because folder is not ok. folder=" + folder.getAbsolutePath()); + File file = new File(folder, ProCoreServer.CONFIG_FILE); + try { + file.createNewFile(); + } catch (IOException e) { + throw new ProCoreException("Can't create configuration file. file=" + file.getAbsolutePath()); + } + } + void deleteGuard() throws ProCoreException { + if (server.isActive()) + throw new ProCoreException("Will not delete guard file when server is alive."); + if (!server.isFolderOk(server.getFolder())) + throw new ProCoreException("Will not delete guard file when server folder is not ok. folder=" + server.getFolder()); + File file = new File(folder, ProCoreServer.GUARD_FILE); + if (!file.exists()) + throw new ProCoreException("Guard file does not exist. file=" + file.getAbsolutePath()); + boolean deleted = file.delete(); + if (!deleted) + throw new ProCoreException("Failed to delete file=" + file.getAbsolutePath()); + } + String execute(String aCommand) throws ProCoreException { + try { + return proCoreServer.execute(aCommand); + } catch (InterruptedException e) { + throw new ProCoreException("Execute call was interrupted.", e); + } + } + Status getStatus() { + Status status = Status.NoDatabase; + String path; + try { + path = folder.getCanonicalPath(); + } catch (IOException e) { + Util.logError("Could not get canonical path for folder. folder=" + folder.getAbsolutePath(), e); + path = ""; + } + if (!isFolderOk(folder)) + status = Status.NoDatabase; + else if (!isAlive()) + status = Status.NotRunning; + else if (isConnected()) { + if (isLocal()) + status = Status.Local; + else + status = Status.Remote; + } else + status = Status.Standalone; + status.message = status.getString() + "@" + path; + return status; + } + File getFolder() { + return folder; + } + boolean isActive() { + try { + return proCoreServer.isActive(); + } catch (ProCoreException e) { + Logger.defaultLogError("IsActive failed.", e); + return false; + } + } + boolean isConnected() { + try { + return proCoreServer.isConnected(); + } catch (ProCoreException e) { + Logger.defaultLogError("IsConnected failed.", e); + return false; + } + } + boolean isLocal() { + try { + return proCoreServer.isLocal(); + } catch (ProCoreException e) { + Logger.defaultLogError("IsLocal faailed.", e); + return false; + } + } + void connect() throws ProCoreException, InterruptedException { + proCoreServer.connect(); + } + void disconnect() { + try { + proCoreServer.disconnect(); + } catch (ProCoreException e) { + Logger.defaultLogError("Could not disconnect.", e); + } + } + /** + * @param aFolder + * @return true if given folder contains database journal file or + * configuration file. + */ + boolean isFolderOk(File aFolder) { + if (!aFolder.isDirectory()) + return false; + File config = new File(aFolder, ProCoreServer.CONFIG_FILE); + if (config.exists()) + return true; + File journal = new File(aFolder, ProCoreServer.JOURNAL_FILE); + if (journal.exists()) + return true; + return false; + } + /** + * @throws ProCoreException if could not stop server. + */ + boolean tryToStop() throws ProCoreException { + return proCoreServer.tryToStop(); + } + Client newClient() throws ProCoreException { + return proCoreServer.newClient(); + } + } +// From interface Database + @Override + public void initFolder(Properties properties) throws ProCoreException { + createFolder(); + serverCreateConfiguration(); + } + @Override + public void deleteFiles() throws ProCoreException { + deleteDatabaseFiles(); + } + @Override + public void start() throws ProCoreException { + connect(); + } + @Override + public boolean isRunning() throws ProCoreException { + return isServerAlive(); + } + @Override + public boolean tryToStop() throws ProCoreException { + return serverTryToStop(); + } + @Override + public String execute(String aCommand) throws ProCoreException { + return server.execute(aCommand); + } + @Override + public void purgeDatabase() throws SDBException { + synchronized (server.proCoreServer.getProCoreClient()) { + if (!server.isLocal()) + throw new ProCoreException("Purge is allowed only for local server."); + List clients = sessionManager.disconnect(this); + purge(); + try { + sessionManager.connect(this, clients); + } catch (InterruptedException e) { + throw new ProCoreException("Failed to connect after purge.", e); + } + } + } + @Override + public Session newSession(ServiceLocator locator) throws ProCoreException { + return sessionManager.newSession(this); + } + @Override + public Path createFromChangeSets(int revision) throws ProCoreException { + if (!isFolderOk()) + throw new ProCoreException("Folder must be valid database folder to create database from journal." ); + File folder = server.getFolder(); + File file = new File(folder, ProCoreServer.DCS_FILE); + if (!file.isFile() && !file.canRead()) + throw new ProCoreException("Dump file must be readable. file=" + file.getAbsolutePath()); + Path db = Paths.get(folder.getAbsolutePath()); + Path temp = createTempFolder(db, "Could not create temporary directory for database to be created from change sets."); + Server s = new Server(temp.toFile()); + s.createConfiguration(); + s.start(); + String t = s.execute("loadChangeSets .. " + revision); + if (t.length() < 1) + throw new ProCoreException("Could not read journal. reply=" + t); + s.tryToStop(); + try { + int cs = Integer.parseInt(t); + if (cs == revision) + return temp; + throw new ProCoreException("Could not load change sets. reply=" + t); + } catch (NumberFormatException e) { + throw new ProCoreException("Could not load change sets. reply=" + t); + } + } + @Override + public void clone(File to, int revision, boolean saveHistory) { + String history = saveHistory ? "with history." : "without history."; + String message = "Clone to " + to.getAbsolutePath() + "@" + revision + " " + history; + Util.trace(message); + } + @Override + public DatabaseUserAgent getUserAgent() { + return dbUserAgent; + } + @Override + public void setUserAgent(DatabaseUserAgent dbUserAgent) { + this.dbUserAgent = dbUserAgent; + } + @Override + public Status getStatus() { + return server.getStatus(); + } + @Override + public File getFolder() { + return server.getFolder(); + } + @Override + public boolean isFolderOk() { + return server.isFolderOk(server.getFolder()); + } + @Override + public boolean isFolderOk(File folder) { + return server.isFolderOk(folder); + } + public boolean isFolderEmpty() { + return isFolderEmpty(server.getFolder()); + } + @Override + public boolean isFolderEmpty(File folder) { + return isFolderEmpty(folder.toPath()); + } + @Override + public void deleteGuard() throws ProCoreException { + server.deleteGuard(); + } + @Override + public boolean isConnected() throws ProCoreException { + return server.isConnected(); + } + @Override + public void connect() throws ProCoreException { + if (!isFolderOk()) + throw new ProCoreException("Could not connect to " + getFolder()); + if (!server.isAlive()) + server.start(); + if (isConnected()) + return; + try { + server.connect(); + } catch (InterruptedException e) { + Util.logError("Server connect was interrupted.", e); + } + if (server.isActive()) + return; + throw new ProCoreException("Could not connect to " + getFolder()); + } + @Override + public void disconnect() throws ProCoreException { + server.disconnect(); + } + @Override + public Path dumpChangeSets() throws ProCoreException { + if (!isFolderOk()) + throw new ProCoreException("Folder must be set to dump change sets."); + if (!server.isActive()) + throw new ProCoreException("Server must be responsive to dump change sets."); + String t = server.execute("dumpChangeSets").replaceAll("\n", ""); + try { + int ncs = Integer.parseInt(t); + if (ncs < 1) + return null; + File file = new File(getFolder(), ProCoreServer.DCS_FILE); + return file.toPath(); + } catch (NumberFormatException e) { + throw new ProCoreException("Could not dump change sets.", e); + } + } + @Override + public long serverGetTailChangeSetId() throws ProCoreException { + try { + return server.proCoreServer.getTailData().nextChangeSetId; + } catch (TailReadException e) { + return 1; + } + } + @Override + public JournalI getJournal() throws ProCoreException { + return journal; + } + class JournalI implements Database.Journal { + class Analyzed { + private final File journalFile; + private long lastModified; + private RandomAccessFile file; // Journal file. + private ArrayList offsets = new ArrayList(); // Offsets to commands in journal file. + private Line lines[] = new Line[256]; + int firstLine = 0; // Index of the first element in lines table. + Analyzed(File dbFolder) { + journalFile = new File(dbFolder, ProCoreServer.JOURNAL_FILE); + for (int i=0; i metadata = (TreeMap) METADATA_SERIALIZER.deserialize(bytes); + CommitMetadata commit = MetadataUtil.getMetadata(metadata, CommitMetadata.class); + String date = ""; + if (null != commit && null != commit.date) + date = commit.date.toString(); + CommentMetadata comment = MetadataUtil.getMetadata(metadata, CommentMetadata.class); + String comments = ""; + if (null != comment) + comments = comment.toString(); + return date + " " + comments; + } catch (IOException e) { + throw new ProCoreException("Failed to interpret metadata.", e); + } + } + private Line getRow(int index, Line line) throws ProCoreException { + if (index < 0 || index >= offsets.size()) + return null; // Index out of range. + if (index < firstLine || index >= firstLine + lines.length) { + readLines(index); + if (index < firstLine) + return null; // Index out of range. + } + int offset = index - firstLine; + line.status = lines[offset].status; + line.request = lines[offset].request; + line.comment = lines[offset].comment; + return line; + } + private void readLines(int first) throws ProCoreException { + open(); + try { + for (int i=0, index = first; i= offsets.size()) { + line.status = false; + line.request = ""; + line.comment = ""; + return; + } + long offset = offsets.get(index); + try { + file.seek(offset); + int i = file.readInt(); + int length = Integer.reverseBytes(i); + byte b = file.readByte(); + boolean ok = b != 0; + b = file.readByte(); + // boolean littleEndian = b != 0; + i = file.readInt(); + int type = Integer.reverseBytes(i); + String comment = ""; + if (length < 6) + throw new ProCoreException("Journal file corrupted at" + file.getFilePointer() + "."); + else if (length > 6) { + if (type != 3) + file.skipBytes(length - 6); + else if (length > 22){ + file.skipBytes(16); + i = file.readInt(); + int size = Integer.reverseBytes(i); + if (size != length - 26) + throw new ProCoreException("Metadata corrupted at" + file.getFilePointer() + "."); + byte[] bytes = new byte[size]; + file.readFully(bytes); + comment = getComment(bytes); + if (null == comment) + comment = ""; + } + } + i = file.readInt(); + int length2 = Integer.reverseBytes(i); + if (length != length2) + throw new ProCoreException("Journal file corrupted at" + file.getFilePointer() + "."); + String command = MessageText.get(type); + line.status = ok; + line.request = command; + line.comment = comment; + } catch (IOException e) { + throw new ProCoreException("Journal file corrupted."); + } + } + private ArrayList readOffsets() throws ProCoreException { + if (!canRead()) { + lastModified = 0; + offsets.clear(); + firstLine = 0; + return offsets; + } + long modified = journalFile.lastModified(); + if (lastModified != 0 && lastModified == modified) + return offsets; // Offsets already up to date. + lastModified = 0; + offsets.clear(); + firstLine = 0; + open(); + try { + file.seek(0); + int i = file.readInt(); + int version = Integer.reverseBytes(i); + if (version != 1) + throw new ProCoreException("Unsupported journal file version. expected=1 got=" + version); + i = file.readInt(); + int major = Integer.reverseBytes(i); + if (major != MessageNumber.ProtocolVersionMajor) + throw new ProCoreException("Unsupported journal request major version. expected=" + MessageNumber.ProtocolVersionMajor + " got=" + major); + i = file.readInt(); + int minor = Integer.reverseBytes(i); + if (minor > MessageNumber.ProtocolVersionMinor) + throw new ProCoreException("Unsupported journal request minor version. expected=" + MessageNumber.ProtocolVersionMinor + " got=" + minor); + i = file.readInt(); + int length = Integer.reverseBytes(i); + while (length > 0) { // Not supporting unsigned integers. + if (length < 6) + throw new ProCoreException("Journal file corrupted at" + file.getFilePointer() + "."); + file.skipBytes(length); + i = file.readInt(); + int length2 = Integer.reverseBytes(i); + if (length != length2) + throw new ProCoreException("Journal file corrupted at " + file.getFilePointer() + "."); + long offset = file.getFilePointer() - length - 8; + offsets.add(offset); + i = file.readInt(); + length = Integer.reverseBytes(i); + } + } catch (EOFException e) { + } catch (IOException e) { + throw new ProCoreException("Failed to get command count.", e); + } finally { + close(); + } + lastModified = modified; + readLines(0); + return offsets; + } + } + private final Analyzed analyzed; + JournalI(File dbFolder) { + this.analyzed = new Analyzed(dbFolder); + } + @Override + public boolean canRead() { + return analyzed.canRead() && !isServerAlive(); + } + @Override + public int count() { + return analyzed.count(); + } + @Override + public int read(int index, Line line) throws ProCoreException { + int count = analyzed.count(); + analyzed.getRow(index, line); + return count; + } + } + @Override + public String getCompression() { + return "FLZ"; + } +}