package org.simantics.db.server.internal; import static java.nio.file.StandardCopyOption.COPY_ATTRIBUTES; import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; import java.io.EOFException; import java.io.File; import java.io.FilenameFilter; import java.io.IOException; import java.io.OutputStream; import java.io.RandomAccessFile; import java.nio.file.DirectoryStream; import java.nio.file.FileSystems; import java.nio.file.FileVisitOption; import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.PathMatcher; import java.nio.file.Paths; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Properties; import java.util.TreeMap; import java.util.UUID; import org.simantics.databoard.Bindings; import org.simantics.databoard.binding.impl.TreeMapBinding; import org.simantics.databoard.serialization.Serializer; import org.simantics.db.Database; import org.simantics.db.DatabaseUserAgent; import org.simantics.db.ServiceLocator; import org.simantics.db.common.CommentMetadata; import org.simantics.db.common.CommitMetadata; import org.simantics.db.common.utils.Logger; import org.simantics.db.common.utils.MetadataUtil; import org.simantics.db.exception.SDBException; import org.simantics.db.server.DatabaseLastExitException; import org.simantics.db.server.ProCoreException; import org.simantics.db.server.internal.ProCoreServer.TailFile; import org.simantics.db.server.protocol.MessageNumber; import org.simantics.db.server.protocol.MessageText; import org.slf4j.LoggerFactory; public class DatabaseI implements Database { private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(DatabaseI.class); private static String NL = System.getProperty("line.separator"); private static String TEMP_PREFIX = "db.temp."; public static DatabaseI newDatabaseI(File dbFolder) { return new DatabaseI(dbFolder); } private final SessionManager sessionManager = new SessionManager(); private final Server server; private final JournalI journal; private DatabaseUserAgent dbUserAgent = null; private DatabaseI(File dbFolder) { server = new Server(dbFolder); journal = new JournalI(dbFolder); } public boolean isSubFolder(Path base, Path sub) throws IOException { if (null == base || null == sub) return false; return isSubFolder(base.toFile(), sub.toFile()); } public boolean isSubFolder(File base, File sub) throws IOException { if (null == base || null == sub) return false; Path basePath = base.getCanonicalFile().toPath(); Path subPath = sub.getCanonicalFile().toPath(); if (subPath.startsWith(basePath)) return true; else return false; } public Path saveDatabase() throws ProCoreException { Path dbFolder = getFolder().toPath(); Path parent = getFolder().getParentFile().toPath(); Path temp = createTempFolder(parent, "Could not create temporary directory for saving database."); copyTree(dbFolder, temp, FileOption.IF_NEW); return temp; } public Path saveDatabase(Path parent) throws ProCoreException { Path dbFolder = getFolder().toPath(); try { boolean yes = isSubFolder(dbFolder, parent); if (yes) throw new ProCoreException("Parent must not be subdirectory of database directory." + NL + "database=" + dbFolder + NL + "parent=" + parent); } catch (IOException e) { throw new ProCoreException("Failed to save database to parent." + NL + "database=" + dbFolder + NL + "parent=" + parent); } Path temp = createTempFolder(parent, "Could not create temporary directory for saving database."); copyTree(dbFolder, temp, FileOption.IF_NEW); return temp; } public void deleteDatabaseFiles() throws ProCoreException { if (server.isAlive()) { server.tryToStop(); if (server.isAlive()) throw new ProCoreException("Server must be dead to delete database."); } Path db = getFolder().toPath(); deleteDatabaseFiles(db); // Redundant, but tests the method. deleteTree(db); } Path copyDatabaseFiles(Path from, Path to) throws ProCoreException { copyTree(from, to, ProCoreServer.BRANCH_DIR, FileOption.IF_EXIST); if (Files.exists(from.resolve(ProCoreServer.TAIL_DIR))) copyTree(from, to, ProCoreServer.TAIL_DIR, FileOption.IF_EXIST); copyPath(from, to, ProCoreServer.CONFIG_FILE, FileOption.IF_EXIST); copyPath(from, to, ProCoreServer.GUARD_FILE, FileOption.IF_EXIST); copyPath(from, to, ProCoreServer.JOURNAL_FILE, FileOption.IF_EXIST); copyPath(from, to, ProCoreServer.LOG_FILE, FileOption.IF_EXIST); copyFiles(from, to, ProCoreServer.PAGE_FILE_PATTERN, FileOption.IF_EXIST); return to; } public static void deleteDatabaseFiles(Path from) throws ProCoreException { deleteFile(from, ProCoreServer.DCS_FILE, FileOption.IF_EXIST); deleteFile(from, ProCoreServer.RECOVERY_NEEDED_FILE, FileOption.IF_EXIST); deleteFile(from, ProCoreServer.RECOVERY_IGNORED_FILE, FileOption.IF_EXIST); deleteFile(from, ProCoreServer.PROTOCOL_IGNORED_FILE, FileOption.IF_EXIST); deleteFile(from, ProCoreServer.LOG_FILE, FileOption.IF_EXIST); deleteFile(from, ProCoreServer.GUARD_FILE, FileOption.IF_EXIST); deleteFile(from, ProCoreServer.CONFIG_FILE, FileOption.IF_EXIST); deleteFiles(from, ProCoreServer.PAGE_FILE_PATTERN); deleteTree(from.resolve(ProCoreServer.BRANCH_DIR)); deleteFile(from, ProCoreServer.JOURNAL_FILE, FileOption.IF_EXIST); deleteTree(from.resolve(ProCoreServer.TAIL_DIR)); } Path moveDatabaseFiles(Path from, Path to) throws ProCoreException { moveFolder(from, to, ProCoreServer.BRANCH_DIR, FileOption.IF_NEW); if (Files.exists(from.resolve(ProCoreServer.TAIL_DIR))) moveFolder(from, to, ProCoreServer.TAIL_DIR, FileOption.IF_NEW); movePath(from, to, ProCoreServer.CONFIG_FILE, FileOption.IF_NEW); movePath(from, to, ProCoreServer.GUARD_FILE, FileOption.IF_NEW); movePath(from, to, ProCoreServer.JOURNAL_FILE, FileOption.IF_NEW); movePath(from, to, ProCoreServer.LOG_FILE, FileOption.IF_NEW); moveFiles(from, to, ProCoreServer.PAGE_FILE_PATTERN, FileOption.IF_NEW); movePath(from, to, ProCoreServer.DCS_FILE, FileOption.IF_NEW); moveFile(from, to, ProCoreServer.RECOVERY_NEEDED_FILE, FileOption.IF_NEW); moveFile(from, to, ProCoreServer.RECOVERY_IGNORED_FILE, FileOption.IF_NEW); moveFile(from, to, ProCoreServer.PROTOCOL_IGNORED_FILE, FileOption.IF_NEW); return to; } private void moveFile(Path from, Path to, String file, FileOption fileOption) throws ProCoreException { if (Files.exists(from.resolve(file))) movePath(from, to, file, fileOption); } public void serverCreateConfiguration() throws ProCoreException { server.createConfiguration(); } public boolean ignoreExit() throws ProCoreException { if (!isFolderOk()) throw new ProCoreException("Folder must be valid database folder to ignore exit." ); server.deleteGuard(); File folder = server.getFolder(); Path db = Paths.get(folder.getAbsolutePath()); Path ri = db.resolve(ProCoreServer.RECOVERY_IGNORED_FILE); if (!Files.exists(ri)) try (OutputStream os = Files.newOutputStream(ri)) { } catch (IOException e) { throw new ProCoreException("Could not create file: " + ri, e); } boolean ok = false; DatabaseUserAgent dbu = dbUserAgent; // Save current user agent. try { dbUserAgent = null; // Recursion not supported. Disable user agent. server.start(); ok = server.isActive(); } finally { try { dbUserAgent = dbu; // Restore used user agent. server.stop(); } finally { try { Files.deleteIfExists(ri); } catch (IOException e) { Logger.defaultLogError("Could not delete file: " + ri, e); } Path rn = db.resolve(ProCoreServer.RECOVERY_NEEDED_FILE); try { Files.deleteIfExists(rn); } catch (IOException e) { Logger.defaultLogError("Could not delete file: " + rn, e); } } } return ok; } public boolean ignoreProtocol() throws ProCoreException { if (!isFolderOk()) throw new ProCoreException("Folder must be valid database folder to ignore exit." ); File folder = server.getFolder(); Path db = Paths.get(folder.getAbsolutePath()); Path ri = db.resolve(ProCoreServer.PROTOCOL_IGNORED_FILE); if (!Files.exists(ri)) try (OutputStream os = Files.newOutputStream(ri)) { } catch (IOException e) { throw new ProCoreException("Could not create file: " + ri, e); } boolean ok = false; try { server.start(); ok = server.isActive(); } finally { server.stop(); } return ok; } private long preJournalCheck() throws ProCoreException { File folder = server.getFolder(); if (!folder.isDirectory()) throw new ProCoreException("Database folder does not exist." + NL + "folder=" + folder); File file = new File(folder, ProCoreServer.JOURNAL_FILE); if (!file.isFile()) throw new ProCoreException("Journal file does not exist." + NL + "file=" + file); else if (!file.canRead()) throw new ProCoreException("Journal file must be readale to create database from journal." + NL + "file=" + file); else if (server.isAlive()) throw new ProCoreException("Server must be dead to create database from journal file." + NL + "file=" + file); return getNextClusterId(folder.toPath()); } private void postJournalFix(long nextFreeId) throws SDBException { Session s = newSession(null); long current = s.reserveIds(0); if (current < nextFreeId) s.reserveIds((int)(nextFreeId - current)); s.isClosed(); } private Path createTempFolder(Path parent, String fail) throws ProCoreException { try { return Files.createTempDirectory(parent, TEMP_PREFIX); } catch (IOException e) { throw new ProCoreException(fail, e); } } public void replaceFromJournal() throws SDBException { long nextFreeId = preJournalCheck(); Path db = Paths.get(server.getFolder().getAbsolutePath()); Path temp = createTempFolder(db, "Could not create temporary directory for database to be created from journal."); movePath(db, temp, ProCoreServer.CONFIG_FILE, FileOption.IF_NEW); movePath(db, temp, ProCoreServer.JOURNAL_FILE, FileOption.IF_NEW); deleteFile(db, ProCoreServer.GUARD_FILE, FileOption.IF_EXIST); deleteFiles(db, ProCoreServer.PAGE_FILE_PATTERN); Path dbHead = db.resolve(ProCoreServer.BRANCH_DIR); deleteFiles(dbHead, ProCoreServer.VALUE_PATTERN); deleteFiles(dbHead, ProCoreServer.DATA_PATTERN); deleteFiles(dbHead, ProCoreServer.INDEX_PATTERN); Path dbHeadTailFile = dbHead.resolve(ProCoreServer.TAIL_FILE); boolean headTailFileExisted = Files.exists(dbHeadTailFile); final long NEXT_REVISION = 1; if (!headTailFileExisted) // Number of change sets and database id not known at this point. Fortunately they are not used by recovery. TailFile.createTailFile(dbHeadTailFile.toFile(), NEXT_REVISION, nextFreeId, UUID.randomUUID().toString()); movePath(dbHead, temp, ProCoreServer.TAIL_FILE, FileOption.IF_NEW); Path dbTail = db.resolve(ProCoreServer.TAIL_DIR); boolean tailExisted = Files.isDirectory(dbTail); if (tailExisted) copyPath(dbTail, dbHead, ProCoreServer.TAIL_FILE, FileOption.IF_NEW); else { try { Files.createDirectory(dbTail); } catch (IOException e) { throw new ProCoreException("Failed to create directory: " + dbTail, e); } copyPath(temp, dbTail, ProCoreServer.TAIL_FILE, FileOption.IF_NEW); } server.createConfiguration(); server.start(); try { String t = server.execute("journalRead " + temp.getFileName()); if (t.length() > 0) throw new ProCoreException("Could not read journal. reply=" + t); postJournalFix(nextFreeId); // Not the right way for implementing this, but better than incorrect recovery. } finally { server.tryToStop(); } movePath(temp, db, ProCoreServer.CONFIG_FILE, FileOption.OVERWRITE); deleteTree(temp); if (!tailExisted) deleteTree(dbTail); } public boolean canPurge() throws ProCoreException { File folder = server.getFolder(); Path db = Paths.get(folder.getAbsolutePath()); Path dbHead = db.resolve(ProCoreServer.BRANCH_DIR); if (!Files.exists(dbHead)) return false; // Already clean. boolean empty = isFolderEmpty(dbHead); if (empty) return false; // Already clean. final boolean[] found = new boolean[1]; found[0] = false; dbHead.toFile().listFiles(new FilenameFilter() { @Override public boolean accept(File dir, String name) { if (found[0]) return false; else if (!name.equals(ProCoreServer.TAIL_FILE)) { found[0] = true; return false; } return true; } }); return found[0]; } /** * Removes old history i.e. change sets. * Note that also cleans last exit status and removes guard file. * This means that all information about last exit status is also lost * . * @throws ProCoreException */ public void purge() throws SDBException { synchronized (server.proCoreServer.getProCoreClient()) { boolean wasAlive = server.isAlive(); if (wasAlive) server.stop(); Path db = Paths.get(server.getFolder().getAbsolutePath()); Path dbHead = db.resolve(ProCoreServer.BRANCH_DIR); Path dbTail = db.resolve(ProCoreServer.TAIL_DIR); if (!Files.isDirectory(dbTail)) { try { Files.createDirectory(dbTail); } catch (IOException e) { throw new ProCoreException("Failed to create directory: " + dbTail, e); } } long nextFreeId = getNextClusterId(db); boolean cleanHead = Files.isDirectory(dbHead) && !isFolderEmpty(dbHead); LOGGER.info("Purging old history and exit information. folder=" + db); if (cleanHead) { deleteClusters(dbHead, dbTail); movePath(dbHead, dbTail, ProCoreServer.TAIL_FILE, FileOption.OVERWRITE); moveFiles(dbHead, dbTail, ProCoreServer.VALUE_PATTERN, FileOption.IF_NEW); moveFiles(dbHead, dbTail, ProCoreServer.DATA_PATTERN, FileOption.OVERWRITE); deleteFiles(dbHead, ProCoreServer.INDEX_PATTERN); } deleteFiles(db, ProCoreServer.PAGE_FILE_PATTERN); deleteFile(db, ProCoreServer.JOURNAL_FILE, FileOption.IF_EXIST); deleteFile(db, ProCoreServer.LOG_FILE, FileOption.IF_EXIST); deleteFile(db, ProCoreServer.DCS_FILE, FileOption.IF_EXIST); deleteFile(db, ProCoreServer.RECOVERY_NEEDED_FILE, FileOption.IF_EXIST); deleteFile(db, ProCoreServer.RECOVERY_IGNORED_FILE, FileOption.IF_EXIST); deleteFile(db, ProCoreServer.PROTOCOL_IGNORED_FILE, FileOption.IF_EXIST); deleteFile(db, ProCoreServer.GUARD_FILE, FileOption.IF_EXIST); purgeValues(dbTail); server.start(); Session s = newSession(null); long current = s.reserveIds(0); if (current < nextFreeId) s.reserveIds((int)(nextFreeId - current)); s.isClosed(); if (!wasAlive) server.tryToStop(); } } private void copyPath(Path fromFolder, Path toFolder, String file, FileOption fileOption) throws ProCoreException { Path from = fromFolder.resolve(file); Path to = toFolder.resolve(file); try { if (FileOption.IF_EXIST.equals(fileOption)) { if (!Files.exists(from)) return; } if (FileOption.OVERWRITE.equals(fileOption)) Files.copy(from, to, COPY_ATTRIBUTES, REPLACE_EXISTING); else Files.copy(from, to, COPY_ATTRIBUTES); } catch (IOException e) { throw new ProCoreException("Could not copy " + from + " to " + to, e); } } private static void deleteFile(Path from, String file, FileOption fileOption) throws ProCoreException { Path path = from.resolve(file); deletePath(path, fileOption); } private static void deletePath(Path path, FileOption fileOption) throws ProCoreException { try { if (FileOption.IF_EXIST.equals(fileOption)) Files.deleteIfExists(path); else Files.delete(path); } catch (IOException e) { throw new ProCoreException("Could not delete " + path, e); } } private boolean isFolderEmpty(final Path folder) { // True if folder exists and is empty. if (!Files.isDirectory(folder)) return false; try(DirectoryStream folderStream = Files.newDirectoryStream(folder)) { return !folderStream.iterator().hasNext(); } catch (IOException e) { Logger.defaultLogError("Failed to open folder stream. folder=" + folder, e); return false; } } private void movePath(Path fromPath, Path toPath, String file, FileOption fileOption) throws ProCoreException { Path from = fromPath.resolve(file); Path to = toPath.resolve(file); try { if (FileOption.OVERWRITE.equals(fileOption)) Files.move(from, to, REPLACE_EXISTING); else Files.move(from, to); } catch (IOException e) { throw new ProCoreException("Could not move " + from + " to " + to, e); } } private static void copyTree(Path from, Path to, String path, final FileOption fileOption) throws ProCoreException { copyTree(from.resolve(path), to.resolve(path), fileOption); } private static void copyTree(Path from, Path to, final FileOption fileOption) throws ProCoreException { class Visitor extends SimpleFileVisitor { private Path fromPath; private Path toPath; Visitor(Path from, Path to) { fromPath = from; toPath = to; } @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { Path targetPath = toPath.resolve(fromPath.relativize(dir)); if (!Files.exists(targetPath)) { Files.createDirectory(targetPath); } return FileVisitResult.CONTINUE; } @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { if (FileOption.OVERWRITE.equals(fileOption)) Files.copy(file, toPath.resolve(fromPath.relativize(file)), COPY_ATTRIBUTES, REPLACE_EXISTING); else Files.copy(file, toPath.resolve(fromPath.relativize(file)), COPY_ATTRIBUTES); return FileVisitResult.CONTINUE; } } try { Visitor v = new Visitor(from, to); EnumSet opts = EnumSet.noneOf(FileVisitOption.class); Files.walkFileTree(from, opts, Integer.MAX_VALUE, v); } catch (IOException e) { throw new ProCoreException("Could not copy " + from + " to " + to, e); } } private static void deleteTree(Path path) throws ProCoreException { if (!Files.exists(path)) return; class Visitor extends SimpleFileVisitor { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { try { Files.delete(file); } catch (IOException ioe) { ioe.printStackTrace(); throw ioe; } return FileVisitResult.CONTINUE; } @Override public FileVisitResult postVisitDirectory(Path dir, IOException e) throws IOException { if (e == null) { try { Files.delete(dir); } catch (IOException ioe) { ioe.printStackTrace(); throw ioe; } return FileVisitResult.CONTINUE; } throw e; } } try { Visitor v = new Visitor(); EnumSet opts = EnumSet.noneOf(FileVisitOption.class); Files.walkFileTree(path, opts, Integer.MAX_VALUE, v); } catch (IOException e) { throw new ProCoreException("Could not delete " + path, e); } } private static void moveFolder(Path fromPath, Path toPath, String folder, FileOption fileOption) throws ProCoreException { Path from = fromPath.resolve(folder); Path to = toPath.resolve(folder); copyTree(from, to, fileOption); deleteTree(from); } enum FileOption { IF_EXIST, IF_NEW, OVERWRITE } private void copyFiles(final Path from, final Path to, String pattern, final FileOption fileOption) throws ProCoreException { class Visitor extends SimpleFileVisitor { private final PathMatcher matcher; Visitor(String pattern) { matcher = FileSystems.getDefault().getPathMatcher("glob:" + pattern); } @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { Path name = file.getFileName(); if (null == name) return FileVisitResult.CONTINUE; else if (!matcher.matches(name)) return FileVisitResult.CONTINUE; if (FileOption.OVERWRITE.equals(fileOption)) Files.copy(file, to.resolve(name), REPLACE_EXISTING); else Files.copy(file, to.resolve(name)); return FileVisitResult.CONTINUE; } @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { if (dir.equals(from)) return FileVisitResult.CONTINUE; else return FileVisitResult.SKIP_SUBTREE; } } try { Visitor v = new Visitor(pattern); EnumSet opts = EnumSet.noneOf(FileVisitOption.class); Files.walkFileTree(from, opts, Integer.MAX_VALUE, v); } catch (IOException e) { throw new ProCoreException("Could not copy " + from + " to " + to, e); } } private void moveFiles(final Path from, final Path to, String pattern, final FileOption fileOption) throws ProCoreException { class Visitor extends SimpleFileVisitor { private final PathMatcher matcher; Visitor(String pattern) { matcher = FileSystems.getDefault().getPathMatcher("glob:" + pattern); } @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { Path name = file.getFileName(); if (null == name) return FileVisitResult.CONTINUE; else if (!matcher.matches(name)) return FileVisitResult.CONTINUE; if (FileOption.OVERWRITE.equals(fileOption)) Files.move(file, to.resolve(name), REPLACE_EXISTING); else Files.move(file, to.resolve(name)); return FileVisitResult.CONTINUE; } @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { if (dir.equals(from)) return FileVisitResult.CONTINUE; else return FileVisitResult.SKIP_SUBTREE; } } try { Visitor v = new Visitor(pattern); EnumSet opts = EnumSet.noneOf(FileVisitOption.class); Files.walkFileTree(from, opts, Integer.MAX_VALUE, v); } catch (IOException e) { throw new ProCoreException("Could not move " + from + " to " + to, e); } } private void deleteClusters(final Path head, final Path tail) throws ProCoreException { class Visitor extends SimpleFileVisitor { private final PathMatcher matcher; Visitor(String pattern) { matcher = FileSystems.getDefault().getPathMatcher("glob:" + ProCoreServer.DELETED_PATTERN); } @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { Path name = file.getFileName(); if (null == name) return FileVisitResult.CONTINUE; else if (!matcher.matches(name)) return FileVisitResult.CONTINUE; String deletedStr = name.toString(); String indexName = deletedStr.replaceFirst(ProCoreServer.DELETED_PREFIX, ProCoreServer.INDEX_PREFIX); String dataName = deletedStr.replaceFirst(ProCoreServer.DELETED_PREFIX, ProCoreServer.DATA_PREFIX); String[] ss = deletedStr.split("\\."); String valuePattern = ProCoreServer.VALUE_PREFIX + ss[1] + "." + ss[2] + "*"; Files.delete(file); Files.delete(head.resolve(indexName)); Files.delete(head.resolve(dataName)); try { deleteFiles(head, valuePattern); if (Files.isDirectory(tail)) { Files.deleteIfExists(tail.resolve(dataName)); deleteFiles(tail, valuePattern); } } catch (ProCoreException e) { throw new IOException("Delete values failed.", e); } return FileVisitResult.CONTINUE; } @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { if (dir.equals(head)) return FileVisitResult.CONTINUE; else return FileVisitResult.SKIP_SUBTREE; } } try { Visitor v = new Visitor(ProCoreServer.DELETED_PREFIX); EnumSet opts = EnumSet.noneOf(FileVisitOption.class); Files.walkFileTree(head, opts, Integer.MAX_VALUE, v); } catch (IOException e) { throw new ProCoreException("Could not delete cluster(s).\nhead=" + head + "\ntail=" + tail); } } private static long convertHexStringToLong(String hex) { if (hex.length() < 16) return Long.parseLong(hex, 16); long f = Long.parseLong(hex.substring(0,1), 16) << 60; long l = Long.parseLong(hex.substring(1,16), 16); return f | l; } private void purgeValues(final Path folder) throws ProCoreException { final class Value { long first; // First part of cluster id. long second; // Secon part of cluster id. int index; // Resource index of the value. transient long cs; // Change set of the value. public Value(long first, long second, int index, long cs) { this.first = first; this.second = second; this.index = index; this.cs = cs; } String getName() { return ProCoreServer.VALUE_PREFIX + toString() + ProCoreServer.VALUE_SUFFIX; } @Override public String toString() { return String.format("%x.%x.%d.%d", first, second, index, cs); } @Override public boolean equals(Object o) { if (this == o) return true; else if (!(o instanceof Value)) return false; Value x = (Value)o; return first == x.first && second == x.second && index == x.index; } @Override public int hashCode() { int result = 17; int f = (int)(first ^ (first >>> 32)); result = 31 * result + f; int s = (int)(second ^ (second >>> 32)); result = 31 * result + s; return result + index; } } File[] files = folder.toFile().listFiles(new FilenameFilter() { @Override public boolean accept(File dir, String name) { return name.startsWith(ProCoreServer.VALUE_PREFIX); } }); HashMap values = new HashMap(); for (int i = 0; i < files.length; ++i) { String s = files[i].getName(); String[] ss = s.split("\\."); if (ss.length != 6) { Logger.defaultLogError("Illegal external value file name. name=" + s); continue; } long first = convertHexStringToLong(ss[1]); long second = convertHexStringToLong(ss[2]); int ri = Integer.parseInt(ss[3]); long cs = Long.parseLong(ss[4]); Value nv = new Value(first, second, ri, cs); Value ov = values.get(nv); if (null == ov) values.put(nv, nv); else if (ov.cs < nv.cs) { deleteFile(folder, ov.getName(), FileOption.IF_EXIST); ov.cs = nv.cs;; } else deleteFile(folder, nv.getName(), FileOption.IF_EXIST); } } private long getNextClusterId(final Path db) throws ProCoreException { long clusterId = 0; Path dbHead = db.resolve(ProCoreServer.BRANCH_DIR); File[] files = dbHead.toFile().listFiles(new FilenameFilter() { @Override public boolean accept(File dir, String name) { return name.startsWith(ProCoreServer.DATA_PREFIX); } }); for (int i = 0; i < files.length; ++i) { String s = files[i].getName(); String[] ss = s.split("\\.", 4); if (ss.length != 4) { Logger.defaultLogError("Illegal cluster file name. name=" + s); continue; } long id = convertHexStringToLong(ss[2]); if (id > clusterId) clusterId = id; } final Path dbTail = db.resolve(ProCoreServer.TAIL_DIR); if (!Files.exists(dbTail)) return clusterId + 1; class Visitor extends SimpleFileVisitor { private final PathMatcher matcher; long clusterId; Visitor(String pattern, long clusterId) { this.clusterId = clusterId; matcher = FileSystems.getDefault().getPathMatcher("glob:" + pattern); } @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { Path name = file.getFileName(); if (null == name) return FileVisitResult.CONTINUE; else if (!matcher.matches(name)) return FileVisitResult.CONTINUE; String s = name.toString(); String[] ss = s.split("\\.", 4); if (ss.length != 4) { Logger.defaultLogError("Illegal cluster file name. name=" + s); return FileVisitResult.CONTINUE; } long id = convertHexStringToLong(ss[2]); if (id > clusterId) clusterId = id; return FileVisitResult.CONTINUE; } @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { if (dir.equals(dbTail)) return FileVisitResult.CONTINUE; else return FileVisitResult.SKIP_SUBTREE; } } try { Visitor v = new Visitor(ProCoreServer.DATA_PATTERN, clusterId); EnumSet opts = EnumSet.noneOf(FileVisitOption.class); Files.walkFileTree(dbTail, opts, Integer.MAX_VALUE, v); return v.clusterId + 1; } catch (IOException e) { throw new ProCoreException("Could not get next free cluster id for " + db, e); } } private static void deleteFiles(final Path folder, String pattern) throws ProCoreException { if (!Files.exists(folder)) return; class Visitor extends SimpleFileVisitor { private final PathMatcher matcher; Visitor(String pattern) { matcher = FileSystems.getDefault().getPathMatcher("glob:" + pattern); } @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { Path name = file.getFileName(); if (null == name) return FileVisitResult.CONTINUE; else if (!matcher.matches(name)) return FileVisitResult.CONTINUE; Files.delete(file); return FileVisitResult.CONTINUE; } @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { if (dir.equals(folder)) return FileVisitResult.CONTINUE; else return FileVisitResult.SKIP_SUBTREE; } } try { Visitor v = new Visitor(pattern); EnumSet opts = EnumSet.noneOf(FileVisitOption.class); Files.walkFileTree(folder, opts, Integer.MAX_VALUE, v); } catch (IOException e) { throw new ProCoreException("Could not delete " + folder + " with " + pattern, e); } } public void copy(File to) throws ProCoreException { copyDatabaseFiles(server.getFolder().toPath(), to.toPath()); } public boolean isServerAlive() { return server.isAlive(); } Client newClient() throws ProCoreException { return server.newClient(); } public void createFolder() throws ProCoreException { if (server.folder.exists()) return; boolean created = server.folder.mkdirs(); if (!created) throw new ProCoreException("Could not create folder=" + server.folder); } /** * @throws ProCoreException if could not stop server. */ public boolean serverTryToStop() throws ProCoreException { return server.tryToStop(); } class Server { class Command { void Do(ProCoreServer proCoreServer) { } } class Event { } private final File folder; // Always non null. private final ProCoreServer proCoreServer; void start() throws ProCoreException { try { proCoreServer.start(); } catch (DatabaseLastExitException e) { if (null == dbUserAgent) return; if (!dbUserAgent.handleStart(e)) throw new ProCoreException(folder, "Failed to handle start exception.", e); proCoreServer.start(); } } void stop() throws ProCoreException { try { proCoreServer.stop(); } catch (InterruptedException e) { if (proCoreServer.isAlive()) throw new ProCoreException("ProCoreServer stop was interrupted.", e); } } boolean isAlive() { try { return proCoreServer.isAlive(); } catch (ProCoreException e) { Logger.defaultLogError(e); return false; } } Server(File aFolder) { if (null == aFolder) throw new RuntimeException("Database folder can not be null."); try { folder = aFolder.getCanonicalFile(); } catch (IOException e) { String t = "Could not get canonical path. file=" + aFolder; Logger.defaultLogError(t); throw new RuntimeException(t); } try { File serverFolder = org.simantics.db.server.internal.Activator.getServerFolder(); if (!folder.isDirectory()) folder.mkdirs(); proCoreServer = ProCoreServer.getProCoreServer(serverFolder, folder); return; } catch (Throwable t) { Logger.defaultLogError(t); throw new RuntimeException(t); } } void createConfiguration() throws ProCoreException { if (!folder.isDirectory()) throw new ProCoreException("Can't create configuration because folder is not ok. folder=" + folder.getAbsolutePath()); File file = new File(folder, ProCoreServer.CONFIG_FILE); try { file.createNewFile(); } catch (IOException e) { throw new ProCoreException("Can't create configuration file. file=" + file.getAbsolutePath()); } } void deleteGuard() throws ProCoreException { if (server.isActive()) throw new ProCoreException("Will not delete guard file when server is alive."); if (!server.isFolderOk(server.getFolder())) throw new ProCoreException("Will not delete guard file when server folder is not ok. folder=" + server.getFolder()); File file = new File(folder, ProCoreServer.GUARD_FILE); if (!file.exists()) throw new ProCoreException("Guard file does not exist. file=" + file.getAbsolutePath()); boolean deleted = file.delete(); if (!deleted) throw new ProCoreException("Failed to delete file=" + file.getAbsolutePath()); } String execute(String aCommand) throws ProCoreException { try { return proCoreServer.execute(aCommand); } catch (InterruptedException e) { throw new ProCoreException("Execute call was interrupted.", e); } } Status getStatus() { Status status = Status.NoDatabase; String path; try { path = folder.getCanonicalPath(); } catch (IOException e) { Util.logError("Could not get canonical path for folder. folder=" + folder.getAbsolutePath(), e); path = ""; } if (!isFolderOk(folder)) status = Status.NoDatabase; else if (!isAlive()) status = Status.NotRunning; else if (isConnected()) { if (isLocal()) status = Status.Local; else status = Status.Remote; } else status = Status.Standalone; status.message = status.getString() + "@" + path; return status; } File getFolder() { return folder; } boolean isActive() { try { return proCoreServer.isActive(); } catch (ProCoreException e) { Logger.defaultLogError("IsActive failed.", e); return false; } } boolean isConnected() { try { return proCoreServer.isConnected(); } catch (ProCoreException e) { Logger.defaultLogError("IsConnected failed.", e); return false; } } boolean isLocal() { try { return proCoreServer.isLocal(); } catch (ProCoreException e) { Logger.defaultLogError("IsLocal faailed.", e); return false; } } void connect() throws ProCoreException, InterruptedException { proCoreServer.connect(); } void disconnect() { try { proCoreServer.disconnect(); } catch (ProCoreException e) { Logger.defaultLogError("Could not disconnect.", e); } } /** * @param aFolder * @return true if given folder contains database journal file or * configuration file. */ boolean isFolderOk(File aFolder) { if (!aFolder.isDirectory()) return false; File config = new File(aFolder, ProCoreServer.CONFIG_FILE); if (config.exists()) return true; File journal = new File(aFolder, ProCoreServer.JOURNAL_FILE); if (journal.exists()) return true; return false; } /** * @throws ProCoreException if could not stop server. */ boolean tryToStop() throws ProCoreException { return proCoreServer.tryToStop(); } Client newClient() throws ProCoreException { return proCoreServer.newClient(); } } // From interface Database @Override public void initFolder(Properties properties) throws ProCoreException { createFolder(); serverCreateConfiguration(); } @Override public void deleteFiles() throws ProCoreException { deleteDatabaseFiles(); } @Override public void start() throws ProCoreException { connect(); } @Override public boolean isRunning() throws ProCoreException { return isServerAlive(); } @Override public boolean tryToStop() throws ProCoreException { return serverTryToStop(); } @Override public String execute(String aCommand) throws ProCoreException { return server.execute(aCommand); } @Override public void purgeDatabase() throws SDBException { synchronized (server.proCoreServer.getProCoreClient()) { if (!server.isLocal()) throw new ProCoreException("Purge is allowed only for local server."); List clients = sessionManager.disconnect(this); purge(); try { sessionManager.connect(this, clients); } catch (InterruptedException e) { throw new ProCoreException("Failed to connect after purge.", e); } } } @Override public Session newSession(ServiceLocator locator) throws ProCoreException { return sessionManager.newSession(this); } @Override public Path createFromChangeSets(int revision) throws ProCoreException { if (!isFolderOk()) throw new ProCoreException("Folder must be valid database folder to create database from journal." ); File folder = server.getFolder(); File file = new File(folder, ProCoreServer.DCS_FILE); if (!file.isFile() && !file.canRead()) throw new ProCoreException("Dump file must be readable. file=" + file.getAbsolutePath()); Path db = Paths.get(folder.getAbsolutePath()); Path temp = createTempFolder(db, "Could not create temporary directory for database to be created from change sets."); Server s = new Server(temp.toFile()); s.createConfiguration(); s.start(); String t = s.execute("loadChangeSets .. " + revision); if (t.length() < 1) throw new ProCoreException("Could not read journal. reply=" + t); s.tryToStop(); try { int cs = Integer.parseInt(t); if (cs == revision) return temp; throw new ProCoreException("Could not load change sets. reply=" + t); } catch (NumberFormatException e) { throw new ProCoreException("Could not load change sets. reply=" + t); } } @Override public void clone(File to, int revision, boolean saveHistory) { String history = saveHistory ? "with history." : "without history."; String message = "Clone to " + to.getAbsolutePath() + "@" + revision + " " + history; Util.trace(message); } @Override public DatabaseUserAgent getUserAgent() { return dbUserAgent; } @Override public void setUserAgent(DatabaseUserAgent dbUserAgent) { this.dbUserAgent = dbUserAgent; } @Override public Status getStatus() { return server.getStatus(); } @Override public File getFolder() { return server.getFolder(); } @Override public boolean isFolderOk() { return server.isFolderOk(server.getFolder()); } @Override public boolean isFolderOk(File folder) { return server.isFolderOk(folder); } public boolean isFolderEmpty() { return isFolderEmpty(server.getFolder()); } @Override public boolean isFolderEmpty(File folder) { return isFolderEmpty(folder.toPath()); } @Override public void deleteGuard() throws ProCoreException { server.deleteGuard(); } @Override public boolean isConnected() throws ProCoreException { return server.isConnected(); } @Override public void connect() throws ProCoreException { if (!isFolderOk()) throw new ProCoreException("Could not connect to " + getFolder()); if (!server.isAlive()) server.start(); if (isConnected()) return; try { server.connect(); } catch (InterruptedException e) { Util.logError("Server connect was interrupted.", e); } if (server.isActive()) return; throw new ProCoreException("Could not connect to " + getFolder()); } @Override public void disconnect() throws ProCoreException { server.disconnect(); } @Override public Path dumpChangeSets() throws ProCoreException { if (!isFolderOk()) throw new ProCoreException("Folder must be set to dump change sets."); if (!server.isActive()) throw new ProCoreException("Server must be responsive to dump change sets."); String t = server.execute("dumpChangeSets").replaceAll("\n", ""); try { int ncs = Integer.parseInt(t); if (ncs < 1) return null; File file = new File(getFolder(), ProCoreServer.DCS_FILE); return file.toPath(); } catch (NumberFormatException e) { throw new ProCoreException("Could not dump change sets.", e); } } @Override public long serverGetTailChangeSetId() throws ProCoreException { try { return server.proCoreServer.getTailData().nextChangeSetId; } catch (TailReadException e) { return 1; } } @Override public JournalI getJournal() throws ProCoreException { return journal; } class JournalI implements Database.Journal { class Analyzed { private final File journalFile; private long lastModified; private RandomAccessFile file; // Journal file. private ArrayList offsets = new ArrayList(); // Offsets to commands in journal file. private Line lines[] = new Line[256]; int firstLine = 0; // Index of the first element in lines table. Analyzed(File dbFolder) { journalFile = new File(dbFolder, ProCoreServer.JOURNAL_FILE); for (int i=0; i metadata = (TreeMap) METADATA_SERIALIZER.deserialize(bytes); CommitMetadata commit = MetadataUtil.getMetadata(metadata, CommitMetadata.class); String date = ""; if (null != commit && null != commit.date) date = commit.date.toString(); CommentMetadata comment = MetadataUtil.getMetadata(metadata, CommentMetadata.class); String comments = ""; if (null != comment) comments = comment.toString(); return date + " " + comments; } catch (IOException e) { throw new ProCoreException("Failed to interpret metadata.", e); } } private Line getRow(int index, Line line) throws ProCoreException { if (index < 0 || index >= offsets.size()) return null; // Index out of range. if (index < firstLine || index >= firstLine + lines.length) { readLines(index); if (index < firstLine) return null; // Index out of range. } int offset = index - firstLine; line.status = lines[offset].status; line.request = lines[offset].request; line.comment = lines[offset].comment; return line; } private void readLines(int first) throws ProCoreException { open(); try { for (int i=0, index = first; i= offsets.size()) { line.status = false; line.request = ""; line.comment = ""; return; } long offset = offsets.get(index); try { file.seek(offset); int i = file.readInt(); int length = Integer.reverseBytes(i); byte b = file.readByte(); boolean ok = b != 0; b = file.readByte(); // boolean littleEndian = b != 0; i = file.readInt(); int type = Integer.reverseBytes(i); String comment = ""; if (length < 6) throw new ProCoreException("Journal file corrupted at" + file.getFilePointer() + "."); else if (length > 6) { if (type != 3) file.skipBytes(length - 6); else if (length > 22){ file.skipBytes(16); i = file.readInt(); int size = Integer.reverseBytes(i); if (size != length - 26) throw new ProCoreException("Metadata corrupted at" + file.getFilePointer() + "."); byte[] bytes = new byte[size]; file.readFully(bytes); comment = getComment(bytes); if (null == comment) comment = ""; } } i = file.readInt(); int length2 = Integer.reverseBytes(i); if (length != length2) throw new ProCoreException("Journal file corrupted at" + file.getFilePointer() + "."); String command = MessageText.get(type); line.status = ok; line.request = command; line.comment = comment; } catch (IOException e) { throw new ProCoreException("Journal file corrupted."); } } private ArrayList readOffsets() throws ProCoreException { if (!canRead()) { lastModified = 0; offsets.clear(); firstLine = 0; return offsets; } long modified = journalFile.lastModified(); if (lastModified != 0 && lastModified == modified) return offsets; // Offsets already up to date. lastModified = 0; offsets.clear(); firstLine = 0; open(); try { file.seek(0); int i = file.readInt(); int version = Integer.reverseBytes(i); if (version != 1) throw new ProCoreException("Unsupported journal file version. expected=1 got=" + version); i = file.readInt(); int major = Integer.reverseBytes(i); if (major != MessageNumber.ProtocolVersionMajor) throw new ProCoreException("Unsupported journal request major version. expected=" + MessageNumber.ProtocolVersionMajor + " got=" + major); i = file.readInt(); int minor = Integer.reverseBytes(i); if (minor > MessageNumber.ProtocolVersionMinor) throw new ProCoreException("Unsupported journal request minor version. expected=" + MessageNumber.ProtocolVersionMinor + " got=" + minor); i = file.readInt(); int length = Integer.reverseBytes(i); while (length > 0) { // Not supporting unsigned integers. if (length < 6) throw new ProCoreException("Journal file corrupted at" + file.getFilePointer() + "."); file.skipBytes(length); i = file.readInt(); int length2 = Integer.reverseBytes(i); if (length != length2) throw new ProCoreException("Journal file corrupted at " + file.getFilePointer() + "."); long offset = file.getFilePointer() - length - 8; offsets.add(offset); i = file.readInt(); length = Integer.reverseBytes(i); } } catch (EOFException e) { } catch (IOException e) { throw new ProCoreException("Failed to get command count.", e); } finally { close(); } lastModified = modified; readLines(0); return offsets; } } private final Analyzed analyzed; JournalI(File dbFolder) { this.analyzed = new Analyzed(dbFolder); } @Override public boolean canRead() { return analyzed.canRead() && !isServerAlive(); } @Override public int count() { return analyzed.count(); } @Override public int read(int index, Line line) throws ProCoreException { int count = analyzed.count(); analyzed.getRow(index, line); return count; } } @Override public String getCompression() { return "FLZ"; } }