From 20dfd0ba5e518a3706cd749c645a0a79480ea36f Mon Sep 17 00:00:00 2001 From: Tuukka Lehtonen Date: Tue, 25 Apr 2017 13:13:45 +0300 Subject: [PATCH] Improved Acorn database rollback logic. MainState will no longer destroy the entire database if the user removes directories but forgets to remove the main.state file. The information stored in main.state is now regarded as cached information only and if it seems invalid or cannot be read, the same normal rollback logic will be performed every time. Another enhancement is that rollback will now automatically store the revisions deleted by the rollback procedure in timestamped /db/recovery/yyyy-M-d_HH-mm-ss/ folders for later inspection and debugging. Previously the code just deleted all the extra revisions. Manually removing the recovery-folder is always a safe operation to perform. Also fixed a bug in databoard Files class readFile methods that take a File as argument. Previously all the functions constructed a BinaryFile using the default mode which is "rw". This unintentionally made the readFile methods create an empty file if the file did not exist. All such methods have been changed to use mode "r". refs #7124 Change-Id: I3ac04d2e33151b33f4982cf7a2edce7ddb896e11 --- .../org/simantics/acorn/ClusterManager.java | 4 +- .../src/org/simantics/acorn/MainState.java | 269 ++++++++++-------- .../src/org/simantics/databoard/Files.java | 8 +- 3 files changed, 160 insertions(+), 121 deletions(-) diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/ClusterManager.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/ClusterManager.java index 40c5de37e..e0bcbebf7 100644 --- a/bundles/org.simantics.acorn/src/org/simantics/acorn/ClusterManager.java +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/ClusterManager.java @@ -2,7 +2,6 @@ package org.simantics.acorn; import java.io.IOException; import java.math.BigInteger; -import java.nio.file.CopyOption; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; @@ -24,7 +23,6 @@ import org.simantics.acorn.lru.ClusterLRU; import org.simantics.acorn.lru.ClusterStreamChunk; import org.simantics.acorn.lru.FileInfo; import org.simantics.acorn.lru.LRU; -import org.simantics.databoard.file.RuntimeIOException; import org.simantics.db.ClusterCreator; import org.simantics.db.Database.Session.ClusterIds; import org.simantics.db.Database.Session.ResourceSegment; @@ -405,7 +403,7 @@ public class ClusterManager { public void load() throws IOException { // Main state - mainState = MainState.load(dbFolder, t -> rollback.set(true)); + mainState = MainState.load(dbFolder, () -> rollback.set(true)); lastSessionDirectory = dbFolder.resolve(Integer.toString(mainState.headDir - 1)); diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/MainState.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/MainState.java index 7d1580421..63fc5d493 100644 --- a/bundles/org.simantics.acorn/src/org/simantics/acorn/MainState.java +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/MainState.java @@ -1,109 +1,124 @@ package org.simantics.acorn; -import java.io.ByteArrayInputStream; +import java.io.FileNotFoundException; import java.io.IOException; -import java.io.OutputStream; import java.io.Serializable; +import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; import java.util.List; -import java.util.function.Consumer; +import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; import org.simantics.acorn.exception.InvalidHeadStateException; import org.simantics.databoard.Bindings; import org.simantics.databoard.binding.mutable.MutableVariant; -import org.simantics.databoard.file.RuntimeIOException; -import org.simantics.databoard.serialization.Serializer; import org.simantics.databoard.util.binary.BinaryMemory; -import org.simantics.utils.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class MainState implements Serializable { private static final long serialVersionUID = 6237383147637270225L; + private static final Logger LOGGER = LoggerFactory.getLogger(MainState.class); + public static final String MAIN_STATE = "main.state"; - - public int headDir = 0; + + public int headDir; public MainState() { + this.headDir = 0; } - + private MainState(int headDir) { this.headDir = headDir; } - public static MainState load(Path directory, Consumer callback) throws IOException { + public static MainState load(Path directory, Runnable rollbackCallback) throws IOException { Files.createDirectories(directory); Path mainState = directory.resolve(MAIN_STATE); try { - byte[] bytes = Files.readAllBytes(mainState); - MainState state = null; - try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes)) { - state = (MainState) org.simantics.databoard.Files.readFile(bais, Bindings.getBindingUnchecked(MainState.class)); + MainState state = (MainState) org.simantics.databoard.Files.readFile( + mainState.toFile(), + Bindings.getBindingUnchecked(MainState.class)); + int latestRevision = state.headDir - 1; + try { + HeadState.validateHeadStateIntegrity(directory.resolve(latestRevision + "/" + HeadState.HEAD_STATE)); + archiveRevisionDirectories(directory, latestRevision, rollbackCallback); + return state; + } catch (InvalidHeadStateException e) { + LOGGER.warn("Failed to start database from revision " + latestRevision + " stored in " + mainState + ". " + HeadState.HEAD_STATE + " is invalid."); + return rollback(directory, rollbackCallback); + } catch (FileNotFoundException e) { + LOGGER.warn("Failed to start database from revision " + latestRevision + " stored in " + mainState + ". Revision does not contain " + HeadState.HEAD_STATE + "."); + return rollback(directory, rollbackCallback); } - - while (true) { - Path latest = directory.resolve(Integer.toString(state.headDir - 1)); - try { - Path headState = latest.resolve(HeadState.HEAD_STATE); - HeadState.validateHeadStateIntegrity(headState); - break; - } catch (InvalidHeadStateException e) { - e.printStackTrace(); - state.headDir--; - callback.accept(e); - } finally { - cleanBaseDirectory(directory, latest, callback); - } - } - return state; - } catch(Exception i) { - callback.accept(i); - int largest = -1; - Path latest = findNewHeadStateDir(directory, callback); - if (latest != null) - largest = safeParseInt(-1, latest.getFileName().toString()); - // +1 because we want to return the next head version to use, - // not the latest existing version. - largest++; - MainState state = new MainState( largest ); - cleanBaseDirectory(directory, latest, callback); - return state; + } catch (FileNotFoundException e) { + // The database may also be totally empty at this point + if (!listRevisionDirs(directory, true, MainState::isInteger).isEmpty()) + return new MainState(0); + + LOGGER.warn("Unclean exit detected, " + mainState + " not found. Initiating automatic rollback."); + return rollback(directory, rollbackCallback); + } catch (Exception e) { + LOGGER.warn("Unclean exit detected. Initiating automatic rollback.", e); + return rollback(directory, rollbackCallback); } finally { - if (Files.exists(mainState)) { - Files.delete(mainState); - } + Files.deleteIfExists(mainState); + } + } + + private static MainState rollback(Path directory, Runnable rollbackCallback) throws IOException { + LOGGER.warn("Database rollback initiated for " + directory); + rollbackCallback.run(); + Path latest = findNewHeadStateDir(directory, rollbackCallback); + int latestRevision = latest != null ? safeParseInt(-1, latest) : -1; + // +1 because we want to return the next head version to use, + // not the latest existing version. + MainState state = new MainState( latestRevision + 1 ); + archiveRevisionDirectories(directory, latestRevision, rollbackCallback); + LOGGER.warn("Database rollback completed. Restarting database from revision " + latest); + return state; + } + + private byte[] toByteArray() throws IOException { + try (BinaryMemory rf = new BinaryMemory(4096)) { + Bindings.getSerializerUnchecked(Bindings.VARIANT).serialize(rf, MutableVariant.ofInstance(this)); + return rf.toByteBuffer().array(); } } public void save(Path directory) throws IOException { Path f = directory.resolve(MAIN_STATE); - BinaryMemory rf = new BinaryMemory(4096); - try { - MutableVariant v = new MutableVariant(Bindings.getBindingUnchecked(MainState.class), this); - Serializer s = Bindings.getSerializerUnchecked( Bindings.VARIANT ); - s.serialize(rf, v); - } finally { - rf.close(); - } - byte[] bytes = rf.toByteBuffer().array(); - try (OutputStream out = Files.newOutputStream(f)) { - out.write(bytes); - } + Files.write(f, toByteArray()); FileIO.syncPath(f); } - private static boolean isInteger(Path p) { + private static int safeParseInt(int defaultValue, Path p) { try { - Integer.parseInt(p.getFileName().toString()); - return true; + return Integer.parseInt(p.getFileName().toString()); } catch (NumberFormatException e) { - return false; + return defaultValue; } } + private static boolean isInteger(Path p) { + return safeParseInt(Integer.MIN_VALUE, p) != Integer.MIN_VALUE; + } + + private static Predicate isGreaterThan(int i) { + return p -> { + int pi = safeParseInt(Integer.MIN_VALUE, p); + return pi != Integer.MIN_VALUE && pi > i; + }; + } + /** * * @param directory @@ -111,70 +126,96 @@ public class MainState implements Serializable { * @return * @throws IOException */ - private static Path findNewHeadStateDir(Path directory, Consumer callback) throws IOException { - try (Stream s = Files.walk(directory, 1)) { - List reverseSortedPaths = s - .filter(p -> !p.equals(directory) && isInteger(p) && Files.isDirectory(p)) - .sorted((p1, p2) -> { - int p1Name = Integer.parseInt(p1.getFileName().toString()); - int p2Name = Integer.parseInt(p2.getFileName().toString()); - return Integer.compare(p2Name, p1Name); - }).collect(Collectors.toList()); - - Path latest = null; - for (Path last : reverseSortedPaths) { - Path headState = last.resolve(HeadState.HEAD_STATE); - try { - HeadState.validateHeadStateIntegrity(headState); - latest = last; - break; - } catch (IOException | InvalidHeadStateException e) { - // Cleanup is done in {@link cleanBaseDirectory} method - callback.accept(e); - } + private static Path findNewHeadStateDir(Path directory, Runnable rollbackCallback) throws IOException { + for (Path last : listRevisionDirs(directory, true, MainState::isInteger)) { + try { + HeadState.validateHeadStateIntegrity(last.resolve(HeadState.HEAD_STATE)); + return last; + } catch (IOException | InvalidHeadStateException e) { + // Cleanup is done in {@link cleanRevisionDirectories} method + rollbackCallback.run(); } - return latest; } + return null; } - private static int safeParseInt(int defaultValue, String s) { - try { - return Integer.parseInt(s); - } catch (NumberFormatException e) { - return defaultValue; + private static void archiveRevisionDirectories(Path directory, int greaterThanRevision, Runnable rollbackCallback) throws IOException { + List reverseSortedPaths = listRevisionDirs(directory, true, isGreaterThan(greaterThanRevision)); + if (reverseSortedPaths.isEmpty()) + return; + + // If none of the revisions to be archived are actually committed revisions + // then just delete them. Commitment is indicated by the head.state file. + if (!anyContainsHeadState(reverseSortedPaths)) { + for (Path p : reverseSortedPaths) { + deleteAll(p); + LOGGER.info("Removed useless working folder " + p); + } + return; } + + // Some kind of rollback is being performed. There is a possibility that + // indexes and virtual graphs are out of sync with the persistent database. + rollbackCallback.run(); + + Path recoveryFolder = getRecoveryFolder(directory); + Files.createDirectories(recoveryFolder); + LOGGER.info("Created new database recovery folder " + recoveryFolder); + for (Path p : reverseSortedPaths) { + Files.move(p, recoveryFolder.resolve(p.getFileName().toString())); + LOGGER.info("Archived revision " + p + " in recovery folder " + recoveryFolder); + } + } + + private static boolean anyContainsHeadState(List paths) { + for (Path p : paths) + if (Files.exists(p.resolve(HeadState.HEAD_STATE))) + return true; + return false; } - private static void cleanBaseDirectory(Path directory, Path latest, Consumer callback) throws IOException { + @SafeVarargs + private static List listRevisionDirs(Path directory, boolean descending, Predicate... filters) throws IOException { + int coef = descending ? -1 : 1; try (Stream s = Files.walk(directory, 1)) { - List reverseSortedPaths = s - .filter(p -> !p.equals(directory) && isInteger(p) && Files.isDirectory(p)) - .sorted((p1, p2) -> { - int p1Name = Integer.parseInt(p1.getFileName().toString()); - int p2Name = Integer.parseInt(p2.getFileName().toString()); - return Integer.compare(p2Name, p1Name); - }).collect(Collectors.toList()); - - for (Path p : reverseSortedPaths) { - if (!p.equals(latest)) { - if (Files.exists(p.resolve(HeadState.HEAD_STATE))) { - // this indicates that there is a possibility that index and vg's are out of sync - // if we are able to find folders with higher number than the current head.state - callback.accept(null); - } - uncheckedDeleteAll(p); - } else { - break; - } - } + Stream fs = s.filter(p -> !p.equals(directory)); + for (Predicate p : filters) + fs = fs.filter(p); + return fs.filter(Files::isDirectory) + .sorted((p1, p2) -> coef * Integer.compare(Integer.parseInt(p1.getFileName().toString()), + Integer.parseInt(p2.getFileName().toString()))) + .collect(Collectors.toList()); } } - private static void uncheckedDeleteAll(Path path) { - try { - FileUtils.deleteAll(path.toFile()); - } catch (IOException e) { - throw new RuntimeIOException(e); + private static void deleteAll(Path dir) throws IOException { + Files.walkFileTree(dir, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } + }); + } + + private static final DateTimeFormatter RECOVERY_DIR_FORMAT = DateTimeFormatter.ofPattern("yyyy-M-d_HH-mm-ss"); + + private static Path getRecoveryFolder(Path directory) { + return findNonexistentDir( + directory.resolve("recovery"), + RECOVERY_DIR_FORMAT.format(ZonedDateTime.now())); + } + + private static Path findNonexistentDir(Path inDirectory, String prefix) { + for (int i = 0;; ++i) { + Path dir = inDirectory.resolve(i == 0 ? prefix : prefix + "-" + i); + if (Files.notExists(dir)) + return dir; } } diff --git a/bundles/org.simantics.databoard/src/org/simantics/databoard/Files.java b/bundles/org.simantics.databoard/src/org/simantics/databoard/Files.java index ae4f46f31..cc41e8e9b 100644 --- a/bundles/org.simantics.databoard/src/org/simantics/databoard/Files.java +++ b/bundles/org.simantics.databoard/src/org/simantics/databoard/Files.java @@ -186,7 +186,7 @@ public class Files { * @throws IOException */ public static Datatype readFileType(File file) throws IOException { - BinaryFile rf = new BinaryFile( file ); + BinaryFile rf = new BinaryFile( file, "r" ); try { Binding datatype_binding = Bindings.getBindingUnchecked( Datatype.class ); return (Datatype) Bindings.getSerializerUnchecked( datatype_binding ).deserialize( rf ); @@ -206,7 +206,7 @@ public class Files { * @throws IOException */ public static Object readFile(File file, Binding binding) throws IOException { - BinaryFile rf = new BinaryFile( file ); + BinaryFile rf = new BinaryFile( file, "r" ); try { Binding datatype_binding = Bindings.getBindingUnchecked( Datatype.class ); Datatype type = (Datatype) Bindings.getSerializerUnchecked( datatype_binding ).deserialize( rf ); @@ -239,7 +239,7 @@ public class Files { * @throws IOException */ public static void readFile(File file, RecordBinding binding, Object dst) throws IOException { - BinaryFile rf = new BinaryFile( file ); + BinaryFile rf = new BinaryFile( file, "r" ); try { Binding datatype_binding = Bindings.getBindingUnchecked( Datatype.class ); Datatype type = (Datatype) Bindings.getSerializerUnchecked( datatype_binding ).deserialize( rf ); @@ -353,7 +353,7 @@ public class Files { public static DataInput openInput( File file ) throws IOException { - return new BinaryFile(file); + return new BinaryFile(file, "r"); } public static DataInput openInput( byte[] data ) -- 2.43.2