From 702195f31e8e94fc0e18172046b644723b0e9ea8 Mon Sep 17 00:00:00 2001 From: Jussi Koskela Date: Fri, 10 Mar 2017 09:38:28 +0200 Subject: [PATCH] Hardening of DB index integrity Mark index dirty as early as possible on write operations (and sync to disk). Clean index dirtiness indicator as last step when deleting the index. Cache dirty statuses to write dirty files only once. refs #7075 Change-Id: I1c8a6882270b5fbd53bc88249ba5f50c05a83b51 --- .../db/indexing/DatabaseIndexing.java | 70 +++++++++++++------ .../db/indexing/IndexedRelationsImpl.java | 9 ++- .../indexing/internal/IndexChangedWriter.java | 32 +++++++++ .../src/org/simantics/utils/FileUtils.java | 6 ++ 4 files changed, 90 insertions(+), 27 deletions(-) create mode 100644 bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/internal/IndexChangedWriter.java diff --git a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/DatabaseIndexing.java b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/DatabaseIndexing.java index c8efdc250..abd6e1ba5 100644 --- a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/DatabaseIndexing.java +++ b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/DatabaseIndexing.java @@ -14,6 +14,7 @@ package org.simantics.db.indexing; import java.io.File; import java.io.FileFilter; import java.io.IOException; +import java.util.ArrayList; import org.simantics.db.Resource; import org.simantics.db.Session; @@ -22,6 +23,7 @@ import org.simantics.db.common.request.IndexRoot; import org.simantics.db.common.request.WriteRequest; import org.simantics.db.common.utils.Logger; import org.simantics.db.exception.DatabaseException; +import org.simantics.db.indexing.internal.IndexChangedWriter; import org.simantics.db.layer0.adapter.GenericRelationIndex; import org.simantics.db.layer0.genericrelation.IndexedRelations; import org.simantics.db.layer0.internal.SimanticsInternal; @@ -75,7 +77,10 @@ public final class DatabaseIndexing { return; if (DEBUG) System.out.println("Marking all indexes dirty"); - getAllDirtyFile().createNewFile(); + File allDirtyFile = getAllDirtyFile(); + if (allDirtyFile.createNewFile()) { + FileUtils.syncFile(allDirtyFile); + } } public static void clearAllDirty() throws IOException { @@ -85,39 +90,61 @@ public final class DatabaseIndexing { File indexBase = getIndexBaseLocation(); if (!indexBase.exists() || !indexBase.isDirectory()) return; - delete(getAllDirtyFile()); forEachIndexPath(new Procedure() { @Override public void execute(File indexPath) throws IOException { - delete(getChangedFile(indexPath)); + getChangedFile(indexPath).delete(); } }); - } + getAllDirtyFile().delete(); + } + /** * Internal to indexing, invoked by {@link IndexedRelationsImpl} which * doesn't want to throw these exceptions forward. Just log it. * * @param indexPath */ - static void markIndexChanged(File indexPath) { - if (!indexPath.exists()) - throw new IllegalArgumentException("index path " + indexPath + " does not exist"); - if (!indexPath.isDirectory()) - throw new IllegalArgumentException("index path " + indexPath + " is not a directory"); + static void markIndexChanged(Session session, File indexPath) { + if (DEBUG) + System.out.println("Marking index dirty: " + indexPath); try { - if (DEBUG) - System.out.println("Marking index dirty: " + indexPath); - getChangedFile(indexPath).createNewFile(); + File changedFile = getChangedFile(indexPath); + // Mark change only once per DB session. + if (getIndexChangedWriter(session).markDirty(changedFile)) { + if (indexPath.mkdirs()) { + if (changedFile.createNewFile()) { + FileUtils.syncFile(changedFile); + } + } + } } catch (IOException e) { Logger.defaultLogError(e); } } + private static IndexChangedWriter getIndexChangedWriter(Session session) { + IndexChangedWriter writer = session.peekService(IndexChangedWriter.class); + if (writer == null) { + synchronized (IndexChangedWriter.class) { + if (writer == null) + session.registerService(IndexChangedWriter.class, writer = new IndexChangedWriter()); + } + } + return writer; + } + public static void deleteAllIndexes() throws IOException { File indexBase = DatabaseIndexing.getIndexBaseLocation(); - delete(indexBase); + + ArrayList filter = new ArrayList<>(2); + filter.add(getAllDirtyFile().getAbsolutePath()); + filter.add(indexBase.getAbsolutePath()); + + FileUtils.deleteAllWithFilter(indexBase, filter); + FileUtils.deleteAll(indexBase); } public static void deleteIndex(final Resource relation, final Resource modelPart) throws DatabaseException { @@ -148,7 +175,13 @@ public final class DatabaseIndexing { public static void deleteIndex(File indexPath) throws IOException { if (DEBUG) System.out.println("Deleting index " + indexPath); - delete(indexPath); + + ArrayList filter = new ArrayList<>(2); + filter.add(getChangedFile(indexPath).getAbsolutePath()); + filter.add(indexPath.getAbsolutePath()); + + FileUtils.deleteAllWithFilter(indexPath, filter); + FileUtils.deleteAll(indexPath); } public static void validateIndexes() throws IOException { @@ -161,7 +194,7 @@ public final class DatabaseIndexing { // Make sure that index-base is a valid directory if (DEBUG) System.out.println(indexBase + " is not a directory! Removing it."); - delete(indexBase); + FileUtils.deleteAll(indexBase); indexBase.mkdirs(); return; } @@ -169,7 +202,6 @@ public final class DatabaseIndexing { if (allDirtyFile.isFile()) { if (DEBUG) System.out.println("All indexes marked dirty, removing them."); - delete(allDirtyFile); deleteAllIndexes(); } else { forEachIndexPath(new Procedure() { @@ -186,12 +218,6 @@ public final class DatabaseIndexing { } } - - private static void delete(File fileOrDir) throws IOException { - if (fileOrDir.exists()) - FileUtils.deleteAll(fileOrDir); - } - interface Procedure { void execute(T t) throws E; } diff --git a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsImpl.java b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsImpl.java index aaadc4704..1b4d62a1d 100644 --- a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsImpl.java +++ b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsImpl.java @@ -392,13 +392,13 @@ public class IndexedRelationsImpl implements IndexedRelations { try { + DatabaseIndexing.markIndexChanged(processor.getSession(), searcher.getIndexPath()); if(!searcher.startAccess(null, processor.getSession(), true)) { // Could not write index for some reason. Ignore and let the next index query reinitialize the index. return; } searcher.insertIndex(progress.newChild(40), relation, 1, documents); - DatabaseIndexing.markIndexChanged(searcher.getIndexPath()); } catch (InvalidResourceReferenceException e) { throw new IndexException(e); @@ -429,13 +429,13 @@ public class IndexedRelationsImpl implements IndexedRelations { LockHandle handle = lock(processor, Pair.make(relationResource, input), true); try { + DatabaseIndexing.markIndexChanged(processor.getSession(), searcher.getIndexPath()); if(!searcher.startAccess(null, processor.getSession(), true)) { // Could not write index for some reason. Ignore and let the next index query reinitialize the index. return; } searcher.removeIndex(progress.newChild(40), relation, processor, key, keyValues); - DatabaseIndexing.markIndexChanged(searcher.getIndexPath()); } catch (DatabaseException e) { throw new IndexException(e); @@ -492,13 +492,12 @@ public class IndexedRelationsImpl implements IndexedRelations { try { + DatabaseIndexing.markIndexChanged(processor.getSession(), searcher.getIndexPath()); if(!searcher.startAccess(null, processor.getSession(), true)) { // Could not write index for some reason. Ignore and let the next index query reinitialize the index. return true; } - didChange = searcher.replaceIndex(progress.newChild(40), key, keyValues, relation, 1, documents); - if(didChange) - DatabaseIndexing.markIndexChanged(searcher.getIndexPath()); + searcher.replaceIndex(progress.newChild(40), key, keyValues, relation, 1, documents); } catch (InvalidResourceReferenceException e) { throw new IndexException(e); diff --git a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/internal/IndexChangedWriter.java b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/internal/IndexChangedWriter.java new file mode 100644 index 000000000..c0e97a2e4 --- /dev/null +++ b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/internal/IndexChangedWriter.java @@ -0,0 +1,32 @@ +/******************************************************************************* + * Copyright (c) 2017 Association for Decentralized Information Management + * in Industry THTH ry. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Semantum Oy - initial API and implementation + *******************************************************************************/ +package org.simantics.db.indexing.internal; + +import java.io.File; +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; + +/** + * An internal in-memory container for telling whether a certain index has been + * changed since the platform was last started up. + * + * @author Jussi Koskela + * @since 1.28.0 + */ +public class IndexChangedWriter { + private Set dirtyFiles = new HashSet<>(); + + public synchronized boolean markDirty(File dirtyFile) throws IOException { + return dirtyFiles.add(dirtyFile.getAbsolutePath()); + } +} diff --git a/bundles/org.simantics.utils/src/org/simantics/utils/FileUtils.java b/bundles/org.simantics.utils/src/org/simantics/utils/FileUtils.java index b3ec46a30..4e68f29c4 100644 --- a/bundles/org.simantics.utils/src/org/simantics/utils/FileUtils.java +++ b/bundles/org.simantics.utils/src/org/simantics/utils/FileUtils.java @@ -1013,4 +1013,10 @@ public class FileUtils { return FileVisitResult.CONTINUE; } } + + public static void syncFile(File file) throws IOException { + try (RandomAccessFile raf = new RandomAccessFile(file, "rw")) { + raf.getFD().sync(); + } + } } -- 2.47.1