Hardening of DB index integrity

Mark index dirty as early as possible on write operations (and sync to
disk). Clean index dirtiness indicator as last step when deleting the
index. Cache dirty statuses to write dirty files only once.

refs #7075

Change-Id: I1c8a6882270b5fbd53bc88249ba5f50c05a83b51

diff --git a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/DatabaseIndexing.java b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/DatabaseIndexing.java
index c8efdc2509edd9e5fedd59f32f9e5b8ad68bfdc6..abd6e1ba54ef1203f7508ee285b8e26a4ed442fd 100644 (file)
--- a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/DatabaseIndexing.java
+++ b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/DatabaseIndexing.java
@@ -14,6 +14,7 @@ package org.simantics.db.indexing;
 import java.io.File;
 import java.io.FileFilter;
 import java.io.IOException;
 import java.io.File;
 import java.io.FileFilter;
 import java.io.IOException;

+import java.util.ArrayList;

 
 import org.simantics.db.Resource;
 import org.simantics.db.Session;
 
 import org.simantics.db.Resource;
 import org.simantics.db.Session;


@@ -22,6 +23,7 @@ import org.simantics.db.common.request.IndexRoot;
 import org.simantics.db.common.request.WriteRequest;
 import org.simantics.db.common.utils.Logger;
 import org.simantics.db.exception.DatabaseException;
 import org.simantics.db.common.request.WriteRequest;
 import org.simantics.db.common.utils.Logger;
 import org.simantics.db.exception.DatabaseException;

+import org.simantics.db.indexing.internal.IndexChangedWriter;

 import org.simantics.db.layer0.adapter.GenericRelationIndex;
 import org.simantics.db.layer0.genericrelation.IndexedRelations;
 import org.simantics.db.layer0.internal.SimanticsInternal;
 import org.simantics.db.layer0.adapter.GenericRelationIndex;
 import org.simantics.db.layer0.genericrelation.IndexedRelations;
 import org.simantics.db.layer0.internal.SimanticsInternal;


@@ -75,7 +77,10 @@ public final class DatabaseIndexing {
             return;
         if (DEBUG)
             System.out.println("Marking all indexes dirty");
             return;
         if (DEBUG)
             System.out.println("Marking all indexes dirty");

-        getAllDirtyFile().createNewFile();
+        File allDirtyFile = getAllDirtyFile();
+        if (allDirtyFile.createNewFile()) {
+            FileUtils.syncFile(allDirtyFile);
+        }

     }
 
     public static void clearAllDirty() throws IOException {
     }
 
     public static void clearAllDirty() throws IOException {


@@ -85,39 +90,61 @@ public final class DatabaseIndexing {
         File indexBase = getIndexBaseLocation();
         if (!indexBase.exists() || !indexBase.isDirectory())
             return;
         File indexBase = getIndexBaseLocation();
         if (!indexBase.exists() || !indexBase.isDirectory())
             return;

-        delete(getAllDirtyFile());

 
         forEachIndexPath(new Procedure<File, IOException>() {
             @Override
             public void execute(File indexPath) throws IOException {
 
         forEachIndexPath(new Procedure<File, IOException>() {
             @Override
             public void execute(File indexPath) throws IOException {

-                delete(getChangedFile(indexPath));
+                getChangedFile(indexPath).delete();

             }
         });
             }
         });

-    }

 
 

+        getAllDirtyFile().delete();
+    }
+    

     /**
      * Internal to indexing, invoked by {@link IndexedRelationsImpl} which
      * doesn't want to throw these exceptions forward. Just log it.
      * 
      * @param indexPath
      */
     /**
      * Internal to indexing, invoked by {@link IndexedRelationsImpl} which
      * doesn't want to throw these exceptions forward. Just log it.
      * 
      * @param indexPath
      */

-    static void markIndexChanged(File indexPath) {
-        if (!indexPath.exists())
-            throw new IllegalArgumentException("index path " + indexPath + " does not exist");
-        if (!indexPath.isDirectory())
-            throw new IllegalArgumentException("index path " + indexPath + " is not a directory");
+    static void markIndexChanged(Session session, File indexPath) {
+        if (DEBUG)
+            System.out.println("Marking index dirty: " + indexPath);

         try {
         try {

-            if (DEBUG)
-                System.out.println("Marking index dirty: " + indexPath);
-            getChangedFile(indexPath).createNewFile();
+            File changedFile = getChangedFile(indexPath);
+            // Mark change only once per DB session.
+            if (getIndexChangedWriter(session).markDirty(changedFile)) {
+                if (indexPath.mkdirs()) {
+                    if (changedFile.createNewFile()) {
+                        FileUtils.syncFile(changedFile);
+                    }
+                }
+            }

         } catch (IOException e) {
             Logger.defaultLogError(e);
         }
     }
 
         } catch (IOException e) {
             Logger.defaultLogError(e);
         }
     }
 

+    private static IndexChangedWriter getIndexChangedWriter(Session session) {
+        IndexChangedWriter writer = session.peekService(IndexChangedWriter.class);
+        if (writer == null) {
+            synchronized (IndexChangedWriter.class) {
+                if (writer == null)
+                    session.registerService(IndexChangedWriter.class, writer = new IndexChangedWriter());
+            }
+        }
+        return writer;
+    }
+

     public static void deleteAllIndexes() throws IOException {
         File indexBase = DatabaseIndexing.getIndexBaseLocation();
     public static void deleteAllIndexes() throws IOException {
         File indexBase = DatabaseIndexing.getIndexBaseLocation();

-        delete(indexBase);
+
+        ArrayList<String> filter = new ArrayList<>(2);
+        filter.add(getAllDirtyFile().getAbsolutePath());
+        filter.add(indexBase.getAbsolutePath());
+
+        FileUtils.deleteAllWithFilter(indexBase, filter);
+        FileUtils.deleteAll(indexBase);

     }
 
     public static void deleteIndex(final Resource relation, final Resource modelPart) throws DatabaseException {
     }
 
     public static void deleteIndex(final Resource relation, final Resource modelPart) throws DatabaseException {


@@ -148,7 +175,13 @@ public final class DatabaseIndexing {
     public static void deleteIndex(File indexPath) throws IOException {
         if (DEBUG)
             System.out.println("Deleting index " + indexPath);
     public static void deleteIndex(File indexPath) throws IOException {
         if (DEBUG)
             System.out.println("Deleting index " + indexPath);

-        delete(indexPath);
+
+        ArrayList<String> filter = new ArrayList<>(2);
+        filter.add(getChangedFile(indexPath).getAbsolutePath());
+        filter.add(indexPath.getAbsolutePath());
+
+        FileUtils.deleteAllWithFilter(indexPath, filter);
+        FileUtils.deleteAll(indexPath);

     }
 
     public static void validateIndexes() throws IOException {
     }
 
     public static void validateIndexes() throws IOException {


@@ -161,7 +194,7 @@ public final class DatabaseIndexing {
             // Make sure that index-base is a valid directory
             if (DEBUG)
                 System.out.println(indexBase + " is not a directory! Removing it.");
             // Make sure that index-base is a valid directory
             if (DEBUG)
                 System.out.println(indexBase + " is not a directory! Removing it.");

-            delete(indexBase);
+            FileUtils.deleteAll(indexBase);

             indexBase.mkdirs();
             return;
         }
             indexBase.mkdirs();
             return;
         }


@@ -169,7 +202,6 @@ public final class DatabaseIndexing {
         if (allDirtyFile.isFile()) {
             if (DEBUG)
                 System.out.println("All indexes marked dirty, removing them.");
         if (allDirtyFile.isFile()) {
             if (DEBUG)
                 System.out.println("All indexes marked dirty, removing them.");

-            delete(allDirtyFile);

             deleteAllIndexes();
         } else {
             forEachIndexPath(new Procedure<File, IOException>() {
             deleteAllIndexes();
         } else {
             forEachIndexPath(new Procedure<File, IOException>() {


@@ -186,12 +218,6 @@ public final class DatabaseIndexing {
         }
     }
 
         }
     }
 

-
-    private static void delete(File fileOrDir) throws IOException {
-        if (fileOrDir.exists())
-            FileUtils.deleteAll(fileOrDir);
-    }
-

     interface Procedure<T, E extends Throwable> {
         void execute(T t) throws E;
     }
     interface Procedure<T, E extends Throwable> {
         void execute(T t) throws E;
     }

diff --git a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsImpl.java b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsImpl.java
index aaadc4704d487ddbdce71af7e59a78522648657b..1b4d62a1d6c6cf9c6d199b7454e713232c763526 100644 (file)
--- a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsImpl.java
+++ b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsImpl.java
@@ -392,13 +392,13 @@ public class IndexedRelationsImpl implements IndexedRelations {
         
         try {
                
         
         try {
                

+               DatabaseIndexing.markIndexChanged(processor.getSession(), searcher.getIndexPath());

                if(!searcher.startAccess(null, processor.getSession(), true)) {
                 // Could not write index for some reason. Ignore and let the next index query reinitialize the index.
                        return;
                }
                
             searcher.insertIndex(progress.newChild(40), relation, 1, documents);
                if(!searcher.startAccess(null, processor.getSession(), true)) {
                 // Could not write index for some reason. Ignore and let the next index query reinitialize the index.
                        return;
                }
                
             searcher.insertIndex(progress.newChild(40), relation, 1, documents);

-            DatabaseIndexing.markIndexChanged(searcher.getIndexPath());

             
         } catch (InvalidResourceReferenceException e) {
             throw new IndexException(e);
             
         } catch (InvalidResourceReferenceException e) {
             throw new IndexException(e);


@@ -429,13 +429,13 @@ public class IndexedRelationsImpl implements IndexedRelations {
         LockHandle handle = lock(processor, Pair.make(relationResource, input), true);
         try {
                
         LockHandle handle = lock(processor, Pair.make(relationResource, input), true);
         try {
                

+            DatabaseIndexing.markIndexChanged(processor.getSession(), searcher.getIndexPath());

                if(!searcher.startAccess(null, processor.getSession(), true)) {
                 // Could not write index for some reason. Ignore and let the next index query reinitialize the index.
                        return;
                }
                
             searcher.removeIndex(progress.newChild(40), relation, processor, key, keyValues);
                if(!searcher.startAccess(null, processor.getSession(), true)) {
                 // Could not write index for some reason. Ignore and let the next index query reinitialize the index.
                        return;
                }
                
             searcher.removeIndex(progress.newChild(40), relation, processor, key, keyValues);

-            DatabaseIndexing.markIndexChanged(searcher.getIndexPath());

             
         } catch (DatabaseException e) {
             throw new IndexException(e);
             
         } catch (DatabaseException e) {
             throw new IndexException(e);


@@ -492,13 +492,12 @@ public class IndexedRelationsImpl implements IndexedRelations {
 
         try {
                
 
         try {
                

+               DatabaseIndexing.markIndexChanged(processor.getSession(), searcher.getIndexPath());

                if(!searcher.startAccess(null, processor.getSession(), true)) {
                 // Could not write index for some reason. Ignore and let the next index query reinitialize the index.
                        return true;
                }
                if(!searcher.startAccess(null, processor.getSession(), true)) {
                 // Could not write index for some reason. Ignore and let the next index query reinitialize the index.
                        return true;
                }

-            didChange = searcher.replaceIndex(progress.newChild(40), key, keyValues, relation, 1, documents);
-            if(didChange)
-               DatabaseIndexing.markIndexChanged(searcher.getIndexPath());
+            searcher.replaceIndex(progress.newChild(40), key, keyValues, relation, 1, documents);

             
         } catch (InvalidResourceReferenceException e) {
             throw new IndexException(e);
             
         } catch (InvalidResourceReferenceException e) {
             throw new IndexException(e);

diff --git a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/internal/IndexChangedWriter.java b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/internal/IndexChangedWriter.java
new file mode 100644 (file)
index 0000000..c0e97a2
--- /dev/null
+++ b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/internal/IndexChangedWriter.java
@@ -0,0 +1,32 @@
+/*******************************************************************************
+ * Copyright (c) 2017 Association for Decentralized Information Management
+ * in Industry THTH ry.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ *
+ * Contributors:
+ *     Semantum Oy - initial API and implementation
+ *******************************************************************************/
+package org.simantics.db.indexing.internal;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * An internal in-memory container for telling whether a certain index has been
+ * changed since the platform was last started up. 
+ * 
+ * @author Jussi Koskela
+ * @since 1.28.0
+ */
+public class IndexChangedWriter {
+       private Set<String> dirtyFiles = new HashSet<>();
+
+       public synchronized boolean markDirty(File dirtyFile) throws IOException {
+               return dirtyFiles.add(dirtyFile.getAbsolutePath());
+       }
+}

diff --git a/bundles/org.simantics.utils/src/org/simantics/utils/FileUtils.java b/bundles/org.simantics.utils/src/org/simantics/utils/FileUtils.java
index b3ec46a30c656b2c19fa626785696620b963f1f8..4e68f29c4ac95246da7d81d2b9e14c7b323dafbc 100644 (file)
--- a/bundles/org.simantics.utils/src/org/simantics/utils/FileUtils.java
+++ b/bundles/org.simantics.utils/src/org/simantics/utils/FileUtils.java
@@ -1013,4 +1013,10 @@ public class FileUtils {
             return FileVisitResult.CONTINUE;
         }
     }
             return FileVisitResult.CONTINUE;
         }
     }

+    
+    public static void syncFile(File file) throws IOException {
+       try (RandomAccessFile raf = new RandomAccessFile(file, "rw")) {
+               raf.getFD().sync();
+               }
+    }

 }
 }