From: jsimomaa Date: Tue, 27 Feb 2018 10:00:15 +0000 (+0200) Subject: Add logging to indexing & replace File-API with NIO Path-API X-Git-Tag: v1.43.0~136^2~573 X-Git-Url: https://gerrit.simantics.org/r/gitweb?p=simantics%2Fplatform.git;a=commitdiff_plain;h=8b8630b93f8e4ec88a8a4eb53b55031407deedc0 Add logging to indexing & replace File-API with NIO Path-API refs #7785 Change-Id: I38a1d2fe7425bf06c97e1178c7b26b3566f1126b --- diff --git a/bundles/org.simantics.db.indexing/META-INF/MANIFEST.MF b/bundles/org.simantics.db.indexing/META-INF/MANIFEST.MF index 981bfe28d..fdec1b5d0 100644 --- a/bundles/org.simantics.db.indexing/META-INF/MANIFEST.MF +++ b/bundles/org.simantics.db.indexing/META-INF/MANIFEST.MF @@ -9,7 +9,8 @@ Require-Bundle: org.eclipse.core.runtime, org.apache.lucene4.core;bundle-version="4.9.0", org.apache.lucene4.queryparser;bundle-version="4.9.0", org.apache.lucene4.analyzers-common;bundle-version="4.9.0", - org.simantics.db.services;bundle-version="0.8.0" + org.simantics.db.services;bundle-version="0.8.0", + org.slf4j.api;bundle-version="1.7.25" Bundle-RequiredExecutionEnvironment: JavaSE-1.8 Export-Package: org.simantics.db.indexing Bundle-Activator: org.simantics.db.indexing.Activator diff --git a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/Activator.java b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/Activator.java index fbcace94b..b4276a70c 100644 --- a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/Activator.java +++ b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/Activator.java @@ -12,6 +12,7 @@ package org.simantics.db.indexing; import java.io.File; +import java.nio.file.Path; import org.eclipse.core.runtime.IPath; import org.eclipse.core.runtime.Platform; @@ -29,7 +30,7 @@ public class Activator extends Plugin { // The shared instance private static Activator plugin; - private File indexBaseFile; + private Path indexBaseFile; /** * The constructor @@ -46,10 +47,10 @@ public class Activator extends Plugin { super.start(context); plugin = this; IPath state = Platform.getStateLocation(context.getBundle()); - indexBaseFile = state.append("index").toFile(); + indexBaseFile = state.append("index").toFile().toPath(); } - public File getIndexBaseFile() { + public Path getIndexBaseFile() { return indexBaseFile; } diff --git a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/DatabaseIndexing.java b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/DatabaseIndexing.java index abd6e1ba5..137d6c5c4 100644 --- a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/DatabaseIndexing.java +++ b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/DatabaseIndexing.java @@ -11,17 +11,19 @@ *******************************************************************************/ package org.simantics.db.indexing; -import java.io.File; -import java.io.FileFilter; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; +import java.util.Iterator; +import java.util.function.Consumer; +import java.util.stream.Stream; import org.simantics.db.Resource; import org.simantics.db.Session; import org.simantics.db.WriteGraph; import org.simantics.db.common.request.IndexRoot; import org.simantics.db.common.request.WriteRequest; -import org.simantics.db.common.utils.Logger; import org.simantics.db.exception.DatabaseException; import org.simantics.db.indexing.internal.IndexChangedWriter; import org.simantics.db.layer0.adapter.GenericRelationIndex; @@ -29,6 +31,7 @@ import org.simantics.db.layer0.genericrelation.IndexedRelations; import org.simantics.db.layer0.internal.SimanticsInternal; import org.simantics.db.service.ServerInformation; import org.simantics.utils.FileUtils; +import org.slf4j.LoggerFactory; /** * A facade for Simantics graph database index management facilities. @@ -38,17 +41,13 @@ import org.simantics.utils.FileUtils; public final class DatabaseIndexing { private static final boolean DEBUG = IndexPolicy.TRACE_INDEX_MANAGEMENT; + private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(DatabaseIndexing.class); - public static File getIndexBaseLocation() { - return Activator.getDefault().getIndexBaseFile(); -// Activator activator = Activator.getDefault(); -// Bundle b = Platform.getBundle(Activator.BUNDLE_ID); -// IPath state = Platform.getStateLocation(b); -// File path = state.append("index").toFile(); -// return path; + public static Path getIndexBaseLocation() { + return Activator.getDefault().getIndexBaseFile(); } - public static File getIndexLocation(Session session, Resource relation, Resource input) { + public static Path getIndexLocation(Session session, Resource relation, Resource input) { if (session == null) throw new NullPointerException("null session"); if (relation == null) @@ -60,45 +59,48 @@ public final class DatabaseIndexing { + "." + relation.getResourceId() + "." + input.getResourceId(); - return new File(getIndexBaseLocation(), dir); + return getIndexBaseLocation().resolve(dir); } - private static File getAllDirtyFile() { - return new File(getIndexBaseLocation(), ".dirty"); + private static Path getAllDirtyFile() { + return getIndexBaseLocation().resolve(".dirty"); } - private static File getChangedFile(File indexPath) { - return new File(indexPath, ".changed"); + private static Path getChangedFile(Path indexPath) { + return indexPath.resolve(".changed"); } public static void markAllDirty() throws IOException { - File indexBase = getIndexBaseLocation(); - if (!indexBase.exists() || !indexBase.isDirectory()) + Path indexBase = getIndexBaseLocation(); + if (!Files.exists(indexBase) || !Files.isDirectory(indexBase)) return; - if (DEBUG) - System.out.println("Marking all indexes dirty"); - File allDirtyFile = getAllDirtyFile(); - if (allDirtyFile.createNewFile()) { - FileUtils.syncFile(allDirtyFile); + if (LOGGER.isDebugEnabled()) + LOGGER.debug("Marking all indexes dirty"); + Path allDirtyFile = getAllDirtyFile(); + if (!Files.exists(allDirtyFile)) { + Files.createFile(allDirtyFile); + FileUtils.sync(allDirtyFile); } } public static void clearAllDirty() throws IOException { - if (DEBUG) - System.out.println("Clearing dirty state of all indexes"); + if (LOGGER.isDebugEnabled()) + LOGGER.debug("Clearing dirty state of all indexes"); - File indexBase = getIndexBaseLocation(); - if (!indexBase.exists() || !indexBase.isDirectory()) + Path indexBase = getIndexBaseLocation(); + if (!Files.exists(indexBase) || !Files.isDirectory(indexBase)) return; - forEachIndexPath(new Procedure() { - @Override - public void execute(File indexPath) throws IOException { - getChangedFile(indexPath).delete(); + forEachIndexPath(indexPath -> { + Path p = getChangedFile(indexPath); + try { + FileUtils.delete(p); + } catch (IOException e) { + LOGGER.error("Could not delete {}", p.toAbsolutePath(), e); } }); - getAllDirtyFile().delete(); + FileUtils.delete(getAllDirtyFile()); } /** @@ -107,21 +109,20 @@ public final class DatabaseIndexing { * * @param indexPath */ - static void markIndexChanged(Session session, File indexPath) { - if (DEBUG) - System.out.println("Marking index dirty: " + indexPath); + static void markIndexChanged(Session session, Path indexPath) { + if (LOGGER.isDebugEnabled()) + LOGGER.debug("Marking index dirty: " + indexPath); + Path changedFile = getChangedFile(indexPath); try { - File changedFile = getChangedFile(indexPath); + // Mark change only once per DB session. if (getIndexChangedWriter(session).markDirty(changedFile)) { - if (indexPath.mkdirs()) { - if (changedFile.createNewFile()) { - FileUtils.syncFile(changedFile); - } - } + Files.createDirectories(indexPath); + Files.createFile(changedFile); + FileUtils.sync(changedFile); } } catch (IOException e) { - Logger.defaultLogError(e); + LOGGER.error("Could not mark index changed for indexPath={} and changedFile={}", indexPath.toAbsolutePath(), changedFile.toAbsolutePath()); } } @@ -137,14 +138,14 @@ public final class DatabaseIndexing { } public static void deleteAllIndexes() throws IOException { - File indexBase = DatabaseIndexing.getIndexBaseLocation(); + Path indexBase = DatabaseIndexing.getIndexBaseLocation(); - ArrayList filter = new ArrayList<>(2); - filter.add(getAllDirtyFile().getAbsolutePath()); - filter.add(indexBase.getAbsolutePath()); + ArrayList filter = new ArrayList<>(2); + filter.add(getAllDirtyFile()); + filter.add(indexBase); - FileUtils.deleteAllWithFilter(indexBase, filter); - FileUtils.deleteAll(indexBase); + FileUtils.deleteWithFilter(indexBase, path -> !filter.contains(path)); + FileUtils.delete(indexBase); } public static void deleteIndex(final Resource relation, final Resource modelPart) throws DatabaseException { @@ -172,64 +173,60 @@ public final class DatabaseIndexing { } - public static void deleteIndex(File indexPath) throws IOException { - if (DEBUG) - System.out.println("Deleting index " + indexPath); + public static void deleteIndex(Path indexPath) throws IOException { + if (LOGGER.isDebugEnabled()) + LOGGER.debug("Deleting index " + indexPath.toAbsolutePath()); - ArrayList filter = new ArrayList<>(2); - filter.add(getChangedFile(indexPath).getAbsolutePath()); - filter.add(indexPath.getAbsolutePath()); + ArrayList filter = new ArrayList<>(2); + filter.add(getChangedFile(indexPath)); + filter.add(indexPath); - FileUtils.deleteAllWithFilter(indexPath, filter); - FileUtils.deleteAll(indexPath); + FileUtils.deleteWithFilter(indexPath, path -> !filter.contains(path)); + FileUtils.delete(indexPath); } public static void validateIndexes() throws IOException { - File indexBase = getIndexBaseLocation(); - if (DEBUG) - System.out.println("Validating indexes at " + indexBase); - if (!indexBase.exists()) + Path indexBase = getIndexBaseLocation(); + if (LOGGER.isDebugEnabled()) + LOGGER.debug("Validating indexes at " + indexBase); + if (!Files.exists(indexBase)) return; - if (!indexBase.isDirectory()) { + if (!Files.isDirectory(indexBase)) { // Make sure that index-base is a valid directory - if (DEBUG) - System.out.println(indexBase + " is not a directory! Removing it."); - FileUtils.deleteAll(indexBase); - indexBase.mkdirs(); + if (LOGGER.isDebugEnabled()) + LOGGER.debug(indexBase + " is not a directory! Removing it."); + FileUtils.delete(indexBase); + Files.createDirectories(indexBase); return; } - File allDirtyFile = getAllDirtyFile(); - if (allDirtyFile.isFile()) { - if (DEBUG) - System.out.println("All indexes marked dirty, removing them."); + Path allDirtyFile = getAllDirtyFile(); + if (Files.isRegularFile(allDirtyFile)) { + if (LOGGER.isDebugEnabled()) + LOGGER.debug("All indexes marked dirty, removing them."); deleteAllIndexes(); } else { - forEachIndexPath(new Procedure() { - @Override - public void execute(File indexPath) throws IOException { - File changed = getChangedFile(indexPath); - if (changed.isFile()) { - if (DEBUG) - System.out.println("Index is dirty, removing: " + indexPath); + forEachIndexPath(indexPath -> { + Path changed = getChangedFile(indexPath); + if (Files.isRegularFile(changed)) { + if (LOGGER.isDebugEnabled()) + LOGGER.debug("Index is dirty, removing: " + indexPath); + try { deleteIndex(indexPath); + } catch (IOException e) { + LOGGER.error("Could not delete index {}", indexPath.toAbsolutePath(), e); } } }); } } - interface Procedure { - void execute(T t) throws E; - } - - private static void forEachIndexPath(Procedure callback) throws E { - for (File indexPath : getIndexBaseLocation().listFiles(new FileFilter() { - @Override - public boolean accept(File pathname) { - return pathname.isDirectory(); + private static void forEachIndexPath(Consumer callback) throws IOException { + try (Stream paths = Files.walk(getIndexBaseLocation(), 1).filter(Files::isDirectory)) { + Iterator iter = paths.iterator(); + while (iter.hasNext()) { + Path p = iter.next(); + callback.accept(p); } - })) { - callback.execute(indexPath); } } diff --git a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/ImmutableIndexedRelationsSearcher.java b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/ImmutableIndexedRelationsSearcher.java index 49f27d4ca..c1eb105ea 100644 --- a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/ImmutableIndexedRelationsSearcher.java +++ b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/ImmutableIndexedRelationsSearcher.java @@ -15,6 +15,8 @@ import org.eclipse.core.runtime.IProgressMonitor; import org.simantics.db.RequestProcessor; import org.simantics.db.Resource; import org.simantics.db.Session; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * @author Tuukka Lehtonen @@ -35,5 +37,10 @@ public class ImmutableIndexedRelationsSearcher extends IndexedRelationsSearcherB Throwable bestEffortClear(IProgressMonitor monitor, Session session) { return new IllegalStateException("Immutable index cannot be cleared"); } + + @Override + protected Logger getLogger() { + return LoggerFactory.getLogger(ImmutableIndexedRelationsSearcher.class); + } } diff --git a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsImpl.java b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsImpl.java index bc9247e37..1d027a825 100644 --- a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsImpl.java +++ b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsImpl.java @@ -13,6 +13,7 @@ package org.simantics.db.indexing; import java.io.File; import java.io.IOException; +import java.nio.file.Path; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -37,6 +38,8 @@ import org.simantics.db.layer0.genericrelation.IndexedRelations; import org.simantics.db.service.QueryControl; import org.simantics.db.service.SerialisationSupport; import org.simantics.utils.datastructures.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * @author Tuukka Lehtonen @@ -44,6 +47,8 @@ import org.simantics.utils.datastructures.Pair; */ public class IndexedRelationsImpl implements IndexedRelations { + private static final Logger LOGGER = LoggerFactory.getLogger(IndexedRelationsImpl.class); + Map indexLocks = new WeakHashMap(); static class LockHandle { @@ -370,7 +375,7 @@ public class IndexedRelationsImpl implements IndexedRelations { @Override public void insert(IProgressMonitor monitor, RequestProcessor processor, GenericRelation relation, - Resource relationResource, Resource input, Collection documents) { + Resource relationResource, Resource input, Collection documents) throws IndexException { // System.out.println("Inserting to index: " + input + " " + documents); @@ -413,7 +418,7 @@ public class IndexedRelationsImpl implements IndexedRelations { @Override public void remove(IProgressMonitor monitor, RequestProcessor processor, GenericRelation relation, - Resource relationResource, Resource input, String key, Collection keyValues) { + Resource relationResource, Resource input, String key, Collection keyValues) throws IndexException { if (relation == null) throw new IllegalArgumentException("null relation"); @@ -473,7 +478,7 @@ public class IndexedRelationsImpl implements IndexedRelations { @Override public boolean replace(IProgressMonitor monitor, RequestProcessor processor, GenericRelation relation, - Resource relationResource, Resource input, String key, Collection keyValues, Collection documents) { + Resource relationResource, Resource input, String key, Collection keyValues, Collection documents) throws IndexException { if (relation == null) throw new IllegalArgumentException("null relation"); @@ -521,16 +526,15 @@ public class IndexedRelationsImpl implements IndexedRelations { IndexedRelationsSearcherBase searcher = makeSearcher(processor, relationResource, input); LockHandle handle = lock(processor, Pair.make(relationResource, input), true); - + Path path = DatabaseIndexing.getIndexLocation(processor.getSession(), relationResource, input); try { searcher.changeState(monitor, processor.getSession(), State.NONE); if (!searcher.checkState(State.NONE)) throw new IndexException("Could not close index for input " + input + " before removing it"); - File path = DatabaseIndexing.getIndexLocation(processor.getSession(), relationResource, input); DatabaseIndexing.deleteIndex(path); - } catch (IOException e) { + LOGGER.error("Could not delete {}", path.toAbsolutePath(), e); throw new IndexException(e); } finally { handle.unlock(); diff --git a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsMemorySearcher.java b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsMemorySearcher.java index ac0918ea5..8fc0ce911 100644 --- a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsMemorySearcher.java +++ b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsMemorySearcher.java @@ -36,6 +36,8 @@ import org.simantics.db.Session; import org.simantics.db.exception.DatabaseException; import org.simantics.db.layer0.adapter.GenericRelation; import org.simantics.utils.datastructures.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * @author Tuukka Lehtonen @@ -43,6 +45,8 @@ import org.simantics.utils.datastructures.Pair; */ public class IndexedRelationsMemorySearcher extends IndexedRelationsSearcherBase { + private static final Logger LOGGER = LoggerFactory.getLogger(IndexedRelationsMemorySearcher.class); + final IndexedRelationsSearcher backend; final GenericRelation r; @@ -173,19 +177,16 @@ public class IndexedRelationsMemorySearcher extends IndexedRelationsSearcherBase @Override Directory getDirectory(Session session) throws IOException { MemoryIndexing mem = MemoryIndexing.getInstance(session); - String path = indexPath.getAbsolutePath(); + String path = indexPath.toAbsolutePath().toString(); return mem.getDirectory(path, Queries.getAnalyzer()); } @Override Throwable bestEffortClear(IProgressMonitor monitor, Session session) { - - setProblem(null); - MemoryIndexing mem = MemoryIndexing.getInstance(session); changed.clear(); - String path = indexPath.getAbsolutePath(); + String path = indexPath.toAbsolutePath().toString(); mem.remove(path); return null; @@ -196,5 +197,10 @@ public class IndexedRelationsMemorySearcher extends IndexedRelationsSearcherBase protected boolean requireChangeInfoOnReplace() { return false; } + + @Override + protected Logger getLogger() { + return LOGGER; + } } diff --git a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsSearcher.java b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsSearcher.java index 115e54595..a535183f2 100644 --- a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsSearcher.java +++ b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsSearcher.java @@ -27,13 +27,17 @@ import org.simantics.db.Session; import org.simantics.db.exception.DatabaseException; import org.simantics.db.layer0.adapter.GenericRelation; import org.simantics.utils.datastructures.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * @author Tuukka Lehtonen * @author Antti Villberg */ public class IndexedRelationsSearcher extends IndexedRelationsSearcherBase { - + + private static final Logger LOGGER = LoggerFactory.getLogger(IndexedRelationsSearcher.class); + IndexedRelationsMemorySearcher cache; IndexedRelationsSearcher(RequestProcessor session, Resource relation, Resource input, GenericRelation r) throws DatabaseException { @@ -126,7 +130,7 @@ public class IndexedRelationsSearcher extends IndexedRelationsSearcherBase { MemoryIndexing mem = MemoryIndexing.getInstance(session.getSession()); - String key = indexPath.getAbsolutePath(); + String key = indexPath.toAbsolutePath().toString(); Map>> cache = mem.persistentCache.get(key); if(cache != null) { @@ -154,7 +158,7 @@ public class IndexedRelationsSearcher extends IndexedRelationsSearcherBase { MemoryIndexing mem = MemoryIndexing.getInstance(session.getSession()); - String key = indexPath.getAbsolutePath(); + String key = indexPath.toAbsolutePath().toString(); Map> cache = mem.persistentCacheResources.get(key); if(cache != null) { @@ -280,7 +284,7 @@ public class IndexedRelationsSearcher extends IndexedRelationsSearcherBase { t = cache.bestEffortClear(monitor, session); if(t != null) return t; - String key = indexPath.getAbsolutePath(); + String key = indexPath.toAbsolutePath().toString(); MemoryIndexing mem = MemoryIndexing.getInstance(session); mem.persistentCache.remove(key); mem.persistentCacheResources.remove(key); @@ -288,5 +292,10 @@ public class IndexedRelationsSearcher extends IndexedRelationsSearcherBase { return null; } + + @Override + protected Logger getLogger() { + return LOGGER; + } } diff --git a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsSearcherBase.java b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsSearcherBase.java index b303dfc4c..9441bb75b 100644 --- a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsSearcherBase.java +++ b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsSearcherBase.java @@ -12,7 +12,6 @@ *******************************************************************************/ package org.simantics.db.indexing; -import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -63,7 +62,6 @@ import org.simantics.db.RequestProcessor; import org.simantics.db.Resource; import org.simantics.db.Session; import org.simantics.db.common.request.SafeName; -import org.simantics.db.common.utils.Logger; import org.simantics.db.common.utils.NameUtils; import org.simantics.db.exception.DatabaseException; import org.simantics.db.indexing.internal.IndexingJob; @@ -73,6 +71,7 @@ import org.simantics.db.service.CollectionSupport; import org.simantics.db.service.SerialisationSupport; import org.simantics.utils.FileUtils; import org.simantics.utils.datastructures.Pair; +import org.slf4j.Logger; import gnu.trove.map.hash.THashMap; @@ -103,6 +102,8 @@ abstract public class IndexedRelationsSearcherBase { } public void setProblem(Throwable t) { + if (t != null) + getLogger().error("Setting problem for {} and previous state {}", this, this.state, t); this.state = State.PROBLEM; this.exception = t; } @@ -131,7 +132,11 @@ abstract public class IndexedRelationsSearcherBase { protected void changeState(IProgressMonitor monitor, Session session, State state, int depth) { - if(this.state == state) return; + if (this.state == state) { + if (getLogger().isDebugEnabled()) + getLogger().debug("Trying to change state {} to the same as previous state {} in depth {} with {}", state, this.state, depth, this); + return; + } if (IndexPolicy.TRACE_INDEX_MANAGEMENT) System.err.println("Index state " + this.state.name() + " => " + state.name() + " " + this); @@ -140,20 +145,29 @@ abstract public class IndexedRelationsSearcherBase { // Try to exit problem state if (State.PROBLEM == this.state && depth > 0) { + getLogger().info("Try to exit problem state for {} and state {}", this, state); Throwable t = bestEffortClear(monitor, session); if(t != null) { + getLogger().error("Best effort clear has failed for state {} and this {}", state, this, t); exception = t; return; } // Managed to get into initial state this.state = State.NONE; + getLogger().info("Managed to get into initial state {}", this.state); return; } // Cannot move into read from no index - if (State.NONE == this.state && State.READ == state) return; + if (State.NONE == this.state && State.READ == state) { + getLogger().info("Cannot move into read from no index in {} with state {}", this, state); + return; + } // Cannot move into write from no index - if (State.NONE == this.state && State.WRITE == state) return; + if (State.NONE == this.state && State.WRITE == state) { + getLogger().info("Cannot move into write from no index in {} with state {}", this, state); + return; + } boolean success = false; @@ -225,9 +239,7 @@ abstract public class IndexedRelationsSearcherBase { } } catch (Throwable t) { - setProblem(t); - } finally { if(!success) { @@ -486,7 +498,7 @@ abstract public class IndexedRelationsSearcherBase { Resource input; - File indexPath; + Path indexPath; Directory directory; @@ -510,7 +522,7 @@ abstract public class IndexedRelationsSearcherBase { } Directory getDirectory(Session session) throws IOException { - return FSDirectory.open(indexPath); + return FSDirectory.open(indexPath.toFile()); } abstract String getDescriptor(); @@ -597,6 +609,7 @@ abstract public class IndexedRelationsSearcherBase { try { initializeIndexImpl(mon, graph, bound, overwrite); } catch (IOException e) { + getLogger().error("Index is in problematic state! {}", this, e); throw new DatabaseException(e); } }); @@ -612,8 +625,11 @@ abstract public class IndexedRelationsSearcherBase { mon.beginTask("Initializing Index", 100); if (overwrite) { - mon.subTask("Erasing previous index"); - FileUtils.deleteAll(indexPath); + if (Files.exists(indexPath)) { + mon.subTask("Erasing previous index"); + getLogger().info("Erasing previous index {}", indexPath.toAbsolutePath()); + FileUtils.delete(indexPath); + } } final AtomicReference directory = new AtomicReference(); @@ -623,7 +639,7 @@ abstract public class IndexedRelationsSearcherBase { mon.subTask("Start index write"); createDirectory(indexPath); - directory.set(FSDirectory.open(indexPath)); + directory.set(FSDirectory.open(indexPath.toFile())); IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_4_9, Queries.getAnalyzer()).setOpenMode(OpenMode.CREATE); writer.set(new IndexWriter(directory.get(), conf)); @@ -668,8 +684,10 @@ abstract public class IndexedRelationsSearcherBase { try { writer.get().addDocument(document); } catch (CorruptIndexException e) { + getLogger().error("Index is corrupted! {}", this, e); throw new IllegalStateException(e); } catch (IOException e) { + getLogger().error("Index is in problematic state! {}", this, e); throw new IllegalStateException(e); } finally { synchronized (mon) { @@ -688,7 +706,7 @@ abstract public class IndexedRelationsSearcherBase { try { s.acquire(INDEXING_THREAD_COUNT); } catch (InterruptedException e) { - Logger.defaultLogError(e); + getLogger().error("Could not initialize index {}", this, e); } // http://www.gossamer-threads.com/lists/lucene/java-dev/47895 @@ -703,9 +721,7 @@ abstract public class IndexedRelationsSearcherBase { System.out.println(getDescriptor() + "Wrote index at " + indexPath + " in " + (1e-9 * (System.nanoTime()-start)) + " seconds."); } catch (DatabaseException e) { - - Logger.defaultLogError(e); - + getLogger().error("Could not initialize index due to db {}", this, e); } finally { try { closeWriter(writer.getAndSet(null)); @@ -738,11 +754,13 @@ abstract public class IndexedRelationsSearcherBase { } result.add(o); - } catch (CorruptIndexException e) { - throw new DatabaseException(e); - } catch (IOException e) { - throw new DatabaseException(e); - } + } catch (CorruptIndexException e) { + getLogger().error("Index is corrupted! {}", this, e); + throw new DatabaseException(e); + } catch (IOException e) { + getLogger().error("Index is in problematic state! {}", this, e); + throw new DatabaseException(e); + } } @@ -823,15 +841,14 @@ abstract public class IndexedRelationsSearcherBase { result.add(entry); } catch (CorruptIndexException e) { + getLogger().error("Index is corrupted! {}", this, e); throw new DatabaseException(e); } catch (IOException e) { + getLogger().error("Index is in problematic state! {}", this, e); throw new DatabaseException(e); } - } - return result; - } }); } @@ -925,22 +942,18 @@ abstract public class IndexedRelationsSearcherBase { ResourceVisitor visitor = new ResourceVisitor(); for (ScoreDoc scoreDoc : docs.scoreDocs) { - try { - reader.document(scoreDoc.doc, visitor); result.add(support.getResource(visitor.id)); - } catch (CorruptIndexException e) { + getLogger().error("Index is corrupted! {}", this, e); throw new DatabaseException(e); } catch (IOException e) { + getLogger().error("Index is in problematic state! {}", this, e); throw new DatabaseException(e); } - } - return result; - } }); } @@ -965,8 +978,10 @@ abstract public class IndexedRelationsSearcherBase { reader.document(scoreDoc.doc, visitor); } catch (CorruptIndexException e) { + getLogger().error("Index is corrupted! {}", this, e); throw new DatabaseException(e); } catch (IOException e) { + getLogger().error("Index is in problematic state! {}", this, e); throw new DatabaseException(e); } @@ -976,59 +991,44 @@ abstract public class IndexedRelationsSearcherBase { } - protected static File getIndexDirectory(Session session, Resource relation, Resource input) { - File path = DatabaseIndexing.getIndexLocation(session, relation, input); + protected static Path getIndexDirectory(Session session, Resource relation, Resource input) { + Path path = DatabaseIndexing.getIndexLocation(session, relation, input); // System.out.println("getIndexDirectory = " + path); return path; } - private static void createDirectory(File path) throws IOException { - Path p = path.toPath(); - if (Files.exists(p) && !Files.isDirectory(p)) + private static void createDirectory(Path path) throws IOException { + if (Files.exists(path) && !Files.isDirectory(path)) throw new IOException("Could not create index directory " + path + ", a file by that name already exists"); - Files.createDirectories(p); + Files.createDirectories(path); } - File getIndexPath() { + Path getIndexPath() { return indexPath; } boolean isIndexAvailable() { - return (indexPath.exists() && indexPath.isDirectory()); + return (Files.exists(indexPath) && Files.isDirectory(indexPath)); } - Throwable bestEffortClear(IProgressMonitor monitor, Session session) { - return null; - } + abstract Throwable bestEffortClear(IProgressMonitor monitor, Session session); /* * Start from scratch. Clear all caches and rebuild the index. */ Throwable clearDirectory(IProgressMonitor monitor, Session session) { - File file = getIndexPath(); + Path file = getIndexPath(); try { - - for(int i=0;i<15;i++) { - FileUtils.deleteDir(file); - if(!file.exists()) { - return null; - } - try { - Thread.sleep(i*100); - } catch (InterruptedException e) { - } - } - + FileUtils.delete(file); } catch (Throwable t) { - + getLogger().error("Could not delete directory {}", file.toAbsolutePath(), t); return t; - } - - return new IllegalStateException("Failed to delete directory " + file.getAbsolutePath()); - + if (Files.exists(file)) + return new IllegalStateException("Failed to delete directory " + file.toAbsolutePath()); + return null; } private Field[] setFields(Field[] fs, Object[] result) { @@ -1043,11 +1043,17 @@ abstract public class IndexedRelationsSearcherBase { System.out.println(getDescriptor() + "index " + fs[i].name() + " = " + value + " : Long"); fs[i].setLongValue((Long) value); } else { - Logger.defaultLogError("Can only index Long and String fields, encountered " + value); + getLogger().error("Can only index Long and String fields, encountered " + value); return null; } } return fs; } + protected abstract Logger getLogger(); + + @Override + public String toString() { + return getClass().getSimpleName() + " [" + String.valueOf(schema) + ", " + String.valueOf(relation) + ", " + String.valueOf(input) + ", " + String.valueOf(indexPath) + ", " + String.valueOf(directory) + ", " + String.valueOf(state) + "]"; + } } diff --git a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/MemoryIndexing.java b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/MemoryIndexing.java index 60b76100c..75c6b79bb 100644 --- a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/MemoryIndexing.java +++ b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/MemoryIndexing.java @@ -1,7 +1,7 @@ package org.simantics.db.indexing; -import java.io.File; import java.io.IOException; +import java.nio.file.Path; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -20,9 +20,9 @@ import org.simantics.db.RequestProcessor; import org.simantics.db.Resource; import org.simantics.db.Session; import org.simantics.db.common.request.Adapt; -import org.simantics.db.common.utils.Logger; import org.simantics.db.indexing.IndexedRelationsSearcherBase.State; import org.simantics.db.layer0.adapter.GenericRelation; +import org.slf4j.LoggerFactory; /** * @author Tuukka Lehtonen @@ -30,6 +30,8 @@ import org.simantics.db.layer0.adapter.GenericRelation; */ public class MemoryIndexing { + private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(MemoryIndexing.class); + final private Session session; final Map>>> persistentCache = new HashMap>>>(); @@ -44,14 +46,14 @@ public class MemoryIndexing { this.session = session; } - protected File getIndexDirectory(Resource relation, Resource input) { + protected Path getIndexDirectory(Resource relation, Resource input) { return DatabaseIndexing.getIndexLocation(session, relation, input); } public IndexedRelationsSearcher get(RequestProcessor processor, Resource relation, Resource input) { + Path location = getIndexDirectory(relation, input); try { - File location = getIndexDirectory(relation, input); - String key = location.getAbsolutePath(); + String key = location.toAbsolutePath().toString(); IndexedRelationsSearcher searcher = searchers.get(key); if (searcher == null) { GenericRelation r = processor.sync(new Adapt(relation, GenericRelation.class)); @@ -60,15 +62,15 @@ public class MemoryIndexing { } return searcher; } catch (Exception e) { - Logger.defaultLogError(e); + LOGGER.error("Could not get searcher for relation {} and input {} in location {}", relation, input, location, e); return null; } } public IndexedRelationsSearcherBase getImmutable(RequestProcessor processor, Resource relation, Resource input) { + Path location = getIndexDirectory(relation, input); try { - File location = getIndexDirectory(relation, input); - String key = location.getAbsolutePath(); + String key = location.toAbsolutePath().toString(); IndexedRelationsSearcherBase searcher = immutableSearchers.get(key); if (searcher == null) { searcher = new ImmutableIndexedRelationsSearcher(processor, relation, input); @@ -76,7 +78,7 @@ public class MemoryIndexing { } return searcher; } catch (Exception e) { - Logger.defaultLogError(e); + LOGGER.error("Could not get searcher base for relation {} and input {} in location {}", relation, input, location, e); return null; } } diff --git a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/internal/IndexChangedWriter.java b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/internal/IndexChangedWriter.java index c0e97a2e4..1404e6022 100644 --- a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/internal/IndexChangedWriter.java +++ b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/internal/IndexChangedWriter.java @@ -11,8 +11,8 @@ *******************************************************************************/ package org.simantics.db.indexing.internal; -import java.io.File; import java.io.IOException; +import java.nio.file.Path; import java.util.HashSet; import java.util.Set; @@ -24,9 +24,9 @@ import java.util.Set; * @since 1.28.0 */ public class IndexChangedWriter { - private Set dirtyFiles = new HashSet<>(); + private Set dirtyFiles = new HashSet<>(); - public synchronized boolean markDirty(File dirtyFile) throws IOException { - return dirtyFiles.add(dirtyFile.getAbsolutePath()); + public synchronized boolean markDirty(Path dirtyFile) throws IOException { + return dirtyFiles.add(dirtyFile); } } diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/genericrelation/DependenciesRelation.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/genericrelation/DependenciesRelation.java index a3e19f1fc..333f3771c 100644 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/genericrelation/DependenciesRelation.java +++ b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/genericrelation/DependenciesRelation.java @@ -40,7 +40,6 @@ import org.simantics.db.common.request.ReadRequest; import org.simantics.db.common.request.SuperTypeString; import org.simantics.db.common.request.TypeString; import org.simantics.db.common.request.UnaryRead; -import org.simantics.db.common.utils.Logger; import org.simantics.db.common.utils.NameUtils; import org.simantics.db.event.ChangeListener; import org.simantics.db.exception.DatabaseException; @@ -62,9 +61,11 @@ import org.simantics.layer0.Layer0; import org.simantics.operation.Layer0X; import org.simantics.utils.datastructures.Pair; import org.simantics.utils.logging.TimeLogger; +import org.slf4j.LoggerFactory; public class DependenciesRelation extends UnsupportedRelation implements GenericRelationIndex { + private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(DependenciesRelation.class); private static final boolean DEBUG = false; static final boolean DEBUG_LISTENERS = false; private static final boolean PROFILE = false; @@ -112,7 +113,7 @@ public class DependenciesRelation extends UnsupportedRelation implements Generic @Override public void exception(AsyncReadGraph graph, Throwable throwable) { - Logger.defaultLogError(throwable); + LOGGER.error("Could not compile possible related value for resource {}", resource, throwable); } }); @@ -126,7 +127,7 @@ public class DependenciesRelation extends UnsupportedRelation implements Generic @Override public void exception(AsyncReadGraph graph, Throwable throwable) { - Logger.defaultLogError(throwable); + LOGGER.error("Could not find type for resource {}", resource, throwable); } }; @@ -150,7 +151,7 @@ public class DependenciesRelation extends UnsupportedRelation implements Generic @Override public void exception(AsyncReadGraph graph, Throwable throwable) { - Logger.defaultLogError(throwable); + LOGGER.error("Could not compile for resource {}", resource, throwable); } }); @@ -171,7 +172,7 @@ public class DependenciesRelation extends UnsupportedRelation implements Generic if(typeString == null) { typeString = graph.syncRequest(new SuperTypeString(e.principalType)); if (typeString.isEmpty()) { - Logger.defaultLogError(new DatabaseException("No name for type " + NameUtils.getURIOrSafeNameInternal(graph, e.resource) + " (" + e.resource + ")")); + LOGGER.error("No name for type", new DatabaseException("No name for type " + NameUtils.getURIOrSafeNameInternal(graph, e.resource) + " (" + e.resource + ")")); } typeStrings.put(e.principalType, typeString); } @@ -507,9 +508,8 @@ public class DependenciesRelation extends UnsupportedRelation implements Generic } catch (Throwable t) { // Just to know if something unexpected happens here. - Logger.defaultLogError("Dependencies index update failed for model " - + model + " and relation " + resource + ".", t); - t.printStackTrace(); + LOGGER.error("Dependencies index update failed for model " + + model + " and relation " + resource + ".", t); // NOTE: Last resort: failure to update index // properly results in removal of the whole index. diff --git a/bundles/org.simantics.utils/src/org/simantics/utils/FileUtils.java b/bundles/org.simantics.utils/src/org/simantics/utils/FileUtils.java index 798fd02b8..3e54e19be 100644 --- a/bundles/org.simantics.utils/src/org/simantics/utils/FileUtils.java +++ b/bundles/org.simantics.utils/src/org/simantics/utils/FileUtils.java @@ -33,12 +33,14 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.SimpleFileVisitor; import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.Arrays; import java.util.Deque; import java.util.LinkedList; import java.util.Random; +import java.util.function.Predicate; import java.util.zip.DataFormatException; import java.util.zip.Deflater; import java.util.zip.Inflater; @@ -959,21 +961,38 @@ public class FileUtils { return read; } - public static void delete(Path databaseLocation) throws IOException { - Files.walkFileTree(databaseLocation, new DeleteDirectoriesVisitor()); + public static void deleteWithFilter(Path path, Predicate filter) throws IOException { + if (Files.exists(path)) + Files.walkFileTree(path, new DeleteDirectoriesVisitor(filter)); } - + + public static void delete(Path path) throws IOException { + deleteWithFilter(path, null); + } + public static void copy(Path from, Path to) throws IOException { Files.walkFileTree(from, new CopyDirectoriesVisitor(from, to)); } public static class DeleteDirectoriesVisitor extends SimpleFileVisitor { - + + private Predicate filter; + + public DeleteDirectoriesVisitor(Predicate filter) { + this.filter = filter; + } + @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - Files.delete(file); - if (Files.exists(file)) + if (filter != null && !filter.test(file)) { + return FileVisitResult.CONTINUE; + } + if (Files.exists(file)) { + Files.delete(file); + } + if (Files.exists(file)) { throw new IOException("Could not delete file " + file.toAbsolutePath().toString()); + } return FileVisitResult.CONTINUE; } @@ -981,7 +1000,12 @@ public class FileUtils { public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { if (exc != null) throw exc; - Files.delete(dir); + if (filter != null && !filter.test(dir)) { + return FileVisitResult.CONTINUE; + } + if (Files.exists(dir)) { + Files.delete(dir); + } if (Files.exists(dir)) throw new IOException("Could not delete file " + dir.toAbsolutePath().toString()); return FileVisitResult.CONTINUE; @@ -1019,4 +1043,9 @@ public class FileUtils { raf.getFD().sync(); } } + + public static void sync(Path path) throws IOException { + try (InputStream stream = Files.newInputStream(path, StandardOpenOption.SYNC)) { + } + } }