X-Git-Url: https://gerrit.simantics.org/r/gitweb?a=blobdiff_plain;f=bundles%2Forg.simantics.db.indexing%2Fsrc%2Forg%2Fsimantics%2Fdb%2Findexing%2FIndexedRelationsSearcherBase.java;h=71db7f6168ac525d69cf1356329f1af081068792;hb=3826e289058a51d09310b7ba1251e959dc0ed3d0;hp=92cbe56d12bacfa76d30696811d90a01cfe4f00f;hpb=0ae2b770234dfc3cbb18bd38f324125cf0faca07;p=simantics%2Fplatform.git diff --git a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsSearcherBase.java b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsSearcherBase.java index 92cbe56d1..71db7f616 100644 --- a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsSearcherBase.java +++ b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsSearcherBase.java @@ -12,8 +12,9 @@ *******************************************************************************/ package org.simantics.db.indexing; -import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -61,7 +62,6 @@ import org.simantics.db.RequestProcessor; import org.simantics.db.Resource; import org.simantics.db.Session; import org.simantics.db.common.request.SafeName; -import org.simantics.db.common.utils.Logger; import org.simantics.db.common.utils.NameUtils; import org.simantics.db.exception.DatabaseException; import org.simantics.db.indexing.internal.IndexingJob; @@ -71,6 +71,7 @@ import org.simantics.db.service.CollectionSupport; import org.simantics.db.service.SerialisationSupport; import org.simantics.utils.FileUtils; import org.simantics.utils.datastructures.Pair; +import org.slf4j.Logger; import gnu.trove.map.hash.THashMap; @@ -101,6 +102,8 @@ abstract public class IndexedRelationsSearcherBase { } public void setProblem(Throwable t) { + if (t != null) + getLogger().error("Setting problem for {} and previous state {}", this, this.state, t); this.state = State.PROBLEM; this.exception = t; } @@ -129,7 +132,11 @@ abstract public class IndexedRelationsSearcherBase { protected void changeState(IProgressMonitor monitor, Session session, State state, int depth) { - if(this.state == state) return; + if (this.state == state) { + if (getLogger().isDebugEnabled()) + getLogger().debug("Trying to change state {} to the same as previous state {} in depth {} with {}", state, this.state, depth, this); + return; + } if (IndexPolicy.TRACE_INDEX_MANAGEMENT) System.err.println("Index state " + this.state.name() + " => " + state.name() + " " + this); @@ -138,20 +145,31 @@ abstract public class IndexedRelationsSearcherBase { // Try to exit problem state if (State.PROBLEM == this.state && depth > 0) { + getLogger().info("Try to exit problem state for {} and state {}", this, state); Throwable t = bestEffortClear(monitor, session); if(t != null) { + getLogger().error("Best effort clear has failed for state {} and this {}", state, this, t); exception = t; return; } // Managed to get into initial state this.state = State.NONE; + getLogger().info("Managed to get into initial state {}", this.state); return; } // Cannot move into read from no index - if (State.NONE == this.state && State.READ == state) return; + if (State.NONE == this.state && State.READ == state) { + if (getLogger().isDebugEnabled()) + getLogger().debug("Cannot move into read from no index in {} with state {}", this, state); + return; + } // Cannot move into write from no index - if (State.NONE == this.state && State.WRITE == state) return; + if (State.NONE == this.state && State.WRITE == state) { + if (getLogger().isDebugEnabled()) + getLogger().debug("Cannot move into write from no index in {} with state {}", this, state); + return; + } boolean success = false; @@ -223,9 +241,7 @@ abstract public class IndexedRelationsSearcherBase { } } catch (Throwable t) { - setProblem(t); - } finally { if(!success) { @@ -484,7 +500,7 @@ abstract public class IndexedRelationsSearcherBase { Resource input; - File indexPath; + Path indexPath; Directory directory; @@ -508,7 +524,7 @@ abstract public class IndexedRelationsSearcherBase { } Directory getDirectory(Session session) throws IOException { - return FSDirectory.open(indexPath); + return FSDirectory.open(indexPath.toFile()); } abstract String getDescriptor(); @@ -577,7 +593,7 @@ abstract public class IndexedRelationsSearcherBase { @Override public Thread newThread(Runnable r) { Thread t = new Thread(r, "Lucene Index Creator"); - if (t.isDaemon()) + if (!t.isDaemon()) t.setDaemon(true); if (t.getPriority() != Thread.NORM_PRIORITY) t.setPriority(Thread.NORM_PRIORITY); @@ -595,6 +611,7 @@ abstract public class IndexedRelationsSearcherBase { try { initializeIndexImpl(mon, graph, bound, overwrite); } catch (IOException e) { + getLogger().error("Index is in problematic state! {}", this, e); throw new DatabaseException(e); } }); @@ -610,8 +627,12 @@ abstract public class IndexedRelationsSearcherBase { mon.beginTask("Initializing Index", 100); if (overwrite) { - mon.subTask("Erasing previous index"); - FileUtils.deleteAll(indexPath); + if (Files.exists(indexPath)) { + mon.subTask("Erasing previous index"); + if (getLogger().isDebugEnabled()) + getLogger().debug("Erasing previous index {}", indexPath.toAbsolutePath()); + FileUtils.delete(indexPath); + } } final AtomicReference directory = new AtomicReference(); @@ -621,7 +642,7 @@ abstract public class IndexedRelationsSearcherBase { mon.subTask("Start index write"); createDirectory(indexPath); - directory.set(FSDirectory.open(indexPath)); + directory.set(FSDirectory.open(indexPath.toFile())); IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_4_9, Queries.getAnalyzer()).setOpenMode(OpenMode.CREATE); writer.set(new IndexWriter(directory.get(), conf)); @@ -666,8 +687,10 @@ abstract public class IndexedRelationsSearcherBase { try { writer.get().addDocument(document); } catch (CorruptIndexException e) { + getLogger().error("Index is corrupted! {}", this, e); throw new IllegalStateException(e); } catch (IOException e) { + getLogger().error("Index is in problematic state! {}", this, e); throw new IllegalStateException(e); } finally { synchronized (mon) { @@ -675,10 +698,10 @@ abstract public class IndexedRelationsSearcherBase { } } } - - s.release(); } catch (DatabaseException e) { throw new IllegalStateException(e); + } finally { + s.release(); } }); } @@ -686,7 +709,7 @@ abstract public class IndexedRelationsSearcherBase { try { s.acquire(INDEXING_THREAD_COUNT); } catch (InterruptedException e) { - Logger.defaultLogError(e); + getLogger().error("Could not initialize index {}", this, e); } // http://www.gossamer-threads.com/lists/lucene/java-dev/47895 @@ -701,14 +724,12 @@ abstract public class IndexedRelationsSearcherBase { System.out.println(getDescriptor() + "Wrote index at " + indexPath + " in " + (1e-9 * (System.nanoTime()-start)) + " seconds."); } catch (DatabaseException e) { - - Logger.defaultLogError(e); - + getLogger().error("Could not initialize index due to db {}", this, e); } finally { try { closeWriter(writer.getAndSet(null)); } finally { - directory.getAndSet(null).close(); + FileUtils.uncheckedClose(directory.getAndSet(null)); } } } @@ -736,11 +757,13 @@ abstract public class IndexedRelationsSearcherBase { } result.add(o); - } catch (CorruptIndexException e) { - throw new DatabaseException(e); - } catch (IOException e) { - throw new DatabaseException(e); - } + } catch (CorruptIndexException e) { + getLogger().error("Index is corrupted! {}", this, e); + throw new DatabaseException(e); + } catch (IOException e) { + getLogger().error("Index is in problematic state! {}", this, e); + throw new DatabaseException(e); + } } @@ -821,15 +844,14 @@ abstract public class IndexedRelationsSearcherBase { result.add(entry); } catch (CorruptIndexException e) { + getLogger().error("Index is corrupted! {}", this, e); throw new DatabaseException(e); } catch (IOException e) { + getLogger().error("Index is in problematic state! {}", this, e); throw new DatabaseException(e); } - } - return result; - } }); } @@ -923,22 +945,18 @@ abstract public class IndexedRelationsSearcherBase { ResourceVisitor visitor = new ResourceVisitor(); for (ScoreDoc scoreDoc : docs.scoreDocs) { - try { - reader.document(scoreDoc.doc, visitor); result.add(support.getResource(visitor.id)); - } catch (CorruptIndexException e) { + getLogger().error("Index is corrupted! {}", this, e); throw new DatabaseException(e); } catch (IOException e) { + getLogger().error("Index is in problematic state! {}", this, e); throw new DatabaseException(e); } - } - return result; - } }); } @@ -963,8 +981,10 @@ abstract public class IndexedRelationsSearcherBase { reader.document(scoreDoc.doc, visitor); } catch (CorruptIndexException e) { + getLogger().error("Index is corrupted! {}", this, e); throw new DatabaseException(e); } catch (IOException e) { + getLogger().error("Index is in problematic state! {}", this, e); throw new DatabaseException(e); } @@ -974,62 +994,44 @@ abstract public class IndexedRelationsSearcherBase { } - protected static File getIndexDirectory(Session session, Resource relation, Resource input) { - File path = DatabaseIndexing.getIndexLocation(session, relation, input); + protected static Path getIndexDirectory(Session session, Resource relation, Resource input) { + Path path = DatabaseIndexing.getIndexLocation(session, relation, input); // System.out.println("getIndexDirectory = " + path); return path; } - private static void createDirectory(File path) throws IOException { - if (path.exists() && !path.isDirectory()) - throw new IOException("Could not create index directory " + path + ", a file by that name already exists"); - path.mkdirs(); - if (!path.exists()) - throw new IOException("Could not create index directory " + path + " for an unknown reason"); - if (!path.isDirectory()) + private static void createDirectory(Path path) throws IOException { + if (Files.exists(path) && !Files.isDirectory(path)) throw new IOException("Could not create index directory " + path + ", a file by that name already exists"); + Files.createDirectories(path); } - File getIndexPath() { + Path getIndexPath() { return indexPath; } boolean isIndexAvailable() { - return (indexPath.exists() && indexPath.isDirectory()); + return (Files.exists(indexPath) && Files.isDirectory(indexPath)); } - Throwable bestEffortClear(IProgressMonitor monitor, Session session) { - return null; - } + abstract Throwable bestEffortClear(IProgressMonitor monitor, Session session); /* * Start from scratch. Clear all caches and rebuild the index. */ Throwable clearDirectory(IProgressMonitor monitor, Session session) { - File file = getIndexPath(); + Path file = getIndexPath(); try { - - for(int i=0;i<15;i++) { - FileUtils.deleteDir(file); - if(!file.exists()) { - return null; - } - try { - Thread.sleep(i*100); - } catch (InterruptedException e) { - } - } - + FileUtils.delete(file); } catch (Throwable t) { - + getLogger().error("Could not delete directory {}", file.toAbsolutePath(), t); return t; - } - - return new IllegalStateException("Failed to delete directory " + file.getAbsolutePath()); - + if (Files.exists(file)) + return new IllegalStateException("Failed to delete directory " + file.toAbsolutePath()); + return null; } private Field[] setFields(Field[] fs, Object[] result) { @@ -1044,11 +1046,17 @@ abstract public class IndexedRelationsSearcherBase { System.out.println(getDescriptor() + "index " + fs[i].name() + " = " + value + " : Long"); fs[i].setLongValue((Long) value); } else { - Logger.defaultLogError("Can only index Long and String fields, encountered " + value); + getLogger().error("Can only index Long and String fields, encountered " + value); return null; } } return fs; } + protected abstract Logger getLogger(); + + @Override + public String toString() { + return getClass().getSimpleName() + " [" + String.valueOf(schema) + ", " + String.valueOf(relation) + ", " + String.valueOf(input) + ", " + String.valueOf(indexPath) + ", " + String.valueOf(directory) + ", " + String.valueOf(state) + "]"; + } }