From: jsimomaa Date: Thu, 8 Mar 2018 11:56:30 +0000 (+0200) Subject: Improve startup time for fresh or rollback'd session in index writing X-Git-Tag: v1.43.0~136^2~553 X-Git-Url: https://gerrit.simantics.org/r/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F36%2F1536%2F3;p=simantics%2Fplatform.git Improve startup time for fresh or rollback'd session in index writing refs #7806 Change-Id: I44ea1e0972bfd360b6af44dbbe62e934e06c59e0 --- diff --git a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/Activator.java b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/Activator.java index b4276a70c..802618855 100644 --- a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/Activator.java +++ b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/Activator.java @@ -11,7 +11,6 @@ *******************************************************************************/ package org.simantics.db.indexing; -import java.io.File; import java.nio.file.Path; import org.eclipse.core.runtime.IPath; diff --git a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/DatabaseIndexing.java b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/DatabaseIndexing.java index 137d6c5c4..f8a19504c 100644 --- a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/DatabaseIndexing.java +++ b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/DatabaseIndexing.java @@ -40,7 +40,6 @@ import org.slf4j.LoggerFactory; */ public final class DatabaseIndexing { - private static final boolean DEBUG = IndexPolicy.TRACE_INDEX_MANAGEMENT; private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(DatabaseIndexing.class); public static Path getIndexBaseLocation() { diff --git a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsImpl.java b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsImpl.java index 1d027a825..390c0d17f 100644 --- a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsImpl.java +++ b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsImpl.java @@ -11,14 +11,18 @@ *******************************************************************************/ package org.simantics.db.indexing; -import java.io.File; import java.io.IOException; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.WeakHashMap; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ForkJoinPool; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -28,15 +32,19 @@ import org.eclipse.core.runtime.SubMonitor; import org.simantics.db.ReadGraph; import org.simantics.db.RequestProcessor; import org.simantics.db.Resource; +import org.simantics.db.common.request.ReadRequest; import org.simantics.db.common.request.UniqueRead; +import org.simantics.db.common.utils.NameUtils; import org.simantics.db.exception.DatabaseException; import org.simantics.db.exception.InvalidResourceReferenceException; import org.simantics.db.indexing.IndexedRelationsSearcherBase.State; import org.simantics.db.layer0.adapter.GenericRelation; import org.simantics.db.layer0.genericrelation.IndexException; import org.simantics.db.layer0.genericrelation.IndexedRelations; +import org.simantics.db.layer0.util.Layer0Utils; import org.simantics.db.service.QueryControl; import org.simantics.db.service.SerialisationSupport; +import org.simantics.operation.Layer0X; import org.simantics.utils.datastructures.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -130,7 +138,7 @@ public class IndexedRelationsImpl implements IndexedRelations { return rwlock.tryLock(processor, write); } - private IndexedRelationsSearcherBase makeSearcher(final RequestProcessor processor, final Resource relation, final Resource input) { + private static IndexedRelationsSearcherBase makeSearcher(final RequestProcessor processor, final Resource relation, final Resource input) { try { return processor.syncRequest(new UniqueRead() { @@ -542,4 +550,79 @@ public class IndexedRelationsImpl implements IndexedRelations { } + @Override + public void fullRebuild(IProgressMonitor monitor, RequestProcessor processor) throws IndexException { + try { + processor.syncRequest(new ReadRequest() { + @Override + public void run(ReadGraph graph) throws DatabaseException { + try { + fullRebuild(monitor, graph); + } catch (IOException e) { + throw new DatabaseException(e); + } + } + }); + } catch (DatabaseException e) { + throw new IndexException(e); + } + } + + private void fullRebuild(IProgressMonitor monitor, ReadGraph graph) throws DatabaseException, IOException { + long startTime = System.currentTimeMillis(); + Resource relation = Layer0X.getInstance(graph).DependenciesRelation; + SerialisationSupport ss = graph.getService(SerialisationSupport.class); + Set indexRoots = Layer0Utils.listIndexRoots(graph); + List> waitFor = new ArrayList<>(indexRoots.size()); + SubMonitor mon = SubMonitor.convert(monitor, indexRoots.size()*2); + + for (Resource indexRoot : indexRoots) { + monitor.subTask(NameUtils.getSafeName(graph, indexRoot)); + + IndexedRelationsSearcherBase searcher = makeSearcher(graph, relation, indexRoot); + + GenericRelation r = graph.adapt(relation, GenericRelation.class); + if (r == null) + throw new DatabaseException("Given resource " + relation + "could not be adapted to GenericRelation."); + + Object[] bound = new Object[] { ss.getRandomAccessId(indexRoot) }; + GenericRelation selection = r.select(IndexedRelationsSearcherBase.getPattern(r, bound.length), bound); + + long relStart = System.currentTimeMillis(); + List results = selection.realize(graph); + if (LOGGER.isDebugEnabled()) { + long relEnd = System.currentTimeMillis() - relStart; + LOGGER.debug(indexRoot + " realized " + relEnd); + } + mon.worked(1); + + CompletableFuture result = new CompletableFuture<>(); + waitFor.add(result); + ForkJoinPool.commonPool().submit(() -> { + long startTime1 = System.currentTimeMillis(); + try { + searcher.initializeIndexImpl(result, mon.newChild(1, SubMonitor.SUPPRESS_ALL_LABELS), r, results, bound, true); + searcher.setReady(); + } catch (IOException e) { + result.completeExceptionally(e); + LOGGER.error("Could not initialize index", e); + } finally { + if (LOGGER.isDebugEnabled()) + LOGGER.debug(indexRoot + " initialized " + (System.currentTimeMillis() - startTime1)); + } + }); + } + for (CompletableFuture fut : waitFor) { + try { + fut.get(); + } catch (InterruptedException | ExecutionException e) { + throw (IOException) e.getCause(); + } + } + if (LOGGER.isInfoEnabled()) { + long endTime = System.currentTimeMillis() - startTime; + LOGGER.info("All indexes rebuilt in {}", endTime); + } + } + } diff --git a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsMemorySearcher.java b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsMemorySearcher.java index 8fc0ce911..d1282c766 100644 --- a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsMemorySearcher.java +++ b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsMemorySearcher.java @@ -11,9 +11,6 @@ *******************************************************************************/ package org.simantics.db.indexing; -import gnu.trove.map.hash.THashMap; -import gnu.trove.set.hash.TLongHashSet; - import java.io.IOException; import java.util.ArrayList; import java.util.Collection; @@ -23,7 +20,6 @@ import java.util.Map; import org.apache.lucene.document.Document; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexableField; -import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; @@ -39,6 +35,9 @@ import org.simantics.utils.datastructures.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import gnu.trove.map.hash.THashMap; +import gnu.trove.set.hash.TLongHashSet; + /** * @author Tuukka Lehtonen * @author Antti Villberg @@ -94,8 +93,7 @@ public class IndexedRelationsMemorySearcher extends IndexedRelationsSearcherBase super.removeIndex(monitor, r, processor, key, keyValues); } - public List allDocs(IProgressMonitor monitor, Session session) throws ParseException, IOException, - DatabaseException { + public List allDocs(IProgressMonitor monitor, Session session) throws IOException { Query query = new MatchAllDocsQuery(); @@ -112,28 +110,19 @@ public class IndexedRelationsMemorySearcher extends IndexedRelationsSearcherBase } for(ScoreDoc scoreDoc:scoreDocs) { - - try { - - Document doc = reader.document(scoreDoc.doc); - List fs = doc.getFields(); - Object[] o = new Object[fs.size()]; - int index = 0; - for (IndexableField f : fs) { - String clazz = classMap.get(f.name()); - if ("Long".equals(clazz)) { - o[index++] = Long.parseLong(f.stringValue()); - } else { - o[index++] = f.stringValue(); - } + Document doc = reader.document(scoreDoc.doc); + List fs = doc.getFields(); + Object[] o = new Object[fs.size()]; + int index = 0; + for (IndexableField f : fs) { + String clazz = classMap.get(f.name()); + if ("Long".equals(clazz)) { + o[index++] = Long.parseLong(f.stringValue()); + } else { + o[index++] = f.stringValue(); } - result.add(o); - } catch (CorruptIndexException e) { - throw new DatabaseException(e); - } catch (IOException e) { - throw new DatabaseException(e); } - + result.add(o); } changeState(monitor, session, State.READY); diff --git a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsSearcherBase.java b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsSearcherBase.java index 71db7f616..136a7bcf6 100644 --- a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsSearcherBase.java +++ b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsSearcherBase.java @@ -21,10 +21,8 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.Semaphore; -import java.util.concurrent.ThreadFactory; import java.util.concurrent.atomic.AtomicReference; import org.apache.lucene.document.Document; @@ -71,6 +69,7 @@ import org.simantics.db.service.CollectionSupport; import org.simantics.db.service.SerialisationSupport; import org.simantics.utils.FileUtils; import org.simantics.utils.datastructures.Pair; +import org.simantics.utils.threads.ThreadUtils; import org.slf4j.Logger; import gnu.trove.map.hash.THashMap; @@ -578,7 +577,7 @@ abstract public class IndexedRelationsSearcherBase { } } - private static String getPattern(GenericRelation relation, int boundCount) { + public static String getPattern(GenericRelation relation, int boundCount) { String result = ""; for (int i = 0; i < boundCount; i++) result += "b"; @@ -586,20 +585,6 @@ abstract public class IndexedRelationsSearcherBase { result += "f"; return result; } - - private static final int INDEXING_THREAD_COUNT = 2; - - private static final ExecutorService executor = Executors.newFixedThreadPool(INDEXING_THREAD_COUNT, new ThreadFactory() { - @Override - public Thread newThread(Runnable r) { - Thread t = new Thread(r, "Lucene Index Creator"); - if (!t.isDaemon()) - t.setDaemon(true); - if (t.getPriority() != Thread.NORM_PRIORITY) - t.setPriority(Thread.NORM_PRIORITY); - return t; - } - }); void initializeIndex(IProgressMonitor monitor, ReadGraph graph, Object[] bound, boolean overwrite) throws IOException, DatabaseException @@ -609,7 +594,14 @@ abstract public class IndexedRelationsSearcherBase { "Reindexing " + NameUtils.getSafeLabel(graph, input), mon -> { try { - initializeIndexImpl(mon, graph, bound, overwrite); + GenericRelation r = graph.adapt(relation, GenericRelation.class); + if (r == null) + throw new DatabaseException("Given resource " + relation + "could not be adapted to GenericRelation."); + + GenericRelation selection = r.select(getPattern(r, bound.length), bound); + + List results = selection.realize(graph); + initializeIndexImpl(new CompletableFuture<>(), mon, r, results, bound, overwrite); } catch (IOException e) { getLogger().error("Index is in problematic state! {}", this, e); throw new DatabaseException(e); @@ -617,120 +609,120 @@ abstract public class IndexedRelationsSearcherBase { }); } - void initializeIndexImpl(IProgressMonitor monitor, ReadGraph graph, final Object[] bound, boolean overwrite) throws IOException, - DatabaseException { - - final SubMonitor mon = SubMonitor.convert(monitor, 100); - - if (IndexPolicy.TRACE_INDEX_INIT) - System.out.println(getDescriptor() + "Initializing index at " + indexPath + " (overwrite = " + overwrite + ")"); - mon.beginTask("Initializing Index", 100); - - if (overwrite) { - if (Files.exists(indexPath)) { - mon.subTask("Erasing previous index"); - if (getLogger().isDebugEnabled()) - getLogger().debug("Erasing previous index {}", indexPath.toAbsolutePath()); - FileUtils.delete(indexPath); - } - } - - final AtomicReference directory = new AtomicReference(); - final AtomicReference writer = new AtomicReference(); + private static final int INDEXING_THREAD_COUNT = 2; // this is quite good parallelism level for lucene + void initializeIndexImpl(CompletableFuture result, IProgressMonitor monitor, GenericRelation r, List results, final Object[] bound, boolean overwrite) throws IOException { try { - mon.subTask("Start index write"); - createDirectory(indexPath); - - directory.set(FSDirectory.open(indexPath.toFile())); - IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_4_9, Queries.getAnalyzer()).setOpenMode(OpenMode.CREATE); - writer.set(new IndexWriter(directory.get(), conf)); - - mon.worked(5); - - final GenericRelation r = graph.adapt(relation, GenericRelation.class); - if (r == null) - throw new DatabaseException("Given resource " + graph.syncRequest(new SafeName(relation)) - + "could not be adapted to GenericRelation."); - - long realizeStart = 0; - if (IndexPolicy.PERF_INDEX_INIT) - realizeStart = System.nanoTime(); - - mon.subTask("Calculating indexed content"); - GenericRelation selection = r.select(getPattern(r, bound.length), bound); - mon.worked(5); - List results = selection.realize(graph); - mon.worked(40); - - if (IndexPolicy.PERF_INDEX_INIT) - System.out.println(getDescriptor() + "Realized index with " + results.size() + " entries at " + indexPath + " in " + (1e-9 * (System.nanoTime()-realizeStart)) + " seconds."); + final SubMonitor mon = SubMonitor.convert(monitor, 100); + if (IndexPolicy.TRACE_INDEX_INIT) - System.out.println(getDescriptor() + "Indexed relation " + r + " produced " + results.size() + " results"); - - long start = IndexPolicy.PERF_INDEX_INIT ? System.nanoTime() : 0; - - mon.subTask("Indexing content"); - final Semaphore s = new Semaphore(0); - mon.setWorkRemaining(results.size()); - - for (int i = 0; i < INDEXING_THREAD_COUNT; i++) { - final int startIndex = i; - executor.submit(() -> { - try { - Document document = new Document(); - Field[] fs = makeFieldsForRelation(r, bound.length, document); - - for (int index = startIndex; index < results.size(); index += INDEXING_THREAD_COUNT) { - if (setFields(fs, results.get(index)) == null) - continue; - try { - writer.get().addDocument(document); - } catch (CorruptIndexException e) { - getLogger().error("Index is corrupted! {}", this, e); - throw new IllegalStateException(e); - } catch (IOException e) { - getLogger().error("Index is in problematic state! {}", this, e); - throw new IllegalStateException(e); - } finally { - synchronized (mon) { - mon.worked(1); + System.out.println(getDescriptor() + "Initializing index at " + indexPath + " (overwrite = " + overwrite + ")"); + mon.beginTask("Initializing Index", 100); + + if (overwrite) { + if (Files.exists(indexPath)) { + mon.subTask("Erasing previous index"); + if (getLogger().isDebugEnabled()) + getLogger().debug("Erasing previous index {}", indexPath.toAbsolutePath()); + FileUtils.delete(indexPath); + } + } + + final AtomicReference directory = new AtomicReference(); + final AtomicReference writer = new AtomicReference(); + + try { + mon.subTask("Start index write"); + createDirectory(indexPath); + + directory.set(FSDirectory.open(indexPath.toFile())); + IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_4_9, Queries.getAnalyzer()).setOpenMode(OpenMode.CREATE); + writer.set(new IndexWriter(directory.get(), conf)); + + mon.worked(5); + + long realizeStart = 0; + if (IndexPolicy.PERF_INDEX_INIT) + realizeStart = System.nanoTime(); + + mon.subTask("Calculating indexed content"); + mon.worked(5); + + mon.worked(40); + + if (IndexPolicy.PERF_INDEX_INIT) + System.out.println(getDescriptor() + "Realized index with " + results.size() + " entries at " + indexPath + " in " + (1e-9 * (System.nanoTime()-realizeStart)) + " seconds."); + + if (IndexPolicy.TRACE_INDEX_INIT) + System.out.println(getDescriptor() + "Indexed relation " + r + " produced " + results.size() + " results"); + + long start = IndexPolicy.PERF_INDEX_INIT ? System.nanoTime() : 0; + + mon.subTask("Indexing content"); + final Semaphore s = new Semaphore(0); + mon.setWorkRemaining(results.size()); + for (int i = 0; i < INDEXING_THREAD_COUNT; i++) { + final int startIndex = i; + ThreadUtils.getBlockingWorkExecutor().submit(() -> { + try { + Document document = new Document(); + Field[] fs = makeFieldsForRelation(r, bound.length, document); + + for (int index = startIndex; index < results.size(); index += INDEXING_THREAD_COUNT) { + if (setFields(fs, results.get(index)) == null) + continue; + try { + writer.get().addDocument(document); + } catch (CorruptIndexException e) { + getLogger().error("Index is corrupted! {}", this, e); + throw new IllegalStateException(e); + } catch (IOException e) { + getLogger().error("Index is in problematic state! {}", this, e); + throw new IllegalStateException(e); + } finally { + synchronized (mon) { + mon.worked(1); + } } } + } catch (DatabaseException e) { + throw new IllegalStateException(e); + } finally { + s.release(); } - } catch (DatabaseException e) { - throw new IllegalStateException(e); - } finally { - s.release(); - } - }); - } - - try { - s.acquire(INDEXING_THREAD_COUNT); - } catch (InterruptedException e) { - getLogger().error("Could not initialize index {}", this, e); - } - - // http://www.gossamer-threads.com/lists/lucene/java-dev/47895 - // and http://lucene.apache.org/java/docs/index.html#27+November+2011+-+Lucene+Core+3.5.0 - // advise against calling optimize at all. So let's not do it anymore. - //writer.get().optimize(); - //writer.get().commit(); - - mon.subTask("Flushing"); - - if (IndexPolicy.PERF_INDEX_INIT) - System.out.println(getDescriptor() + "Wrote index at " + indexPath + " in " + (1e-9 * (System.nanoTime()-start)) + " seconds."); - - } catch (DatabaseException e) { - getLogger().error("Could not initialize index due to db {}", this, e); - } finally { - try { - closeWriter(writer.getAndSet(null)); + }); + } + + try { + s.acquire(INDEXING_THREAD_COUNT); + } catch (InterruptedException e) { + getLogger().error("Could not initialize index {}", this, e); + } + + // http://www.gossamer-threads.com/lists/lucene/java-dev/47895 + // and http://lucene.apache.org/java/docs/index.html#27+November+2011+-+Lucene+Core+3.5.0 + // advise against calling optimize at all. So let's not do it anymore. + //writer.get().optimize(); + //writer.get().commit(); + + mon.subTask("Flushing"); + + if (IndexPolicy.PERF_INDEX_INIT) + System.out.println(getDescriptor() + "Wrote index at " + indexPath + " in " + (1e-9 * (System.nanoTime()-start)) + " seconds."); + + result.complete(null); + // } catch (DatabaseException e) { + // getLogger().error("Could not initialize index due to db {}", this, e); } finally { - FileUtils.uncheckedClose(directory.getAndSet(null)); + try { + closeWriter(writer.getAndSet(null)); + } finally { + FileUtils.uncheckedClose(directory.getAndSet(null)); + } } + } catch (Throwable t) { + getLogger().error("Could not initialize index", t); + result.completeExceptionally(t); } } diff --git a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/MemoryIndexing.java b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/MemoryIndexing.java index 75c6b79bb..cee5eb26b 100644 --- a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/MemoryIndexing.java +++ b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/MemoryIndexing.java @@ -2,10 +2,12 @@ package org.simantics.db.indexing; import java.io.IOException; import java.nio.file.Path; -import java.util.HashMap; +import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Semaphore; +import java.util.stream.Stream; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.IndexWriter; @@ -32,118 +34,110 @@ public class MemoryIndexing { private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(MemoryIndexing.class); - final private Session session; - - final Map>>> persistentCache = new HashMap>>>(); - final Map>> persistentCacheResources = new HashMap>>(); - - final private Map directories = new HashMap(); - - final private Map immutableSearchers = new HashMap(); - final private Map searchers = new HashMap(); + final private Session session; + + final Map>>> persistentCache = new ConcurrentHashMap<>(); + final Map>> persistentCacheResources = new ConcurrentHashMap<>(); + + final private ConcurrentHashMap directories = new ConcurrentHashMap<>(); + + final private ConcurrentHashMap immutableSearchers = new ConcurrentHashMap<>(); + final private ConcurrentHashMap searchers = new ConcurrentHashMap<>(); public MemoryIndexing(Session session) { this.session = session; } - + protected Path getIndexDirectory(Resource relation, Resource input) { return DatabaseIndexing.getIndexLocation(session, relation, input); } - + public IndexedRelationsSearcher get(RequestProcessor processor, Resource relation, Resource input) { Path location = getIndexDirectory(relation, input); - try { - String key = location.toAbsolutePath().toString(); - IndexedRelationsSearcher searcher = searchers.get(key); - if (searcher == null) { + String key = location.toAbsolutePath().toString(); + return searchers.computeIfAbsent(key, t -> { + try { GenericRelation r = processor.sync(new Adapt(relation, GenericRelation.class)); - searcher = new IndexedRelationsSearcher(processor, relation, input, r); - searchers.put(key, searcher); + return new IndexedRelationsSearcher(processor, relation, input, r); + } catch (Exception e) { + LOGGER.error("Could not get searcher for relation {} and input {} in location {}", relation, input, location, e); + return null; } - return searcher; - } catch (Exception e) { - LOGGER.error("Could not get searcher for relation {} and input {} in location {}", relation, input, location, e); - return null; - } + }); } public IndexedRelationsSearcherBase getImmutable(RequestProcessor processor, Resource relation, Resource input) { Path location = getIndexDirectory(relation, input); - try { - String key = location.toAbsolutePath().toString(); - IndexedRelationsSearcherBase searcher = immutableSearchers.get(key); - if (searcher == null) { - searcher = new ImmutableIndexedRelationsSearcher(processor, relation, input); - immutableSearchers.put(key, searcher); + String key = location.toAbsolutePath().toString(); + return immutableSearchers.computeIfAbsent(key, t -> { + try { + return new ImmutableIndexedRelationsSearcher(processor, relation, input); + } catch (Exception e) { + LOGGER.error("Could not get searcher base for relation {} and input {} in location {}", relation, input, location, e); + return null; } - return searcher; - } catch (Exception e) { - LOGGER.error("Could not get searcher base for relation {} and input {} in location {}", relation, input, location, e); - return null; - } + }); } public static MemoryIndexing getInstance(Session session) { - MemoryIndexing ret = session.peekService(MemoryIndexing.class); + MemoryIndexing ret = session.peekService(MemoryIndexing.class); if(ret == null) { - ret = new MemoryIndexing(session); + ret = new MemoryIndexing(session); session.registerService(MemoryIndexing.class, ret); } return ret; } - public synchronized Directory getDirectory(String path, Analyzer analyzer) throws IOException { - RAMDirectory directory = directories.get(path); - if (directory == null) { - synchronized (directories) { - directory = directories.get(path); - if (directory == null) { - directory = new RAMDirectory(); + public Directory getDirectory(String path, Analyzer analyzer) throws IOException { + try { + return directories.computeIfAbsent(path, t -> { + try { + RAMDirectory directory = new RAMDirectory(); IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_4_9, analyzer); new IndexWriter(directory, config.setOpenMode(OpenMode.CREATE)).close(); - directories.put(path, directory); + return directory; + } catch (IOException e) { + throw new RuntimeException(e); } - } + }); + } catch (RuntimeException e) { + throw (IOException) e.getCause(); } - return directory; - } - + public void remove(String path) { directories.remove(path); } - + public void flush(IProgressMonitor progress) throws Exception { - + long startTime = System.currentTimeMillis(); SubMonitor monitor = SubMonitor.convert(progress); - - Set> set = searchers.entrySet(); - Set> iset = immutableSearchers.entrySet(); - - monitor.setWorkRemaining(set.size()+iset.size()); - - for(Map.Entry entry : set) { - - IndexedRelationsSearcher persistent = entry.getValue(); - IndexedRelationsMemorySearcher searcher = persistent.cache; - - if(persistent.isIndexAvailable()) { - List os = searcher.allDocs(monitor, session); - persistent.applyChanges(monitor, session, searcher.r, os); - } - - monitor.worked(1); - entry.getValue().changeState(monitor, session, State.READY); - - } - - for(Map.Entry entry : iset) { - - entry.getValue().changeState(monitor, session, State.READY); - monitor.worked(1); - - } - + Collection searcherEntries = searchers.values(); + Collection immutableSearcherEntries = immutableSearchers.values(); + int count = searcherEntries.size() + immutableSearcherEntries.size(); + Semaphore sema = new Semaphore(0); + Stream.concat(searcherEntries.stream(), immutableSearcherEntries.stream()).parallel().forEach(base -> { + try { + if (base.isIndexAvailable()) { + if (base instanceof IndexedRelationsSearcher) { + IndexedRelationsMemorySearcher searcher = ((IndexedRelationsSearcher) base).cache; + try { + List os = searcher.allDocs(monitor, session); + ((IndexedRelationsSearcher) base).applyChanges(monitor, session, searcher.r, os); + } catch (Exception e) { + LOGGER.error("Could not flush", e); + } + } + } + monitor.worked(1); + base.changeState(monitor, session, State.READY); + } finally { + sema.release(); + } + }); + sema.acquire(count); + long totalTime = System.currentTimeMillis() - startTime; + LOGGER.info("index flush " + totalTime); } } diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/genericrelation/IndexedRelations.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/genericrelation/IndexedRelations.java index a52edc4b9..c334691d3 100644 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/genericrelation/IndexedRelations.java +++ b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/genericrelation/IndexedRelations.java @@ -96,4 +96,6 @@ public interface IndexedRelations { public void reset(IProgressMonitor monitor, RequestProcessor processor, Resource resource, Resource input) throws IndexException; + public void fullRebuild(IProgressMonitor monitor, RequestProcessor processor) throws IndexException; + } diff --git a/bundles/org.simantics.project/src/org/simantics/project/SessionDescriptor.java b/bundles/org.simantics.project/src/org/simantics/project/SessionDescriptor.java new file mode 100644 index 000000000..3043479d1 --- /dev/null +++ b/bundles/org.simantics.project/src/org/simantics/project/SessionDescriptor.java @@ -0,0 +1,27 @@ +package org.simantics.project; + +import org.simantics.db.Session; + +/** + * @author Jani Simomaa + * @since 1.34.0 + */ +public class SessionDescriptor { + + private final Session session; + private final boolean fresh; + + public SessionDescriptor(Session session, boolean fresh) { + this.session = session; + this.fresh = fresh; + } + + public Session getSession() { + return session; + } + + public boolean isFreshDatabase() { + return fresh; + } + +} \ No newline at end of file diff --git a/bundles/org.simantics.project/src/org/simantics/project/management/PlatformUtil.java b/bundles/org.simantics.project/src/org/simantics/project/management/PlatformUtil.java index 7655f9ca5..ebd987e3f 100644 --- a/bundles/org.simantics.project/src/org/simantics/project/management/PlatformUtil.java +++ b/bundles/org.simantics.project/src/org/simantics/project/management/PlatformUtil.java @@ -25,11 +25,16 @@ import java.io.UnsupportedEncodingException; import java.net.URL; import java.net.URLDecoder; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Enumeration; import java.util.Map.Entry; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import java.util.jar.Attributes; import java.util.jar.Manifest; +import java.util.stream.Collectors; import org.eclipse.core.internal.runtime.PlatformActivator; import org.eclipse.core.runtime.FileLocator; @@ -38,12 +43,10 @@ import org.eclipse.equinox.p2.metadata.IVersionedId; import org.eclipse.equinox.p2.metadata.Version; import org.eclipse.equinox.p2.metadata.VersionedId; import org.osgi.framework.Bundle; -import org.simantics.databoard.adapter.AdaptException; import org.simantics.databoard.binding.Binding; import org.simantics.databoard.binding.mutable.Variant; import org.simantics.databoard.container.DataContainer; import org.simantics.databoard.container.DataContainers; -import org.simantics.databoard.serialization.SerializationException; import org.simantics.graph.compiler.CompilationResult; import org.simantics.graph.compiler.GraphCompiler; import org.simantics.graph.compiler.GraphCompilerPreferences; @@ -310,12 +313,30 @@ public class PlatformUtil { * @param collection * @throws IOException */ - public static void getAllGraphs(Collection collection) throws IOException { - for (Bundle bundle : getBundles()) { - GraphBundle entry = getGraph(bundle); - if (entry!=null) collection.add(entry); - } - } + public static Collection getAllGraphs() throws IOException { + CompletableFuture f = new CompletableFuture<>(); + Bundle[] bundles = getBundles(); + Collection gbundles = Arrays.stream(bundles).map(t -> { // this could be done in parallel in the future? + if (f.isCompletedExceptionally()) + return null; + try { + return PlatformUtil.getGraph(t); + } catch (IOException e) { + if (LOGGER.isDebugEnabled()) + LOGGER.debug("Could not get graph {}", t, e); + f.completeExceptionally(e); + return null; + } + }).filter(Objects::nonNull).collect(Collectors.toList()); + if (f.isCompletedExceptionally()) { + try { + f.get(); + } catch (ExecutionException | InterruptedException e) { + throw (IOException) e.getCause(); + } + } + return gbundles; + } /** * Get bundle diff --git a/bundles/org.simantics.project/src/org/simantics/project/management/ServerManager.java b/bundles/org.simantics.project/src/org/simantics/project/management/ServerManager.java index 936ded8b3..6570da33a 100644 --- a/bundles/org.simantics.project/src/org/simantics/project/management/ServerManager.java +++ b/bundles/org.simantics.project/src/org/simantics/project/management/ServerManager.java @@ -39,6 +39,7 @@ import org.simantics.db.service.XSupport; import org.simantics.graph.db.CoreInitialization; import org.simantics.layer0.DatabaseManagementResource; import org.simantics.layer0.Layer0; +import org.simantics.project.SessionDescriptor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -81,7 +82,7 @@ public class ServerManager { * @param initialGraphs initialGraphs to install * @throws DatabaseException */ - public Session createDatabase(File databaseDirectory) throws DatabaseException { + public SessionDescriptor createDatabase(File databaseDirectory) throws DatabaseException { try { LOGGER.debug("Creating database to "+ databaseDirectory); @@ -110,7 +111,7 @@ public class ServerManager { } }; session.syncRequest(req); - return session; + return new SessionDescriptor(session, false); } CoreInitialization.initializeBuiltins(session); // This will try to initialize Builtins.class but because there is no statements @@ -142,7 +143,7 @@ public class ServerManager { mgt.createGraphBundle(graph, l0ex); graph.flushCluster(); }}); - return session; + return new SessionDescriptor(session, true); } finally { if (null == session) server1.stop(); diff --git a/bundles/org.simantics.workbench/src/org/simantics/workbench/internal/SimanticsWorkbenchAdvisor.java b/bundles/org.simantics.workbench/src/org/simantics/workbench/internal/SimanticsWorkbenchAdvisor.java index 9169c64b3..591c7fd42 100644 --- a/bundles/org.simantics.workbench/src/org/simantics/workbench/internal/SimanticsWorkbenchAdvisor.java +++ b/bundles/org.simantics.workbench/src/org/simantics/workbench/internal/SimanticsWorkbenchAdvisor.java @@ -91,6 +91,7 @@ import org.simantics.Simantics; import org.simantics.SimanticsPlatform; import org.simantics.SimanticsPlatform.OntologyRecoveryPolicy; import org.simantics.SimanticsPlatform.RecoveryPolicy; +import org.simantics.TimingProgressMonitor; import org.simantics.application.arguments.IArguments; import org.simantics.application.arguments.SimanticsArguments; import org.simantics.db.common.Indexing; diff --git a/bundles/org.simantics/src/org/simantics/SimanticsPlatform.java b/bundles/org.simantics/src/org/simantics/SimanticsPlatform.java index 211b286a9..e529938e2 100644 --- a/bundles/org.simantics/src/org/simantics/SimanticsPlatform.java +++ b/bundles/org.simantics/src/org/simantics/SimanticsPlatform.java @@ -72,6 +72,8 @@ import org.simantics.db.exception.DatabaseException; import org.simantics.db.exception.ResourceNotFoundException; import org.simantics.db.indexing.DatabaseIndexing; import org.simantics.db.layer0.genericrelation.DependenciesRelation; +import org.simantics.db.layer0.genericrelation.IndexException; +import org.simantics.db.layer0.genericrelation.IndexedRelations; import org.simantics.db.layer0.util.SimanticsClipboardImpl; import org.simantics.db.layer0.util.SimanticsKeys; import org.simantics.db.layer0.util.TGTransferableGraphSource; @@ -101,6 +103,7 @@ import org.simantics.project.IProject; import org.simantics.project.ProjectFeatures; import org.simantics.project.ProjectKeys; import org.simantics.project.Projects; +import org.simantics.project.SessionDescriptor; import org.simantics.project.exception.ProjectException; import org.simantics.project.features.registry.GroupReference; import org.simantics.project.management.DatabaseManagement; @@ -229,7 +232,7 @@ public class SimanticsPlatform implements LifecycleListener { return application != null ? application : UUID.randomUUID().toString(); } - private Session setupDatabase(String databaseDriverId, IProgressMonitor progressMonitor, RecoveryPolicy workspacePolicy, PlatformUserAgent userAgent) throws PlatformException { + private SessionDescriptor setupDatabase(String databaseDriverId, IProgressMonitor progressMonitor, RecoveryPolicy workspacePolicy, PlatformUserAgent userAgent) throws PlatformException { if (progressMonitor == null) progressMonitor = new NullProgressMonitor(); Path workspaceLocation = Platform.getLocation().toFile().toPath(); @@ -274,12 +277,16 @@ public class SimanticsPlatform implements LifecycleListener { PlatformUtil.compileAllDynamicOntologies(); - progressMonitor.setTaskName("Asserting all ontologies are installed"); + String message = "Asserting all ontologies are installed"; + LOGGER.info(message); + progressMonitor.setTaskName(message); final Map platformTGs = new HashMap(); try { // Get a list of bundles installed into the database - progressMonitor.subTask("find installed bundles from database"); + message = "find installed bundles from database"; + progressMonitor.subTask(message); + LOGGER.info(message); Map installedTGs = new HashMap(); for (GraphBundle b : session.syncRequest( mgmt.GraphBundleQuery )) { installedTGs.put(GraphBundleRef.of(b), GraphBundleEx.extend(b)); @@ -289,10 +296,13 @@ public class SimanticsPlatform implements LifecycleListener { // if(installedTGs.size() > 1) return; // Get a list of all bundles in the platform (Bundle Context) - List tgs = new ArrayList(); - progressMonitor.subTask("load all transferable graphs from platform"); - PlatformUtil.getAllGraphs(tgs); - progressMonitor.subTask("extend bundles to compile versions"); + message = "load all transferable graphs from platform"; + progressMonitor.subTask(message); + LOGGER.info(message); + Collection tgs = PlatformUtil.getAllGraphs(); + message = "extend bundles to compile versions"; + progressMonitor.subTask(message); + LOGGER.info(message); for (GraphBundle b : tgs) { GraphBundleEx gbe = GraphBundleEx.extend(b); gbe.build(); @@ -300,7 +310,9 @@ public class SimanticsPlatform implements LifecycleListener { } // Compile a list of TGs that need to be installed or reinstalled in the database - progressMonitor.subTask("check bundle reinstallation demand"); + message = "check bundle reinstallation demand"; + progressMonitor.subTask(message); + LOGGER.info(message); List installTGs = new ArrayList(); // Create list of TGs to update, Map reinstallTGs = new TreeMap(); @@ -373,7 +385,9 @@ public class SimanticsPlatform implements LifecycleListener { } if (ontologyPolicy == OntologyRecoveryPolicy.Merge) { - progressMonitor.subTask("Merging ontology changes"); + message = "Merging ontology changes"; + progressMonitor.subTask(message); + LOGGER.info(message); // Sort missing TGs into install order GraphDependencyAnalyzer analyzer = new GraphDependencyAnalyzer(); for(GraphBundle tg : installTGs) analyzer.addGraph(tg, tg.getGraph()); @@ -526,11 +540,11 @@ public class SimanticsPlatform implements LifecycleListener { public boolean assertProject(IProgressMonitor progressMonitor, RecoveryPolicy workspacePolicy, boolean installProject) throws PlatformException { - if (progressMonitor == null) progressMonitor = new NullProgressMonitor(); + SubMonitor monitor = SubMonitor.convert(progressMonitor, 10); final DatabaseManagement mgmt = new DatabaseManagement(); - progressMonitor.setTaskName("Asserting project resource exists in the database"); + monitor.setTaskName("Asserting project resource exists in the database"); try { projectResource = session.syncRequest( Queries.resource( projectURI ) ); } catch (ResourceNotFoundException nfe) { @@ -560,7 +574,7 @@ public class SimanticsPlatform implements LifecycleListener { } catch (DatabaseException e) { throw new PlatformException("Failed to create "+projectURI, e); } - progressMonitor.worked(10); + monitor.worked(10); return installProject; @@ -794,8 +808,8 @@ public class SimanticsPlatform implements LifecycleListener { TimeLogger.log("Beginning of SimanticsPlatform.startUp"); LOGGER.info("Beginning of SimanticsPlatform.startUp"); - - if (progressMonitor == null) progressMonitor = new NullProgressMonitor(); + + SubMonitor monitor = SubMonitor.convert(progressMonitor, 1000); // For debugging on what kind of platform automatic tests are running in // case there are problems. @@ -817,10 +831,12 @@ public class SimanticsPlatform implements LifecycleListener { boolean usingBaseline = handleBaselineDatabase(); // 1. Assert there is a database at /db - session = setupDatabase(databaseDriverId, progressMonitor, workspacePolicy, userAgent); + SessionDescriptor sessionDescriptor = setupDatabase(databaseDriverId, monitor.newChild(200, SubMonitor.SUPPRESS_NONE), workspacePolicy, userAgent); + session = sessionDescriptor.getSession(); TimeLogger.log("Database setup complete"); - // 1.1 + // 1.1 Delete all indexes if we cannot be certain they are up-to-date + // A full index rebuild will be done later, before project activation. XSupport support = session.getService(XSupport.class); if (support.rolledback()) { try { @@ -832,26 +848,27 @@ public class SimanticsPlatform implements LifecycleListener { // 2. Assert all graphs, and correct versions, are installed to the database if(!usingBaseline) { - synchronizeOntologies(progressMonitor, ontologyPolicy, requireSynchronize); + synchronizeOntologies(monitor.newChild(400, SubMonitor.SUPPRESS_NONE), ontologyPolicy, requireSynchronize); TimeLogger.log("Synchronized ontologies"); } // 4. Assert simantics.cfg exists - boolean installProject = assertConfiguration(progressMonitor,workspacePolicy); + boolean installProject = assertConfiguration(monitor.newChild(25, SubMonitor.SUPPRESS_NONE),workspacePolicy); // 5. Assert Project Resource is installed in the database - installProject = assertProject(progressMonitor, workspacePolicy, installProject); + installProject = assertProject(monitor.newChild(25, SubMonitor.SUPPRESS_NONE), workspacePolicy, installProject); // 6. Install all features into project, if in debug mode - updateInstalledGroups(progressMonitor, true); //installProject); + updateInstalledGroups(monitor.newChild(25), true); //installProject); TimeLogger.log("Installed all features into project"); // 7. Assert L0.Session in database for this session - assertSessionModel(progressMonitor); + assertSessionModel(monitor.newChild(25, SubMonitor.SUPPRESS_NONE)); session.getService(XSupport.class).setServiceMode(false, false); try { + monitor.setTaskName("Flush query cache"); session.syncRequest((Write) graph -> { QueryControl qc = graph.getService(QueryControl.class); qc.flush(graph); @@ -863,6 +880,7 @@ public class SimanticsPlatform implements LifecycleListener { boolean loadProject = true; try { + monitor.setTaskName("Open database session"); sessionContext = SimanticsPlatform.INSTANCE.createSessionContext(true); // This must be before setSessionContext since some listeners might query this sessionContext.setHint(SimanticsKeys.KEY_PROJECT, SimanticsPlatform.INSTANCE.projectResource); @@ -886,16 +904,28 @@ public class SimanticsPlatform implements LifecycleListener { Bindings.defaultBindingFactory.getRepository().put(RGB.Integer.BINDING.type(), RGB.Integer.BINDING); Bindings.defaultBindingFactory.getRepository().put(Font.BINDING.type(), Font.BINDING); - if(loadProject) { + if (support.rolledback() || sessionDescriptor.isFreshDatabase()) { + monitor.setTaskName("Rebuilding all indexes"); + try { + session.getService(IndexedRelations.class).fullRebuild(monitor.newChild(100), session); + } catch (IndexException e) { + LOGGER.error("Failed to re-build all indexes", e); + } + } else { + monitor.worked(100); + } + if(loadProject) { TimeLogger.log("Load projects"); project = Projects.loadProject(sessionContext.getSession(), SimanticsPlatform.INSTANCE.projectResource); + monitor.worked(100); sessionContext.setHint(ProjectKeys.KEY_PROJECT, project); TimeLogger.log("Loading projects complete"); project.activate(); TimeLogger.log("Project activated"); + monitor.worked(100); } } catch (DatabaseException e) { diff --git a/bundles/org.simantics.workbench/src/org/simantics/workbench/internal/TimingProgressMonitor.java b/bundles/org.simantics/src/org/simantics/TimingProgressMonitor.java similarity index 62% rename from bundles/org.simantics.workbench/src/org/simantics/workbench/internal/TimingProgressMonitor.java rename to bundles/org.simantics/src/org/simantics/TimingProgressMonitor.java index bf41202f3..5c43d17b3 100644 --- a/bundles/org.simantics.workbench/src/org/simantics/workbench/internal/TimingProgressMonitor.java +++ b/bundles/org.simantics/src/org/simantics/TimingProgressMonitor.java @@ -9,14 +9,18 @@ * Contributors: * VTT Technical Research Centre of Finland - initial API and implementation *******************************************************************************/ -package org.simantics.workbench.internal; +package org.simantics; import org.eclipse.core.runtime.NullProgressMonitor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * @author Tuukka Lehtonen */ -class TimingProgressMonitor extends NullProgressMonitor { +public class TimingProgressMonitor extends NullProgressMonitor { + + private static final Logger LOGGER = LoggerFactory.getLogger(TimingProgressMonitor.class); private final long start; @@ -24,46 +28,50 @@ class TimingProgressMonitor extends NullProgressMonitor { this.start = System.nanoTime(); } + private void log(String msg) { + LOGGER.info(msg + " @ " + (System.nanoTime() - start)*1e-9); + } + @Override public void worked(int work) { - System.out.println("worked(" + work + ") @ " + (System.nanoTime() - start)*1e-9); + log("worked(" + work + ")"); } @Override public void subTask(String name) { - System.out.println("subTask(" + name + ") @ " + (System.nanoTime() - start)*1e-9); + log("subTask(" + name + ")"); } @Override public void setTaskName(String name) { - System.out.println("setTaskName(" + name + ") @ " + (System.nanoTime() - start)*1e-9); + log("setTaskName(" + name + ")"); } @Override public void setCanceled(boolean value) { - System.out.println("setCanceled(" + value+ ") @ " + (System.nanoTime() - start)*1e-9); + log("setCanceled(" + value+ ")"); super.setCanceled(value); } @Override public boolean isCanceled() { - System.out.println("isCanceled() @ " + (System.nanoTime() - start)*1e-9); + log("isCanceled()"); return super.isCanceled(); } @Override public void internalWorked(double work) { - System.out.println("internalWorked(" + work + ") @ " + (System.nanoTime() - start)*1e-9); + log("internalWorked(" + work + ")"); } @Override public void done() { - System.out.println("done() @ " + (System.nanoTime() - start)*1e-9); + log("done()"); } @Override public void beginTask(String name, int totalWork) { - System.out.println("beginTask(" + name + ", " + totalWork + ") @ " + (System.nanoTime() - start)*1e-9); + log("beginTask(" + name + ", " + totalWork + ")"); } } \ No newline at end of file