X-Git-Url: https://gerrit.simantics.org/r/gitweb?a=blobdiff_plain;f=bundles%2Forg.simantics.db.indexing%2Fsrc%2Forg%2Fsimantics%2Fdb%2Findexing%2FIndexedRelationsImpl.java;h=4caa102529d4adb5851fc799377a42637719abf5;hb=da749c56c5caad4252cd27276bf19b08f4dcb778;hp=bc9247e37818302ce53b3bdbfd5b35f9c406d21d;hpb=480b0e3c16a3958a21de00707b9957c422c21c88;p=simantics%2Fplatform.git diff --git a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsImpl.java b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsImpl.java index bc9247e37..4caa10252 100644 --- a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsImpl.java +++ b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsImpl.java @@ -11,13 +11,19 @@ *******************************************************************************/ package org.simantics.db.indexing; -import java.io.File; import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.WeakHashMap; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.Semaphore; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -27,16 +33,24 @@ import org.eclipse.core.runtime.SubMonitor; import org.simantics.db.ReadGraph; import org.simantics.db.RequestProcessor; import org.simantics.db.Resource; +import org.simantics.db.Session; +import org.simantics.db.common.request.ReadRequest; import org.simantics.db.common.request.UniqueRead; +import org.simantics.db.common.utils.NameUtils; import org.simantics.db.exception.DatabaseException; import org.simantics.db.exception.InvalidResourceReferenceException; import org.simantics.db.indexing.IndexedRelationsSearcherBase.State; +import org.simantics.db.indexing.exception.IndexingException; import org.simantics.db.layer0.adapter.GenericRelation; import org.simantics.db.layer0.genericrelation.IndexException; import org.simantics.db.layer0.genericrelation.IndexedRelations; +import org.simantics.db.layer0.util.Layer0Utils; import org.simantics.db.service.QueryControl; import org.simantics.db.service.SerialisationSupport; +import org.simantics.operation.Layer0X; import org.simantics.utils.datastructures.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * @author Tuukka Lehtonen @@ -44,6 +58,8 @@ import org.simantics.utils.datastructures.Pair; */ public class IndexedRelationsImpl implements IndexedRelations { + private static final Logger LOGGER = LoggerFactory.getLogger(IndexedRelationsImpl.class); + Map indexLocks = new WeakHashMap(); static class LockHandle { @@ -125,7 +141,7 @@ public class IndexedRelationsImpl implements IndexedRelations { return rwlock.tryLock(processor, write); } - private IndexedRelationsSearcherBase makeSearcher(final RequestProcessor processor, final Resource relation, final Resource input) { + private static IndexedRelationsSearcherBase makeSearcher(final RequestProcessor processor, final Resource relation, final Resource input) { try { return processor.syncRequest(new UniqueRead() { @@ -161,9 +177,8 @@ public class IndexedRelationsImpl implements IndexedRelations { // 3. something was wrong, but the index has been successfully cleaned } - if(!searcher.checkState(State.NONE)) - throw new IndexException("Illegal searcher state, contact application support."); - + searcher.assertState(State.NONE); + // We loop until the index is loaded while(true) { @@ -218,16 +233,18 @@ public class IndexedRelationsImpl implements IndexedRelations { searcher.startAccess(null, graph.getSession(), false); // At this point we have three options: // 1. we have access - if(searcher.hasAccess(false)) loaded = true; + if(searcher.hasAccess(false)) + loaded = true; // 2. something is wrong and the index cannot be cleaned - if(searcher.checkState(State.PROBLEM)) throw new DatabaseException("Searcher is in problematic state", searcher.getException()); + if(searcher.checkState(State.PROBLEM)) + throw new IndexingException("Searcher is in problematic state", searcher.getException()); // 3. something was wrong, but the index has been successfully cleaned } if(!loaded) { if(!searcher.checkState(State.NONE)) - throw new DatabaseException("Illegal searcher state, contact application support."); + throw new IndexingException("Illegal searcher state " + searcher.state()); try { SerialisationSupport ss = graph.getService(SerialisationSupport.class); @@ -235,7 +252,7 @@ public class IndexedRelationsImpl implements IndexedRelations { searcher.setReady(); } catch (IOException e) { searcher.setProblem(e); - throw new DatabaseException(e); + throw new IndexingException(e); } } @@ -260,7 +277,7 @@ public class IndexedRelationsImpl implements IndexedRelations { } if(!success) - throw new IndexException("Did not manage to load index. Contact application support."); + throw new IndexException("Did not manage to load index."); // Try again @@ -270,7 +287,7 @@ public class IndexedRelationsImpl implements IndexedRelations { if (searcher.isIndexAvailable()) { searcher.startAccess(progress.newChild(50), processor.getSession(), false); if(searcher.hasAccess(false)) return lock; - throw new IndexException("Illegal searcher state, contact application support."); + throw new IndexException("Illegal searcher state " + searcher.state()); } } @@ -370,7 +387,7 @@ public class IndexedRelationsImpl implements IndexedRelations { @Override public void insert(IProgressMonitor monitor, RequestProcessor processor, GenericRelation relation, - Resource relationResource, Resource input, Collection documents) { + Resource relationResource, Resource input, Collection documents) throws IndexException { // System.out.println("Inserting to index: " + input + " " + documents); @@ -413,7 +430,7 @@ public class IndexedRelationsImpl implements IndexedRelations { @Override public void remove(IProgressMonitor monitor, RequestProcessor processor, GenericRelation relation, - Resource relationResource, Resource input, String key, Collection keyValues) { + Resource relationResource, Resource input, String key, Collection keyValues) throws IndexException { if (relation == null) throw new IllegalArgumentException("null relation"); @@ -473,7 +490,7 @@ public class IndexedRelationsImpl implements IndexedRelations { @Override public boolean replace(IProgressMonitor monitor, RequestProcessor processor, GenericRelation relation, - Resource relationResource, Resource input, String key, Collection keyValues, Collection documents) { + Resource relationResource, Resource input, String key, Collection keyValues, Collection documents) throws IndexException { if (relation == null) throw new IllegalArgumentException("null relation"); @@ -521,16 +538,15 @@ public class IndexedRelationsImpl implements IndexedRelations { IndexedRelationsSearcherBase searcher = makeSearcher(processor, relationResource, input); LockHandle handle = lock(processor, Pair.make(relationResource, input), true); - + Path path = DatabaseIndexing.getIndexLocation(processor.getSession(), relationResource, input); try { searcher.changeState(monitor, processor.getSession(), State.NONE); if (!searcher.checkState(State.NONE)) throw new IndexException("Could not close index for input " + input + " before removing it"); - File path = DatabaseIndexing.getIndexLocation(processor.getSession(), relationResource, input); DatabaseIndexing.deleteIndex(path); - } catch (IOException e) { + LOGGER.error("Could not delete {}", path.toAbsolutePath(), e); throw new IndexException(e); } finally { handle.unlock(); @@ -538,4 +554,116 @@ public class IndexedRelationsImpl implements IndexedRelations { } + @Override + public void fullRebuild(IProgressMonitor monitor, RequestProcessor processor) throws IndexException { + try { + processor.syncRequest(new ReadRequest() { + @Override + public void run(ReadGraph graph) throws DatabaseException { + try { + fullRebuild(monitor, graph); + } catch (IOException e) { + throw new IndexingException(e); + } + } + }); + } catch (DatabaseException e) { + throw new IndexException(e); + } + } + + private void fullRebuild(IProgressMonitor monitor, ReadGraph graph) throws DatabaseException, IOException { + long startTime = System.currentTimeMillis(); + Resource relation = Layer0X.getInstance(graph).DependenciesRelation; + SerialisationSupport ss = graph.getService(SerialisationSupport.class); + Set indexRoots = Layer0Utils.listIndexRoots(graph); + List> waitFor = new ArrayList<>(indexRoots.size()); + SubMonitor mon = SubMonitor.convert(monitor, indexRoots.size()*2); + + for (Resource indexRoot : indexRoots) { + monitor.subTask(NameUtils.getSafeName(graph, indexRoot)); + + IndexedRelationsSearcherBase searcher = makeSearcher(graph, relation, indexRoot); + + GenericRelation r = graph.adapt(relation, GenericRelation.class); + if (r == null) + throw new IndexingException("Given resource " + relation + "could not be adapted to GenericRelation."); + + Object[] bound = new Object[] { ss.getRandomAccessId(indexRoot) }; + GenericRelation selection = r.select(IndexedRelationsSearcherBase.getPattern(r, bound.length), bound); + + long relStart = System.currentTimeMillis(); + List results = selection.realize(graph); + if (LOGGER.isDebugEnabled()) { + long relEnd = System.currentTimeMillis() - relStart; + LOGGER.debug(indexRoot + " realized " + relEnd); + } + mon.worked(1); + + CompletableFuture result = new CompletableFuture<>(); + waitFor.add(result); + ForkJoinPool.commonPool().submit(() -> { + long startTime1 = System.currentTimeMillis(); + try { + searcher.initializeIndexImpl(result, mon.newChild(1, SubMonitor.SUPPRESS_ALL_LABELS), r, results, bound, true); + searcher.setReady(); + } catch (IOException e) { + result.completeExceptionally(e); + LOGGER.error("Could not initialize index", e); + } finally { + if (LOGGER.isDebugEnabled()) + LOGGER.debug(indexRoot + " initialized " + (System.currentTimeMillis() - startTime1)); + } + }); + } + for (CompletableFuture fut : waitFor) { + try { + fut.get(); + } catch (InterruptedException | ExecutionException e) { + throw (IOException) e.getCause(); + } + } + if (LOGGER.isInfoEnabled()) { + long endTime = System.currentTimeMillis() - startTime; + LOGGER.info("All indexes rebuilt in {}", endTime); + } + } + + @Override + public void flush(IProgressMonitor progress, Session session) throws IndexException { + long startTime = System.currentTimeMillis(); + SubMonitor monitor = SubMonitor.convert(progress); + MemoryIndexing mem = MemoryIndexing.getInstance(session); + + try { + List searchers = mem.getAllSearchers(); + int count = searchers.size(); + Semaphore sema = new Semaphore(0); + searchers.stream().parallel().forEach(s -> { + LockHandle handle = lock(session, Pair.make(s.getRelation(), s.getInput()), true); + try { + if (s.isIndexAvailable() && s instanceof IndexedRelationsSearcher) { + IndexedRelationsSearcher searcher = (IndexedRelationsSearcher) s; + try { + List os = searcher.cache.allDocs(monitor, session); + searcher.applyChanges(monitor, session, searcher.cache.r, os); + } catch (Exception e) { + LOGGER.error("Could not flush in-memory changes to on-disk index", e); + } + } + monitor.worked(1); + s.changeState(monitor, session, State.READY); + } finally { + handle.unlock(); + sema.release(); + } + }); + sema.acquire(count); + long totalTime = System.currentTimeMillis() - startTime; + LOGGER.info("Database index cache flush done in {} ms", totalTime); + } catch (InterruptedException e) { + LOGGER.error("Index searcher flush interrupted", e); + } + } + }