X-Git-Url: https://gerrit.simantics.org/r/gitweb?p=simantics%2Fplatform.git;a=blobdiff_plain;f=bundles%2Forg.simantics.db.indexing%2Fsrc%2Forg%2Fsimantics%2Fdb%2Findexing%2FIndexedRelationsImpl.java;h=60e824cad0bf4f84592e25dbeda6b0a1b4939112;hp=1b4d62a1d6c6cf9c6d199b7454e713232c763526;hb=4aba159170fc72d39c2f930ea224aa71f4cdc2e7;hpb=c26409b1caf2f1e560d37c5befd11b442399c3fe diff --git a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsImpl.java b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsImpl.java index 1b4d62a1d..60e824cad 100644 --- a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsImpl.java +++ b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsImpl.java @@ -11,13 +11,18 @@ *******************************************************************************/ package org.simantics.db.indexing; -import java.io.File; import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.WeakHashMap; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ForkJoinPool; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -27,16 +32,22 @@ import org.eclipse.core.runtime.SubMonitor; import org.simantics.db.ReadGraph; import org.simantics.db.RequestProcessor; import org.simantics.db.Resource; +import org.simantics.db.common.request.ReadRequest; import org.simantics.db.common.request.UniqueRead; +import org.simantics.db.common.utils.NameUtils; import org.simantics.db.exception.DatabaseException; import org.simantics.db.exception.InvalidResourceReferenceException; import org.simantics.db.indexing.IndexedRelationsSearcherBase.State; import org.simantics.db.layer0.adapter.GenericRelation; import org.simantics.db.layer0.genericrelation.IndexException; import org.simantics.db.layer0.genericrelation.IndexedRelations; +import org.simantics.db.layer0.util.Layer0Utils; import org.simantics.db.service.QueryControl; import org.simantics.db.service.SerialisationSupport; +import org.simantics.operation.Layer0X; import org.simantics.utils.datastructures.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * @author Tuukka Lehtonen @@ -44,6 +55,8 @@ import org.simantics.utils.datastructures.Pair; */ public class IndexedRelationsImpl implements IndexedRelations { + private static final Logger LOGGER = LoggerFactory.getLogger(IndexedRelationsImpl.class); + Map indexLocks = new WeakHashMap(); static class LockHandle { @@ -125,7 +138,7 @@ public class IndexedRelationsImpl implements IndexedRelations { return rwlock.tryLock(processor, write); } - private IndexedRelationsSearcherBase makeSearcher(final RequestProcessor processor, final Resource relation, final Resource input) { + private static IndexedRelationsSearcherBase makeSearcher(final RequestProcessor processor, final Resource relation, final Resource input) { try { return processor.syncRequest(new UniqueRead() { @@ -161,9 +174,8 @@ public class IndexedRelationsImpl implements IndexedRelations { // 3. something was wrong, but the index has been successfully cleaned } - if(!searcher.checkState(State.NONE)) - throw new IndexException("Illegal searcher state, contact application support."); - + searcher.assertState(State.NONE); + // We loop until the index is loaded while(true) { @@ -227,7 +239,7 @@ public class IndexedRelationsImpl implements IndexedRelations { if(!loaded) { if(!searcher.checkState(State.NONE)) - throw new DatabaseException("Illegal searcher state, contact application support."); + throw new DatabaseException("Illegal searcher state " + searcher.state()); try { SerialisationSupport ss = graph.getService(SerialisationSupport.class); @@ -260,7 +272,7 @@ public class IndexedRelationsImpl implements IndexedRelations { } if(!success) - throw new IndexException("Did not manage to load index. Contact application support."); + throw new IndexException("Did not manage to load index."); // Try again @@ -270,7 +282,7 @@ public class IndexedRelationsImpl implements IndexedRelations { if (searcher.isIndexAvailable()) { searcher.startAccess(progress.newChild(50), processor.getSession(), false); if(searcher.hasAccess(false)) return lock; - throw new IndexException("Illegal searcher state, contact application support."); + throw new IndexException("Illegal searcher state " + searcher.state()); } } @@ -370,7 +382,7 @@ public class IndexedRelationsImpl implements IndexedRelations { @Override public void insert(IProgressMonitor monitor, RequestProcessor processor, GenericRelation relation, - Resource relationResource, Resource input, Collection documents) { + Resource relationResource, Resource input, Collection documents) throws IndexException { // System.out.println("Inserting to index: " + input + " " + documents); @@ -413,7 +425,7 @@ public class IndexedRelationsImpl implements IndexedRelations { @Override public void remove(IProgressMonitor monitor, RequestProcessor processor, GenericRelation relation, - Resource relationResource, Resource input, String key, Collection keyValues) { + Resource relationResource, Resource input, String key, Collection keyValues) throws IndexException { if (relation == null) throw new IllegalArgumentException("null relation"); @@ -473,7 +485,7 @@ public class IndexedRelationsImpl implements IndexedRelations { @Override public boolean replace(IProgressMonitor monitor, RequestProcessor processor, GenericRelation relation, - Resource relationResource, Resource input, String key, Collection keyValues, Collection documents) { + Resource relationResource, Resource input, String key, Collection keyValues, Collection documents) throws IndexException { if (relation == null) throw new IllegalArgumentException("null relation"); @@ -497,7 +509,7 @@ public class IndexedRelationsImpl implements IndexedRelations { // Could not write index for some reason. Ignore and let the next index query reinitialize the index. return true; } - searcher.replaceIndex(progress.newChild(40), key, keyValues, relation, 1, documents); + didChange |= searcher.replaceIndex(progress.newChild(40), key, keyValues, relation, 1, documents); } catch (InvalidResourceReferenceException e) { throw new IndexException(e); @@ -521,16 +533,15 @@ public class IndexedRelationsImpl implements IndexedRelations { IndexedRelationsSearcherBase searcher = makeSearcher(processor, relationResource, input); LockHandle handle = lock(processor, Pair.make(relationResource, input), true); - + Path path = DatabaseIndexing.getIndexLocation(processor.getSession(), relationResource, input); try { searcher.changeState(monitor, processor.getSession(), State.NONE); if (!searcher.checkState(State.NONE)) throw new IndexException("Could not close index for input " + input + " before removing it"); - File path = DatabaseIndexing.getIndexLocation(processor.getSession(), relationResource, input); DatabaseIndexing.deleteIndex(path); - } catch (IOException e) { + LOGGER.error("Could not delete {}", path.toAbsolutePath(), e); throw new IndexException(e); } finally { handle.unlock(); @@ -538,4 +549,79 @@ public class IndexedRelationsImpl implements IndexedRelations { } + @Override + public void fullRebuild(IProgressMonitor monitor, RequestProcessor processor) throws IndexException { + try { + processor.syncRequest(new ReadRequest() { + @Override + public void run(ReadGraph graph) throws DatabaseException { + try { + fullRebuild(monitor, graph); + } catch (IOException e) { + throw new DatabaseException(e); + } + } + }); + } catch (DatabaseException e) { + throw new IndexException(e); + } + } + + private void fullRebuild(IProgressMonitor monitor, ReadGraph graph) throws DatabaseException, IOException { + long startTime = System.currentTimeMillis(); + Resource relation = Layer0X.getInstance(graph).DependenciesRelation; + SerialisationSupport ss = graph.getService(SerialisationSupport.class); + Set indexRoots = Layer0Utils.listIndexRoots(graph); + List> waitFor = new ArrayList<>(indexRoots.size()); + SubMonitor mon = SubMonitor.convert(monitor, indexRoots.size()*2); + + for (Resource indexRoot : indexRoots) { + monitor.subTask(NameUtils.getSafeName(graph, indexRoot)); + + IndexedRelationsSearcherBase searcher = makeSearcher(graph, relation, indexRoot); + + GenericRelation r = graph.adapt(relation, GenericRelation.class); + if (r == null) + throw new DatabaseException("Given resource " + relation + "could not be adapted to GenericRelation."); + + Object[] bound = new Object[] { ss.getRandomAccessId(indexRoot) }; + GenericRelation selection = r.select(IndexedRelationsSearcherBase.getPattern(r, bound.length), bound); + + long relStart = System.currentTimeMillis(); + List results = selection.realize(graph); + if (LOGGER.isDebugEnabled()) { + long relEnd = System.currentTimeMillis() - relStart; + LOGGER.debug(indexRoot + " realized " + relEnd); + } + mon.worked(1); + + CompletableFuture result = new CompletableFuture<>(); + waitFor.add(result); + ForkJoinPool.commonPool().submit(() -> { + long startTime1 = System.currentTimeMillis(); + try { + searcher.initializeIndexImpl(result, mon.newChild(1, SubMonitor.SUPPRESS_ALL_LABELS), r, results, bound, true); + searcher.setReady(); + } catch (IOException e) { + result.completeExceptionally(e); + LOGGER.error("Could not initialize index", e); + } finally { + if (LOGGER.isDebugEnabled()) + LOGGER.debug(indexRoot + " initialized " + (System.currentTimeMillis() - startTime1)); + } + }); + } + for (CompletableFuture fut : waitFor) { + try { + fut.get(); + } catch (InterruptedException | ExecutionException e) { + throw (IOException) e.getCause(); + } + } + if (LOGGER.isInfoEnabled()) { + long endTime = System.currentTimeMillis() - startTime; + LOGGER.info("All indexes rebuilt in {}", endTime); + } + } + }