1 /*******************************************************************************
2 * Copyright (c) 2007, 2010 Association for Decentralized Information Management
4 * All rights reserved. This program and the accompanying materials
5 * are made available under the terms of the Eclipse Public License v1.0
6 * which accompanies this distribution, and is available at
7 * http://www.eclipse.org/legal/epl-v10.html
10 * VTT Technical Research Centre of Finland - initial API and implementation
11 *******************************************************************************/
12 package org.simantics.db.indexing;
14 import java.io.IOException;
15 import java.nio.file.Path;
16 import java.util.ArrayList;
17 import java.util.Collection;
18 import java.util.Collections;
19 import java.util.List;
22 import java.util.WeakHashMap;
23 import java.util.concurrent.CompletableFuture;
24 import java.util.concurrent.ExecutionException;
25 import java.util.concurrent.ForkJoinPool;
26 import java.util.concurrent.locks.Lock;
27 import java.util.concurrent.locks.ReentrantReadWriteLock;
29 import org.apache.lucene.queryparser.classic.ParseException;
30 import org.eclipse.core.runtime.IProgressMonitor;
31 import org.eclipse.core.runtime.SubMonitor;
32 import org.simantics.db.ReadGraph;
33 import org.simantics.db.RequestProcessor;
34 import org.simantics.db.Resource;
35 import org.simantics.db.common.request.ReadRequest;
36 import org.simantics.db.common.request.UniqueRead;
37 import org.simantics.db.common.utils.NameUtils;
38 import org.simantics.db.exception.DatabaseException;
39 import org.simantics.db.exception.InvalidResourceReferenceException;
40 import org.simantics.db.indexing.IndexedRelationsSearcherBase.State;
41 import org.simantics.db.layer0.adapter.GenericRelation;
42 import org.simantics.db.layer0.genericrelation.IndexException;
43 import org.simantics.db.layer0.genericrelation.IndexedRelations;
44 import org.simantics.db.layer0.util.Layer0Utils;
45 import org.simantics.db.service.QueryControl;
46 import org.simantics.db.service.SerialisationSupport;
47 import org.simantics.operation.Layer0X;
48 import org.simantics.utils.datastructures.Pair;
49 import org.slf4j.Logger;
50 import org.slf4j.LoggerFactory;
53 * @author Tuukka Lehtonen
54 * @author Antti Villberg
56 public class IndexedRelationsImpl implements IndexedRelations {
58 private static final Logger LOGGER = LoggerFactory.getLogger(IndexedRelationsImpl.class);
60 Map<Object, RWLock> indexLocks = new WeakHashMap<Object, RWLock>();
62 static class LockHandle {
63 public final Object id;
64 public final Lock lock;
66 public LockHandle(Object id, Lock lock) {
71 public void unlock() {
72 if (IndexPolicy.TRACE_INDEX_LOCKING)
73 System.out.println("Unlocking index " + id);
79 public final Object id;
80 public final ReentrantReadWriteLock lock;
82 public RWLock(Object id) {
84 this.lock = new ReentrantReadWriteLock(true);
87 LockHandle lock(RequestProcessor processor, boolean write) {
88 Lock l = write ? lock.writeLock() : lock.readLock();
89 if(processor instanceof ReadGraph) {
90 ReadGraph graph = (ReadGraph)processor;
92 QueryControl qc = processor.getService(QueryControl.class);
93 boolean executed = qc.resume(graph);
97 } catch (InterruptedException e) {
104 if (IndexPolicy.TRACE_INDEX_LOCKING)
105 System.out.println("Locked index " + id);
106 return new LockHandle(id, l);
109 LockHandle tryLock(RequestProcessor processor, boolean write) {
110 Lock l = write ? lock.writeLock() : lock.readLock();
111 if(l.tryLock()) return new LockHandle(id, l);
117 private LockHandle lock(RequestProcessor processor, Object lockIdentifier, boolean write) {
118 RWLock rwlock = null;
119 synchronized (indexLocks) {
120 rwlock = indexLocks.get(lockIdentifier);
121 if (rwlock == null) {
122 rwlock = new RWLock(lockIdentifier);
123 indexLocks.put(lockIdentifier, rwlock);
126 return rwlock.lock(processor, write);
129 private LockHandle tryLock(RequestProcessor processor, Object lockIdentifier, boolean write) {
130 RWLock rwlock = null;
131 synchronized (indexLocks) {
132 rwlock = indexLocks.get(lockIdentifier);
133 if (rwlock == null) {
134 rwlock = new RWLock(lockIdentifier);
135 indexLocks.put(lockIdentifier, rwlock);
138 return rwlock.tryLock(processor, write);
141 private static IndexedRelationsSearcherBase makeSearcher(final RequestProcessor processor, final Resource relation, final Resource input) {
143 return processor.syncRequest(new UniqueRead<IndexedRelationsSearcherBase>() {
146 public IndexedRelationsSearcherBase perform(ReadGraph graph) throws DatabaseException {
147 if(graph.isImmutable(input)) {
148 return MemoryIndexing.getInstance(processor.getSession()).getImmutable(processor, relation, input);
150 return MemoryIndexing.getInstance(processor.getSession()).get(processor, relation, input);
155 } catch (DatabaseException e) {
156 throw new IllegalStateException(e);
160 private LockHandle waitLoaded(SubMonitor progress, final IndexedRelationsSearcherBase searcher, RequestProcessor processor, LockHandle lock, final Object lockId, final Resource input) throws IndexException {
162 // Initial state: we are locked, no news about the index
163 // Final state: we are locked and the index has been loaded, the current lock is returned
165 // First just check if the index is loaded
166 if (searcher.isIndexAvailable()) {
167 // We have an index - try to start access
168 searcher.startAccess(progress.newChild(50), processor.getSession(), false);
169 // At this point we have three options:
171 if(searcher.hasAccess(false)) return lock;
172 // 2. something is wrong and the index cannot be cleaned
173 if(searcher.checkState(State.PROBLEM)) throw new IndexException("Searcher is in problematic state", searcher.getException());
174 // 3. something was wrong, but the index has been successfully cleaned
177 searcher.assertState(State.NONE);
179 // We loop until the index is loaded
182 // With ReadGraph we can proceed to initialize
183 if(processor instanceof ReadGraph) {
185 // (re)create the index.
187 SerialisationSupport ss = processor.getService(SerialisationSupport.class);
188 searcher.initializeIndex(progress.newChild(40), (ReadGraph)processor, new Object[] { ss.getRandomAccessId(input) }, true);
190 searcher.startAccess(progress.newChild(10), processor.getSession(), false);
191 if(searcher.hasAccess(false)) return lock;
192 } catch (IOException e) {
193 searcher.setProblem(e);
194 throw new IndexException(e);
195 } catch (DatabaseException e) {
196 searcher.setProblem(e);
197 throw new IndexException(e);
201 // With session we schedule the job
207 // final Semaphore s = new Semaphore(0);
211 boolean success = false;
213 while(!success && (++tries)<10) {
217 success = processor.sync(new UniqueRead<Boolean>() {
220 public Boolean perform(ReadGraph graph) throws DatabaseException {
223 LockHandle lock = tryLock(graph, lockId, true);
224 if(lock == null) return false;
228 boolean loaded = false;
229 if (searcher.isIndexAvailable()) {
230 searcher.startAccess(null, graph.getSession(), false);
231 // At this point we have three options:
233 if(searcher.hasAccess(false)) loaded = true;
234 // 2. something is wrong and the index cannot be cleaned
235 if(searcher.checkState(State.PROBLEM)) throw new DatabaseException("Searcher is in problematic state", searcher.getException());
236 // 3. something was wrong, but the index has been successfully cleaned
241 if(!searcher.checkState(State.NONE))
242 throw new DatabaseException("Illegal searcher state " + searcher.state());
245 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
246 searcher.initializeIndex(null, graph, new Object[] { ss.getRandomAccessId(input) }, true);
248 } catch (IOException e) {
249 searcher.setProblem(e);
250 throw new DatabaseException(e);
268 } catch (DatabaseException e) {
269 throw new IndexException(e);
275 throw new IndexException("Did not manage to load index.");
280 lock= lock(processor, lockId, true);
282 if (searcher.isIndexAvailable()) {
283 searcher.startAccess(progress.newChild(50), processor.getSession(), false);
284 if(searcher.hasAccess(false)) return lock;
285 throw new IndexException("Illegal searcher state " + searcher.state());
296 public List<Resource> queryResources(IProgressMonitor monitor, String search, RequestProcessor processor,
297 Resource relation, final Resource input, int maxResultCount) {
298 if (processor == null)
299 throw new IllegalArgumentException("null processor");
300 if (relation == null)
301 throw new IllegalArgumentException("null relation");
303 throw new IllegalArgumentException("null input");
305 throw new IllegalArgumentException("null search criterion");
307 SubMonitor progress = SubMonitor.convert(monitor, 100);
309 // Look for existing index.
310 // Indexes always exist in secondary storage, i.e. disk.
311 // Indexes can be cached in memory when necessary performance-wise.
313 final IndexedRelationsSearcherBase searcher = makeSearcher(processor, relation, input);
315 final Object lockId = Pair.make(relation, input);
317 LockHandle lock = lock(processor, lockId, false);
319 // Ensure that index is loaded & ready
320 lock = waitLoaded(progress, searcher, processor, lock, lockId, input);
324 return searcher.doSearchResources(progress.newChild(50), processor, search, maxResultCount);
325 } catch (ParseException e) {
326 // FIXME: should throw an exception, not just ignore.
328 return Collections.emptyList();
329 //throw new IndexException(e);
330 } catch (IOException e) {
331 throw new IndexException(e);
332 } catch (DatabaseException e) {
333 throw new IndexException(e);
340 public List<Map<String, Object>> query(IProgressMonitor monitor, String search, RequestProcessor processor,
341 Resource relation, final Resource input, int maxResultCount) {
342 if (processor == null)
343 throw new IllegalArgumentException("null processor");
344 if (relation == null)
345 throw new IllegalArgumentException("null relation");
347 throw new IllegalArgumentException("null input");
349 throw new IllegalArgumentException("null search criterion");
351 SubMonitor progress = SubMonitor.convert(monitor, 100);
353 // Look for existing index.
354 // Indexes always exist in secondary storage, i.e. disk.
355 // Indexes can be cached in memory when necessary performance-wise.
357 final IndexedRelationsSearcherBase searcher = makeSearcher(processor, relation, input);
359 final Object lockId = Pair.make(relation, input);
361 LockHandle lock = lock(processor, lockId, false);
363 // Ensure that index is loaded & ready
364 lock = waitLoaded(progress, searcher, processor, lock, lockId, input);
368 return searcher.doSearch(progress.newChild(50), processor, search, maxResultCount);
369 } catch (ParseException e) {
370 // FIXME: should throw an exception, not just ignore.
372 return Collections.emptyList();
373 //throw new IndexException(e);
374 } catch (IOException e) {
375 throw new IndexException(e);
376 } catch (DatabaseException e) {
377 throw new IndexException(e);
384 public void insert(IProgressMonitor monitor, RequestProcessor processor, GenericRelation relation,
385 Resource relationResource, Resource input, Collection<Object[]> documents) throws IndexException {
387 // System.out.println("Inserting to index: " + input + " " + documents);
389 if (relation == null)
390 throw new IllegalArgumentException("null relation");
392 throw new IllegalArgumentException("null input");
393 if (documents == null)
394 throw new IllegalArgumentException("null documents");
396 if (documents.isEmpty())
399 final SubMonitor progress = SubMonitor.convert(monitor, 100);
401 final IndexedRelationsSearcherBase searcher = makeSearcher(processor, relationResource, input);
403 LockHandle handle = lock(processor, Pair.make(relationResource, input), true);
407 DatabaseIndexing.markIndexChanged(processor.getSession(), searcher.getIndexPath());
408 if(!searcher.startAccess(null, processor.getSession(), true)) {
409 // Could not write index for some reason. Ignore and let the next index query reinitialize the index.
413 searcher.insertIndex(progress.newChild(40), relation, 1, documents);
415 } catch (InvalidResourceReferenceException e) {
416 throw new IndexException(e);
417 } catch (IOException e) {
418 throw new IndexException(e);
419 } catch (DatabaseException e) {
420 throw new IndexException(e);
427 public void remove(IProgressMonitor monitor, RequestProcessor processor, GenericRelation relation,
428 Resource relationResource, Resource input, String key, Collection<Object> keyValues) throws IndexException {
430 if (relation == null)
431 throw new IllegalArgumentException("null relation");
433 throw new IllegalArgumentException("null input");
435 throw new IllegalArgumentException("null key");
437 SubMonitor progress = SubMonitor.convert(monitor, 100);
439 IndexedRelationsSearcherBase searcher = makeSearcher(processor, relationResource, input);
441 LockHandle handle = lock(processor, Pair.make(relationResource, input), true);
444 DatabaseIndexing.markIndexChanged(processor.getSession(), searcher.getIndexPath());
445 if(!searcher.startAccess(null, processor.getSession(), true)) {
446 // Could not write index for some reason. Ignore and let the next index query reinitialize the index.
450 searcher.removeIndex(progress.newChild(40), relation, processor, key, keyValues);
452 } catch (DatabaseException e) {
453 throw new IndexException(e);
454 } catch (IOException e) {
455 throw new IndexException(e);
462 public void removeAll(IProgressMonitor monitor, RequestProcessor processor, GenericRelation relation,
463 Resource relationResource, Resource input) {
465 if (relation == null)
466 throw new IllegalArgumentException("null relation");
468 throw new IllegalArgumentException("null input");
470 IndexedRelationsSearcherBase searcher = makeSearcher(processor, relationResource, input);
472 LockHandle handle = lock(processor, Pair.make(relationResource, input), true);
476 Throwable t = searcher.bestEffortClear(monitor, processor.getSession());
477 if(t != null) searcher.setProblem(t);
478 else searcher.setNone();
487 public boolean replace(IProgressMonitor monitor, RequestProcessor processor, GenericRelation relation,
488 Resource relationResource, Resource input, String key, Collection<Object> keyValues, Collection<Object[]> documents) throws IndexException {
490 if (relation == null)
491 throw new IllegalArgumentException("null relation");
493 throw new IllegalArgumentException("null input");
495 throw new IllegalArgumentException("null key");
497 SubMonitor progress = SubMonitor.convert(monitor, 100);
499 IndexedRelationsSearcherBase searcher = makeSearcher(processor, relationResource, input);
501 LockHandle handle = lock(processor, Pair.make(relationResource, input), true);
503 boolean didChange = false;
507 DatabaseIndexing.markIndexChanged(processor.getSession(), searcher.getIndexPath());
508 if(!searcher.startAccess(null, processor.getSession(), true)) {
509 // Could not write index for some reason. Ignore and let the next index query reinitialize the index.
512 didChange |= searcher.replaceIndex(progress.newChild(40), key, keyValues, relation, 1, documents);
514 } catch (InvalidResourceReferenceException e) {
515 throw new IndexException(e);
516 } catch (IOException e) {
517 throw new IndexException(e);
518 } catch (DatabaseException e) {
519 throw new IndexException(e);
520 } catch (Throwable t) {
521 throw new IndexException(t);
531 public void reset(IProgressMonitor monitor, RequestProcessor processor, Resource relationResource, Resource input) throws IndexException {
533 IndexedRelationsSearcherBase searcher = makeSearcher(processor, relationResource, input);
535 LockHandle handle = lock(processor, Pair.make(relationResource, input), true);
536 Path path = DatabaseIndexing.getIndexLocation(processor.getSession(), relationResource, input);
538 searcher.changeState(monitor, processor.getSession(), State.NONE);
539 if (!searcher.checkState(State.NONE))
540 throw new IndexException("Could not close index for input " + input + " before removing it");
542 DatabaseIndexing.deleteIndex(path);
543 } catch (IOException e) {
544 LOGGER.error("Could not delete {}", path.toAbsolutePath(), e);
545 throw new IndexException(e);
553 public void fullRebuild(IProgressMonitor monitor, RequestProcessor processor) throws IndexException {
555 processor.syncRequest(new ReadRequest() {
557 public void run(ReadGraph graph) throws DatabaseException {
559 fullRebuild(monitor, graph);
560 } catch (IOException e) {
561 throw new DatabaseException(e);
565 } catch (DatabaseException e) {
566 throw new IndexException(e);
570 private void fullRebuild(IProgressMonitor monitor, ReadGraph graph) throws DatabaseException, IOException {
571 long startTime = System.currentTimeMillis();
572 Resource relation = Layer0X.getInstance(graph).DependenciesRelation;
573 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
574 Set<Resource> indexRoots = Layer0Utils.listIndexRoots(graph);
575 List<CompletableFuture<?>> waitFor = new ArrayList<>(indexRoots.size());
576 SubMonitor mon = SubMonitor.convert(monitor, indexRoots.size()*2);
578 for (Resource indexRoot : indexRoots) {
579 monitor.subTask(NameUtils.getSafeName(graph, indexRoot));
581 IndexedRelationsSearcherBase searcher = makeSearcher(graph, relation, indexRoot);
583 GenericRelation r = graph.adapt(relation, GenericRelation.class);
585 throw new DatabaseException("Given resource " + relation + "could not be adapted to GenericRelation.");
587 Object[] bound = new Object[] { ss.getRandomAccessId(indexRoot) };
588 GenericRelation selection = r.select(IndexedRelationsSearcherBase.getPattern(r, bound.length), bound);
590 long relStart = System.currentTimeMillis();
591 List<Object[]> results = selection.realize(graph);
592 if (LOGGER.isDebugEnabled()) {
593 long relEnd = System.currentTimeMillis() - relStart;
594 LOGGER.debug(indexRoot + " realized " + relEnd);
598 CompletableFuture<?> result = new CompletableFuture<>();
600 ForkJoinPool.commonPool().submit(() -> {
601 long startTime1 = System.currentTimeMillis();
603 searcher.initializeIndexImpl(result, mon.newChild(1, SubMonitor.SUPPRESS_ALL_LABELS), r, results, bound, true);
605 } catch (IOException e) {
606 result.completeExceptionally(e);
607 LOGGER.error("Could not initialize index", e);
609 if (LOGGER.isDebugEnabled())
610 LOGGER.debug(indexRoot + " initialized " + (System.currentTimeMillis() - startTime1));
614 for (CompletableFuture<?> fut : waitFor) {
617 } catch (InterruptedException | ExecutionException e) {
618 throw (IOException) e.getCause();
621 if (LOGGER.isInfoEnabled()) {
622 long endTime = System.currentTimeMillis() - startTime;
623 LOGGER.info("All indexes rebuilt in {}", endTime);