1 /*******************************************************************************
\r
2 * Copyright (c) 2007, 2010 Association for Decentralized Information Management
\r
3 * in Industry THTH ry.
\r
4 * All rights reserved. This program and the accompanying materials
\r
5 * are made available under the terms of the Eclipse Public License v1.0
\r
6 * which accompanies this distribution, and is available at
\r
7 * http://www.eclipse.org/legal/epl-v10.html
\r
10 * VTT Technical Research Centre of Finland - initial API and implementation
\r
11 *******************************************************************************/
\r
12 package org.simantics.db.indexing;
\r
14 import java.io.File;
\r
15 import java.io.IOException;
\r
16 import java.util.Collection;
\r
17 import java.util.Collections;
\r
18 import java.util.List;
\r
19 import java.util.Map;
\r
20 import java.util.WeakHashMap;
\r
21 import java.util.concurrent.locks.Lock;
\r
22 import java.util.concurrent.locks.ReentrantReadWriteLock;
\r
24 import org.apache.lucene.queryparser.classic.ParseException;
\r
25 import org.eclipse.core.runtime.IProgressMonitor;
\r
26 import org.eclipse.core.runtime.SubMonitor;
\r
27 import org.simantics.db.ReadGraph;
\r
28 import org.simantics.db.RequestProcessor;
\r
29 import org.simantics.db.Resource;
\r
30 import org.simantics.db.common.request.UniqueRead;
\r
31 import org.simantics.db.exception.DatabaseException;
\r
32 import org.simantics.db.exception.InvalidResourceReferenceException;
\r
33 import org.simantics.db.indexing.IndexedRelationsSearcherBase.State;
\r
34 import org.simantics.db.layer0.adapter.GenericRelation;
\r
35 import org.simantics.db.layer0.genericrelation.IndexException;
\r
36 import org.simantics.db.layer0.genericrelation.IndexedRelations;
\r
37 import org.simantics.db.service.QueryControl;
\r
38 import org.simantics.db.service.SerialisationSupport;
\r
39 import org.simantics.utils.datastructures.Pair;
\r
42 * @author Tuukka Lehtonen
\r
43 * @author Antti Villberg
\r
45 public class IndexedRelationsImpl implements IndexedRelations {
\r
47 Map<Object, RWLock> indexLocks = new WeakHashMap<Object, RWLock>();
\r
49 static class LockHandle {
\r
50 public final Object id;
\r
51 public final Lock lock;
\r
53 public LockHandle(Object id, Lock lock) {
\r
58 public void unlock() {
\r
59 if (IndexPolicy.TRACE_INDEX_LOCKING)
\r
60 System.out.println("Unlocking index " + id);
\r
65 static class RWLock {
\r
66 public final Object id;
\r
67 public final ReentrantReadWriteLock lock;
\r
69 public RWLock(Object id) {
\r
71 this.lock = new ReentrantReadWriteLock(true);
\r
74 LockHandle lock(RequestProcessor processor, boolean write) {
\r
75 Lock l = write ? lock.writeLock() : lock.readLock();
\r
76 if(processor instanceof ReadGraph) {
\r
77 ReadGraph graph = (ReadGraph)processor;
\r
78 while(!l.tryLock()) {
\r
79 QueryControl qc = processor.getService(QueryControl.class);
\r
80 boolean executed = qc.resume(graph);
\r
84 } catch (InterruptedException e) {
\r
91 if (IndexPolicy.TRACE_INDEX_LOCKING)
\r
92 System.out.println("Locked index " + id);
\r
93 return new LockHandle(id, l);
\r
96 LockHandle tryLock(RequestProcessor processor, boolean write) {
\r
97 Lock l = write ? lock.writeLock() : lock.readLock();
\r
98 if(l.tryLock()) return new LockHandle(id, l);
\r
104 private LockHandle lock(RequestProcessor processor, Object lockIdentifier, boolean write) {
\r
105 RWLock rwlock = null;
\r
106 synchronized (indexLocks) {
\r
107 rwlock = indexLocks.get(lockIdentifier);
\r
108 if (rwlock == null) {
\r
109 rwlock = new RWLock(lockIdentifier);
\r
110 indexLocks.put(lockIdentifier, rwlock);
\r
113 return rwlock.lock(processor, write);
\r
116 private LockHandle tryLock(RequestProcessor processor, Object lockIdentifier, boolean write) {
\r
117 RWLock rwlock = null;
\r
118 synchronized (indexLocks) {
\r
119 rwlock = indexLocks.get(lockIdentifier);
\r
120 if (rwlock == null) {
\r
121 rwlock = new RWLock(lockIdentifier);
\r
122 indexLocks.put(lockIdentifier, rwlock);
\r
125 return rwlock.tryLock(processor, write);
\r
128 private IndexedRelationsSearcherBase makeSearcher(final RequestProcessor processor, final Resource relation, final Resource input) {
\r
130 return processor.syncRequest(new UniqueRead<IndexedRelationsSearcherBase>() {
\r
133 public IndexedRelationsSearcherBase perform(ReadGraph graph) throws DatabaseException {
\r
134 if(graph.isImmutable(input)) {
\r
135 return MemoryIndexing.getInstance(processor.getSession()).getImmutable(processor, relation, input);
\r
137 return MemoryIndexing.getInstance(processor.getSession()).get(processor, relation, input);
\r
142 } catch (DatabaseException e) {
\r
143 throw new IllegalStateException(e);
\r
147 private LockHandle waitLoaded(SubMonitor progress, final IndexedRelationsSearcherBase searcher, RequestProcessor processor, LockHandle lock, final Object lockId, final Resource input) throws IndexException {
\r
149 // Initial state: we are locked, no news about the index
\r
150 // Final state: we are locked and the index has been loaded, the current lock is returned
\r
152 // First just check if the index is loaded
\r
153 if (searcher.isIndexAvailable()) {
\r
154 // We have an index - try to start access
\r
155 searcher.startAccess(progress.newChild(50), processor.getSession(), false);
\r
156 // At this point we have three options:
\r
157 // 1. we have access
\r
158 if(searcher.hasAccess(false)) return lock;
\r
159 // 2. something is wrong and the index cannot be cleaned
\r
160 if(searcher.checkState(State.PROBLEM)) throw new IndexException("Searcher is in problematic state", searcher.getException());
\r
161 // 3. something was wrong, but the index has been successfully cleaned
\r
164 if(!searcher.checkState(State.NONE))
\r
165 throw new IndexException("Illegal searcher state, contact application support.");
\r
167 // We loop until the index is loaded
\r
170 // With ReadGraph we can proceed to initialize
\r
171 if(processor instanceof ReadGraph) {
\r
173 // (re)create the index.
\r
175 SerialisationSupport ss = processor.getService(SerialisationSupport.class);
\r
176 searcher.initializeIndex(progress.newChild(40), (ReadGraph)processor, new Object[] { ss.getRandomAccessId(input) }, true);
\r
177 searcher.setReady();
\r
178 searcher.startAccess(progress.newChild(10), processor.getSession(), false);
\r
179 if(searcher.hasAccess(false)) return lock;
\r
180 } catch (IOException e) {
\r
181 searcher.setProblem(e);
\r
182 throw new IndexException(e);
\r
183 } catch (DatabaseException e) {
\r
184 searcher.setProblem(e);
\r
185 throw new IndexException(e);
\r
189 // With session we schedule the job
\r
195 // final Semaphore s = new Semaphore(0);
\r
199 boolean success = false;
\r
201 while(!success && (++tries)<10) {
\r
205 success = processor.sync(new UniqueRead<Boolean>() {
\r
208 public Boolean perform(ReadGraph graph) throws DatabaseException {
\r
211 LockHandle lock = tryLock(graph, lockId, true);
\r
212 if(lock == null) return false;
\r
216 boolean loaded = false;
\r
217 if (searcher.isIndexAvailable()) {
\r
218 searcher.startAccess(null, graph.getSession(), false);
\r
219 // At this point we have three options:
\r
220 // 1. we have access
\r
221 if(searcher.hasAccess(false)) loaded = true;
\r
222 // 2. something is wrong and the index cannot be cleaned
\r
223 if(searcher.checkState(State.PROBLEM)) throw new DatabaseException("Searcher is in problematic state", searcher.getException());
\r
224 // 3. something was wrong, but the index has been successfully cleaned
\r
229 if(!searcher.checkState(State.NONE))
\r
230 throw new DatabaseException("Illegal searcher state, contact application support.");
\r
233 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
\r
234 searcher.initializeIndex(null, graph, new Object[] { ss.getRandomAccessId(input) }, true);
\r
235 searcher.setReady();
\r
236 } catch (IOException e) {
\r
237 searcher.setProblem(e);
\r
238 throw new DatabaseException(e);
\r
256 } catch (DatabaseException e) {
\r
257 throw new IndexException(e);
\r
263 throw new IndexException("Did not manage to load index. Contact application support.");
\r
268 lock= lock(processor, lockId, true);
\r
270 if (searcher.isIndexAvailable()) {
\r
271 searcher.startAccess(progress.newChild(50), processor.getSession(), false);
\r
272 if(searcher.hasAccess(false)) return lock;
\r
273 throw new IndexException("Illegal searcher state, contact application support.");
\r
284 public List<Resource> queryResources(IProgressMonitor monitor, String search, RequestProcessor processor,
\r
285 Resource relation, final Resource input, int maxResultCount) {
\r
286 if (processor == null)
\r
287 throw new IllegalArgumentException("null processor");
\r
288 if (relation == null)
\r
289 throw new IllegalArgumentException("null relation");
\r
291 throw new IllegalArgumentException("null input");
\r
292 if (search == null)
\r
293 throw new IllegalArgumentException("null search criterion");
\r
295 SubMonitor progress = SubMonitor.convert(monitor, 100);
\r
297 // Look for existing index.
\r
298 // Indexes always exist in secondary storage, i.e. disk.
\r
299 // Indexes can be cached in memory when necessary performance-wise.
\r
301 final IndexedRelationsSearcherBase searcher = makeSearcher(processor, relation, input);
\r
303 final Object lockId = Pair.make(relation, input);
\r
305 LockHandle lock = lock(processor, lockId, false);
\r
307 // Ensure that index is loaded & ready
\r
308 lock = waitLoaded(progress, searcher, processor, lock, lockId, input);
\r
312 return searcher.doSearchResources(progress.newChild(50), processor, search, maxResultCount);
\r
313 } catch (ParseException e) {
\r
314 // FIXME: should throw an exception, not just ignore.
\r
315 e.printStackTrace();
\r
316 return Collections.emptyList();
\r
317 //throw new IndexException(e);
\r
318 } catch (IOException e) {
\r
319 throw new IndexException(e);
\r
320 } catch (DatabaseException e) {
\r
321 throw new IndexException(e);
\r
328 public List<Map<String, Object>> query(IProgressMonitor monitor, String search, RequestProcessor processor,
\r
329 Resource relation, final Resource input, int maxResultCount) {
\r
330 if (processor == null)
\r
331 throw new IllegalArgumentException("null processor");
\r
332 if (relation == null)
\r
333 throw new IllegalArgumentException("null relation");
\r
335 throw new IllegalArgumentException("null input");
\r
336 if (search == null)
\r
337 throw new IllegalArgumentException("null search criterion");
\r
339 SubMonitor progress = SubMonitor.convert(monitor, 100);
\r
341 // Look for existing index.
\r
342 // Indexes always exist in secondary storage, i.e. disk.
\r
343 // Indexes can be cached in memory when necessary performance-wise.
\r
345 final IndexedRelationsSearcherBase searcher = makeSearcher(processor, relation, input);
\r
347 final Object lockId = Pair.make(relation, input);
\r
349 LockHandle lock = lock(processor, lockId, false);
\r
351 // Ensure that index is loaded & ready
\r
352 lock = waitLoaded(progress, searcher, processor, lock, lockId, input);
\r
356 return searcher.doSearch(progress.newChild(50), processor, search, maxResultCount);
\r
357 } catch (ParseException e) {
\r
358 // FIXME: should throw an exception, not just ignore.
\r
359 e.printStackTrace();
\r
360 return Collections.emptyList();
\r
361 //throw new IndexException(e);
\r
362 } catch (IOException e) {
\r
363 throw new IndexException(e);
\r
364 } catch (DatabaseException e) {
\r
365 throw new IndexException(e);
\r
372 public void insert(IProgressMonitor monitor, RequestProcessor processor, GenericRelation relation,
\r
373 Resource relationResource, Resource input, Collection<Object[]> documents) {
\r
375 // System.out.println("Inserting to index: " + input + " " + documents);
\r
377 if (relation == null)
\r
378 throw new IllegalArgumentException("null relation");
\r
380 throw new IllegalArgumentException("null input");
\r
381 if (documents == null)
\r
382 throw new IllegalArgumentException("null documents");
\r
384 if (documents.isEmpty())
\r
387 final SubMonitor progress = SubMonitor.convert(monitor, 100);
\r
389 final IndexedRelationsSearcherBase searcher = makeSearcher(processor, relationResource, input);
\r
391 LockHandle handle = lock(processor, Pair.make(relationResource, input), true);
\r
395 if(!searcher.startAccess(null, processor.getSession(), true)) {
\r
396 // Could not write index for some reason. Ignore and let the next index query reinitialize the index.
\r
400 searcher.insertIndex(progress.newChild(40), relation, 1, documents);
\r
401 DatabaseIndexing.markIndexChanged(searcher.getIndexPath());
\r
403 } catch (InvalidResourceReferenceException e) {
\r
404 throw new IndexException(e);
\r
405 } catch (IOException e) {
\r
406 throw new IndexException(e);
\r
407 } catch (DatabaseException e) {
\r
408 throw new IndexException(e);
\r
415 public void remove(IProgressMonitor monitor, RequestProcessor processor, GenericRelation relation,
\r
416 Resource relationResource, Resource input, String key, Collection<Object> keyValues) {
\r
418 if (relation == null)
\r
419 throw new IllegalArgumentException("null relation");
\r
421 throw new IllegalArgumentException("null input");
\r
423 throw new IllegalArgumentException("null key");
\r
425 SubMonitor progress = SubMonitor.convert(monitor, 100);
\r
427 IndexedRelationsSearcherBase searcher = makeSearcher(processor, relationResource, input);
\r
429 LockHandle handle = lock(processor, Pair.make(relationResource, input), true);
\r
432 if(!searcher.startAccess(null, processor.getSession(), true)) {
\r
433 // Could not write index for some reason. Ignore and let the next index query reinitialize the index.
\r
437 searcher.removeIndex(progress.newChild(40), relation, processor, key, keyValues);
\r
438 DatabaseIndexing.markIndexChanged(searcher.getIndexPath());
\r
440 } catch (DatabaseException e) {
\r
441 throw new IndexException(e);
\r
442 } catch (IOException e) {
\r
443 throw new IndexException(e);
\r
450 public void removeAll(IProgressMonitor monitor, RequestProcessor processor, GenericRelation relation,
\r
451 Resource relationResource, Resource input) {
\r
453 if (relation == null)
\r
454 throw new IllegalArgumentException("null relation");
\r
456 throw new IllegalArgumentException("null input");
\r
458 IndexedRelationsSearcherBase searcher = makeSearcher(processor, relationResource, input);
\r
460 LockHandle handle = lock(processor, Pair.make(relationResource, input), true);
\r
464 Throwable t = searcher.bestEffortClear(monitor, processor.getSession());
\r
465 if(t != null) searcher.setProblem(t);
\r
466 else searcher.setNone();
\r
475 public boolean replace(IProgressMonitor monitor, RequestProcessor processor, GenericRelation relation,
\r
476 Resource relationResource, Resource input, String key, Collection<Object> keyValues, Collection<Object[]> documents) {
\r
478 if (relation == null)
\r
479 throw new IllegalArgumentException("null relation");
\r
481 throw new IllegalArgumentException("null input");
\r
483 throw new IllegalArgumentException("null key");
\r
485 SubMonitor progress = SubMonitor.convert(monitor, 100);
\r
487 IndexedRelationsSearcherBase searcher = makeSearcher(processor, relationResource, input);
\r
489 LockHandle handle = lock(processor, Pair.make(relationResource, input), true);
\r
491 boolean didChange = false;
\r
495 if(!searcher.startAccess(null, processor.getSession(), true)) {
\r
496 // Could not write index for some reason. Ignore and let the next index query reinitialize the index.
\r
499 didChange = searcher.replaceIndex(progress.newChild(40), key, keyValues, relation, 1, documents);
\r
501 DatabaseIndexing.markIndexChanged(searcher.getIndexPath());
\r
503 } catch (InvalidResourceReferenceException e) {
\r
504 throw new IndexException(e);
\r
505 } catch (IOException e) {
\r
506 throw new IndexException(e);
\r
507 } catch (DatabaseException e) {
\r
508 throw new IndexException(e);
\r
509 } catch (Throwable t) {
\r
510 throw new IndexException(t);
\r
520 public void reset(IProgressMonitor monitor, RequestProcessor processor, Resource relationResource, Resource input) throws IndexException {
\r
522 IndexedRelationsSearcherBase searcher = makeSearcher(processor, relationResource, input);
\r
524 LockHandle handle = lock(processor, Pair.make(relationResource, input), true);
\r
527 searcher.changeState(monitor, processor.getSession(), State.NONE);
\r
528 if (!searcher.checkState(State.NONE))
\r
529 throw new IndexException("Could not close index for input " + input + " before removing it");
\r
531 File path = DatabaseIndexing.getIndexLocation(processor.getSession(), relationResource, input);
\r
532 DatabaseIndexing.deleteIndex(path);
\r
534 } catch (IOException e) {
\r
535 throw new IndexException(e);
\r