1 /*******************************************************************************
2 * Copyright (c) 2007, 2015 Association for Decentralized Information Management
4 * All rights reserved. This program and the accompanying materials
5 * are made available under the terms of the Eclipse Public License v1.0
6 * which accompanies this distribution, and is available at
7 * http://www.eclipse.org/legal/epl-v10.html
10 * VTT Technical Research Centre of Finland - initial API and implementation
11 * Semantum Oy - Fix for simantics issue #6053
12 *******************************************************************************/
13 package org.simantics.db.indexing;
15 import java.io.IOException;
16 import java.nio.file.Files;
17 import java.nio.file.Path;
18 import java.util.ArrayList;
19 import java.util.Collection;
20 import java.util.Collections;
21 import java.util.Iterator;
22 import java.util.List;
24 import java.util.concurrent.CompletableFuture;
25 import java.util.concurrent.Semaphore;
26 import java.util.concurrent.atomic.AtomicReference;
28 import org.apache.lucene.document.Document;
29 import org.apache.lucene.document.DocumentStoredFieldVisitor;
30 import org.apache.lucene.document.Field;
31 import org.apache.lucene.document.FieldType;
32 import org.apache.lucene.document.LongField;
33 import org.apache.lucene.document.TextField;
34 import org.apache.lucene.index.CorruptIndexException;
35 import org.apache.lucene.index.DirectoryReader;
36 import org.apache.lucene.index.FieldInfo;
37 import org.apache.lucene.index.IndexNotFoundException;
38 import org.apache.lucene.index.IndexReader;
39 import org.apache.lucene.index.IndexWriter;
40 import org.apache.lucene.index.IndexWriterConfig;
41 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
42 import org.apache.lucene.index.IndexableField;
43 import org.apache.lucene.index.StoredFieldVisitor;
44 import org.apache.lucene.index.Term;
45 import org.apache.lucene.queryparser.classic.ParseException;
46 import org.apache.lucene.search.IndexSearcher;
47 import org.apache.lucene.search.MatchAllDocsQuery;
48 import org.apache.lucene.search.Query;
49 import org.apache.lucene.search.ScoreDoc;
50 import org.apache.lucene.search.TermQuery;
51 import org.apache.lucene.search.TopDocs;
52 import org.apache.lucene.store.Directory;
53 import org.apache.lucene.store.FSDirectory;
54 import org.apache.lucene.util.Version;
55 import org.eclipse.core.runtime.IProgressMonitor;
56 import org.eclipse.core.runtime.SubMonitor;
57 import org.simantics.databoard.util.ObjectUtils;
58 import org.simantics.db.ReadGraph;
59 import org.simantics.db.RequestProcessor;
60 import org.simantics.db.Resource;
61 import org.simantics.db.Session;
62 import org.simantics.db.common.request.SafeName;
63 import org.simantics.db.common.utils.NameUtils;
64 import org.simantics.db.exception.DatabaseException;
65 import org.simantics.db.indexing.exception.IndexCorruptedException;
66 import org.simantics.db.indexing.exception.IndexingException;
67 import org.simantics.db.indexing.internal.IndexingJob;
68 import org.simantics.db.layer0.adapter.GenericRelation;
69 import org.simantics.db.layer0.genericrelation.IndexException;
70 import org.simantics.db.request.Read;
71 import org.simantics.db.service.CollectionSupport;
72 import org.simantics.db.service.SerialisationSupport;
73 import org.simantics.utils.FileUtils;
74 import org.simantics.utils.datastructures.Pair;
75 import org.simantics.utils.threads.ThreadUtils;
76 import org.slf4j.Logger;
78 import gnu.trove.map.hash.THashMap;
81 * @author Tuukka Lehtonen
82 * @author Antti Villberg
84 abstract public class IndexedRelationsSearcherBase {
86 protected enum State {
87 // No index is available
89 // An index is available, but there is a problem with it
91 // An index is available but no reader or writer is ready
95 // A writer (and a reader) is ready
99 private State state = State.READY;
100 private Throwable exception;
102 public Throwable getException() {
106 public void setProblem(Throwable t) {
108 getLogger().error("Setting problem for {} and previous state {}", this, this.state, t);
109 this.state = State.PROBLEM;
113 public void setNone() {
114 this.state = State.NONE;
117 public void setReady() {
118 this.state = State.READY;
121 public State state() {
125 protected boolean checkState(State state) {
126 return this.state == state;
129 protected void assertState(State state) throws IndexException {
130 State s = this.state;
132 throw new IndexException("Illegal index searcher state, expected " + state.name() + " but state was " + s.name());
135 public void changeState(IProgressMonitor monitor, Session session, State state) {
136 changeState(monitor, session, state, 0);
139 protected void changeState(IProgressMonitor monitor, Session session, State state, int depth) {
141 if (this.state == state) {
142 if (getLogger().isDebugEnabled())
143 getLogger().debug("Trying to change state {} to the same as previous state {} in depth {} with {}", state, this.state, depth, this);
147 if (IndexPolicy.TRACE_INDEX_MANAGEMENT)
148 System.err.println("Index state " + this.state.name() + " => " + state.name() + " " + this);
152 // Try to exit problem state
153 if (State.PROBLEM == this.state && depth > 0) {
154 getLogger().info("Try to exit problem state for {} and state {}", this, state);
155 Throwable t = bestEffortClear(monitor, session);
157 getLogger().error("Best effort clear has failed for state {} and this {}", state, this, t);
161 // Managed to get into initial state
162 this.state = State.NONE;
163 getLogger().info("Managed to get into initial state {}", this.state);
167 // Cannot move into read from no index
168 if (State.NONE == this.state && State.READ == state) {
169 if (getLogger().isDebugEnabled())
170 getLogger().debug("Cannot move into read from no index in {} with state {}", this, state);
173 // Cannot move into write from no index
174 if (State.NONE == this.state && State.WRITE == state) {
175 if (getLogger().isDebugEnabled())
176 getLogger().debug("Cannot move into write from no index in {} with state {}", this, state);
180 boolean success = false;
184 if (searcher != null) {
187 if (reader != null) {
197 if (State.READ == state || State.WRITE == state) {
201 boolean forWriting = State.WRITE == state;
203 if (directory != null)
204 throw new IllegalStateException(getDescriptor() + "Index already loaded");
206 SubMonitor mon = SubMonitor.convert(monitor, 100);
208 mon.beginTask("Loading index", 100);
210 if (IndexPolicy.TRACE_INDEX_LOAD)
211 System.out.println(getDescriptor() + "Loading Lucene index from " + indexPath + " for " + (forWriting ? "writing" : "reading"));
213 long start = System.nanoTime();
215 directory = getDirectory(session);
218 // Never overwrite an index that is about to be loaded.
219 // TODO: could use OpenMode.CREATE_OR_APPEND but must test first
220 IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_4_9, Queries.getAnalyzer()).setOpenMode(OpenMode.APPEND);
222 // FIXME: platform #4676
223 writer = new IndexWriter(directory, config);
224 } catch (IndexNotFoundException e) {
225 // There was no pre-existing index on disk. Create it now.
226 writer = new IndexWriter(directory, config.setOpenMode(OpenMode.CREATE));
229 reader = DirectoryReader.open(directory);
230 searcher = new IndexSearcher(reader);
232 reader = DirectoryReader.open(directory);
233 searcher = new IndexSearcher(reader);
236 long end = System.nanoTime();
240 if (IndexPolicy.PERF_INDEX_LOAD) {
241 double time = (end - start) * 1e-6;
242 System.out.println(getDescriptor() + "Loaded Lucene index from " + indexPath + " for " + (forWriting ? "writing" : "reading") + " in " + time + " ms");
249 } catch (Throwable t) {
254 this.state = State.PROBLEM;
255 changeState(monitor, session, State.NONE, depth+1);
265 public static final FieldType STRING_TYPE = new FieldType();
268 STRING_TYPE.setIndexed(true);
269 STRING_TYPE.setStored(true);
270 STRING_TYPE.setTokenized(true);
271 STRING_TYPE.freeze();
274 protected static Field makeField(String fieldName, String fieldClass) throws IndexingException {
275 switch (fieldClass) {
276 case "Long": return new LongField(fieldName, 0L, Field.Store.YES);
277 case "String": return new Field (fieldName, "", STRING_TYPE);
278 case "Text": return new TextField(fieldName, "", Field.Store.YES);
280 throw new IndexingException("Can only index Long, String and Text fields, encountered field type " + fieldClass);
284 protected static Field[] makeFieldsForRelation(GenericRelation r, int boundLength, Document document) throws DatabaseException {
285 Pair<String, String>[] fields = r.getFields();
286 Field[] fs = new Field[Math.max(0, fields.length - boundLength)];
287 for (int i = boundLength; i < fields.length; i++) {
288 Field f = makeField(fields[i].first, fields[i].second);
289 fs[i - boundLength] = f;
290 if (document != null)
296 void insertIndex(IProgressMonitor monitor, GenericRelation r, int boundLength, Collection<Object[]> documentsData)
297 throws CorruptIndexException, IOException, DatabaseException {
298 assertAccessOpen(true);
300 if (IndexPolicy.TRACE_INDEX_UPDATE)
301 System.out.println(getDescriptor() + "Inserting " + documentsData.size() + " documents into index at " + indexPath);
303 long start = 0, end = 0;
304 if (IndexPolicy.PERF_INDEX_UPDATE)
305 start = System.nanoTime();
308 Document document = new Document();
309 Field[] fs = makeFieldsForRelation(r, boundLength, document);
311 for (Object[] documentData : documentsData) {
312 if (setFields(fs, documentData) == null)
315 if (IndexPolicy.TRACE_INDEX_UPDATE)
316 System.out.println(getDescriptor() + "Inserting document " + document);
318 writer.addDocument(document);
321 if (IndexPolicy.PERF_INDEX_UPDATE) {
322 end = System.nanoTime();
323 double ms = (end - start) * 1e-6;
324 System.out.println(getDescriptor() + "Inserted " + documentsData.size() + " documents into index at " + indexPath + " in " + ms + " ms");
331 void removeIndex(IProgressMonitor monitor, GenericRelation r, RequestProcessor processor, String key, Collection<Object> keyValues) throws DatabaseException, CorruptIndexException, IOException {
332 assertAccessOpen(true);
334 if (IndexPolicy.TRACE_INDEX_UPDATE)
335 System.out.println(getDescriptor() + "Removing " + keyValues.size() + " documents from index at " + indexPath);
337 long start = 0, end = 0;
338 if (IndexPolicy.PERF_INDEX_UPDATE)
339 start = System.nanoTime();
342 for (Object keyValue : keyValues) {
343 Term removedTerm = null;
344 if (keyValue instanceof Long) {
345 removedTerm = IndexUtils.longTerm(key, (Long) keyValue);
346 } else if (keyValue instanceof String) {
347 removedTerm = new Term(key, (String) keyValue);
349 // FIXME: should throw an exception for illegal input data but this would leave the index in an incoherent state
350 getLogger().error("Attempting to remove document from index of {} with key {} and unrecognized key value type {} : {}", input, key, keyValue, keyValue != null ? keyValue.getClass() : "null");
354 if (IndexPolicy.TRACE_INDEX_UPDATE)
355 System.out.println(getDescriptor() + "Removing document with key " + removedTerm);
356 writer.deleteDocuments(removedTerm);
359 if (IndexPolicy.PERF_INDEX_UPDATE) {
360 end = System.nanoTime();
361 double ms = (end - start) * 1e-6;
362 System.out.println(getDescriptor() + "Removed " + keyValues.size() + " documents from index at " + indexPath + " in " + ms + " ms");
369 void removeIndex(IProgressMonitor monitor) throws DatabaseException, CorruptIndexException, IOException {
370 assertAccessOpen(true);
372 long start = 0, end = 0;
373 if (IndexPolicy.PERF_INDEX_UPDATE)
374 start = System.nanoTime();
380 if (IndexPolicy.PERF_INDEX_UPDATE) {
381 end = System.nanoTime();
382 double ms = (end - start) * 1e-6;
383 System.out.println(getDescriptor() + "Removed all documents from index at " + indexPath + " in " + ms + " ms");
390 boolean replaceIndex(IProgressMonitor monitor, String key, Collection<Object> keyValues, GenericRelation r, int boundLength, Collection<Object[]> documentsData) throws CorruptIndexException, IOException, DatabaseException {
392 boolean didReplace = false;
394 assertAccessOpen(true);
395 if (keyValues.size() != documentsData.size())
396 throw new IllegalArgumentException("keyValues size does not match documents data size, " + keyValues.size() + " <> " + documentsData.size());
398 if (IndexPolicy.TRACE_INDEX_UPDATE)
399 System.out.println(getDescriptor() + "Replacing " + keyValues.size() + " documents from index at " + indexPath);
401 long start = 0, end = 0;
402 if (IndexPolicy.PERF_INDEX_UPDATE)
403 start = System.nanoTime();
406 Iterator<Object> keyIt = keyValues.iterator();
407 Iterator<Object[]> documentDataIt = documentsData.iterator();
409 Document document = new Document();
410 Field[] fs = makeFieldsForRelation(r, boundLength, document);
413 while (keyIt.hasNext()) {
414 Object keyValue = keyIt.next();
415 Object[] documentData = documentDataIt.next();
417 Term removedTerm = null;
418 if (keyValue instanceof Long) {
419 removedTerm = IndexUtils.longTerm(key, (Long) keyValue);
420 } else if (keyValue instanceof String) {
421 removedTerm = new Term(key, (String) keyValue);
423 // FIXME: should throw an exception for illegal input data but this would leave the index in an incoherent state
424 System.err.println("[" + getClass().getSimpleName() + "] Unrecognized document key to remove '" + keyValue + "', only " + String.class + " and " + Resource.class + " are supported.");
425 continue nextDocument;
428 if (setFields(fs, documentData) == null)
429 continue nextDocument;
431 if (IndexPolicy.TRACE_INDEX_UPDATE)
432 System.out.println(getDescriptor() + "Replacing document with key " + removedTerm + " with " + document);
434 boolean done = false;
435 if(requireChangeInfoOnReplace()) {
436 TopDocs exist = searcher.search(new TermQuery(removedTerm), null, 2);
437 if(exist.scoreDocs.length == 1) {
438 Document doc = reader.document(exist.scoreDocs[0].doc);
439 if(!areSame(doc, document)) {
440 writer.deleteDocuments(removedTerm);
441 writer.addDocument(document);
443 if (IndexPolicy.TRACE_INDEX_UPDATE)
444 System.out.println("-replaced single existing");
446 if (IndexPolicy.TRACE_INDEX_UPDATE)
447 System.out.println("-was actually same than single existing");
453 writer.deleteDocuments(removedTerm);
454 writer.addDocument(document);
456 if (IndexPolicy.TRACE_INDEX_UPDATE)
457 System.out.println("-had many or none - removed all existing");
462 if (IndexPolicy.PERF_INDEX_UPDATE) {
463 end = System.nanoTime();
464 double ms = (end - start) * 1e-6;
465 System.out.println(getDescriptor() + "Replaced " + keyValues.size() + " documents from index at " + indexPath + " in " + ms + " ms");
475 protected boolean requireChangeInfoOnReplace() {
479 private boolean areSame(Document d1, Document d2) {
480 List<IndexableField> fs1 = d1.getFields();
481 List<IndexableField> fs2 = d2.getFields();
482 if(fs1.size() != fs2.size()) return false;
483 for(int i=0;i<fs1.size();i++) {
484 IndexableField f1 = fs1.get(i);
485 IndexableField f2 = fs2.get(i);
486 String s1 = f1.stringValue();
487 String s2 = f2.stringValue();
488 if (IndexPolicy.TRACE_INDEX_UPDATE)
489 System.err.println("areSame " + s1 + " vs. " + s2 );
490 if(!ObjectUtils.objectEquals(s1,s2)) return false;
495 final RequestProcessor session;
497 final Resource relation;
500 * The schema of the index, i.e. the fields that will be indexed per
501 * document for the specified relation. Since the relation stays the same
502 * throughout the lifetime of this class, the index schema is also assumed
503 * to the same. This means that {@link GenericRelation#getFields()} is
504 * assumed to stay the same.
506 final IndexSchema schema;
518 IndexSearcher searcher;
520 IndexedRelationsSearcherBase(RequestProcessor session, Resource relation, Resource input) {
521 this.session = session;
522 this.relation = relation;
524 this.indexPath = getIndexDirectory(session.getSession(), relation, input);
525 if(isIndexAvailable()) {
530 this.schema = IndexSchema.readFromRelation(session, relation);
533 Directory getDirectory(Session session) throws IOException {
534 return FSDirectory.open(indexPath.toFile());
537 abstract String getDescriptor();
540 * Ensures that searcher is in read or write state.
542 * @param forWriting <code>true</code> to open index for writing,
543 * <code>false</code> for reading
544 * @return true is required state was reached
547 boolean startAccess(IProgressMonitor monitor, Session session, boolean forWriting) {
549 changeState(monitor, session, State.WRITE);
550 return checkState(State.WRITE);
552 changeState(monitor, session, State.READ);
553 return checkState(State.READ);
557 boolean hasAccess(boolean forWriting) {
560 return checkState(State.WRITE);
562 return checkState(State.WRITE) || checkState(State.READ);
566 void assertAccessOpen(boolean forWriting) {
568 if(!checkState(State.WRITE))
569 throw new IllegalStateException("index not opened for writing (directory=" + directory + ", reader=" + reader + ")");
571 if(!(checkState(State.WRITE) || checkState(State.READ)))
572 throw new IllegalStateException("index not opened for reading (directory=" + directory + ", writer=" + writer + ")");
575 void closeWriter(IndexWriter writer) throws CorruptIndexException, IOException {
580 // May throw OOME, see IndexWriter javadoc for the correct actions.
582 } catch (OutOfMemoryError e) {
588 public static String getPattern(GenericRelation relation, int boundCount) {
590 for (int i = 0; i < boundCount; i++)
592 for (int i = 0; i < relation.getFields().length - boundCount; i++)
597 void initializeIndex(IProgressMonitor monitor, ReadGraph graph, Object[] bound, boolean overwrite)
598 throws IOException, DatabaseException
600 IndexingJob.jobifyIfPossible(
602 "Reindexing " + NameUtils.getSafeLabel(graph, input),
605 GenericRelation r = graph.adapt(relation, GenericRelation.class);
607 throw new IndexingException("Given resource " + relation + "could not be adapted to GenericRelation.");
609 GenericRelation selection = r.select(getPattern(r, bound.length), bound);
611 List<Object[]> results = selection.realize(graph);
612 initializeIndexImpl(new CompletableFuture<>(), mon, r, results, bound, overwrite);
613 } catch (IOException e) {
614 getLogger().error("Index is in problematic state! {}", this, e);
615 throw new IndexingException(e);
620 private static final int INDEXING_THREAD_COUNT = 2; // this is quite good parallelism level for lucene
622 void initializeIndexImpl(CompletableFuture<?> result, IProgressMonitor monitor, GenericRelation r, List<Object[]> results, final Object[] bound, boolean overwrite) throws IOException {
624 final SubMonitor mon = SubMonitor.convert(monitor, 100);
626 if (IndexPolicy.TRACE_INDEX_INIT)
627 System.out.println(getDescriptor() + "Initializing index at " + indexPath + " (overwrite = " + overwrite + ")");
628 mon.beginTask("Initializing Index", 100);
631 if (Files.exists(indexPath)) {
632 mon.subTask("Erasing previous index");
633 if (getLogger().isDebugEnabled())
634 getLogger().debug("Erasing previous index {}", indexPath.toAbsolutePath());
635 FileUtils.emptyDirectory(indexPath);
639 final AtomicReference<FSDirectory> directory = new AtomicReference<FSDirectory>();
640 final AtomicReference<IndexWriter> writer = new AtomicReference<IndexWriter>();
643 mon.subTask("Start index write");
644 Files.createDirectories(indexPath);
646 directory.set(FSDirectory.open(indexPath.toFile()));
647 IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_4_9, Queries.getAnalyzer()).setOpenMode(OpenMode.CREATE);
648 writer.set(new IndexWriter(directory.get(), conf));
652 long realizeStart = 0;
653 if (IndexPolicy.PERF_INDEX_INIT)
654 realizeStart = System.nanoTime();
656 mon.subTask("Calculating indexed content");
661 if (IndexPolicy.PERF_INDEX_INIT)
662 System.out.println(getDescriptor() + "Realized index with " + results.size() + " entries at " + indexPath + " in " + (1e-9 * (System.nanoTime()-realizeStart)) + " seconds.");
664 if (IndexPolicy.TRACE_INDEX_INIT)
665 System.out.println(getDescriptor() + "Indexed relation " + r + " produced " + results.size() + " results");
667 long start = IndexPolicy.PERF_INDEX_INIT ? System.nanoTime() : 0;
669 mon.subTask("Indexing content");
670 final Semaphore s = new Semaphore(0);
671 mon.setWorkRemaining(results.size());
672 for (int i = 0; i < INDEXING_THREAD_COUNT; i++) {
673 final int startIndex = i;
674 ThreadUtils.getBlockingWorkExecutor().submit(() -> {
676 Document document = new Document();
677 Field[] fs = makeFieldsForRelation(r, bound.length, document);
679 for (int index = startIndex; index < results.size(); index += INDEXING_THREAD_COUNT) {
680 if (setFields(fs, results.get(index)) == null)
683 writer.get().addDocument(document);
684 } catch (CorruptIndexException e) {
685 getLogger().error("Index is corrupted! {}", this, e);
686 throw new IllegalStateException(e);
687 } catch (IOException e) {
688 getLogger().error("Index is in problematic state! {}", this, e);
689 throw new IllegalStateException(e);
696 } catch (DatabaseException e) {
697 throw new IllegalStateException(e);
705 s.acquire(INDEXING_THREAD_COUNT);
706 } catch (InterruptedException e) {
707 getLogger().error("Could not initialize index {}", this, e);
710 // http://www.gossamer-threads.com/lists/lucene/java-dev/47895
711 // and http://lucene.apache.org/java/docs/index.html#27+November+2011+-+Lucene+Core+3.5.0
712 // advise against calling optimize at all. So let's not do it anymore.
713 //writer.get().optimize();
714 //writer.get().commit();
716 mon.subTask("Flushing");
718 if (IndexPolicy.PERF_INDEX_INIT)
719 System.out.println(getDescriptor() + "Wrote index at " + indexPath + " in " + (1e-9 * (System.nanoTime()-start)) + " seconds.");
721 result.complete(null);
722 // } catch (DatabaseException e) {
723 // getLogger().error("Could not initialize index due to db {}", this, e);
726 closeWriter(writer.getAndSet(null));
728 FileUtils.uncheckedClose(directory.getAndSet(null));
731 } catch (Throwable t) {
732 getLogger().error("Could not initialize index", t);
733 result.completeExceptionally(t);
738 public List<Object[]> debugDocs(IProgressMonitor monitor) throws ParseException, IOException, IndexingException {
740 Query query = new MatchAllDocsQuery();
742 TopDocs td = searcher.search(query, Integer.MAX_VALUE);
744 ScoreDoc[ ] scoreDocs = td.scoreDocs;
745 List<Object[]> result = new ArrayList<Object[]>(scoreDocs.length);
747 for(ScoreDoc scoreDoc:scoreDocs) {
751 Document doc = reader.document(scoreDoc.doc);
752 List<IndexableField> fs = doc.getFields();
753 Object[] o = new Object[fs.size()];
755 for (IndexableField f : fs) {
756 o[index++] = f.stringValue();
760 } catch (CorruptIndexException e) {
761 getLogger().error("Index is corrupted! {}", this, e);
762 throw new IndexCorruptedException("Index is corrupted! " + this, e);
763 } catch (IOException e) {
764 getLogger().error("Index is in problematic state! {}", this, e);
765 throw new IndexingException(e);
775 List<Map<String, Object>> doSearch(IProgressMonitor monitor, RequestProcessor processor, String search, int maxResultCount) throws ParseException, IOException,
778 // An empty search string will crash QueryParser
779 // Just return no results for empty queries.
780 //System.out.println("search: '" + search + "'");
781 if (search.isEmpty())
782 return Collections.emptyList();
784 assertAccessOpen(false);
786 Query query = Queries.parse(search, schema);
788 long start = System.nanoTime();
790 maxResultCount = Math.min(maxResultCount, searcher.getIndexReader().numDocs());
791 if (maxResultCount == 0)
792 return Collections.emptyList();
794 final TopDocs docs = searcher.search(query, null, maxResultCount);
796 // for(Object[] o : debugDocs(monitor)) {
797 // System.err.println("-" + Arrays.toString(o));
800 if (IndexPolicy.PERF_INDEX_QUERY) {
801 long end = System.nanoTime();
802 System.out.println(getDescriptor() + "search(" + search + ", " + maxResultCount + ") into index at " + indexPath + " took " + (1e-9 * (end-start)) + " seconds.");
805 if (docs.totalHits == 0) {
806 return Collections.emptyList();
810 return processor.syncRequest(new Read<List<Map<String, Object>>>() {
813 public List<Map<String, Object>> perform(ReadGraph graph) throws DatabaseException {
815 GenericRelation r = graph.adapt(relation, GenericRelation.class);
817 throw new IndexingException("Given resource " + graph.syncRequest(new SafeName(relation))
818 + "could not be adapted to GenericRelation.");
820 SerialisationSupport support = graph.getService(SerialisationSupport.class);
822 List<Map<String, Object>> result = new ArrayList<Map<String, Object>>(docs.scoreDocs.length);
824 final DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor();
826 for (ScoreDoc scoreDoc : docs.scoreDocs) {
830 reader.document(scoreDoc.doc, visitor);
832 Document doc = visitor.getDocument();
834 List<IndexableField> fs = doc.getFields();
835 Map<String, Object> entry = new THashMap<String, Object>(fs.size());
836 for (IndexableField f : fs) {
837 IndexSchema.Type type = schema.typeMap.get(f.name());
838 if (type == IndexSchema.Type.LONG) {
839 entry.put(f.name(), support.getResource(f.numericValue().longValue()));
841 entry.put(f.name(), f.stringValue());
847 } catch (CorruptIndexException e) {
848 getLogger().error("Index is corrupted! {}", this, e);
849 throw new IndexCorruptedException("Index is corrupted! " + " " + this + " " + scoreDoc, e);
850 } catch (IOException e) {
851 getLogger().error("Index is in problematic state! {}", this, e);
852 throw new IndexingException(e);
858 } catch (DatabaseException e) {
859 if (e instanceof IndexingException) {
860 throw (IndexingException) e;
862 throw new IndexingException(e);
867 static class ResourceVisitor extends StoredFieldVisitor {
872 public Status needsField(FieldInfo fieldInfo) throws IOException {
873 if("Resource".equals(fieldInfo.name)) return Status.YES;
878 public void longField(FieldInfo fieldInfo, long value) throws IOException {
884 static class DumpVisitor extends StoredFieldVisitor {
886 public List<Object> values;
888 DumpVisitor(List<Object> values) {
889 this.values = values;
893 public Status needsField(FieldInfo fieldInfo) throws IOException {
898 public void longField(FieldInfo fieldInfo, long value) throws IOException {
903 public void stringField(FieldInfo fieldInfo, String value) throws IOException {
909 List<Resource> doSearchResources(IProgressMonitor monitor, RequestProcessor processor, String search, int maxResultCount) throws ParseException, IOException,
912 // An empty search string will crash QueryParser
913 // Just return no results for empty queries.
914 //System.out.println("search: '" + search + "'");
915 if (search.isEmpty())
916 return Collections.emptyList();
918 assertAccessOpen(false);
920 Query query = Queries.parse(search, schema);
922 long start = System.nanoTime();
924 maxResultCount = Math.min(maxResultCount, searcher.getIndexReader().numDocs());
925 if (maxResultCount == 0)
926 return Collections.emptyList();
928 final TopDocs docs = searcher.search(query, null, maxResultCount);
930 // for(Object[] o : debugDocs(monitor)) {
931 // System.err.println("-" + Arrays.toString(o));
934 if (IndexPolicy.PERF_INDEX_QUERY) {
935 long end = System.nanoTime();
936 System.out.println(getDescriptor() + "search(" + search + ", " + maxResultCount + ") into index at " + indexPath + " took " + (1e-9 * (end-start)) + " seconds.");
939 if (docs.totalHits == 0) {
940 return Collections.emptyList();
944 return processor.syncRequest(new Read<List<Resource>>() {
947 public List<Resource> perform(ReadGraph graph) throws DatabaseException {
949 CollectionSupport cs = graph.getService(CollectionSupport.class);
950 SerialisationSupport support = graph.getService(SerialisationSupport.class);
952 List<Resource> result = cs.createList();
954 ResourceVisitor visitor = new ResourceVisitor();
956 for (ScoreDoc scoreDoc : docs.scoreDocs) {
958 reader.document(scoreDoc.doc, visitor);
959 result.add(support.getResource(visitor.id));
960 } catch (CorruptIndexException e) {
961 getLogger().error("Index is corrupted! {}", this, e);
962 throw new IndexCorruptedException("Index is corrupted! " + " " + this + " " + scoreDoc, e);
963 } catch (IOException e) {
964 getLogger().error("Index is in problematic state! {}", this, e);
965 throw new IndexingException(e);
971 } catch (DatabaseException e) {
972 if (e instanceof IndexingException) {
973 throw (IndexingException) e;
975 throw new IndexingException(e);
980 List<Object> doList(IProgressMonitor monitor, RequestProcessor processor) throws ParseException, IOException,
983 assertAccessOpen(false);
985 Query query = new MatchAllDocsQuery();
987 final TopDocs docs = searcher.search(query, Integer.MAX_VALUE);
989 ArrayList<Object> result = new ArrayList<Object>();
991 DumpVisitor visitor = new DumpVisitor(result);
993 for (ScoreDoc scoreDoc : docs.scoreDocs) {
997 reader.document(scoreDoc.doc, visitor);
999 } catch (CorruptIndexException e) {
1000 getLogger().error("Index is corrupted! {}", this, e);
1001 throw new IndexCorruptedException("Index is corrupted! " + " " + this + " " + scoreDoc, e);
1002 } catch (IOException e) {
1003 getLogger().error("Index is in problematic state! {}", this, e);
1004 throw new IndexingException(e);
1013 protected static Path getIndexDirectory(Session session, Resource relation, Resource input) {
1014 Path path = DatabaseIndexing.getIndexLocation(session, relation, input);
1015 // System.out.println("getIndexDirectory = " + path);
1019 Path getIndexPath() {
1023 boolean isIndexAvailable() {
1024 return Files.isDirectory(indexPath);
1027 abstract Throwable bestEffortClear(IProgressMonitor monitor, Session session);
1030 * Start from scratch. Clear all caches and rebuild the index.
1032 Throwable clearDirectory(IProgressMonitor monitor, Session session) {
1034 Path file = getIndexPath();
1037 FileUtils.delete(file);
1038 } catch (Throwable t) {
1039 getLogger().error("Could not delete directory {}", file.toAbsolutePath(), t);
1042 if (Files.exists(file))
1043 return new IllegalStateException("Failed to delete directory " + file.toAbsolutePath());
1047 private Field[] setFields(Field[] fs, Object[] result) {
1048 for (int i = 0; i < result.length; i++) {
1049 Object value = result[i];
1050 if (value instanceof String) {
1051 if (IndexPolicy.DEBUG_INDEX_INIT)
1052 System.out.println(getDescriptor() + "index " + fs[i].name() + " = " + value + " : String");
1053 fs[i].setStringValue((String) value);
1054 } else if (value instanceof Long) {
1055 if (IndexPolicy.DEBUG_INDEX_INIT)
1056 System.out.println(getDescriptor() + "index " + fs[i].name() + " = " + value + " : Long");
1057 fs[i].setLongValue((Long) value);
1059 getLogger().error("Can only index Long and String fields, encountered " + value);
1066 protected abstract Logger getLogger();
1069 public String toString() {
1070 return getClass().getSimpleName() + " [" + String.valueOf(schema) + ", " + String.valueOf(relation) + ", " + String.valueOf(input) + ", " + String.valueOf(indexPath) + ", " + String.valueOf(directory) + ", " + String.valueOf(state) + "]";