1 /*******************************************************************************
2 * Copyright (c) 2007, 2015 Association for Decentralized Information Management
4 * All rights reserved. This program and the accompanying materials
5 * are made available under the terms of the Eclipse Public License v1.0
6 * which accompanies this distribution, and is available at
7 * http://www.eclipse.org/legal/epl-v10.html
10 * VTT Technical Research Centre of Finland - initial API and implementation
11 * Semantum Oy - Fix for simantics issue #6053
12 *******************************************************************************/
13 package org.simantics.db.indexing;
16 import java.io.IOException;
17 import java.nio.file.Files;
18 import java.util.ArrayList;
19 import java.util.Collection;
20 import java.util.Collections;
21 import java.util.Iterator;
22 import java.util.List;
24 import java.util.concurrent.ExecutorService;
25 import java.util.concurrent.Executors;
26 import java.util.concurrent.Semaphore;
27 import java.util.concurrent.ThreadFactory;
28 import java.util.concurrent.atomic.AtomicReference;
30 import org.apache.lucene.document.Document;
31 import org.apache.lucene.document.DocumentStoredFieldVisitor;
32 import org.apache.lucene.document.Field;
33 import org.apache.lucene.document.FieldType;
34 import org.apache.lucene.document.LongField;
35 import org.apache.lucene.document.TextField;
36 import org.apache.lucene.index.CorruptIndexException;
37 import org.apache.lucene.index.DirectoryReader;
38 import org.apache.lucene.index.FieldInfo;
39 import org.apache.lucene.index.IndexNotFoundException;
40 import org.apache.lucene.index.IndexReader;
41 import org.apache.lucene.index.IndexWriter;
42 import org.apache.lucene.index.IndexWriterConfig;
43 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
44 import org.apache.lucene.index.IndexableField;
45 import org.apache.lucene.index.StoredFieldVisitor;
46 import org.apache.lucene.index.Term;
47 import org.apache.lucene.queryparser.classic.ParseException;
48 import org.apache.lucene.search.IndexSearcher;
49 import org.apache.lucene.search.MatchAllDocsQuery;
50 import org.apache.lucene.search.Query;
51 import org.apache.lucene.search.ScoreDoc;
52 import org.apache.lucene.search.TermQuery;
53 import org.apache.lucene.search.TopDocs;
54 import org.apache.lucene.store.Directory;
55 import org.apache.lucene.store.FSDirectory;
56 import org.apache.lucene.util.Version;
57 import org.eclipse.core.runtime.IProgressMonitor;
58 import org.eclipse.core.runtime.SubMonitor;
59 import org.simantics.databoard.util.ObjectUtils;
60 import org.simantics.db.ReadGraph;
61 import org.simantics.db.RequestProcessor;
62 import org.simantics.db.Resource;
63 import org.simantics.db.Session;
64 import org.simantics.db.common.request.SafeName;
65 import org.simantics.db.common.utils.Logger;
66 import org.simantics.db.common.utils.NameUtils;
67 import org.simantics.db.exception.DatabaseException;
68 import org.simantics.db.indexing.internal.IndexingJob;
69 import org.simantics.db.layer0.adapter.GenericRelation;
70 import org.simantics.db.request.Read;
71 import org.simantics.db.service.CollectionSupport;
72 import org.simantics.db.service.SerialisationSupport;
73 import org.simantics.utils.FileUtils;
74 import org.simantics.utils.datastructures.Pair;
76 import gnu.trove.map.hash.THashMap;
79 * @author Tuukka Lehtonen
80 * @author Antti Villberg
82 abstract public class IndexedRelationsSearcherBase {
84 protected enum State {
85 // No index is available
87 // An index is available, but there is a problem with it
89 // An index is available but no reader or writer is ready
93 // A writer (and a reader) is ready
97 private State state = State.READY;
98 private Throwable exception;
100 public Throwable getException() {
104 public void setProblem(Throwable t) {
105 this.state = State.PROBLEM;
109 public void setNone() {
110 this.state = State.NONE;
113 public void setReady() {
114 this.state = State.READY;
117 protected boolean checkState(State state) {
118 return this.state == state;
121 protected void assertState(State state) throws AssertionError {
123 if(this.state != state) throw new AssertionError("Illegal state, expected " + state.name() + " but was in " + this.state.name());
127 public void changeState(IProgressMonitor monitor, Session session, State state) {
128 changeState(monitor, session, state, 0);
131 protected void changeState(IProgressMonitor monitor, Session session, State state, int depth) {
133 if(this.state == state) return;
135 if (IndexPolicy.TRACE_INDEX_MANAGEMENT)
136 System.err.println("Index state " + this.state.name() + " => " + state.name() + " " + this);
140 // Try to exit problem state
141 if (State.PROBLEM == this.state && depth > 0) {
142 Throwable t = bestEffortClear(monitor, session);
147 // Managed to get into initial state
148 this.state = State.NONE;
152 // Cannot move into read from no index
153 if (State.NONE == this.state && State.READ == state) return;
154 // Cannot move into write from no index
155 if (State.NONE == this.state && State.WRITE == state) return;
157 boolean success = false;
161 if (searcher != null) {
164 if (reader != null) {
174 if (State.READ == state || State.WRITE == state) {
178 boolean forWriting = State.WRITE == state;
180 if (directory != null)
181 throw new IllegalStateException(getDescriptor() + "Index already loaded");
183 SubMonitor mon = SubMonitor.convert(monitor, 100);
185 mon.beginTask("Loading index", 100);
187 if (IndexPolicy.TRACE_INDEX_LOAD)
188 System.out.println(getDescriptor() + "Loading Lucene index from " + indexPath + " for " + (forWriting ? "writing" : "reading"));
190 long start = System.nanoTime();
192 directory = getDirectory(session);
195 // Never overwrite an index that is about to be loaded.
196 // TODO: could use OpenMode.CREATE_OR_APPEND but must test first
197 IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_4_9, Queries.getAnalyzer()).setOpenMode(OpenMode.APPEND);
199 // FIXME: platform #4676
200 writer = new IndexWriter(directory, config);
201 } catch (IndexNotFoundException e) {
202 // There was no pre-existing index on disk. Create it now.
203 writer = new IndexWriter(directory, config.setOpenMode(OpenMode.CREATE));
206 reader = DirectoryReader.open(directory);
207 searcher = new IndexSearcher(reader);
209 reader = DirectoryReader.open(directory);
210 searcher = new IndexSearcher(reader);
213 long end = System.nanoTime();
217 if (IndexPolicy.PERF_INDEX_LOAD) {
218 double time = (end - start) * 1e-6;
219 System.out.println(getDescriptor() + "Loaded Lucene index from " + indexPath + " for " + (forWriting ? "writing" : "reading") + " in " + time + " ms");
226 } catch (Throwable t) {
233 this.state = State.PROBLEM;
234 changeState(monitor, session, State.NONE, depth+1);
244 public static final FieldType STRING_TYPE = new FieldType();
247 STRING_TYPE.setIndexed(true);
248 STRING_TYPE.setStored(true);
249 STRING_TYPE.setTokenized(true);
250 STRING_TYPE.freeze();
253 protected static Field makeField(String fieldName, String fieldClass) throws DatabaseException {
254 switch (fieldClass) {
255 case "Long": return new LongField(fieldName, 0L, Field.Store.YES);
256 case "String": return new Field (fieldName, "", STRING_TYPE);
257 case "Text": return new TextField(fieldName, "", Field.Store.YES);
259 throw new DatabaseException("Can only index Long, String and Text fields, encountered field type " + fieldClass);
263 protected static Field[] makeFieldsForRelation(GenericRelation r, int boundLength, Document document) throws DatabaseException {
264 Pair<String, String>[] fields = r.getFields();
265 Field[] fs = new Field[Math.max(0, fields.length - boundLength)];
266 for (int i = boundLength; i < fields.length; i++) {
267 Field f = makeField(fields[i].first, fields[i].second);
268 fs[i - boundLength] = f;
269 if (document != null)
275 void insertIndex(IProgressMonitor monitor, GenericRelation r, int boundLength, Collection<Object[]> documentsData)
276 throws CorruptIndexException, IOException, DatabaseException {
277 assertAccessOpen(true);
279 if (IndexPolicy.TRACE_INDEX_UPDATE)
280 System.out.println(getDescriptor() + "Inserting " + documentsData.size() + " documents into index at " + indexPath);
282 long start = 0, end = 0;
283 if (IndexPolicy.PERF_INDEX_UPDATE)
284 start = System.nanoTime();
287 Document document = new Document();
288 Field[] fs = makeFieldsForRelation(r, boundLength, document);
290 for (Object[] documentData : documentsData) {
291 if (setFields(fs, documentData) == null)
294 if (IndexPolicy.TRACE_INDEX_UPDATE)
295 System.out.println(getDescriptor() + "Inserting document " + document);
297 writer.addDocument(document);
300 if (IndexPolicy.PERF_INDEX_UPDATE) {
301 end = System.nanoTime();
302 double ms = (end - start) * 1e-6;
303 System.out.println(getDescriptor() + "Inserted " + documentsData.size() + " documents into index at " + indexPath + " in " + ms + " ms");
310 void removeIndex(IProgressMonitor monitor, GenericRelation r, RequestProcessor processor, String key, Collection<Object> keyValues) throws DatabaseException, CorruptIndexException, IOException {
311 assertAccessOpen(true);
313 if (IndexPolicy.TRACE_INDEX_UPDATE)
314 System.out.println(getDescriptor() + "Removing " + keyValues.size() + " documents from index at " + indexPath);
316 long start = 0, end = 0;
317 if (IndexPolicy.PERF_INDEX_UPDATE)
318 start = System.nanoTime();
321 for (Object keyValue : keyValues) {
322 Term removedTerm = null;
323 if (keyValue instanceof Long) {
324 removedTerm = IndexUtils.longTerm(key, (Long) keyValue);
325 } else if (keyValue instanceof String) {
326 removedTerm = new Term(key, (String) keyValue);
328 // FIXME: should throw an exception for illegal input data but this would leave the index in an incoherent state
332 if (IndexPolicy.TRACE_INDEX_UPDATE)
333 System.out.println(getDescriptor() + "Removing document with key " + removedTerm);
334 writer.deleteDocuments(removedTerm);
337 if (IndexPolicy.PERF_INDEX_UPDATE) {
338 end = System.nanoTime();
339 double ms = (end - start) * 1e-6;
340 System.out.println(getDescriptor() + "Removed " + keyValues.size() + " documents from index at " + indexPath + " in " + ms + " ms");
347 void removeIndex(IProgressMonitor monitor) throws DatabaseException, CorruptIndexException, IOException {
348 assertAccessOpen(true);
350 long start = 0, end = 0;
351 if (IndexPolicy.PERF_INDEX_UPDATE)
352 start = System.nanoTime();
358 if (IndexPolicy.PERF_INDEX_UPDATE) {
359 end = System.nanoTime();
360 double ms = (end - start) * 1e-6;
361 System.out.println(getDescriptor() + "Removed all documents from index at " + indexPath + " in " + ms + " ms");
368 boolean replaceIndex(IProgressMonitor monitor, String key, Collection<Object> keyValues, GenericRelation r, int boundLength, Collection<Object[]> documentsData) throws CorruptIndexException, IOException, DatabaseException {
370 boolean didReplace = false;
372 assertAccessOpen(true);
373 if (keyValues.size() != documentsData.size())
374 throw new IllegalArgumentException("keyValues size does not match documents data size, " + keyValues.size() + " <> " + documentsData.size());
376 if (IndexPolicy.TRACE_INDEX_UPDATE)
377 System.out.println(getDescriptor() + "Replacing " + keyValues.size() + " documents from index at " + indexPath);
379 long start = 0, end = 0;
380 if (IndexPolicy.PERF_INDEX_UPDATE)
381 start = System.nanoTime();
384 Iterator<Object> keyIt = keyValues.iterator();
385 Iterator<Object[]> documentDataIt = documentsData.iterator();
387 Document document = new Document();
388 Field[] fs = makeFieldsForRelation(r, boundLength, document);
391 while (keyIt.hasNext()) {
392 Object keyValue = keyIt.next();
393 Object[] documentData = documentDataIt.next();
395 Term removedTerm = null;
396 if (keyValue instanceof Long) {
397 removedTerm = IndexUtils.longTerm(key, (Long) keyValue);
398 } else if (keyValue instanceof String) {
399 removedTerm = new Term(key, (String) keyValue);
401 // FIXME: should throw an exception for illegal input data but this would leave the index in an incoherent state
402 System.err.println("[" + getClass().getSimpleName() + "] Unrecognized document key to remove '" + keyValue + "', only " + String.class + " and " + Resource.class + " are supported.");
403 continue nextDocument;
406 if (setFields(fs, documentData) == null)
407 continue nextDocument;
409 if (IndexPolicy.TRACE_INDEX_UPDATE)
410 System.out.println(getDescriptor() + "Replacing document with key " + removedTerm + " with " + document);
412 boolean done = false;
413 if(requireChangeInfoOnReplace()) {
414 TopDocs exist = searcher.search(new TermQuery(removedTerm), null, 2);
415 if(exist.scoreDocs.length == 1 && requireChangeInfoOnReplace()) {
416 Document doc = reader.document(exist.scoreDocs[0].doc);
417 if(!areSame(doc, document)) {
418 writer.deleteDocuments(removedTerm);
419 writer.addDocument(document);
421 if (IndexPolicy.TRACE_INDEX_UPDATE)
422 System.out.println("-replaced single existing");
424 if (IndexPolicy.TRACE_INDEX_UPDATE)
425 System.out.println("-was actually same than single existing");
431 writer.deleteDocuments(removedTerm);
432 writer.addDocument(document);
434 if (IndexPolicy.TRACE_INDEX_UPDATE)
435 System.out.println("-had many or none - removed all existing");
440 if (IndexPolicy.PERF_INDEX_UPDATE) {
441 end = System.nanoTime();
442 double ms = (end - start) * 1e-6;
443 System.out.println(getDescriptor() + "Replaced " + keyValues.size() + " documents from index at " + indexPath + " in " + ms + " ms");
453 protected boolean requireChangeInfoOnReplace() {
457 private boolean areSame(Document d1, Document d2) {
458 List<IndexableField> fs1 = d1.getFields();
459 List<IndexableField> fs2 = d2.getFields();
460 if(fs1.size() != fs2.size()) return false;
461 for(int i=0;i<fs1.size();i++) {
462 IndexableField f1 = fs1.get(i);
463 IndexableField f2 = fs2.get(i);
464 String s1 = f1.stringValue();
465 String s2 = f2.stringValue();
466 if (IndexPolicy.TRACE_INDEX_UPDATE)
467 System.err.println("areSame " + s1 + " vs. " + s2 );
468 if(!ObjectUtils.objectEquals(s1,s2)) return false;
473 final RequestProcessor session;
475 final Resource relation;
478 * The schema of the index, i.e. the fields that will be indexed per
479 * document for the specified relation. Since the relation stays the same
480 * throughout the lifetime of this class, the index schema is also assumed
481 * to the same. This means that {@link GenericRelation#getFields()} is
482 * assumed to stay the same.
484 final IndexSchema schema;
496 IndexSearcher searcher;
498 IndexedRelationsSearcherBase(RequestProcessor session, Resource relation, Resource input) {
499 this.session = session;
500 this.relation = relation;
502 this.indexPath = getIndexDirectory(session.getSession(), relation, input);
503 if(isIndexAvailable()) {
508 this.schema = IndexSchema.readFromRelation(session, relation);
511 Directory getDirectory(Session session) throws IOException {
512 return FSDirectory.open(indexPath);
515 abstract String getDescriptor();
518 * Ensures that searcher is in read or write state.
520 * @param forWriting <code>true</code> to open index for writing,
521 * <code>false</code> for reading
522 * @return true is required state was reached
525 boolean startAccess(IProgressMonitor monitor, Session session, boolean forWriting) {
527 changeState(monitor, session, State.WRITE);
528 return checkState(State.WRITE);
530 changeState(monitor, session, State.READ);
531 return checkState(State.READ);
535 boolean hasAccess(boolean forWriting) {
538 return checkState(State.WRITE);
540 return checkState(State.WRITE) || checkState(State.READ);
544 void assertAccessOpen(boolean forWriting) {
546 if(!checkState(State.WRITE))
547 throw new IllegalStateException("index not opened for writing (directory=" + directory + ", reader=" + reader + ")");
549 if(!(checkState(State.WRITE) || checkState(State.READ)))
550 throw new IllegalStateException("index not opened for reading (directory=" + directory + ", writer=" + writer + ")");
553 void closeWriter(IndexWriter writer) throws CorruptIndexException, IOException {
558 // May throw OOME, see IndexWriter javadoc for the correct actions.
560 } catch (OutOfMemoryError e) {
566 private static String getPattern(GenericRelation relation, int boundCount) {
568 for (int i = 0; i < boundCount; i++)
570 for (int i = 0; i < relation.getFields().length - boundCount; i++)
575 private static final int INDEXING_THREAD_COUNT = 2;
577 private static final ExecutorService executor = Executors.newFixedThreadPool(INDEXING_THREAD_COUNT, new ThreadFactory() {
579 public Thread newThread(Runnable r) {
580 Thread t = new Thread(r, "Lucene Index Creator");
583 if (t.getPriority() != Thread.NORM_PRIORITY)
584 t.setPriority(Thread.NORM_PRIORITY);
589 void initializeIndex(IProgressMonitor monitor, ReadGraph graph, Object[] bound, boolean overwrite)
590 throws IOException, DatabaseException
592 IndexingJob.jobifyIfPossible(
594 "Reindexing " + NameUtils.getSafeLabel(graph, input),
597 initializeIndexImpl(mon, graph, bound, overwrite);
598 } catch (IOException e) {
599 throw new DatabaseException(e);
604 void initializeIndexImpl(IProgressMonitor monitor, ReadGraph graph, final Object[] bound, boolean overwrite) throws IOException,
607 final SubMonitor mon = SubMonitor.convert(monitor, 100);
609 if (IndexPolicy.TRACE_INDEX_INIT)
610 System.out.println(getDescriptor() + "Initializing index at " + indexPath + " (overwrite = " + overwrite + ")");
611 mon.beginTask("Initializing Index", 100);
614 mon.subTask("Erasing previous index");
615 FileUtils.deleteAll(indexPath);
618 final AtomicReference<FSDirectory> directory = new AtomicReference<FSDirectory>();
619 final AtomicReference<IndexWriter> writer = new AtomicReference<IndexWriter>();
622 mon.subTask("Start index write");
623 Files.createDirectories(indexPath.toPath());
625 directory.set(FSDirectory.open(indexPath));
626 IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_4_9, Queries.getAnalyzer()).setOpenMode(OpenMode.CREATE);
627 writer.set(new IndexWriter(directory.get(), conf));
631 final GenericRelation r = graph.adapt(relation, GenericRelation.class);
633 throw new DatabaseException("Given resource " + graph.syncRequest(new SafeName(relation))
634 + "could not be adapted to GenericRelation.");
636 long realizeStart = 0;
637 if (IndexPolicy.PERF_INDEX_INIT)
638 realizeStart = System.nanoTime();
640 mon.subTask("Calculating indexed content");
641 GenericRelation selection = r.select(getPattern(r, bound.length), bound);
643 List<Object[]> results = selection.realize(graph);
646 if (IndexPolicy.PERF_INDEX_INIT)
647 System.out.println(getDescriptor() + "Realized index with " + results.size() + " entries at " + indexPath + " in " + (1e-9 * (System.nanoTime()-realizeStart)) + " seconds.");
648 if (IndexPolicy.TRACE_INDEX_INIT)
649 System.out.println(getDescriptor() + "Indexed relation " + r + " produced " + results.size() + " results");
651 long start = IndexPolicy.PERF_INDEX_INIT ? System.nanoTime() : 0;
653 mon.subTask("Indexing content");
654 final Semaphore s = new Semaphore(0);
655 mon.setWorkRemaining(results.size());
657 for (int i = 0; i < INDEXING_THREAD_COUNT; i++) {
658 final int startIndex = i;
659 executor.submit(() -> {
661 Document document = new Document();
662 Field[] fs = makeFieldsForRelation(r, bound.length, document);
664 for (int index = startIndex; index < results.size(); index += INDEXING_THREAD_COUNT) {
665 if (setFields(fs, results.get(index)) == null)
668 writer.get().addDocument(document);
669 } catch (CorruptIndexException e) {
670 throw new IllegalStateException(e);
671 } catch (IOException e) {
672 throw new IllegalStateException(e);
679 } catch (DatabaseException e) {
680 throw new IllegalStateException(e);
688 s.acquire(INDEXING_THREAD_COUNT);
689 } catch (InterruptedException e) {
690 Logger.defaultLogError(e);
693 // http://www.gossamer-threads.com/lists/lucene/java-dev/47895
694 // and http://lucene.apache.org/java/docs/index.html#27+November+2011+-+Lucene+Core+3.5.0
695 // advise against calling optimize at all. So let's not do it anymore.
696 //writer.get().optimize();
697 //writer.get().commit();
699 mon.subTask("Flushing");
701 if (IndexPolicy.PERF_INDEX_INIT)
702 System.out.println(getDescriptor() + "Wrote index at " + indexPath + " in " + (1e-9 * (System.nanoTime()-start)) + " seconds.");
704 } catch (DatabaseException e) {
706 Logger.defaultLogError(e);
710 closeWriter(writer.getAndSet(null));
712 FileUtils.uncheckedClose(directory.getAndSet(null));
718 public List<Object[]> debugDocs(IProgressMonitor monitor) throws ParseException, IOException, DatabaseException {
720 Query query = new MatchAllDocsQuery();
722 TopDocs td = searcher.search(query, Integer.MAX_VALUE);
724 ScoreDoc[ ] scoreDocs = td.scoreDocs;
725 List<Object[]> result = new ArrayList<Object[]>(scoreDocs.length);
727 for(ScoreDoc scoreDoc:scoreDocs) {
731 Document doc = reader.document(scoreDoc.doc);
732 List<IndexableField> fs = doc.getFields();
733 Object[] o = new Object[fs.size()];
735 for (IndexableField f : fs) {
736 o[index++] = f.stringValue();
740 } catch (CorruptIndexException e) {
741 throw new DatabaseException(e);
742 } catch (IOException e) {
743 throw new DatabaseException(e);
753 List<Map<String, Object>> doSearch(IProgressMonitor monitor, RequestProcessor processor, String search, int maxResultCount) throws ParseException, IOException,
756 // An empty search string will crash QueryParser
757 // Just return no results for empty queries.
758 //System.out.println("search: '" + search + "'");
759 if (search.isEmpty())
760 return Collections.emptyList();
762 assertAccessOpen(false);
764 Query query = Queries.parse(search, schema);
766 long start = System.nanoTime();
768 maxResultCount = Math.min(maxResultCount, searcher.getIndexReader().numDocs());
769 if (maxResultCount == 0)
770 return Collections.emptyList();
772 final TopDocs docs = searcher.search(query, null, maxResultCount);
774 // for(Object[] o : debugDocs(monitor)) {
775 // System.err.println("-" + Arrays.toString(o));
778 if (IndexPolicy.PERF_INDEX_QUERY) {
779 long end = System.nanoTime();
780 System.out.println(getDescriptor() + "search(" + search + ", " + maxResultCount + ") into index at " + indexPath + " took " + (1e-9 * (end-start)) + " seconds.");
783 if (docs.totalHits == 0) {
784 return Collections.emptyList();
787 return processor.syncRequest(new Read<List<Map<String, Object>>>() {
790 public List<Map<String, Object>> perform(ReadGraph graph) throws DatabaseException {
792 GenericRelation r = graph.adapt(relation, GenericRelation.class);
794 throw new DatabaseException("Given resource " + graph.syncRequest(new SafeName(relation))
795 + "could not be adapted to GenericRelation.");
797 SerialisationSupport support = graph.getService(SerialisationSupport.class);
799 List<Map<String, Object>> result = new ArrayList<Map<String, Object>>(docs.scoreDocs.length);
801 final DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor();
803 for (ScoreDoc scoreDoc : docs.scoreDocs) {
807 reader.document(scoreDoc.doc, visitor);
809 Document doc = visitor.getDocument();
811 List<IndexableField> fs = doc.getFields();
812 Map<String, Object> entry = new THashMap<String, Object>(fs.size());
813 for (IndexableField f : fs) {
814 IndexSchema.Type type = schema.typeMap.get(f.name());
815 if (type == IndexSchema.Type.LONG) {
816 entry.put(f.name(), support.getResource(f.numericValue().longValue()));
818 entry.put(f.name(), f.stringValue());
824 } catch (CorruptIndexException e) {
825 throw new DatabaseException(e);
826 } catch (IOException e) {
827 throw new DatabaseException(e);
838 static class ResourceVisitor extends StoredFieldVisitor {
843 public Status needsField(FieldInfo fieldInfo) throws IOException {
844 if("Resource".equals(fieldInfo.name)) return Status.YES;
849 public void longField(FieldInfo fieldInfo, long value) throws IOException {
855 static class DumpVisitor extends StoredFieldVisitor {
857 public List<Object> values;
859 DumpVisitor(List<Object> values) {
860 this.values = values;
864 public Status needsField(FieldInfo fieldInfo) throws IOException {
869 public void longField(FieldInfo fieldInfo, long value) throws IOException {
874 public void stringField(FieldInfo fieldInfo, String value) throws IOException {
880 List<Resource> doSearchResources(IProgressMonitor monitor, RequestProcessor processor, String search, int maxResultCount) throws ParseException, IOException,
883 // An empty search string will crash QueryParser
884 // Just return no results for empty queries.
885 //System.out.println("search: '" + search + "'");
886 if (search.isEmpty())
887 return Collections.emptyList();
889 assertAccessOpen(false);
891 Query query = Queries.parse(search, schema);
893 long start = System.nanoTime();
895 maxResultCount = Math.min(maxResultCount, searcher.getIndexReader().numDocs());
896 if (maxResultCount == 0)
897 return Collections.emptyList();
899 final TopDocs docs = searcher.search(query, null, maxResultCount);
901 // for(Object[] o : debugDocs(monitor)) {
902 // System.err.println("-" + Arrays.toString(o));
905 if (IndexPolicy.PERF_INDEX_QUERY) {
906 long end = System.nanoTime();
907 System.out.println(getDescriptor() + "search(" + search + ", " + maxResultCount + ") into index at " + indexPath + " took " + (1e-9 * (end-start)) + " seconds.");
910 if (docs.totalHits == 0) {
911 return Collections.emptyList();
914 return processor.syncRequest(new Read<List<Resource>>() {
917 public List<Resource> perform(ReadGraph graph) throws DatabaseException {
919 CollectionSupport cs = graph.getService(CollectionSupport.class);
920 SerialisationSupport support = graph.getService(SerialisationSupport.class);
922 List<Resource> result = cs.createList();
924 ResourceVisitor visitor = new ResourceVisitor();
926 for (ScoreDoc scoreDoc : docs.scoreDocs) {
930 reader.document(scoreDoc.doc, visitor);
931 result.add(support.getResource(visitor.id));
933 } catch (CorruptIndexException e) {
934 throw new DatabaseException(e);
935 } catch (IOException e) {
936 throw new DatabaseException(e);
947 List<Object> doList(IProgressMonitor monitor, RequestProcessor processor) throws ParseException, IOException,
950 assertAccessOpen(false);
952 Query query = new MatchAllDocsQuery();
954 final TopDocs docs = searcher.search(query, Integer.MAX_VALUE);
956 ArrayList<Object> result = new ArrayList<Object>();
958 DumpVisitor visitor = new DumpVisitor(result);
960 for (ScoreDoc scoreDoc : docs.scoreDocs) {
964 reader.document(scoreDoc.doc, visitor);
966 } catch (CorruptIndexException e) {
967 throw new DatabaseException(e);
968 } catch (IOException e) {
969 throw new DatabaseException(e);
978 protected static File getIndexDirectory(Session session, Resource relation, Resource input) {
979 File path = DatabaseIndexing.getIndexLocation(session, relation, input);
980 // System.out.println("getIndexDirectory = " + path);
984 File getIndexPath() {
988 boolean isIndexAvailable() {
989 return (indexPath.exists() && indexPath.isDirectory());
992 Throwable bestEffortClear(IProgressMonitor monitor, Session session) {
997 * Start from scratch. Clear all caches and rebuild the index.
999 Throwable clearDirectory(IProgressMonitor monitor, Session session) {
1001 File file = getIndexPath();
1005 for(int i=0;i<15;i++) {
1006 FileUtils.deleteDir(file);
1007 if(!file.exists()) {
1011 Thread.sleep(i*100);
1012 } catch (InterruptedException e) {
1016 } catch (Throwable t) {
1022 return new IllegalStateException("Failed to delete directory " + file.getAbsolutePath());
1026 private Field[] setFields(Field[] fs, Object[] result) {
1027 for (int i = 0; i < result.length; i++) {
1028 Object value = result[i];
1029 if (value instanceof String) {
1030 if (IndexPolicy.DEBUG_INDEX_INIT)
1031 System.out.println(getDescriptor() + "index " + fs[i].name() + " = " + value + " : String");
1032 fs[i].setStringValue((String) value);
1033 } else if (value instanceof Long) {
1034 if (IndexPolicy.DEBUG_INDEX_INIT)
1035 System.out.println(getDescriptor() + "index " + fs[i].name() + " = " + value + " : Long");
1036 fs[i].setLongValue((Long) value);
1038 Logger.defaultLogError("Can only index Long and String fields, encountered " + value);