1 /*******************************************************************************
2 * Copyright (c) 2007, 2015 Association for Decentralized Information Management
4 * All rights reserved. This program and the accompanying materials
5 * are made available under the terms of the Eclipse Public License v1.0
6 * which accompanies this distribution, and is available at
7 * http://www.eclipse.org/legal/epl-v10.html
10 * VTT Technical Research Centre of Finland - initial API and implementation
11 * Semantum Oy - Fix for simantics issue #6053
12 *******************************************************************************/
13 package org.simantics.db.indexing;
16 import java.io.IOException;
17 import java.nio.file.Files;
18 import java.nio.file.Path;
19 import java.util.ArrayList;
20 import java.util.Collection;
21 import java.util.Collections;
22 import java.util.Iterator;
23 import java.util.List;
25 import java.util.concurrent.ExecutorService;
26 import java.util.concurrent.Executors;
27 import java.util.concurrent.Semaphore;
28 import java.util.concurrent.ThreadFactory;
29 import java.util.concurrent.atomic.AtomicReference;
31 import org.apache.lucene.document.Document;
32 import org.apache.lucene.document.DocumentStoredFieldVisitor;
33 import org.apache.lucene.document.Field;
34 import org.apache.lucene.document.FieldType;
35 import org.apache.lucene.document.LongField;
36 import org.apache.lucene.document.TextField;
37 import org.apache.lucene.index.CorruptIndexException;
38 import org.apache.lucene.index.DirectoryReader;
39 import org.apache.lucene.index.FieldInfo;
40 import org.apache.lucene.index.IndexNotFoundException;
41 import org.apache.lucene.index.IndexReader;
42 import org.apache.lucene.index.IndexWriter;
43 import org.apache.lucene.index.IndexWriterConfig;
44 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
45 import org.apache.lucene.index.IndexableField;
46 import org.apache.lucene.index.StoredFieldVisitor;
47 import org.apache.lucene.index.Term;
48 import org.apache.lucene.queryparser.classic.ParseException;
49 import org.apache.lucene.search.IndexSearcher;
50 import org.apache.lucene.search.MatchAllDocsQuery;
51 import org.apache.lucene.search.Query;
52 import org.apache.lucene.search.ScoreDoc;
53 import org.apache.lucene.search.TermQuery;
54 import org.apache.lucene.search.TopDocs;
55 import org.apache.lucene.store.Directory;
56 import org.apache.lucene.store.FSDirectory;
57 import org.apache.lucene.util.Version;
58 import org.eclipse.core.runtime.IProgressMonitor;
59 import org.eclipse.core.runtime.SubMonitor;
60 import org.simantics.databoard.util.ObjectUtils;
61 import org.simantics.db.ReadGraph;
62 import org.simantics.db.RequestProcessor;
63 import org.simantics.db.Resource;
64 import org.simantics.db.Session;
65 import org.simantics.db.common.request.SafeName;
66 import org.simantics.db.common.utils.Logger;
67 import org.simantics.db.common.utils.NameUtils;
68 import org.simantics.db.exception.DatabaseException;
69 import org.simantics.db.indexing.internal.IndexingJob;
70 import org.simantics.db.layer0.adapter.GenericRelation;
71 import org.simantics.db.request.Read;
72 import org.simantics.db.service.CollectionSupport;
73 import org.simantics.db.service.SerialisationSupport;
74 import org.simantics.utils.FileUtils;
75 import org.simantics.utils.datastructures.Pair;
77 import gnu.trove.map.hash.THashMap;
80 * @author Tuukka Lehtonen
81 * @author Antti Villberg
83 abstract public class IndexedRelationsSearcherBase {
85 protected enum State {
86 // No index is available
88 // An index is available, but there is a problem with it
90 // An index is available but no reader or writer is ready
94 // A writer (and a reader) is ready
98 private State state = State.READY;
99 private Throwable exception;
101 public Throwable getException() {
105 public void setProblem(Throwable t) {
106 this.state = State.PROBLEM;
110 public void setNone() {
111 this.state = State.NONE;
114 public void setReady() {
115 this.state = State.READY;
118 protected boolean checkState(State state) {
119 return this.state == state;
122 protected void assertState(State state) throws AssertionError {
124 if(this.state != state) throw new AssertionError("Illegal state, expected " + state.name() + " but was in " + this.state.name());
128 public void changeState(IProgressMonitor monitor, Session session, State state) {
129 changeState(monitor, session, state, 0);
132 protected void changeState(IProgressMonitor monitor, Session session, State state, int depth) {
134 if(this.state == state) return;
136 if (IndexPolicy.TRACE_INDEX_MANAGEMENT)
137 System.err.println("Index state " + this.state.name() + " => " + state.name() + " " + this);
141 // Try to exit problem state
142 if (State.PROBLEM == this.state && depth > 0) {
143 Throwable t = bestEffortClear(monitor, session);
148 // Managed to get into initial state
149 this.state = State.NONE;
153 // Cannot move into read from no index
154 if (State.NONE == this.state && State.READ == state) return;
155 // Cannot move into write from no index
156 if (State.NONE == this.state && State.WRITE == state) return;
158 boolean success = false;
162 if (searcher != null) {
165 if (reader != null) {
175 if (State.READ == state || State.WRITE == state) {
179 boolean forWriting = State.WRITE == state;
181 if (directory != null)
182 throw new IllegalStateException(getDescriptor() + "Index already loaded");
184 SubMonitor mon = SubMonitor.convert(monitor, 100);
186 mon.beginTask("Loading index", 100);
188 if (IndexPolicy.TRACE_INDEX_LOAD)
189 System.out.println(getDescriptor() + "Loading Lucene index from " + indexPath + " for " + (forWriting ? "writing" : "reading"));
191 long start = System.nanoTime();
193 directory = getDirectory(session);
196 // Never overwrite an index that is about to be loaded.
197 // TODO: could use OpenMode.CREATE_OR_APPEND but must test first
198 IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_4_9, Queries.getAnalyzer()).setOpenMode(OpenMode.APPEND);
200 // FIXME: platform #4676
201 writer = new IndexWriter(directory, config);
202 } catch (IndexNotFoundException e) {
203 // There was no pre-existing index on disk. Create it now.
204 writer = new IndexWriter(directory, config.setOpenMode(OpenMode.CREATE));
207 reader = DirectoryReader.open(directory);
208 searcher = new IndexSearcher(reader);
210 reader = DirectoryReader.open(directory);
211 searcher = new IndexSearcher(reader);
214 long end = System.nanoTime();
218 if (IndexPolicy.PERF_INDEX_LOAD) {
219 double time = (end - start) * 1e-6;
220 System.out.println(getDescriptor() + "Loaded Lucene index from " + indexPath + " for " + (forWriting ? "writing" : "reading") + " in " + time + " ms");
227 } catch (Throwable t) {
234 this.state = State.PROBLEM;
235 changeState(monitor, session, State.NONE, depth+1);
245 public static final FieldType STRING_TYPE = new FieldType();
248 STRING_TYPE.setIndexed(true);
249 STRING_TYPE.setStored(true);
250 STRING_TYPE.setTokenized(true);
251 STRING_TYPE.freeze();
254 protected static Field makeField(String fieldName, String fieldClass) throws DatabaseException {
255 switch (fieldClass) {
256 case "Long": return new LongField(fieldName, 0L, Field.Store.YES);
257 case "String": return new Field (fieldName, "", STRING_TYPE);
258 case "Text": return new TextField(fieldName, "", Field.Store.YES);
260 throw new DatabaseException("Can only index Long, String and Text fields, encountered field type " + fieldClass);
264 protected static Field[] makeFieldsForRelation(GenericRelation r, int boundLength, Document document) throws DatabaseException {
265 Pair<String, String>[] fields = r.getFields();
266 Field[] fs = new Field[Math.max(0, fields.length - boundLength)];
267 for (int i = boundLength; i < fields.length; i++) {
268 Field f = makeField(fields[i].first, fields[i].second);
269 fs[i - boundLength] = f;
270 if (document != null)
276 void insertIndex(IProgressMonitor monitor, GenericRelation r, int boundLength, Collection<Object[]> documentsData)
277 throws CorruptIndexException, IOException, DatabaseException {
278 assertAccessOpen(true);
280 if (IndexPolicy.TRACE_INDEX_UPDATE)
281 System.out.println(getDescriptor() + "Inserting " + documentsData.size() + " documents into index at " + indexPath);
283 long start = 0, end = 0;
284 if (IndexPolicy.PERF_INDEX_UPDATE)
285 start = System.nanoTime();
288 Document document = new Document();
289 Field[] fs = makeFieldsForRelation(r, boundLength, document);
291 for (Object[] documentData : documentsData) {
292 if (setFields(fs, documentData) == null)
295 if (IndexPolicy.TRACE_INDEX_UPDATE)
296 System.out.println(getDescriptor() + "Inserting document " + document);
298 writer.addDocument(document);
301 if (IndexPolicy.PERF_INDEX_UPDATE) {
302 end = System.nanoTime();
303 double ms = (end - start) * 1e-6;
304 System.out.println(getDescriptor() + "Inserted " + documentsData.size() + " documents into index at " + indexPath + " in " + ms + " ms");
311 void removeIndex(IProgressMonitor monitor, GenericRelation r, RequestProcessor processor, String key, Collection<Object> keyValues) throws DatabaseException, CorruptIndexException, IOException {
312 assertAccessOpen(true);
314 if (IndexPolicy.TRACE_INDEX_UPDATE)
315 System.out.println(getDescriptor() + "Removing " + keyValues.size() + " documents from index at " + indexPath);
317 long start = 0, end = 0;
318 if (IndexPolicy.PERF_INDEX_UPDATE)
319 start = System.nanoTime();
322 for (Object keyValue : keyValues) {
323 Term removedTerm = null;
324 if (keyValue instanceof Long) {
325 removedTerm = IndexUtils.longTerm(key, (Long) keyValue);
326 } else if (keyValue instanceof String) {
327 removedTerm = new Term(key, (String) keyValue);
329 // FIXME: should throw an exception for illegal input data but this would leave the index in an incoherent state
333 if (IndexPolicy.TRACE_INDEX_UPDATE)
334 System.out.println(getDescriptor() + "Removing document with key " + removedTerm);
335 writer.deleteDocuments(removedTerm);
338 if (IndexPolicy.PERF_INDEX_UPDATE) {
339 end = System.nanoTime();
340 double ms = (end - start) * 1e-6;
341 System.out.println(getDescriptor() + "Removed " + keyValues.size() + " documents from index at " + indexPath + " in " + ms + " ms");
348 void removeIndex(IProgressMonitor monitor) throws DatabaseException, CorruptIndexException, IOException {
349 assertAccessOpen(true);
351 long start = 0, end = 0;
352 if (IndexPolicy.PERF_INDEX_UPDATE)
353 start = System.nanoTime();
359 if (IndexPolicy.PERF_INDEX_UPDATE) {
360 end = System.nanoTime();
361 double ms = (end - start) * 1e-6;
362 System.out.println(getDescriptor() + "Removed all documents from index at " + indexPath + " in " + ms + " ms");
369 boolean replaceIndex(IProgressMonitor monitor, String key, Collection<Object> keyValues, GenericRelation r, int boundLength, Collection<Object[]> documentsData) throws CorruptIndexException, IOException, DatabaseException {
371 boolean didReplace = false;
373 assertAccessOpen(true);
374 if (keyValues.size() != documentsData.size())
375 throw new IllegalArgumentException("keyValues size does not match documents data size, " + keyValues.size() + " <> " + documentsData.size());
377 if (IndexPolicy.TRACE_INDEX_UPDATE)
378 System.out.println(getDescriptor() + "Replacing " + keyValues.size() + " documents from index at " + indexPath);
380 long start = 0, end = 0;
381 if (IndexPolicy.PERF_INDEX_UPDATE)
382 start = System.nanoTime();
385 Iterator<Object> keyIt = keyValues.iterator();
386 Iterator<Object[]> documentDataIt = documentsData.iterator();
388 Document document = new Document();
389 Field[] fs = makeFieldsForRelation(r, boundLength, document);
392 while (keyIt.hasNext()) {
393 Object keyValue = keyIt.next();
394 Object[] documentData = documentDataIt.next();
396 Term removedTerm = null;
397 if (keyValue instanceof Long) {
398 removedTerm = IndexUtils.longTerm(key, (Long) keyValue);
399 } else if (keyValue instanceof String) {
400 removedTerm = new Term(key, (String) keyValue);
402 // FIXME: should throw an exception for illegal input data but this would leave the index in an incoherent state
403 System.err.println("[" + getClass().getSimpleName() + "] Unrecognized document key to remove '" + keyValue + "', only " + String.class + " and " + Resource.class + " are supported.");
404 continue nextDocument;
407 if (setFields(fs, documentData) == null)
408 continue nextDocument;
410 if (IndexPolicy.TRACE_INDEX_UPDATE)
411 System.out.println(getDescriptor() + "Replacing document with key " + removedTerm + " with " + document);
413 boolean done = false;
414 if(requireChangeInfoOnReplace()) {
415 TopDocs exist = searcher.search(new TermQuery(removedTerm), null, 2);
416 if(exist.scoreDocs.length == 1 && requireChangeInfoOnReplace()) {
417 Document doc = reader.document(exist.scoreDocs[0].doc);
418 if(!areSame(doc, document)) {
419 writer.deleteDocuments(removedTerm);
420 writer.addDocument(document);
422 if (IndexPolicy.TRACE_INDEX_UPDATE)
423 System.out.println("-replaced single existing");
425 if (IndexPolicy.TRACE_INDEX_UPDATE)
426 System.out.println("-was actually same than single existing");
432 writer.deleteDocuments(removedTerm);
433 writer.addDocument(document);
435 if (IndexPolicy.TRACE_INDEX_UPDATE)
436 System.out.println("-had many or none - removed all existing");
441 if (IndexPolicy.PERF_INDEX_UPDATE) {
442 end = System.nanoTime();
443 double ms = (end - start) * 1e-6;
444 System.out.println(getDescriptor() + "Replaced " + keyValues.size() + " documents from index at " + indexPath + " in " + ms + " ms");
454 protected boolean requireChangeInfoOnReplace() {
458 private boolean areSame(Document d1, Document d2) {
459 List<IndexableField> fs1 = d1.getFields();
460 List<IndexableField> fs2 = d2.getFields();
461 if(fs1.size() != fs2.size()) return false;
462 for(int i=0;i<fs1.size();i++) {
463 IndexableField f1 = fs1.get(i);
464 IndexableField f2 = fs2.get(i);
465 String s1 = f1.stringValue();
466 String s2 = f2.stringValue();
467 if (IndexPolicy.TRACE_INDEX_UPDATE)
468 System.err.println("areSame " + s1 + " vs. " + s2 );
469 if(!ObjectUtils.objectEquals(s1,s2)) return false;
474 final RequestProcessor session;
476 final Resource relation;
479 * The schema of the index, i.e. the fields that will be indexed per
480 * document for the specified relation. Since the relation stays the same
481 * throughout the lifetime of this class, the index schema is also assumed
482 * to the same. This means that {@link GenericRelation#getFields()} is
483 * assumed to stay the same.
485 final IndexSchema schema;
497 IndexSearcher searcher;
499 IndexedRelationsSearcherBase(RequestProcessor session, Resource relation, Resource input) {
500 this.session = session;
501 this.relation = relation;
503 this.indexPath = getIndexDirectory(session.getSession(), relation, input);
504 if(isIndexAvailable()) {
509 this.schema = IndexSchema.readFromRelation(session, relation);
512 Directory getDirectory(Session session) throws IOException {
513 return FSDirectory.open(indexPath);
516 abstract String getDescriptor();
519 * Ensures that searcher is in read or write state.
521 * @param forWriting <code>true</code> to open index for writing,
522 * <code>false</code> for reading
523 * @return true is required state was reached
526 boolean startAccess(IProgressMonitor monitor, Session session, boolean forWriting) {
528 changeState(monitor, session, State.WRITE);
529 return checkState(State.WRITE);
531 changeState(monitor, session, State.READ);
532 return checkState(State.READ);
536 boolean hasAccess(boolean forWriting) {
539 return checkState(State.WRITE);
541 return checkState(State.WRITE) || checkState(State.READ);
545 void assertAccessOpen(boolean forWriting) {
547 if(!checkState(State.WRITE))
548 throw new IllegalStateException("index not opened for writing (directory=" + directory + ", reader=" + reader + ")");
550 if(!(checkState(State.WRITE) || checkState(State.READ)))
551 throw new IllegalStateException("index not opened for reading (directory=" + directory + ", writer=" + writer + ")");
554 void closeWriter(IndexWriter writer) throws CorruptIndexException, IOException {
559 // May throw OOME, see IndexWriter javadoc for the correct actions.
561 } catch (OutOfMemoryError e) {
567 private static String getPattern(GenericRelation relation, int boundCount) {
569 for (int i = 0; i < boundCount; i++)
571 for (int i = 0; i < relation.getFields().length - boundCount; i++)
576 private static final int INDEXING_THREAD_COUNT = 2;
578 private static final ExecutorService executor = Executors.newFixedThreadPool(INDEXING_THREAD_COUNT, new ThreadFactory() {
580 public Thread newThread(Runnable r) {
581 Thread t = new Thread(r, "Lucene Index Creator");
584 if (t.getPriority() != Thread.NORM_PRIORITY)
585 t.setPriority(Thread.NORM_PRIORITY);
590 void initializeIndex(IProgressMonitor monitor, ReadGraph graph, Object[] bound, boolean overwrite)
591 throws IOException, DatabaseException
593 IndexingJob.jobifyIfPossible(
595 "Reindexing " + NameUtils.getSafeLabel(graph, input),
598 initializeIndexImpl(mon, graph, bound, overwrite);
599 } catch (IOException e) {
600 throw new DatabaseException(e);
605 void initializeIndexImpl(IProgressMonitor monitor, ReadGraph graph, final Object[] bound, boolean overwrite) throws IOException,
608 final SubMonitor mon = SubMonitor.convert(monitor, 100);
610 if (IndexPolicy.TRACE_INDEX_INIT)
611 System.out.println(getDescriptor() + "Initializing index at " + indexPath + " (overwrite = " + overwrite + ")");
612 mon.beginTask("Initializing Index", 100);
615 mon.subTask("Erasing previous index");
616 FileUtils.deleteAll(indexPath);
619 final AtomicReference<FSDirectory> directory = new AtomicReference<FSDirectory>();
620 final AtomicReference<IndexWriter> writer = new AtomicReference<IndexWriter>();
623 mon.subTask("Start index write");
624 createDirectory(indexPath);
626 directory.set(FSDirectory.open(indexPath));
627 IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_4_9, Queries.getAnalyzer()).setOpenMode(OpenMode.CREATE);
628 writer.set(new IndexWriter(directory.get(), conf));
632 final GenericRelation r = graph.adapt(relation, GenericRelation.class);
634 throw new DatabaseException("Given resource " + graph.syncRequest(new SafeName(relation))
635 + "could not be adapted to GenericRelation.");
637 long realizeStart = 0;
638 if (IndexPolicy.PERF_INDEX_INIT)
639 realizeStart = System.nanoTime();
641 mon.subTask("Calculating indexed content");
642 GenericRelation selection = r.select(getPattern(r, bound.length), bound);
644 List<Object[]> results = selection.realize(graph);
647 if (IndexPolicy.PERF_INDEX_INIT)
648 System.out.println(getDescriptor() + "Realized index with " + results.size() + " entries at " + indexPath + " in " + (1e-9 * (System.nanoTime()-realizeStart)) + " seconds.");
649 if (IndexPolicy.TRACE_INDEX_INIT)
650 System.out.println(getDescriptor() + "Indexed relation " + r + " produced " + results.size() + " results");
652 long start = IndexPolicy.PERF_INDEX_INIT ? System.nanoTime() : 0;
654 mon.subTask("Indexing content");
655 final Semaphore s = new Semaphore(0);
656 mon.setWorkRemaining(results.size());
658 for (int i = 0; i < INDEXING_THREAD_COUNT; i++) {
659 final int startIndex = i;
660 executor.submit(() -> {
662 Document document = new Document();
663 Field[] fs = makeFieldsForRelation(r, bound.length, document);
665 for (int index = startIndex; index < results.size(); index += INDEXING_THREAD_COUNT) {
666 if (setFields(fs, results.get(index)) == null)
669 writer.get().addDocument(document);
670 } catch (CorruptIndexException e) {
671 throw new IllegalStateException(e);
672 } catch (IOException e) {
673 throw new IllegalStateException(e);
682 } catch (DatabaseException e) {
683 throw new IllegalStateException(e);
689 s.acquire(INDEXING_THREAD_COUNT);
690 } catch (InterruptedException e) {
691 Logger.defaultLogError(e);
694 // http://www.gossamer-threads.com/lists/lucene/java-dev/47895
695 // and http://lucene.apache.org/java/docs/index.html#27+November+2011+-+Lucene+Core+3.5.0
696 // advise against calling optimize at all. So let's not do it anymore.
697 //writer.get().optimize();
698 //writer.get().commit();
700 mon.subTask("Flushing");
702 if (IndexPolicy.PERF_INDEX_INIT)
703 System.out.println(getDescriptor() + "Wrote index at " + indexPath + " in " + (1e-9 * (System.nanoTime()-start)) + " seconds.");
705 } catch (DatabaseException e) {
707 Logger.defaultLogError(e);
711 closeWriter(writer.getAndSet(null));
713 FileUtils.uncheckedClose(directory.getAndSet(null));
719 public List<Object[]> debugDocs(IProgressMonitor monitor) throws ParseException, IOException, DatabaseException {
721 Query query = new MatchAllDocsQuery();
723 TopDocs td = searcher.search(query, Integer.MAX_VALUE);
725 ScoreDoc[ ] scoreDocs = td.scoreDocs;
726 List<Object[]> result = new ArrayList<Object[]>(scoreDocs.length);
728 for(ScoreDoc scoreDoc:scoreDocs) {
732 Document doc = reader.document(scoreDoc.doc);
733 List<IndexableField> fs = doc.getFields();
734 Object[] o = new Object[fs.size()];
736 for (IndexableField f : fs) {
737 o[index++] = f.stringValue();
741 } catch (CorruptIndexException e) {
742 throw new DatabaseException(e);
743 } catch (IOException e) {
744 throw new DatabaseException(e);
754 List<Map<String, Object>> doSearch(IProgressMonitor monitor, RequestProcessor processor, String search, int maxResultCount) throws ParseException, IOException,
757 // An empty search string will crash QueryParser
758 // Just return no results for empty queries.
759 //System.out.println("search: '" + search + "'");
760 if (search.isEmpty())
761 return Collections.emptyList();
763 assertAccessOpen(false);
765 Query query = Queries.parse(search, schema);
767 long start = System.nanoTime();
769 maxResultCount = Math.min(maxResultCount, searcher.getIndexReader().numDocs());
770 if (maxResultCount == 0)
771 return Collections.emptyList();
773 final TopDocs docs = searcher.search(query, null, maxResultCount);
775 // for(Object[] o : debugDocs(monitor)) {
776 // System.err.println("-" + Arrays.toString(o));
779 if (IndexPolicy.PERF_INDEX_QUERY) {
780 long end = System.nanoTime();
781 System.out.println(getDescriptor() + "search(" + search + ", " + maxResultCount + ") into index at " + indexPath + " took " + (1e-9 * (end-start)) + " seconds.");
784 if (docs.totalHits == 0) {
785 return Collections.emptyList();
788 return processor.syncRequest(new Read<List<Map<String, Object>>>() {
791 public List<Map<String, Object>> perform(ReadGraph graph) throws DatabaseException {
793 GenericRelation r = graph.adapt(relation, GenericRelation.class);
795 throw new DatabaseException("Given resource " + graph.syncRequest(new SafeName(relation))
796 + "could not be adapted to GenericRelation.");
798 SerialisationSupport support = graph.getService(SerialisationSupport.class);
800 List<Map<String, Object>> result = new ArrayList<Map<String, Object>>(docs.scoreDocs.length);
802 final DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor();
804 for (ScoreDoc scoreDoc : docs.scoreDocs) {
808 reader.document(scoreDoc.doc, visitor);
810 Document doc = visitor.getDocument();
812 List<IndexableField> fs = doc.getFields();
813 Map<String, Object> entry = new THashMap<String, Object>(fs.size());
814 for (IndexableField f : fs) {
815 IndexSchema.Type type = schema.typeMap.get(f.name());
816 if (type == IndexSchema.Type.LONG) {
817 entry.put(f.name(), support.getResource(f.numericValue().longValue()));
819 entry.put(f.name(), f.stringValue());
825 } catch (CorruptIndexException e) {
826 throw new DatabaseException(e);
827 } catch (IOException e) {
828 throw new DatabaseException(e);
839 static class ResourceVisitor extends StoredFieldVisitor {
844 public Status needsField(FieldInfo fieldInfo) throws IOException {
845 if("Resource".equals(fieldInfo.name)) return Status.YES;
850 public void longField(FieldInfo fieldInfo, long value) throws IOException {
856 static class DumpVisitor extends StoredFieldVisitor {
858 public List<Object> values;
860 DumpVisitor(List<Object> values) {
861 this.values = values;
865 public Status needsField(FieldInfo fieldInfo) throws IOException {
870 public void longField(FieldInfo fieldInfo, long value) throws IOException {
875 public void stringField(FieldInfo fieldInfo, String value) throws IOException {
881 List<Resource> doSearchResources(IProgressMonitor monitor, RequestProcessor processor, String search, int maxResultCount) throws ParseException, IOException,
884 // An empty search string will crash QueryParser
885 // Just return no results for empty queries.
886 //System.out.println("search: '" + search + "'");
887 if (search.isEmpty())
888 return Collections.emptyList();
890 assertAccessOpen(false);
892 Query query = Queries.parse(search, schema);
894 long start = System.nanoTime();
896 maxResultCount = Math.min(maxResultCount, searcher.getIndexReader().numDocs());
897 if (maxResultCount == 0)
898 return Collections.emptyList();
900 final TopDocs docs = searcher.search(query, null, maxResultCount);
902 // for(Object[] o : debugDocs(monitor)) {
903 // System.err.println("-" + Arrays.toString(o));
906 if (IndexPolicy.PERF_INDEX_QUERY) {
907 long end = System.nanoTime();
908 System.out.println(getDescriptor() + "search(" + search + ", " + maxResultCount + ") into index at " + indexPath + " took " + (1e-9 * (end-start)) + " seconds.");
911 if (docs.totalHits == 0) {
912 return Collections.emptyList();
915 return processor.syncRequest(new Read<List<Resource>>() {
918 public List<Resource> perform(ReadGraph graph) throws DatabaseException {
920 CollectionSupport cs = graph.getService(CollectionSupport.class);
921 SerialisationSupport support = graph.getService(SerialisationSupport.class);
923 List<Resource> result = cs.createList();
925 ResourceVisitor visitor = new ResourceVisitor();
927 for (ScoreDoc scoreDoc : docs.scoreDocs) {
931 reader.document(scoreDoc.doc, visitor);
932 result.add(support.getResource(visitor.id));
934 } catch (CorruptIndexException e) {
935 throw new DatabaseException(e);
936 } catch (IOException e) {
937 throw new DatabaseException(e);
948 List<Object> doList(IProgressMonitor monitor, RequestProcessor processor) throws ParseException, IOException,
951 assertAccessOpen(false);
953 Query query = new MatchAllDocsQuery();
955 final TopDocs docs = searcher.search(query, Integer.MAX_VALUE);
957 ArrayList<Object> result = new ArrayList<Object>();
959 DumpVisitor visitor = new DumpVisitor(result);
961 for (ScoreDoc scoreDoc : docs.scoreDocs) {
965 reader.document(scoreDoc.doc, visitor);
967 } catch (CorruptIndexException e) {
968 throw new DatabaseException(e);
969 } catch (IOException e) {
970 throw new DatabaseException(e);
979 protected static File getIndexDirectory(Session session, Resource relation, Resource input) {
980 File path = DatabaseIndexing.getIndexLocation(session, relation, input);
981 // System.out.println("getIndexDirectory = " + path);
985 private static void createDirectory(File path) throws IOException {
986 Path p = path.toPath();
987 if (Files.exists(p) && !Files.isDirectory(p))
988 throw new IOException("Could not create index directory " + path + ", a file by that name already exists");
989 Files.createDirectories(p);
992 File getIndexPath() {
996 boolean isIndexAvailable() {
997 return (indexPath.exists() && indexPath.isDirectory());
1000 Throwable bestEffortClear(IProgressMonitor monitor, Session session) {
1005 * Start from scratch. Clear all caches and rebuild the index.
1007 Throwable clearDirectory(IProgressMonitor monitor, Session session) {
1009 File file = getIndexPath();
1013 for(int i=0;i<15;i++) {
1014 FileUtils.deleteDir(file);
1015 if(!file.exists()) {
1019 Thread.sleep(i*100);
1020 } catch (InterruptedException e) {
1024 } catch (Throwable t) {
1030 return new IllegalStateException("Failed to delete directory " + file.getAbsolutePath());
1034 private Field[] setFields(Field[] fs, Object[] result) {
1035 for (int i = 0; i < result.length; i++) {
1036 Object value = result[i];
1037 if (value instanceof String) {
1038 if (IndexPolicy.DEBUG_INDEX_INIT)
1039 System.out.println(getDescriptor() + "index " + fs[i].name() + " = " + value + " : String");
1040 fs[i].setStringValue((String) value);
1041 } else if (value instanceof Long) {
1042 if (IndexPolicy.DEBUG_INDEX_INIT)
1043 System.out.println(getDescriptor() + "index " + fs[i].name() + " = " + value + " : Long");
1044 fs[i].setLongValue((Long) value);
1046 Logger.defaultLogError("Can only index Long and String fields, encountered " + value);