bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsSearcherBase.java

   1 /*******************************************************************************
   2  * Copyright (c) 2007, 2015 Association for Decentralized Information Management
   3  * in Industry THTH ry.
   4  * All rights reserved. This program and the accompanying materials
   5  * are made available under the terms of the Eclipse Public License v1.0
   6  * which accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *     VTT Technical Research Centre of Finland - initial API and implementation
  11  *     Semantum Oy - Fix for simantics issue #6053
  12  *******************************************************************************/
  13 package org.simantics.db.indexing;
  14
  15 import java.io.File;
  16 import java.io.IOException;
  17 import java.util.ArrayList;
  18 import java.util.Collection;
  19 import java.util.Collections;
  20 import java.util.Iterator;
  21 import java.util.List;
  22 import java.util.Map;
  23 import java.util.concurrent.ExecutorService;
  24 import java.util.concurrent.Executors;
  25 import java.util.concurrent.Semaphore;
  26 import java.util.concurrent.ThreadFactory;
  27 import java.util.concurrent.atomic.AtomicReference;
  28
  29 import org.apache.lucene.document.Document;
  30 import org.apache.lucene.document.DocumentStoredFieldVisitor;
  31 import org.apache.lucene.document.Field;
  32 import org.apache.lucene.document.FieldType;
  33 import org.apache.lucene.document.LongField;
  34 import org.apache.lucene.document.TextField;
  35 import org.apache.lucene.index.CorruptIndexException;
  36 import org.apache.lucene.index.DirectoryReader;
  37 import org.apache.lucene.index.FieldInfo;
  38 import org.apache.lucene.index.IndexNotFoundException;
  39 import org.apache.lucene.index.IndexReader;
  40 import org.apache.lucene.index.IndexWriter;
  41 import org.apache.lucene.index.IndexWriterConfig;
  42 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
  43 import org.apache.lucene.index.IndexableField;
  44 import org.apache.lucene.index.StoredFieldVisitor;
  45 import org.apache.lucene.index.Term;
  46 import org.apache.lucene.queryparser.classic.ParseException;
  47 import org.apache.lucene.search.IndexSearcher;
  48 import org.apache.lucene.search.MatchAllDocsQuery;
  49 import org.apache.lucene.search.Query;
  50 import org.apache.lucene.search.ScoreDoc;
  51 import org.apache.lucene.search.TermQuery;
  52 import org.apache.lucene.search.TopDocs;
  53 import org.apache.lucene.store.Directory;
  54 import org.apache.lucene.store.FSDirectory;
  55 import org.apache.lucene.util.Version;
  56 import org.eclipse.core.runtime.IProgressMonitor;
  57 import org.eclipse.core.runtime.SubMonitor;
  58 import org.simantics.databoard.util.ObjectUtils;
  59 import org.simantics.db.ReadGraph;
  60 import org.simantics.db.RequestProcessor;
  61 import org.simantics.db.Resource;
  62 import org.simantics.db.Session;
  63 import org.simantics.db.common.request.SafeName;
  64 import org.simantics.db.common.utils.Logger;
  65 import org.simantics.db.common.utils.NameUtils;
  66 import org.simantics.db.exception.DatabaseException;
  67 import org.simantics.db.indexing.internal.IndexingJob;
  68 import org.simantics.db.layer0.adapter.GenericRelation;
  69 import org.simantics.db.request.Read;
  70 import org.simantics.db.service.CollectionSupport;
  71 import org.simantics.db.service.SerialisationSupport;
  72 import org.simantics.utils.FileUtils;
  73 import org.simantics.utils.datastructures.Pair;
  74
  75 import gnu.trove.map.hash.THashMap;
  76
  77 /**
  78  * @author Tuukka Lehtonen
  79  * @author Antti Villberg
  80  */
  81 abstract public class IndexedRelationsSearcherBase {
  82
  83     protected enum State {
  84         // No index is available
  85         NONE,
  86         // An index is available, but there is a problem with it
  87         PROBLEM,
  88         // An index is available but no reader or writer is ready
  89         READY,
  90         // A reader is ready
  91         READ,
  92         // A writer (and a reader) is ready
  93         WRITE
  94     }
  95
  96     private State state = State.READY;
  97     private Throwable exception;
  98
  99     public Throwable getException() {
 100         return exception;
 101     }
 102
 103     public void setProblem(Throwable t) {
 104         this.state = State.PROBLEM;
 105         this.exception = t;
 106     }
 107
 108     public void setNone() {
 109         this.state = State.NONE;
 110     }
 111
 112     public void setReady() {
 113         this.state = State.READY;
 114     }
 115
 116     protected boolean checkState(State state) {
 117         return this.state == state;
 118     }
 119
 120     protected void assertState(State state) throws AssertionError {
 121
 122         if(this.state != state) throw new AssertionError("Illegal state, expected " + state.name() + " but was in " + this.state.name());
 123
 124     }
 125
 126     public void changeState(IProgressMonitor monitor, Session session, State state) {
 127         changeState(monitor, session, state, 0);
 128     }
 129
 130     protected void changeState(IProgressMonitor monitor, Session session, State state, int depth) {
 131
 132         if(this.state == state) return;
 133
 134         if (IndexPolicy.TRACE_INDEX_MANAGEMENT)
 135                 System.err.println("Index state " + this.state.name() + " => " + state.name() + " " + this);
 136
 137         // Check transitions
 138
 139         // Try to exit problem state
 140         if (State.PROBLEM == this.state && depth > 0) {
 141                 Throwable t = bestEffortClear(monitor, session);
 142                 if(t != null) {
 143                                 exception = t;
 144                                 return;
 145                 }
 146                 // Managed to get into initial state
 147                 this.state = State.NONE;
 148                 return;
 149         }
 150
 151         // Cannot move into read from no index
 152         if (State.NONE ==  this.state && State.READ == state) return;
 153         // Cannot move into write from no index
 154         if (State.NONE ==  this.state && State.WRITE == state) return;
 155
 156                 boolean success = false;
 157
 158         try {
 159
 160                 if (searcher != null) {
 161                         searcher = null;
 162                 }
 163                 if (reader != null) {
 164                         reader.close();
 165                         reader = null;
 166                 }
 167                         closeWriter(writer);
 168                         directory = null;
 169
 170                         success = true;
 171
 172                 // Enter new state
 173                 if (State.READ == state || State.WRITE == state) {
 174
 175                         success = false;
 176
 177                         boolean forWriting = State.WRITE == state;
 178
 179                         if (directory != null)
 180                                 throw new IllegalStateException(getDescriptor() + "Index already loaded");
 181
 182                         SubMonitor mon = SubMonitor.convert(monitor, 100);
 183
 184                         mon.beginTask("Loading index", 100);
 185
 186                         if (IndexPolicy.TRACE_INDEX_LOAD)
 187                     System.out.println(getDescriptor() + "Loading Lucene index from " + indexPath + " for " + (forWriting ? "writing" : "reading"));
 188
 189                 long start = System.nanoTime();
 190
 191                 directory = getDirectory(session);
 192
 193                 if (forWriting) {
 194                     // Never overwrite an index that is about to be loaded.
 195                     // TODO: could use OpenMode.CREATE_OR_APPEND but must test first
 196                     IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_4_9, Queries.getAnalyzer()).setOpenMode(OpenMode.APPEND);
 197                     try {
 198                         // FIXME: platform #4676
 199                         writer = new IndexWriter(directory, config);
 200                     } catch (IndexNotFoundException e) {
 201                         // There was no pre-existing index on disk. Create it now.
 202                         writer = new IndexWriter(directory, config.setOpenMode(OpenMode.CREATE));
 203                         writer.commit();
 204                     }
 205                     reader = DirectoryReader.open(directory);
 206                     searcher = new IndexSearcher(reader);
 207                 } else {
 208                     reader = DirectoryReader.open(directory);
 209                     searcher = new IndexSearcher(reader);
 210                 }
 211
 212                 long end = System.nanoTime();
 213
 214                 mon.worked(100);
 215
 216                 if (IndexPolicy.PERF_INDEX_LOAD) {
 217                     double time = (end - start) * 1e-6;
 218                     System.out.println(getDescriptor() + "Loaded Lucene index from " + indexPath + " for " + (forWriting ? "writing" : "reading") + " in " + time + " ms");
 219                 }
 220
 221                 success = true;
 222
 223                 }
 224
 225         } catch (Throwable t) {
 226
 227                 setProblem(t);
 228
 229         } finally {
 230
 231                 if(!success) {
 232                         this.state = State.PROBLEM;
 233                         changeState(monitor, session, State.NONE, depth+1);
 234                         return;
 235                 }
 236
 237         }
 238
 239         this.state = state;
 240
 241     }
 242
 243     public static final FieldType STRING_TYPE = new FieldType();
 244
 245     static {
 246       STRING_TYPE.setIndexed(true);
 247       STRING_TYPE.setStored(true);
 248       STRING_TYPE.setTokenized(true);
 249       STRING_TYPE.freeze();
 250     }
 251
 252     protected static Field makeField(String fieldName, String fieldClass) throws DatabaseException {
 253         switch (fieldClass) {
 254         case "Long":   return new LongField(fieldName, 0L, Field.Store.YES);
 255         case "String": return new Field    (fieldName, "", STRING_TYPE);
 256         case "Text":   return new TextField(fieldName, "", Field.Store.YES);
 257         default:
 258             throw new DatabaseException("Can only index Long, String and Text fields, encountered field type " + fieldClass);
 259         }
 260     }
 261
 262     protected static Field[] makeFieldsForRelation(GenericRelation r, int boundLength, Document document) throws DatabaseException {
 263         Pair<String, String>[] fields = r.getFields();
 264         Field[] fs = new Field[Math.max(0, fields.length - boundLength)];
 265         for (int i = boundLength; i < fields.length; i++) {
 266             Field f = makeField(fields[i].first, fields[i].second);
 267             fs[i - boundLength] = f;
 268             if (document != null)
 269                 document.add(f);
 270         }
 271         return fs;
 272     }
 273
 274     void insertIndex(IProgressMonitor monitor, GenericRelation r, int boundLength, Collection<Object[]> documentsData)
 275     throws CorruptIndexException, IOException, DatabaseException {
 276         assertAccessOpen(true);
 277
 278         if (IndexPolicy.TRACE_INDEX_UPDATE)
 279             System.out.println(getDescriptor() + "Inserting " + documentsData.size() + " documents into index at " + indexPath);
 280
 281         long start = 0, end = 0;
 282         if (IndexPolicy.PERF_INDEX_UPDATE)
 283             start = System.nanoTime();
 284
 285         try {
 286             Document document = new Document();
 287             Field[] fs = makeFieldsForRelation(r, boundLength, document);
 288
 289             for (Object[] documentData : documentsData) {
 290                 if (setFields(fs, documentData) == null)
 291                     continue;
 292
 293                 if (IndexPolicy.TRACE_INDEX_UPDATE)
 294                     System.out.println(getDescriptor() + "Inserting document " + document);
 295
 296                 writer.addDocument(document);
 297             }
 298
 299             if (IndexPolicy.PERF_INDEX_UPDATE) {
 300                 end = System.nanoTime();
 301                 double ms = (end - start) * 1e-6;
 302                 System.out.println(getDescriptor() + "Inserted " + documentsData.size() + " documents into index at " + indexPath + " in " + ms + " ms");
 303             }
 304
 305         } finally {
 306         }
 307     }
 308
 309     void removeIndex(IProgressMonitor monitor, GenericRelation r, RequestProcessor processor, String key, Collection<Object> keyValues) throws DatabaseException, CorruptIndexException, IOException {
 310         assertAccessOpen(true);
 311
 312         if (IndexPolicy.TRACE_INDEX_UPDATE)
 313             System.out.println(getDescriptor() + "Removing " + keyValues.size() + " documents from index at " + indexPath);
 314
 315         long start = 0, end = 0;
 316         if (IndexPolicy.PERF_INDEX_UPDATE)
 317             start = System.nanoTime();
 318
 319         try {
 320             for (Object keyValue : keyValues) {
 321                 Term removedTerm = null;
 322                 if (keyValue instanceof Long) {
 323                     removedTerm = IndexUtils.longTerm(key, (Long) keyValue);
 324                 } else if (keyValue instanceof String) {
 325                     removedTerm = new Term(key, (String) keyValue);
 326                 } else {
 327                     // FIXME: should throw an exception for illegal input data but this would leave the index in an incoherent state
 328                     continue;
 329                 }
 330
 331                 if (IndexPolicy.TRACE_INDEX_UPDATE)
 332                     System.out.println(getDescriptor() + "Removing document with key " + removedTerm);
 333                 writer.deleteDocuments(removedTerm);
 334             }
 335
 336             if (IndexPolicy.PERF_INDEX_UPDATE) {
 337                 end = System.nanoTime();
 338                 double ms = (end - start) * 1e-6;
 339                 System.out.println(getDescriptor() + "Removed " + keyValues.size() + " documents from index at " + indexPath + " in " + ms + " ms");
 340             }
 341
 342         } finally {
 343         }
 344     }
 345
 346     void removeIndex(IProgressMonitor monitor) throws DatabaseException, CorruptIndexException, IOException {
 347         assertAccessOpen(true);
 348
 349         long start = 0, end = 0;
 350         if (IndexPolicy.PERF_INDEX_UPDATE)
 351             start = System.nanoTime();
 352
 353         try {
 354
 355             writer.deleteAll();
 356
 357             if (IndexPolicy.PERF_INDEX_UPDATE) {
 358                 end = System.nanoTime();
 359                 double ms = (end - start) * 1e-6;
 360                 System.out.println(getDescriptor() + "Removed all documents from index at " + indexPath + " in " + ms + " ms");
 361             }
 362
 363         } finally {
 364         }
 365     }
 366
 367     boolean replaceIndex(IProgressMonitor monitor, String key, Collection<Object> keyValues, GenericRelation r, int boundLength, Collection<Object[]> documentsData) throws CorruptIndexException, IOException, DatabaseException {
 368
 369         boolean didReplace = false;
 370
 371         assertAccessOpen(true);
 372         if (keyValues.size() != documentsData.size())
 373             throw new IllegalArgumentException("keyValues size does not match documents data size, " + keyValues.size() + " <> " + documentsData.size());
 374
 375         if (IndexPolicy.TRACE_INDEX_UPDATE)
 376             System.out.println(getDescriptor() + "Replacing " + keyValues.size() + " documents from index at " + indexPath);
 377
 378         long start = 0, end = 0;
 379         if (IndexPolicy.PERF_INDEX_UPDATE)
 380             start = System.nanoTime();
 381
 382         try {
 383             Iterator<Object> keyIt = keyValues.iterator();
 384             Iterator<Object[]> documentDataIt = documentsData.iterator();
 385
 386             Document document = new Document();
 387             Field[] fs = makeFieldsForRelation(r, boundLength, document);
 388
 389             nextDocument:
 390                 while (keyIt.hasNext()) {
 391                     Object keyValue = keyIt.next();
 392                     Object[] documentData = documentDataIt.next();
 393
 394                     Term removedTerm = null;
 395                     if (keyValue instanceof Long) {
 396                         removedTerm = IndexUtils.longTerm(key, (Long) keyValue);
 397                     } else if (keyValue instanceof String) {
 398                         removedTerm = new Term(key, (String) keyValue);
 399                     } else {
 400                         // FIXME: should throw an exception for illegal input data but this would leave the index in an incoherent state
 401                         System.err.println("[" + getClass().getSimpleName() + "] Unrecognized document key to remove '" + keyValue + "', only " + String.class + " and " + Resource.class + " are supported.");
 402                         continue nextDocument;
 403                     }
 404
 405                     if (setFields(fs, documentData) == null)
 406                         continue nextDocument;
 407
 408                     if (IndexPolicy.TRACE_INDEX_UPDATE)
 409                         System.out.println(getDescriptor() + "Replacing document with key " + removedTerm + " with " + document);
 410
 411                     boolean done = false;
 412                     if(requireChangeInfoOnReplace()) {
 413                             TopDocs exist = searcher.search(new TermQuery(removedTerm), null, 2);
 414                             if(exist.scoreDocs.length == 1 && requireChangeInfoOnReplace()) {
 415                                 Document doc = reader.document(exist.scoreDocs[0].doc);
 416                                 if(!areSame(doc, document)) {
 417                                     writer.deleteDocuments(removedTerm);
 418                                     writer.addDocument(document);
 419                                     didReplace |= true;
 420                                     if (IndexPolicy.TRACE_INDEX_UPDATE)
 421                                         System.out.println("-replaced single existing");
 422                                 } else {
 423                                     if (IndexPolicy.TRACE_INDEX_UPDATE)
 424                                         System.out.println("-was actually same than single existing");
 425                                 }
 426                                 done = true;
 427                             }
 428                     }
 429                     if(!done) {
 430                         writer.deleteDocuments(removedTerm);
 431                         writer.addDocument(document);
 432                         didReplace |= true;
 433                         if (IndexPolicy.TRACE_INDEX_UPDATE)
 434                                 System.out.println("-had many or none - removed all existing");
 435                     }
 436
 437                 }
 438
 439             if (IndexPolicy.PERF_INDEX_UPDATE) {
 440                 end = System.nanoTime();
 441                 double ms = (end - start) * 1e-6;
 442                 System.out.println(getDescriptor() + "Replaced " + keyValues.size() + " documents from index at " + indexPath + " in " + ms + " ms");
 443             }
 444
 445         } finally {
 446         }
 447
 448         return didReplace;
 449
 450     }
 451
 452     protected boolean requireChangeInfoOnReplace() {
 453         return true;
 454     }
 455
 456     private boolean areSame(Document d1, Document d2) {
 457         List<IndexableField> fs1 = d1.getFields();
 458         List<IndexableField> fs2 = d2.getFields();
 459         if(fs1.size() != fs2.size()) return false;
 460         for(int i=0;i<fs1.size();i++) {
 461                 IndexableField f1 = fs1.get(i);
 462                 IndexableField f2 = fs2.get(i);
 463                 String s1 = f1.stringValue();
 464                 String s2 = f2.stringValue();
 465             if (IndexPolicy.TRACE_INDEX_UPDATE)
 466                 System.err.println("areSame " + s1 + " vs. " + s2 );
 467                 if(!ObjectUtils.objectEquals(s1,s2)) return false;
 468         }
 469         return true;
 470     }
 471
 472     final RequestProcessor session;
 473
 474     final Resource         relation;
 475
 476     /**
 477      * The schema of the index, i.e. the fields that will be indexed per
 478      * document for the specified relation. Since the relation stays the same
 479      * throughout the lifetime of this class, the index schema is also assumed
 480      * to the same. This means that {@link GenericRelation#getFields()} is
 481      * assumed to stay the same.
 482      */
 483     final IndexSchema      schema;
 484
 485     Resource         input;
 486
 487     File             indexPath;
 488
 489     Directory        directory;
 490
 491     IndexReader      reader;
 492
 493     IndexWriter      writer;
 494
 495     IndexSearcher    searcher;
 496
 497     IndexedRelationsSearcherBase(RequestProcessor session, Resource relation, Resource input) {
 498         this.session = session;
 499         this.relation = relation;
 500         this.input = input;
 501         this.indexPath = getIndexDirectory(session.getSession(), relation, input);
 502         if(isIndexAvailable()) {
 503                 state = State.READY;
 504         } else {
 505                 state = State.NONE;
 506         }
 507         this.schema = IndexSchema.readFromRelation(session, relation);
 508     }
 509
 510     Directory getDirectory(Session session) throws IOException {
 511         return FSDirectory.open(indexPath);
 512     }
 513
 514     abstract String getDescriptor();
 515
 516     /**
 517      * Ensures that searcher is in read or write state.
 518      *
 519      * @param forWriting <code>true</code> to open index for writing,
 520      *        <code>false</code> for reading
 521      * @return true is required state was reached
 522      *
 523      */
 524     boolean startAccess(IProgressMonitor monitor, Session session, boolean forWriting) {
 525         if(forWriting) {
 526                 changeState(monitor, session, State.WRITE);
 527                 return checkState(State.WRITE);
 528         } else {
 529                 changeState(monitor, session, State.READ);
 530                 return checkState(State.READ);
 531         }
 532     }
 533
 534     boolean hasAccess(boolean forWriting) {
 535
 536         if (forWriting)
 537                 return checkState(State.WRITE);
 538         else
 539                 return checkState(State.WRITE) || checkState(State.READ);
 540
 541     }
 542
 543     void assertAccessOpen(boolean forWriting) {
 544         if (forWriting)
 545                 if(!checkState(State.WRITE))
 546                 throw new IllegalStateException("index not opened for writing (directory=" + directory + ", reader=" + reader + ")");
 547         else
 548                 if(!(checkState(State.WRITE) || checkState(State.READ)))
 549                 throw new IllegalStateException("index not opened for reading (directory=" + directory + ", writer=" + writer + ")");
 550     }
 551
 552     void closeWriter(IndexWriter writer) throws CorruptIndexException, IOException {
 553         if (writer == null)
 554             return;
 555
 556         try {
 557             // May throw OOME, see IndexWriter javadoc for the correct actions.
 558             writer.close(false);
 559         } catch (OutOfMemoryError e) {
 560             writer.close();
 561             throw e;
 562         }
 563     }
 564
 565     private static String getPattern(GenericRelation relation, int boundCount) {
 566         String result = "";
 567         for (int i = 0; i < boundCount; i++)
 568             result += "b";
 569         for (int i = 0; i < relation.getFields().length - boundCount; i++)
 570             result += "f";
 571         return result;
 572     }
 573
 574     private static final int INDEXING_THREAD_COUNT = 2;
 575
 576     private static final ExecutorService executor = Executors.newFixedThreadPool(INDEXING_THREAD_COUNT, new ThreadFactory() {
 577         @Override
 578         public Thread newThread(Runnable r) {
 579             Thread t = new Thread(r, "Lucene Index Creator");
 580             if (!t.isDaemon())
 581                 t.setDaemon(true);
 582             if (t.getPriority() != Thread.NORM_PRIORITY)
 583                 t.setPriority(Thread.NORM_PRIORITY);
 584             return t;
 585         }
 586     });
 587
 588     void initializeIndex(IProgressMonitor monitor, ReadGraph graph, Object[] bound, boolean overwrite)
 589             throws IOException, DatabaseException
 590     {
 591         IndexingJob.jobifyIfPossible(
 592                 monitor,
 593                 "Reindexing " + NameUtils.getSafeLabel(graph, input),
 594                 mon -> {
 595                     try {
 596                         initializeIndexImpl(mon, graph, bound, overwrite);
 597                     } catch (IOException e) {
 598                         throw new DatabaseException(e);
 599                     }
 600                 });
 601     }
 602
 603     void initializeIndexImpl(IProgressMonitor monitor, ReadGraph graph, final Object[] bound, boolean overwrite) throws IOException,
 604     DatabaseException {
 605
 606         final SubMonitor mon = SubMonitor.convert(monitor, 100);
 607
 608         if (IndexPolicy.TRACE_INDEX_INIT)
 609             System.out.println(getDescriptor() + "Initializing index at " + indexPath + " (overwrite = " + overwrite + ")");
 610         mon.beginTask("Initializing Index", 100);
 611
 612         if (overwrite) {
 613             mon.subTask("Erasing previous index");
 614             FileUtils.deleteAll(indexPath);
 615         }
 616
 617         final AtomicReference<FSDirectory> directory = new AtomicReference<FSDirectory>();
 618         final AtomicReference<IndexWriter> writer = new AtomicReference<IndexWriter>();
 619
 620         try {
 621             mon.subTask("Start index write");
 622             createDirectory(indexPath);
 623
 624             directory.set(FSDirectory.open(indexPath));
 625             IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_4_9, Queries.getAnalyzer()).setOpenMode(OpenMode.CREATE);
 626             writer.set(new IndexWriter(directory.get(), conf));
 627
 628             mon.worked(5);
 629
 630             final GenericRelation r = graph.adapt(relation, GenericRelation.class);
 631             if (r == null)
 632                 throw new DatabaseException("Given resource " + graph.syncRequest(new SafeName(relation))
 633                 + "could not be adapted to GenericRelation.");
 634
 635             long realizeStart = 0;
 636             if (IndexPolicy.PERF_INDEX_INIT)
 637                 realizeStart = System.nanoTime();
 638
 639             mon.subTask("Calculating indexed content");
 640             GenericRelation selection = r.select(getPattern(r, bound.length), bound);
 641             mon.worked(5);
 642             List<Object[]> results = selection.realize(graph);
 643             mon.worked(40);
 644
 645             if (IndexPolicy.PERF_INDEX_INIT)
 646                 System.out.println(getDescriptor() + "Realized index with " + results.size() + " entries at " + indexPath + " in " + (1e-9 * (System.nanoTime()-realizeStart)) + " seconds.");
 647             if (IndexPolicy.TRACE_INDEX_INIT)
 648                 System.out.println(getDescriptor() + "Indexed relation " + r + " produced " + results.size() + " results");
 649
 650             long start = IndexPolicy.PERF_INDEX_INIT ? System.nanoTime() : 0;
 651
 652             mon.subTask("Indexing content");
 653             final Semaphore s = new Semaphore(0);
 654             mon.setWorkRemaining(results.size());
 655
 656             for (int i = 0; i < INDEXING_THREAD_COUNT; i++) {
 657                 final int startIndex = i;
 658                 executor.submit(() -> {
 659                     try {
 660                         Document document = new Document();
 661                         Field[] fs = makeFieldsForRelation(r, bound.length, document);
 662
 663                         for (int index = startIndex; index < results.size(); index += INDEXING_THREAD_COUNT) {
 664                             if (setFields(fs, results.get(index)) == null)
 665                                 continue;
 666                             try {
 667                                 writer.get().addDocument(document);
 668                             } catch (CorruptIndexException e) {
 669                                 throw new IllegalStateException(e);
 670                             } catch (IOException e) {
 671                                 throw new IllegalStateException(e);
 672                             } finally {
 673                                 synchronized (mon) {
 674                                     mon.worked(1);
 675                                 }
 676                             }
 677                         }
 678
 679                         s.release();
 680                     } catch (DatabaseException e) {
 681                         throw new IllegalStateException(e);
 682                     }
 683                 });
 684             }
 685
 686             try {
 687                 s.acquire(INDEXING_THREAD_COUNT);
 688             } catch (InterruptedException e) {
 689                 Logger.defaultLogError(e);
 690             }
 691
 692             // http://www.gossamer-threads.com/lists/lucene/java-dev/47895
 693             // and http://lucene.apache.org/java/docs/index.html#27+November+2011+-+Lucene+Core+3.5.0
 694             // advise against calling optimize at all. So let's not do it anymore.
 695             //writer.get().optimize();
 696             //writer.get().commit();
 697
 698             mon.subTask("Flushing");
 699
 700             if (IndexPolicy.PERF_INDEX_INIT)
 701                 System.out.println(getDescriptor() + "Wrote index at " + indexPath + " in " + (1e-9 * (System.nanoTime()-start)) + " seconds.");
 702
 703         } catch (DatabaseException e) {
 704
 705             Logger.defaultLogError(e);
 706
 707         } finally {
 708             try {
 709                 closeWriter(writer.getAndSet(null));
 710             } finally {
 711                 FileUtils.uncheckedClose(directory.getAndSet(null));
 712             }
 713         }
 714     }
 715
 716
 717     public List<Object[]> debugDocs(IProgressMonitor monitor) throws ParseException, IOException, DatabaseException {
 718
 719             Query query = new MatchAllDocsQuery();
 720
 721             TopDocs td = searcher.search(query, Integer.MAX_VALUE);
 722
 723             ScoreDoc[ ] scoreDocs = td.scoreDocs;
 724             List<Object[]> result = new ArrayList<Object[]>(scoreDocs.length);
 725
 726             for(ScoreDoc scoreDoc:scoreDocs) {
 727
 728                 try {
 729
 730                     Document doc = reader.document(scoreDoc.doc);
 731                     List<IndexableField> fs = doc.getFields();
 732                     Object[] o = new Object[fs.size()];
 733                     int index = 0;
 734                     for (IndexableField f : fs) {
 735                     o[index++] = f.stringValue();
 736                     }
 737                     result.add(o);
 738
 739                 } catch (CorruptIndexException e) {
 740                     throw new DatabaseException(e);
 741                 } catch (IOException e) {
 742                     throw new DatabaseException(e);
 743                 }
 744
 745             }
 746
 747             return result;
 748
 749     }
 750
 751
 752     List<Map<String, Object>> doSearch(IProgressMonitor monitor, RequestProcessor processor, String search, int maxResultCount) throws ParseException, IOException,
 753     DatabaseException {
 754
 755         // An empty search string will crash QueryParser
 756         // Just return no results for empty queries.
 757         //System.out.println("search: '" + search + "'");
 758         if (search.isEmpty())
 759             return Collections.emptyList();
 760
 761         assertAccessOpen(false);
 762
 763         Query query = Queries.parse(search, schema);
 764
 765         long start = System.nanoTime();
 766
 767         maxResultCount = Math.min(maxResultCount, searcher.getIndexReader().numDocs());
 768         if (maxResultCount == 0)
 769             return Collections.emptyList();
 770
 771         final TopDocs docs = searcher.search(query, null, maxResultCount);
 772
 773 //        for(Object[] o : debugDocs(monitor)) {
 774 //            System.err.println("-" + Arrays.toString(o));
 775 //        }
 776
 777         if (IndexPolicy.PERF_INDEX_QUERY) {
 778             long end = System.nanoTime();
 779             System.out.println(getDescriptor() + "search(" + search + ", " + maxResultCount + ") into index at " + indexPath + " took " + (1e-9 * (end-start)) + " seconds.");
 780         }
 781
 782         if (docs.totalHits == 0) {
 783             return Collections.emptyList();
 784         }
 785
 786         return processor.syncRequest(new Read<List<Map<String, Object>>>() {
 787
 788             @Override
 789             public List<Map<String, Object>> perform(ReadGraph graph) throws DatabaseException {
 790
 791                 GenericRelation r = graph.adapt(relation, GenericRelation.class);
 792                 if (r == null)
 793                     throw new DatabaseException("Given resource " + graph.syncRequest(new SafeName(relation))
 794                             + "could not be adapted to GenericRelation.");
 795
 796                 SerialisationSupport support = graph.getService(SerialisationSupport.class);
 797
 798                 List<Map<String, Object>> result = new ArrayList<Map<String, Object>>(docs.scoreDocs.length);
 799
 800                 final DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor();
 801
 802                 for (ScoreDoc scoreDoc : docs.scoreDocs) {
 803
 804                     try {
 805
 806                         reader.document(scoreDoc.doc, visitor);
 807
 808                         Document doc = visitor.getDocument();
 809
 810                         List<IndexableField> fs = doc.getFields();
 811                         Map<String, Object> entry = new THashMap<String, Object>(fs.size());
 812                         for (IndexableField f : fs) {
 813                             IndexSchema.Type type = schema.typeMap.get(f.name());
 814                             if (type == IndexSchema.Type.LONG) {
 815                                 entry.put(f.name(), support.getResource(f.numericValue().longValue()));
 816                             } else {
 817                                 entry.put(f.name(), f.stringValue());
 818                             }
 819                         }
 820
 821                         result.add(entry);
 822
 823                     } catch (CorruptIndexException e) {
 824                         throw new DatabaseException(e);
 825                     } catch (IOException e) {
 826                         throw new DatabaseException(e);
 827                     }
 828
 829                 }
 830
 831                 return result;
 832
 833             }
 834         });
 835     }
 836
 837     static class ResourceVisitor extends StoredFieldVisitor {
 838
 839         public long id;
 840
 841                 @Override
 842                 public Status needsField(FieldInfo fieldInfo) throws IOException {
 843                         if("Resource".equals(fieldInfo.name)) return Status.YES;
 844                         return Status.NO;
 845                 }
 846
 847                 @Override
 848                 public void longField(FieldInfo fieldInfo, long value) throws IOException {
 849                         id = value;
 850                 }
 851
 852     };
 853
 854     static class DumpVisitor extends StoredFieldVisitor {
 855
 856         public List<Object> values;
 857
 858         DumpVisitor(List<Object> values) {
 859                 this.values = values;
 860         }
 861
 862                 @Override
 863                 public Status needsField(FieldInfo fieldInfo) throws IOException {
 864                         return Status.YES;
 865                 }
 866
 867                 @Override
 868                 public void longField(FieldInfo fieldInfo, long value) throws IOException {
 869                         values.add(value);
 870                 }
 871
 872                 @Override
 873                 public void stringField(FieldInfo fieldInfo, String value) throws IOException {
 874                         values.add(value);
 875                 }
 876
 877     }
 878
 879     List<Resource> doSearchResources(IProgressMonitor monitor, RequestProcessor processor, String search, int maxResultCount) throws ParseException, IOException,
 880     DatabaseException {
 881
 882         // An empty search string will crash QueryParser
 883         // Just return no results for empty queries.
 884         //System.out.println("search: '" + search + "'");
 885         if (search.isEmpty())
 886             return Collections.emptyList();
 887
 888         assertAccessOpen(false);
 889
 890         Query query = Queries.parse(search, schema);
 891
 892         long start = System.nanoTime();
 893
 894         maxResultCount = Math.min(maxResultCount, searcher.getIndexReader().numDocs());
 895         if (maxResultCount == 0)
 896             return Collections.emptyList();
 897
 898         final TopDocs docs = searcher.search(query, null, maxResultCount);
 899
 900 //        for(Object[] o : debugDocs(monitor)) {
 901 //            System.err.println("-" + Arrays.toString(o));
 902 //        }
 903
 904         if (IndexPolicy.PERF_INDEX_QUERY) {
 905             long end = System.nanoTime();
 906             System.out.println(getDescriptor() + "search(" + search + ", " + maxResultCount + ") into index at " + indexPath + " took " + (1e-9 * (end-start)) + " seconds.");
 907         }
 908
 909         if (docs.totalHits == 0) {
 910             return Collections.emptyList();
 911         }
 912
 913         return processor.syncRequest(new Read<List<Resource>>() {
 914
 915             @Override
 916             public List<Resource> perform(ReadGraph graph) throws DatabaseException {
 917
 918                 CollectionSupport cs = graph.getService(CollectionSupport.class);
 919                 SerialisationSupport support = graph.getService(SerialisationSupport.class);
 920
 921                 List<Resource> result = cs.createList();
 922
 923                 ResourceVisitor visitor = new ResourceVisitor();
 924
 925                 for (ScoreDoc scoreDoc : docs.scoreDocs) {
 926
 927                     try {
 928
 929                         reader.document(scoreDoc.doc, visitor);
 930                         result.add(support.getResource(visitor.id));
 931
 932                     } catch (CorruptIndexException e) {
 933                         throw new DatabaseException(e);
 934                     } catch (IOException e) {
 935                         throw new DatabaseException(e);
 936                     }
 937
 938                 }
 939
 940                 return result;
 941
 942             }
 943         });
 944     }
 945
 946     List<Object> doList(IProgressMonitor monitor, RequestProcessor processor) throws ParseException, IOException,
 947     DatabaseException {
 948
 949         assertAccessOpen(false);
 950
 951         Query query = new MatchAllDocsQuery();
 952
 953         final TopDocs docs = searcher.search(query, Integer.MAX_VALUE);
 954
 955         ArrayList<Object> result = new ArrayList<Object>();
 956
 957         DumpVisitor visitor = new DumpVisitor(result);
 958
 959         for (ScoreDoc scoreDoc : docs.scoreDocs) {
 960
 961                 try {
 962
 963                         reader.document(scoreDoc.doc, visitor);
 964
 965                 } catch (CorruptIndexException e) {
 966                         throw new DatabaseException(e);
 967                 } catch (IOException e) {
 968                         throw new DatabaseException(e);
 969                 }
 970
 971         }
 972
 973         return result;
 974
 975     }
 976
 977     protected static File getIndexDirectory(Session session, Resource relation, Resource input) {
 978         File path = DatabaseIndexing.getIndexLocation(session, relation, input);
 979 //        System.out.println("getIndexDirectory = " + path);
 980         return path;
 981     }
 982
 983     private static void createDirectory(File path) throws IOException {
 984         if (path.exists() && !path.isDirectory())
 985             throw new IOException("Could not create index directory " + path + ", a file by that name already exists");
 986         path.mkdirs();
 987         if (!path.exists())
 988             throw new IOException("Could not create index directory " + path + " for an unknown reason");
 989         if (!path.isDirectory())
 990             throw new IOException("Could not create index directory " + path + ", a file by that name already exists");
 991     }
 992
 993     File getIndexPath() {
 994         return indexPath;
 995     }
 996
 997     boolean isIndexAvailable() {
 998         return (indexPath.exists() && indexPath.isDirectory());
 999     }
1000
1001     Throwable bestEffortClear(IProgressMonitor monitor, Session session) {
1002         return null;
1003     }
1004
1005     /*
1006      * Start from scratch. Clear all caches and rebuild the index.
1007      */
1008     Throwable clearDirectory(IProgressMonitor monitor, Session session) {
1009
1010                 File file = getIndexPath();
1011
1012         try {
1013
1014                 for(int i=0;i<15;i++) {
1015                         FileUtils.deleteDir(file);
1016                         if(!file.exists()) {
1017                                 return null;
1018                         }
1019                         try {
1020                                 Thread.sleep(i*100);
1021                         } catch (InterruptedException e) {
1022                         }
1023                 }
1024
1025         } catch (Throwable t) {
1026
1027                 return t;
1028
1029         }
1030
1031         return new IllegalStateException("Failed to delete directory " + file.getAbsolutePath());
1032
1033     }
1034
1035     private Field[] setFields(Field[] fs, Object[] result) {
1036         for (int i = 0; i < result.length; i++) {
1037             Object value = result[i];
1038             if (value instanceof String) {
1039                 if (IndexPolicy.DEBUG_INDEX_INIT)
1040                     System.out.println(getDescriptor() + "index " + fs[i].name() + " = " + value + " : String");
1041                 fs[i].setStringValue((String) value);
1042             } else if (value instanceof Long) {
1043                 if (IndexPolicy.DEBUG_INDEX_INIT)
1044                     System.out.println(getDescriptor() + "index " + fs[i].name() + " = " + value + " : Long");
1045                 fs[i].setLongValue((Long) value);
1046             } else {
1047                 Logger.defaultLogError("Can only index Long and String fields, encountered " + value);
1048                 return null;
1049             }
1050         }
1051         return fs;
1052     }
1053
1054 }