bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsSearcherBase.java

   1 /*******************************************************************************
   2  * Copyright (c) 2007, 2015 Association for Decentralized Information Management
   3  * in Industry THTH ry.
   4  * All rights reserved. This program and the accompanying materials
   5  * are made available under the terms of the Eclipse Public License v1.0
   6  * which accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *     VTT Technical Research Centre of Finland - initial API and implementation
  11  *     Semantum Oy - Fix for simantics issue #6053
  12  *******************************************************************************/
  13 package org.simantics.db.indexing;
  14
  15 import java.io.IOException;
  16 import java.nio.file.Files;
  17 import java.nio.file.Path;
  18 import java.util.ArrayList;
  19 import java.util.Collection;
  20 import java.util.Collections;
  21 import java.util.Iterator;
  22 import java.util.List;
  23 import java.util.Map;
  24 import java.util.concurrent.CompletableFuture;
  25 import java.util.concurrent.Semaphore;
  26 import java.util.concurrent.atomic.AtomicReference;
  27
  28 import org.apache.lucene.document.Document;
  29 import org.apache.lucene.document.DocumentStoredFieldVisitor;
  30 import org.apache.lucene.document.Field;
  31 import org.apache.lucene.document.FieldType;
  32 import org.apache.lucene.document.LongField;
  33 import org.apache.lucene.document.TextField;
  34 import org.apache.lucene.index.CorruptIndexException;
  35 import org.apache.lucene.index.DirectoryReader;
  36 import org.apache.lucene.index.FieldInfo;
  37 import org.apache.lucene.index.IndexNotFoundException;
  38 import org.apache.lucene.index.IndexReader;
  39 import org.apache.lucene.index.IndexWriter;
  40 import org.apache.lucene.index.IndexWriterConfig;
  41 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
  42 import org.apache.lucene.index.IndexableField;
  43 import org.apache.lucene.index.StoredFieldVisitor;
  44 import org.apache.lucene.index.Term;
  45 import org.apache.lucene.queryparser.classic.ParseException;
  46 import org.apache.lucene.search.IndexSearcher;
  47 import org.apache.lucene.search.MatchAllDocsQuery;
  48 import org.apache.lucene.search.Query;
  49 import org.apache.lucene.search.ScoreDoc;
  50 import org.apache.lucene.search.TermQuery;
  51 import org.apache.lucene.search.TopDocs;
  52 import org.apache.lucene.store.Directory;
  53 import org.apache.lucene.store.FSDirectory;
  54 import org.apache.lucene.util.Version;
  55 import org.eclipse.core.runtime.IProgressMonitor;
  56 import org.eclipse.core.runtime.SubMonitor;
  57 import org.simantics.databoard.util.ObjectUtils;
  58 import org.simantics.db.ReadGraph;
  59 import org.simantics.db.RequestProcessor;
  60 import org.simantics.db.Resource;
  61 import org.simantics.db.Session;
  62 import org.simantics.db.common.request.SafeName;
  63 import org.simantics.db.common.utils.NameUtils;
  64 import org.simantics.db.exception.DatabaseException;
  65 import org.simantics.db.indexing.internal.IndexingJob;
  66 import org.simantics.db.layer0.adapter.GenericRelation;
  67 import org.simantics.db.request.Read;
  68 import org.simantics.db.service.CollectionSupport;
  69 import org.simantics.db.service.SerialisationSupport;
  70 import org.simantics.utils.FileUtils;
  71 import org.simantics.utils.datastructures.Pair;
  72 import org.simantics.utils.threads.ThreadUtils;
  73 import org.slf4j.Logger;
  74
  75 import gnu.trove.map.hash.THashMap;
  76
  77 /**
  78  * @author Tuukka Lehtonen
  79  * @author Antti Villberg
  80  */
  81 abstract public class IndexedRelationsSearcherBase {
  82
  83     protected enum State {
  84         // No index is available
  85         NONE,
  86         // An index is available, but there is a problem with it
  87         PROBLEM,
  88         // An index is available but no reader or writer is ready
  89         READY,
  90         // A reader is ready
  91         READ,
  92         // A writer (and a reader) is ready
  93         WRITE
  94     }
  95
  96     private State state = State.READY;
  97     private Throwable exception;
  98
  99     public Throwable getException() {
 100         return exception;
 101     }
 102
 103     public void setProblem(Throwable t) {
 104         if (t != null)
 105             getLogger().error("Setting problem for {} and previous state {}", this, this.state, t);
 106         this.state = State.PROBLEM;
 107         this.exception = t;
 108     }
 109
 110     public void setNone() {
 111         this.state = State.NONE;
 112     }
 113
 114     public void setReady() {
 115         this.state = State.READY;
 116     }
 117
 118     protected boolean checkState(State state) {
 119         return this.state == state;
 120     }
 121
 122     protected void assertState(State state) throws AssertionError {
 123
 124         if(this.state != state) throw new AssertionError("Illegal state, expected " + state.name() + " but was in " + this.state.name());
 125
 126     }
 127
 128     public void changeState(IProgressMonitor monitor, Session session, State state) {
 129         changeState(monitor, session, state, 0);
 130     }
 131
 132     protected void changeState(IProgressMonitor monitor, Session session, State state, int depth) {
 133
 134         if (this.state == state) {
 135             if (getLogger().isDebugEnabled())
 136                 getLogger().debug("Trying to change state {} to the same as previous state {} in depth {} with {}", state, this.state, depth, this);
 137             return;
 138         }
 139
 140         if (IndexPolicy.TRACE_INDEX_MANAGEMENT)
 141                 System.err.println("Index state " + this.state.name() + " => " + state.name() + " " + this);
 142
 143         // Check transitions
 144
 145         // Try to exit problem state
 146         if (State.PROBLEM == this.state && depth > 0) {
 147             getLogger().info("Try to exit problem state for {} and state {}", this, state);
 148                 Throwable t = bestEffortClear(monitor, session);
 149                 if(t != null) {
 150                     getLogger().error("Best effort clear has failed for state {} and this {}", state, this, t);
 151                                 exception = t;
 152                                 return;
 153                 }
 154                 // Managed to get into initial state
 155                 this.state = State.NONE;
 156                 getLogger().info("Managed to get into initial state {}", this.state);
 157                 return;
 158         }
 159
 160         // Cannot move into read from no index
 161         if (State.NONE ==  this.state && State.READ == state) {
 162             if (getLogger().isDebugEnabled())
 163                 getLogger().debug("Cannot move into read from no index in {} with state {}", this, state);
 164             return;
 165         }
 166         // Cannot move into write from no index
 167         if (State.NONE ==  this.state && State.WRITE == state) {
 168             if (getLogger().isDebugEnabled())
 169                 getLogger().debug("Cannot move into write from no index in {} with state {}", this, state);
 170             return;
 171         }
 172
 173                 boolean success = false;
 174
 175         try {
 176
 177                 if (searcher != null) {
 178                         searcher = null;
 179                 }
 180                 if (reader != null) {
 181                         reader.close();
 182                         reader = null;
 183                 }
 184                         closeWriter(writer);
 185                         directory = null;
 186
 187                         success = true;
 188
 189                 // Enter new state
 190                 if (State.READ == state || State.WRITE == state) {
 191
 192                         success = false;
 193
 194                         boolean forWriting = State.WRITE == state;
 195
 196                         if (directory != null)
 197                                 throw new IllegalStateException(getDescriptor() + "Index already loaded");
 198
 199                         SubMonitor mon = SubMonitor.convert(monitor, 100);
 200
 201                         mon.beginTask("Loading index", 100);
 202
 203                         if (IndexPolicy.TRACE_INDEX_LOAD)
 204                     System.out.println(getDescriptor() + "Loading Lucene index from " + indexPath + " for " + (forWriting ? "writing" : "reading"));
 205
 206                 long start = System.nanoTime();
 207
 208                 directory = getDirectory(session);
 209
 210                 if (forWriting) {
 211                     // Never overwrite an index that is about to be loaded.
 212                     // TODO: could use OpenMode.CREATE_OR_APPEND but must test first
 213                     IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_4_9, Queries.getAnalyzer()).setOpenMode(OpenMode.APPEND);
 214                     try {
 215                         // FIXME: platform #4676
 216                         writer = new IndexWriter(directory, config);
 217                     } catch (IndexNotFoundException e) {
 218                         // There was no pre-existing index on disk. Create it now.
 219                         writer = new IndexWriter(directory, config.setOpenMode(OpenMode.CREATE));
 220                         writer.commit();
 221                     }
 222                     reader = DirectoryReader.open(directory);
 223                     searcher = new IndexSearcher(reader);
 224                 } else {
 225                     reader = DirectoryReader.open(directory);
 226                     searcher = new IndexSearcher(reader);
 227                 }
 228
 229                 long end = System.nanoTime();
 230
 231                 mon.worked(100);
 232
 233                 if (IndexPolicy.PERF_INDEX_LOAD) {
 234                     double time = (end - start) * 1e-6;
 235                     System.out.println(getDescriptor() + "Loaded Lucene index from " + indexPath + " for " + (forWriting ? "writing" : "reading") + " in " + time + " ms");
 236                 }
 237
 238                 success = true;
 239
 240                 }
 241
 242         } catch (Throwable t) {
 243                 setProblem(t);
 244         } finally {
 245
 246                 if(!success) {
 247                         this.state = State.PROBLEM;
 248                         changeState(monitor, session, State.NONE, depth+1);
 249                         return;
 250                 }
 251
 252         }
 253
 254         this.state = state;
 255
 256     }
 257
 258     public static final FieldType STRING_TYPE = new FieldType();
 259
 260     static {
 261       STRING_TYPE.setIndexed(true);
 262       STRING_TYPE.setStored(true);
 263       STRING_TYPE.setTokenized(true);
 264       STRING_TYPE.freeze();
 265     }
 266
 267     protected static Field makeField(String fieldName, String fieldClass) throws DatabaseException {
 268         switch (fieldClass) {
 269         case "Long":   return new LongField(fieldName, 0L, Field.Store.YES);
 270         case "String": return new Field    (fieldName, "", STRING_TYPE);
 271         case "Text":   return new TextField(fieldName, "", Field.Store.YES);
 272         default:
 273             throw new DatabaseException("Can only index Long, String and Text fields, encountered field type " + fieldClass);
 274         }
 275     }
 276
 277     protected static Field[] makeFieldsForRelation(GenericRelation r, int boundLength, Document document) throws DatabaseException {
 278         Pair<String, String>[] fields = r.getFields();
 279         Field[] fs = new Field[Math.max(0, fields.length - boundLength)];
 280         for (int i = boundLength; i < fields.length; i++) {
 281             Field f = makeField(fields[i].first, fields[i].second);
 282             fs[i - boundLength] = f;
 283             if (document != null)
 284                 document.add(f);
 285         }
 286         return fs;
 287     }
 288
 289     void insertIndex(IProgressMonitor monitor, GenericRelation r, int boundLength, Collection<Object[]> documentsData)
 290     throws CorruptIndexException, IOException, DatabaseException {
 291         assertAccessOpen(true);
 292
 293         if (IndexPolicy.TRACE_INDEX_UPDATE)
 294             System.out.println(getDescriptor() + "Inserting " + documentsData.size() + " documents into index at " + indexPath);
 295
 296         long start = 0, end = 0;
 297         if (IndexPolicy.PERF_INDEX_UPDATE)
 298             start = System.nanoTime();
 299
 300         try {
 301             Document document = new Document();
 302             Field[] fs = makeFieldsForRelation(r, boundLength, document);
 303
 304             for (Object[] documentData : documentsData) {
 305                 if (setFields(fs, documentData) == null)
 306                     continue;
 307
 308                 if (IndexPolicy.TRACE_INDEX_UPDATE)
 309                     System.out.println(getDescriptor() + "Inserting document " + document);
 310
 311                 writer.addDocument(document);
 312             }
 313
 314             if (IndexPolicy.PERF_INDEX_UPDATE) {
 315                 end = System.nanoTime();
 316                 double ms = (end - start) * 1e-6;
 317                 System.out.println(getDescriptor() + "Inserted " + documentsData.size() + " documents into index at " + indexPath + " in " + ms + " ms");
 318             }
 319
 320         } finally {
 321         }
 322     }
 323
 324     void removeIndex(IProgressMonitor monitor, GenericRelation r, RequestProcessor processor, String key, Collection<Object> keyValues) throws DatabaseException, CorruptIndexException, IOException {
 325         assertAccessOpen(true);
 326
 327         if (IndexPolicy.TRACE_INDEX_UPDATE)
 328             System.out.println(getDescriptor() + "Removing " + keyValues.size() + " documents from index at " + indexPath);
 329
 330         long start = 0, end = 0;
 331         if (IndexPolicy.PERF_INDEX_UPDATE)
 332             start = System.nanoTime();
 333
 334         try {
 335             for (Object keyValue : keyValues) {
 336                 Term removedTerm = null;
 337                 if (keyValue instanceof Long) {
 338                     removedTerm = IndexUtils.longTerm(key, (Long) keyValue);
 339                 } else if (keyValue instanceof String) {
 340                     removedTerm = new Term(key, (String) keyValue);
 341                 } else {
 342                     // FIXME: should throw an exception for illegal input data but this would leave the index in an incoherent state
 343                     continue;
 344                 }
 345
 346                 if (IndexPolicy.TRACE_INDEX_UPDATE)
 347                     System.out.println(getDescriptor() + "Removing document with key " + removedTerm);
 348                 writer.deleteDocuments(removedTerm);
 349             }
 350
 351             if (IndexPolicy.PERF_INDEX_UPDATE) {
 352                 end = System.nanoTime();
 353                 double ms = (end - start) * 1e-6;
 354                 System.out.println(getDescriptor() + "Removed " + keyValues.size() + " documents from index at " + indexPath + " in " + ms + " ms");
 355             }
 356
 357         } finally {
 358         }
 359     }
 360
 361     void removeIndex(IProgressMonitor monitor) throws DatabaseException, CorruptIndexException, IOException {
 362         assertAccessOpen(true);
 363
 364         long start = 0, end = 0;
 365         if (IndexPolicy.PERF_INDEX_UPDATE)
 366             start = System.nanoTime();
 367
 368         try {
 369
 370             writer.deleteAll();
 371
 372             if (IndexPolicy.PERF_INDEX_UPDATE) {
 373                 end = System.nanoTime();
 374                 double ms = (end - start) * 1e-6;
 375                 System.out.println(getDescriptor() + "Removed all documents from index at " + indexPath + " in " + ms + " ms");
 376             }
 377
 378         } finally {
 379         }
 380     }
 381
 382     boolean replaceIndex(IProgressMonitor monitor, String key, Collection<Object> keyValues, GenericRelation r, int boundLength, Collection<Object[]> documentsData) throws CorruptIndexException, IOException, DatabaseException {
 383
 384         boolean didReplace = false;
 385
 386         assertAccessOpen(true);
 387         if (keyValues.size() != documentsData.size())
 388             throw new IllegalArgumentException("keyValues size does not match documents data size, " + keyValues.size() + " <> " + documentsData.size());
 389
 390         if (IndexPolicy.TRACE_INDEX_UPDATE)
 391             System.out.println(getDescriptor() + "Replacing " + keyValues.size() + " documents from index at " + indexPath);
 392
 393         long start = 0, end = 0;
 394         if (IndexPolicy.PERF_INDEX_UPDATE)
 395             start = System.nanoTime();
 396
 397         try {
 398             Iterator<Object> keyIt = keyValues.iterator();
 399             Iterator<Object[]> documentDataIt = documentsData.iterator();
 400
 401             Document document = new Document();
 402             Field[] fs = makeFieldsForRelation(r, boundLength, document);
 403
 404             nextDocument:
 405                 while (keyIt.hasNext()) {
 406                     Object keyValue = keyIt.next();
 407                     Object[] documentData = documentDataIt.next();
 408
 409                     Term removedTerm = null;
 410                     if (keyValue instanceof Long) {
 411                         removedTerm = IndexUtils.longTerm(key, (Long) keyValue);
 412                     } else if (keyValue instanceof String) {
 413                         removedTerm = new Term(key, (String) keyValue);
 414                     } else {
 415                         // FIXME: should throw an exception for illegal input data but this would leave the index in an incoherent state
 416                         System.err.println("[" + getClass().getSimpleName() + "] Unrecognized document key to remove '" + keyValue + "', only " + String.class + " and " + Resource.class + " are supported.");
 417                         continue nextDocument;
 418                     }
 419
 420                     if (setFields(fs, documentData) == null)
 421                         continue nextDocument;
 422
 423                     if (IndexPolicy.TRACE_INDEX_UPDATE)
 424                         System.out.println(getDescriptor() + "Replacing document with key " + removedTerm + " with " + document);
 425
 426                     boolean done = false;
 427                     if(requireChangeInfoOnReplace()) {
 428                             TopDocs exist = searcher.search(new TermQuery(removedTerm), null, 2);
 429                             if(exist.scoreDocs.length == 1 && requireChangeInfoOnReplace()) {
 430                                 Document doc = reader.document(exist.scoreDocs[0].doc);
 431                                 if(!areSame(doc, document)) {
 432                                     writer.deleteDocuments(removedTerm);
 433                                     writer.addDocument(document);
 434                                     didReplace |= true;
 435                                     if (IndexPolicy.TRACE_INDEX_UPDATE)
 436                                         System.out.println("-replaced single existing");
 437                                 } else {
 438                                     if (IndexPolicy.TRACE_INDEX_UPDATE)
 439                                         System.out.println("-was actually same than single existing");
 440                                 }
 441                                 done = true;
 442                             }
 443                     }
 444                     if(!done) {
 445                         writer.deleteDocuments(removedTerm);
 446                         writer.addDocument(document);
 447                         didReplace |= true;
 448                         if (IndexPolicy.TRACE_INDEX_UPDATE)
 449                                 System.out.println("-had many or none - removed all existing");
 450                     }
 451
 452                 }
 453
 454             if (IndexPolicy.PERF_INDEX_UPDATE) {
 455                 end = System.nanoTime();
 456                 double ms = (end - start) * 1e-6;
 457                 System.out.println(getDescriptor() + "Replaced " + keyValues.size() + " documents from index at " + indexPath + " in " + ms + " ms");
 458             }
 459
 460         } finally {
 461         }
 462
 463         return didReplace;
 464
 465     }
 466
 467     protected boolean requireChangeInfoOnReplace() {
 468         return true;
 469     }
 470
 471     private boolean areSame(Document d1, Document d2) {
 472         List<IndexableField> fs1 = d1.getFields();
 473         List<IndexableField> fs2 = d2.getFields();
 474         if(fs1.size() != fs2.size()) return false;
 475         for(int i=0;i<fs1.size();i++) {
 476                 IndexableField f1 = fs1.get(i);
 477                 IndexableField f2 = fs2.get(i);
 478                 String s1 = f1.stringValue();
 479                 String s2 = f2.stringValue();
 480             if (IndexPolicy.TRACE_INDEX_UPDATE)
 481                 System.err.println("areSame " + s1 + " vs. " + s2 );
 482                 if(!ObjectUtils.objectEquals(s1,s2)) return false;
 483         }
 484         return true;
 485     }
 486
 487     final RequestProcessor session;
 488
 489     final Resource         relation;
 490
 491     /**
 492      * The schema of the index, i.e. the fields that will be indexed per
 493      * document for the specified relation. Since the relation stays the same
 494      * throughout the lifetime of this class, the index schema is also assumed
 495      * to the same. This means that {@link GenericRelation#getFields()} is
 496      * assumed to stay the same.
 497      */
 498     final IndexSchema      schema;
 499
 500     Resource         input;
 501
 502     Path             indexPath;
 503
 504     Directory        directory;
 505
 506     IndexReader      reader;
 507
 508     IndexWriter      writer;
 509
 510     IndexSearcher    searcher;
 511
 512     IndexedRelationsSearcherBase(RequestProcessor session, Resource relation, Resource input) {
 513         this.session = session;
 514         this.relation = relation;
 515         this.input = input;
 516         this.indexPath = getIndexDirectory(session.getSession(), relation, input);
 517         if(isIndexAvailable()) {
 518                 state = State.READY;
 519         } else {
 520                 state = State.NONE;
 521         }
 522         this.schema = IndexSchema.readFromRelation(session, relation);
 523     }
 524
 525     Directory getDirectory(Session session) throws IOException {
 526         return FSDirectory.open(indexPath.toFile());
 527     }
 528
 529     abstract String getDescriptor();
 530
 531     /**
 532      * Ensures that searcher is in read or write state.
 533      *
 534      * @param forWriting <code>true</code> to open index for writing,
 535      *        <code>false</code> for reading
 536      * @return true is required state was reached
 537      *
 538      */
 539     boolean startAccess(IProgressMonitor monitor, Session session, boolean forWriting) {
 540         if(forWriting) {
 541                 changeState(monitor, session, State.WRITE);
 542                 return checkState(State.WRITE);
 543         } else {
 544                 changeState(monitor, session, State.READ);
 545                 return checkState(State.READ);
 546         }
 547     }
 548
 549     boolean hasAccess(boolean forWriting) {
 550
 551         if (forWriting)
 552                 return checkState(State.WRITE);
 553         else
 554                 return checkState(State.WRITE) || checkState(State.READ);
 555
 556     }
 557
 558     void assertAccessOpen(boolean forWriting) {
 559         if (forWriting)
 560                 if(!checkState(State.WRITE))
 561                 throw new IllegalStateException("index not opened for writing (directory=" + directory + ", reader=" + reader + ")");
 562         else
 563                 if(!(checkState(State.WRITE) || checkState(State.READ)))
 564                 throw new IllegalStateException("index not opened for reading (directory=" + directory + ", writer=" + writer + ")");
 565     }
 566
 567     void closeWriter(IndexWriter writer) throws CorruptIndexException, IOException {
 568         if (writer == null)
 569             return;
 570
 571         try {
 572             // May throw OOME, see IndexWriter javadoc for the correct actions.
 573             writer.close(false);
 574         } catch (OutOfMemoryError e) {
 575             writer.close();
 576             throw e;
 577         }
 578     }
 579
 580     public static String getPattern(GenericRelation relation, int boundCount) {
 581         String result = "";
 582         for (int i = 0; i < boundCount; i++)
 583             result += "b";
 584         for (int i = 0; i < relation.getFields().length - boundCount; i++)
 585             result += "f";
 586         return result;
 587     }
 588
 589     void initializeIndex(IProgressMonitor monitor, ReadGraph graph, Object[] bound, boolean overwrite)
 590             throws IOException, DatabaseException
 591     {
 592         IndexingJob.jobifyIfPossible(
 593                 monitor,
 594                 "Reindexing " + NameUtils.getSafeLabel(graph, input),
 595                 mon -> {
 596                     try {
 597                         GenericRelation r = graph.adapt(relation, GenericRelation.class);
 598                         if (r == null)
 599                             throw new DatabaseException("Given resource " + relation + "could not be adapted to GenericRelation.");
 600
 601                         GenericRelation selection = r.select(getPattern(r, bound.length), bound);
 602
 603                         List<Object[]> results = selection.realize(graph);
 604                         initializeIndexImpl(new CompletableFuture<>(), mon, r, results, bound, overwrite);
 605                     } catch (IOException e) {
 606                         getLogger().error("Index is in problematic state! {}", this, e);
 607                         throw new DatabaseException(e);
 608                     }
 609                 });
 610     }
 611
 612     private static final int INDEXING_THREAD_COUNT = 2; // this is quite good parallelism level for lucene
 613
 614     void initializeIndexImpl(CompletableFuture<?> result, IProgressMonitor monitor, GenericRelation r, List<Object[]> results, final Object[] bound, boolean overwrite) throws IOException {
 615         try {
 616             final SubMonitor mon = SubMonitor.convert(monitor, 100);
 617
 618             if (IndexPolicy.TRACE_INDEX_INIT)
 619                 System.out.println(getDescriptor() + "Initializing index at " + indexPath + " (overwrite = " + overwrite + ")");
 620             mon.beginTask("Initializing Index", 100);
 621
 622             if (overwrite) {
 623                 if (Files.exists(indexPath)) {
 624                     mon.subTask("Erasing previous index");
 625                     if (getLogger().isDebugEnabled())
 626                         getLogger().debug("Erasing previous index {}", indexPath.toAbsolutePath());
 627                     FileUtils.emptyDirectory(indexPath);
 628                 }
 629             }
 630
 631             final AtomicReference<FSDirectory> directory = new AtomicReference<FSDirectory>();
 632             final AtomicReference<IndexWriter> writer = new AtomicReference<IndexWriter>();
 633
 634             try {
 635                 mon.subTask("Start index write");
 636                 Files.createDirectories(indexPath);
 637
 638                 directory.set(FSDirectory.open(indexPath.toFile()));
 639                 IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_4_9, Queries.getAnalyzer()).setOpenMode(OpenMode.CREATE);
 640                 writer.set(new IndexWriter(directory.get(), conf));
 641
 642                 mon.worked(5);
 643
 644                 long realizeStart = 0;
 645                 if (IndexPolicy.PERF_INDEX_INIT)
 646                     realizeStart = System.nanoTime();
 647
 648                 mon.subTask("Calculating indexed content");
 649                 mon.worked(5);
 650
 651                 mon.worked(40);
 652
 653                 if (IndexPolicy.PERF_INDEX_INIT)
 654                     System.out.println(getDescriptor() + "Realized index with " + results.size() + " entries at " + indexPath + " in " + (1e-9 * (System.nanoTime()-realizeStart)) + " seconds.");
 655
 656                 if (IndexPolicy.TRACE_INDEX_INIT)
 657                     System.out.println(getDescriptor() + "Indexed relation " + r + " produced " + results.size() + " results");
 658
 659                 long start = IndexPolicy.PERF_INDEX_INIT ? System.nanoTime() : 0;
 660
 661                 mon.subTask("Indexing content");
 662                 final Semaphore s = new Semaphore(0);
 663                 mon.setWorkRemaining(results.size());
 664                 for (int i = 0; i < INDEXING_THREAD_COUNT; i++) {
 665                     final int startIndex = i;
 666                     ThreadUtils.getBlockingWorkExecutor().submit(() -> {
 667                         try {
 668                             Document document = new Document();
 669                             Field[] fs = makeFieldsForRelation(r, bound.length, document);
 670
 671                             for (int index = startIndex; index < results.size(); index += INDEXING_THREAD_COUNT) {
 672                                 if (setFields(fs, results.get(index)) == null)
 673                                     continue;
 674                                 try {
 675                                     writer.get().addDocument(document);
 676                                 } catch (CorruptIndexException e) {
 677                                     getLogger().error("Index is corrupted! {}", this, e);
 678                                     throw new IllegalStateException(e);
 679                                 } catch (IOException e) {
 680                                     getLogger().error("Index is in problematic state! {}", this, e);
 681                                     throw new IllegalStateException(e);
 682                                 } finally {
 683                                     synchronized (mon) {
 684                                         mon.worked(1);
 685                                     }
 686                                 }
 687                             }
 688                         } catch (DatabaseException e) {
 689                             throw new IllegalStateException(e);
 690                         } finally {
 691                             s.release();
 692                         }
 693                     });
 694                 }
 695
 696                 try {
 697                     s.acquire(INDEXING_THREAD_COUNT);
 698                 } catch (InterruptedException e) {
 699                     getLogger().error("Could not initialize index {}", this, e);
 700                 }
 701
 702                 // http://www.gossamer-threads.com/lists/lucene/java-dev/47895
 703                 // and http://lucene.apache.org/java/docs/index.html#27+November+2011+-+Lucene+Core+3.5.0
 704                 // advise against calling optimize at all. So let's not do it anymore.
 705                 //writer.get().optimize();
 706                 //writer.get().commit();
 707
 708                 mon.subTask("Flushing");
 709
 710                 if (IndexPolicy.PERF_INDEX_INIT)
 711                     System.out.println(getDescriptor() + "Wrote index at " + indexPath + " in " + (1e-9 * (System.nanoTime()-start)) + " seconds.");
 712
 713                 result.complete(null);
 714     //        } catch (DatabaseException e) {
 715     //            getLogger().error("Could not initialize index due to db {}", this, e);
 716             } finally {
 717                 try {
 718                     closeWriter(writer.getAndSet(null));
 719                 } finally {
 720                     FileUtils.uncheckedClose(directory.getAndSet(null));
 721                 }
 722             }
 723         } catch (Throwable t) {
 724             getLogger().error("Could not initialize index", t);
 725             result.completeExceptionally(t);
 726         }
 727     }
 728
 729
 730     public List<Object[]> debugDocs(IProgressMonitor monitor) throws ParseException, IOException, DatabaseException {
 731
 732             Query query = new MatchAllDocsQuery();
 733
 734             TopDocs td = searcher.search(query, Integer.MAX_VALUE);
 735
 736             ScoreDoc[ ] scoreDocs = td.scoreDocs;
 737             List<Object[]> result = new ArrayList<Object[]>(scoreDocs.length);
 738
 739             for(ScoreDoc scoreDoc:scoreDocs) {
 740
 741                 try {
 742
 743                     Document doc = reader.document(scoreDoc.doc);
 744                     List<IndexableField> fs = doc.getFields();
 745                     Object[] o = new Object[fs.size()];
 746                     int index = 0;
 747                     for (IndexableField f : fs) {
 748                     o[index++] = f.stringValue();
 749                     }
 750                     result.add(o);
 751
 752             } catch (CorruptIndexException e) {
 753                 getLogger().error("Index is corrupted! {}", this, e);
 754                 throw new DatabaseException(e);
 755             } catch (IOException e) {
 756                 getLogger().error("Index is in problematic state! {}", this, e);
 757                 throw new DatabaseException(e);
 758             }
 759
 760             }
 761
 762             return result;
 763
 764     }
 765
 766
 767     List<Map<String, Object>> doSearch(IProgressMonitor monitor, RequestProcessor processor, String search, int maxResultCount) throws ParseException, IOException,
 768     DatabaseException {
 769
 770         // An empty search string will crash QueryParser
 771         // Just return no results for empty queries.
 772         //System.out.println("search: '" + search + "'");
 773         if (search.isEmpty())
 774             return Collections.emptyList();
 775
 776         assertAccessOpen(false);
 777
 778         Query query = Queries.parse(search, schema);
 779
 780         long start = System.nanoTime();
 781
 782         maxResultCount = Math.min(maxResultCount, searcher.getIndexReader().numDocs());
 783         if (maxResultCount == 0)
 784             return Collections.emptyList();
 785
 786         final TopDocs docs = searcher.search(query, null, maxResultCount);
 787
 788 //        for(Object[] o : debugDocs(monitor)) {
 789 //            System.err.println("-" + Arrays.toString(o));
 790 //        }
 791
 792         if (IndexPolicy.PERF_INDEX_QUERY) {
 793             long end = System.nanoTime();
 794             System.out.println(getDescriptor() + "search(" + search + ", " + maxResultCount + ") into index at " + indexPath + " took " + (1e-9 * (end-start)) + " seconds.");
 795         }
 796
 797         if (docs.totalHits == 0) {
 798             return Collections.emptyList();
 799         }
 800
 801         return processor.syncRequest(new Read<List<Map<String, Object>>>() {
 802
 803             @Override
 804             public List<Map<String, Object>> perform(ReadGraph graph) throws DatabaseException {
 805
 806                 GenericRelation r = graph.adapt(relation, GenericRelation.class);
 807                 if (r == null)
 808                     throw new DatabaseException("Given resource " + graph.syncRequest(new SafeName(relation))
 809                             + "could not be adapted to GenericRelation.");
 810
 811                 SerialisationSupport support = graph.getService(SerialisationSupport.class);
 812
 813                 List<Map<String, Object>> result = new ArrayList<Map<String, Object>>(docs.scoreDocs.length);
 814
 815                 final DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor();
 816
 817                 for (ScoreDoc scoreDoc : docs.scoreDocs) {
 818
 819                     try {
 820
 821                         reader.document(scoreDoc.doc, visitor);
 822
 823                         Document doc = visitor.getDocument();
 824
 825                         List<IndexableField> fs = doc.getFields();
 826                         Map<String, Object> entry = new THashMap<String, Object>(fs.size());
 827                         for (IndexableField f : fs) {
 828                             IndexSchema.Type type = schema.typeMap.get(f.name());
 829                             if (type == IndexSchema.Type.LONG) {
 830                                 entry.put(f.name(), support.getResource(f.numericValue().longValue()));
 831                             } else {
 832                                 entry.put(f.name(), f.stringValue());
 833                             }
 834                         }
 835
 836                         result.add(entry);
 837
 838                     } catch (CorruptIndexException e) {
 839                         getLogger().error("Index is corrupted! {}", this, e);
 840                         throw new DatabaseException(e);
 841                     } catch (IOException e) {
 842                         getLogger().error("Index is in problematic state! {}", this, e);
 843                         throw new DatabaseException(e);
 844                     }
 845                 }
 846                 return result;
 847             }
 848         });
 849     }
 850
 851     static class ResourceVisitor extends StoredFieldVisitor {
 852
 853         public long id;
 854
 855                 @Override
 856                 public Status needsField(FieldInfo fieldInfo) throws IOException {
 857                         if("Resource".equals(fieldInfo.name)) return Status.YES;
 858                         return Status.NO;
 859                 }
 860
 861                 @Override
 862                 public void longField(FieldInfo fieldInfo, long value) throws IOException {
 863                         id = value;
 864                 }
 865
 866     };
 867
 868     static class DumpVisitor extends StoredFieldVisitor {
 869
 870         public List<Object> values;
 871
 872         DumpVisitor(List<Object> values) {
 873                 this.values = values;
 874         }
 875
 876                 @Override
 877                 public Status needsField(FieldInfo fieldInfo) throws IOException {
 878                         return Status.YES;
 879                 }
 880
 881                 @Override
 882                 public void longField(FieldInfo fieldInfo, long value) throws IOException {
 883                         values.add(value);
 884                 }
 885
 886                 @Override
 887                 public void stringField(FieldInfo fieldInfo, String value) throws IOException {
 888                         values.add(value);
 889                 }
 890
 891     }
 892
 893     List<Resource> doSearchResources(IProgressMonitor monitor, RequestProcessor processor, String search, int maxResultCount) throws ParseException, IOException,
 894     DatabaseException {
 895
 896         // An empty search string will crash QueryParser
 897         // Just return no results for empty queries.
 898         //System.out.println("search: '" + search + "'");
 899         if (search.isEmpty())
 900             return Collections.emptyList();
 901
 902         assertAccessOpen(false);
 903
 904         Query query = Queries.parse(search, schema);
 905
 906         long start = System.nanoTime();
 907
 908         maxResultCount = Math.min(maxResultCount, searcher.getIndexReader().numDocs());
 909         if (maxResultCount == 0)
 910             return Collections.emptyList();
 911
 912         final TopDocs docs = searcher.search(query, null, maxResultCount);
 913
 914 //        for(Object[] o : debugDocs(monitor)) {
 915 //            System.err.println("-" + Arrays.toString(o));
 916 //        }
 917
 918         if (IndexPolicy.PERF_INDEX_QUERY) {
 919             long end = System.nanoTime();
 920             System.out.println(getDescriptor() + "search(" + search + ", " + maxResultCount + ") into index at " + indexPath + " took " + (1e-9 * (end-start)) + " seconds.");
 921         }
 922
 923         if (docs.totalHits == 0) {
 924             return Collections.emptyList();
 925         }
 926
 927         return processor.syncRequest(new Read<List<Resource>>() {
 928
 929             @Override
 930             public List<Resource> perform(ReadGraph graph) throws DatabaseException {
 931
 932                 CollectionSupport cs = graph.getService(CollectionSupport.class);
 933                 SerialisationSupport support = graph.getService(SerialisationSupport.class);
 934
 935                 List<Resource> result = cs.createList();
 936
 937                 ResourceVisitor visitor = new ResourceVisitor();
 938
 939                 for (ScoreDoc scoreDoc : docs.scoreDocs) {
 940                     try {
 941                         reader.document(scoreDoc.doc, visitor);
 942                         result.add(support.getResource(visitor.id));
 943                     } catch (CorruptIndexException e) {
 944                         getLogger().error("Index is corrupted! {}", this, e);
 945                         throw new DatabaseException(e);
 946                     } catch (IOException e) {
 947                         getLogger().error("Index is in problematic state! {}", this, e);
 948                         throw new DatabaseException(e);
 949                     }
 950                 }
 951                 return result;
 952             }
 953         });
 954     }
 955
 956     List<Object> doList(IProgressMonitor monitor, RequestProcessor processor) throws ParseException, IOException,
 957     DatabaseException {
 958
 959         assertAccessOpen(false);
 960
 961         Query query = new MatchAllDocsQuery();
 962
 963         final TopDocs docs = searcher.search(query, Integer.MAX_VALUE);
 964
 965         ArrayList<Object> result = new ArrayList<Object>();
 966
 967         DumpVisitor visitor = new DumpVisitor(result);
 968
 969         for (ScoreDoc scoreDoc : docs.scoreDocs) {
 970
 971                 try {
 972
 973                         reader.document(scoreDoc.doc, visitor);
 974
 975                 } catch (CorruptIndexException e) {
 976                     getLogger().error("Index is corrupted! {}", this, e);
 977                         throw new DatabaseException(e);
 978                 } catch (IOException e) {
 979                     getLogger().error("Index is in problematic state! {}", this, e);
 980                         throw new DatabaseException(e);
 981                 }
 982
 983         }
 984
 985         return result;
 986
 987     }
 988
 989     protected static Path getIndexDirectory(Session session, Resource relation, Resource input) {
 990         Path path = DatabaseIndexing.getIndexLocation(session, relation, input);
 991 //        System.out.println("getIndexDirectory = " + path);
 992         return path;
 993     }
 994
 995     Path getIndexPath() {
 996         return indexPath;
 997     }
 998
 999     boolean isIndexAvailable() {
1000         return Files.isDirectory(indexPath);
1001     }
1002
1003     abstract Throwable bestEffortClear(IProgressMonitor monitor, Session session);
1004
1005     /*
1006      * Start from scratch. Clear all caches and rebuild the index.
1007      */
1008     Throwable clearDirectory(IProgressMonitor monitor, Session session) {
1009
1010                 Path file = getIndexPath();
1011
1012         try {
1013                         FileUtils.delete(file);
1014         } catch (Throwable t) {
1015                 getLogger().error("Could not delete directory {}", file.toAbsolutePath(), t);
1016                 return t;
1017         }
1018         if (Files.exists(file))
1019             return new IllegalStateException("Failed to delete directory " + file.toAbsolutePath());
1020         return null;
1021     }
1022
1023     private Field[] setFields(Field[] fs, Object[] result) {
1024         for (int i = 0; i < result.length; i++) {
1025             Object value = result[i];
1026             if (value instanceof String) {
1027                 if (IndexPolicy.DEBUG_INDEX_INIT)
1028                     System.out.println(getDescriptor() + "index " + fs[i].name() + " = " + value + " : String");
1029                 fs[i].setStringValue((String) value);
1030             } else if (value instanceof Long) {
1031                 if (IndexPolicy.DEBUG_INDEX_INIT)
1032                     System.out.println(getDescriptor() + "index " + fs[i].name() + " = " + value + " : Long");
1033                 fs[i].setLongValue((Long) value);
1034             } else {
1035                 getLogger().error("Can only index Long and String fields, encountered " + value);
1036                 return null;
1037             }
1038         }
1039         return fs;
1040     }
1041
1042     protected abstract Logger getLogger();
1043
1044     @Override
1045     public String toString() {
1046         return getClass().getSimpleName() + " [" + String.valueOf(schema) + ", " + String.valueOf(relation) + ", " + String.valueOf(input) + ", " + String.valueOf(indexPath) + ", " + String.valueOf(directory) + ", " + String.valueOf(state) + "]";
1047     }
1048 }