bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsSearcherBase.java

   1 /*******************************************************************************
   2  * Copyright (c) 2007, 2015 Association for Decentralized Information Management
   3  * in Industry THTH ry.
   4  * All rights reserved. This program and the accompanying materials
   5  * are made available under the terms of the Eclipse Public License v1.0
   6  * which accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *     VTT Technical Research Centre of Finland - initial API and implementation
  11  *     Semantum Oy - Fix for simantics issue #6053
  12  *******************************************************************************/
  13 package org.simantics.db.indexing;
  14
  15 import java.io.File;
  16 import java.io.IOException;
  17 import java.nio.file.Files;
  18 import java.util.ArrayList;
  19 import java.util.Collection;
  20 import java.util.Collections;
  21 import java.util.Iterator;
  22 import java.util.List;
  23 import java.util.Map;
  24 import java.util.concurrent.ExecutorService;
  25 import java.util.concurrent.Executors;
  26 import java.util.concurrent.Semaphore;
  27 import java.util.concurrent.ThreadFactory;
  28 import java.util.concurrent.atomic.AtomicReference;
  29
  30 import org.apache.lucene.document.Document;
  31 import org.apache.lucene.document.DocumentStoredFieldVisitor;
  32 import org.apache.lucene.document.Field;
  33 import org.apache.lucene.document.FieldType;
  34 import org.apache.lucene.document.LongField;
  35 import org.apache.lucene.document.TextField;
  36 import org.apache.lucene.index.CorruptIndexException;
  37 import org.apache.lucene.index.DirectoryReader;
  38 import org.apache.lucene.index.FieldInfo;
  39 import org.apache.lucene.index.IndexNotFoundException;
  40 import org.apache.lucene.index.IndexReader;
  41 import org.apache.lucene.index.IndexWriter;
  42 import org.apache.lucene.index.IndexWriterConfig;
  43 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
  44 import org.apache.lucene.index.IndexableField;
  45 import org.apache.lucene.index.StoredFieldVisitor;
  46 import org.apache.lucene.index.Term;
  47 import org.apache.lucene.queryparser.classic.ParseException;
  48 import org.apache.lucene.search.IndexSearcher;
  49 import org.apache.lucene.search.MatchAllDocsQuery;
  50 import org.apache.lucene.search.Query;
  51 import org.apache.lucene.search.ScoreDoc;
  52 import org.apache.lucene.search.TermQuery;
  53 import org.apache.lucene.search.TopDocs;
  54 import org.apache.lucene.store.Directory;
  55 import org.apache.lucene.store.FSDirectory;
  56 import org.apache.lucene.util.Version;
  57 import org.eclipse.core.runtime.IProgressMonitor;
  58 import org.eclipse.core.runtime.SubMonitor;
  59 import org.simantics.databoard.util.ObjectUtils;
  60 import org.simantics.db.ReadGraph;
  61 import org.simantics.db.RequestProcessor;
  62 import org.simantics.db.Resource;
  63 import org.simantics.db.Session;
  64 import org.simantics.db.common.request.SafeName;
  65 import org.simantics.db.common.utils.Logger;
  66 import org.simantics.db.common.utils.NameUtils;
  67 import org.simantics.db.exception.DatabaseException;
  68 import org.simantics.db.indexing.internal.IndexingJob;
  69 import org.simantics.db.layer0.adapter.GenericRelation;
  70 import org.simantics.db.request.Read;
  71 import org.simantics.db.service.CollectionSupport;
  72 import org.simantics.db.service.SerialisationSupport;
  73 import org.simantics.utils.FileUtils;
  74 import org.simantics.utils.datastructures.Pair;
  75
  76 import gnu.trove.map.hash.THashMap;
  77
  78 /**
  79  * @author Tuukka Lehtonen
  80  * @author Antti Villberg
  81  */
  82 abstract public class IndexedRelationsSearcherBase {
  83
  84     protected enum State {
  85         // No index is available
  86         NONE,
  87         // An index is available, but there is a problem with it
  88         PROBLEM,
  89         // An index is available but no reader or writer is ready
  90         READY,
  91         // A reader is ready
  92         READ,
  93         // A writer (and a reader) is ready
  94         WRITE
  95     }
  96
  97     private State state = State.READY;
  98     private Throwable exception;
  99
 100     public Throwable getException() {
 101         return exception;
 102     }
 103
 104     public void setProblem(Throwable t) {
 105         this.state = State.PROBLEM;
 106         this.exception = t;
 107     }
 108
 109     public void setNone() {
 110         this.state = State.NONE;
 111     }
 112
 113     public void setReady() {
 114         this.state = State.READY;
 115     }
 116
 117     protected boolean checkState(State state) {
 118         return this.state == state;
 119     }
 120
 121     protected void assertState(State state) throws AssertionError {
 122
 123         if(this.state != state) throw new AssertionError("Illegal state, expected " + state.name() + " but was in " + this.state.name());
 124
 125     }
 126
 127     public void changeState(IProgressMonitor monitor, Session session, State state) {
 128         changeState(monitor, session, state, 0);
 129     }
 130
 131     protected void changeState(IProgressMonitor monitor, Session session, State state, int depth) {
 132
 133         if(this.state == state) return;
 134
 135         if (IndexPolicy.TRACE_INDEX_MANAGEMENT)
 136                 System.err.println("Index state " + this.state.name() + " => " + state.name() + " " + this);
 137
 138         // Check transitions
 139
 140         // Try to exit problem state
 141         if (State.PROBLEM == this.state && depth > 0) {
 142                 Throwable t = bestEffortClear(monitor, session);
 143                 if(t != null) {
 144                                 exception = t;
 145                                 return;
 146                 }
 147                 // Managed to get into initial state
 148                 this.state = State.NONE;
 149                 return;
 150         }
 151
 152         // Cannot move into read from no index
 153         if (State.NONE ==  this.state && State.READ == state) return;
 154         // Cannot move into write from no index
 155         if (State.NONE ==  this.state && State.WRITE == state) return;
 156
 157                 boolean success = false;
 158
 159         try {
 160
 161                 if (searcher != null) {
 162                         searcher = null;
 163                 }
 164                 if (reader != null) {
 165                         reader.close();
 166                         reader = null;
 167                 }
 168                         closeWriter(writer);
 169                         directory = null;
 170
 171                         success = true;
 172
 173                 // Enter new state
 174                 if (State.READ == state || State.WRITE == state) {
 175
 176                         success = false;
 177
 178                         boolean forWriting = State.WRITE == state;
 179
 180                         if (directory != null)
 181                                 throw new IllegalStateException(getDescriptor() + "Index already loaded");
 182
 183                         SubMonitor mon = SubMonitor.convert(monitor, 100);
 184
 185                         mon.beginTask("Loading index", 100);
 186
 187                         if (IndexPolicy.TRACE_INDEX_LOAD)
 188                     System.out.println(getDescriptor() + "Loading Lucene index from " + indexPath + " for " + (forWriting ? "writing" : "reading"));
 189
 190                 long start = System.nanoTime();
 191
 192                 directory = getDirectory(session);
 193
 194                 if (forWriting) {
 195                     // Never overwrite an index that is about to be loaded.
 196                     // TODO: could use OpenMode.CREATE_OR_APPEND but must test first
 197                     IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_4_9, Queries.getAnalyzer()).setOpenMode(OpenMode.APPEND);
 198                     try {
 199                         // FIXME: platform #4676
 200                         writer = new IndexWriter(directory, config);
 201                     } catch (IndexNotFoundException e) {
 202                         // There was no pre-existing index on disk. Create it now.
 203                         writer = new IndexWriter(directory, config.setOpenMode(OpenMode.CREATE));
 204                         writer.commit();
 205                     }
 206                     reader = DirectoryReader.open(directory);
 207                     searcher = new IndexSearcher(reader);
 208                 } else {
 209                     reader = DirectoryReader.open(directory);
 210                     searcher = new IndexSearcher(reader);
 211                 }
 212
 213                 long end = System.nanoTime();
 214
 215                 mon.worked(100);
 216
 217                 if (IndexPolicy.PERF_INDEX_LOAD) {
 218                     double time = (end - start) * 1e-6;
 219                     System.out.println(getDescriptor() + "Loaded Lucene index from " + indexPath + " for " + (forWriting ? "writing" : "reading") + " in " + time + " ms");
 220                 }
 221
 222                 success = true;
 223
 224                 }
 225
 226         } catch (Throwable t) {
 227
 228                 setProblem(t);
 229
 230         } finally {
 231
 232                 if(!success) {
 233                         this.state = State.PROBLEM;
 234                         changeState(monitor, session, State.NONE, depth+1);
 235                         return;
 236                 }
 237
 238         }
 239
 240         this.state = state;
 241
 242     }
 243
 244     public static final FieldType STRING_TYPE = new FieldType();
 245
 246     static {
 247       STRING_TYPE.setIndexed(true);
 248       STRING_TYPE.setStored(true);
 249       STRING_TYPE.setTokenized(true);
 250       STRING_TYPE.freeze();
 251     }
 252
 253     protected static Field makeField(String fieldName, String fieldClass) throws DatabaseException {
 254         switch (fieldClass) {
 255         case "Long":   return new LongField(fieldName, 0L, Field.Store.YES);
 256         case "String": return new Field    (fieldName, "", STRING_TYPE);
 257         case "Text":   return new TextField(fieldName, "", Field.Store.YES);
 258         default:
 259             throw new DatabaseException("Can only index Long, String and Text fields, encountered field type " + fieldClass);
 260         }
 261     }
 262
 263     protected static Field[] makeFieldsForRelation(GenericRelation r, int boundLength, Document document) throws DatabaseException {
 264         Pair<String, String>[] fields = r.getFields();
 265         Field[] fs = new Field[Math.max(0, fields.length - boundLength)];
 266         for (int i = boundLength; i < fields.length; i++) {
 267             Field f = makeField(fields[i].first, fields[i].second);
 268             fs[i - boundLength] = f;
 269             if (document != null)
 270                 document.add(f);
 271         }
 272         return fs;
 273     }
 274
 275     void insertIndex(IProgressMonitor monitor, GenericRelation r, int boundLength, Collection<Object[]> documentsData)
 276     throws CorruptIndexException, IOException, DatabaseException {
 277         assertAccessOpen(true);
 278
 279         if (IndexPolicy.TRACE_INDEX_UPDATE)
 280             System.out.println(getDescriptor() + "Inserting " + documentsData.size() + " documents into index at " + indexPath);
 281
 282         long start = 0, end = 0;
 283         if (IndexPolicy.PERF_INDEX_UPDATE)
 284             start = System.nanoTime();
 285
 286         try {
 287             Document document = new Document();
 288             Field[] fs = makeFieldsForRelation(r, boundLength, document);
 289
 290             for (Object[] documentData : documentsData) {
 291                 if (setFields(fs, documentData) == null)
 292                     continue;
 293
 294                 if (IndexPolicy.TRACE_INDEX_UPDATE)
 295                     System.out.println(getDescriptor() + "Inserting document " + document);
 296
 297                 writer.addDocument(document);
 298             }
 299
 300             if (IndexPolicy.PERF_INDEX_UPDATE) {
 301                 end = System.nanoTime();
 302                 double ms = (end - start) * 1e-6;
 303                 System.out.println(getDescriptor() + "Inserted " + documentsData.size() + " documents into index at " + indexPath + " in " + ms + " ms");
 304             }
 305
 306         } finally {
 307         }
 308     }
 309
 310     void removeIndex(IProgressMonitor monitor, GenericRelation r, RequestProcessor processor, String key, Collection<Object> keyValues) throws DatabaseException, CorruptIndexException, IOException {
 311         assertAccessOpen(true);
 312
 313         if (IndexPolicy.TRACE_INDEX_UPDATE)
 314             System.out.println(getDescriptor() + "Removing " + keyValues.size() + " documents from index at " + indexPath);
 315
 316         long start = 0, end = 0;
 317         if (IndexPolicy.PERF_INDEX_UPDATE)
 318             start = System.nanoTime();
 319
 320         try {
 321             for (Object keyValue : keyValues) {
 322                 Term removedTerm = null;
 323                 if (keyValue instanceof Long) {
 324                     removedTerm = IndexUtils.longTerm(key, (Long) keyValue);
 325                 } else if (keyValue instanceof String) {
 326                     removedTerm = new Term(key, (String) keyValue);
 327                 } else {
 328                     // FIXME: should throw an exception for illegal input data but this would leave the index in an incoherent state
 329                     continue;
 330                 }
 331
 332                 if (IndexPolicy.TRACE_INDEX_UPDATE)
 333                     System.out.println(getDescriptor() + "Removing document with key " + removedTerm);
 334                 writer.deleteDocuments(removedTerm);
 335             }
 336
 337             if (IndexPolicy.PERF_INDEX_UPDATE) {
 338                 end = System.nanoTime();
 339                 double ms = (end - start) * 1e-6;
 340                 System.out.println(getDescriptor() + "Removed " + keyValues.size() + " documents from index at " + indexPath + " in " + ms + " ms");
 341             }
 342
 343         } finally {
 344         }
 345     }
 346
 347     void removeIndex(IProgressMonitor monitor) throws DatabaseException, CorruptIndexException, IOException {
 348         assertAccessOpen(true);
 349
 350         long start = 0, end = 0;
 351         if (IndexPolicy.PERF_INDEX_UPDATE)
 352             start = System.nanoTime();
 353
 354         try {
 355
 356             writer.deleteAll();
 357
 358             if (IndexPolicy.PERF_INDEX_UPDATE) {
 359                 end = System.nanoTime();
 360                 double ms = (end - start) * 1e-6;
 361                 System.out.println(getDescriptor() + "Removed all documents from index at " + indexPath + " in " + ms + " ms");
 362             }
 363
 364         } finally {
 365         }
 366     }
 367
 368     boolean replaceIndex(IProgressMonitor monitor, String key, Collection<Object> keyValues, GenericRelation r, int boundLength, Collection<Object[]> documentsData) throws CorruptIndexException, IOException, DatabaseException {
 369
 370         boolean didReplace = false;
 371
 372         assertAccessOpen(true);
 373         if (keyValues.size() != documentsData.size())
 374             throw new IllegalArgumentException("keyValues size does not match documents data size, " + keyValues.size() + " <> " + documentsData.size());
 375
 376         if (IndexPolicy.TRACE_INDEX_UPDATE)
 377             System.out.println(getDescriptor() + "Replacing " + keyValues.size() + " documents from index at " + indexPath);
 378
 379         long start = 0, end = 0;
 380         if (IndexPolicy.PERF_INDEX_UPDATE)
 381             start = System.nanoTime();
 382
 383         try {
 384             Iterator<Object> keyIt = keyValues.iterator();
 385             Iterator<Object[]> documentDataIt = documentsData.iterator();
 386
 387             Document document = new Document();
 388             Field[] fs = makeFieldsForRelation(r, boundLength, document);
 389
 390             nextDocument:
 391                 while (keyIt.hasNext()) {
 392                     Object keyValue = keyIt.next();
 393                     Object[] documentData = documentDataIt.next();
 394
 395                     Term removedTerm = null;
 396                     if (keyValue instanceof Long) {
 397                         removedTerm = IndexUtils.longTerm(key, (Long) keyValue);
 398                     } else if (keyValue instanceof String) {
 399                         removedTerm = new Term(key, (String) keyValue);
 400                     } else {
 401                         // FIXME: should throw an exception for illegal input data but this would leave the index in an incoherent state
 402                         System.err.println("[" + getClass().getSimpleName() + "] Unrecognized document key to remove '" + keyValue + "', only " + String.class + " and " + Resource.class + " are supported.");
 403                         continue nextDocument;
 404                     }
 405
 406                     if (setFields(fs, documentData) == null)
 407                         continue nextDocument;
 408
 409                     if (IndexPolicy.TRACE_INDEX_UPDATE)
 410                         System.out.println(getDescriptor() + "Replacing document with key " + removedTerm + " with " + document);
 411
 412                     boolean done = false;
 413                     if(requireChangeInfoOnReplace()) {
 414                             TopDocs exist = searcher.search(new TermQuery(removedTerm), null, 2);
 415                             if(exist.scoreDocs.length == 1 && requireChangeInfoOnReplace()) {
 416                                 Document doc = reader.document(exist.scoreDocs[0].doc);
 417                                 if(!areSame(doc, document)) {
 418                                     writer.deleteDocuments(removedTerm);
 419                                     writer.addDocument(document);
 420                                     didReplace |= true;
 421                                     if (IndexPolicy.TRACE_INDEX_UPDATE)
 422                                         System.out.println("-replaced single existing");
 423                                 } else {
 424                                     if (IndexPolicy.TRACE_INDEX_UPDATE)
 425                                         System.out.println("-was actually same than single existing");
 426                                 }
 427                                 done = true;
 428                             }
 429                     }
 430                     if(!done) {
 431                         writer.deleteDocuments(removedTerm);
 432                         writer.addDocument(document);
 433                         didReplace |= true;
 434                         if (IndexPolicy.TRACE_INDEX_UPDATE)
 435                                 System.out.println("-had many or none - removed all existing");
 436                     }
 437
 438                 }
 439
 440             if (IndexPolicy.PERF_INDEX_UPDATE) {
 441                 end = System.nanoTime();
 442                 double ms = (end - start) * 1e-6;
 443                 System.out.println(getDescriptor() + "Replaced " + keyValues.size() + " documents from index at " + indexPath + " in " + ms + " ms");
 444             }
 445
 446         } finally {
 447         }
 448
 449         return didReplace;
 450
 451     }
 452
 453     protected boolean requireChangeInfoOnReplace() {
 454         return true;
 455     }
 456
 457     private boolean areSame(Document d1, Document d2) {
 458         List<IndexableField> fs1 = d1.getFields();
 459         List<IndexableField> fs2 = d2.getFields();
 460         if(fs1.size() != fs2.size()) return false;
 461         for(int i=0;i<fs1.size();i++) {
 462                 IndexableField f1 = fs1.get(i);
 463                 IndexableField f2 = fs2.get(i);
 464                 String s1 = f1.stringValue();
 465                 String s2 = f2.stringValue();
 466             if (IndexPolicy.TRACE_INDEX_UPDATE)
 467                 System.err.println("areSame " + s1 + " vs. " + s2 );
 468                 if(!ObjectUtils.objectEquals(s1,s2)) return false;
 469         }
 470         return true;
 471     }
 472
 473     final RequestProcessor session;
 474
 475     final Resource         relation;
 476
 477     /**
 478      * The schema of the index, i.e. the fields that will be indexed per
 479      * document for the specified relation. Since the relation stays the same
 480      * throughout the lifetime of this class, the index schema is also assumed
 481      * to the same. This means that {@link GenericRelation#getFields()} is
 482      * assumed to stay the same.
 483      */
 484     final IndexSchema      schema;
 485
 486     Resource         input;
 487
 488     File             indexPath;
 489
 490     Directory        directory;
 491
 492     IndexReader      reader;
 493
 494     IndexWriter      writer;
 495
 496     IndexSearcher    searcher;
 497
 498     IndexedRelationsSearcherBase(RequestProcessor session, Resource relation, Resource input) {
 499         this.session = session;
 500         this.relation = relation;
 501         this.input = input;
 502         this.indexPath = getIndexDirectory(session.getSession(), relation, input);
 503         if(isIndexAvailable()) {
 504                 state = State.READY;
 505         } else {
 506                 state = State.NONE;
 507         }
 508         this.schema = IndexSchema.readFromRelation(session, relation);
 509     }
 510
 511     Directory getDirectory(Session session) throws IOException {
 512         return FSDirectory.open(indexPath);
 513     }
 514
 515     abstract String getDescriptor();
 516
 517     /**
 518      * Ensures that searcher is in read or write state.
 519      *
 520      * @param forWriting <code>true</code> to open index for writing,
 521      *        <code>false</code> for reading
 522      * @return true is required state was reached
 523      *
 524      */
 525     boolean startAccess(IProgressMonitor monitor, Session session, boolean forWriting) {
 526         if(forWriting) {
 527                 changeState(monitor, session, State.WRITE);
 528                 return checkState(State.WRITE);
 529         } else {
 530                 changeState(monitor, session, State.READ);
 531                 return checkState(State.READ);
 532         }
 533     }
 534
 535     boolean hasAccess(boolean forWriting) {
 536
 537         if (forWriting)
 538                 return checkState(State.WRITE);
 539         else
 540                 return checkState(State.WRITE) || checkState(State.READ);
 541
 542     }
 543
 544     void assertAccessOpen(boolean forWriting) {
 545         if (forWriting)
 546                 if(!checkState(State.WRITE))
 547                 throw new IllegalStateException("index not opened for writing (directory=" + directory + ", reader=" + reader + ")");
 548         else
 549                 if(!(checkState(State.WRITE) || checkState(State.READ)))
 550                 throw new IllegalStateException("index not opened for reading (directory=" + directory + ", writer=" + writer + ")");
 551     }
 552
 553     void closeWriter(IndexWriter writer) throws CorruptIndexException, IOException {
 554         if (writer == null)
 555             return;
 556
 557         try {
 558             // May throw OOME, see IndexWriter javadoc for the correct actions.
 559             writer.close(false);
 560         } catch (OutOfMemoryError e) {
 561             writer.close();
 562             throw e;
 563         }
 564     }
 565
 566     private static String getPattern(GenericRelation relation, int boundCount) {
 567         String result = "";
 568         for (int i = 0; i < boundCount; i++)
 569             result += "b";
 570         for (int i = 0; i < relation.getFields().length - boundCount; i++)
 571             result += "f";
 572         return result;
 573     }
 574
 575     private static final int INDEXING_THREAD_COUNT = 2;
 576
 577     private static final ExecutorService executor = Executors.newFixedThreadPool(INDEXING_THREAD_COUNT, new ThreadFactory() {
 578         @Override
 579         public Thread newThread(Runnable r) {
 580             Thread t = new Thread(r, "Lucene Index Creator");
 581             if (!t.isDaemon())
 582                 t.setDaemon(true);
 583             if (t.getPriority() != Thread.NORM_PRIORITY)
 584                 t.setPriority(Thread.NORM_PRIORITY);
 585             return t;
 586         }
 587     });
 588
 589     void initializeIndex(IProgressMonitor monitor, ReadGraph graph, Object[] bound, boolean overwrite)
 590             throws IOException, DatabaseException
 591     {
 592         IndexingJob.jobifyIfPossible(
 593                 monitor,
 594                 "Reindexing " + NameUtils.getSafeLabel(graph, input),
 595                 mon -> {
 596                     try {
 597                         initializeIndexImpl(mon, graph, bound, overwrite);
 598                     } catch (IOException e) {
 599                         throw new DatabaseException(e);
 600                     }
 601                 });
 602     }
 603
 604     void initializeIndexImpl(IProgressMonitor monitor, ReadGraph graph, final Object[] bound, boolean overwrite) throws IOException,
 605     DatabaseException {
 606
 607         final SubMonitor mon = SubMonitor.convert(monitor, 100);
 608
 609         if (IndexPolicy.TRACE_INDEX_INIT)
 610             System.out.println(getDescriptor() + "Initializing index at " + indexPath + " (overwrite = " + overwrite + ")");
 611         mon.beginTask("Initializing Index", 100);
 612
 613         if (overwrite) {
 614             mon.subTask("Erasing previous index");
 615             FileUtils.deleteAll(indexPath);
 616         }
 617
 618         final AtomicReference<FSDirectory> directory = new AtomicReference<FSDirectory>();
 619         final AtomicReference<IndexWriter> writer = new AtomicReference<IndexWriter>();
 620
 621         try {
 622             mon.subTask("Start index write");
 623             Files.createDirectories(indexPath.toPath());
 624
 625             directory.set(FSDirectory.open(indexPath));
 626             IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_4_9, Queries.getAnalyzer()).setOpenMode(OpenMode.CREATE);
 627             writer.set(new IndexWriter(directory.get(), conf));
 628
 629             mon.worked(5);
 630
 631             final GenericRelation r = graph.adapt(relation, GenericRelation.class);
 632             if (r == null)
 633                 throw new DatabaseException("Given resource " + graph.syncRequest(new SafeName(relation))
 634                 + "could not be adapted to GenericRelation.");
 635
 636             long realizeStart = 0;
 637             if (IndexPolicy.PERF_INDEX_INIT)
 638                 realizeStart = System.nanoTime();
 639
 640             mon.subTask("Calculating indexed content");
 641             GenericRelation selection = r.select(getPattern(r, bound.length), bound);
 642             mon.worked(5);
 643             List<Object[]> results = selection.realize(graph);
 644             mon.worked(40);
 645
 646             if (IndexPolicy.PERF_INDEX_INIT)
 647                 System.out.println(getDescriptor() + "Realized index with " + results.size() + " entries at " + indexPath + " in " + (1e-9 * (System.nanoTime()-realizeStart)) + " seconds.");
 648             if (IndexPolicy.TRACE_INDEX_INIT)
 649                 System.out.println(getDescriptor() + "Indexed relation " + r + " produced " + results.size() + " results");
 650
 651             long start = IndexPolicy.PERF_INDEX_INIT ? System.nanoTime() : 0;
 652
 653             mon.subTask("Indexing content");
 654             final Semaphore s = new Semaphore(0);
 655             mon.setWorkRemaining(results.size());
 656
 657             for (int i = 0; i < INDEXING_THREAD_COUNT; i++) {
 658                 final int startIndex = i;
 659                 executor.submit(() -> {
 660                     try {
 661                         Document document = new Document();
 662                         Field[] fs = makeFieldsForRelation(r, bound.length, document);
 663
 664                         for (int index = startIndex; index < results.size(); index += INDEXING_THREAD_COUNT) {
 665                             if (setFields(fs, results.get(index)) == null)
 666                                 continue;
 667                             try {
 668                                 writer.get().addDocument(document);
 669                             } catch (CorruptIndexException e) {
 670                                 throw new IllegalStateException(e);
 671                             } catch (IOException e) {
 672                                 throw new IllegalStateException(e);
 673                             } finally {
 674                                 synchronized (mon) {
 675                                     mon.worked(1);
 676                                 }
 677                             }
 678                         }
 679                     } catch (DatabaseException e) {
 680                         throw new IllegalStateException(e);
 681                     } finally {
 682                         s.release();
 683                     }
 684                 });
 685             }
 686
 687             try {
 688                 s.acquire(INDEXING_THREAD_COUNT);
 689             } catch (InterruptedException e) {
 690                 Logger.defaultLogError(e);
 691             }
 692
 693             // http://www.gossamer-threads.com/lists/lucene/java-dev/47895
 694             // and http://lucene.apache.org/java/docs/index.html#27+November+2011+-+Lucene+Core+3.5.0
 695             // advise against calling optimize at all. So let's not do it anymore.
 696             //writer.get().optimize();
 697             //writer.get().commit();
 698
 699             mon.subTask("Flushing");
 700
 701             if (IndexPolicy.PERF_INDEX_INIT)
 702                 System.out.println(getDescriptor() + "Wrote index at " + indexPath + " in " + (1e-9 * (System.nanoTime()-start)) + " seconds.");
 703
 704         } catch (DatabaseException e) {
 705
 706             Logger.defaultLogError(e);
 707
 708         } finally {
 709             try {
 710                 closeWriter(writer.getAndSet(null));
 711             } finally {
 712                 FileUtils.uncheckedClose(directory.getAndSet(null));
 713             }
 714         }
 715     }
 716
 717
 718     public List<Object[]> debugDocs(IProgressMonitor monitor) throws ParseException, IOException, DatabaseException {
 719
 720             Query query = new MatchAllDocsQuery();
 721
 722             TopDocs td = searcher.search(query, Integer.MAX_VALUE);
 723
 724             ScoreDoc[ ] scoreDocs = td.scoreDocs;
 725             List<Object[]> result = new ArrayList<Object[]>(scoreDocs.length);
 726
 727             for(ScoreDoc scoreDoc:scoreDocs) {
 728
 729                 try {
 730
 731                     Document doc = reader.document(scoreDoc.doc);
 732                     List<IndexableField> fs = doc.getFields();
 733                     Object[] o = new Object[fs.size()];
 734                     int index = 0;
 735                     for (IndexableField f : fs) {
 736                     o[index++] = f.stringValue();
 737                     }
 738                     result.add(o);
 739
 740                 } catch (CorruptIndexException e) {
 741                     throw new DatabaseException(e);
 742                 } catch (IOException e) {
 743                     throw new DatabaseException(e);
 744                 }
 745
 746             }
 747
 748             return result;
 749
 750     }
 751
 752
 753     List<Map<String, Object>> doSearch(IProgressMonitor monitor, RequestProcessor processor, String search, int maxResultCount) throws ParseException, IOException,
 754     DatabaseException {
 755
 756         // An empty search string will crash QueryParser
 757         // Just return no results for empty queries.
 758         //System.out.println("search: '" + search + "'");
 759         if (search.isEmpty())
 760             return Collections.emptyList();
 761
 762         assertAccessOpen(false);
 763
 764         Query query = Queries.parse(search, schema);
 765
 766         long start = System.nanoTime();
 767
 768         maxResultCount = Math.min(maxResultCount, searcher.getIndexReader().numDocs());
 769         if (maxResultCount == 0)
 770             return Collections.emptyList();
 771
 772         final TopDocs docs = searcher.search(query, null, maxResultCount);
 773
 774 //        for(Object[] o : debugDocs(monitor)) {
 775 //            System.err.println("-" + Arrays.toString(o));
 776 //        }
 777
 778         if (IndexPolicy.PERF_INDEX_QUERY) {
 779             long end = System.nanoTime();
 780             System.out.println(getDescriptor() + "search(" + search + ", " + maxResultCount + ") into index at " + indexPath + " took " + (1e-9 * (end-start)) + " seconds.");
 781         }
 782
 783         if (docs.totalHits == 0) {
 784             return Collections.emptyList();
 785         }
 786
 787         return processor.syncRequest(new Read<List<Map<String, Object>>>() {
 788
 789             @Override
 790             public List<Map<String, Object>> perform(ReadGraph graph) throws DatabaseException {
 791
 792                 GenericRelation r = graph.adapt(relation, GenericRelation.class);
 793                 if (r == null)
 794                     throw new DatabaseException("Given resource " + graph.syncRequest(new SafeName(relation))
 795                             + "could not be adapted to GenericRelation.");
 796
 797                 SerialisationSupport support = graph.getService(SerialisationSupport.class);
 798
 799                 List<Map<String, Object>> result = new ArrayList<Map<String, Object>>(docs.scoreDocs.length);
 800
 801                 final DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor();
 802
 803                 for (ScoreDoc scoreDoc : docs.scoreDocs) {
 804
 805                     try {
 806
 807                         reader.document(scoreDoc.doc, visitor);
 808
 809                         Document doc = visitor.getDocument();
 810
 811                         List<IndexableField> fs = doc.getFields();
 812                         Map<String, Object> entry = new THashMap<String, Object>(fs.size());
 813                         for (IndexableField f : fs) {
 814                             IndexSchema.Type type = schema.typeMap.get(f.name());
 815                             if (type == IndexSchema.Type.LONG) {
 816                                 entry.put(f.name(), support.getResource(f.numericValue().longValue()));
 817                             } else {
 818                                 entry.put(f.name(), f.stringValue());
 819                             }
 820                         }
 821
 822                         result.add(entry);
 823
 824                     } catch (CorruptIndexException e) {
 825                         throw new DatabaseException(e);
 826                     } catch (IOException e) {
 827                         throw new DatabaseException(e);
 828                     }
 829
 830                 }
 831
 832                 return result;
 833
 834             }
 835         });
 836     }
 837
 838     static class ResourceVisitor extends StoredFieldVisitor {
 839
 840         public long id;
 841
 842                 @Override
 843                 public Status needsField(FieldInfo fieldInfo) throws IOException {
 844                         if("Resource".equals(fieldInfo.name)) return Status.YES;
 845                         return Status.NO;
 846                 }
 847
 848                 @Override
 849                 public void longField(FieldInfo fieldInfo, long value) throws IOException {
 850                         id = value;
 851                 }
 852
 853     };
 854
 855     static class DumpVisitor extends StoredFieldVisitor {
 856
 857         public List<Object> values;
 858
 859         DumpVisitor(List<Object> values) {
 860                 this.values = values;
 861         }
 862
 863                 @Override
 864                 public Status needsField(FieldInfo fieldInfo) throws IOException {
 865                         return Status.YES;
 866                 }
 867
 868                 @Override
 869                 public void longField(FieldInfo fieldInfo, long value) throws IOException {
 870                         values.add(value);
 871                 }
 872
 873                 @Override
 874                 public void stringField(FieldInfo fieldInfo, String value) throws IOException {
 875                         values.add(value);
 876                 }
 877
 878     }
 879
 880     List<Resource> doSearchResources(IProgressMonitor monitor, RequestProcessor processor, String search, int maxResultCount) throws ParseException, IOException,
 881     DatabaseException {
 882
 883         // An empty search string will crash QueryParser
 884         // Just return no results for empty queries.
 885         //System.out.println("search: '" + search + "'");
 886         if (search.isEmpty())
 887             return Collections.emptyList();
 888
 889         assertAccessOpen(false);
 890
 891         Query query = Queries.parse(search, schema);
 892
 893         long start = System.nanoTime();
 894
 895         maxResultCount = Math.min(maxResultCount, searcher.getIndexReader().numDocs());
 896         if (maxResultCount == 0)
 897             return Collections.emptyList();
 898
 899         final TopDocs docs = searcher.search(query, null, maxResultCount);
 900
 901 //        for(Object[] o : debugDocs(monitor)) {
 902 //            System.err.println("-" + Arrays.toString(o));
 903 //        }
 904
 905         if (IndexPolicy.PERF_INDEX_QUERY) {
 906             long end = System.nanoTime();
 907             System.out.println(getDescriptor() + "search(" + search + ", " + maxResultCount + ") into index at " + indexPath + " took " + (1e-9 * (end-start)) + " seconds.");
 908         }
 909
 910         if (docs.totalHits == 0) {
 911             return Collections.emptyList();
 912         }
 913
 914         return processor.syncRequest(new Read<List<Resource>>() {
 915
 916             @Override
 917             public List<Resource> perform(ReadGraph graph) throws DatabaseException {
 918
 919                 CollectionSupport cs = graph.getService(CollectionSupport.class);
 920                 SerialisationSupport support = graph.getService(SerialisationSupport.class);
 921
 922                 List<Resource> result = cs.createList();
 923
 924                 ResourceVisitor visitor = new ResourceVisitor();
 925
 926                 for (ScoreDoc scoreDoc : docs.scoreDocs) {
 927
 928                     try {
 929
 930                         reader.document(scoreDoc.doc, visitor);
 931                         result.add(support.getResource(visitor.id));
 932
 933                     } catch (CorruptIndexException e) {
 934                         throw new DatabaseException(e);
 935                     } catch (IOException e) {
 936                         throw new DatabaseException(e);
 937                     }
 938
 939                 }
 940
 941                 return result;
 942
 943             }
 944         });
 945     }
 946
 947     List<Object> doList(IProgressMonitor monitor, RequestProcessor processor) throws ParseException, IOException,
 948     DatabaseException {
 949
 950         assertAccessOpen(false);
 951
 952         Query query = new MatchAllDocsQuery();
 953
 954         final TopDocs docs = searcher.search(query, Integer.MAX_VALUE);
 955
 956         ArrayList<Object> result = new ArrayList<Object>();
 957
 958         DumpVisitor visitor = new DumpVisitor(result);
 959
 960         for (ScoreDoc scoreDoc : docs.scoreDocs) {
 961
 962                 try {
 963
 964                         reader.document(scoreDoc.doc, visitor);
 965
 966                 } catch (CorruptIndexException e) {
 967                         throw new DatabaseException(e);
 968                 } catch (IOException e) {
 969                         throw new DatabaseException(e);
 970                 }
 971
 972         }
 973
 974         return result;
 975
 976     }
 977
 978     protected static File getIndexDirectory(Session session, Resource relation, Resource input) {
 979         File path = DatabaseIndexing.getIndexLocation(session, relation, input);
 980 //        System.out.println("getIndexDirectory = " + path);
 981         return path;
 982     }
 983
 984     File getIndexPath() {
 985         return indexPath;
 986     }
 987
 988     boolean isIndexAvailable() {
 989         return (indexPath.exists() && indexPath.isDirectory());
 990     }
 991
 992     Throwable bestEffortClear(IProgressMonitor monitor, Session session) {
 993         return null;
 994     }
 995
 996     /*
 997      * Start from scratch. Clear all caches and rebuild the index.
 998      */
 999     Throwable clearDirectory(IProgressMonitor monitor, Session session) {
1000
1001                 File file = getIndexPath();
1002
1003         try {
1004
1005                 for(int i=0;i<15;i++) {
1006                         FileUtils.deleteDir(file);
1007                         if(!file.exists()) {
1008                                 return null;
1009                         }
1010                         try {
1011                                 Thread.sleep(i*100);
1012                         } catch (InterruptedException e) {
1013                         }
1014                 }
1015
1016         } catch (Throwable t) {
1017
1018                 return t;
1019
1020         }
1021
1022         return new IllegalStateException("Failed to delete directory " + file.getAbsolutePath());
1023
1024     }
1025
1026     private Field[] setFields(Field[] fs, Object[] result) {
1027         for (int i = 0; i < result.length; i++) {
1028             Object value = result[i];
1029             if (value instanceof String) {
1030                 if (IndexPolicy.DEBUG_INDEX_INIT)
1031                     System.out.println(getDescriptor() + "index " + fs[i].name() + " = " + value + " : String");
1032                 fs[i].setStringValue((String) value);
1033             } else if (value instanceof Long) {
1034                 if (IndexPolicy.DEBUG_INDEX_INIT)
1035                     System.out.println(getDescriptor() + "index " + fs[i].name() + " = " + value + " : Long");
1036                 fs[i].setLongValue((Long) value);
1037             } else {
1038                 Logger.defaultLogError("Can only index Long and String fields, encountered " + value);
1039                 return null;
1040             }
1041         }
1042         return fs;
1043     }
1044
1045 }