bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsSearcherBase.java

   1 /*******************************************************************************
   2  * Copyright (c) 2007, 2015 Association for Decentralized Information Management
   3  * in Industry THTH ry.
   4  * All rights reserved. This program and the accompanying materials
   5  * are made available under the terms of the Eclipse Public License v1.0
   6  * which accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *     VTT Technical Research Centre of Finland - initial API and implementation
  11  *     Semantum Oy - Fix for simantics issue #6053
  12  *******************************************************************************/
  13 package org.simantics.db.indexing;
  14
  15 import java.io.File;
  16 import java.io.IOException;
  17 import java.nio.file.Files;
  18 import java.nio.file.Path;
  19 import java.util.ArrayList;
  20 import java.util.Collection;
  21 import java.util.Collections;
  22 import java.util.Iterator;
  23 import java.util.List;
  24 import java.util.Map;
  25 import java.util.concurrent.ExecutorService;
  26 import java.util.concurrent.Executors;
  27 import java.util.concurrent.Semaphore;
  28 import java.util.concurrent.ThreadFactory;
  29 import java.util.concurrent.atomic.AtomicReference;
  30
  31 import org.apache.lucene.document.Document;
  32 import org.apache.lucene.document.DocumentStoredFieldVisitor;
  33 import org.apache.lucene.document.Field;
  34 import org.apache.lucene.document.FieldType;
  35 import org.apache.lucene.document.LongField;
  36 import org.apache.lucene.document.TextField;
  37 import org.apache.lucene.index.CorruptIndexException;
  38 import org.apache.lucene.index.DirectoryReader;
  39 import org.apache.lucene.index.FieldInfo;
  40 import org.apache.lucene.index.IndexNotFoundException;
  41 import org.apache.lucene.index.IndexReader;
  42 import org.apache.lucene.index.IndexWriter;
  43 import org.apache.lucene.index.IndexWriterConfig;
  44 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
  45 import org.apache.lucene.index.IndexableField;
  46 import org.apache.lucene.index.StoredFieldVisitor;
  47 import org.apache.lucene.index.Term;
  48 import org.apache.lucene.queryparser.classic.ParseException;
  49 import org.apache.lucene.search.IndexSearcher;
  50 import org.apache.lucene.search.MatchAllDocsQuery;
  51 import org.apache.lucene.search.Query;
  52 import org.apache.lucene.search.ScoreDoc;
  53 import org.apache.lucene.search.TermQuery;
  54 import org.apache.lucene.search.TopDocs;
  55 import org.apache.lucene.store.Directory;
  56 import org.apache.lucene.store.FSDirectory;
  57 import org.apache.lucene.util.Version;
  58 import org.eclipse.core.runtime.IProgressMonitor;
  59 import org.eclipse.core.runtime.SubMonitor;
  60 import org.simantics.databoard.util.ObjectUtils;
  61 import org.simantics.db.ReadGraph;
  62 import org.simantics.db.RequestProcessor;
  63 import org.simantics.db.Resource;
  64 import org.simantics.db.Session;
  65 import org.simantics.db.common.request.SafeName;
  66 import org.simantics.db.common.utils.Logger;
  67 import org.simantics.db.common.utils.NameUtils;
  68 import org.simantics.db.exception.DatabaseException;
  69 import org.simantics.db.indexing.internal.IndexingJob;
  70 import org.simantics.db.layer0.adapter.GenericRelation;
  71 import org.simantics.db.request.Read;
  72 import org.simantics.db.service.CollectionSupport;
  73 import org.simantics.db.service.SerialisationSupport;
  74 import org.simantics.utils.FileUtils;
  75 import org.simantics.utils.datastructures.Pair;
  76
  77 import gnu.trove.map.hash.THashMap;
  78
  79 /**
  80  * @author Tuukka Lehtonen
  81  * @author Antti Villberg
  82  */
  83 abstract public class IndexedRelationsSearcherBase {
  84
  85     protected enum State {
  86         // No index is available
  87         NONE,
  88         // An index is available, but there is a problem with it
  89         PROBLEM,
  90         // An index is available but no reader or writer is ready
  91         READY,
  92         // A reader is ready
  93         READ,
  94         // A writer (and a reader) is ready
  95         WRITE
  96     }
  97
  98     private State state = State.READY;
  99     private Throwable exception;
 100
 101     public Throwable getException() {
 102         return exception;
 103     }
 104
 105     public void setProblem(Throwable t) {
 106         this.state = State.PROBLEM;
 107         this.exception = t;
 108     }
 109
 110     public void setNone() {
 111         this.state = State.NONE;
 112     }
 113
 114     public void setReady() {
 115         this.state = State.READY;
 116     }
 117
 118     protected boolean checkState(State state) {
 119         return this.state == state;
 120     }
 121
 122     protected void assertState(State state) throws AssertionError {
 123
 124         if(this.state != state) throw new AssertionError("Illegal state, expected " + state.name() + " but was in " + this.state.name());
 125
 126     }
 127
 128     public void changeState(IProgressMonitor monitor, Session session, State state) {
 129         changeState(monitor, session, state, 0);
 130     }
 131
 132     protected void changeState(IProgressMonitor monitor, Session session, State state, int depth) {
 133
 134         if(this.state == state) return;
 135
 136         if (IndexPolicy.TRACE_INDEX_MANAGEMENT)
 137                 System.err.println("Index state " + this.state.name() + " => " + state.name() + " " + this);
 138
 139         // Check transitions
 140
 141         // Try to exit problem state
 142         if (State.PROBLEM == this.state && depth > 0) {
 143                 Throwable t = bestEffortClear(monitor, session);
 144                 if(t != null) {
 145                                 exception = t;
 146                                 return;
 147                 }
 148                 // Managed to get into initial state
 149                 this.state = State.NONE;
 150                 return;
 151         }
 152
 153         // Cannot move into read from no index
 154         if (State.NONE ==  this.state && State.READ == state) return;
 155         // Cannot move into write from no index
 156         if (State.NONE ==  this.state && State.WRITE == state) return;
 157
 158                 boolean success = false;
 159
 160         try {
 161
 162                 if (searcher != null) {
 163                         searcher = null;
 164                 }
 165                 if (reader != null) {
 166                         reader.close();
 167                         reader = null;
 168                 }
 169                         closeWriter(writer);
 170                         directory = null;
 171
 172                         success = true;
 173
 174                 // Enter new state
 175                 if (State.READ == state || State.WRITE == state) {
 176
 177                         success = false;
 178
 179                         boolean forWriting = State.WRITE == state;
 180
 181                         if (directory != null)
 182                                 throw new IllegalStateException(getDescriptor() + "Index already loaded");
 183
 184                         SubMonitor mon = SubMonitor.convert(monitor, 100);
 185
 186                         mon.beginTask("Loading index", 100);
 187
 188                         if (IndexPolicy.TRACE_INDEX_LOAD)
 189                     System.out.println(getDescriptor() + "Loading Lucene index from " + indexPath + " for " + (forWriting ? "writing" : "reading"));
 190
 191                 long start = System.nanoTime();
 192
 193                 directory = getDirectory(session);
 194
 195                 if (forWriting) {
 196                     // Never overwrite an index that is about to be loaded.
 197                     // TODO: could use OpenMode.CREATE_OR_APPEND but must test first
 198                     IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_4_9, Queries.getAnalyzer()).setOpenMode(OpenMode.APPEND);
 199                     try {
 200                         // FIXME: platform #4676
 201                         writer = new IndexWriter(directory, config);
 202                     } catch (IndexNotFoundException e) {
 203                         // There was no pre-existing index on disk. Create it now.
 204                         writer = new IndexWriter(directory, config.setOpenMode(OpenMode.CREATE));
 205                         writer.commit();
 206                     }
 207                     reader = DirectoryReader.open(directory);
 208                     searcher = new IndexSearcher(reader);
 209                 } else {
 210                     reader = DirectoryReader.open(directory);
 211                     searcher = new IndexSearcher(reader);
 212                 }
 213
 214                 long end = System.nanoTime();
 215
 216                 mon.worked(100);
 217
 218                 if (IndexPolicy.PERF_INDEX_LOAD) {
 219                     double time = (end - start) * 1e-6;
 220                     System.out.println(getDescriptor() + "Loaded Lucene index from " + indexPath + " for " + (forWriting ? "writing" : "reading") + " in " + time + " ms");
 221                 }
 222
 223                 success = true;
 224
 225                 }
 226
 227         } catch (Throwable t) {
 228
 229                 setProblem(t);
 230
 231         } finally {
 232
 233                 if(!success) {
 234                         this.state = State.PROBLEM;
 235                         changeState(monitor, session, State.NONE, depth+1);
 236                         return;
 237                 }
 238
 239         }
 240
 241         this.state = state;
 242
 243     }
 244
 245     public static final FieldType STRING_TYPE = new FieldType();
 246
 247     static {
 248       STRING_TYPE.setIndexed(true);
 249       STRING_TYPE.setStored(true);
 250       STRING_TYPE.setTokenized(true);
 251       STRING_TYPE.freeze();
 252     }
 253
 254     protected static Field makeField(String fieldName, String fieldClass) throws DatabaseException {
 255         switch (fieldClass) {
 256         case "Long":   return new LongField(fieldName, 0L, Field.Store.YES);
 257         case "String": return new Field    (fieldName, "", STRING_TYPE);
 258         case "Text":   return new TextField(fieldName, "", Field.Store.YES);
 259         default:
 260             throw new DatabaseException("Can only index Long, String and Text fields, encountered field type " + fieldClass);
 261         }
 262     }
 263
 264     protected static Field[] makeFieldsForRelation(GenericRelation r, int boundLength, Document document) throws DatabaseException {
 265         Pair<String, String>[] fields = r.getFields();
 266         Field[] fs = new Field[Math.max(0, fields.length - boundLength)];
 267         for (int i = boundLength; i < fields.length; i++) {
 268             Field f = makeField(fields[i].first, fields[i].second);
 269             fs[i - boundLength] = f;
 270             if (document != null)
 271                 document.add(f);
 272         }
 273         return fs;
 274     }
 275
 276     void insertIndex(IProgressMonitor monitor, GenericRelation r, int boundLength, Collection<Object[]> documentsData)
 277     throws CorruptIndexException, IOException, DatabaseException {
 278         assertAccessOpen(true);
 279
 280         if (IndexPolicy.TRACE_INDEX_UPDATE)
 281             System.out.println(getDescriptor() + "Inserting " + documentsData.size() + " documents into index at " + indexPath);
 282
 283         long start = 0, end = 0;
 284         if (IndexPolicy.PERF_INDEX_UPDATE)
 285             start = System.nanoTime();
 286
 287         try {
 288             Document document = new Document();
 289             Field[] fs = makeFieldsForRelation(r, boundLength, document);
 290
 291             for (Object[] documentData : documentsData) {
 292                 if (setFields(fs, documentData) == null)
 293                     continue;
 294
 295                 if (IndexPolicy.TRACE_INDEX_UPDATE)
 296                     System.out.println(getDescriptor() + "Inserting document " + document);
 297
 298                 writer.addDocument(document);
 299             }
 300
 301             if (IndexPolicy.PERF_INDEX_UPDATE) {
 302                 end = System.nanoTime();
 303                 double ms = (end - start) * 1e-6;
 304                 System.out.println(getDescriptor() + "Inserted " + documentsData.size() + " documents into index at " + indexPath + " in " + ms + " ms");
 305             }
 306
 307         } finally {
 308         }
 309     }
 310
 311     void removeIndex(IProgressMonitor monitor, GenericRelation r, RequestProcessor processor, String key, Collection<Object> keyValues) throws DatabaseException, CorruptIndexException, IOException {
 312         assertAccessOpen(true);
 313
 314         if (IndexPolicy.TRACE_INDEX_UPDATE)
 315             System.out.println(getDescriptor() + "Removing " + keyValues.size() + " documents from index at " + indexPath);
 316
 317         long start = 0, end = 0;
 318         if (IndexPolicy.PERF_INDEX_UPDATE)
 319             start = System.nanoTime();
 320
 321         try {
 322             for (Object keyValue : keyValues) {
 323                 Term removedTerm = null;
 324                 if (keyValue instanceof Long) {
 325                     removedTerm = IndexUtils.longTerm(key, (Long) keyValue);
 326                 } else if (keyValue instanceof String) {
 327                     removedTerm = new Term(key, (String) keyValue);
 328                 } else {
 329                     // FIXME: should throw an exception for illegal input data but this would leave the index in an incoherent state
 330                     continue;
 331                 }
 332
 333                 if (IndexPolicy.TRACE_INDEX_UPDATE)
 334                     System.out.println(getDescriptor() + "Removing document with key " + removedTerm);
 335                 writer.deleteDocuments(removedTerm);
 336             }
 337
 338             if (IndexPolicy.PERF_INDEX_UPDATE) {
 339                 end = System.nanoTime();
 340                 double ms = (end - start) * 1e-6;
 341                 System.out.println(getDescriptor() + "Removed " + keyValues.size() + " documents from index at " + indexPath + " in " + ms + " ms");
 342             }
 343
 344         } finally {
 345         }
 346     }
 347
 348     void removeIndex(IProgressMonitor monitor) throws DatabaseException, CorruptIndexException, IOException {
 349         assertAccessOpen(true);
 350
 351         long start = 0, end = 0;
 352         if (IndexPolicy.PERF_INDEX_UPDATE)
 353             start = System.nanoTime();
 354
 355         try {
 356
 357             writer.deleteAll();
 358
 359             if (IndexPolicy.PERF_INDEX_UPDATE) {
 360                 end = System.nanoTime();
 361                 double ms = (end - start) * 1e-6;
 362                 System.out.println(getDescriptor() + "Removed all documents from index at " + indexPath + " in " + ms + " ms");
 363             }
 364
 365         } finally {
 366         }
 367     }
 368
 369     boolean replaceIndex(IProgressMonitor monitor, String key, Collection<Object> keyValues, GenericRelation r, int boundLength, Collection<Object[]> documentsData) throws CorruptIndexException, IOException, DatabaseException {
 370
 371         boolean didReplace = false;
 372
 373         assertAccessOpen(true);
 374         if (keyValues.size() != documentsData.size())
 375             throw new IllegalArgumentException("keyValues size does not match documents data size, " + keyValues.size() + " <> " + documentsData.size());
 376
 377         if (IndexPolicy.TRACE_INDEX_UPDATE)
 378             System.out.println(getDescriptor() + "Replacing " + keyValues.size() + " documents from index at " + indexPath);
 379
 380         long start = 0, end = 0;
 381         if (IndexPolicy.PERF_INDEX_UPDATE)
 382             start = System.nanoTime();
 383
 384         try {
 385             Iterator<Object> keyIt = keyValues.iterator();
 386             Iterator<Object[]> documentDataIt = documentsData.iterator();
 387
 388             Document document = new Document();
 389             Field[] fs = makeFieldsForRelation(r, boundLength, document);
 390
 391             nextDocument:
 392                 while (keyIt.hasNext()) {
 393                     Object keyValue = keyIt.next();
 394                     Object[] documentData = documentDataIt.next();
 395
 396                     Term removedTerm = null;
 397                     if (keyValue instanceof Long) {
 398                         removedTerm = IndexUtils.longTerm(key, (Long) keyValue);
 399                     } else if (keyValue instanceof String) {
 400                         removedTerm = new Term(key, (String) keyValue);
 401                     } else {
 402                         // FIXME: should throw an exception for illegal input data but this would leave the index in an incoherent state
 403                         System.err.println("[" + getClass().getSimpleName() + "] Unrecognized document key to remove '" + keyValue + "', only " + String.class + " and " + Resource.class + " are supported.");
 404                         continue nextDocument;
 405                     }
 406
 407                     if (setFields(fs, documentData) == null)
 408                         continue nextDocument;
 409
 410                     if (IndexPolicy.TRACE_INDEX_UPDATE)
 411                         System.out.println(getDescriptor() + "Replacing document with key " + removedTerm + " with " + document);
 412
 413                     boolean done = false;
 414                     if(requireChangeInfoOnReplace()) {
 415                             TopDocs exist = searcher.search(new TermQuery(removedTerm), null, 2);
 416                             if(exist.scoreDocs.length == 1 && requireChangeInfoOnReplace()) {
 417                                 Document doc = reader.document(exist.scoreDocs[0].doc);
 418                                 if(!areSame(doc, document)) {
 419                                     writer.deleteDocuments(removedTerm);
 420                                     writer.addDocument(document);
 421                                     didReplace |= true;
 422                                     if (IndexPolicy.TRACE_INDEX_UPDATE)
 423                                         System.out.println("-replaced single existing");
 424                                 } else {
 425                                     if (IndexPolicy.TRACE_INDEX_UPDATE)
 426                                         System.out.println("-was actually same than single existing");
 427                                 }
 428                                 done = true;
 429                             }
 430                     }
 431                     if(!done) {
 432                         writer.deleteDocuments(removedTerm);
 433                         writer.addDocument(document);
 434                         didReplace |= true;
 435                         if (IndexPolicy.TRACE_INDEX_UPDATE)
 436                                 System.out.println("-had many or none - removed all existing");
 437                     }
 438
 439                 }
 440
 441             if (IndexPolicy.PERF_INDEX_UPDATE) {
 442                 end = System.nanoTime();
 443                 double ms = (end - start) * 1e-6;
 444                 System.out.println(getDescriptor() + "Replaced " + keyValues.size() + " documents from index at " + indexPath + " in " + ms + " ms");
 445             }
 446
 447         } finally {
 448         }
 449
 450         return didReplace;
 451
 452     }
 453
 454     protected boolean requireChangeInfoOnReplace() {
 455         return true;
 456     }
 457
 458     private boolean areSame(Document d1, Document d2) {
 459         List<IndexableField> fs1 = d1.getFields();
 460         List<IndexableField> fs2 = d2.getFields();
 461         if(fs1.size() != fs2.size()) return false;
 462         for(int i=0;i<fs1.size();i++) {
 463                 IndexableField f1 = fs1.get(i);
 464                 IndexableField f2 = fs2.get(i);
 465                 String s1 = f1.stringValue();
 466                 String s2 = f2.stringValue();
 467             if (IndexPolicy.TRACE_INDEX_UPDATE)
 468                 System.err.println("areSame " + s1 + " vs. " + s2 );
 469                 if(!ObjectUtils.objectEquals(s1,s2)) return false;
 470         }
 471         return true;
 472     }
 473
 474     final RequestProcessor session;
 475
 476     final Resource         relation;
 477
 478     /**
 479      * The schema of the index, i.e. the fields that will be indexed per
 480      * document for the specified relation. Since the relation stays the same
 481      * throughout the lifetime of this class, the index schema is also assumed
 482      * to the same. This means that {@link GenericRelation#getFields()} is
 483      * assumed to stay the same.
 484      */
 485     final IndexSchema      schema;
 486
 487     Resource         input;
 488
 489     File             indexPath;
 490
 491     Directory        directory;
 492
 493     IndexReader      reader;
 494
 495     IndexWriter      writer;
 496
 497     IndexSearcher    searcher;
 498
 499     IndexedRelationsSearcherBase(RequestProcessor session, Resource relation, Resource input) {
 500         this.session = session;
 501         this.relation = relation;
 502         this.input = input;
 503         this.indexPath = getIndexDirectory(session.getSession(), relation, input);
 504         if(isIndexAvailable()) {
 505                 state = State.READY;
 506         } else {
 507                 state = State.NONE;
 508         }
 509         this.schema = IndexSchema.readFromRelation(session, relation);
 510     }
 511
 512     Directory getDirectory(Session session) throws IOException {
 513         return FSDirectory.open(indexPath);
 514     }
 515
 516     abstract String getDescriptor();
 517
 518     /**
 519      * Ensures that searcher is in read or write state.
 520      *
 521      * @param forWriting <code>true</code> to open index for writing,
 522      *        <code>false</code> for reading
 523      * @return true is required state was reached
 524      *
 525      */
 526     boolean startAccess(IProgressMonitor monitor, Session session, boolean forWriting) {
 527         if(forWriting) {
 528                 changeState(monitor, session, State.WRITE);
 529                 return checkState(State.WRITE);
 530         } else {
 531                 changeState(monitor, session, State.READ);
 532                 return checkState(State.READ);
 533         }
 534     }
 535
 536     boolean hasAccess(boolean forWriting) {
 537
 538         if (forWriting)
 539                 return checkState(State.WRITE);
 540         else
 541                 return checkState(State.WRITE) || checkState(State.READ);
 542
 543     }
 544
 545     void assertAccessOpen(boolean forWriting) {
 546         if (forWriting)
 547                 if(!checkState(State.WRITE))
 548                 throw new IllegalStateException("index not opened for writing (directory=" + directory + ", reader=" + reader + ")");
 549         else
 550                 if(!(checkState(State.WRITE) || checkState(State.READ)))
 551                 throw new IllegalStateException("index not opened for reading (directory=" + directory + ", writer=" + writer + ")");
 552     }
 553
 554     void closeWriter(IndexWriter writer) throws CorruptIndexException, IOException {
 555         if (writer == null)
 556             return;
 557
 558         try {
 559             // May throw OOME, see IndexWriter javadoc for the correct actions.
 560             writer.close(false);
 561         } catch (OutOfMemoryError e) {
 562             writer.close();
 563             throw e;
 564         }
 565     }
 566
 567     private static String getPattern(GenericRelation relation, int boundCount) {
 568         String result = "";
 569         for (int i = 0; i < boundCount; i++)
 570             result += "b";
 571         for (int i = 0; i < relation.getFields().length - boundCount; i++)
 572             result += "f";
 573         return result;
 574     }
 575
 576     private static final int INDEXING_THREAD_COUNT = 2;
 577
 578     private static final ExecutorService executor = Executors.newFixedThreadPool(INDEXING_THREAD_COUNT, new ThreadFactory() {
 579         @Override
 580         public Thread newThread(Runnable r) {
 581             Thread t = new Thread(r, "Lucene Index Creator");
 582             if (!t.isDaemon())
 583                 t.setDaemon(true);
 584             if (t.getPriority() != Thread.NORM_PRIORITY)
 585                 t.setPriority(Thread.NORM_PRIORITY);
 586             return t;
 587         }
 588     });
 589
 590     void initializeIndex(IProgressMonitor monitor, ReadGraph graph, Object[] bound, boolean overwrite)
 591             throws IOException, DatabaseException
 592     {
 593         IndexingJob.jobifyIfPossible(
 594                 monitor,
 595                 "Reindexing " + NameUtils.getSafeLabel(graph, input),
 596                 mon -> {
 597                     try {
 598                         initializeIndexImpl(mon, graph, bound, overwrite);
 599                     } catch (IOException e) {
 600                         throw new DatabaseException(e);
 601                     }
 602                 });
 603     }
 604
 605     void initializeIndexImpl(IProgressMonitor monitor, ReadGraph graph, final Object[] bound, boolean overwrite) throws IOException,
 606     DatabaseException {
 607
 608         final SubMonitor mon = SubMonitor.convert(monitor, 100);
 609
 610         if (IndexPolicy.TRACE_INDEX_INIT)
 611             System.out.println(getDescriptor() + "Initializing index at " + indexPath + " (overwrite = " + overwrite + ")");
 612         mon.beginTask("Initializing Index", 100);
 613
 614         if (overwrite) {
 615             mon.subTask("Erasing previous index");
 616             FileUtils.deleteAll(indexPath);
 617         }
 618
 619         final AtomicReference<FSDirectory> directory = new AtomicReference<FSDirectory>();
 620         final AtomicReference<IndexWriter> writer = new AtomicReference<IndexWriter>();
 621
 622         try {
 623             mon.subTask("Start index write");
 624             createDirectory(indexPath);
 625
 626             directory.set(FSDirectory.open(indexPath));
 627             IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_4_9, Queries.getAnalyzer()).setOpenMode(OpenMode.CREATE);
 628             writer.set(new IndexWriter(directory.get(), conf));
 629
 630             mon.worked(5);
 631
 632             final GenericRelation r = graph.adapt(relation, GenericRelation.class);
 633             if (r == null)
 634                 throw new DatabaseException("Given resource " + graph.syncRequest(new SafeName(relation))
 635                 + "could not be adapted to GenericRelation.");
 636
 637             long realizeStart = 0;
 638             if (IndexPolicy.PERF_INDEX_INIT)
 639                 realizeStart = System.nanoTime();
 640
 641             mon.subTask("Calculating indexed content");
 642             GenericRelation selection = r.select(getPattern(r, bound.length), bound);
 643             mon.worked(5);
 644             List<Object[]> results = selection.realize(graph);
 645             mon.worked(40);
 646
 647             if (IndexPolicy.PERF_INDEX_INIT)
 648                 System.out.println(getDescriptor() + "Realized index with " + results.size() + " entries at " + indexPath + " in " + (1e-9 * (System.nanoTime()-realizeStart)) + " seconds.");
 649             if (IndexPolicy.TRACE_INDEX_INIT)
 650                 System.out.println(getDescriptor() + "Indexed relation " + r + " produced " + results.size() + " results");
 651
 652             long start = IndexPolicy.PERF_INDEX_INIT ? System.nanoTime() : 0;
 653
 654             mon.subTask("Indexing content");
 655             final Semaphore s = new Semaphore(0);
 656             mon.setWorkRemaining(results.size());
 657
 658             for (int i = 0; i < INDEXING_THREAD_COUNT; i++) {
 659                 final int startIndex = i;
 660                 executor.submit(() -> {
 661                     try {
 662                         Document document = new Document();
 663                         Field[] fs = makeFieldsForRelation(r, bound.length, document);
 664
 665                         for (int index = startIndex; index < results.size(); index += INDEXING_THREAD_COUNT) {
 666                             if (setFields(fs, results.get(index)) == null)
 667                                 continue;
 668                             try {
 669                                 writer.get().addDocument(document);
 670                             } catch (CorruptIndexException e) {
 671                                 throw new IllegalStateException(e);
 672                             } catch (IOException e) {
 673                                 throw new IllegalStateException(e);
 674                             } finally {
 675                                 synchronized (mon) {
 676                                     mon.worked(1);
 677                                 }
 678                             }
 679                         }
 680
 681                         s.release();
 682                     } catch (DatabaseException e) {
 683                         throw new IllegalStateException(e);
 684                     }
 685                 });
 686             }
 687
 688             try {
 689                 s.acquire(INDEXING_THREAD_COUNT);
 690             } catch (InterruptedException e) {
 691                 Logger.defaultLogError(e);
 692             }
 693
 694             // http://www.gossamer-threads.com/lists/lucene/java-dev/47895
 695             // and http://lucene.apache.org/java/docs/index.html#27+November+2011+-+Lucene+Core+3.5.0
 696             // advise against calling optimize at all. So let's not do it anymore.
 697             //writer.get().optimize();
 698             //writer.get().commit();
 699
 700             mon.subTask("Flushing");
 701
 702             if (IndexPolicy.PERF_INDEX_INIT)
 703                 System.out.println(getDescriptor() + "Wrote index at " + indexPath + " in " + (1e-9 * (System.nanoTime()-start)) + " seconds.");
 704
 705         } catch (DatabaseException e) {
 706
 707             Logger.defaultLogError(e);
 708
 709         } finally {
 710             try {
 711                 closeWriter(writer.getAndSet(null));
 712             } finally {
 713                 FileUtils.uncheckedClose(directory.getAndSet(null));
 714             }
 715         }
 716     }
 717
 718
 719     public List<Object[]> debugDocs(IProgressMonitor monitor) throws ParseException, IOException, DatabaseException {
 720
 721             Query query = new MatchAllDocsQuery();
 722
 723             TopDocs td = searcher.search(query, Integer.MAX_VALUE);
 724
 725             ScoreDoc[ ] scoreDocs = td.scoreDocs;
 726             List<Object[]> result = new ArrayList<Object[]>(scoreDocs.length);
 727
 728             for(ScoreDoc scoreDoc:scoreDocs) {
 729
 730                 try {
 731
 732                     Document doc = reader.document(scoreDoc.doc);
 733                     List<IndexableField> fs = doc.getFields();
 734                     Object[] o = new Object[fs.size()];
 735                     int index = 0;
 736                     for (IndexableField f : fs) {
 737                     o[index++] = f.stringValue();
 738                     }
 739                     result.add(o);
 740
 741                 } catch (CorruptIndexException e) {
 742                     throw new DatabaseException(e);
 743                 } catch (IOException e) {
 744                     throw new DatabaseException(e);
 745                 }
 746
 747             }
 748
 749             return result;
 750
 751     }
 752
 753
 754     List<Map<String, Object>> doSearch(IProgressMonitor monitor, RequestProcessor processor, String search, int maxResultCount) throws ParseException, IOException,
 755     DatabaseException {
 756
 757         // An empty search string will crash QueryParser
 758         // Just return no results for empty queries.
 759         //System.out.println("search: '" + search + "'");
 760         if (search.isEmpty())
 761             return Collections.emptyList();
 762
 763         assertAccessOpen(false);
 764
 765         Query query = Queries.parse(search, schema);
 766
 767         long start = System.nanoTime();
 768
 769         maxResultCount = Math.min(maxResultCount, searcher.getIndexReader().numDocs());
 770         if (maxResultCount == 0)
 771             return Collections.emptyList();
 772
 773         final TopDocs docs = searcher.search(query, null, maxResultCount);
 774
 775 //        for(Object[] o : debugDocs(monitor)) {
 776 //            System.err.println("-" + Arrays.toString(o));
 777 //        }
 778
 779         if (IndexPolicy.PERF_INDEX_QUERY) {
 780             long end = System.nanoTime();
 781             System.out.println(getDescriptor() + "search(" + search + ", " + maxResultCount + ") into index at " + indexPath + " took " + (1e-9 * (end-start)) + " seconds.");
 782         }
 783
 784         if (docs.totalHits == 0) {
 785             return Collections.emptyList();
 786         }
 787
 788         return processor.syncRequest(new Read<List<Map<String, Object>>>() {
 789
 790             @Override
 791             public List<Map<String, Object>> perform(ReadGraph graph) throws DatabaseException {
 792
 793                 GenericRelation r = graph.adapt(relation, GenericRelation.class);
 794                 if (r == null)
 795                     throw new DatabaseException("Given resource " + graph.syncRequest(new SafeName(relation))
 796                             + "could not be adapted to GenericRelation.");
 797
 798                 SerialisationSupport support = graph.getService(SerialisationSupport.class);
 799
 800                 List<Map<String, Object>> result = new ArrayList<Map<String, Object>>(docs.scoreDocs.length);
 801
 802                 final DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor();
 803
 804                 for (ScoreDoc scoreDoc : docs.scoreDocs) {
 805
 806                     try {
 807
 808                         reader.document(scoreDoc.doc, visitor);
 809
 810                         Document doc = visitor.getDocument();
 811
 812                         List<IndexableField> fs = doc.getFields();
 813                         Map<String, Object> entry = new THashMap<String, Object>(fs.size());
 814                         for (IndexableField f : fs) {
 815                             IndexSchema.Type type = schema.typeMap.get(f.name());
 816                             if (type == IndexSchema.Type.LONG) {
 817                                 entry.put(f.name(), support.getResource(f.numericValue().longValue()));
 818                             } else {
 819                                 entry.put(f.name(), f.stringValue());
 820                             }
 821                         }
 822
 823                         result.add(entry);
 824
 825                     } catch (CorruptIndexException e) {
 826                         throw new DatabaseException(e);
 827                     } catch (IOException e) {
 828                         throw new DatabaseException(e);
 829                     }
 830
 831                 }
 832
 833                 return result;
 834
 835             }
 836         });
 837     }
 838
 839     static class ResourceVisitor extends StoredFieldVisitor {
 840
 841         public long id;
 842
 843                 @Override
 844                 public Status needsField(FieldInfo fieldInfo) throws IOException {
 845                         if("Resource".equals(fieldInfo.name)) return Status.YES;
 846                         return Status.NO;
 847                 }
 848
 849                 @Override
 850                 public void longField(FieldInfo fieldInfo, long value) throws IOException {
 851                         id = value;
 852                 }
 853
 854     };
 855
 856     static class DumpVisitor extends StoredFieldVisitor {
 857
 858         public List<Object> values;
 859
 860         DumpVisitor(List<Object> values) {
 861                 this.values = values;
 862         }
 863
 864                 @Override
 865                 public Status needsField(FieldInfo fieldInfo) throws IOException {
 866                         return Status.YES;
 867                 }
 868
 869                 @Override
 870                 public void longField(FieldInfo fieldInfo, long value) throws IOException {
 871                         values.add(value);
 872                 }
 873
 874                 @Override
 875                 public void stringField(FieldInfo fieldInfo, String value) throws IOException {
 876                         values.add(value);
 877                 }
 878
 879     }
 880
 881     List<Resource> doSearchResources(IProgressMonitor monitor, RequestProcessor processor, String search, int maxResultCount) throws ParseException, IOException,
 882     DatabaseException {
 883
 884         // An empty search string will crash QueryParser
 885         // Just return no results for empty queries.
 886         //System.out.println("search: '" + search + "'");
 887         if (search.isEmpty())
 888             return Collections.emptyList();
 889
 890         assertAccessOpen(false);
 891
 892         Query query = Queries.parse(search, schema);
 893
 894         long start = System.nanoTime();
 895
 896         maxResultCount = Math.min(maxResultCount, searcher.getIndexReader().numDocs());
 897         if (maxResultCount == 0)
 898             return Collections.emptyList();
 899
 900         final TopDocs docs = searcher.search(query, null, maxResultCount);
 901
 902 //        for(Object[] o : debugDocs(monitor)) {
 903 //            System.err.println("-" + Arrays.toString(o));
 904 //        }
 905
 906         if (IndexPolicy.PERF_INDEX_QUERY) {
 907             long end = System.nanoTime();
 908             System.out.println(getDescriptor() + "search(" + search + ", " + maxResultCount + ") into index at " + indexPath + " took " + (1e-9 * (end-start)) + " seconds.");
 909         }
 910
 911         if (docs.totalHits == 0) {
 912             return Collections.emptyList();
 913         }
 914
 915         return processor.syncRequest(new Read<List<Resource>>() {
 916
 917             @Override
 918             public List<Resource> perform(ReadGraph graph) throws DatabaseException {
 919
 920                 CollectionSupport cs = graph.getService(CollectionSupport.class);
 921                 SerialisationSupport support = graph.getService(SerialisationSupport.class);
 922
 923                 List<Resource> result = cs.createList();
 924
 925                 ResourceVisitor visitor = new ResourceVisitor();
 926
 927                 for (ScoreDoc scoreDoc : docs.scoreDocs) {
 928
 929                     try {
 930
 931                         reader.document(scoreDoc.doc, visitor);
 932                         result.add(support.getResource(visitor.id));
 933
 934                     } catch (CorruptIndexException e) {
 935                         throw new DatabaseException(e);
 936                     } catch (IOException e) {
 937                         throw new DatabaseException(e);
 938                     }
 939
 940                 }
 941
 942                 return result;
 943
 944             }
 945         });
 946     }
 947
 948     List<Object> doList(IProgressMonitor monitor, RequestProcessor processor) throws ParseException, IOException,
 949     DatabaseException {
 950
 951         assertAccessOpen(false);
 952
 953         Query query = new MatchAllDocsQuery();
 954
 955         final TopDocs docs = searcher.search(query, Integer.MAX_VALUE);
 956
 957         ArrayList<Object> result = new ArrayList<Object>();
 958
 959         DumpVisitor visitor = new DumpVisitor(result);
 960
 961         for (ScoreDoc scoreDoc : docs.scoreDocs) {
 962
 963                 try {
 964
 965                         reader.document(scoreDoc.doc, visitor);
 966
 967                 } catch (CorruptIndexException e) {
 968                         throw new DatabaseException(e);
 969                 } catch (IOException e) {
 970                         throw new DatabaseException(e);
 971                 }
 972
 973         }
 974
 975         return result;
 976
 977     }
 978
 979     protected static File getIndexDirectory(Session session, Resource relation, Resource input) {
 980         File path = DatabaseIndexing.getIndexLocation(session, relation, input);
 981 //        System.out.println("getIndexDirectory = " + path);
 982         return path;
 983     }
 984
 985     private static void createDirectory(File path) throws IOException {
 986         Path p = path.toPath();
 987         if (Files.exists(p) && !Files.isDirectory(p))
 988             throw new IOException("Could not create index directory " + path + ", a file by that name already exists");
 989         Files.createDirectories(p);
 990     }
 991
 992     File getIndexPath() {
 993         return indexPath;
 994     }
 995
 996     boolean isIndexAvailable() {
 997         return (indexPath.exists() && indexPath.isDirectory());
 998     }
 999
1000     Throwable bestEffortClear(IProgressMonitor monitor, Session session) {
1001         return null;
1002     }
1003
1004     /*
1005      * Start from scratch. Clear all caches and rebuild the index.
1006      */
1007     Throwable clearDirectory(IProgressMonitor monitor, Session session) {
1008
1009                 File file = getIndexPath();
1010
1011         try {
1012
1013                 for(int i=0;i<15;i++) {
1014                         FileUtils.deleteDir(file);
1015                         if(!file.exists()) {
1016                                 return null;
1017                         }
1018                         try {
1019                                 Thread.sleep(i*100);
1020                         } catch (InterruptedException e) {
1021                         }
1022                 }
1023
1024         } catch (Throwable t) {
1025
1026                 return t;
1027
1028         }
1029
1030         return new IllegalStateException("Failed to delete directory " + file.getAbsolutePath());
1031
1032     }
1033
1034     private Field[] setFields(Field[] fs, Object[] result) {
1035         for (int i = 0; i < result.length; i++) {
1036             Object value = result[i];
1037             if (value instanceof String) {
1038                 if (IndexPolicy.DEBUG_INDEX_INIT)
1039                     System.out.println(getDescriptor() + "index " + fs[i].name() + " = " + value + " : String");
1040                 fs[i].setStringValue((String) value);
1041             } else if (value instanceof Long) {
1042                 if (IndexPolicy.DEBUG_INDEX_INIT)
1043                     System.out.println(getDescriptor() + "index " + fs[i].name() + " = " + value + " : Long");
1044                 fs[i].setLongValue((Long) value);
1045             } else {
1046                 Logger.defaultLogError("Can only index Long and String fields, encountered " + value);
1047                 return null;
1048             }
1049         }
1050         return fs;
1051     }
1052
1053 }