bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsImpl.java

   1 /*******************************************************************************\r
   2  * Copyright (c) 2007, 2010 Association for Decentralized Information Management\r
   3  * in Industry THTH ry.\r
   4  * All rights reserved. This program and the accompanying materials\r
   5  * are made available under the terms of the Eclipse Public License v1.0\r
   6  * which accompanies this distribution, and is available at\r
   7  * http://www.eclipse.org/legal/epl-v10.html\r
   8  *\r
   9  * Contributors:\r
  10  *     VTT Technical Research Centre of Finland - initial API and implementation\r
  11  *******************************************************************************/\r
  12 package org.simantics.db.indexing;\r
  13 \r
  14 import java.io.File;\r
  15 import java.io.IOException;\r
  16 import java.util.Collection;\r
  17 import java.util.Collections;\r
  18 import java.util.List;\r
  19 import java.util.Map;\r
  20 import java.util.WeakHashMap;\r
  21 import java.util.concurrent.locks.Lock;\r
  22 import java.util.concurrent.locks.ReentrantReadWriteLock;\r
  23 \r
  24 import org.apache.lucene.queryparser.classic.ParseException;\r
  25 import org.eclipse.core.runtime.IProgressMonitor;\r
  26 import org.eclipse.core.runtime.SubMonitor;\r
  27 import org.simantics.db.ReadGraph;\r
  28 import org.simantics.db.RequestProcessor;\r
  29 import org.simantics.db.Resource;\r
  30 import org.simantics.db.common.request.UniqueRead;\r
  31 import org.simantics.db.exception.DatabaseException;\r
  32 import org.simantics.db.exception.InvalidResourceReferenceException;\r
  33 import org.simantics.db.indexing.IndexedRelationsSearcherBase.State;\r
  34 import org.simantics.db.layer0.adapter.GenericRelation;\r
  35 import org.simantics.db.layer0.genericrelation.IndexException;\r
  36 import org.simantics.db.layer0.genericrelation.IndexedRelations;\r
  37 import org.simantics.db.service.QueryControl;\r
  38 import org.simantics.db.service.SerialisationSupport;\r
  39 import org.simantics.utils.datastructures.Pair;\r
  40 \r
  41 /**\r
  42  * @author Tuukka Lehtonen\r
  43  * @author Antti Villberg\r
  44  */\r
  45 public class IndexedRelationsImpl implements IndexedRelations {\r
  46 \r
  47     Map<Object, RWLock> indexLocks = new WeakHashMap<Object, RWLock>();\r
  48 \r
  49     static class LockHandle {\r
  50         public final Object id;\r
  51         public final Lock   lock;\r
  52 \r
  53         public LockHandle(Object id, Lock lock) {\r
  54             this.id = id;\r
  55             this.lock = lock;\r
  56         }\r
  57 \r
  58         public void unlock() {\r
  59             if (IndexPolicy.TRACE_INDEX_LOCKING)\r
  60                 System.out.println("Unlocking index " + id);\r
  61             lock.unlock();\r
  62         }\r
  63     }\r
  64 \r
  65     static class RWLock {\r
  66         public final Object                 id;\r
  67         public final ReentrantReadWriteLock lock;\r
  68 \r
  69         public RWLock(Object id) {\r
  70             this.id = id;\r
  71             this.lock = new ReentrantReadWriteLock(true);\r
  72         }\r
  73 \r
  74         LockHandle lock(RequestProcessor processor, boolean write) {\r
  75             Lock l = write ? lock.writeLock() : lock.readLock();\r
  76             if(processor instanceof ReadGraph) {\r
  77                 ReadGraph graph = (ReadGraph)processor;\r
  78                 while(!l.tryLock()) {\r
  79                         QueryControl qc = processor.getService(QueryControl.class);\r
  80                         boolean executed = qc.resume(graph);\r
  81                         if(!executed) {\r
  82                                                 try {\r
  83                                                         Thread.sleep(1);\r
  84                                                 } catch (InterruptedException e) {\r
  85                                                 }\r
  86                         }\r
  87                 }\r
  88             } else {\r
  89                 l.lock();\r
  90             }\r
  91             if (IndexPolicy.TRACE_INDEX_LOCKING)\r
  92                 System.out.println("Locked index " + id);\r
  93             return new LockHandle(id, l);\r
  94         }\r
  95         \r
  96         LockHandle tryLock(RequestProcessor processor, boolean write) {\r
  97             Lock l = write ? lock.writeLock() : lock.readLock();\r
  98             if(l.tryLock()) return new LockHandle(id, l);\r
  99             else return null;\r
 100         }\r
 101         \r
 102     }\r
 103 \r
 104     private LockHandle lock(RequestProcessor processor, Object lockIdentifier, boolean write) {\r
 105         RWLock rwlock = null;\r
 106         synchronized (indexLocks) {\r
 107             rwlock = indexLocks.get(lockIdentifier);\r
 108             if (rwlock == null) {\r
 109                 rwlock = new RWLock(lockIdentifier);\r
 110                 indexLocks.put(lockIdentifier, rwlock);\r
 111             }\r
 112         }\r
 113         return rwlock.lock(processor, write);\r
 114     }\r
 115 \r
 116     private LockHandle tryLock(RequestProcessor processor, Object lockIdentifier, boolean write) {\r
 117         RWLock rwlock = null;\r
 118         synchronized (indexLocks) {\r
 119             rwlock = indexLocks.get(lockIdentifier);\r
 120             if (rwlock == null) {\r
 121                 rwlock = new RWLock(lockIdentifier);\r
 122                 indexLocks.put(lockIdentifier, rwlock);\r
 123             }\r
 124         }\r
 125         return rwlock.tryLock(processor, write);\r
 126     }\r
 127 \r
 128     private IndexedRelationsSearcherBase makeSearcher(final RequestProcessor processor, final Resource relation, final Resource input) {\r
 129         try {\r
 130                         return processor.syncRequest(new UniqueRead<IndexedRelationsSearcherBase>() {\r
 131 \r
 132                                 @Override\r
 133                                 public IndexedRelationsSearcherBase perform(ReadGraph graph) throws DatabaseException {\r
 134                                         if(graph.isImmutable(input)) {\r
 135                                                 return MemoryIndexing.getInstance(processor.getSession()).getImmutable(processor, relation, input);\r
 136                                         } else {\r
 137                                                 return MemoryIndexing.getInstance(processor.getSession()).get(processor, relation, input);\r
 138                                         }\r
 139                                 }\r
 140                                 \r
 141                         });\r
 142                 } catch (DatabaseException e) {\r
 143                         throw new IllegalStateException(e);\r
 144                 }\r
 145     }\r
 146     \r
 147     private LockHandle waitLoaded(SubMonitor progress, final IndexedRelationsSearcherBase searcher, RequestProcessor processor, LockHandle lock, final Object lockId, final Resource input) throws IndexException {\r
 148         \r
 149         // Initial state: we are locked, no news about the index\r
 150         // Final state: we are locked and the index has been loaded, the current lock is returned\r
 151 \r
 152         // First just check if the index is loaded\r
 153         if (searcher.isIndexAvailable()) {\r
 154                 // We have an index - try to start access\r
 155                 searcher.startAccess(progress.newChild(50), processor.getSession(), false);\r
 156                 // At this point we have three options:\r
 157                 // 1. we have access\r
 158                 if(searcher.hasAccess(false)) return lock;\r
 159                 // 2. something is wrong and the index cannot be cleaned\r
 160                 if(searcher.checkState(State.PROBLEM)) throw new IndexException("Searcher is in problematic state", searcher.getException());\r
 161                 // 3. something was wrong, but the index has been successfully cleaned\r
 162         }\r
 163 \r
 164                 if(!searcher.checkState(State.NONE)) \r
 165                         throw new IndexException("Illegal searcher state, contact application support.");\r
 166         \r
 167         // We loop until the index is loaded\r
 168         while(true) {\r
 169                 \r
 170                 // With ReadGraph we can proceed to initialize\r
 171                 if(processor instanceof ReadGraph) {\r
 172 \r
 173                 // (re)create the index.\r
 174                 try {\r
 175                     SerialisationSupport ss = processor.getService(SerialisationSupport.class);\r
 176                     searcher.initializeIndex(progress.newChild(40), (ReadGraph)processor, new Object[] { ss.getRandomAccessId(input) }, true);\r
 177                                         searcher.setReady();\r
 178                     searcher.startAccess(progress.newChild(10), processor.getSession(), false);\r
 179                         if(searcher.hasAccess(false)) return lock;\r
 180                 } catch (IOException e) {\r
 181                                         searcher.setProblem(e);\r
 182                     throw new IndexException(e);\r
 183                 } catch (DatabaseException e) {\r
 184                                         searcher.setProblem(e);\r
 185                     throw new IndexException(e);\r
 186                 }\r
 187                         \r
 188                 }\r
 189                 // With session we schedule the job\r
 190                 else {\r
 191 \r
 192                         // Release lock\r
 193                         lock.unlock();\r
 194                         \r
 195 //                      final Semaphore s = new Semaphore(0);\r
 196                         \r
 197                         // Schedule job\r
 198                         \r
 199                         boolean success = false;\r
 200                         int tries = 0;\r
 201                         while(!success && (++tries)<10) {\r
 202 \r
 203                                 try {\r
 204 \r
 205                                         success = processor.sync(new UniqueRead<Boolean>() {\r
 206 \r
 207                                                 @Override\r
 208                                                 public Boolean perform(ReadGraph graph) throws DatabaseException {\r
 209 \r
 210                                                         // Obtain lock\r
 211                                                         LockHandle lock = tryLock(graph, lockId, true);\r
 212                                                         if(lock == null) return false;\r
 213 \r
 214                                                         try {\r
 215 \r
 216                                                                 boolean loaded = false;\r
 217                                                                 if (searcher.isIndexAvailable()) {\r
 218                                                                         searcher.startAccess(null, graph.getSession(), false);\r
 219                                                                         // At this point we have three options:\r
 220                                                                         // 1. we have access\r
 221                                                                         if(searcher.hasAccess(false)) loaded = true;\r
 222                                                                         // 2. something is wrong and the index cannot be cleaned\r
 223                                                                         if(searcher.checkState(State.PROBLEM)) throw new DatabaseException("Searcher is in problematic state", searcher.getException());\r
 224                                                                         // 3. something was wrong, but the index has been successfully cleaned\r
 225                                                                 }\r
 226 \r
 227                                                                 if(!loaded) {\r
 228                                                                         \r
 229                                                                 if(!searcher.checkState(State.NONE)) \r
 230                                                                         throw new DatabaseException("Illegal searcher state, contact application support.");                                                            \r
 231 \r
 232                                                                         try {\r
 233                                                                             SerialisationSupport ss = graph.getService(SerialisationSupport.class);\r
 234                                                                                 searcher.initializeIndex(null, graph, new Object[] { ss.getRandomAccessId(input) }, true);\r
 235                                                                                 searcher.setReady();\r
 236                                                                         } catch (IOException e) {\r
 237                                                                                 searcher.setProblem(e);\r
 238                                                                                 throw new DatabaseException(e);\r
 239                                                                         }    \r
 240 \r
 241                                                                 }\r
 242 \r
 243                                                         } finally {\r
 244                                                                 \r
 245                                                                 lock.unlock();\r
 246 //                                                              s.release();\r
 247                                                                 \r
 248                                                         }\r
 249 \r
 250                                                         return true;\r
 251 \r
 252                                                 }\r
 253 \r
 254                                         });\r
 255 \r
 256                                 } catch (DatabaseException e) {\r
 257                                         throw new IndexException(e);\r
 258                                 }\r
 259 \r
 260                         }\r
 261                         \r
 262                         if(!success)\r
 263                                 throw new IndexException("Did not manage to load index. Contact application support.");\r
 264                 \r
 265                 // Try again\r
 266 \r
 267                 // Obtain lock\r
 268                                 lock= lock(processor, lockId, true);                    \r
 269                         \r
 270                         if (searcher.isIndexAvailable()) {\r
 271                                 searcher.startAccess(progress.newChild(50), processor.getSession(), false);\r
 272                                 if(searcher.hasAccess(false)) return lock;\r
 273                                 throw new IndexException("Illegal searcher state, contact application support.");\r
 274                         }\r
 275                         \r
 276                 }\r
 277                 \r
 278         }\r
 279 \r
 280         \r
 281     }\r
 282     \r
 283     @Override\r
 284     public List<Resource> queryResources(IProgressMonitor monitor, String search, RequestProcessor processor,\r
 285             Resource relation, final Resource input, int maxResultCount) {\r
 286         if (processor == null)\r
 287             throw new IllegalArgumentException("null processor");\r
 288         if (relation == null)\r
 289             throw new IllegalArgumentException("null relation");\r
 290         if (input == null)\r
 291             throw new IllegalArgumentException("null input");\r
 292         if (search == null)\r
 293             throw new IllegalArgumentException("null search criterion");\r
 294 \r
 295         SubMonitor progress = SubMonitor.convert(monitor, 100);\r
 296 \r
 297         // Look for existing index.\r
 298         // Indexes always exist in secondary storage, i.e. disk.\r
 299         // Indexes can be cached in memory when necessary performance-wise.\r
 300 \r
 301         final IndexedRelationsSearcherBase searcher = makeSearcher(processor, relation, input);\r
 302 \r
 303         final Object lockId = Pair.make(relation, input);\r
 304 \r
 305         LockHandle lock = lock(processor, lockId, false);\r
 306 \r
 307         // Ensure that index is loaded & ready\r
 308         lock = waitLoaded(progress, searcher, processor, lock, lockId, input);\r
 309 \r
 310         // Perform query\r
 311         try {\r
 312             return searcher.doSearchResources(progress.newChild(50), processor, search, maxResultCount);\r
 313         } catch (ParseException e) {\r
 314             // FIXME: should throw an exception, not just ignore.\r
 315             e.printStackTrace();\r
 316             return Collections.emptyList();\r
 317             //throw new IndexException(e);\r
 318         } catch (IOException e) {\r
 319             throw new IndexException(e);\r
 320         } catch (DatabaseException e) {\r
 321             throw new IndexException(e);\r
 322         } finally {\r
 323                 lock.unlock();\r
 324         }\r
 325     }\r
 326 \r
 327     @Override\r
 328     public List<Map<String, Object>> query(IProgressMonitor monitor, String search, RequestProcessor processor,\r
 329             Resource relation, final Resource input, int maxResultCount) {\r
 330         if (processor == null)\r
 331             throw new IllegalArgumentException("null processor");\r
 332         if (relation == null)\r
 333             throw new IllegalArgumentException("null relation");\r
 334         if (input == null)\r
 335             throw new IllegalArgumentException("null input");\r
 336         if (search == null)\r
 337             throw new IllegalArgumentException("null search criterion");\r
 338 \r
 339         SubMonitor progress = SubMonitor.convert(monitor, 100);\r
 340 \r
 341         // Look for existing index.\r
 342         // Indexes always exist in secondary storage, i.e. disk.\r
 343         // Indexes can be cached in memory when necessary performance-wise.\r
 344 \r
 345         final IndexedRelationsSearcherBase searcher = makeSearcher(processor, relation, input);\r
 346 \r
 347         final Object lockId = Pair.make(relation, input);\r
 348 \r
 349         LockHandle lock = lock(processor, lockId, false);\r
 350 \r
 351         // Ensure that index is loaded & ready\r
 352         lock = waitLoaded(progress, searcher, processor, lock, lockId, input);\r
 353 \r
 354         // Perform query\r
 355         try {\r
 356             return searcher.doSearch(progress.newChild(50), processor, search, maxResultCount);\r
 357         } catch (ParseException e) {\r
 358             // FIXME: should throw an exception, not just ignore.\r
 359             e.printStackTrace();\r
 360             return Collections.emptyList();\r
 361             //throw new IndexException(e);\r
 362         } catch (IOException e) {\r
 363             throw new IndexException(e);\r
 364         } catch (DatabaseException e) {\r
 365             throw new IndexException(e);\r
 366         } finally {\r
 367                 lock.unlock();\r
 368         }\r
 369     }\r
 370 \r
 371     @Override\r
 372     public void insert(IProgressMonitor monitor, RequestProcessor processor, GenericRelation relation,  \r
 373             Resource relationResource, Resource input, Collection<Object[]> documents) {\r
 374 \r
 375 //        System.out.println("Inserting to index: " + input + " " + documents);\r
 376 \r
 377         if (relation == null)\r
 378             throw new IllegalArgumentException("null relation");\r
 379         if (input == null)\r
 380             throw new IllegalArgumentException("null input");\r
 381         if (documents == null)\r
 382             throw new IllegalArgumentException("null documents");\r
 383 \r
 384         if (documents.isEmpty())\r
 385             return;\r
 386 \r
 387         final SubMonitor progress = SubMonitor.convert(monitor, 100);\r
 388 \r
 389         final IndexedRelationsSearcherBase searcher = makeSearcher(processor, relationResource, input);\r
 390 \r
 391         LockHandle handle = lock(processor, Pair.make(relationResource, input), true);\r
 392         \r
 393         try {\r
 394                 \r
 395                 if(!searcher.startAccess(null, processor.getSession(), true)) {\r
 396                 // Could not write index for some reason. Ignore and let the next index query reinitialize the index.\r
 397                         return;\r
 398                 }\r
 399                 \r
 400             searcher.insertIndex(progress.newChild(40), relation, 1, documents);\r
 401             DatabaseIndexing.markIndexChanged(searcher.getIndexPath());\r
 402             \r
 403         } catch (InvalidResourceReferenceException e) {\r
 404             throw new IndexException(e);\r
 405         } catch (IOException e) {\r
 406             throw new IndexException(e);\r
 407         } catch (DatabaseException e) {\r
 408             throw new IndexException(e);\r
 409         } finally {\r
 410                 handle.unlock();\r
 411         }\r
 412     }\r
 413 \r
 414     @Override\r
 415     public void remove(IProgressMonitor monitor, RequestProcessor processor, GenericRelation relation,\r
 416             Resource relationResource, Resource input, String key, Collection<Object> keyValues) {\r
 417 \r
 418         if (relation == null)\r
 419             throw new IllegalArgumentException("null relation");\r
 420         if (input == null)\r
 421             throw new IllegalArgumentException("null input");\r
 422         if (key == null)\r
 423             throw new IllegalArgumentException("null key");\r
 424 \r
 425         SubMonitor progress = SubMonitor.convert(monitor, 100);\r
 426 \r
 427         IndexedRelationsSearcherBase searcher = makeSearcher(processor, relationResource, input);\r
 428 \r
 429         LockHandle handle = lock(processor, Pair.make(relationResource, input), true);\r
 430         try {\r
 431                 \r
 432                 if(!searcher.startAccess(null, processor.getSession(), true)) {\r
 433                 // Could not write index for some reason. Ignore and let the next index query reinitialize the index.\r
 434                         return;\r
 435                 }\r
 436                 \r
 437             searcher.removeIndex(progress.newChild(40), relation, processor, key, keyValues);\r
 438             DatabaseIndexing.markIndexChanged(searcher.getIndexPath());\r
 439             \r
 440         } catch (DatabaseException e) {\r
 441             throw new IndexException(e);\r
 442         } catch (IOException e) {\r
 443             throw new IndexException(e);\r
 444         } finally {\r
 445                 handle.unlock();\r
 446         }\r
 447     }\r
 448 \r
 449     @Override\r
 450     public void removeAll(IProgressMonitor monitor, RequestProcessor processor, GenericRelation relation,\r
 451             Resource relationResource, Resource input) {\r
 452 \r
 453         if (relation == null)\r
 454             throw new IllegalArgumentException("null relation");\r
 455         if (input == null)\r
 456             throw new IllegalArgumentException("null input");\r
 457 \r
 458         IndexedRelationsSearcherBase searcher = makeSearcher(processor, relationResource, input);\r
 459         \r
 460         LockHandle handle = lock(processor, Pair.make(relationResource, input), true);\r
 461 \r
 462         try {\r
 463                 \r
 464                 Throwable t = searcher.bestEffortClear(monitor, processor.getSession());\r
 465                 if(t != null) searcher.setProblem(t);\r
 466                 else searcher.setNone();\r
 467                 \r
 468                 } finally {\r
 469                         handle.unlock();\r
 470                 }\r
 471         \r
 472     }\r
 473     \r
 474     @Override\r
 475     public boolean replace(IProgressMonitor monitor, RequestProcessor processor, GenericRelation relation,\r
 476             Resource relationResource, Resource input, String key, Collection<Object> keyValues, Collection<Object[]> documents) {\r
 477 \r
 478         if (relation == null)\r
 479             throw new IllegalArgumentException("null relation");\r
 480         if (input == null)\r
 481             throw new IllegalArgumentException("null input");\r
 482         if (key == null)\r
 483             throw new IllegalArgumentException("null key");\r
 484 \r
 485         SubMonitor progress = SubMonitor.convert(monitor, 100);\r
 486 \r
 487         IndexedRelationsSearcherBase searcher = makeSearcher(processor, relationResource, input);\r
 488 \r
 489         LockHandle handle = lock(processor, Pair.make(relationResource, input), true);\r
 490 \r
 491         boolean didChange = false;\r
 492 \r
 493         try {\r
 494                 \r
 495                 if(!searcher.startAccess(null, processor.getSession(), true)) {\r
 496                 // Could not write index for some reason. Ignore and let the next index query reinitialize the index.\r
 497                         return true;\r
 498                 }\r
 499             didChange = searcher.replaceIndex(progress.newChild(40), key, keyValues, relation, 1, documents);\r
 500             if(didChange)\r
 501                 DatabaseIndexing.markIndexChanged(searcher.getIndexPath());\r
 502             \r
 503         } catch (InvalidResourceReferenceException e) {\r
 504             throw new IndexException(e);\r
 505         } catch (IOException e) {\r
 506             throw new IndexException(e);\r
 507         } catch (DatabaseException e) {\r
 508             throw new IndexException(e);\r
 509         } catch (Throwable t) {\r
 510             throw new IndexException(t);\r
 511         } finally {\r
 512                 handle.unlock();\r
 513         }\r
 514         \r
 515         return didChange;\r
 516 \r
 517     }\r
 518     \r
 519     @Override\r
 520     public void reset(IProgressMonitor monitor, RequestProcessor processor, Resource relationResource, Resource input) throws IndexException {\r
 521 \r
 522         IndexedRelationsSearcherBase searcher = makeSearcher(processor, relationResource, input);\r
 523 \r
 524         LockHandle handle = lock(processor, Pair.make(relationResource, input), true);\r
 525 \r
 526         try {\r
 527             searcher.changeState(monitor, processor.getSession(), State.NONE);\r
 528             if (!searcher.checkState(State.NONE))\r
 529                 throw new IndexException("Could not close index for input " + input + " before removing it");\r
 530 \r
 531             File path = DatabaseIndexing.getIndexLocation(processor.getSession(), relationResource, input);\r
 532             DatabaseIndexing.deleteIndex(path);\r
 533 \r
 534         } catch (IOException e) {\r
 535             throw new IndexException(e);\r
 536         } finally {\r
 537             handle.unlock();\r
 538         }\r
 539 \r
 540     }\r
 541 \r
 542 }\r