bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsSearcher.java

   1 /*******************************************************************************
   2  * Copyright (c) 2007, 2010 Association for Decentralized Information Management
   3  * in Industry THTH ry.
   4  * All rights reserved. This program and the accompanying materials
   5  * are made available under the terms of the Eclipse Public License v1.0
   6  * which accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *     VTT Technical Research Centre of Finland - initial API and implementation
  11  *******************************************************************************/
  12 package org.simantics.db.indexing;
  13
  14 import java.io.IOException;
  15 import java.util.ArrayList;
  16 import java.util.Collection;
  17 import java.util.HashMap;
  18 import java.util.List;
  19 import java.util.Map;
  20
  21 import org.apache.lucene.index.CorruptIndexException;
  22 import org.apache.lucene.queryparser.classic.ParseException;
  23 import org.eclipse.core.runtime.IProgressMonitor;
  24 import org.simantics.db.RequestProcessor;
  25 import org.simantics.db.Resource;
  26 import org.simantics.db.Session;
  27 import org.simantics.db.exception.DatabaseException;
  28 import org.simantics.db.layer0.adapter.GenericRelation;
  29 import org.simantics.utils.datastructures.Pair;
  30 import org.slf4j.Logger;
  31 import org.slf4j.LoggerFactory;
  32
  33 /**
  34  * @author Tuukka Lehtonen
  35  * @author Antti Villberg
  36  */
  37 public class IndexedRelationsSearcher extends IndexedRelationsSearcherBase {
  38
  39     private static final Logger LOGGER = LoggerFactory.getLogger(IndexedRelationsSearcher.class);
  40
  41     IndexedRelationsMemorySearcher cache;
  42
  43     IndexedRelationsSearcher(RequestProcessor session, Resource relation, Resource input, GenericRelation r) throws DatabaseException {
  44         super(session, relation, input);
  45         this.cache = new IndexedRelationsMemorySearcher(session, this, relation, input, r);
  46     }
  47
  48     @Override
  49     String getDescriptor() {
  50         return "DISK: ";
  51     }
  52
  53     @Override
  54     public void setProblem(Throwable t) {
  55         super.setProblem(t);
  56         cache.setProblem(t);
  57     }
  58
  59     @Override
  60     boolean startAccess(IProgressMonitor monitor, Session session, boolean forWriting) {
  61         boolean success = super.startAccess(monitor, session, false);
  62         if(!success) return false;
  63         success = cache.startAccess(monitor, session, forWriting);
  64         if(!success) {
  65                 setProblem(cache.getException());
  66                 return false;
  67         } else return true;
  68     }
  69
  70     @Override
  71     void insertIndex(IProgressMonitor monitor, GenericRelation r, int boundLength, Collection<Object[]> documentsData)
  72             throws CorruptIndexException, IOException, DatabaseException {
  73
  74         Collection<Object> keyValues = new ArrayList<>();
  75         for(Object[] data : documentsData) {
  76             keyValues.add(data[1]);
  77         }
  78
  79         cache.replaceIndex(monitor, "Resource", keyValues, r, boundLength, documentsData);
  80         //cache.commit();
  81
  82     }
  83
  84     @Override
  85     void removeIndex(IProgressMonitor monitor) throws DatabaseException, CorruptIndexException, IOException {
  86         super.removeIndex(monitor);
  87         cache.removeIndex(monitor);
  88     }
  89
  90     @Override
  91     void removeIndex(IProgressMonitor monitor, GenericRelation r, RequestProcessor processor, String key, Collection<Object> keyValues) throws DatabaseException,CorruptIndexException, IOException {
  92
  93         Collection<Object[]> documentsData = new ArrayList<>();
  94
  95         Pair<String,String>[] fields = r.getFields();
  96
  97         for(Object keyValue : keyValues) {
  98             Object[] data = new Object[fields.length-1];
  99             int index = 0;
 100             for(int i=1;i<fields.length;i++) {
 101                 String fieldName = fields[i].first;
 102                 if(key.equals(fieldName)) {
 103                     data[index++] = keyValue;
 104                 } else {
 105                     String fieldClass = fields[i].second;
 106                     if ("Long".equals(fieldClass)) {
 107                         data[index++] = 0L;
 108                     } else if ("String".equals(fieldClass) || "Text".equals(fieldClass)) {
 109                         data[index++] = "";
 110                     } else {
 111                         throw new DatabaseException("Can only index Long and String fields, encountered class " + fieldClass);
 112                     }
 113                 }
 114             }
 115             documentsData.add(data);
 116         }
 117
 118         cache.replaceIndex(monitor, key, keyValues, r, 1, documentsData);
 119 //        cache.commit();
 120
 121     }
 122
 123     @Override
 124     boolean replaceIndex(IProgressMonitor monitor, String key, Collection<Object> keyValues, GenericRelation r,
 125             int boundLength, Collection<Object[]> documentsData) throws CorruptIndexException, IOException,
 126             DatabaseException {
 127
 128         boolean result = cache.replaceIndex(monitor, key, keyValues, r, boundLength, documentsData);
 129 //        cache.commit();
 130         return result;
 131
 132     }
 133
 134     List<Map<String, Object>> persistentCachedSearch(IProgressMonitor monitor, RequestProcessor processor, String search,
 135             int maxResultCount) throws ParseException, IOException, DatabaseException {
 136
 137         MemoryIndexing mem = MemoryIndexing.getInstance(session.getSession());
 138
 139         String key = indexPath.toAbsolutePath().toString();
 140
 141         Map<String,List<Map<String, Object>>> cache = mem.persistentCache.get(key);
 142         if(cache != null) {
 143             List<Map<String,Object>> result = cache.get(search);
 144             if(result != null) return result;
 145         }
 146
 147         startAccess(monitor, processor.getSession(), false);
 148
 149         List<Map<String, Object>> results = super.doSearch(monitor, processor, search, maxResultCount);
 150         if(cache == null) {
 151             cache = new HashMap<>();
 152             mem.persistentCache.put(key, cache);
 153         }
 154
 155         if(results.size() < 500)
 156             cache.put(search, results);
 157
 158         return results;
 159
 160     }
 161
 162     List<Resource> persistentCachedSearchResources(IProgressMonitor monitor, RequestProcessor processor, String search,
 163             int maxResultCount) throws ParseException, IOException, DatabaseException {
 164
 165         MemoryIndexing mem = MemoryIndexing.getInstance(session.getSession());
 166
 167         String key = indexPath.toAbsolutePath().toString();
 168
 169         Map<String,List<Resource>> cache = mem.persistentCacheResources.get(key);
 170         if(cache != null) {
 171             List<Resource> result = cache.get(search);
 172             if(result != null) return result;
 173         }
 174
 175         startAccess(monitor, processor.getSession(), false);
 176
 177         List<Resource> results = super.doSearchResources(monitor, processor, search, maxResultCount);
 178         if(cache == null) {
 179             cache = new HashMap<>();
 180             mem.persistentCacheResources.put(key, cache);
 181         }
 182
 183         if(results.size() < 500)
 184             cache.put(search, results);
 185
 186         return results;
 187
 188     }
 189
 190     List<Object> persistentCachedList(IProgressMonitor monitor, RequestProcessor processor) throws ParseException, IOException, DatabaseException {
 191
 192         startAccess(monitor, processor.getSession(), false);
 193
 194         List<Object> results = super.doList(monitor, processor);
 195
 196         return results;
 197
 198     }
 199
 200     @Override
 201     List<Map<String, Object>> doSearch(IProgressMonitor monitor, RequestProcessor processor, String search,
 202             int maxResultCount) throws ParseException, IOException, DatabaseException {
 203
 204         List<Map<String,Object>> persistent = persistentCachedSearch(monitor, processor, search, maxResultCount);
 205         List<Map<String,Object>> cached = cache.doSearch(monitor, processor, search, maxResultCount);
 206
 207         ArrayList<Map<String,Object>> result = new ArrayList<>();
 208
 209         for(Map<String,Object> m : persistent) {
 210             Resource r = (Resource)m.get("Resource");
 211             if(!cache.changed.contains(r.getResourceId())) {
 212                 result.add(m);
 213             }
 214         }
 215         result.addAll(cached);
 216         return result;
 217
 218     }
 219
 220     @Override
 221     List<Resource> doSearchResources(IProgressMonitor monitor, RequestProcessor processor, String search,
 222                 int maxResultCount) throws ParseException, IOException, DatabaseException {
 223
 224         List<Resource> persistent = persistentCachedSearchResources(monitor, processor, search, maxResultCount);
 225         List<Resource> cached = cache.doSearchResources(monitor, processor, search, maxResultCount);
 226
 227         ArrayList<Resource> result = new ArrayList<>();
 228         for(Resource r : persistent) {
 229             if(!cache.changed.contains(r.getResourceId())) {
 230                 result.add(r);
 231             }
 232         }
 233         result.addAll(cached);
 234         return result;
 235
 236     }
 237
 238     List<Object> doList(IProgressMonitor monitor, RequestProcessor processor) throws ParseException, IOException, DatabaseException {
 239
 240         List<Object> persistent = persistentCachedList(monitor, processor);
 241
 242         // TODO: check that caches have been properly flushed
 243         //List<Object> cached = cache.doList(monitor, processor);
 244         //if(!cached.isEmpty()) throw new DatabaseException("doList does not support caching");
 245
 246         return persistent;
 247
 248     }
 249
 250     void applyChanges(IProgressMonitor monitor, Session session, GenericRelation r, Collection<Object[]> os) throws Exception {
 251
 252         if(!os.isEmpty()) {
 253
 254                 ArrayList<Object> replaceKeys = new ArrayList<>();
 255                 ArrayList<Object[]> replaceValues = new ArrayList<>();
 256                 ArrayList<Object> removeKeys = new ArrayList<>();
 257                 for(Object[] o : os) {
 258                     Long parent = (Long)o[0];
 259                     Long key = (Long)o[1];
 260                     if(parent != 0) {
 261                         replaceKeys.add(key);
 262                         replaceValues.add(o);
 263                     } else {
 264                         removeKeys.add(key);
 265                     }
 266                 }
 267
 268                 changeState(monitor, session, State.READY);
 269
 270                 super.startAccess(null, session, true);
 271
 272                 super.replaceIndex(null, "Resource", replaceKeys, r, 1, replaceValues);
 273                 super.removeIndex(null, r, null, "Resource", removeKeys);
 274
 275         }
 276
 277         changeState(monitor, session, State.READY);
 278
 279     }
 280
 281     @Override
 282     Throwable bestEffortClear(IProgressMonitor monitor, Session session) {
 283
 284         // Free the index
 285         changeState(monitor, session, State.NONE);
 286
 287         Throwable t = clearDirectory(monitor, session);
 288         if(t != null) return t;
 289
 290         t = cache.bestEffortClear(monitor, session);
 291         if(t != null) return t;
 292
 293         String key = indexPath.toAbsolutePath().toString();
 294         MemoryIndexing mem = MemoryIndexing.getInstance(session);
 295         mem.persistentCache.remove(key);
 296         mem.persistentCacheResources.remove(key);
 297
 298         return null;
 299
 300     }
 301
 302     @Override
 303     protected Logger getLogger() {
 304         return LOGGER;
 305     }
 306
 307 }