bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsSearcher.java

   1 /*******************************************************************************
   2  * Copyright (c) 2007, 2010 Association for Decentralized Information Management
   3  * in Industry THTH ry.
   4  * All rights reserved. This program and the accompanying materials
   5  * are made available under the terms of the Eclipse Public License v1.0
   6  * which accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *     VTT Technical Research Centre of Finland - initial API and implementation
  11  *******************************************************************************/
  12 package org.simantics.db.indexing;
  13
  14 import java.io.IOException;
  15 import java.util.ArrayList;
  16 import java.util.Collection;
  17 import java.util.HashMap;
  18 import java.util.List;
  19 import java.util.Map;
  20
  21 import org.apache.lucene.index.CorruptIndexException;
  22 import org.apache.lucene.queryparser.classic.ParseException;
  23 import org.eclipse.core.runtime.IProgressMonitor;
  24 import org.simantics.db.RequestProcessor;
  25 import org.simantics.db.Resource;
  26 import org.simantics.db.Session;
  27 import org.simantics.db.exception.DatabaseException;
  28 import org.simantics.db.indexing.exception.IndexingException;
  29 import org.simantics.db.layer0.adapter.GenericRelation;
  30 import org.simantics.utils.datastructures.Pair;
  31 import org.slf4j.Logger;
  32 import org.slf4j.LoggerFactory;
  33
  34 /**
  35  * @author Tuukka Lehtonen
  36  * @author Antti Villberg
  37  */
  38 public class IndexedRelationsSearcher extends IndexedRelationsSearcherBase {
  39
  40     private static final Logger LOGGER = LoggerFactory.getLogger(IndexedRelationsSearcher.class);
  41
  42     IndexedRelationsMemorySearcher cache;
  43
  44     IndexedRelationsSearcher(RequestProcessor session, Resource relation, Resource input, GenericRelation r) throws DatabaseException {
  45         super(session, relation, input);
  46         this.cache = new IndexedRelationsMemorySearcher(session, this, relation, input, r);
  47     }
  48
  49     @Override
  50     String getDescriptor() {
  51         return "DISK: ";
  52     }
  53
  54     @Override
  55     public void setProblem(Throwable t) {
  56         super.setProblem(t);
  57         cache.setProblem(t);
  58     }
  59
  60     @Override
  61     boolean startAccess(IProgressMonitor monitor, Session session, boolean forWriting) {
  62         boolean success = super.startAccess(monitor, session, false);
  63         if(!success) return false;
  64         success = cache.startAccess(monitor, session, forWriting);
  65         if(!success) {
  66                 setProblem(cache.getException());
  67                 return false;
  68         } else return true;
  69     }
  70
  71     @Override
  72     void insertIndex(IProgressMonitor monitor, GenericRelation r, int boundLength, Collection<Object[]> documentsData)
  73             throws CorruptIndexException, IOException, DatabaseException {
  74
  75         Collection<Object> keyValues = new ArrayList<Object>();
  76         for(Object[] data : documentsData) {
  77             keyValues.add(data[1]);
  78         }
  79
  80         cache.replaceIndex(monitor, "Resource", keyValues, r, boundLength, documentsData);
  81         //cache.commit();
  82
  83     }
  84
  85     @Override
  86     void removeIndex(IProgressMonitor monitor, GenericRelation r, RequestProcessor processor, String key, Collection<Object> keyValues) throws DatabaseException,CorruptIndexException, IOException {
  87
  88         Collection<Object[]> documentsData = new ArrayList<Object[]>();
  89
  90         Pair<String,String>[] fields = r.getFields();
  91
  92         for(Object keyValue : keyValues) {
  93             Object[] data = new Object[fields.length-1];
  94             int index = 0;
  95             for(int i=1;i<fields.length;i++) {
  96                 String fieldName = fields[i].first;
  97                 if(key.equals(fieldName)) {
  98                     data[index++] = keyValue;
  99                 } else {
 100                     String fieldClass = fields[i].second;
 101                     if ("Long".equals(fieldClass)) {
 102                         data[index++] = 0L;
 103                     } else if ("String".equals(fieldClass) || "Text".equals(fieldClass)) {
 104                         data[index++] = "";
 105                     } else {
 106                         throw new IndexingException("Can only index Long and String fields, encountered class " + fieldClass);
 107                     }
 108                 }
 109             }
 110             documentsData.add(data);
 111         }
 112
 113         cache.replaceIndex(monitor, key, keyValues, r, 1, documentsData);
 114 //        cache.commit();
 115
 116     }
 117
 118     @Override
 119     boolean replaceIndex(IProgressMonitor monitor, String key, Collection<Object> keyValues, GenericRelation r,
 120             int boundLength, Collection<Object[]> documentsData) throws CorruptIndexException, IOException,
 121             DatabaseException {
 122
 123         boolean result = cache.replaceIndex(monitor, key, keyValues, r, boundLength, documentsData);
 124 //        cache.commit();
 125         return result;
 126
 127     }
 128
 129     List<Map<String, Object>> persistentCachedSearch(IProgressMonitor monitor, RequestProcessor processor, String search,
 130             int maxResultCount) throws ParseException, IOException, IndexingException {
 131
 132         MemoryIndexing mem = MemoryIndexing.getInstance(session.getSession());
 133
 134         String key = indexPath.toAbsolutePath().toString();
 135
 136         Map<String,List<Map<String, Object>>> cache = mem.persistentCache.get(key);
 137         if(cache != null) {
 138             List<Map<String,Object>> result = cache.get(search);
 139             if(result != null) return result;
 140         }
 141
 142         startAccess(monitor, processor.getSession(), false);
 143
 144         List<Map<String, Object>> results = super.doSearch(monitor, processor, search, maxResultCount);
 145         if(cache == null) {
 146             cache = new HashMap<String,List<Map<String,Object>>>();
 147             mem.persistentCache.put(key, cache);
 148         }
 149
 150         if(results.size() < 500)
 151             cache.put(search, results);
 152
 153         return results;
 154
 155     }
 156
 157     List<Resource> persistentCachedSearchResources(IProgressMonitor monitor, RequestProcessor processor, String search,
 158             int maxResultCount) throws ParseException, IOException, IndexingException {
 159
 160         MemoryIndexing mem = MemoryIndexing.getInstance(session.getSession());
 161
 162         String key = indexPath.toAbsolutePath().toString();
 163
 164         Map<String,List<Resource>> cache = mem.persistentCacheResources.get(key);
 165         if(cache != null) {
 166             List<Resource> result = cache.get(search);
 167             if(result != null) return result;
 168         }
 169
 170         startAccess(monitor, processor.getSession(), false);
 171
 172         List<Resource> results = super.doSearchResources(monitor, processor, search, maxResultCount);
 173         if(cache == null) {
 174             cache = new HashMap<String,List<Resource>>();
 175             mem.persistentCacheResources.put(key, cache);
 176         }
 177
 178         if(results.size() < 500)
 179             cache.put(search, results);
 180
 181         return results;
 182
 183     }
 184
 185     List<Object> persistentCachedList(IProgressMonitor monitor, RequestProcessor processor) throws ParseException, IOException, IndexingException {
 186
 187         startAccess(monitor, processor.getSession(), false);
 188
 189         List<Object> results = super.doList(monitor, processor);
 190
 191         return results;
 192
 193     }
 194
 195     @Override
 196     List<Map<String, Object>> doSearch(IProgressMonitor monitor, RequestProcessor processor, String search,
 197             int maxResultCount) throws ParseException, IOException, IndexingException {
 198
 199         List<Map<String,Object>> persistent = persistentCachedSearch(monitor, processor, search, maxResultCount);
 200         List<Map<String,Object>> cached = cache.doSearch(monitor, processor, search, maxResultCount);
 201
 202         ArrayList<Map<String,Object>> result = new ArrayList<Map<String,Object>>();
 203
 204         for(Map<String,Object> m : persistent) {
 205             Resource r = (Resource)m.get("Resource");
 206             if(!cache.changed.contains(r.getResourceId())) {
 207                 result.add(m);
 208             }
 209         }
 210         result.addAll(cached);
 211         return result;
 212
 213     }
 214
 215     @Override
 216     List<Resource> doSearchResources(IProgressMonitor monitor, RequestProcessor processor, String search,
 217                 int maxResultCount) throws ParseException, IOException, IndexingException {
 218
 219         List<Resource> persistent = persistentCachedSearchResources(monitor, processor, search, maxResultCount);
 220         List<Resource> cached = cache.doSearchResources(monitor, processor, search, maxResultCount);
 221
 222         ArrayList<Resource> result = new ArrayList<Resource>();
 223         for(Resource r : persistent) {
 224             if(!cache.changed.contains(r.getResourceId())) {
 225                 result.add(r);
 226             }
 227         }
 228         result.addAll(cached);
 229         return result;
 230
 231     }
 232
 233     List<Object> doList(IProgressMonitor monitor, RequestProcessor processor) throws ParseException, IOException, IndexingException {
 234
 235         List<Object> persistent = persistentCachedList(monitor, processor);
 236
 237         // TODO: check that caches have been properly flushed
 238         //List<Object> cached = cache.doList(monitor, processor);
 239         //if(!cached.isEmpty()) throw new DatabaseException("doList does not support caching");
 240
 241         return persistent;
 242
 243     }
 244
 245     void applyChanges(IProgressMonitor monitor, Session session, GenericRelation r, Collection<Object[]> os) throws Exception {
 246
 247         if(!os.isEmpty()) {
 248
 249                 ArrayList<Object> replaceKeys = new ArrayList<Object>();
 250                 ArrayList<Object[]> replaceValues = new ArrayList<Object[]>();
 251                 ArrayList<Object> removeKeys = new ArrayList<Object>();
 252                 for(Object[] o : os) {
 253                     Long parent = (Long)o[0];
 254                     Long key = (Long)o[1];
 255                     if(parent != 0) {
 256                         replaceKeys.add(key);
 257                         replaceValues.add(o);
 258                     } else {
 259                         removeKeys.add(key);
 260                     }
 261                 }
 262
 263                 changeState(monitor, session, State.READY);
 264
 265                 super.startAccess(null, session, true);
 266
 267                 super.replaceIndex(null, "Resource", replaceKeys, r, 1, replaceValues);
 268                 super.removeIndex(null, r, null, "Resource", removeKeys);
 269
 270         }
 271
 272         changeState(monitor, session, State.READY);
 273
 274     }
 275
 276     @Override
 277     Throwable bestEffortClear(IProgressMonitor monitor, Session session) {
 278
 279         // Free the index
 280         changeState(monitor, session, State.NONE);
 281
 282         Throwable t = clearDirectory(monitor, session);
 283         if(t != null) return t;
 284
 285         t = cache.bestEffortClear(monitor, session);
 286         if(t != null) return t;
 287
 288         String key = indexPath.toAbsolutePath().toString();
 289         MemoryIndexing mem = MemoryIndexing.getInstance(session);
 290         mem.persistentCache.remove(key);
 291         mem.persistentCacheResources.remove(key);
 292
 293         return null;
 294
 295     }
 296
 297     @Override
 298     protected Logger getLogger() {
 299         return LOGGER;
 300     }
 301
 302 }