bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsSearcher.java

   1 /*******************************************************************************
   2  * Copyright (c) 2007, 2010 Association for Decentralized Information Management
   3  * in Industry THTH ry.
   4  * All rights reserved. This program and the accompanying materials
   5  * are made available under the terms of the Eclipse Public License v1.0
   6  * which accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *     VTT Technical Research Centre of Finland - initial API and implementation
  11  *******************************************************************************/
  12 package org.simantics.db.indexing;
  13
  14 import java.io.IOException;
  15 import java.util.ArrayList;
  16 import java.util.Collection;
  17 import java.util.HashMap;
  18 import java.util.List;
  19 import java.util.Map;
  20
  21 import org.apache.lucene.index.CorruptIndexException;
  22 import org.apache.lucene.queryparser.classic.ParseException;
  23 import org.eclipse.core.runtime.IProgressMonitor;
  24 import org.simantics.db.RequestProcessor;
  25 import org.simantics.db.Resource;
  26 import org.simantics.db.Session;
  27 import org.simantics.db.exception.DatabaseException;
  28 import org.simantics.db.layer0.adapter.GenericRelation;
  29 import org.simantics.utils.datastructures.Pair;
  30
  31 /**
  32  * @author Tuukka Lehtonen
  33  * @author Antti Villberg
  34  */
  35 public class IndexedRelationsSearcher extends IndexedRelationsSearcherBase {
  36
  37     IndexedRelationsMemorySearcher cache;
  38
  39     IndexedRelationsSearcher(RequestProcessor session, Resource relation, Resource input, GenericRelation r) throws DatabaseException {
  40         super(session, relation, input);
  41         this.cache = new IndexedRelationsMemorySearcher(session, this, relation, input, r);
  42     }
  43
  44     @Override
  45     String getDescriptor() {
  46         return "DISK: ";
  47     }
  48
  49     @Override
  50     public void setProblem(Throwable t) {
  51         super.setProblem(t);
  52         cache.setProblem(t);
  53     }
  54
  55     @Override
  56     boolean startAccess(IProgressMonitor monitor, Session session, boolean forWriting) {
  57         boolean success = super.startAccess(monitor, session, false);
  58         if(!success) return false;
  59         success = cache.startAccess(monitor, session, forWriting);
  60         if(!success) {
  61                 setProblem(cache.getException());
  62                 return false;
  63         } else return true;
  64     }
  65
  66     @Override
  67     void insertIndex(IProgressMonitor monitor, GenericRelation r, int boundLength, Collection<Object[]> documentsData)
  68             throws CorruptIndexException, IOException, DatabaseException {
  69
  70         Collection<Object> keyValues = new ArrayList<Object>();
  71         for(Object[] data : documentsData) {
  72             keyValues.add(data[1]);
  73         }
  74
  75         cache.replaceIndex(monitor, "Resource", keyValues, r, boundLength, documentsData);
  76         //cache.commit();
  77
  78     }
  79
  80     @Override
  81     void removeIndex(IProgressMonitor monitor, GenericRelation r, RequestProcessor processor, String key, Collection<Object> keyValues) throws DatabaseException,CorruptIndexException, IOException {
  82
  83         Collection<Object[]> documentsData = new ArrayList<Object[]>();
  84
  85         Pair<String,String>[] fields = r.getFields();
  86
  87         for(Object keyValue : keyValues) {
  88             Object[] data = new Object[fields.length-1];
  89             int index = 0;
  90             for(int i=1;i<fields.length;i++) {
  91                 String fieldName = fields[i].first;
  92                 if(key.equals(fieldName)) {
  93                     data[index++] = keyValue;
  94                 } else {
  95                     String fieldClass = fields[i].second;
  96                     if ("Long".equals(fieldClass)) {
  97                         data[index++] = 0L;
  98                     } else if ("String".equals(fieldClass) || "Text".equals(fieldClass)) {
  99                         data[index++] = "";
 100                     } else {
 101                         throw new DatabaseException("Can only index Long and String fields, encountered class " + fieldClass);
 102                     }
 103                 }
 104             }
 105             documentsData.add(data);
 106         }
 107
 108         cache.replaceIndex(monitor, key, keyValues, r, 1, documentsData);
 109 //        cache.commit();
 110
 111     }
 112
 113     @Override
 114     boolean replaceIndex(IProgressMonitor monitor, String key, Collection<Object> keyValues, GenericRelation r,
 115             int boundLength, Collection<Object[]> documentsData) throws CorruptIndexException, IOException,
 116             DatabaseException {
 117
 118         boolean result = cache.replaceIndex(monitor, key, keyValues, r, boundLength, documentsData);
 119 //        cache.commit();
 120         return result;
 121
 122     }
 123
 124     List<Map<String, Object>> persistentCachedSearch(IProgressMonitor monitor, RequestProcessor processor, String search,
 125             int maxResultCount) throws ParseException, IOException, DatabaseException {
 126
 127         MemoryIndexing mem = MemoryIndexing.getInstance(session.getSession());
 128
 129         String key = indexPath.getAbsolutePath();
 130
 131         Map<String,List<Map<String, Object>>> cache = mem.persistentCache.get(key);
 132         if(cache != null) {
 133             List<Map<String,Object>> result = cache.get(search);
 134             if(result != null) return result;
 135         }
 136
 137         startAccess(monitor, processor.getSession(), false);
 138
 139         List<Map<String, Object>> results = super.doSearch(monitor, processor, search, maxResultCount);
 140         if(cache == null) {
 141             cache = new HashMap<String,List<Map<String,Object>>>();
 142             mem.persistentCache.put(key, cache);
 143         }
 144
 145         if(results.size() < 500)
 146             cache.put(search, results);
 147
 148         return results;
 149
 150     }
 151
 152     List<Resource> persistentCachedSearchResources(IProgressMonitor monitor, RequestProcessor processor, String search,
 153             int maxResultCount) throws ParseException, IOException, DatabaseException {
 154
 155         MemoryIndexing mem = MemoryIndexing.getInstance(session.getSession());
 156
 157         String key = indexPath.getAbsolutePath();
 158
 159         Map<String,List<Resource>> cache = mem.persistentCacheResources.get(key);
 160         if(cache != null) {
 161             List<Resource> result = cache.get(search);
 162             if(result != null) return result;
 163         }
 164
 165         startAccess(monitor, processor.getSession(), false);
 166
 167         List<Resource> results = super.doSearchResources(monitor, processor, search, maxResultCount);
 168         if(cache == null) {
 169             cache = new HashMap<String,List<Resource>>();
 170             mem.persistentCacheResources.put(key, cache);
 171         }
 172
 173         if(results.size() < 500)
 174             cache.put(search, results);
 175
 176         return results;
 177
 178     }
 179
 180     List<Object> persistentCachedList(IProgressMonitor monitor, RequestProcessor processor) throws ParseException, IOException, DatabaseException {
 181
 182         startAccess(monitor, processor.getSession(), false);
 183
 184         List<Object> results = super.doList(monitor, processor);
 185
 186         return results;
 187
 188     }
 189
 190     @Override
 191     List<Map<String, Object>> doSearch(IProgressMonitor monitor, RequestProcessor processor, String search,
 192             int maxResultCount) throws ParseException, IOException, DatabaseException {
 193
 194         List<Map<String,Object>> persistent = persistentCachedSearch(monitor, processor, search, maxResultCount);
 195         List<Map<String,Object>> cached = cache.doSearch(monitor, processor, search, maxResultCount);
 196
 197         ArrayList<Map<String,Object>> result = new ArrayList<Map<String,Object>>();
 198
 199         for(Map<String,Object> m : persistent) {
 200             Resource r = (Resource)m.get("Resource");
 201             if(!cache.changed.contains(r.getResourceId())) {
 202                 result.add(m);
 203             }
 204         }
 205         result.addAll(cached);
 206         return result;
 207
 208     }
 209
 210     @Override
 211     List<Resource> doSearchResources(IProgressMonitor monitor, RequestProcessor processor, String search,
 212                 int maxResultCount) throws ParseException, IOException, DatabaseException {
 213
 214         List<Resource> persistent = persistentCachedSearchResources(monitor, processor, search, maxResultCount);
 215         List<Resource> cached = cache.doSearchResources(monitor, processor, search, maxResultCount);
 216
 217         ArrayList<Resource> result = new ArrayList<Resource>();
 218         for(Resource r : persistent) {
 219             if(!cache.changed.contains(r.getResourceId())) {
 220                 result.add(r);
 221             }
 222         }
 223         result.addAll(cached);
 224         return result;
 225
 226     }
 227
 228     List<Object> doList(IProgressMonitor monitor, RequestProcessor processor) throws ParseException, IOException, DatabaseException {
 229
 230         List<Object> persistent = persistentCachedList(monitor, processor);
 231
 232         // TODO: check that caches have been properly flushed
 233         //List<Object> cached = cache.doList(monitor, processor);
 234         //if(!cached.isEmpty()) throw new DatabaseException("doList does not support caching");
 235
 236         return persistent;
 237
 238     }
 239
 240     void applyChanges(IProgressMonitor monitor, Session session, GenericRelation r, Collection<Object[]> os) throws Exception {
 241
 242         if(!os.isEmpty()) {
 243
 244                 ArrayList<Object> replaceKeys = new ArrayList<Object>();
 245                 ArrayList<Object[]> replaceValues = new ArrayList<Object[]>();
 246                 ArrayList<Object> removeKeys = new ArrayList<Object>();
 247                 for(Object[] o : os) {
 248                     Long parent = (Long)o[0];
 249                     Long key = (Long)o[1];
 250                     if(parent != 0) {
 251                         replaceKeys.add(key);
 252                         replaceValues.add(o);
 253                     } else {
 254                         removeKeys.add(key);
 255                     }
 256                 }
 257
 258                 changeState(monitor, session, State.READY);
 259
 260                 super.startAccess(null, session, true);
 261
 262                 super.replaceIndex(null, "Resource", replaceKeys, r, 1, replaceValues);
 263                 super.removeIndex(null, r, null, "Resource", removeKeys);
 264
 265         }
 266
 267         changeState(monitor, session, State.READY);
 268
 269     }
 270
 271     @Override
 272     Throwable bestEffortClear(IProgressMonitor monitor, Session session) {
 273
 274         // Free the index
 275         changeState(monitor, session, State.NONE);
 276
 277         Throwable t = clearDirectory(monitor, session);
 278         if(t != null) return t;
 279
 280         t = cache.bestEffortClear(monitor, session);
 281         if(t != null) return t;
 282
 283         String key = indexPath.getAbsolutePath();
 284         MemoryIndexing mem = MemoryIndexing.getInstance(session);
 285         mem.persistentCache.remove(key);
 286         mem.persistentCacheResources.remove(key);
 287
 288         return null;
 289
 290     }
 291
 292 }