X-Git-Url: https://gerrit.simantics.org/r/gitweb?a=blobdiff_plain;f=bundles%2Forg.simantics.db.indexing%2Fsrc%2Forg%2Fsimantics%2Fdb%2Findexing%2FMemoryIndexing.java;h=cee5eb26bfb616799c522baca31cf186b2a46a17;hb=cc084b2c81c8b47adfd1d33b507164c6c42728fa;hp=adfdbd1961a2f3ab4ebff53e5b93b723763ee734;hpb=969bd23cab98a79ca9101af33334000879fb60c5;p=simantics%2Fplatform.git diff --git a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/MemoryIndexing.java b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/MemoryIndexing.java index adfdbd196..cee5eb26b 100644 --- a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/MemoryIndexing.java +++ b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/MemoryIndexing.java @@ -1,147 +1,143 @@ -package org.simantics.db.indexing; - -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.RAMDirectory; -import org.apache.lucene.util.Version; -import org.eclipse.core.runtime.IProgressMonitor; -import org.eclipse.core.runtime.SubMonitor; -import org.simantics.db.RequestProcessor; -import org.simantics.db.Resource; -import org.simantics.db.Session; -import org.simantics.db.common.request.Adapt; -import org.simantics.db.common.utils.Logger; -import org.simantics.db.indexing.IndexedRelationsSearcherBase.State; -import org.simantics.db.layer0.adapter.GenericRelation; - -/** - * @author Tuukka Lehtonen - * @author Antti Villberg - */ -public class MemoryIndexing { - - final private Session session; - - final Map>>> persistentCache = new HashMap>>>(); - final Map>> persistentCacheResources = new HashMap>>(); - - final private Map directories = new HashMap(); - - final private Map immutableSearchers = new HashMap(); - final private Map searchers = new HashMap(); - - public MemoryIndexing(Session session) { - this.session = session; - } - - protected File getIndexDirectory(Resource relation, Resource input) { - return DatabaseIndexing.getIndexLocation(session, relation, input); - } - - public IndexedRelationsSearcher get(RequestProcessor processor, Resource relation, Resource input) { - try { - File location = getIndexDirectory(relation, input); - String key = location.getAbsolutePath(); - IndexedRelationsSearcher searcher = searchers.get(key); - if (searcher == null) { - GenericRelation r = processor.sync(new Adapt(relation, GenericRelation.class)); - searcher = new IndexedRelationsSearcher(processor, relation, input, r); - searchers.put(key, searcher); - } - return searcher; - } catch (Exception e) { - Logger.defaultLogError(e); - return null; - } - } - - public IndexedRelationsSearcherBase getImmutable(RequestProcessor processor, Resource relation, Resource input) { - try { - File location = getIndexDirectory(relation, input); - String key = location.getAbsolutePath(); - IndexedRelationsSearcherBase searcher = immutableSearchers.get(key); - if (searcher == null) { - searcher = new ImmutableIndexedRelationsSearcher(processor, relation, input); - immutableSearchers.put(key, searcher); - } - return searcher; - } catch (Exception e) { - Logger.defaultLogError(e); - return null; - } - } - - public static MemoryIndexing getInstance(Session session) { - MemoryIndexing ret = session.peekService(MemoryIndexing.class); - if(ret == null) { - ret = new MemoryIndexing(session); - session.registerService(MemoryIndexing.class, ret); - } - return ret; - } - - public synchronized Directory getDirectory(String path, Analyzer analyzer) throws IOException { - RAMDirectory directory = directories.get(path); - if (directory == null) { - synchronized (directories) { - directory = directories.get(path); - if (directory == null) { - directory = new RAMDirectory(); - IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_4_9, analyzer); - new IndexWriter(directory, config.setOpenMode(OpenMode.CREATE)).close(); - directories.put(path, directory); - } - } - } - return directory; - - } - - public void remove(String path) { - directories.remove(path); - } - - public void flush(IProgressMonitor progress) throws Exception { - - SubMonitor monitor = SubMonitor.convert(progress); - - Set> set = searchers.entrySet(); - Set> iset = immutableSearchers.entrySet(); - - monitor.setWorkRemaining(set.size()+iset.size()); - - for(Map.Entry entry : set) { - - IndexedRelationsSearcher persistent = entry.getValue(); - IndexedRelationsMemorySearcher searcher = persistent.cache; - - if(persistent.isIndexAvailable()) { - List os = searcher.allDocs(monitor, session); - persistent.applyChanges(monitor, session, searcher.r, os); - } - - monitor.worked(1); - entry.getValue().changeState(monitor, session, State.READY); - - } - - for(Map.Entry entry : iset) { - - entry.getValue().changeState(monitor, session, State.READY); - monitor.worked(1); - - } - - } - -} +package org.simantics.db.indexing; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Semaphore; +import java.util.stream.Stream; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.Version; +import org.eclipse.core.runtime.IProgressMonitor; +import org.eclipse.core.runtime.SubMonitor; +import org.simantics.db.RequestProcessor; +import org.simantics.db.Resource; +import org.simantics.db.Session; +import org.simantics.db.common.request.Adapt; +import org.simantics.db.indexing.IndexedRelationsSearcherBase.State; +import org.simantics.db.layer0.adapter.GenericRelation; +import org.slf4j.LoggerFactory; + +/** + * @author Tuukka Lehtonen + * @author Antti Villberg + */ +public class MemoryIndexing { + + private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(MemoryIndexing.class); + + final private Session session; + + final Map>>> persistentCache = new ConcurrentHashMap<>(); + final Map>> persistentCacheResources = new ConcurrentHashMap<>(); + + final private ConcurrentHashMap directories = new ConcurrentHashMap<>(); + + final private ConcurrentHashMap immutableSearchers = new ConcurrentHashMap<>(); + final private ConcurrentHashMap searchers = new ConcurrentHashMap<>(); + + public MemoryIndexing(Session session) { + this.session = session; + } + + protected Path getIndexDirectory(Resource relation, Resource input) { + return DatabaseIndexing.getIndexLocation(session, relation, input); + } + + public IndexedRelationsSearcher get(RequestProcessor processor, Resource relation, Resource input) { + Path location = getIndexDirectory(relation, input); + String key = location.toAbsolutePath().toString(); + return searchers.computeIfAbsent(key, t -> { + try { + GenericRelation r = processor.sync(new Adapt(relation, GenericRelation.class)); + return new IndexedRelationsSearcher(processor, relation, input, r); + } catch (Exception e) { + LOGGER.error("Could not get searcher for relation {} and input {} in location {}", relation, input, location, e); + return null; + } + }); + } + + public IndexedRelationsSearcherBase getImmutable(RequestProcessor processor, Resource relation, Resource input) { + Path location = getIndexDirectory(relation, input); + String key = location.toAbsolutePath().toString(); + return immutableSearchers.computeIfAbsent(key, t -> { + try { + return new ImmutableIndexedRelationsSearcher(processor, relation, input); + } catch (Exception e) { + LOGGER.error("Could not get searcher base for relation {} and input {} in location {}", relation, input, location, e); + return null; + } + }); + } + + public static MemoryIndexing getInstance(Session session) { + MemoryIndexing ret = session.peekService(MemoryIndexing.class); + if(ret == null) { + ret = new MemoryIndexing(session); + session.registerService(MemoryIndexing.class, ret); + } + return ret; + } + + public Directory getDirectory(String path, Analyzer analyzer) throws IOException { + try { + return directories.computeIfAbsent(path, t -> { + try { + RAMDirectory directory = new RAMDirectory(); + IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_4_9, analyzer); + new IndexWriter(directory, config.setOpenMode(OpenMode.CREATE)).close(); + return directory; + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } catch (RuntimeException e) { + throw (IOException) e.getCause(); + } + } + + public void remove(String path) { + directories.remove(path); + } + + public void flush(IProgressMonitor progress) throws Exception { + long startTime = System.currentTimeMillis(); + SubMonitor monitor = SubMonitor.convert(progress); + Collection searcherEntries = searchers.values(); + Collection immutableSearcherEntries = immutableSearchers.values(); + int count = searcherEntries.size() + immutableSearcherEntries.size(); + Semaphore sema = new Semaphore(0); + Stream.concat(searcherEntries.stream(), immutableSearcherEntries.stream()).parallel().forEach(base -> { + try { + if (base.isIndexAvailable()) { + if (base instanceof IndexedRelationsSearcher) { + IndexedRelationsMemorySearcher searcher = ((IndexedRelationsSearcher) base).cache; + try { + List os = searcher.allDocs(monitor, session); + ((IndexedRelationsSearcher) base).applyChanges(monitor, session, searcher.r, os); + } catch (Exception e) { + LOGGER.error("Could not flush", e); + } + } + } + monitor.worked(1); + base.changeState(monitor, session, State.READY); + } finally { + sema.release(); + } + }); + sema.acquire(count); + long totalTime = System.currentTimeMillis() - startTime; + LOGGER.info("index flush " + totalTime); + } + +}