Improvements to Lucene indexing 28/1428/2
authorJussi Koskela <jussi.koskela@semantum.fi>
Wed, 7 Feb 2018 11:50:07 +0000 (13:50 +0200)
committerTuukka Lehtonen <tuukka.lehtonen@semantum.fi>
Wed, 13 Jun 2018 20:07:24 +0000 (23:07 +0300)
refs #7750

Change-Id: Ie93ff3058882bc61835d25d72440031c240be494

13 files changed:
bundles/org.simantics.db.common/src/org/simantics/db/common/request/SuperTypeString.java
bundles/org.simantics.db.common/src/org/simantics/db/common/request/TypeString.java
bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexUtils.java
bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/Queries.java
bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/QueryIndexUtils.java
bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/adapter/impl/EntityInstances.java
bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/genericrelation/IndexQueries.java
bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/request/IndexedInstances.java
bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TGRepresentationUtils.java
bundles/org.simantics.debug.ui/src/org/simantics/debug/ui/SearchResourceDialog.java
bundles/org.simantics.modeling/src/org/simantics/modeling/ModelingUtils.java
bundles/org.simantics.modeling/src/org/simantics/modeling/adapters/Removers.java
bundles/org.simantics.modeling/src/org/simantics/modeling/services/CaseInsensitiveComponentFunctionNamingStrategy.java

index f1d83fc6b3b97e01922ee1c9e6c37db7a14a597d..05fdc008293470f08a14060a45395c72d0664a3a 100644 (file)
@@ -40,12 +40,12 @@ public class SuperTypeString extends BinaryRead<Resource, String, String> {
         final StringBuilder b = new StringBuilder();
         String name = graph.getPossibleRelatedValue(parameter, L0.HasName, Bindings.STRING);
         if(name != null) {
-               b.append(name);
+               b.append(TypeString.escapeToken(name));
             for (Resource r : supers) {
                name = graph.getPossibleRelatedValue(r, L0.HasName, Bindings.STRING);
                if(name != null) {
                        b.append(parameter2);
-                       b.append(name);
+                       b.append(TypeString.escapeToken(name));
                } else {
                        Logger.defaultLogError(new DatabaseException("No name for type " + r));
                }
@@ -53,7 +53,7 @@ public class SuperTypeString extends BinaryRead<Resource, String, String> {
         } else {
                Logger.defaultLogError(new DatabaseException("No name for type " + parameter));
         }
-        
+
         return b.toString();
 
     }
index 9439ad28e75b5e6aa54fc34630a3cc7c7ffa4d46..1813b0744f0318b5418a64c7e0d439a468429103 100644 (file)
@@ -46,13 +46,19 @@ public class TypeString extends BinaryRead<ResourceSet, String, String> {
         boolean first = true;
        for(Resource r : parameter) {
            String name = graph.getPossibleRelatedValue(r, L0.HasName, Bindings.STRING);
-            if(!first) result.append(parameter2);
-            else first = false;
-            result.append(name);
+           if (name != null) {
+                if(!first) result.append(parameter2);
+                else first = false;
+                result.append(escapeToken(name));
+           }
        }
        
        return result.toString();
        
     }
+    
+    public static String escapeToken(String token) {
+       return token.replaceAll("( |\\\\)", "\\\\$0");
+       }
 
 }
\ No newline at end of file
index 5262d99a2d85690715f4264a9a01cfaf664bcf95..23e93ea1c9a7c743f80118dab680ca3026d4fa09 100644 (file)
@@ -17,8 +17,8 @@ import org.simantics.db.Resource;
 import org.simantics.db.Session;
 import org.simantics.db.common.procedure.adapter.TransientCacheListener;
 import org.simantics.db.common.request.ObjectsWithType;
-import org.simantics.db.common.utils.NameUtils;
 import org.simantics.db.exception.DatabaseException;
+import org.simantics.db.layer0.genericrelation.IndexQueries;
 import org.simantics.db.layer0.util.Layer0Utils;
 import org.simantics.db.service.CollectionSupport;
 import org.simantics.layer0.Layer0;
@@ -96,8 +96,9 @@ public class IndexUtils {
     public static Collection<Resource> findByType(ReadGraph graph, Resource model, Resource type) throws DatabaseException {
        
         HashSet<Resource> results = new HashSet<Resource>();
-        
-               String search = "Types:*" + NameUtils.getSafeName(graph, type);
+        Layer0 L0 = Layer0.getInstance(graph);
+        String typeName = graph.getRelatedValue(type, L0.HasName, Bindings.STRING);
+               String search = "Types:" + IndexQueries.quoteTerm(typeName);
 
         for(Map<String, Object> entry : find(graph, model, search)) {
                Resource resource = (Resource)entry.get("Resource");
@@ -111,12 +112,12 @@ public class IndexUtils {
         Layer0 L0 = Layer0.getInstance(graph);
         
         HashSet<Resource> results = new HashSet<Resource>();
-        
-               String search = "Types:*" + type + " AND Name:" + name;
+        String typeName = graph.getRelatedValue(type, L0.HasName, Bindings.STRING);
+               String search = "Types:" + IndexQueries.quoteTerm(typeName) + " AND Name:" + IndexQueries.quoteTerm(name);
 
         for(Map<String, Object> entry : find(graph, model, search)) {
                Resource resource = (Resource)entry.get("Resource");
-               if(graph.isInstanceOf(resource, type) && name.equals(graph.getPossibleRelatedValue(resource, L0.HasName, Bindings.STRING))) results.add(resource);
+               if(graph.isInstanceOf(resource, type)) results.add(resource);
         }
         return results;
     }
index 1ea64d0f2bf159418c942d4b3c3cdd01fc130924..21a25d7d63a79caba3742c5457005222d3845e3b 100644 (file)
@@ -16,12 +16,16 @@ import java.io.Reader;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
 
 import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.core.KeywordAnalyzer;
 import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
 import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
+import org.apache.lucene.analysis.pattern.PatternReplaceFilter;
+import org.apache.lucene.analysis.pattern.PatternTokenizer;
 import org.apache.lucene.analysis.util.CharTokenizer;
 import org.apache.lucene.queryparser.classic.ParseException;
 import org.apache.lucene.queryparser.classic.QueryParser;
@@ -90,6 +94,18 @@ public class Queries {
         }
     }
 
+    static final class TypeStringAnalyzer extends Analyzer {
+
+               @Override
+               protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+                       Tokenizer tokenizer = new PatternTokenizer(reader, Pattern.compile("(([^\\\\ ]|\\\\\\\\|\\\\ )+)( *)"), 1);
+                       TokenFilter filter = new PatternReplaceFilter(tokenizer, Pattern.compile("(\\\\(\\\\| ))"), "$2", true);
+                       
+                       return new TokenStreamComponents(tokenizer, filter);
+               }
+               
+       }
+
     private static AtomicReference<Pair<Query, String>> queryCache = new AtomicReference<>();
 
     final static PerFieldAnalyzerWrapper analyzer = createAnalyzer();
@@ -101,6 +117,8 @@ public class Queries {
        analyzerPerField.put("Parent", new KeywordAnalyzer());
        analyzerPerField.put("Resource", new KeywordAnalyzer());
        analyzerPerField.put("GUID", new KeywordAnalyzer());
+       analyzerPerField.put("Name", new KeywordAnalyzer());
+       analyzerPerField.put("Types", new TypeStringAnalyzer());
        
         PerFieldAnalyzerWrapper analyzer = new PerFieldAnalyzerWrapper(new LowerCaseWhitespaceAnalyzer(Version.LUCENE_4_9), analyzerPerField);
         return analyzer;
@@ -118,6 +136,7 @@ public class Queries {
 
         //System.err.println("parse " + search + " (cached=" + (cachedQuery != null ? cachedQuery.second : "null") + ")" );
         CustomQueryParser parser = new CustomQueryParser(Version.LUCENE_4_9, "Name", getAnalyzer(), schema);
+        parser.setLowercaseExpandedTerms(false);
         Query query = parser.parse(search);
 
         queryCache.set(Pair.make(query, search));
index 4a066a30c06b817cf186e93eeb2701567d8d6fc7..ed6886abecec8aab22a9dd9e6c0e0d1226b0d64c 100644 (file)
@@ -12,6 +12,7 @@ import org.simantics.db.common.utils.NameUtils;
 import org.simantics.db.exception.DatabaseException;
 import org.simantics.db.layer0.adapter.Instances;
 import org.simantics.db.layer0.adapter.impl.EntityInstances.QueryIndex;
+import org.simantics.db.layer0.genericrelation.IndexQueries;
 import org.simantics.db.layer0.util.Layer0Utils;
 import org.simantics.layer0.Layer0;
 import org.simantics.scl.runtime.function.Function1;
@@ -33,7 +34,7 @@ public final class QueryIndexUtils {
     }
     
     public static List<Resource> searchByGUID(ReadGraph graph, Resource indexRoot, String indexString) throws DatabaseException {
-        return searchByQueryShallow(graph, indexRoot, "GUID:" + indexString);
+        return searchByQueryShallow(graph, indexRoot, "GUID:" + IndexQueries.quoteTerm(indexString));
     }
 
     public static List<Resource> searchByQueryShallow(ReadGraph graph, Resource model, String query) throws DatabaseException {
@@ -90,6 +91,6 @@ public final class QueryIndexUtils {
     }
 
     public static List<Resource> searchByTypeAndNameShallow(ReadGraph graph, Resource model, Resource type, String name) throws DatabaseException {
-        return graph.syncRequest(new QueryIndex(model, type, name), TransientCacheListener.<List<Resource>>instance());
+        return graph.syncRequest(new QueryIndex(model, type, IndexQueries.quoteTerm(name)), TransientCacheListener.<List<Resource>>instance());
     }
 }
index 07ddce7bd176e1fe2d8eca40819a0183ca3b64bf..93903fb9b2d4371e01f521eca67c55cc925d33bf 100644 (file)
@@ -119,13 +119,16 @@ public class EntityInstances implements Instances {
                 String typeName = graph.getPossibleRelatedValue(type, L0.HasName, Bindings.STRING);
                 if (typeName == null || typeName.isEmpty())
                     return null;
-                sb.append("Types:").append( IndexQueries.escape( typeName, true ) );
+                sb.append("Types:").append( IndexQueries.quoteTerm(typeName) );
             }
             if (!emptyFilter) {
                 if (sb.length() > 0)
                     sb.append(" AND ");
                 sb.append(filter);
             }
+            if (sb.length() == 0) {
+               sb.append("*:*");
+            }
             return sb.toString();
         }
 
@@ -141,7 +144,8 @@ public class EntityInstances implements Instances {
 
                @Override
                public int getType() {
-                       return RequestFlags.IMMEDIATE_UPDATE;
+                       // This query should not be immediate update since it takes a long time!
+                       return RequestFlags.INVALIDATE;
                }
 
     }
@@ -198,13 +202,7 @@ public class EntityInstances implements Instances {
     @Override
     public Collection<Resource> findByName(ReadGraph graph, Resource model,
             String name) throws DatabaseException {
-        Layer0 L0 = Layer0.getInstance(graph);
-       CollectionSupport coll = graph.getService(CollectionSupport.class);
-        List<Resource> results = coll.createList();
-        for(Resource match : find(graph, model, name)) {
-            if(name.equals(graph.getPossibleRelatedValue(match, L0.HasName, Bindings.STRING))) results.add(match);
-        }
-        return results;
+        return find(graph, model, IndexQueries.quoteTerm(name));
     }
 
 }
index 135c5ebc2a107b02d20923db0cdd0eda52f082a8..212da3830a1decfaac2ef541ba0f0d5155070252 100644 (file)
@@ -52,7 +52,7 @@ public class IndexQueries {
                        // These characters are part of the query syntax and must be escaped
                        if (c == '\\' || c == '+' || c == '-' || c == '!' || c == '(' || c == ')' || c == ':'
                                        || c == '^' || c == '[' || c == ']' || c == '\"' || c == '{' || c == '}' || c == '~'
-                                       || c == '|' || c == '&' || c == '/' || (escapeWildcards && (c == '*' || c == '?'))) {
+                                       || c == '|' || c == '&' || c == '/' || c == ' ' || (escapeWildcards && (c == '*' || c == '?'))) {
                                sb.append('\\');
                                sb.append(c);
                                lastWhitespace = false;
@@ -95,7 +95,7 @@ public class IndexQueries {
                        // These characters are part of the query syntax and must be escaped
                        if (c == '\\' || c == '+' || c == '-' || c == '!' || c == '(' || c == ')' || c == ':'
                                        || c == '^' || c == '[' || c == ']' || c == '\"' || c == '{' || c == '}' || c == '~'
-                                       || c == '|' || c == '&' || c == '/' || (escapeWildcards && (c == '*' || c == '?'))) {
+                                       || c == '|' || c == '&' || c == '/' || c == ' ' || (escapeWildcards && (c == '*' || c == '?'))) {
                                return true;
                        } else if (Character.isWhitespace(c)) {
                                lastWhitespace = true;
@@ -113,8 +113,9 @@ public class IndexQueries {
        }
 
        private static final String[] RESERVED_WORDS = {
-               "AND", "and",
-               "OR", "or"
+               "AND", "\\AND",
+               "OR", "\\OR",
+               "NOT", "\\NOT",
        };
 
        /**
@@ -161,6 +162,14 @@ public class IndexQueries {
                StringBuilder sb = new StringBuilder();
                return escapeTerm(field, term, escapeWildcards, sb).toString();
        }
+       
+       public static String quoteTerm(String term) {
+               StringBuilder sb = new StringBuilder();
+               sb.append("\"");
+               sb.append(term.replaceAll("(\"|\\\\)", "\\\\$0"));
+               sb.append("\"");
+               return sb.toString();
+       }
 
 //     public static void main(String[] args) {
 //             System.out.println("esc: " + escape("AND01", true, true));
index d37805403f77f59255db1442753d1751a44bbee2..09a1fc9021c77ee6440e93aeb84c87a68baf175d 100644 (file)
@@ -10,6 +10,7 @@ import org.simantics.db.ReadGraph;
 import org.simantics.db.Resource;
 import org.simantics.db.common.request.ResourceRead2;
 import org.simantics.db.exception.DatabaseException;
+import org.simantics.db.layer0.genericrelation.IndexQueries;
 import org.simantics.layer0.Layer0;
 import org.simantics.operation.Layer0X;
 import org.simantics.scl.runtime.function.Function;
@@ -30,7 +31,7 @@ public class IndexedInstances extends ResourceRead2<Set<Resource>> {
 
         Function dependencies = graph.adapt(L0X.Dependencies, Function.class);
         
-        Collection<Map<String, Object>> results = (Collection<Map<String, Object>>)dependencies.apply(graph, resource2, "Types:*" + typeName);
+        Collection<Map<String, Object>> results = (Collection<Map<String, Object>>)dependencies.apply(graph, resource2, "Types:" + IndexQueries.quoteTerm(typeName));
         if (results == null)
             return Collections.emptySet();
 
index 89da10f4493e738a0d61b32c5e6ff3271f81f903..f02b8eb9e29252d2cc36c86ad12d77883db0b495 100644 (file)
@@ -23,6 +23,7 @@ import org.simantics.db.Statement;
 import org.simantics.db.common.request.PossibleIndexRoot;
 import org.simantics.db.exception.DatabaseException;
 import org.simantics.db.layer0.adapter.impl.EntityInstances.QueryIndex;
+import org.simantics.db.layer0.genericrelation.IndexQueries;
 import org.simantics.db.layer0.util.ConsistsOfProcess.ConsistsOfProcessEntry;
 import org.simantics.db.layer0.util.DomainProcessor3.ExclusionDecision;
 import org.simantics.db.layer0.util.TransferableGraphConfiguration2.SeedSpec;
@@ -41,7 +42,7 @@ public class TGRepresentationUtils {
         Layer0 L0 = Layer0.getInstance(graph);
         GUID guid = graph.getPossibleRelatedValue(source, L0.identifier, GUID.BINDING);
         if(guid != null) {
-            List<Resource> exist = graph.syncRequest(new QueryIndex(targetIndex, L0.Entity, "GUID:" + guid.indexString()));
+            List<Resource> exist = graph.syncRequest(new QueryIndex(targetIndex, L0.Entity, "GUID:" + IndexQueries.quoteTerm(guid.indexString())));
             return !exist.isEmpty();
         }
         return false;
index 29cfd63bd45d9786bed42ad25a04b9e0cdc670a7..4eca4e550638513f0d65d00ecf45ab57540a32a4 100644 (file)
@@ -193,7 +193,9 @@ public class SearchResourceDialog extends FilteredItemsSelectionDialog {
         if (buttonId == SHOW_IN_BROWSER_ID) {
             okPressed();
             LabeledResource lr = (LabeledResource) getFirstResult();
-            ShowInBrowser.defaultExecute(new StructuredSelection(new ResourceWorkbenchSelectionElement(lr.resource)));
+            if (lr != null) {
+               ShowInBrowser.defaultExecute(new StructuredSelection(new ResourceWorkbenchSelectionElement(lr.resource)));
+            }
             return;
         }
         super.buttonPressed(buttonId);
index de388b2576d5df5d1883faed042d27ca2ecfefde..b10ad9da009065604f184ed0c9311799ff08d74d 100644 (file)
@@ -96,7 +96,6 @@ import org.simantics.db.layer0.adapter.CopyHandler;
 import org.simantics.db.layer0.adapter.GenericRelationIndex;
 import org.simantics.db.layer0.adapter.Instances;
 import org.simantics.db.layer0.adapter.impl.DefaultPasteImportAdvisor;
-import org.simantics.db.layer0.adapter.impl.EntityInstances.QueryIndex;
 import org.simantics.db.layer0.adapter.impl.ImportAdvisorFactory;
 import org.simantics.db.layer0.genericrelation.DependenciesRelation.DependencyChangesRequest;
 import org.simantics.db.layer0.genericrelation.DependencyChanges;
@@ -158,7 +157,6 @@ import org.simantics.simulation.ontology.SimulationResource;
 import org.simantics.structural.stubs.StructuralResource2;
 import org.simantics.structural2.scl.StructuralComponent;
 import org.simantics.structural2.utils.StructuralUtils;
-import org.simantics.ui.SimanticsUI;
 import org.simantics.utils.ObjectUtils;
 import org.simantics.utils.datastructures.Pair;
 import org.simantics.utils.datastructures.Triple;
@@ -413,7 +411,7 @@ public class ModelingUtils {
 
        public static void importModel(String fileName) {
 
-               Resource project = SimanticsUI.getProject().get();
+               Resource project = Simantics.getProject().get();
 
                try (StreamingTransferableGraphFileReader importer = new StreamingTransferableGraphFileReader(new File(fileName))) {
                        TransferableGraphSource tg = importer.readTG();
index 80809c959ab23e5eea1cc06c152b0dbf07c93fdf..90ee3db2ffa3fb8e5a1928b1c479677d5e567944 100644 (file)
@@ -28,6 +28,7 @@ import org.simantics.db.common.request.PossibleObjectWithType;
 import org.simantics.db.common.utils.NameUtils;
 import org.simantics.db.exception.DatabaseException;
 import org.simantics.db.layer0.exception.VariableException;
+import org.simantics.db.layer0.genericrelation.IndexQueries;
 import org.simantics.db.layer0.request.PossibleModel;
 import org.simantics.db.layer0.variable.RVI;
 import org.simantics.db.layer0.variable.Variable;
@@ -123,7 +124,7 @@ final class Removers {
         @SuppressWarnings("rawtypes")
         Function modules = graph.adapt(Layer0X.getInstance(graph).Dependencies, Function.class);
         @SuppressWarnings("unchecked")
-        List<Map<String, Object>> rows = (List<Map<String, Object>>) modules.apply(graph, root, "Types:\"" + result.componentType.getName() + "\"");
+        List<Map<String, Object>> rows = (List<Map<String, Object>>) modules.apply(graph, root, "Types:" + IndexQueries.quoteTerm(result.componentType.getName()));
         if (rows.isEmpty())
             return;
 
index feef9c8992fce52eec3e4417e45897762bf06b03..ad48d1f2f809ab89e48a545da83ae88f26e094ca 100644 (file)
@@ -267,7 +267,7 @@ public class CaseInsensitiveComponentFunctionNamingStrategy extends ComponentNam
                 if (propositionPreFilter != null)
                     proposition = propositionPreFilter.apply(proposition);
 
-                String search = "Name:" + IndexQueries.escape( proposition ) + "*";
+                String search = "Name:" + IndexQueries.quoteTerm( proposition );
                 @SuppressWarnings("unchecked")
                 List<Resource> components = (List<Resource>) index.apply(graph, indexRoot, search, Integer.MAX_VALUE);