From: Jussi Koskela Date: Wed, 7 Feb 2018 11:50:07 +0000 (+0200) Subject: Improvements to Lucene indexing X-Git-Tag: v1.43.0~136^2~466 X-Git-Url: https://gerrit.simantics.org/r/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F28%2F1428%2F2;p=simantics%2Fplatform.git Improvements to Lucene indexing refs #7750 Change-Id: Ie93ff3058882bc61835d25d72440031c240be494 --- diff --git a/bundles/org.simantics.db.common/src/org/simantics/db/common/request/SuperTypeString.java b/bundles/org.simantics.db.common/src/org/simantics/db/common/request/SuperTypeString.java index f1d83fc6b..05fdc0082 100644 --- a/bundles/org.simantics.db.common/src/org/simantics/db/common/request/SuperTypeString.java +++ b/bundles/org.simantics.db.common/src/org/simantics/db/common/request/SuperTypeString.java @@ -40,12 +40,12 @@ public class SuperTypeString extends BinaryRead { final StringBuilder b = new StringBuilder(); String name = graph.getPossibleRelatedValue(parameter, L0.HasName, Bindings.STRING); if(name != null) { - b.append(name); + b.append(TypeString.escapeToken(name)); for (Resource r : supers) { name = graph.getPossibleRelatedValue(r, L0.HasName, Bindings.STRING); if(name != null) { b.append(parameter2); - b.append(name); + b.append(TypeString.escapeToken(name)); } else { Logger.defaultLogError(new DatabaseException("No name for type " + r)); } @@ -53,7 +53,7 @@ public class SuperTypeString extends BinaryRead { } else { Logger.defaultLogError(new DatabaseException("No name for type " + parameter)); } - + return b.toString(); } diff --git a/bundles/org.simantics.db.common/src/org/simantics/db/common/request/TypeString.java b/bundles/org.simantics.db.common/src/org/simantics/db/common/request/TypeString.java index 9439ad28e..1813b0744 100644 --- a/bundles/org.simantics.db.common/src/org/simantics/db/common/request/TypeString.java +++ b/bundles/org.simantics.db.common/src/org/simantics/db/common/request/TypeString.java @@ -46,13 +46,19 @@ public class TypeString extends BinaryRead { boolean first = true; for(Resource r : parameter) { String name = graph.getPossibleRelatedValue(r, L0.HasName, Bindings.STRING); - if(!first) result.append(parameter2); - else first = false; - result.append(name); + if (name != null) { + if(!first) result.append(parameter2); + else first = false; + result.append(escapeToken(name)); + } } return result.toString(); } + + public static String escapeToken(String token) { + return token.replaceAll("( |\\\\)", "\\\\$0"); + } } \ No newline at end of file diff --git a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexUtils.java b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexUtils.java index 5262d99a2..23e93ea1c 100644 --- a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexUtils.java +++ b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexUtils.java @@ -17,8 +17,8 @@ import org.simantics.db.Resource; import org.simantics.db.Session; import org.simantics.db.common.procedure.adapter.TransientCacheListener; import org.simantics.db.common.request.ObjectsWithType; -import org.simantics.db.common.utils.NameUtils; import org.simantics.db.exception.DatabaseException; +import org.simantics.db.layer0.genericrelation.IndexQueries; import org.simantics.db.layer0.util.Layer0Utils; import org.simantics.db.service.CollectionSupport; import org.simantics.layer0.Layer0; @@ -96,8 +96,9 @@ public class IndexUtils { public static Collection findByType(ReadGraph graph, Resource model, Resource type) throws DatabaseException { HashSet results = new HashSet(); - - String search = "Types:*" + NameUtils.getSafeName(graph, type); + Layer0 L0 = Layer0.getInstance(graph); + String typeName = graph.getRelatedValue(type, L0.HasName, Bindings.STRING); + String search = "Types:" + IndexQueries.quoteTerm(typeName); for(Map entry : find(graph, model, search)) { Resource resource = (Resource)entry.get("Resource"); @@ -111,12 +112,12 @@ public class IndexUtils { Layer0 L0 = Layer0.getInstance(graph); HashSet results = new HashSet(); - - String search = "Types:*" + type + " AND Name:" + name; + String typeName = graph.getRelatedValue(type, L0.HasName, Bindings.STRING); + String search = "Types:" + IndexQueries.quoteTerm(typeName) + " AND Name:" + IndexQueries.quoteTerm(name); for(Map entry : find(graph, model, search)) { Resource resource = (Resource)entry.get("Resource"); - if(graph.isInstanceOf(resource, type) && name.equals(graph.getPossibleRelatedValue(resource, L0.HasName, Bindings.STRING))) results.add(resource); + if(graph.isInstanceOf(resource, type)) results.add(resource); } return results; } diff --git a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/Queries.java b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/Queries.java index 1ea64d0f2..21a25d7d6 100644 --- a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/Queries.java +++ b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/Queries.java @@ -16,12 +16,16 @@ import java.io.Reader; import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Pattern; import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.analysis.core.WhitespaceAnalyzer; import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper; +import org.apache.lucene.analysis.pattern.PatternReplaceFilter; +import org.apache.lucene.analysis.pattern.PatternTokenizer; import org.apache.lucene.analysis.util.CharTokenizer; import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.queryparser.classic.QueryParser; @@ -90,6 +94,18 @@ public class Queries { } } + static final class TypeStringAnalyzer extends Analyzer { + + @Override + protected TokenStreamComponents createComponents(String fieldName, Reader reader) { + Tokenizer tokenizer = new PatternTokenizer(reader, Pattern.compile("(([^\\\\ ]|\\\\\\\\|\\\\ )+)( *)"), 1); + TokenFilter filter = new PatternReplaceFilter(tokenizer, Pattern.compile("(\\\\(\\\\| ))"), "$2", true); + + return new TokenStreamComponents(tokenizer, filter); + } + + } + private static AtomicReference> queryCache = new AtomicReference<>(); final static PerFieldAnalyzerWrapper analyzer = createAnalyzer(); @@ -101,6 +117,8 @@ public class Queries { analyzerPerField.put("Parent", new KeywordAnalyzer()); analyzerPerField.put("Resource", new KeywordAnalyzer()); analyzerPerField.put("GUID", new KeywordAnalyzer()); + analyzerPerField.put("Name", new KeywordAnalyzer()); + analyzerPerField.put("Types", new TypeStringAnalyzer()); PerFieldAnalyzerWrapper analyzer = new PerFieldAnalyzerWrapper(new LowerCaseWhitespaceAnalyzer(Version.LUCENE_4_9), analyzerPerField); return analyzer; @@ -118,6 +136,7 @@ public class Queries { //System.err.println("parse " + search + " (cached=" + (cachedQuery != null ? cachedQuery.second : "null") + ")" ); CustomQueryParser parser = new CustomQueryParser(Version.LUCENE_4_9, "Name", getAnalyzer(), schema); + parser.setLowercaseExpandedTerms(false); Query query = parser.parse(search); queryCache.set(Pair.make(query, search)); diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/QueryIndexUtils.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/QueryIndexUtils.java index 4a066a30c..ed6886abe 100644 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/QueryIndexUtils.java +++ b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/QueryIndexUtils.java @@ -12,6 +12,7 @@ import org.simantics.db.common.utils.NameUtils; import org.simantics.db.exception.DatabaseException; import org.simantics.db.layer0.adapter.Instances; import org.simantics.db.layer0.adapter.impl.EntityInstances.QueryIndex; +import org.simantics.db.layer0.genericrelation.IndexQueries; import org.simantics.db.layer0.util.Layer0Utils; import org.simantics.layer0.Layer0; import org.simantics.scl.runtime.function.Function1; @@ -33,7 +34,7 @@ public final class QueryIndexUtils { } public static List searchByGUID(ReadGraph graph, Resource indexRoot, String indexString) throws DatabaseException { - return searchByQueryShallow(graph, indexRoot, "GUID:" + indexString); + return searchByQueryShallow(graph, indexRoot, "GUID:" + IndexQueries.quoteTerm(indexString)); } public static List searchByQueryShallow(ReadGraph graph, Resource model, String query) throws DatabaseException { @@ -90,6 +91,6 @@ public final class QueryIndexUtils { } public static List searchByTypeAndNameShallow(ReadGraph graph, Resource model, Resource type, String name) throws DatabaseException { - return graph.syncRequest(new QueryIndex(model, type, name), TransientCacheListener.>instance()); + return graph.syncRequest(new QueryIndex(model, type, IndexQueries.quoteTerm(name)), TransientCacheListener.>instance()); } } diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/adapter/impl/EntityInstances.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/adapter/impl/EntityInstances.java index 07ddce7bd..93903fb9b 100644 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/adapter/impl/EntityInstances.java +++ b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/adapter/impl/EntityInstances.java @@ -119,13 +119,16 @@ public class EntityInstances implements Instances { String typeName = graph.getPossibleRelatedValue(type, L0.HasName, Bindings.STRING); if (typeName == null || typeName.isEmpty()) return null; - sb.append("Types:").append( IndexQueries.escape( typeName, true ) ); + sb.append("Types:").append( IndexQueries.quoteTerm(typeName) ); } if (!emptyFilter) { if (sb.length() > 0) sb.append(" AND "); sb.append(filter); } + if (sb.length() == 0) { + sb.append("*:*"); + } return sb.toString(); } @@ -141,7 +144,8 @@ public class EntityInstances implements Instances { @Override public int getType() { - return RequestFlags.IMMEDIATE_UPDATE; + // This query should not be immediate update since it takes a long time! + return RequestFlags.INVALIDATE; } } @@ -198,13 +202,7 @@ public class EntityInstances implements Instances { @Override public Collection findByName(ReadGraph graph, Resource model, String name) throws DatabaseException { - Layer0 L0 = Layer0.getInstance(graph); - CollectionSupport coll = graph.getService(CollectionSupport.class); - List results = coll.createList(); - for(Resource match : find(graph, model, name)) { - if(name.equals(graph.getPossibleRelatedValue(match, L0.HasName, Bindings.STRING))) results.add(match); - } - return results; + return find(graph, model, IndexQueries.quoteTerm(name)); } } diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/genericrelation/IndexQueries.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/genericrelation/IndexQueries.java index 135c5ebc2..212da3830 100644 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/genericrelation/IndexQueries.java +++ b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/genericrelation/IndexQueries.java @@ -52,7 +52,7 @@ public class IndexQueries { // These characters are part of the query syntax and must be escaped if (c == '\\' || c == '+' || c == '-' || c == '!' || c == '(' || c == ')' || c == ':' || c == '^' || c == '[' || c == ']' || c == '\"' || c == '{' || c == '}' || c == '~' - || c == '|' || c == '&' || c == '/' || (escapeWildcards && (c == '*' || c == '?'))) { + || c == '|' || c == '&' || c == '/' || c == ' ' || (escapeWildcards && (c == '*' || c == '?'))) { sb.append('\\'); sb.append(c); lastWhitespace = false; @@ -95,7 +95,7 @@ public class IndexQueries { // These characters are part of the query syntax and must be escaped if (c == '\\' || c == '+' || c == '-' || c == '!' || c == '(' || c == ')' || c == ':' || c == '^' || c == '[' || c == ']' || c == '\"' || c == '{' || c == '}' || c == '~' - || c == '|' || c == '&' || c == '/' || (escapeWildcards && (c == '*' || c == '?'))) { + || c == '|' || c == '&' || c == '/' || c == ' ' || (escapeWildcards && (c == '*' || c == '?'))) { return true; } else if (Character.isWhitespace(c)) { lastWhitespace = true; @@ -113,8 +113,9 @@ public class IndexQueries { } private static final String[] RESERVED_WORDS = { - "AND", "and", - "OR", "or" + "AND", "\\AND", + "OR", "\\OR", + "NOT", "\\NOT", }; /** @@ -161,6 +162,14 @@ public class IndexQueries { StringBuilder sb = new StringBuilder(); return escapeTerm(field, term, escapeWildcards, sb).toString(); } + + public static String quoteTerm(String term) { + StringBuilder sb = new StringBuilder(); + sb.append("\""); + sb.append(term.replaceAll("(\"|\\\\)", "\\\\$0")); + sb.append("\""); + return sb.toString(); + } // public static void main(String[] args) { // System.out.println("esc: " + escape("AND01", true, true)); diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/request/IndexedInstances.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/request/IndexedInstances.java index d37805403..09a1fc902 100644 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/request/IndexedInstances.java +++ b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/request/IndexedInstances.java @@ -10,6 +10,7 @@ import org.simantics.db.ReadGraph; import org.simantics.db.Resource; import org.simantics.db.common.request.ResourceRead2; import org.simantics.db.exception.DatabaseException; +import org.simantics.db.layer0.genericrelation.IndexQueries; import org.simantics.layer0.Layer0; import org.simantics.operation.Layer0X; import org.simantics.scl.runtime.function.Function; @@ -30,7 +31,7 @@ public class IndexedInstances extends ResourceRead2> { Function dependencies = graph.adapt(L0X.Dependencies, Function.class); - Collection> results = (Collection>)dependencies.apply(graph, resource2, "Types:*" + typeName); + Collection> results = (Collection>)dependencies.apply(graph, resource2, "Types:" + IndexQueries.quoteTerm(typeName)); if (results == null) return Collections.emptySet(); diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TGRepresentationUtils.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TGRepresentationUtils.java index 89da10f44..f02b8eb9e 100644 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TGRepresentationUtils.java +++ b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TGRepresentationUtils.java @@ -23,6 +23,7 @@ import org.simantics.db.Statement; import org.simantics.db.common.request.PossibleIndexRoot; import org.simantics.db.exception.DatabaseException; import org.simantics.db.layer0.adapter.impl.EntityInstances.QueryIndex; +import org.simantics.db.layer0.genericrelation.IndexQueries; import org.simantics.db.layer0.util.ConsistsOfProcess.ConsistsOfProcessEntry; import org.simantics.db.layer0.util.DomainProcessor3.ExclusionDecision; import org.simantics.db.layer0.util.TransferableGraphConfiguration2.SeedSpec; @@ -41,7 +42,7 @@ public class TGRepresentationUtils { Layer0 L0 = Layer0.getInstance(graph); GUID guid = graph.getPossibleRelatedValue(source, L0.identifier, GUID.BINDING); if(guid != null) { - List exist = graph.syncRequest(new QueryIndex(targetIndex, L0.Entity, "GUID:" + guid.indexString())); + List exist = graph.syncRequest(new QueryIndex(targetIndex, L0.Entity, "GUID:" + IndexQueries.quoteTerm(guid.indexString()))); return !exist.isEmpty(); } return false; diff --git a/bundles/org.simantics.debug.ui/src/org/simantics/debug/ui/SearchResourceDialog.java b/bundles/org.simantics.debug.ui/src/org/simantics/debug/ui/SearchResourceDialog.java index 29cfd63bd..4eca4e550 100644 --- a/bundles/org.simantics.debug.ui/src/org/simantics/debug/ui/SearchResourceDialog.java +++ b/bundles/org.simantics.debug.ui/src/org/simantics/debug/ui/SearchResourceDialog.java @@ -193,7 +193,9 @@ public class SearchResourceDialog extends FilteredItemsSelectionDialog { if (buttonId == SHOW_IN_BROWSER_ID) { okPressed(); LabeledResource lr = (LabeledResource) getFirstResult(); - ShowInBrowser.defaultExecute(new StructuredSelection(new ResourceWorkbenchSelectionElement(lr.resource))); + if (lr != null) { + ShowInBrowser.defaultExecute(new StructuredSelection(new ResourceWorkbenchSelectionElement(lr.resource))); + } return; } super.buttonPressed(buttonId); diff --git a/bundles/org.simantics.modeling/src/org/simantics/modeling/ModelingUtils.java b/bundles/org.simantics.modeling/src/org/simantics/modeling/ModelingUtils.java index de388b257..b10ad9da0 100644 --- a/bundles/org.simantics.modeling/src/org/simantics/modeling/ModelingUtils.java +++ b/bundles/org.simantics.modeling/src/org/simantics/modeling/ModelingUtils.java @@ -96,7 +96,6 @@ import org.simantics.db.layer0.adapter.CopyHandler; import org.simantics.db.layer0.adapter.GenericRelationIndex; import org.simantics.db.layer0.adapter.Instances; import org.simantics.db.layer0.adapter.impl.DefaultPasteImportAdvisor; -import org.simantics.db.layer0.adapter.impl.EntityInstances.QueryIndex; import org.simantics.db.layer0.adapter.impl.ImportAdvisorFactory; import org.simantics.db.layer0.genericrelation.DependenciesRelation.DependencyChangesRequest; import org.simantics.db.layer0.genericrelation.DependencyChanges; @@ -158,7 +157,6 @@ import org.simantics.simulation.ontology.SimulationResource; import org.simantics.structural.stubs.StructuralResource2; import org.simantics.structural2.scl.StructuralComponent; import org.simantics.structural2.utils.StructuralUtils; -import org.simantics.ui.SimanticsUI; import org.simantics.utils.ObjectUtils; import org.simantics.utils.datastructures.Pair; import org.simantics.utils.datastructures.Triple; @@ -413,7 +411,7 @@ public class ModelingUtils { public static void importModel(String fileName) { - Resource project = SimanticsUI.getProject().get(); + Resource project = Simantics.getProject().get(); try (StreamingTransferableGraphFileReader importer = new StreamingTransferableGraphFileReader(new File(fileName))) { TransferableGraphSource tg = importer.readTG(); diff --git a/bundles/org.simantics.modeling/src/org/simantics/modeling/adapters/Removers.java b/bundles/org.simantics.modeling/src/org/simantics/modeling/adapters/Removers.java index 80809c959..90ee3db2f 100644 --- a/bundles/org.simantics.modeling/src/org/simantics/modeling/adapters/Removers.java +++ b/bundles/org.simantics.modeling/src/org/simantics/modeling/adapters/Removers.java @@ -28,6 +28,7 @@ import org.simantics.db.common.request.PossibleObjectWithType; import org.simantics.db.common.utils.NameUtils; import org.simantics.db.exception.DatabaseException; import org.simantics.db.layer0.exception.VariableException; +import org.simantics.db.layer0.genericrelation.IndexQueries; import org.simantics.db.layer0.request.PossibleModel; import org.simantics.db.layer0.variable.RVI; import org.simantics.db.layer0.variable.Variable; @@ -123,7 +124,7 @@ final class Removers { @SuppressWarnings("rawtypes") Function modules = graph.adapt(Layer0X.getInstance(graph).Dependencies, Function.class); @SuppressWarnings("unchecked") - List> rows = (List>) modules.apply(graph, root, "Types:\"" + result.componentType.getName() + "\""); + List> rows = (List>) modules.apply(graph, root, "Types:" + IndexQueries.quoteTerm(result.componentType.getName())); if (rows.isEmpty()) return; diff --git a/bundles/org.simantics.modeling/src/org/simantics/modeling/services/CaseInsensitiveComponentFunctionNamingStrategy.java b/bundles/org.simantics.modeling/src/org/simantics/modeling/services/CaseInsensitiveComponentFunctionNamingStrategy.java index feef9c899..ad48d1f2f 100644 --- a/bundles/org.simantics.modeling/src/org/simantics/modeling/services/CaseInsensitiveComponentFunctionNamingStrategy.java +++ b/bundles/org.simantics.modeling/src/org/simantics/modeling/services/CaseInsensitiveComponentFunctionNamingStrategy.java @@ -267,7 +267,7 @@ public class CaseInsensitiveComponentFunctionNamingStrategy extends ComponentNam if (propositionPreFilter != null) proposition = propositionPreFilter.apply(proposition); - String search = "Name:" + IndexQueries.escape( proposition ) + "*"; + String search = "Name:" + IndexQueries.quoteTerm( proposition ); @SuppressWarnings("unchecked") List components = (List) index.apply(graph, indexRoot, search, Integer.MAX_VALUE);