From 091591094efd9ad65f51f3cc64616ed0c167a1ab Mon Sep 17 00:00:00 2001 From: Tuukka Lehtonen Date: Thu, 6 Oct 2016 00:29:28 +0300 Subject: [PATCH] Sync git svn branch with SVN repository r33308. refs #6742 refs #6741 --- .../modifiers/StringPropertyModifierRule.java | 26 +-- .../IndexedRelationsSearcherBase.java | 213 ++++++++---------- .../db/indexing/internal/IndexingJob.java | 92 ++++++++ .../procore/internal/GraphSession.java | 2 +- 4 files changed, 200 insertions(+), 133 deletions(-) create mode 100644 bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/internal/IndexingJob.java diff --git a/bundles/org.simantics.browsing.ui.model/src/org/simantics/browsing/ui/model/modifiers/StringPropertyModifierRule.java b/bundles/org.simantics.browsing.ui.model/src/org/simantics/browsing/ui/model/modifiers/StringPropertyModifierRule.java index 28e712750..530c97ea8 100644 --- a/bundles/org.simantics.browsing.ui.model/src/org/simantics/browsing/ui/model/modifiers/StringPropertyModifierRule.java +++ b/bundles/org.simantics.browsing.ui.model/src/org/simantics/browsing/ui/model/modifiers/StringPropertyModifierRule.java @@ -69,12 +69,6 @@ public class StringPropertyModifierRule implements ModifierRule { @Override public String getValue() { return sm.getValue(); -// try { -// return session.syncRequest(new PossibleAdapter(valueResource, String.class)); -// } catch (DatabaseException e) { -// ErrorLogger.defaultLogError(e); -// return null; -// } } @Override public String isValid(String label) { @@ -82,18 +76,14 @@ public class StringPropertyModifierRule implements ModifierRule { } @Override public void modify(final String label) { - try { - session.syncRequest(new WriteRequest() { - @Override - public void perform(WriteGraph graph) throws DatabaseException { - Layer0Utils.addCommentMetadata(graph, "Modify string"); - graph.markUndoPoint(); - sm.modify(graph, label); - } - }); - } catch (DatabaseException e) { - ErrorLogger.defaultLogError(e); - } + session.asyncRequest(new WriteRequest() { + @Override + public void perform(WriteGraph graph) throws DatabaseException { + Layer0Utils.addCommentMetadata(graph, "Modify string"); + graph.markUndoPoint(); + sm.modify(graph, label); + } + }, e -> { if (e != null) ErrorLogger.defaultLogError(e); }); } }; } diff --git a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsSearcherBase.java b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsSearcherBase.java index 4afbb174e..c52588af0 100644 --- a/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsSearcherBase.java +++ b/bundles/org.simantics.db.indexing/src/org/simantics/db/indexing/IndexedRelationsSearcherBase.java @@ -60,15 +60,15 @@ import org.simantics.db.ReadGraph; import org.simantics.db.RequestProcessor; import org.simantics.db.Resource; import org.simantics.db.Session; -import org.simantics.db.common.request.ReadRequest; import org.simantics.db.common.request.SafeName; import org.simantics.db.common.utils.Logger; +import org.simantics.db.common.utils.NameUtils; import org.simantics.db.exception.DatabaseException; +import org.simantics.db.indexing.internal.IndexingJob; import org.simantics.db.layer0.adapter.GenericRelation; import org.simantics.db.request.Read; import org.simantics.db.service.CollectionSupport; import org.simantics.db.service.SerialisationSupport; -import org.simantics.utils.DataContainer; import org.simantics.utils.FileUtils; import org.simantics.utils.datastructures.Pair; @@ -287,19 +287,8 @@ abstract public class IndexedRelationsSearcherBase { Field[] fs = makeFieldsForRelation(r, boundLength, document); for (Object[] documentData : documentsData) { - for (int i = 0; i < documentData.length; i++) { - Object value = documentData[i]; - if (value instanceof String) { - fs[i].setStringValue((String) value); - } else if (value instanceof Long) { - fs[i].setLongValue((Long) value); - } else { - System.err.println("Can only index Long and String fields, encountered " + value); - // FIXME: should throw an exception for illegal input data but this would leave the index in an incoherent state - continue; - } -// System.out.println("index " + fs[i].name() + " = " + result[i]); - } + if (setFields(fs, documentData) == null) + continue; if (IndexPolicy.TRACE_INDEX_UPDATE) System.out.println(getDescriptor() + "Inserting document " + document); @@ -413,18 +402,8 @@ abstract public class IndexedRelationsSearcherBase { continue nextDocument; } - for (int i = 0; i < documentData.length; i++) { - Object value = documentData[i]; - if (value instanceof String) { - fs[i].setStringValue((String) value); - } else if (keyValue instanceof Long) { - fs[i].setLongValue((Long)value); - } else { - // FIXME: should throw an exception for illegal input data but this would leave the index in an incoherent state - System.err.println("[" + getClass().getSimpleName() + "] Unrecognized document value '" + value + "' for field '" + fs[i].toString() + "', only " + String.class + " and " + Resource.class + " are supported."); - continue nextDocument; - } - } + if (setFields(fs, documentData) == null) + continue nextDocument; if (IndexPolicy.TRACE_INDEX_UPDATE) System.out.println(getDescriptor() + "Replacing document with key " + removedTerm + " with " + document); @@ -606,7 +585,22 @@ abstract public class IndexedRelationsSearcherBase { } }); - void initializeIndex(IProgressMonitor monitor, ReadGraph graph, final Object[] bound, boolean overwrite) throws IOException, + void initializeIndex(IProgressMonitor monitor, ReadGraph graph, Object[] bound, boolean overwrite) + throws IOException, DatabaseException + { + IndexingJob.jobifyIfPossible( + monitor, + "Reindexing " + NameUtils.getSafeLabel(graph, input), + mon -> { + try { + initializeIndexImpl(mon, graph, bound, overwrite); + } catch (IOException e) { + throw new DatabaseException(e); + } + }); + } + + void initializeIndexImpl(IProgressMonitor monitor, ReadGraph graph, final Object[] bound, boolean overwrite) throws IOException, DatabaseException { final SubMonitor mon = SubMonitor.convert(monitor, 100); @@ -624,7 +618,7 @@ abstract public class IndexedRelationsSearcherBase { final AtomicReference writer = new AtomicReference(); try { - mon.subTask("Create index at " + indexPath.toString()); + mon.subTask("Start index write"); createDirectory(indexPath); directory.set(FSDirectory.open(indexPath)); @@ -633,95 +627,67 @@ abstract public class IndexedRelationsSearcherBase { mon.worked(5); - final DataContainer start = new DataContainer(); - - graph.syncRequest(new ReadRequest() { + final GenericRelation r = graph.adapt(relation, GenericRelation.class); + if (r == null) + throw new DatabaseException("Given resource " + graph.syncRequest(new SafeName(relation)) + + "could not be adapted to GenericRelation."); - @Override - public void run(ReadGraph graph) throws DatabaseException { + long realizeStart = 0; + if (IndexPolicy.PERF_INDEX_INIT) + realizeStart = System.nanoTime(); - final GenericRelation r = graph.adapt(relation, GenericRelation.class); - if (r == null) - throw new DatabaseException("Given resource " + graph.syncRequest(new SafeName(relation)) - + "could not be adapted to GenericRelation."); - - mon.worked(45); - - GenericRelation selection = r.select(getPattern(r, bound.length), bound); - - long perfStart = 0; - if (IndexPolicy.PERF_INDEX_INIT) - perfStart = System.nanoTime(); + mon.subTask("Calculating indexed content"); + GenericRelation selection = r.select(getPattern(r, bound.length), bound); + mon.worked(5); + List results = selection.realize(graph); + mon.worked(40); - final List results = selection.realize(graph); + if (IndexPolicy.PERF_INDEX_INIT) + System.out.println(getDescriptor() + "Realized index with " + results.size() + " entries at " + indexPath + " in " + (1e-9 * (System.nanoTime()-realizeStart)) + " seconds."); + if (IndexPolicy.TRACE_INDEX_INIT) + System.out.println(getDescriptor() + "Indexed relation " + r + " produced " + results.size() + " results"); - if (IndexPolicy.PERF_INDEX_INIT) - System.out.println(getDescriptor() + "Realized index at " + indexPath + " in " + (1e-9 * (System.nanoTime()-perfStart)) + " seconds."); - - if (IndexPolicy.TRACE_INDEX_INIT) - System.out.println(getDescriptor() + "Indexed relation " + r + " produced " + results.size() + " results"); + long start = IndexPolicy.PERF_INDEX_INIT ? System.nanoTime() : 0; - if (IndexPolicy.PERF_INDEX_INIT) - start.set(System.nanoTime()); + mon.subTask("Indexing content"); + final Semaphore s = new Semaphore(0); + mon.setWorkRemaining(results.size()); - final Semaphore s = new Semaphore(0); + for (int i = 0; i < INDEXING_THREAD_COUNT; i++) { + final int startIndex = i; + executor.submit(() -> { + try { + Document document = new Document(); + Field[] fs = makeFieldsForRelation(r, bound.length, document); + + for (int index = startIndex; index < results.size(); index += INDEXING_THREAD_COUNT) { + if (setFields(fs, results.get(index)) == null) + continue; + try { + writer.get().addDocument(document); + } catch (CorruptIndexException e) { + throw new IllegalStateException(e); + } catch (IOException e) { + throw new IllegalStateException(e); + } finally { + synchronized (mon) { + mon.worked(1); + } + } + } - for(int i=0;iorg.simantics.DatabaseJob.DATABASE_JOB_FAMILY in order for + * this job to cause the same UI-behavior as DatabaseJob. + */ + private static final String DATABASE_JOB_FAMILY = "org.simantics.db.inDatabaseJob"; + + public IndexingJob(String name) { + super(name); + } + + @Override + public boolean belongsTo(Object family) { + return DATABASE_JOB_FAMILY.equals(family); + } + + /** + * @param monitor + * @param jobName + * @param consumer + * @throws DatabaseException + */ + public static void jobifyIfPossible( + IProgressMonitor monitor, + String jobName, + DbConsumer consumer) + throws DatabaseException + { + // Prevent deadlocks by checking preconditions for using a job. + // JobManager is suspended e.g. during workbench startup. + if (Job.getJobManager().isSuspended() || Job.getJobManager().currentJob() != null) { + consumer.accept(monitor); + return; + } + + Semaphore barrier = new Semaphore(0); + Throwable[] err = { null }; + IndexingJob job = new IndexingJob(jobName) { + @Override + protected IStatus run(IProgressMonitor monitor) { + try { + consumer.accept(monitor); + } catch (Throwable t) { + err[0] = t; + } finally { + monitor.done(); + barrier.release(); + } + return org.eclipse.core.runtime.Status.OK_STATUS; + } + }; + job.schedule(); + try { + barrier.acquire(); + if (err[0] != null) { + if (err[0] instanceof DatabaseException) + throw (DatabaseException) err[0]; + throw new DatabaseException(err[0]); + } + } catch (InterruptedException e) { + throw new DatabaseException(e); + } + } + +} diff --git a/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/GraphSession.java b/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/GraphSession.java index 58917ae74..4ba3722fc 100644 --- a/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/GraphSession.java +++ b/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/GraphSession.java @@ -518,7 +518,7 @@ public abstract class GraphSession { @Override public int read() throws IOException { - if(left <= 0) throw new IllegalStateException(); + if(left <= 0) return -1; if(offset == _s.bytes.length) { short slen = (short)Math.min(left, IMAX); -- 2.43.2