]> gerrit.simantics Code Review - simantics/platform.git/commitdiff
Better emptying of trash bin 22/1922/2
authorAntti Villberg <antti.villberg@semantum.fi>
Tue, 17 Jul 2018 09:09:14 +0000 (12:09 +0300)
committerAntti Villberg <antti.villberg@semantum.fi>
Tue, 17 Jul 2018 10:16:58 +0000 (13:16 +0300)
Statement removal was broken in write only request. TG remover did not
correctly manage internal resources. Added hardening against errors.

gitlab #32

Change-Id: I120fcbdb7077e0f037104331480a2b1ab090d798

bundles/org.simantics.db.layer0/src/org/simantics/db/common/procedure/adapter/DirectStatementProcedure.java [new file with mode: 0644]
bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/adapter/impl/TGRemover.java
bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/Layer0Utils.java
bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/ModelTransferableGraphSource.java
bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/ClusterChange.java
bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/ClusterChange2.java
bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/ClusterTable.java
bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/DirectStatementsImpl.java
bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/SessionImplSocket.java

diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/common/procedure/adapter/DirectStatementProcedure.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/common/procedure/adapter/DirectStatementProcedure.java
new file mode 100644 (file)
index 0000000..398b185
--- /dev/null
@@ -0,0 +1,32 @@
+package org.simantics.db.common.procedure.adapter;
+
+import org.simantics.db.AsyncReadGraph;
+import org.simantics.db.DirectStatements;
+import org.simantics.db.exception.DatabaseException;
+import org.simantics.db.procedure.AsyncProcedure;
+
+public class DirectStatementProcedure implements AsyncProcedure<DirectStatements> {
+
+       DirectStatements result = null;
+       DatabaseException exception = null;
+
+       @Override
+       public void execute(AsyncReadGraph graph, final DirectStatements ds) {
+               result = ds;
+       }
+
+       @Override
+       public void exception(AsyncReadGraph graph, Throwable throwable) {
+               if(throwable instanceof DatabaseException) {
+                       exception = (DatabaseException)throwable;
+               } else {
+                       exception = new DatabaseException(throwable);
+               }
+       }
+       
+       public DirectStatements getOrThrow() throws DatabaseException {
+               if(exception != null) throw exception;
+               return result;
+       }
+
+}
\ No newline at end of file
index 88594883488f71059dd6ef128b53e9d5176ce85c..be34bb9ba5165c9714e3e8573caed16077ccbcd6 100644 (file)
  *******************************************************************************/
 package org.simantics.db.layer0.adapter.impl;
 
  *******************************************************************************/
 package org.simantics.db.layer0.adapter.impl;
 
+import java.util.ArrayList;
+import java.util.List;
+
 import org.eclipse.core.runtime.IProgressMonitor;
 import org.simantics.db.Resource;
 import org.eclipse.core.runtime.IProgressMonitor;
 import org.simantics.db.Resource;
+import org.simantics.db.Statement;
 import org.simantics.db.WriteGraph;
 import org.simantics.db.WriteGraph;
+import org.simantics.db.common.procedure.adapter.DirectStatementProcedure;
 import org.simantics.db.exception.DatabaseException;
 import org.simantics.db.layer0.util.ModelTransferableGraphSource;
 import org.simantics.db.layer0.util.ModelTransferableGraphSourceRequest;
 import org.simantics.db.layer0.util.TransferableGraphConfiguration2;
 import org.simantics.db.exception.DatabaseException;
 import org.simantics.db.layer0.util.ModelTransferableGraphSource;
 import org.simantics.db.layer0.util.ModelTransferableGraphSourceRequest;
 import org.simantics.db.layer0.util.TransferableGraphConfiguration2;
+import org.simantics.db.service.DirectQuerySupport;
 import org.simantics.db.service.SerialisationSupport;
 import org.simantics.graph.db.TransferableGraphSource.TransferableGraphSourceProcedure;
 import org.simantics.db.service.SerialisationSupport;
 import org.simantics.graph.db.TransferableGraphSource.TransferableGraphSourceProcedure;
+import org.simantics.graph.representation.Identity;
+import org.simantics.graph.representation.Internal;
+import org.simantics.graph.representation.Root;
+import org.simantics.layer0.Layer0;
 
 /**
  * @author Tuukka Lehtonen
 
 /**
  * @author Tuukka Lehtonen
@@ -27,7 +37,9 @@ import org.simantics.graph.db.TransferableGraphSource.TransferableGraphSourcePro
 public class TGRemover extends AbstractRemover {
 
     @SuppressWarnings("unused")
 public class TGRemover extends AbstractRemover {
 
     @SuppressWarnings("unused")
-    private IProgressMonitor monitor;
+    private IProgressMonitor    monitor;
+
+    private ArrayList<Resource> roots = new ArrayList<>();
 
     public TGRemover(Resource resource) {
         super(resource);
 
     public TGRemover(Resource resource) {
         super(resource);
@@ -38,46 +50,78 @@ public class TGRemover extends AbstractRemover {
         this.monitor = monitor;
     }
 
         this.monitor = monitor;
     }
 
+    public List<Resource> getRoots() {
+        return roots;
+    }
+
     @Override
     public void remove(final WriteGraph graph) throws DatabaseException {
     @Override
     public void remove(final WriteGraph graph) throws DatabaseException {
-       
-       TransferableGraphConfiguration2 conf = new TransferableGraphConfiguration2(graph, resource);
-       conf.values = false;
-       final SerialisationSupport ss = graph.getService(SerialisationSupport.class);
-       
-       try (ModelTransferableGraphSource source = graph.syncRequest(new ModelTransferableGraphSourceRequest(conf))) {
-                       source.forResourceStatements(graph, new TransferableGraphSourceProcedure<int[]>() {
-                               
-                               @Override
-                               public void execute(int[] value) throws Exception {
-                                       Resource s = ss.getResource(value[0]);
-                                       Resource p = ss.getResource(value[1]);
-                                       Resource i = null;
-                                       if(value[2] != -1) i = ss.getResource(value[2]);
-                                       Resource o = ss.getResource(value[3]);
-                                       
-//                                     System.err.println("s=" + s + " p=" + graph.getPossibleURI(p) + " o=" + o + " p:" + p + " i:" + i);
-                                       
-                                       graph.deny(s,p,i,o);
-                               }
-                               
-                       });
-                       
-                       source.forValueResources(graph, new TransferableGraphSourceProcedure<int[]>() {
-                               
-                               @Override
-                               public void execute(int[] value) throws Exception {
-                                       Resource s = ss.getResource(value[0]);
-//                                     System.err.println("s=" + s + " p=" + graph.getPossibleURI(p) + " o=" + o + " " + i);
-                                       graph.denyValue(s);
-                               }
-                               
-                       });
-                       
-               } catch (Exception e) {
-                       throw new DatabaseException(e);
-               }
-       
+
+        Layer0 L0 = Layer0.getInstance(graph);
+
+        TransferableGraphConfiguration2 conf = new TransferableGraphConfiguration2(graph, resource);
+        conf.values = false;
+        final SerialisationSupport ss = graph.getService(SerialisationSupport.class);
+        final DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class);
+
+        try (ModelTransferableGraphSource source = graph.syncRequest(new ModelTransferableGraphSourceRequest(conf))) {
+
+            long[] rev = source.getResourceArray(graph);
+
+            source.forIdentities(graph, new TransferableGraphSourceProcedure<Identity>() {
+
+                @Override
+                public void execute(Identity value) throws Exception {
+                    if (value.definition instanceof Internal) {
+                        long res = rev[value.resource];
+                        Resource r = ss.getResource(res);
+                        Resource name = graph.getPossibleObject(r, L0.HasName);
+                        if (name != null) {
+                            graph.deny(r, L0.HasName, L0.NameOf, name);
+                            graph.denyValue(name);
+                            DirectStatementProcedure proc = new DirectStatementProcedure();
+                            dqs.forEachDirectPersistentStatement(graph, name, proc);
+                            for (Statement stm : proc.getOrThrow()) {
+                                graph.deny(name, stm.getPredicate(), stm.getObject());
+                            }
+                        }
+                    } else if (value.definition instanceof Root) {
+                        long res = rev[value.resource];
+                        Resource r = ss.getResource(res);
+                        roots.add(r);
+                    }
+                }
+
+            });
+
+            source.forResourceStatements(graph, new TransferableGraphSourceProcedure<int[]>() {
+
+                @Override
+                public void execute(int[] value) throws Exception {
+                    Resource s = ss.getResource(value[0]);
+                    Resource p = ss.getResource(value[1]);
+                    Resource i = null;
+                    if (value[2] != -1)
+                        i = ss.getResource(value[2]);
+                    Resource o = ss.getResource(value[3]);
+                    graph.deny(s, p, i, o);
+                }
+
+            });
+
+            source.forValueResources(graph, new TransferableGraphSourceProcedure<int[]>() {
+
+                @Override
+                public void execute(int[] value) throws Exception {
+                    graph.denyValue(ss.getResource(value[0]));
+                }
+
+            });
+
+        } catch (Exception e) {
+            throw new DatabaseException(e);
+        }
+
     }
 
 }
     }
 
 }
index 6b9b2fc6502845763871aa1383d8d8a1835ef33b..56e1e11f23a10893b0b2818c31845e5bbe7d1c48 100644 (file)
@@ -73,6 +73,7 @@ import org.simantics.db.common.request.DelayedWriteRequest;
 import org.simantics.db.common.request.ObjectsWithType;
 import org.simantics.db.common.request.PossibleChild;
 import org.simantics.db.common.request.PossibleIndexRoot;
 import org.simantics.db.common.request.ObjectsWithType;
 import org.simantics.db.common.request.PossibleChild;
 import org.simantics.db.common.request.PossibleIndexRoot;
+import org.simantics.db.common.request.WriteRequest;
 import org.simantics.db.common.utils.NameUtils;
 import org.simantics.db.event.ChangeListener;
 import org.simantics.db.exception.CancelTransactionException;
 import org.simantics.db.common.utils.NameUtils;
 import org.simantics.db.event.ChangeListener;
 import org.simantics.db.exception.CancelTransactionException;
@@ -83,6 +84,7 @@ import org.simantics.db.layer0.adapter.CopyHandler2;
 import org.simantics.db.layer0.adapter.GenericRelationIndex;
 import org.simantics.db.layer0.adapter.PasteHandler;
 import org.simantics.db.layer0.adapter.impl.DefaultPasteHandler;
 import org.simantics.db.layer0.adapter.GenericRelationIndex;
 import org.simantics.db.layer0.adapter.PasteHandler;
 import org.simantics.db.layer0.adapter.impl.DefaultPasteHandler;
+import org.simantics.db.layer0.adapter.impl.EntityRemover;
 import org.simantics.db.layer0.adapter.impl.TGRemover;
 import org.simantics.db.layer0.genericrelation.IndexedRelations;
 import org.simantics.db.layer0.internal.SimanticsInternal;
 import org.simantics.db.layer0.adapter.impl.TGRemover;
 import org.simantics.db.layer0.genericrelation.IndexedRelations;
 import org.simantics.db.layer0.internal.SimanticsInternal;
@@ -1247,9 +1249,11 @@ public class Layer0Utils {
         emptyTrashBin(monitor, SimanticsInternal.getSession(), SimanticsInternal.getProject());
     }
 
         emptyTrashBin(monitor, SimanticsInternal.getSession(), SimanticsInternal.getProject());
     }
 
-    public static void emptyTrashBin(final IProgressMonitor monitor, Session session, final Resource project) throws ServiceException {
+    public static void emptyTrashBin(final IProgressMonitor monitor, Session session, final Resource project)
+            throws ServiceException {
         final SubMonitor mon = SubMonitor.convert(monitor, "Emptying Trash Bin...", 10000);
         try {
         final SubMonitor mon = SubMonitor.convert(monitor, "Emptying Trash Bin...", 10000);
         try {
+            ArrayList<Resource> unhandled = new ArrayList<Resource>();
             session.syncRequest(new DelayedWriteRequest() {
                 @Override
                 public void perform(WriteGraph graph) throws DatabaseException {
             session.syncRequest(new DelayedWriteRequest() {
                 @Override
                 public void perform(WriteGraph graph) throws DatabaseException {
@@ -1258,26 +1262,34 @@ public class Layer0Utils {
                     Layer0X L0X = Layer0X.getInstance(graph);
                     Resource parent = graph.getSingleObject(project, L0.PartOf);
                     Resource trashBin = Layer0Utils.getPossibleChild(graph, parent, "TrashBin");
                     Layer0X L0X = Layer0X.getInstance(graph);
                     Resource parent = graph.getSingleObject(project, L0.PartOf);
                     Resource trashBin = Layer0Utils.getPossibleChild(graph, parent, "TrashBin");
-                    Collection<Resource> trashes = trashBin != null
-                            ? graph.getObjects(trashBin, L0.ConsistsOf)
-                            : Collections.<Resource>emptyList();
+                    Collection<Resource> trashes = trashBin != null ? graph.getObjects(trashBin, L0.ConsistsOf)
+                            : Collections.<Resource> emptyList();
                     if (trashes.isEmpty())
                         throw new CancelTransactionException();
                     mon.setWorkRemaining((2 + trashes.size()) * 1000);
                     if (trashes.isEmpty())
                         throw new CancelTransactionException();
                     mon.setWorkRemaining((2 + trashes.size()) * 1000);
-                    for(Resource trash : trashes) {
+                    for (Resource trash : trashes) {
                         if (mon.isCanceled())
                             throw new CancelTransactionException();
                         mon.subTask(NameUtils.getSafeName(graph, trash));
                         if (mon.isCanceled())
                             throw new CancelTransactionException();
                         mon.subTask(NameUtils.getSafeName(graph, trash));
+                        boolean isIndexRoot = graph.isInstanceOf(trash, L0.IndexRoot);
                         TGRemover remo = new TGRemover(mon.newChild(1000, SubMonitor.SUPPRESS_ALL_LABELS), trash);
                         TGRemover remo = new TGRemover(mon.newChild(1000, SubMonitor.SUPPRESS_ALL_LABELS), trash);
-                        remo.remove(graph);
-                        if(graph.isInstanceOf(trash, L0.IndexRoot)) {
-                               // TODO: this should be an utility 
-                                       GenericRelationIndex index = graph.adapt(L0X.DependenciesRelation, GenericRelationIndex.class);
-                                       IndexedRelations ir = graph.getService(IndexedRelations.class);
-                                       // Deletes index files
-                                       ir.reset(null, graph, L0X.DependenciesRelation, trash);
-                                       // Notifies DB listeners
-                                       index.reset(graph, trash);
+                        try {
+                            remo.remove(graph);
+                            unhandled.addAll(remo.getRoots());
+                        } catch (DatabaseException e) {
+                            // Something went wrong - try to remove this later
+                            // with EntityRemover
+                            unhandled.add(trash);
+                        }
+                        if (isIndexRoot) {
+                            // TODO: this should be an utility
+                            GenericRelationIndex index = graph.adapt(L0X.DependenciesRelation,
+                                    GenericRelationIndex.class);
+                            IndexedRelations ir = graph.getService(IndexedRelations.class);
+                            // Deletes index files
+                            ir.reset(null, graph, L0X.DependenciesRelation, trash);
+                            // Notifies DB listeners
+                            index.reset(graph, trash);
                         }
                     }
                     if (mon.isCanceled())
                         }
                     }
                     if (mon.isCanceled())
@@ -1286,6 +1298,15 @@ public class Layer0Utils {
                     mon.newChild(1000);
                 }
             });
                     mon.newChild(1000);
                 }
             });
+
+            session.syncRequest(new WriteRequest() {
+                @Override
+                public void perform(WriteGraph graph) throws DatabaseException {
+                    for (Resource r : unhandled)
+                        EntityRemover.remove(graph, r);
+                }
+            });
+
             if (mon.isCanceled())
                 return;
             mon.subTask("Purging Database");
             if (mon.isCanceled())
                 return;
             mon.subTask("Purging Database");
index 95923c79946206805ef467211e5523c313c766ee..b0922dd1c444760637967e4be03a4ec4286e44e8 100644 (file)
@@ -38,6 +38,8 @@ import org.simantics.graph.representation.Internal;
 import org.simantics.graph.representation.Root;
 import org.simantics.graph.representation.Value;
 import org.simantics.layer0.Layer0;
 import org.simantics.graph.representation.Root;
 import org.simantics.graph.representation.Value;
 import org.simantics.layer0.Layer0;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import gnu.trove.list.array.TIntArrayList;
 import gnu.trove.map.TIntObjectMap;
 
 import gnu.trove.list.array.TIntArrayList;
 import gnu.trove.map.TIntObjectMap;
@@ -46,552 +48,638 @@ import gnu.trove.procedure.TIntIntProcedure;
 
 public class ModelTransferableGraphSource implements TransferableGraphSource {
 
 
 public class ModelTransferableGraphSource implements TransferableGraphSource {
 
-       final private TransferableGraphConfiguration2 configuration;
-       final private DomainProcessorState state;
-       final private int externalBase;
-       final private int resourceCount;
-       final private File[] files;
-       final private TGValueModifier valueModifier;
-
-       private volatile boolean closed = false;
-
-       TIntArrayList externalParents = new TIntArrayList();
-       ArrayList<String> externalNames = new ArrayList<>();
-       TreeMap<String,String> downloads = new TreeMap<String,String>();
-
-       public ModelTransferableGraphSource(final ReadGraph graph, TransferableGraphConfiguration2 configuration, final DomainProcessorState state, File ... fs) throws DatabaseException {
-
-               this.configuration = configuration;
-               this.state = state;
-               this.files = fs;
-               this.valueModifier = state.valueModifier;
-
-               SerialisationSupport ss = graph.getService(SerialisationSupport.class);
-
-               // At this point ids contains all internal resources. Now add roots and externals.
-
-               // Root Library
-               state.ids.put(ss.getTransientId(graph.getRootLibrary()), state.id++);
-
-               // External roots - internal roots were already processed as internal resources by domain processor
-               for(SeedSpec spec : configuration.seeds) {
-                       if(SeedSpecType.SPECIAL_ROOT.equals(spec.specType)) {
-                               int resourceId = ss.getTransientId(spec.resource);
-                               state.ids.put(resourceId, state.id++);
-                               // The fixed roots have been seen as externals by domain processor. Now remove them from external set.
-                               state.externals.remove(resourceId);
-                       }
-               }
-
-               this.externalBase = state.id;
-
-               final Collection<String> errors = new HashSet<>();
-
-               // All resource considered as not internal by domain processor. Can also contain roots.
-               int[] externals = state.externals.toArray();
-
-               // Build up the state.externals, externalNames and externalParents
-               for(int i=0;i<externals.length;i++) {
-                       getId(graph, externals[i], errors);
-               }
-
-               state.inverses.forEachEntry(new TIntIntProcedure() {
-
-                       @Override
-                       public boolean execute(int predicate, int inverse) {
-                               try {
-                                       getId(graph, predicate, errors);
-                                       if(inverse != 0) getId(graph, inverse, errors);
-                               } catch (DatabaseException e) {
-                                       throw new RuntimeDatabaseException(e);
-                               }
-                               return true;
-                       }
-
-               });
-
-               if(!errors.isEmpty()) {
-                       ArrayList<String> sorted = new ArrayList<>(errors);
-                       Collections.sort(sorted);
-                       StringBuilder message = new StringBuilder();
-                       message.append("Errors in exported model:\n");
-                       for(String error : sorted) {
-                               message.append(error);
-                               message.append("\n");
-                       }
-                       throw new DatabaseException(message.toString());
-               }
-
-               this.resourceCount = state.id;
-
-               state.extensions.put(ExternalDownloadBean.EXTENSION_KEY, new Variant(ExternalDownloadBean.BINDING, new ExternalDownloadBean(downloads)));
-
-       }
-
-       int indent = 0;
-
-       public boolean validateExternal(Resource ext) {
-               if(configuration.validate) {
-                       ExtentStatus status = configuration.preStatus.get(ext);
-                       if(status != null) {
-                               if(ExtentStatus.INTERNAL.equals(status)) return false;
-                               else if(ExtentStatus.EXCLUDED.equals(status)) return false;
-                       }
-               }
-               return true;
-       }
-
-       private Resource getResource(ReadGraph graph, int r) throws DatabaseException {
-               SerialisationSupport ss = graph.getService(SerialisationSupport.class);
-               return ss.getResource(r);
-       }
-
-       final public int getExistingId(ReadGraph graph, int r) throws DatabaseException {
-
-               int ret = state.ids.get(r);
-               if(ret != -1) {
-                       return ret;
-               } else {
-                       SerialisationSupport ss = graph.getService(SerialisationSupport.class);
-                       throw new DatabaseException("Id has not been created for " + NameUtils.getSafeName(graph, ss.getResource(r)));
-               }
-
-       }
-
-       /*
-        * 
-        * @return -2 if r is not really external and the statement should be excluded
-        * 
-        */
-       public int getId(ReadGraph graph, int r, Collection<String> errors) throws DatabaseException {
+    private static final Logger                   LOGGER          = LoggerFactory
+            .getLogger(ModelTransferableGraphSource.class);
+
+    final private TransferableGraphConfiguration2 configuration;
+
+    final private DomainProcessorState            state;
+
+    final private int                             externalBase;
+
+    final private int                             resourceCount;
+
+    final private File[]                          files;
+
+    final private TGValueModifier                 valueModifier;
+
+    private volatile boolean                      closed          = false;
+
+    TIntArrayList                                 externalParents = new TIntArrayList();
+
+    ArrayList<String>                             externalNames   = new ArrayList<>();
+
+    TreeMap<String, String>                       downloads       = new TreeMap<String, String>();
+
+    public ModelTransferableGraphSource(final ReadGraph graph, TransferableGraphConfiguration2 configuration,
+            final DomainProcessorState state, File... fs) throws DatabaseException {
+
+        this.configuration = configuration;
+        this.state = state;
+        this.files = fs;
+        this.valueModifier = state.valueModifier;
+
+        SerialisationSupport ss = graph.getService(SerialisationSupport.class);
+
+        // At this point ids contains all internal resources. Now add roots and
+        // externals.
+
+        // Root Library
+        state.ids.put(ss.getTransientId(graph.getRootLibrary()), state.id++);
+
+        // External roots - internal roots were already processed as internal
+        // resources by domain processor
+        for (SeedSpec spec : configuration.seeds) {
+            if (SeedSpecType.SPECIAL_ROOT.equals(spec.specType)) {
+                int resourceId = ss.getTransientId(spec.resource);
+                state.ids.put(resourceId, state.id++);
+                // The fixed roots have been seen as externals by domain
+                // processor. Now remove them from external set.
+                state.externals.remove(resourceId);
+            }
+        }
+
+        this.externalBase = state.id;
+
+        final Collection<String> errors = new HashSet<>();
+
+        // All resource considered as not internal by domain processor. Can also
+        // contain roots.
+        int[] externals = state.externals.toArray();
+
+        // Build up the state.externals, externalNames and externalParents
+        for (int i = 0; i < externals.length; i++) {
+            getId(graph, externals[i], errors);
+        }
+
+        state.inverses.forEachEntry(new TIntIntProcedure() {
+
+            @Override
+            public boolean execute(int predicate, int inverse) {
+                try {
+                    getId(graph, predicate, errors);
+                    if (inverse != 0)
+                        getId(graph, inverse, errors);
+                } catch (DatabaseException e) {
+                    throw new RuntimeDatabaseException(e);
+                }
+                return true;
+            }
+
+        });
+
+        if (!errors.isEmpty()) {
+            ArrayList<String> sorted = new ArrayList<>(errors);
+            Collections.sort(sorted);
+            StringBuilder message = new StringBuilder();
+            message.append("Errors in exported model:\n");
+            for (String error : sorted) {
+                message.append(error);
+                message.append("\n");
+            }
+            throw new DatabaseException(message.toString());
+        }
+
+        this.resourceCount = state.id;
+
+        state.extensions.put(ExternalDownloadBean.EXTENSION_KEY,
+                new Variant(ExternalDownloadBean.BINDING, new ExternalDownloadBean(downloads)));
+
+    }
+
+    int indent = 0;
+
+    public boolean validateExternal(Resource ext) {
+        if (configuration.validate) {
+            ExtentStatus status = configuration.preStatus.get(ext);
+            if (status != null) {
+                if (ExtentStatus.INTERNAL.equals(status))
+                    return false;
+                else if (ExtentStatus.EXCLUDED.equals(status))
+                    return false;
+            }
+        }
+        return true;
+    }
+
+    private Resource getResource(ReadGraph graph, int r) throws DatabaseException {
+        SerialisationSupport ss = graph.getService(SerialisationSupport.class);
+        return ss.getResource(r);
+    }
+
+    final public int getExistingId(ReadGraph graph, int r) throws DatabaseException {
+
+        int ret = state.ids.get(r);
+        if (ret != -1) {
+            return ret;
+        } else {
+            SerialisationSupport ss = graph.getService(SerialisationSupport.class);
+            throw new DatabaseException(
+                    "Id has not been created for " + NameUtils.getSafeName(graph, ss.getResource(r)));
+        }
+
+    }
+
+    /*
+     * 
+     * @return -2 if r is not really external and the statement should be
+     * excluded
+     * 
+     */
+    public int getId(ReadGraph graph, int r, Collection<String> errors) throws DatabaseException {
 
 //             // First external is root library
 //             if(r == rootId) return internalCount;
 
 
 //             // First external is root library
 //             if(r == rootId) return internalCount;
 
-               SerialisationSupport ss = graph.getService(SerialisationSupport.class);
-               Layer0 L0 = Layer0.getInstance(graph);
-
-               if(state.ids.containsKey(r)) {
-                   int ret = state.ids.get(r);
-                   if(ret == -1) {
-                       for(int i=0;i<=indent;++i)
-                           System.out.print("  ");
-                       System.out.println("Cycle!!!"); // with " + GraphUtils.getReadableName(g, r));
-                   }
-                       return ret;
-               }
-               else {
-                       Resource res = getResource(graph, r);
-            if(!validateExternal(res)) {
-               errors.add("Illegal reference to " + graph.getPossibleURI(getResource(graph, r)));
-               return -2;
+        SerialisationSupport ss = graph.getService(SerialisationSupport.class);
+        Layer0 L0 = Layer0.getInstance(graph);
+
+        if (state.ids.containsKey(r)) {
+            int ret = state.ids.get(r);
+            if (ret == -1) {
+                for (int i = 0; i <= indent; ++i)
+                    System.out.print("  ");
+                System.out.println("Cycle!!!"); // with " +
+                                                // GraphUtils.getReadableName(g,
+                                                // r));
+            }
+            return ret;
+        } else {
+            Resource res = getResource(graph, r);
+            if (!validateExternal(res)) {
+                errors.add("Illegal reference to " + graph.getPossibleURI(getResource(graph, r)));
+                return -2;
+            }
+            Collection<Resource> parents = graph.getObjects(res, L0.PartOf);
+            if (parents.size() != 1) {
+                throw new ValidationException(
+                        "Reference to external resource " + NameUtils.getSafeName(graph, getResource(graph, r), true)
+                                + " without unique uri (" + parents.size() + " parents).");
             }
             }
-                       Collection<Resource> parents = graph.getObjects(res, L0.PartOf);
-                       if(parents.size() != 1) {
-                               throw new ValidationException("Reference to external resource " 
-                                               + NameUtils.getSafeName(graph, getResource(graph, r), true) + " without unique uri (" + parents.size() + " parents).");
-                       }
-                       int pid = 0;
-                       for(Resource p : parents) {
-                           ++indent;
-                           pid = getId(graph, ss.getTransientId(p), errors);
-                           if(pid == -2) {
-                       errors.add("Illegal reference to " + graph.getPossibleURI(getResource(graph, r)));
-                               return -2;
-                           }
-                       }
-                       --indent;
-                       String name = graph.getRelatedValue(res, L0.HasName);
-                       // Record the external entry
-                       externalParents.add(pid);
+            int pid = 0;
+            for (Resource p : parents) {
+                ++indent;
+                pid = getId(graph, ss.getTransientId(p), errors);
+                if (pid == -2) {
+                    errors.add("Illegal reference to " + graph.getPossibleURI(getResource(graph, r)));
+                    return -2;
+                }
+            }
+            --indent;
+            String name = graph.getRelatedValue(res, L0.HasName);
+            // Record the external entry
+            externalParents.add(pid);
             externalNames.add(name);
             externalNames.add(name);
-                       state.ids.put(r, state.id);
-                       // Ensure that this resource is included into the set of externals to maintain the total number of externals 
-                       state.externals.add(r);
-                       String download = graph.getPossibleRelatedValue(res, L0.Ontology_download, Bindings.STRING);
-                       if(download != null) {
-                               String uri = graph.getURI(res);
-                               downloads.put(uri, download);
-                       }
-                       return state.id++;
-               }
-       }
-
-       @Override
-       public DataContainer getHeader() throws Exception {
-           return null;
-       }
-
-       @Override
-       public int getResourceCount() {
-               return resourceCount;
-       }
-
-       private int countRootSeeds() {
-               int result = 0;
-               for(SeedSpec spec : configuration.seeds) {
-                       if(SeedSpecType.INTERNAL.equals(spec.specType)) continue;
-                       result++;
-               }
-               return result;
-       }
-
-       @Override
-       public int getIdentityCount() {
-               return countRootSeeds() + state.externals.size() + state.internalEntries.size() + 1;
-       }
-
-       @Override
-       public int getStatementCount() {
-               return state.statementCount;
-       }
-
-       @Override
-       public int getValueCount() {
-               return state.valueCount;
-       }
-
-       @Override
-       public void forStatements(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
-
-               int[] value = new int[4];
-               long length = state.otherStatementsInput.length();
-               state.otherStatementsInput.position(0);
-
-               while(state.otherStatementsInput.position() < length && !state.monitor.isCanceled()) {
-
-                       int s = state.otherStatementsInput.readInt();
-                       int subjectId = state.ids.get(s);
-
-                       boolean exclude = subjectId == -1;
-
-                       int size = state.otherStatementsInput.readInt();
-                       for(int i=0;i<size;i++) {
-                               int p = state.otherStatementsInput.readInt();
-                               int o = state.otherStatementsInput.readInt();
-                               if(!exclude) {
-                                       if(state.pending.contains(o)) {
-                                               System.err.println("excluding garbage statement " + s + " " + p + " " + o + ", object resource is garbage");
-                                       } else if(state.excludedShared.contains(o)) {
-                                               System.err.println("excluding shared " + s + " " + p + " " + o);
-                                       } else {
-
-                                               int objectId = getExistingId(graph, o);
-                                               // The statement can be denied still
-                                               if(objectId != -2) {
-                                                       value[0] = subjectId;
-                                                       value[1] = getExistingId(graph, p);
-                                                       int inverse = state.inverses.get(p);
-                                                       if(inverse != 0) {
-                                                               value[2] = getExistingId(graph, inverse);
-                                                       } else {
-                                                               value[2] = -1;
-                                                       }
-                                                       value[3] = objectId;
-
-                                                       procedure.execute(value);
-
-                                               } else {
-                                                       System.err.println("Denied (" + NameUtils.getSafeName(graph, getResource(graph, s)) + ", " + NameUtils.getSafeName(graph, getResource(graph, p)) + "," + NameUtils.getSafeName(graph, getResource(graph, o)) + ")");
-                                               }
-
-                                       }
-                               } else {
-                                       System.err.println("excluding shared " + s);
-                               }
-                       }
-               }
-       }
-
-       @Override
-       public void forValues(ReadGraph graph, TransferableGraphSourceProcedure<Value> procedure) throws Exception {
-
-               Serializer variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT);
-               List<Object> idContext = new ArrayList<>();
-               long length = state.valueInput.length();
-               state.valueInput.position(0);
-
-               while(state.valueInput.position() < length && !state.monitor.isCanceled()) {
-
-                       // Ignore value type tag
-                       int s = state.valueInput.readInt();
-                       byte valueType = state.valueInput.readByte();
-                       switch (valueType) {
-                       case TAG_RAW_COPY_VARIANT_VALUE:
-                               state.valueInput.readInt();
-                               // Intentional fallthrough.
-
-                       case TAG_POTENTIALLY_MODIFIED_VARIANT_VALUE: {
-                               idContext.clear();
-                               Variant variant = (Variant) variantSerializer.deserialize((DataInput)state.valueInput, idContext);
-                               if (valueModifier.mayNeedModification(variant.type())) {
-                                       Object currentObject = variant.getValue();
-                                       Object newObject = valueModifier.modify(state, variant.getBinding(), currentObject);
-                                       if (newObject != currentObject)
-                                               variant = new Variant(variant.getBinding(), newObject);
-                               }
-                               procedure.execute(new Value(state.ids.get(s), variant));
-                               break;
-                       }
-
-                       default:
-                               throw new IllegalArgumentException("Unrecognized variant value type encountered: " + valueType);
-                       }
-               }
-       }
+            state.ids.put(r, state.id);
+            // Ensure that this resource is included into the set of externals
+            // to maintain the total number of externals
+            state.externals.add(r);
+            String download = graph.getPossibleRelatedValue(res, L0.Ontology_download, Bindings.STRING);
+            if (download != null) {
+                String uri = graph.getURI(res);
+                downloads.put(uri, download);
+            }
+            return state.id++;
+        }
+    }
+
+    @Override
+    public DataContainer getHeader() throws Exception {
+        return null;
+    }
+
+    @Override
+    public int getResourceCount() {
+        return resourceCount;
+    }
+
+    private int countRootSeeds() {
+        int result = 0;
+        for (SeedSpec spec : configuration.seeds) {
+            if (SeedSpecType.INTERNAL.equals(spec.specType))
+                continue;
+            result++;
+        }
+        return result;
+    }
+
+    @Override
+    public int getIdentityCount() {
+        return countRootSeeds() + state.externals.size() + state.internalEntries.size() + 1;
+    }
+
+    @Override
+    public int getStatementCount() {
+        return state.statementCount;
+    }
+
+    @Override
+    public int getValueCount() {
+        return state.valueCount;
+    }
+
+    @Override
+    public void forStatements(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
+
+        int[] value = new int[4];
+        long length = state.otherStatementsInput.length();
+        state.otherStatementsInput.position(0);
+
+        while (state.otherStatementsInput.position() < length && !state.monitor.isCanceled()) {
+
+            int s = state.otherStatementsInput.readInt();
+            int subjectId = state.ids.get(s);
+
+            boolean exclude = subjectId == -1;
+
+            int size = state.otherStatementsInput.readInt();
+            for (int i = 0; i < size; i++) {
+                int p = state.otherStatementsInput.readInt();
+                int o = state.otherStatementsInput.readInt();
+                if (!exclude) {
+                    if (state.pending.contains(o)) {
+                        System.err.println("excluding garbage statement " + s + " " + p + " " + o
+                                + ", object resource is garbage");
+                    } else if (state.excludedShared.contains(o)) {
+                        System.err.println("excluding shared " + s + " " + p + " " + o);
+                    } else {
+
+                        int objectId = getExistingId(graph, o);
+                        // The statement can be denied still
+                        if (objectId != -2) {
+                            value[0] = subjectId;
+                            value[1] = getExistingId(graph, p);
+                            int inverse = state.inverses.get(p);
+                            if (inverse != 0) {
+                                value[2] = getExistingId(graph, inverse);
+                            } else {
+                                value[2] = -1;
+                            }
+                            value[3] = objectId;
+
+                            try {
+                                procedure.execute(value);
+                            } catch (Exception e) {
+                                LOGGER.error("Error while processing a statement in transferable graph source", e);
+                            }
+
+                        } else {
+                            System.err.println("Denied (" + NameUtils.getSafeName(graph, getResource(graph, s)) + ", "
+                                    + NameUtils.getSafeName(graph, getResource(graph, p)) + ","
+                                    + NameUtils.getSafeName(graph, getResource(graph, o)) + ")");
+                        }
+
+                    }
+                } else {
+                    System.err.println("excluding shared " + s);
+                }
+            }
+        }
+    }
+
+    @Override
+    public void forValues(ReadGraph graph, TransferableGraphSourceProcedure<Value> procedure) throws Exception {
+
+        Serializer variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT);
+        List<Object> idContext = new ArrayList<>();
+        long length = state.valueInput.length();
+        state.valueInput.position(0);
+
+        while (state.valueInput.position() < length && !state.monitor.isCanceled()) {
+
+            // Ignore value type tag
+            int s = state.valueInput.readInt();
+            byte valueType = state.valueInput.readByte();
+            switch (valueType) {
+                case TAG_RAW_COPY_VARIANT_VALUE:
+                    state.valueInput.readInt();
+                    // Intentional fallthrough.
+
+                case TAG_POTENTIALLY_MODIFIED_VARIANT_VALUE: {
+                    idContext.clear();
+                    Variant variant = (Variant) variantSerializer.deserialize((DataInput) state.valueInput, idContext);
+                    if (valueModifier.mayNeedModification(variant.type())) {
+                        Object currentObject = variant.getValue();
+                        Object newObject = valueModifier.modify(state, variant.getBinding(), currentObject);
+                        if (newObject != currentObject)
+                            variant = new Variant(variant.getBinding(), newObject);
+                    }
+                    try {
+                        procedure.execute(new Value(state.ids.get(s), variant));
+                    } catch (Exception e) {
+                        LOGGER.error("Error while processing a value in transferable graph source", e);
+                    }
+                    break;
+                }
+
+                default:
+                    throw new IllegalArgumentException("Unrecognized variant value type encountered: " + valueType);
+            }
+        }
+    }
 
     @Override
     public void forValues2(ReadGraph graph, TransferableGraphSourceValueProcedure procedure) throws Exception {
 
 
     @Override
     public void forValues2(ReadGraph graph, TransferableGraphSourceValueProcedure procedure) throws Exception {
 
-        Serializer datatypeSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.getBindingUnchecked(Datatype.class));
+        Serializer datatypeSerializer = graph.getService(Databoard.class)
+                .getSerializerUnchecked(Bindings.getBindingUnchecked(Datatype.class));
         List<Object> idContext = new ArrayList<>();
         long length = state.valueInput.length();
         state.valueInput.position(0);
 
         List<Object> idContext = new ArrayList<>();
         long length = state.valueInput.length();
         state.valueInput.position(0);
 
-        while(state.valueInput.position() < length && !state.monitor.isCanceled()) {
+        while (state.valueInput.position() < length && !state.monitor.isCanceled()) {
 
             int s = state.valueInput.readInt();
             byte valueType = state.valueInput.readByte();
 
             switch (valueType) {
 
             int s = state.valueInput.readInt();
             byte valueType = state.valueInput.readByte();
 
             switch (valueType) {
-            case TAG_RAW_COPY_VARIANT_VALUE: {
-                // Variant data could be copied raw, no need for modifications
-                // Note that the variant data is a concatenation of the
-                // variant datatype serialization and the value serialization.
-                // variantLength contains both datatype and value data.
-                int variantLength = state.valueInput.readInt();
-                procedure.rawCopy(state.ids.get(s), variantLength, state.valueInput);
-                break;
+                case TAG_RAW_COPY_VARIANT_VALUE: {
+                    // Variant data could be copied raw, no need for
+                    // modifications
+                    // Note that the variant data is a concatenation of the
+                    // variant datatype serialization and the value
+                    // serialization.
+                    // variantLength contains both datatype and value data.
+                    int variantLength = state.valueInput.readInt();
+                    try {
+                        procedure.rawCopy(state.ids.get(s), variantLength, state.valueInput);
+                    } catch (Exception e) {
+                        LOGGER.error("Error while processing a raw value in transferable graph source", e);
+                    }
+                    break;
+                }
+
+                case TAG_POTENTIALLY_MODIFIED_VARIANT_VALUE: {
+                    // Variant data may need to be modified.
+                    // Cannot optimize this case with raw copying.
+                    idContext.clear();
+                    Datatype type = (Datatype) datatypeSerializer.deserialize((DataInput) state.valueInput, idContext);
+
+                    if (valueModifier.mayNeedModification(type)) {
+                        Binding binding = Bindings.getBinding(type);
+                        Serializer serializer = Bindings.getSerializerUnchecked(binding);
+                        Object value = serializer.deserialize((DataInput) state.valueInput);
+                        value = valueModifier.modify(state, binding, value);
+                        byte[] bytes = serializer.serialize(value);
+                        try {
+                            procedure.execute(state.ids.get(s), type, new ByteBufferReadable(bytes));
+                        } catch (Exception e) {
+                            LOGGER.error("Error while processing a data type in transferable graph source", e);
+                        }
+                    } else {
+                        try {
+                            procedure.execute(state.ids.get(s), type, state.valueInput);
+                        } catch (Exception e) {
+                            LOGGER.error("Error while processing a raw value in transferable graph source", e);
+                        }
+                    }
+                    break;
+                }
+
+                default:
+                    throw new IllegalArgumentException("Unrecognized variant value type encountered: " + valueType);
             }
             }
+        }
+    }
+
+    protected Identity getRootIdentity(DomainProcessorState state, SerialisationSupport support, Resource rootLibrary)
+            throws DatabaseException {
+        return new Identity(state.ids.get(support.getTransientId(rootLibrary)), new External(-1, ""));
+    }
+
+    @Override
+    public void forIdentities(ReadGraph graph, TransferableGraphSourceProcedure<Identity> procedure) throws Exception {
+
+        SerialisationSupport support = graph.getService(SerialisationSupport.class);
+        Layer0 L0 = Layer0.getInstance(graph);
+
+        // TODO: this should be Root with name ""
+        try {
+            procedure.execute(getRootIdentity(state, support, graph.getRootLibrary()));
+        } catch (Exception e) {
+            LOGGER.error("Error while processing a root identity in transferable graph source", e);
+        }
+
+        TIntObjectMap<Identity> internalMap = new TIntObjectHashMap<>(100, 0.5f, Integer.MIN_VALUE);
 
 
-            case TAG_POTENTIALLY_MODIFIED_VARIANT_VALUE: {
-                // Variant data may need to be modified.
-                // Cannot optimize this case with raw copying.
-                idContext.clear();
-                Datatype type = (Datatype)datatypeSerializer.deserialize((DataInput)state.valueInput, idContext);
-
-                if (valueModifier.mayNeedModification(type)) {
-                    Binding binding = Bindings.getBinding(type);
-                    Serializer serializer = Bindings.getSerializerUnchecked(binding);
-                    Object value = serializer.deserialize((DataInput)state.valueInput);
-                    value = valueModifier.modify(state, binding, value);
-                    byte[] bytes = serializer.serialize(value);
-                    procedure.execute(state.ids.get(s), type, new ByteBufferReadable(bytes));
+        // Declare internal and external roots
+        for (SeedSpec r : configuration.seeds) {
+            if (SeedSpecType.INTERNAL.equals(r.specType))
+                continue;
+            String typeId = r.type;
+            if (typeId == null) {
+                Resource type = graph.getPossibleType(r.resource, L0.Entity);
+                typeId = type != null ? graph.getURI(type) : Layer0.URIs.Entity;
+            }
+            int id = state.ids.get(support.getTransientId(r.resource));
+            Root root = new Root(r.name, typeId);
+            Identity rootId = new Identity(id, root);
+            internalMap.put(id, rootId);
+            try {
+                procedure.execute(rootId);
+            } catch (Exception e) {
+                LOGGER.error("Error while processing a root in transferable graph source", e);
+            }
+        }
+
+        for (int i = 0; i < state.externals.size(); i++) {
+            int parent = externalParents.get(i);
+            String name = externalNames.get(i);
+            try {
+                procedure.execute(new Identity(externalBase + i, new External(parent, name)));
+            } catch (Exception e) {
+                LOGGER.error("Error while processing a an external identity in transferable graph source", e);
+            }
+        }
+
+        if (state.internalEntries != null) {
+            for (ConsistsOfProcessEntry ie : state.internalEntries) {
+                if (ie.parent != null) {
+                    if (ie.name != null) {
+                        try {
+                            procedure.execute(resolveInternal(graph, support, ie, internalMap));
+                        } catch (Exception e) {
+                            LOGGER.error("Error while processing an internal identity in transferable graph source", e);
+                        }
+                    } else {
+                        // In this case there is a child that has no HasName =>
+                        // this should be treated as a blank
+                    }
                 } else {
                 } else {
-                    procedure.execute(state.ids.get(s), type, state.valueInput);
+                    try {
+                        procedure.execute(resolveInternal(graph, support, ie, internalMap));
+                    } catch (Exception e) {
+                        LOGGER.error("Error while processing an internal identity in transferable graph source", e);
+                    }
                 }
                 }
-                break;
             }
             }
+        }
+
+    }
+
+    private Identity resolveInternal(ReadGraph graph, SerialisationSupport ss, ConsistsOfProcessEntry entry,
+            TIntObjectMap<Identity> internalMap) throws DatabaseException {
+        int id = state.ids.get(ss.getTransientId(entry.resource));
+        Identity existing = internalMap.get(id);
+        if (existing != null)
+            return existing;
+
+        if (entry.parent == null) {
+            Layer0 L0 = Layer0.getInstance(graph);
+            Resource possibleParent = graph.getPossibleObject(entry.resource, L0.PartOf);
+            if (possibleParent == null)
+                throw new DatabaseException("Invalid root or internal parent path: " + entry.resource);
+            int externalId = state.ids.get(ss.getTransientId(possibleParent));
+            Identity result = new Identity(id, new Internal(externalId, entry.name));
+            internalMap.put(id, result);
+            return result;
+        } else {
+            Identity parent = resolveInternal(graph, ss, entry.parent, internalMap);
+            Identity result = new Identity(id, new Internal(parent.resource, entry.name));
+            internalMap.put(id, result);
+            return result;
+        }
+    }
+
+    @Override
+    public TreeMap<String, Variant> getExtensions() {
+        return state.extensions;
+    }
+
+    public File[] getFiles() {
+        return files;
+    }
+
+    private static <T> T tryClose(T c) throws IOException {
+        if (c != null && c instanceof Closeable)
+            ((Closeable) c).close();
+        return null;
+    }
+
+    public void closeStreams() throws IOException {
+        state.valueInput = tryClose(state.valueInput);
+        state.otherStatementsInput = tryClose(state.otherStatementsInput);
+        state.statementsOutput = tryClose(state.statementsOutput);
+        state.valueOutput = tryClose(state.valueOutput);
+    }
+
+    @Override
+    public void reset() throws Exception {
+        throw new UnsupportedOperationException();
+    }
+
+    public long[] getResourceArray(ReadGraph graph) throws DatabaseException {
+        final SerialisationSupport ss = graph.getService(SerialisationSupport.class);
+        final long[] result = new long[state.ids.size()];
+        state.ids.forEachEntry(new TIntIntProcedure() {
+
+            @Override
+            public boolean execute(int a, int b) {
+
+                try {
+                    Resource r = ss.getResource(a);
+                    result[b] = r.getResourceId();
+                } catch (DatabaseException e) {
+                    e.printStackTrace();
+                }
+
+                return true;
 
 
-            default:
-                throw new IllegalArgumentException("Unrecognized variant value type encountered: " + valueType);
+            }
+        });
+        return result;
+    }
+
+    public DomainProcessorState getState() {
+        return state;
+    }
+
+    public void forResourceStatements(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure)
+            throws Exception {
+
+        int[] value = new int[4];
+        long length = state.otherStatementsInput.length();
+        state.otherStatementsInput.position(0);
+
+        while (state.otherStatementsInput.position() < length) {
+
+            int s = state.otherStatementsInput.readInt();
+            int subjectId = state.ids.get(s);
+
+            boolean exclude = subjectId == -1;
+
+            int size = state.otherStatementsInput.readInt();
+            for (int i = 0; i < size; i++) {
+                int p = state.otherStatementsInput.readInt();
+                int o = state.otherStatementsInput.readInt();
+                if (!exclude) {
+                    if (state.excludedShared.contains(o)) {
+                        System.err.println("excluding shared " + s + " " + p + " " + o);
+                    } else {
+
+                        int objectId = getExistingId(graph, o);
+                        // The statement can be denied still
+                        if (objectId != -2) {
+                            value[0] = s;
+                            value[1] = p;
+                            int inverse = state.inverses.get(p);
+                            if (inverse != 0) {
+                                value[2] = inverse;
+                            } else {
+                                value[2] = -1;
+                            }
+                            value[3] = o;
+
+                            try {
+                                procedure.execute(value);
+                            } catch (Exception e) {
+                                LOGGER.error("Error while processing a statement in transferable graph source", e);
+                            }
+
+                        } else {
+                            System.err.println("Denied (" + NameUtils.getSafeName(graph, getResource(graph, s)) + ", "
+                                    + NameUtils.getSafeName(graph, getResource(graph, p)) + ","
+                                    + NameUtils.getSafeName(graph, getResource(graph, o)) + ")");
+                        }
+
+                    }
+                } else {
+                    System.err.println("excluding shared " + s);
+                }
             }
         }
     }
 
             }
         }
     }
 
-       protected Identity getRootIdentity(DomainProcessorState state, SerialisationSupport support, Resource rootLibrary) throws DatabaseException {
-               return new Identity(state.ids.get(support.getTransientId(rootLibrary)), new External(-1, ""));
-       }
-
-       @Override
-       public void forIdentities(ReadGraph graph, TransferableGraphSourceProcedure<Identity> procedure) throws Exception {
-
-               SerialisationSupport support = graph.getService(SerialisationSupport.class);
-               Layer0 L0 = Layer0.getInstance(graph);
-
-               // TODO: this should be Root with name ""
-               procedure.execute(getRootIdentity(state, support, graph.getRootLibrary()));
-
-               TIntObjectMap<Identity> internalMap = new TIntObjectHashMap<>(100, 0.5f, Integer.MIN_VALUE);
-
-               // Declare internal and external roots
-               for(SeedSpec r : configuration.seeds) {
-                       if(SeedSpecType.INTERNAL.equals(r.specType)) continue;
-                       String typeId = r.type;
-                       if (typeId == null) {
-                               Resource type = graph.getPossibleType(r.resource, L0.Entity);
-                               typeId = type != null ? graph.getURI(type) : Layer0.URIs.Entity;
-                       }
-                       int id = state.ids.get(support.getTransientId(r.resource));
-                       Root root = new Root(r.name, typeId);
-                       Identity rootId = new Identity(id,root);
-                       internalMap.put(id, rootId);
-                       procedure.execute(rootId);
-               }
-
-               for(int i = 0; i < state.externals.size() ; i++) {
-                       int parent = externalParents.get(i);
-                       String name = externalNames.get(i);
-                       procedure.execute(new Identity(externalBase + i, new External(parent,name)));
-       }
-
-               if(state.internalEntries != null) {
-                       for(ConsistsOfProcessEntry ie : state.internalEntries) {
-                               if(ie.parent != null) {
-                                       if(ie.name != null) {
-                                               procedure.execute(resolveInternal(graph, support, ie, internalMap));
-                                       } else {
-                                               // In this case there is a child that has no HasName => this should be treated as a blank
-                                       }
-                               } else {
-                                       procedure.execute(resolveInternal(graph, support, ie, internalMap));
-                               }
-                       }
-               }
-
-       }
-
-       private Identity resolveInternal(ReadGraph graph, SerialisationSupport ss, ConsistsOfProcessEntry entry, TIntObjectMap<Identity> internalMap) throws DatabaseException {
-               int id = state.ids.get(ss.getTransientId(entry.resource));
-               Identity existing = internalMap.get(id);
-               if(existing != null) return existing;
-
-               if(entry.parent == null) {
-                       Layer0 L0 = Layer0.getInstance(graph);
-                       Resource possibleParent = graph.getPossibleObject(entry.resource, L0.PartOf);
-                       if(possibleParent == null) throw new DatabaseException("Invalid root or internal parent path: " + entry.resource);
-                       int externalId = state.ids.get(ss.getTransientId(possibleParent));
-                       Identity result = new Identity(id,
-                                       new Internal(externalId, entry.name));
-                       internalMap.put(id, result);
-                       return result;
-               } else {
-                       Identity parent = resolveInternal(graph, ss, entry.parent, internalMap);
-                       Identity result = new Identity(id,
-                                       new Internal(parent.resource, entry.name));
-                       internalMap.put(id, result);
-                       return result;
-               }
-       }
-
-       @Override
-       public TreeMap<String, Variant> getExtensions() {
-               return state.extensions;
-       }
-
-       public File[] getFiles() {
-           return files;
-       }
-
-       private static <T> T tryClose(T c) throws IOException {
-               if (c != null && c instanceof Closeable)
-                       ((Closeable) c).close();
-               return null;
-       }
-
-       public void closeStreams() throws IOException {
-               state.valueInput = tryClose(state.valueInput);
-               state.otherStatementsInput = tryClose(state.otherStatementsInput);
-               state.statementsOutput = tryClose(state.statementsOutput);
-               state.valueOutput = tryClose(state.valueOutput);
-       }
-
-       @Override
-       public void reset() throws Exception {
-           throw new UnsupportedOperationException();
-       }
-
-       public long[] getResourceArray(ReadGraph graph) throws DatabaseException {
-               final SerialisationSupport ss = graph.getService(SerialisationSupport.class);
-               final long[] result = new long[state.ids.size()];
-               state.ids.forEachEntry(new TIntIntProcedure() {
-
-                       @Override
-                       public boolean execute(int a, int b) {
-
-                               try {
-                                       Resource r = ss.getResource(a);
-                                       result[b] = r.getResourceId();
-                               } catch (DatabaseException e) {
-                                       e.printStackTrace();
-                               }
-
-                               return true;
-
-                       }
-               });
-               return result;
-       }
-
-       public DomainProcessorState getState() {
-               return state;
-       }
-
-       public void forResourceStatements(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
-
-               int[] value = new int[4];
-               long length = state.otherStatementsInput.length();
-               state.otherStatementsInput.position(0);
-
-               while(state.otherStatementsInput.position() < length) {
-
-                       int s = state.otherStatementsInput.readInt();
-                       int subjectId = state.ids.get(s);
-
-                       boolean exclude = subjectId == -1;
-
-                       int size = state.otherStatementsInput.readInt();
-                       for(int i=0;i<size;i++) {
-                               int p = state.otherStatementsInput.readInt();
-                               int o = state.otherStatementsInput.readInt();
-                               if(!exclude) {
-                                       if(state.excludedShared.contains(o)) {
-                                               System.err.println("excluding shared " + s + " " + p + " " + o);
-                                       } else {
-
-                                               int objectId = getExistingId(graph, o);
-                                               // The statement can be denied still
-                                               if(objectId != -2) {
-                                                       value[0] = s;
-                                                       value[1] = p;
-                                                       int inverse = state.inverses.get(p);
-                                                       if(inverse != 0) {
-                                                               value[2] = inverse;
-                                                       } else {
-                                                               value[2] = -1;
-                                                       }
-                                                       value[3] = o;
-
-                                                       procedure.execute(value);
-
-                                               } else {
-                                                       System.err.println("Denied (" + NameUtils.getSafeName(graph, getResource(graph, s)) + ", " + NameUtils.getSafeName(graph, getResource(graph, p)) + "," + NameUtils.getSafeName(graph, getResource(graph, o)) + ")");
-                                               }
-
-                                       }
-                               } else {
-                                       System.err.println("excluding shared " + s);
-                               }
-                       }
-               }
-       }
-
-       public void forValueResources(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
-               int[] value = { 0 };
-               long length = state.valueInput.length();
-               while (state.valueInput.position() < length) {
-                       value[0] = state.valueInput.readInt();
-                       procedure.execute(value);
-               }
-       }
-
-       public TransferableGraphConfiguration2 getConfiguration() {
-               return configuration;
-       }
-
-       @Override
-       public void close() throws IOException {
-               synchronized (this) {
-                       if (closed)
-                               return;
-                       closed = true;
-               }
-               closeStreams();
-               if (files != null) {
-                       for (File f : files) {
-                               Files.deleteIfExists(f.toPath());
-                       }
-               }
-       }
+    public void forValueResources(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
+        int[] value = { 0 };
+        long length = state.valueInput.length();
+        while (state.valueInput.position() < length) {
+            value[0] = state.valueInput.readInt();
+            try {
+                procedure.execute(value);
+            } catch (Exception e) {
+                LOGGER.error("Error while processing a value in transferable graph source", e);
+            }
+        }
+    }
+
+    public TransferableGraphConfiguration2 getConfiguration() {
+        return configuration;
+    }
+
+    @Override
+    public void close() throws IOException {
+        synchronized (this) {
+            if (closed)
+                return;
+            closed = true;
+        }
+        closeStreams();
+        if (files != null) {
+            for (File f : files) {
+                Files.deleteIfExists(f.toPath());
+            }
+        }
+    }
 
 }
\ No newline at end of file
 
 }
\ No newline at end of file
index 1a264dc515d5329d22ec1f285f1043cf08771d1a..fa0b0797f2a2e0e1c4fb34d8af4cbea3bf4c9ea4 100644 (file)
@@ -54,7 +54,12 @@ public final class ClusterChange {
     public ClusterImpl clusterImpl;
 
     public ClusterChange(ClusterStream clusterStream, ClusterImpl clusterImpl) {
     public ClusterImpl clusterImpl;
 
     public ClusterChange(ClusterStream clusterStream, ClusterImpl clusterImpl) {
+       
         this.clusterImpl = clusterImpl;
         this.clusterImpl = clusterImpl;
+
+        if (!clusterImpl.isLoaded())
+            new IllegalStateException("Change to proxy cluster " + clusterImpl.getClusterUID()).printStackTrace();
+
         clusterUID = clusterImpl.getClusterUID();
         long[] longs = new long[ClusterUID.getLongLength()];
         clusterUID.toLong(longs, 0);
         clusterUID = clusterImpl.getClusterUID();
         long[] longs = new long[ClusterUID.getLongLength()];
         clusterUID.toLong(longs, 0);
@@ -69,9 +74,13 @@ public final class ClusterChange {
             Bytes.writeLE(header, offset, longs[i]);
         //initBuffer();
         this.clusterStream = clusterStream;
             Bytes.writeLE(header, offset, longs[i]);
         //initBuffer();
         this.clusterStream = clusterStream;
-        this.clusterChange2 = new ClusterChange2(clusterUID, clusterImpl);
+        this.clusterChange2 = new ClusterChange2(clusterUID);
         clusterStream.changes.add(this);
     }
         clusterStream.changes.add(this);
     }
+    
+    public void adopt(ClusterImpl impl) {
+        this.clusterImpl = impl;
+    }
 
     private void setHeaderVectorSize(int size) {
         if (size < 0)
 
     private void setHeaderVectorSize(int size) {
         if (size < 0)
@@ -81,7 +90,7 @@ public final class ClusterChange {
     }
     @Override
     public String toString() {
     }
     @Override
     public String toString() {
-        return super.toString() + " cluster=" + clusterImpl.getClusterUID() + " id=" + clusterImpl.getClusterId() + " off=" + byteIndex;
+        return super.toString() + " cluster=" + clusterUID + " id=" + clusterImpl.getClusterId() + " off=" + byteIndex;
     }
     private final void initBuffer() {
         flushed = false;
     }
     private final void initBuffer() {
         flushed = false;
index cebcdcf4294f270b8a17a79a12632fa52a9dfda2..234f2ada433ab8d858038cb8b26d732bf1121add 100644 (file)
@@ -2,7 +2,6 @@ package fi.vtt.simantics.procore.internal;
 
 import java.util.Arrays;
 
 
 import java.util.Arrays;
 
-import org.simantics.db.procore.cluster.ClusterImpl;
 import org.simantics.db.service.Bytes;
 import org.simantics.db.service.ClusterUID;
 
 import org.simantics.db.service.Bytes;
 import org.simantics.db.service.ClusterUID;
 
@@ -16,7 +15,7 @@ public class ClusterChange2 {
     private byte[] bytes;
     private int byteIndex;
     private ClusterUID clusterUID;
     private byte[] bytes;
     private int byteIndex;
     private ClusterUID clusterUID;
-    ClusterChange2(ClusterUID clusterUID, ClusterImpl cluster) {
+    ClusterChange2(ClusterUID clusterUID) {
         this.clusterUID = clusterUID;
         init();
     }
         this.clusterUID = clusterUID;
         init();
     }
index c6fd11c4eb35900763e1176b51fe90f7001819e3..7df13315f8f68b093fdbf6cb91b0359b5bb909e1 100644 (file)
@@ -289,14 +289,29 @@ public final class ClusterTable implements IClusterTable {
         }
     }
 
         }
     }
 
-    synchronized void replaceCluster(ClusterI cluster) {
-        //printMaps("replaceCluster");
+    synchronized void replaceCluster(ClusterI cluster_) {
+        ClusterImpl cluster = (ClusterImpl) cluster_;
         checkCollect();
         int clusterKey = cluster.getClusterKey();
         checkCollect();
         int clusterKey = cluster.getClusterKey();
-        ClusterI existing = clusterArray[clusterKey];
+        ClusterImpl existing = (ClusterImpl) clusterArray[clusterKey];
         if (existing.hasVirtual())
             cluster.markVirtual();
 
         if (existing.hasVirtual())
             cluster.markVirtual();
 
+        if (existing.cc != null) {
+            if (existing.isLoaded()) {
+                // This shall be promoted to actual exception in the future -
+                // for now, minimal changes
+                new Exception("Trying to replace cluster with pending changes " + existing.getClusterUID())
+                        .printStackTrace();
+            } else {
+                // Adopt changes to loaded cluster
+                cluster.cc = existing.cc;
+                cluster.cc.adopt(cluster);
+                cluster.foreignLookup = existing.foreignLookup;
+                cluster.change = existing.change;
+            }
+        }
+        
         importanceMap.remove(existing.getImportance());
         if (collectorPolicy != null)
             collectorPolicy.removed((ClusterImpl)existing);
         importanceMap.remove(existing.getImportance());
         if (collectorPolicy != null)
             collectorPolicy.removed((ClusterImpl)existing);
@@ -589,7 +604,6 @@ public final class ClusterTable implements IClusterTable {
             public boolean execute(int clusterKey) {
                 ClusterImpl proxy = clusterArray[clusterKey];
                 ClusterUID clusterUID = proxy.getClusterUID();
             public boolean execute(int clusterKey) {
                 ClusterImpl proxy = clusterArray[clusterKey];
                 ClusterUID clusterUID = proxy.getClusterUID();
-                System.err.println("writeOnlyInvalidate " + clusterUID);
                 clusters.freeProxy(proxy);
                 return true;
             }
                 clusters.freeProxy(proxy);
                 return true;
             }
index 8313beecad40da561e4e9916dc115d35b99aa5c8..10511979176a64c21dbd90c1c2ab69f9176fba0a 100644 (file)
@@ -24,116 +24,121 @@ import org.simantics.db.impl.support.ResourceSupport;
 
 final public class DirectStatementsImpl implements DirectStatements {
 
 
 final public class DirectStatementsImpl implements DirectStatements {
 
-       final ResourceSupport support;
-       final int subject;
-       final TIntArrayList statements = new TIntArrayList();
-       
-       DirectStatementsImpl(ResourceSupport support, int subject) {
-               this.support = support;
-               this.subject = subject;
-       }
-       
-       void addStatement(int p, int o) {
-               statements.add(p);
-               statements.add(o);
-       }
-       
-       @Override
-       public Resource getSubject() {
-               return new ResourceImpl(support, subject);
-       }
-
-       @Override
-       public boolean add(Statement arg0) {
-               throw new Error("Not supported");
-       }
-
-       @Override
-       public boolean addAll(Collection<? extends Statement> arg0) {
-               throw new Error("Not supported");
-       }
-
-       @Override
-       public void clear() {
-               throw new Error("Not supported");
-       }
-
-       @Override
-       public boolean contains(Object arg0) {
-        Statement stm = (Statement)arg0;
-        ResourceImpl p = (ResourceImpl)stm.getPredicate();
-        ResourceImpl o = (ResourceImpl)stm.getObject();
-        for(int i=0;i<statements.size();i+=2) {
+    final ResourceSupport support;
+
+    final int             subject;
+
+    final TIntArrayList   statements = new TIntArrayList();
+
+    DirectStatementsImpl(ResourceSupport support, int subject) {
+        this.support = support;
+        this.subject = subject;
+    }
+
+    void addStatement(int p, int o) {
+        statements.add(p);
+        statements.add(o);
+    }
+
+    @Override
+    public Resource getSubject() {
+        return new ResourceImpl(support, subject);
+    }
+
+    @Override
+    public boolean add(Statement arg0) {
+        throw new Error("Not supported");
+    }
+
+    @Override
+    public boolean addAll(Collection<? extends Statement> arg0) {
+        throw new Error("Not supported");
+    }
+
+    @Override
+    public void clear() {
+        throw new Error("Not supported");
+    }
+
+    @Override
+    public boolean contains(Object arg0) {
+        Statement stm = (Statement) arg0;
+        ResourceImpl p = (ResourceImpl) stm.getPredicate();
+        ResourceImpl o = (ResourceImpl) stm.getObject();
+        for (int i = 0; i < statements.size(); i += 2) {
             int pi = statements.getQuick(i);
             int pi = statements.getQuick(i);
-            int oi = statements.getQuick(i+1);
-            if(p.id == pi && o.id == oi) return true;
+            int oi = statements.getQuick(i + 1);
+            if (p.id == pi && o.id == oi)
+                return true;
         }
         return false;
         }
         return false;
-       }
-
-       @Override
-       public boolean containsAll(Collection<?> arg0) {
-               throw new Error("Not supported");
-       }
-
-       @Override
-       public boolean isEmpty() {
-               throw new Error("Not supported");
-       }
-
-       @Override
-       public Iterator<Statement> iterator() {
-               return new Iterator<Statement>() {
-
-                       int index = 0;
-                       int max = statements.size();
-                       
-                       @Override
-                       public boolean hasNext() {
-                               return index < max; 
-                       }
-
-                       @Override
-                       public Statement next() {
-                               return new DirectStatementImpl(support, subject, statements.getQuick(index++), statements.getQuick(index++));
-                       }
-
-                       @Override
-                       public void remove() {
-                               throw new Error("Not supported");
-                       }
-                       
-               };
-       }
-
-       @Override
-       public boolean remove(Object arg0) {
-               throw new Error("Not supported");
-       }
-
-       @Override
-       public boolean removeAll(Collection<?> arg0) {
-               throw new Error("Not supported");
-       }
-
-       @Override
-       public boolean retainAll(Collection<?> arg0) {
-               throw new Error("Not supported");
-       }
-
-       @Override
-       public int size() {
-               return statements.size() >> 1;
-       }
-
-       @Override
-       public Object[] toArray() {
-               throw new Error("Not supported");
-       }
-
-       @Override
-       public <T> T[] toArray(T[] arg0) {
-               throw new Error("Not supported");
-       }
-       
+    }
+
+    @Override
+    public boolean containsAll(Collection<?> arg0) {
+        throw new Error("Not supported");
+    }
+
+    @Override
+    public boolean isEmpty() {
+        throw new Error("Not supported");
+    }
+
+    @Override
+    public Iterator<Statement> iterator() {
+        return new Iterator<Statement>() {
+
+            int index = 0;
+
+            int max   = statements.size();
+
+            @Override
+            public boolean hasNext() {
+                return index < max;
+            }
+
+            @Override
+            public Statement next() {
+                return new DirectStatementImpl(support, subject, statements.getQuick(index++),
+                        statements.getQuick(index++));
+            }
+
+            @Override
+            public void remove() {
+                throw new Error("Not supported");
+            }
+
+        };
+    }
+
+    @Override
+    public boolean remove(Object arg0) {
+        throw new Error("Not supported");
+    }
+
+    @Override
+    public boolean removeAll(Collection<?> arg0) {
+        throw new Error("Not supported");
+    }
+
+    @Override
+    public boolean retainAll(Collection<?> arg0) {
+        throw new Error("Not supported");
+    }
+
+    @Override
+    public int size() {
+        return statements.size() >> 1;
+    }
+
+    @Override
+    public Object[] toArray() {
+        throw new Error("Not supported");
+    }
+
+    @Override
+    public <T> T[] toArray(T[] arg0) {
+        throw new Error("Not supported");
+    }
+
 }
 }
index 9290ac99fdd98a5792a542a0b51b7309941f1704..d71a8bfc81c491d52dc7afe30cae65a5e8e95d60 100644 (file)
@@ -1022,15 +1022,13 @@ public abstract class SessionImplSocket implements Session, WriteRequestSchedule
                        try {
 
                                int sResourceKey = ClusterTraits.getResourceIndexFromResourceKey(s);
                        try {
 
                                int sResourceKey = ClusterTraits.getResourceIndexFromResourceKey(s);
-                               int pResourceKey = ClusterTraits.getResourceIndexFromResourceKey(p);
-                               int oResourceKey = ClusterTraits.getResourceIndexFromResourceKey(o);
 
                                ClusterI pc = clusterTable.getClusterProxyByResourceKey(p);
                                ClusterI oc = clusterTable.getClusterProxyByResourceKey(o);
 
                                clusterTranslator.addStatementIndex(cluster, sResourceKey, cluster.getClusterUID(), ClusterChange.REMOVE_OPERATION);
 
                                ClusterI pc = clusterTable.getClusterProxyByResourceKey(p);
                                ClusterI oc = clusterTable.getClusterProxyByResourceKey(o);
 
                                clusterTranslator.addStatementIndex(cluster, sResourceKey, cluster.getClusterUID(), ClusterChange.REMOVE_OPERATION);
-                               clusterTranslator.addStatementIndex(cluster, pResourceKey, pc.getClusterUID(), ClusterStream.NULL_OPERATION);
-                               clusterTranslator.addStatementIndex(cluster, oResourceKey, oc.getClusterUID(), ClusterStream.NULL_OPERATION);
+                               clusterTranslator.addStatementIndex(cluster, p, pc.getClusterUID(), ClusterStream.NULL_OPERATION);
+                               clusterTranslator.addStatementIndex(cluster, o, oc.getClusterUID(), ClusterStream.NULL_OPERATION);
                                clusterTranslator.removeStatement(cluster);
 
                                queryProvider2.invalidateResource(s);
                                clusterTranslator.removeStatement(cluster);
 
                                queryProvider2.invalidateResource(s);