From: Antti Villberg Date: Wed, 8 Mar 2017 14:50:59 +0000 (+0200) Subject: Import/export changes. A load. X-Git-Tag: v1.29.0~129 X-Git-Url: https://gerrit.simantics.org/r/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F51%2F351%2F13;p=simantics%2Fplatform.git Import/export changes. A load. May cause some problems. Exports create Internals instead of blanks if possible. FixExportedOntology treatment now produces tgs that are compatible with Graph Compiler. Import creates L0.ExternalEntity instances for unresolved external references (thus making it fail safe). Note that addBuiltin("http://Projects") was removed from class CoreInitialization because its existence resulted in a non-functioning URI space because: * http://Projects was added as a "built-in resource" when the database is first initialized which means that ReadGraph.getResource("http://Projects") would succeed even when the resource was not yet properly imported from the project ontology * When the project ontology is imported, a new resource is created to represent http://Projects instead of using the existing uninitialized "built-in resource". The L0.ExternalEntity changes made to the standard transferable graph import in this patch caused the code to find the built-in resource for "http://Projects" and think that it has to be a resource that's already properly initialized. This led to the TG import not initializing the resource at all. Previously there was a bug here as well but it was hidden by the fact that the system would import the actual/new "http://Projects" resource from the ontology anyway. This effectively left the uninitialized built-in resource just hanging in the database as trash. refs #7016 Change-Id: I5d9dd8dea4df58cc54d4a6664c112a770ae7f7b3 --- diff --git a/bundles/org.simantics.db.impl/src/org/simantics/db/impl/query/URIToResource.java b/bundles/org.simantics.db.impl/src/org/simantics/db/impl/query/URIToResource.java index e5a14f77c..c466cb261 100644 --- a/bundles/org.simantics.db.impl/src/org/simantics/db/impl/query/URIToResource.java +++ b/bundles/org.simantics.db.impl/src/org/simantics/db/impl/query/URIToResource.java @@ -141,8 +141,9 @@ public class URIToResource extends StringQuery> { } } - if(entry != null) entry.addOrSet(graph, graph.processor, new Integer(0)); - procedure.execute(graph, new Integer(0)); + Integer zero = 0; + if(entry != null) entry.addOrSet(graph, graph.processor, zero); + procedure.execute(graph, zero); } diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/adapter/impl/SharedOntologyImportAdvisor.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/adapter/impl/SharedOntologyImportAdvisor.java index e16d07cf0..2a575a578 100644 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/adapter/impl/SharedOntologyImportAdvisor.java +++ b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/adapter/impl/SharedOntologyImportAdvisor.java @@ -33,6 +33,7 @@ import org.simantics.graph.db.MissingDependencyException; import org.simantics.graph.db.TransferableGraphImporter; import org.simantics.graph.representation.Root; import org.simantics.layer0.Layer0; +import org.simantics.utils.datastructures.Pair; import org.simantics.utils.datastructures.Triple; public class SharedOntologyImportAdvisor extends AbstractImportAdvisor2 { @@ -87,7 +88,15 @@ public class SharedOntologyImportAdvisor extends AbstractImportAdvisor2 { } Resource existing = graph.getPossibleResource(uri); - if(existing != null) throw new DatabaseException("Shared library " + uri + " exists already."); + if(existing != null) { + if(graph.isInstanceOf(existing, L0.ExternalEntity)) { + created.put(r, existing); + addRootInfo(r, r.name, existing); + return null; + } else { + throw new DatabaseException("Shared library " + uri + " exists already."); + } + } Resource type = graph.getPossibleResource(r.type); if(type == null) { @@ -124,43 +133,62 @@ public class SharedOntologyImportAdvisor extends AbstractImportAdvisor2 { public void beforeWrite(WriteOnlyGraph graph, TransferableGraphImporter process) throws DatabaseException { - graph.markUndoPoint(); - if(published) { - XSupport xs = graph.getService(XSupport.class); - xs.setServiceMode(false, true); - } - - graph.setClusterSet4NewResource(graph.getRootLibrary()); - graph.flushCluster(); - - for(Map.Entry>> entry : toCreate.entrySet()) { - - Triple> recipe = entry.getValue(); - - Resource base = recipe.first; - for(int i=0;i mode = xs.getServiceMode(); + + try { + + graph.markUndoPoint(); + + if(published) { + xs.setServiceMode(true, true); + } else { + xs.setServiceMode(true, mode.second); + } + + graph.setClusterSet4NewResource(graph.getRootLibrary()); + graph.flushCluster(); + + for(RootInfo info : getRootInfo()) { + // At this stage these are existing L0.ExternalEntity instances that are now being imported. + graph.deny(info.resource, L0.InstanceOf, null, L0.ExternalEntity, null); + } + + for(Map.Entry>> entry : toCreate.entrySet()) { + + Triple> recipe = entry.getValue(); + + Resource base = recipe.first; + for(int i=0;i roots; + + public final ImportResult tgResult; + + public MigratedImportResult(Collection roots, ImportResult tgResult) { + this.roots = roots; + this.tgResult = tgResult; + } + + public Resource singleRoot() throws DatabaseException { + int s = roots.size(); + if (s != 1) + throw new AssumptionException("No single imported root found, roots are: " + roots); + return roots.iterator().next(); + } + + public boolean hasMissingExternals() { + return tgResult != null ? tgResult.hasMissingExternals() : false; + } + +} diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/migration/MigrationStateImpl.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/migration/MigrationStateImpl.java index bf860e44d..2b19da586 100644 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/migration/MigrationStateImpl.java +++ b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/migration/MigrationStateImpl.java @@ -37,12 +37,14 @@ import org.simantics.db.layer0.adapter.impl.DefaultPasteImportAdvisor; import org.simantics.db.layer0.internal.SimanticsInternal; import org.simantics.db.layer0.util.Layer0Utils; import org.simantics.db.layer0.util.ModelTransferableGraphSourceRequest; +import org.simantics.db.layer0.util.TGTransferableGraphSource; import org.simantics.db.layer0.util.TransferableGraphConfiguration2; import org.simantics.db.request.Read; import org.simantics.db.service.ManagementSupport; import org.simantics.db.service.SerialisationSupport; import org.simantics.graph.db.IImportAdvisor; import org.simantics.graph.db.IImportAdvisor2; +import org.simantics.graph.db.ImportResult; import org.simantics.graph.db.StreamingTransferableGraphFileReader; import org.simantics.graph.db.TGStatusMonitor; import org.simantics.graph.db.TransferableGraphImporter; @@ -201,8 +203,8 @@ public class MigrationStateImpl implements MigrationState { }; // Make sure that the supplied advisor is redirected to temp advisor.redirect(indexRoot); - - TransferableGraphs.importGraph1WithMonitor(session, tg, advisor, new TGStatusMonitor() { + + ImportResult ir = TransferableGraphs.importGraph1(session, new TGTransferableGraphSource(tg), advisor, new TGStatusMonitor() { @Override public void status(int percentage) { monitor.subTask("Importing model from file (" + percentage + "%)"); @@ -212,7 +214,8 @@ public class MigrationStateImpl implements MigrationState { return monitor.isCanceled(); } }); - + + setProperty(MigrationStateKeys.IMPORT_RESULT, ir); setProperty(MigrationStateKeys.CURRENT_RESOURCE, indexRoot); setProperty(MigrationStateKeys.CURRENT_ROOT_RESOURCES, new ArrayList<>(advisor.getRoots())); setProperty(MigrationStateKeys.DATABASE_REVISION_AFTER_TG_IMPORT, session.getService(ManagementSupport.class).getHeadRevisionId()); @@ -258,7 +261,7 @@ public class MigrationStateImpl implements MigrationState { // Make sure that the supplied advisor is redirected to temp advisor.redirect(indexRoot); - TransferableGraphs.importGraph1(session, tgs, advisor, new TGStatusMonitor() { + ImportResult ir = TransferableGraphs.importGraph1(session, tgs, advisor, new TGStatusMonitor() { @Override public void status(int percentage) { monitor.subTask("Importing model from file (" + percentage + "%)"); @@ -269,6 +272,7 @@ public class MigrationStateImpl implements MigrationState { } }); + setProperty(MigrationStateKeys.IMPORT_RESULT, ir); setProperty(MigrationStateKeys.CURRENT_RESOURCE, indexRoot); setProperty(MigrationStateKeys.CURRENT_ROOT_RESOURCES, new ArrayList<>(advisor.getRoots())); setProperty(MigrationStateKeys.DATABASE_REVISION_AFTER_TG_IMPORT, session.getService(ManagementSupport.class).getHeadRevisionId()); diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/migration/MigrationStateKeys.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/migration/MigrationStateKeys.java index a1a7a82ae..9a5ac6fda 100644 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/migration/MigrationStateKeys.java +++ b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/migration/MigrationStateKeys.java @@ -15,6 +15,7 @@ import java.io.PrintWriter; import org.eclipse.core.runtime.IProgressMonitor; import org.simantics.db.Session; +import org.simantics.graph.db.ImportResult; public interface MigrationStateKeys { @@ -51,6 +52,12 @@ public interface MigrationStateKeys { public final static String TG_EXTENSIONS = "tgExtensions"; public final static String CURRENT_RESOURCE = "currentResource"; public final static String IMPORT_ADVISOR = "importAdvisor"; + + /** + * Used for storing the {@link ImportResult} resulting from the TG import. + */ + public final static String IMPORT_RESULT = "importResult"; + /** * All root resources of the imported material as * Collection<Resource>. diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/migration/MigrationUtils.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/migration/MigrationUtils.java index 7af76baf3..5f20036a5 100644 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/migration/MigrationUtils.java +++ b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/migration/MigrationUtils.java @@ -55,6 +55,7 @@ import org.simantics.db.layer0.util.Layer0Utils; import org.simantics.db.layer0.util.TGTransferableGraphSource; import org.simantics.db.service.XSupport; import org.simantics.graph.db.IImportAdvisor; +import org.simantics.graph.db.ImportResult; import org.simantics.graph.db.MissingDependencyException; import org.simantics.graph.db.TransferableGraphException; import org.simantics.graph.representation.Identity; @@ -164,7 +165,6 @@ public class MigrationUtils { } - @SuppressWarnings("deprecation") public static Collection getMigrationSteps(DataContainer header) throws DatabaseException { return SimanticsInternal.sync(new BinaryRead>(header.format, header.version) { @@ -308,11 +308,11 @@ public class MigrationUtils { return t != null ? t : defaultValue; } - public static Resource importSharedOntology(Session session, TransferableGraph1 tg, boolean published) throws DatabaseException { + public static MigratedImportResult importSharedOntology(Session session, TransferableGraph1 tg, boolean published) throws DatabaseException { return importSharedOntology(null, session, tg, published); } - public static Resource importSharedOntology(IProgressMonitor monitor, Session session, TransferableGraph1 tg, boolean published) throws DatabaseException { + public static MigratedImportResult importSharedOntology(IProgressMonitor monitor, Session session, TransferableGraph1 tg, boolean published) throws DatabaseException { if(monitor == null) monitor = new NullProgressMonitor(); @@ -344,7 +344,6 @@ public class MigrationUtils { Collection roots = TransferableGraphUtils.getRoots(tg); if(roots.size() == 1) { try { - TGTransferableGraphSource tgSource = new TGTransferableGraphSource(tg); SharedOntologyImportAdvisor advisor = new SharedOntologyImportAdvisor(published); @@ -355,8 +354,11 @@ public class MigrationUtils { state.setProperty(MigrationStateKeys.PROGRESS_MONITOR, monitor); state.setProperty(MigrationStateKeys.CURRENT_DATA_CONTAINER, new DataContainer("sharedLibrary", 1, new Variant(TransferableGraph1.BINDING, tg))); - return MigrationUtils.importMigrated(monitor, session, null, state, advisor, null); - + MigrationUtils.importMigrated(monitor, session, null, state, advisor, null); + + Collection resultRoots = state.getProperty(MigrationStateKeys.CURRENT_ROOT_RESOURCES); + ImportResult result = state.getProperty(MigrationStateKeys.IMPORT_RESULT); + return new MigratedImportResult(resultRoots, result); } catch (TransferableGraphException e) { throw new DatabaseException(e); } catch (MissingDependencyException e) { diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/ConsistsOfProcess.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/ConsistsOfProcess.java index 39c661e4b..d910f13ff 100644 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/ConsistsOfProcess.java +++ b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/ConsistsOfProcess.java @@ -1,48 +1,100 @@ package org.simantics.db.layer0.util; +import java.util.ArrayList; import java.util.Collection; +import java.util.List; import java.util.Set; import org.simantics.db.AsyncReadGraph; import org.simantics.db.ReadGraph; import org.simantics.db.Resource; +import org.simantics.db.ResourceMap; import org.simantics.db.common.request.ReadRequest; import org.simantics.db.common.utils.Logger; import org.simantics.db.exception.DatabaseException; +import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus; import org.simantics.db.procedure.AsyncContextMultiProcedure; -import org.simantics.db.service.CollectionSupport; +import org.simantics.db.procedure.Procedure; import org.simantics.db.service.DirectQuerySupport; import org.simantics.layer0.Layer0; class ConsistsOfProcess { - final Set result; - final AsyncContextMultiProcedure structure; + final List result; + final AsyncContextMultiProcedure structure; + final AsyncContextMultiProcedure names; - public static Set walk(ReadGraph graph, Collection resources, Set exclusions, boolean ignoreVirtual) throws DatabaseException { - ConsistsOfProcess process = new ConsistsOfProcess(graph, resources, exclusions, ignoreVirtual); + public static List walk(ReadGraph graph, ResourceMap status, Collection resources, Set exclusions, boolean ignoreVirtual) throws DatabaseException { + ConsistsOfProcess process = new ConsistsOfProcess(graph, status, resources, exclusions, ignoreVirtual); return process.result; } + + static class InternalEntry { + public InternalEntry parent; + public Resource resource; + public String name; + InternalEntry(InternalEntry parent, Resource resource, String name) { + this.parent = parent; + this.resource = resource; + this.name = name; + } + } - private ConsistsOfProcess(ReadGraph graph, final Collection resources, final Set exclusions, final boolean ignoreVirtual) throws DatabaseException { + private ConsistsOfProcess(ReadGraph graph, ResourceMap status, final Collection resources, final Set exclusions, final boolean ignoreVirtual) throws DatabaseException { final Layer0 L0 = Layer0.getInstance(graph); final DirectQuerySupport dqs = graph.getService(DirectQuerySupport.class); - CollectionSupport cs = graph.getService(CollectionSupport.class); - result = cs.createSet(); + result = new ArrayList(); + + names = dqs.compileForEachObject(graph, L0.HasName, new AsyncContextMultiProcedure() { + + @Override + public void execute(AsyncReadGraph graph, InternalEntry entry, Resource nameResource) { + + if(status != null) + status.put(nameResource, ExtentStatus.EXCLUDED); + + graph.forPossibleValue(nameResource, new Procedure() { + + @Override + public void execute(String result) { + entry.name = result; + } + + @Override + public void exception(Throwable t) { + Logger.defaultLogError(t); + } + + }); + } + + @Override + public void exception(AsyncReadGraph graph, Throwable throwable) { + Logger.defaultLogError(throwable); + } + + @Override + public void finished(AsyncReadGraph graph) { + } + + }); - structure = dqs.compileForEachObject(graph, L0.ConsistsOf, new AsyncContextMultiProcedure() { + structure = dqs.compileForEachObject(graph, L0.ConsistsOf, new AsyncContextMultiProcedure() { @Override - public void execute(AsyncReadGraph graph, Resource parent, Resource child) { + public void execute(AsyncReadGraph graph, InternalEntry parent, Resource child) { if(exclusions.contains(child)) return; - if(!ignoreVirtual || child.isPersistent()) - if(result.add(child)) { - dqs.forEachObjectCompiled(graph, child, child, structure); + if(!ignoreVirtual || child.isPersistent()) { + InternalEntry entry = new InternalEntry(parent, child, null); + if(result.add(entry)) { + dqs.forEachObjectCompiled(graph, child, entry, structure); + dqs.forEachObjectCompiled(graph, child, entry, names); } + } } @@ -61,8 +113,10 @@ class ConsistsOfProcess { @Override public void run(ReadGraph graph) throws DatabaseException { - for(Resource r : resources) - dqs.forEachObjectCompiled(graph, r, r, structure); + for(Resource r : resources) { + InternalEntry root = new InternalEntry(null, r, null); + dqs.forEachObjectCompiled(graph, r, root, structure); + } } }); diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/DomainProcessor3.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/DomainProcessor3.java index b55f0a3d2..bd67680db 100644 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/DomainProcessor3.java +++ b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/DomainProcessor3.java @@ -5,6 +5,7 @@ import java.io.InputStream; import java.util.Collection; import java.util.HashMap; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeSet; @@ -25,6 +26,7 @@ import org.simantics.db.common.utils.NameUtils; import org.simantics.db.exception.CancelTransactionException; import org.simantics.db.exception.DatabaseException; import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus; +import org.simantics.db.layer0.util.ConsistsOfProcess.InternalEntry; import org.simantics.db.layer0.util.ModelTransferableGraphSourceRequest.Expansion3; import org.simantics.db.service.CollectionSupport; import org.simantics.db.service.SerialisationSupport; @@ -502,7 +504,10 @@ public class DomainProcessor3 { this.datatypeBinding = Bindings.getBindingUnchecked(Datatype.class); this.datatypeSerializer = graph.getService(Databoard.class).getSerializerUnchecked(this.datatypeBinding); - for(Resource r : ConsistsOfProcess.walk(graph, fringe, exclusions, ignoreVirtual)) { + state.internalEntries = ConsistsOfProcess.walk(graph, status, fringe, exclusions, ignoreVirtual); + + for(InternalEntry entry : state.internalEntries) { + Resource r = entry.resource; if (status.put(r, ExtentStatus.INTERNAL) == null) { if(ModelTransferableGraphSourceRequest.LOG) { String URI = graph.getPossibleURI(r); diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/DomainProcessorState.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/DomainProcessorState.java index f1dc02242..3f2541018 100644 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/DomainProcessorState.java +++ b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/DomainProcessorState.java @@ -1,6 +1,7 @@ package org.simantics.db.layer0.util; import java.io.DataOutputStream; +import java.util.List; import java.util.TreeMap; import org.eclipse.core.runtime.SubMonitor; @@ -31,5 +32,6 @@ public class DomainProcessorState { public SubMonitor monitor; public TGValueModifier valueModifier; + public List internalEntries; } \ No newline at end of file diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/ModelTransferableGraphSource.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/ModelTransferableGraphSource.java index cbd3130ee..e7fa24124 100644 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/ModelTransferableGraphSource.java +++ b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/ModelTransferableGraphSource.java @@ -27,16 +27,20 @@ import org.simantics.db.exception.DatabaseException; import org.simantics.db.exception.RuntimeDatabaseException; import org.simantics.db.exception.ValidationException; import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus; +import org.simantics.db.layer0.util.ConsistsOfProcess.InternalEntry; import org.simantics.db.layer0.util.TransferableGraphConfiguration2.RootSpec; import org.simantics.db.service.SerialisationSupport; import org.simantics.graph.db.TransferableGraphSource; import org.simantics.graph.representation.External; import org.simantics.graph.representation.Identity; +import org.simantics.graph.representation.Internal; import org.simantics.graph.representation.Root; import org.simantics.graph.representation.Value; import org.simantics.layer0.Layer0; import gnu.trove.list.array.TIntArrayList; +import gnu.trove.map.TIntObjectMap; +import gnu.trove.map.hash.TIntObjectHashMap; import gnu.trove.procedure.TIntIntProcedure; public class ModelTransferableGraphSource implements TransferableGraphSource { @@ -51,7 +55,7 @@ public class ModelTransferableGraphSource implements TransferableGraphSource { private volatile boolean closed = false; TIntArrayList externalParents = new TIntArrayList(); - ArrayList externalNames = new ArrayList(); + ArrayList externalNames = new ArrayList<>(); TreeMap downloads = new TreeMap(); public ModelTransferableGraphSource(final ReadGraph graph, TransferableGraphConfiguration2 configuration, final DomainProcessorState state, File ... fs) throws DatabaseException { @@ -80,7 +84,7 @@ public class ModelTransferableGraphSource implements TransferableGraphSource { this.externalBase = state.id; - final Collection errors = new HashSet(); + final Collection errors = new HashSet<>(); // All resource considered as not internal by domain processor. Can also contain roots. int[] externals = state.externals.toArray(); @@ -106,7 +110,7 @@ public class ModelTransferableGraphSource implements TransferableGraphSource { }); if(!errors.isEmpty()) { - ArrayList sorted = new ArrayList(errors); + ArrayList sorted = new ArrayList<>(errors); Collections.sort(sorted); StringBuilder message = new StringBuilder(); message.append("Errors in exported model:\n"); @@ -388,13 +392,17 @@ public class ModelTransferableGraphSource implements TransferableGraphSource { // TODO: this should be Root with name "" procedure.execute(getRootIdentity(state, support, graph.getRootLibrary())); + TIntObjectMap internalMap = new TIntObjectHashMap<>(100, 0.5f, Integer.MIN_VALUE); + // Declare internal and external roots for(RootSpec r : configuration.roots) { Resource type = graph.getPossibleType(r.resource, L0.Entity); if(type == null) type = L0.Entity; - procedure.execute(new Identity( - state.ids.get(support.getTransientId(r.resource)), - new Root(r.name, graph.getURI(type)))); + int id = state.ids.get(support.getTransientId(r.resource)); + Root root = new Root(r.name, graph.getURI(type)); + Identity rootId = new Identity(id,root); + internalMap.put(id, rootId); + procedure.execute(rootId); } for(int i = 0; i < state.externals.size() ; i++) { @@ -402,8 +410,27 @@ public class ModelTransferableGraphSource implements TransferableGraphSource { String name = externalNames.get(i); procedure.execute(new Identity(externalBase + i, new External(parent,name))); } + + if(state.internalEntries != null) { + for(InternalEntry ie : state.internalEntries) { + if(ie.parent != null && ie.name != null) { + procedure.execute(resolveInternal(graph, support, ie, internalMap)); + } + } + } } + + private Identity resolveInternal(ReadGraph graph, SerialisationSupport ss, InternalEntry entry, TIntObjectMap internalMap) throws DatabaseException { + int id = state.ids.get(ss.getTransientId(entry.resource)); + Identity existing = internalMap.get(id); + if(existing != null) return existing; + Identity parent = resolveInternal(graph, ss, entry.parent, internalMap); + Identity result = new Identity(id, + new Internal(parent.resource, entry.name)); + internalMap.put(id, result); + return result; + } @Override public TreeMap getExtensions() { diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/Subgraphs.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/Subgraphs.java index 9b85adf8f..c7e697e62 100644 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/Subgraphs.java +++ b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/Subgraphs.java @@ -11,15 +11,6 @@ *******************************************************************************/ package org.simantics.db.layer0.util; -import gnu.trove.list.array.TIntArrayList; -import gnu.trove.map.hash.TIntIntHashMap; -import gnu.trove.map.hash.TLongObjectHashMap; -import gnu.trove.procedure.TIntProcedure; -import gnu.trove.procedure.TLongObjectProcedure; -import gnu.trove.set.TIntSet; -import gnu.trove.set.hash.THashSet; -import gnu.trove.set.hash.TIntHashSet; - import java.io.DataOutput; import java.io.DataOutputStream; import java.io.FileNotFoundException; @@ -30,6 +21,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; @@ -54,6 +46,7 @@ import org.simantics.db.common.utils.NameUtils; import org.simantics.db.exception.DatabaseException; import org.simantics.db.layer0.adapter.SubgraphAdvisor; import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus; +import org.simantics.db.layer0.util.ConsistsOfProcess.InternalEntry; import org.simantics.db.procedure.AsyncProcedure; import org.simantics.db.request.AsyncRead; import org.simantics.db.service.ClusteringSupport; @@ -70,6 +63,15 @@ import org.simantics.utils.datastructures.Pair; import org.simantics.utils.threads.logger.ITask; import org.simantics.utils.threads.logger.ThreadLogger; +import gnu.trove.list.array.TIntArrayList; +import gnu.trove.map.hash.TIntIntHashMap; +import gnu.trove.map.hash.TLongObjectHashMap; +import gnu.trove.procedure.TIntProcedure; +import gnu.trove.procedure.TLongObjectProcedure; +import gnu.trove.set.TIntSet; +import gnu.trove.set.hash.THashSet; +import gnu.trove.set.hash.TIntHashSet; + public class Subgraphs { public static String LOG_FILE = "export.log"; @@ -989,7 +991,9 @@ public class Subgraphs { * � All o are internal * � All stm are included */ - for(Resource r : ConsistsOfProcess.walk(graph, fringe, exclusions, true)) { + List entries = ConsistsOfProcess.walk(graph, null, fringe, exclusions, true); + for(InternalEntry entry : entries) { + Resource r = entry.resource; if (status.put(r, ExtentStatus.INTERNAL) == null) { String URI = graph.getPossibleURI(r); if(URI != null) log("URI INTERNAL " + URI); diff --git a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TGRepresentationUtils.java b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TGRepresentationUtils.java index 82c20514c..ffa399743 100644 --- a/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TGRepresentationUtils.java +++ b/bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/TGRepresentationUtils.java @@ -22,6 +22,7 @@ import org.simantics.db.Statement; import org.simantics.db.common.request.PossibleIndexRoot; import org.simantics.db.exception.DatabaseException; import org.simantics.db.layer0.adapter.impl.EntityInstances.QueryIndex; +import org.simantics.db.layer0.util.ConsistsOfProcess.InternalEntry; import org.simantics.db.layer0.util.DomainProcessor3.ExclusionDecision; import org.simantics.layer0.Layer0; import org.simantics.scl.runtime.function.Function1; @@ -62,8 +63,9 @@ public class TGRepresentationUtils { return new GUIDExclusionFunction(graph); // The root is OK - check everything beneath - for(Resource part : ConsistsOfProcess.walk(graph, Collections.singleton(r), Collections.emptySet(), true)) { - if(findByIdentifier(graph, targetRoot, part)) + List entries = ConsistsOfProcess.walk(graph, null, Collections.singleton(r), Collections.emptySet(), true); + for(InternalEntry entry : entries) { + if(findByIdentifier(graph, targetRoot, entry.resource)) return new GUIDExclusionFunction(graph); } } diff --git a/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/XSupportImpl.java b/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/XSupportImpl.java index 1e8d2731f..4ac08fc0b 100644 --- a/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/XSupportImpl.java +++ b/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/XSupportImpl.java @@ -16,6 +16,7 @@ import org.simantics.db.request.WriteTraits; import org.simantics.db.service.ClusterUID; import org.simantics.db.service.SerialisationSupport; import org.simantics.db.service.XSupport; +import org.simantics.utils.datastructures.Pair; public class XSupportImpl implements XSupport { final private boolean DEBUG = false; @@ -112,6 +113,12 @@ public class XSupportImpl implements XSupport { session.clusterSetsSupport.clear(); } } + + @Override + public Pair getServiceMode() { + return Pair.make((session.serviceMode & 1) == 1, (session.serviceMode & 2) == 2); + } + @Override public Resource convertDelayedResourceToResource(Resource resource) { return DelayedWriteGraph.convertDelayedResource(resource); diff --git a/bundles/org.simantics.db/src/org/simantics/db/service/XSupport.java b/bundles/org.simantics.db/src/org/simantics/db/service/XSupport.java index 73594cddb..8e08147c5 100644 --- a/bundles/org.simantics.db/src/org/simantics/db/service/XSupport.java +++ b/bundles/org.simantics.db/src/org/simantics/db/service/XSupport.java @@ -17,6 +17,7 @@ import org.simantics.db.WriteOnlyGraph; import org.simantics.db.exception.DatabaseException; import org.simantics.db.exception.ServiceException; import org.simantics.db.request.WriteTraits; +import org.simantics.utils.datastructures.Pair; /** * @author TUOKSK @@ -103,6 +104,7 @@ public interface XSupport { * @param createAsImmutable true to make all created clusters immutable */ void setServiceMode(boolean allowImmutableWrites, boolean createAsImmutable); + Pair getServiceMode(); /** * If resource is acquired from DelayedWriteGraph during delayed write request then diff --git a/bundles/org.simantics.graph.db/src/org/simantics/graph/db/CoreInitialization.java b/bundles/org.simantics.graph.db/src/org/simantics/graph/db/CoreInitialization.java index 3e15e4b31..096f0a00f 100644 --- a/bundles/org.simantics.graph.db/src/org/simantics/graph/db/CoreInitialization.java +++ b/bundles/org.simantics.graph.db/src/org/simantics/graph/db/CoreInitialization.java @@ -38,6 +38,7 @@ public class CoreInitialization { addLayer0Builtin("ConsistsOf"); addLayer0Builtin("PartOf"); + addLayer0Builtin("ExternalEntity"); addLayer0Builtin("String"); addLayer0Builtin("Library"); @@ -49,7 +50,6 @@ public class CoreInitialization { addLayer0Builtin("HasPredicateInverse"); addLayer0Builtin("HasObject"); - addBuiltin("http://Projects"); addBuiltin("http:/"); } diff --git a/bundles/org.simantics.graph.db/src/org/simantics/graph/db/ImportResult.java b/bundles/org.simantics.graph.db/src/org/simantics/graph/db/ImportResult.java new file mode 100644 index 000000000..1d986da09 --- /dev/null +++ b/bundles/org.simantics.graph.db/src/org/simantics/graph/db/ImportResult.java @@ -0,0 +1,36 @@ +/******************************************************************************* + * Copyright (c) 2017 Association for Decentralized Information Management + * in Industry THTH ry. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Semamtum Oy - initial API and implementation + *******************************************************************************/ +package org.simantics.graph.db; + +import java.util.Set; + +/** + * @author Tuukka Lehtonen + * @since 1.28.0 + */ +public class ImportResult { + + /** + * This lists externals that the import was missing and has created in the + * database as L0.ExternalEntity place-holders for now. + */ + public final Set missingExternals; + + public ImportResult(Set missingExternals) { + this.missingExternals = missingExternals; + } + + public boolean hasMissingExternals() { + return !missingExternals.isEmpty(); + } + +} diff --git a/bundles/org.simantics.graph.db/src/org/simantics/graph/db/StreamingTransferableGraphImportProcess.java b/bundles/org.simantics.graph.db/src/org/simantics/graph/db/StreamingTransferableGraphImportProcess.java index 48cebf342..3599eefc0 100644 --- a/bundles/org.simantics.graph.db/src/org/simantics/graph/db/StreamingTransferableGraphImportProcess.java +++ b/bundles/org.simantics.graph.db/src/org/simantics/graph/db/StreamingTransferableGraphImportProcess.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2012, 2016 Association for Decentralized Information Management + * Copyright (c) 2012, 2017 Association for Decentralized Information Management * in Industry THTH ry. * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Public License v1.0 @@ -8,17 +8,16 @@ * * Contributors: * VTT Technical Research Centre of Finland - initial API and implementation - * Semantum Oy + * Semantum Oy - e.g. #7016 *******************************************************************************/ package org.simantics.graph.db; import java.io.DataInput; -import java.io.DataOutput; -import java.io.DataOutputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -30,6 +29,7 @@ import org.simantics.databoard.binding.Binding; import org.simantics.databoard.binding.mutable.Variant; import org.simantics.databoard.serialization.Serializer; import org.simantics.databoard.type.Datatype; +import org.simantics.databoard.util.URIStringUtils; import org.simantics.db.ReadGraph; import org.simantics.db.Resource; import org.simantics.db.Session; @@ -45,6 +45,7 @@ import org.simantics.db.service.ClusterBuilder2; import org.simantics.db.service.ClusterBuilderFactory; import org.simantics.db.service.ClusteringSupport; import org.simantics.db.service.SerialisationSupport; +import org.simantics.db.service.XSupport; import org.simantics.graph.db.TransferableGraphSource.TransferableGraphSourceProcedure; import org.simantics.graph.db.TransferableGraphSource.TransferableGraphSourceValueProcedure; import org.simantics.graph.representation.Extensions; @@ -57,37 +58,15 @@ import org.simantics.graph.representation.Root; import org.simantics.graph.representation.TransferableGraphUtils; import org.simantics.graph.utils.TGResourceUtil; import org.simantics.graph.utils.TGResourceUtil.LongAdapter; +import org.simantics.utils.datastructures.Pair; +import org.slf4j.LoggerFactory; + +import gnu.trove.map.TIntObjectMap; +import gnu.trove.map.hash.TIntObjectHashMap; public class StreamingTransferableGraphImportProcess implements TransferableGraphImporter { - - public static String LOG_FILE = "transferableGraphs.log"; - final static private boolean LOG = false; - - static DataOutput log; - static { - - if (LOG) { - - try { - FileOutputStream stream = new FileOutputStream(LOG_FILE); - log = new DataOutputStream(stream); - } catch (FileNotFoundException e) { - e.printStackTrace(); - } - } - - } - - private static void log(String line) { - if (LOG) { - try { - log.writeUTF(line + "\n"); - } catch (IOException e) { - e.printStackTrace(); - } - } - } + private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(StreamingTransferableGraphImportProcess.class); Resource indexRoot; TransferableGraphSource tg; @@ -95,32 +74,36 @@ public class StreamingTransferableGraphImportProcess implements TransferableGrap IImportAdvisor2 advisor; ClusterBuilder2 builder; final TGResourceUtil resourceUtil = new TGResourceUtil(); - + int[] handles; - - Set missingExternals = new HashSet(); - + + Map allMissingExternals = new HashMap<>(); + Set missingExternals = new HashSet<>(); + Map resolvedParents = new HashMap<>(); + TIntObjectHashMap existingInternalMap = new TIntObjectHashMap<>(); + int resourceCount; Identity[] identities; TreeMap extensions; - + // Builtins Resource RootLibrary; Resource String; + Resource ExternalEntity; Resource Library; - + Resource InstanceOf; Resource ConsistsOf; Resource PartOf; Resource HasName; Resource NameOf; - + public StreamingTransferableGraphImportProcess(Session session, VirtualGraph vg, TransferableGraphSource tg, IImportAdvisor2 advisor) { this.tg = tg; this.vg = vg; this.advisor = advisor; } - + public void readIdentities(ReadGraph g) throws Exception { extensions = tg.getExtensions(); resourceCount = tg.getResourceCount(); @@ -135,7 +118,7 @@ public class StreamingTransferableGraphImportProcess implements TransferableGrap } }); } - + public void findBuiltins(WriteOnlyGraph g) throws DatabaseException { RootLibrary = g.getBuiltin("http:/"); String = g.getBuiltin(CoreInitialization.LAYER0 + "String"); @@ -145,8 +128,9 @@ public class StreamingTransferableGraphImportProcess implements TransferableGrap PartOf = g.getBuiltin(CoreInitialization.LAYER0 + "PartOf"); HasName = g.getBuiltin(CoreInitialization.LAYER0 + "HasName"); NameOf = g.getBuiltin(CoreInitialization.LAYER0 + "NameOf"); + ExternalEntity = g.getBuiltin(CoreInitialization.LAYER0 + "ExternalEntity"); } - + public void findBuiltins(ReadGraph g) throws DatabaseException { RootLibrary = g.getBuiltin("http:/"); String = g.getBuiltin(CoreInitialization.LAYER0 + "String"); @@ -156,59 +140,18 @@ public class StreamingTransferableGraphImportProcess implements TransferableGrap PartOf = g.getBuiltin(CoreInitialization.LAYER0 + "PartOf"); HasName = g.getBuiltin(CoreInitialization.LAYER0 + "HasName"); NameOf = g.getBuiltin(CoreInitialization.LAYER0 + "NameOf"); + ExternalEntity = g.getBuiltin(CoreInitialization.LAYER0 + "ExternalEntity"); } -// /* Preparation that is used when the core is empty. -// */ -// void initialPrepare(WriteOnlyGraph graph) throws DatabaseException { -// findBuiltins(graph); -// -// resources = new Resource[tg.resourceCount]; -// -// int Root = -1; -// int SimanticsDomain = -1; -// int Layer0 = -1; -// -// for(Identity identity : tg.identities) { -// if(identity.definition instanceof Internal) { -// Internal def = (Internal)identity.definition; -// Resource res = null; -// if(def.parent == Layer0) { -// try { -// res = graph.getBuiltin(CoreInitialization.LAYER0 + def.name); -// } catch(ResourceNotFoundException e) { -// } -// } -// else if(def.parent == SimanticsDomain) { -// if(def.name.equals("Layer0-1.0")) -// Layer0 = identity.resource; -// } -// else if(def.parent == Root) { -// if(def.name.equals("www.simantics.org")) -// SimanticsDomain = identity.resource; -// } -// -// if(res == null) -// res = createChild(graph, resources[def.parent], def.name); -// else -// createChild(graph, res, resources[def.parent], def.name); -// resources[identity.resource] = res; -// } -// else if(identity.definition instanceof Root) { -// Root = identity.resource; -// resources[identity.resource] = RootLibrary; -// } -// } -// } - - void addMissing(String external) { - Set removals = new HashSet(); + void addMissing(int handleIndex, String external) { + allMissingExternals.put(external, handleIndex); + Set removals = new HashSet<>(); for(String ext : missingExternals) if(ext.startsWith(external)) return; for(String ext : missingExternals) if(external.startsWith(ext)) removals.add(ext); missingExternals.removeAll(removals); missingExternals.add(external); } - + void prepare(ReadGraph graph) throws Exception { Resource target = advisor.getTarget(); @@ -225,7 +168,8 @@ public class StreamingTransferableGraphImportProcess implements TransferableGrap ClusterBuilderFactory factory = graph.getService(ClusterBuilderFactory.class); ClusterBuilder2 builder = factory.create(vg, false); - this.handles = new int[resourceCount]; + this.handles = new int[resourceCount]; + TIntObjectMap identityMap = TransferableGraphUtils.mapIdentities(identities); for(Identity identity : identities) { IdentityDefinition definition = identity.definition; @@ -243,23 +187,28 @@ public class StreamingTransferableGraphImportProcess implements TransferableGrap Resource parent = handle != 0 ? builder.resource(handle) : null; // TODO: escape should be removed when names become well-behaving if(parent != null) { + resolvedParents.put(graph.getURI(parent), parent); Map childMap = graph .syncRequest(new UnescapedChildMapOfResource(parent), - new TransientCacheAsyncListener>()); + TransientCacheAsyncListener.instance()); Resource child = childMap.get(def.name); if(child == null) { - addMissing(graph.getURI(parent) + "/" + def.name); + addMissing(identity.resource, graph.getURI(parent) + "/" + def.name); } else { handles[identity.resource] = builder.handle(child); } } else { - addMissing(TransferableGraphUtils.getURI(resourceCount, identities, def.parent) + "/" + def.name); + addMissing(identity.resource, TransferableGraphUtils.getURI(resourceCount, identityMap, def.parent) + "/" + def.name); } } } } else if(definition instanceof Internal) { - // Do not do anything for now + String uri = TransferableGraphUtils.getURI(resourceCount, identityMap, identity.resource); + Resource existing = graph.getPossibleResource(uri); + if(existing != null) { + existingInternalMap.put(identity.resource, existing); + } } else if(definition instanceof Root) { Root root = (Root)definition; @@ -279,7 +228,7 @@ public class StreamingTransferableGraphImportProcess implements TransferableGrap } } - if(!missingExternals.isEmpty()) throw new MissingDependencyException(this); + //if(!missingExternals.isEmpty()) throw new MissingDependencyException(this); } @@ -292,7 +241,7 @@ public class StreamingTransferableGraphImportProcess implements TransferableGrap graph.claim(child, HasName, NameOf, nameResource); return child; } - + int[] getClustering() { Variant v = extensions.get(Extensions.CLUSTERING); if(v == null) return null; @@ -318,7 +267,7 @@ public class StreamingTransferableGraphImportProcess implements TransferableGrap boolean needTranslation(Datatype type) { return resourceUtil.mayHaveResource(type); } - + void findClusterSet(WriteOnlyGraph graph, Resource rootLibrary, int[] clustering, int[] clusterSets, long[] clusters, int id) throws DatabaseException { ClusteringSupport support = graph.getService(ClusteringSupport.class); if(id == Extensions.ROOT_LIBRARY_CLUSTER_SET || id == Extensions.INDEX_ROOT_CLUSTER_SET) return; @@ -352,11 +301,47 @@ public class StreamingTransferableGraphImportProcess implements TransferableGrap } } } - + + void createMissing(final WriteOnlyGraph graph) throws Exception { + + if(allMissingExternals.isEmpty()) return; + + XSupport xs = graph.getService(XSupport.class); + Pair serviceMode = xs.getServiceMode(); + xs.setServiceMode(true, false); + try { + ArrayList missing = new ArrayList<>(allMissingExternals.keySet()); + Collections.sort(missing); + for(String uri : missing) { + String[] parts = URIStringUtils.splitURI(uri); + + Resource parent = resolvedParents.get(parts[0]); + // TODO: proper exception message + if(parent == null) throw new IllegalStateException("!!"); + + Resource childResource = graph.newResource(); + graph.claim(childResource, InstanceOf, null, ExternalEntity); + + Resource nameResource = graph.newResource(); + graph.claim(nameResource, InstanceOf, null, String); + graph.claimValue(nameResource, parts[1], WriteBindings.STRING); + graph.claim(childResource, HasName, NameOf, nameResource); + + graph.claim(parent, ConsistsOf, PartOf, childResource); + + resolvedParents.put(uri, childResource); + + handles[allMissingExternals.get(uri)] = builder.handle(childResource); + } + } finally { + xs.setServiceMode(serviceMode.first, serviceMode.second); + } + } + void write(final WriteOnlyGraph graph) throws Exception { final SerialisationSupport ss = graph.getService(SerialisationSupport.class); - + ClusterBuilderFactory factory = graph.getService(ClusterBuilderFactory.class); if(advisor instanceof IImportAdvisor2) { boolean allowImmutable = ((IImportAdvisor2)advisor).allowImmutableModifications(); @@ -365,6 +350,8 @@ public class StreamingTransferableGraphImportProcess implements TransferableGrap builder = factory.create(vg, false); } + createMissing(graph); + final int[] handles = this.handles; int[] clustering = getClustering(); @@ -439,10 +426,17 @@ public class StreamingTransferableGraphImportProcess implements TransferableGrap } else if(definition instanceof Internal) { Internal def = (Internal)definition; - if(handles[identity.resource] != 0) - handles[identity.resource] = builder.handle(advisor.createChild(graph, this, builder.resource(handles[def.parent]), builder.resource(handles[identity.resource]), def.name)); - else - handles[identity.resource] = builder.handle(advisor.createChild(graph, this, builder.resource(handles[def.parent]), null, def.name)); + + Resource external = existingInternalMap.get(identity.resource); + if(external != null) { + handles[identity.resource] = builder.handle(external); + } else { + if(handles[identity.resource] != 0) + handles[identity.resource] = builder.handle(advisor.createChild(graph, this, builder.resource(handles[def.parent]), builder.resource(handles[identity.resource]), def.name)); + else + handles[identity.resource] = builder.handle(advisor.createChild(graph, this, builder.resource(handles[def.parent]), null, def.name)); + } + } else if(definition instanceof Root) { @@ -492,7 +486,7 @@ public class StreamingTransferableGraphImportProcess implements TransferableGrap }); tg.getValueCount(); - + class ValueProcedure extends InputStream implements TransferableGraphSourceValueProcedure { private TGResourceUtil util = new TGResourceUtil(); @@ -541,7 +535,7 @@ public class StreamingTransferableGraphImportProcess implements TransferableGrap try { builder.appendValue(value); } catch (DatabaseException e) { - e.printStackTrace(); + LOGGER.error("Failed to write value into database", e); } return value; } @@ -558,8 +552,12 @@ public class StreamingTransferableGraphImportProcess implements TransferableGrap tg.forValues2(null, new ValueProcedure()); + for(Resource r : existingInternalMap.valueCollection()) { + graph.deny(r, InstanceOf, null, ExternalEntity, null); + } + } - + @Override public long[] getResourceIds(SerialisationSupport serializer) throws DatabaseException { final int count = handles.length; diff --git a/bundles/org.simantics.graph.db/src/org/simantics/graph/db/TransferableGraphs.java b/bundles/org.simantics.graph.db/src/org/simantics/graph/db/TransferableGraphs.java index bdfd3de6e..4d5cc7e3c 100644 --- a/bundles/org.simantics.graph.db/src/org/simantics/graph/db/TransferableGraphs.java +++ b/bundles/org.simantics.graph.db/src/org/simantics/graph/db/TransferableGraphs.java @@ -173,15 +173,15 @@ public class TransferableGraphs { }); } - public static void importGraph1(Session session, final TransferableGraphSource tg, IImportAdvisor advisor) throws Exception { - importGraph1(session, tg, advisor, null); + public static ImportResult importGraph1(Session session, final TransferableGraphSource tg, IImportAdvisor advisor) throws Exception { + return importGraph1(session, tg, advisor, null); } - public static void importGraph1(Session session, final TransferableGraphSource tg, IImportAdvisor advisor, TGStatusMonitor monitor) throws DatabaseException { - importGraph1(session, null, tg, advisor, monitor); + public static ImportResult importGraph1(Session session, final TransferableGraphSource tg, IImportAdvisor advisor, TGStatusMonitor monitor) throws DatabaseException { + return importGraph1(session, null, tg, advisor, monitor); } - public static void importGraph1(Session session, VirtualGraph vg, final TransferableGraphSource tg, IImportAdvisor advisor_, TGStatusMonitor monitor) throws DatabaseException { + public static ImportResult importGraph1(Session session, VirtualGraph vg, final TransferableGraphSource tg, IImportAdvisor advisor_, TGStatusMonitor monitor) throws DatabaseException { final IImportAdvisor2 advisor = (advisor_ instanceof IImportAdvisor2) ? ((IImportAdvisor2)advisor_) : new WrapperAdvisor(advisor_); @@ -210,6 +210,8 @@ public class TransferableGraphs { } } }); + + return new ImportResult(process.missingExternals); } public static void importGraph1WithMonitor(Session session, final TransferableGraph1 tg, IImportAdvisor advisor_, TGStatusMonitor monitor) throws DatabaseException { diff --git a/bundles/org.simantics.graph/src/org/simantics/graph/refactoring/GraphRefactoringUtils.java b/bundles/org.simantics.graph/src/org/simantics/graph/refactoring/GraphRefactoringUtils.java index 55b9cbe2e..9882048b7 100644 --- a/bundles/org.simantics.graph/src/org/simantics/graph/refactoring/GraphRefactoringUtils.java +++ b/bundles/org.simantics.graph/src/org/simantics/graph/refactoring/GraphRefactoringUtils.java @@ -1,7 +1,9 @@ package org.simantics.graph.refactoring; import java.util.ArrayList; +import java.util.Arrays; +import org.simantics.databoard.util.URIStringUtils; import org.simantics.graph.query.Path; import org.simantics.graph.query.PathChild; import org.simantics.graph.query.TransferableGraphConversion; @@ -17,9 +19,11 @@ import org.simantics.graph.representation.TransferableGraph1; import org.simantics.graph.representation.TransferableGraphUtils; import org.simantics.graph.representation.old.OldTransferableGraph1; import org.simantics.graph.representation.old.OldValue1; +import org.simantics.graph.store.GraphStore; import org.simantics.graph.store.IdentityStore; import gnu.trove.list.array.TIntArrayList; +import gnu.trove.map.hash.TIntIntHashMap; import gnu.trove.set.hash.TIntHashSet; public class GraphRefactoringUtils { @@ -89,6 +93,20 @@ public class GraphRefactoringUtils { } + private static Identity recursePath(TransferableGraph1 tg, String path) { + + Identity extId = TransferableGraphUtils.findExternal(tg, path); + if(extId != null) return extId; + if("http://".equals(path)) return TransferableGraphUtils.findRootWithName(tg, ""); + String[] parts = URIStringUtils.splitURI(path); + Identity parentId = recursePath(tg, parts[0]); + tg.identities = Arrays.copyOf(tg.identities, tg.identities.length+1); + Identity childIdentity = new Identity(tg.resourceCount++, new External(parentId.resource, parts[1])); + tg.identities[tg.identities.length-1] = childIdentity; + return childIdentity; + + } + public static void fixOntologyRoot(TransferableGraph1 tg, boolean tryToFix) { Identity[] ids = tg.identities; @@ -97,37 +115,36 @@ public class GraphRefactoringUtils { if(id.definition instanceof Root) { Root ext = (Root)id.definition; if(ext.name.startsWith("http://")) { - String rootName = ext.name.substring(ext.name.lastIndexOf("/")+1); - String path = ext.name.substring(0, ext.name.lastIndexOf("/")); - Identity pathId = TransferableGraphUtils.findExternal(tg, path); - System.err.println("GraphRefactoringUtils.rootName=" + rootName); - System.err.println("GraphRefactoringUtils.path2=" + path); - if(pathId == null) { - if(!tryToFix) return; - IdentityStore idStore = TransferableGraphConversion.extractIdentities(tg); - idStore.createPathToId(UriUtils.uriToPath(path)); - tg.resourceCount = idStore.getResourceCount(); - tg.identities = idStore.toArray(); - fixOntologyRoot(tg, false); - return; - } else { - id.definition = new Internal(pathId.resource, rootName); - TIntArrayList stms = new TIntArrayList(tg.statements); - Identity consistsOf = TransferableGraphUtils.findExternal(tg, "http://www.simantics.org/Layer0-1.1/ConsistsOf"); - Identity partOf = TransferableGraphUtils.findExternal(tg, "http://www.simantics.org/Layer0-1.1/PartOf"); - stms.add(id.resource); - stms.add(partOf.resource); - stms.add(consistsOf.resource); - stms.add(pathId.resource); - tg.statements = stms.toArray(); - return; - } + + String[] parts = URIStringUtils.splitURI(ext.name); + Identity path = recursePath(tg, parts[0]); + id.definition = new Internal(path.resource, parts[1]); + + GraphStore store = TransferableGraphConversion.convert(tg); + int rootId = store.identities.createPathToId(UriUtils.uriToPath(ext.name)); + propagateNewMarks(store.identities, rootId); + + TransferableGraph1 tgNew = TransferableGraphConversion.convert(store); + + tg.resourceCount = tgNew.resourceCount; + tg.identities = tgNew.identities; + tg.values = tgNew.values; + tg.statements = tgNew.statements; + + return; } } } } + + private static void propagateNewMarks(IdentityStore identities, int resource) { + if(identities.markNew(resource)) { + for(int child : identities.getChildren(resource)) + propagateNewMarks(identities, child); + } + } public static void unfixIncorrectRoot(Identity[] ids) { for(int i=0;i infos = new TIntObjectHashMap<>(); + + ResourceInfo recurseURI(TransferableGraph1 graph, Identity parent, String parentName) { + String name = parentName + ".\"" + getName(parent) + "\""; + ResourceInfo info = new ResourceInfo(true, name, parent.resource); + infos.put(parent.resource, info); + for(Identity child : getChildren(graph, parent)) { + recurseURI(graph, child, name); + } + return info; + } + + void discoverBlank(TransferableGraph1 graph, int resource, TIntArrayList todo) { + TIntArrayList statements = getStatements(graph, resource); + for(int i=0;i " + predicateURI + " " + existing.name); + } else { + existing.owner = -1; + //System.err.println("Multiple owners " + info.name + " => " + predicateURI + " " + existing.name); + } + } + info.statements = statements; + } + + void fixInstanceOf(TransferableGraph1 graph, ResourceInfo info) { + Identity id = getIdentity(graph, info.resource); + if(id == null) return; + if(id.definition instanceof Internal) { + Identity instanceOf = findExternal(graph, "http://www.simantics.org/Layer0-1.1/InstanceOf"); + Identity library = findExternal(graph, "http://www.simantics.org/Layer0-1.1/Library"); + info.statements.add(instanceOf.resource); + info.statements.add(library.resource); + } + } + + public static String getExternalURI(TransferableGraph1 tg, External ext) { + String name = ext.name; + if(name.contains(" ")) name = name.replace(" ", "_").replaceAll("@", "_");//name = "\"" + name + "\""; + int parentId = ext.parent; + if(parentId == 0) return ext.name; + else { + Identity id = getIdentity(tg, parentId); + if(id.definition instanceof External) { + return getExternalURI(tg, (External)id.definition) + "/" + name; + } else if(id.definition instanceof Root) { + Root root = (Root)id.definition; + return "http:/" + root.name + "/" + name; + } else { + return null; + } + } + } + + public static String getExternalURI(TransferableGraph1 tg, int resource) { + Identity id = getIdentity(tg, resource); + if(id == null) return null; + if(id.definition instanceof External) { + External ext = (External)id.definition; + return getExternalURI(tg, ext); + } + return null; + } + + String rewritePredicateURI(TransferableGraph1 graph, int predicate) { + String uri = getExternalURI(graph, predicate); + if(uri == null) return null; + uri = uri.replace("http://www.simantics.org/Modeling-1.2", "MOD"); + uri = uri.replace("http://www.simantics.org/Layer0-1.1", "L0"); + uri = uri.replace("http://www.simantics.org/Layer0X-1.1", "L0X"); + uri = uri.replace("http://www.simantics.org/Diagram-2.2", "DIA"); + uri = uri.replace("http://www.simantics.org/Structural-1.2", "STR"); + uri = uri.replace("http://www.simantics.org/Documentation-1.2", "DOCU"); + uri = uri.replace("http://www.simantics.org/Document-1.2", "DOC"); + uri = uri.replace("http://www.simantics.org/G2D-1.1", "G2D"); + uri = uri.replace("http://www.simantics.org/Image2-1.2", "IMAGE2"); + uri = uri.replace("http://www.simantics.org/SelectionView-1.2", "SEL"); + uri = uri.replace("http://www.simantics.org/GraphFile-0.1", "GRAPHFILE"); + uri = uri.replace("http://www.semantum.fi/Simupedia-1.0", "SIMUPEDIA"); + uri = uri.replace("http://www.semantum.fi/SimupediaWorkbench-1.0", "SIMUPEDIA_WB"); + uri = uri.replace("http://www.apros.fi/OperationUI-6.6", "APROS_OPER"); + uri = uri.replace("http://semantum.fi/SimupediaStandardLibrary@1.3-trunk", "SIMUPEDIA_STD"); + uri = uri.replace("/", "."); + return uri; + } + + void printBlank(TransferableGraph1 graph, String predicateURI2, ResourceInfo info) { + + if(info.hasURI) return; + output.append(" " + predicateURI2 + " " + info.name + "\n"); + + Value value = findValue(graph, info.resource); + if(value != null) { + + } + +// for(int i=0;i\n"); + } else { + output.append(info.name + "\n"); + } + if(info.newResource) + output.append(" @L0.new\n"); + TLongHashSet processed = new TLongHashSet(); + for(int i=0;i 0) { + ResourceInfo ownerInfo = infos.get(info.owner); + ownerInfo.owned.add(info.ownerPredicate); + ownerInfo.owned.add(info.resource); + } else if (info.owner == 0) { + //System.err.println("faf1"); + } else if (info.owner == -1) { + //System.err.println("faf2"); + } + + TreeMap order = new TreeMap<>(); + for(ResourceInfo info : infos.valueCollection()) + order.put(info.name, info); + + this.output.append("MOD = \n"); + this.output.append("L0 = \n"); + this.output.append("L0X = \n"); + this.output.append("DIA = \n"); + this.output.append("STR = \n"); + this.output.append("DOCU = \n"); + this.output.append("DOC = \n"); + this.output.append("G2D = \n"); + this.output.append("SEL = \n"); + this.output.append("IMAGE2 = \n"); + this.output.append("GRAPHFILE = \n"); + this.output.append("APROS_OPER = \n"); + this.output.append("SIMUPEDIA = \n"); + this.output.append("SIMUPEDIA_WB = \n"); + this.output.append("SIMUPEDIA_STD = \n"); + +// uri = uri.replace("http://semantum.fi/SimupediaStandardLibrary@1.3-trunk/", "SIMUPEDIA_STD."); + + + for(ResourceInfo info : order.values()) + printURI(graph, info); + + Files.write(output, this.output.toString().getBytes()); + + } + } + + public static void main(String[] args) throws Exception { + if (args.length < 1) { + System.out.println("Required arguments: []"); + } else if (args.length < 2) { + Path input = Paths.get(args[0]); + Path output = input.getParent().resolve(input.getName(input.getNameCount()-1) + ".fixed"); + new PrettyPrintTG().prettyPrint(input, output); + } else { + new PrettyPrintTG().prettyPrint(Paths.get(args[0]), Paths.get(args[1])); + } + } + +} diff --git a/bundles/org.simantics.graph/src/org/simantics/graph/representation/TransferableGraphUtils.java b/bundles/org.simantics.graph/src/org/simantics/graph/representation/TransferableGraphUtils.java index 82ac178fb..4646116dc 100644 --- a/bundles/org.simantics.graph/src/org/simantics/graph/representation/TransferableGraphUtils.java +++ b/bundles/org.simantics.graph/src/org/simantics/graph/representation/TransferableGraphUtils.java @@ -4,6 +4,14 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.Map; +import java.util.TreeMap; + +import org.simantics.databoard.Bindings; +import org.simantics.databoard.adapter.AdaptException; + +import gnu.trove.list.array.TIntArrayList; +import gnu.trove.map.TIntObjectMap; +import gnu.trove.map.hash.TIntObjectHashMap; public class TransferableGraphUtils { @@ -67,36 +75,89 @@ public class TransferableGraphUtils { } + public static Identity getIdentity(TransferableGraph1 tg, int resource) { + for(Identity id : tg.identities) { + if(id.resource == resource) return id; + } + return null; + } + + public static TIntArrayList getStatements(TransferableGraph1 tg, int resource) { + TIntArrayList result = new TIntArrayList(); + for(int i=0;i getChildren(TransferableGraph1 tg, Identity parent) { - ArrayList result = new ArrayList(); - System.err.println("children for " + parent.resource); + TreeMap result = new TreeMap<>(); for(Identity id : tg.identities) { if(id.definition instanceof Internal) { Internal internal = (Internal)id.definition; - System.err.println("internal with parent " + internal.parent); - if(internal.parent == parent.resource) result.add(id); + if(internal.parent == parent.resource) result.put(internal.name, id); + } + } + Identity consistsOf = findExternal(tg, "http://www.simantics.org/Layer0-1.1/ConsistsOf"); + Identity hasName = findExternal(tg, "http://www.simantics.org/Layer0-1.1/HasName"); + for(int i=0;i getNames(TransferableGraph1 tg, Collection ids) { Map result = new HashMap(); @@ -159,11 +220,11 @@ public class TransferableGraphUtils { else return getURI(resourceCount, identities, def.parent) + "/" + def.name; } else if(definition instanceof Root) { Root def = (Root)definition; + if(def.name.isEmpty()) return "http:/"; return def.name; } else if (definition instanceof Internal) { Internal def = (Internal)definition; - System.err.println("External URI error: parent was internal '" + def.name + "'"); - return ""; + return getURI(resourceCount, identities, def.parent) + "/" + def.name; } else { return ""; } @@ -171,5 +232,39 @@ public class TransferableGraphUtils { } return ":"; } - + + public static TIntObjectMap mapIdentities(TransferableGraph1 tg) { + return mapIdentities(tg.identities); + } + + public static TIntObjectMap mapIdentities(Identity[] identities) { + // Integer.MIN_VALUE cannot be the value of Identity.resource + TIntObjectMap map = new TIntObjectHashMap<>(identities.length, 0.5f, Integer.MIN_VALUE); + for (Identity id : identities) + map.put(id.resource, id); + return map; + } + + public static String getURI(int resourceCount, TIntObjectMap identities, int id) { + Identity identity = identities.get(id); + if(identity != null) { + IdentityDefinition definition = identity.definition; + if(definition instanceof External) { + External def = (External)definition; + if(def.parent == -1) return "http:/"; + else return getURI(resourceCount, identities, def.parent) + "/" + def.name; + } else if(definition instanceof Root) { + Root def = (Root)definition; + if(def.name.isEmpty()) return "http:/"; + return def.name; + } else if (definition instanceof Internal) { + Internal def = (Internal)definition; + return getURI(resourceCount, identities, def.parent) + "/" + def.name; + } else { + return ""; + } + } + return ":"; + } + } diff --git a/bundles/org.simantics.layer0/graph/Layer0.pgraph b/bundles/org.simantics.layer0/graph/Layer0.pgraph index 93d348f05..482f606fc 100644 --- a/bundles/org.simantics.layer0/graph/Layer0.pgraph +++ b/bundles/org.simantics.layer0/graph/Layer0.pgraph @@ -124,4 +124,6 @@ L0.ResourceIdArray -- L0.PGraph.definition --> L0.String handler1 = new FormatHandler() { + FormatHandler handler1 = new FormatHandler() { @Override public Binding getBinding() { return TransferableGraph1.BINDING; } @Override - public Object process(DataContainer container) throws Exception { + public MigratedImportResult process(DataContainer container) throws Exception { mon.worked(100); mon.setTaskName("Importing shared library into database"); Variant draftStatus = container.metadata.get(DraftStatusBean.EXTENSION_KEY); TransferableGraph1 tg = (TransferableGraph1) container.content.getValue(); - MigrationUtils.importSharedOntology(mon.newChild(850, SubMonitor.SUPPRESS_NONE), session, tg, draftStatus == null); - return null; + return MigrationUtils.importSharedOntology(mon.newChild(850, SubMonitor.SUPPRESS_NONE), session, tg, draftStatus == null); } }; - Map> handlers = new HashMap<>(); + Map> handlers = new HashMap<>(); handlers.put(Constants.SHARED_LIBRARY_FORMAT_V1, handler1); - DataContainers.readFile(modelFile, handlers); + MigratedImportResult result = DataContainers.readFile(modelFile, handlers); mon.setTaskName("Postprocessing"); mon.subTask(""); mon.newChild(50).done(); + + return result; } } diff --git a/bundles/org.simantics.utils.ui/src/org/simantics/utils/ui/dialogs/InfoDialog.java b/bundles/org.simantics.utils.ui/src/org/simantics/utils/ui/dialogs/InfoDialog.java new file mode 100644 index 000000000..53a0957d9 --- /dev/null +++ b/bundles/org.simantics.utils.ui/src/org/simantics/utils/ui/dialogs/InfoDialog.java @@ -0,0 +1,119 @@ +/******************************************************************************* + * Copyright (c) 2017 Association for Decentralized Information Management in + * Industry THTH ry. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Semantum Oy - initial API and implementation + *******************************************************************************/ +package org.simantics.utils.ui.dialogs; + +import org.eclipse.jface.dialogs.IDialogConstants; +import org.eclipse.jface.dialogs.IconAndMessageDialog; +import org.eclipse.jface.layout.GridDataFactory; +import org.eclipse.swt.SWT; +import org.eclipse.swt.graphics.Image; +import org.eclipse.swt.layout.GridData; +import org.eclipse.swt.layout.GridLayout; +import org.eclipse.swt.widgets.Composite; +import org.eclipse.swt.widgets.Control; +import org.eclipse.swt.widgets.Label; +import org.eclipse.swt.widgets.Shell; +import org.eclipse.swt.widgets.Text; + +/** + * @author Tuukka Lehtonen + * @since 1.28.0 + */ +public class InfoDialog extends IconAndMessageDialog { + + private String title; + private Text messageText; + + protected InfoDialog(Shell parentShell, String title, String message) { + super(parentShell); + this.title = title; + this.message = message; + setShellStyle(getShellStyle() | SWT.RESIZE); + } + + @Override + protected Image getImage() { + return getInfoImage(); + } + + @Override + protected void configureShell(Shell newShell) { + super.configureShell(newShell); + newShell.setText(title); + newShell.setImage(newShell.getDisplay().getSystemImage(SWT.ICON_INFORMATION)); + } + + protected Control createMessageArea(Composite composite) { + // create image + Image image = getImage(); + if (image != null) { + imageLabel = new Label(composite, SWT.NULL); + image.setBackground(imageLabel.getBackground()); + imageLabel.setImage(image); + GridDataFactory.fillDefaults().align(SWT.CENTER, SWT.BEGINNING) + .applyTo(imageLabel); + } + // create message + if (message != null) { + messageText = new Text(composite, SWT.MULTI | SWT.BORDER | SWT.FLAT | SWT.WRAP | SWT.H_SCROLL | SWT.V_SCROLL); + messageText.setEditable(false); + messageText.setText(message); + GridDataFactory + .fillDefaults() + .grab(true, true) + .hint( + convertHorizontalDLUsToPixels(IDialogConstants.MINIMUM_MESSAGE_AREA_WIDTH), + SWT.DEFAULT).applyTo(messageText); + } + return composite; + } + + protected Control createDialogArea(Composite parent) { + Composite composite = new Composite(parent, SWT.NONE); + + createMessageArea(composite); + + GridLayout layout = new GridLayout(); + layout.marginHeight = convertVerticalDLUsToPixels(IDialogConstants.VERTICAL_MARGIN); + layout.marginWidth = convertHorizontalDLUsToPixels(IDialogConstants.HORIZONTAL_MARGIN); + layout.verticalSpacing = convertVerticalDLUsToPixels(IDialogConstants.VERTICAL_SPACING); + layout.horizontalSpacing = convertHorizontalDLUsToPixels(IDialogConstants.HORIZONTAL_SPACING); + layout.numColumns = 2; + composite.setLayout(layout); + GridData childData = new GridData(GridData.FILL_BOTH); + childData.horizontalSpan = 2; + childData.grabExcessVerticalSpace = true; + composite.setLayoutData(childData); + applyDialogFont(composite); + + return composite; + } + + protected void createButtonsForButtonBar(Composite parent) { + // create OK button by default + createButton(parent, IDialogConstants.OK_ID, IDialogConstants.OK_LABEL, + true); + } + + @Override + protected void buttonPressed(int buttonId) { + super.buttonPressed(buttonId); + } + + public static boolean open(Shell parent, String title, String message, int style) { + InfoDialog dialog = new InfoDialog(parent, title, message); + style &= SWT.SHEET; + dialog.setShellStyle(dialog.getShellStyle() | style); + return dialog.open() == 0; + } + +} \ No newline at end of file diff --git a/bundles/org.simantics/src/org/simantics/OntologyImportAdvisor.java b/bundles/org.simantics/src/org/simantics/OntologyImportAdvisor.java index aa528a652..7cc4cf473 100644 --- a/bundles/org.simantics/src/org/simantics/OntologyImportAdvisor.java +++ b/bundles/org.simantics/src/org/simantics/OntologyImportAdvisor.java @@ -120,6 +120,7 @@ public class OntologyImportAdvisor extends AbstractImportAdvisor2 { graph.claim(ontologyName, L0.InstanceOf, null, L0.String); graph.claimValue(ontologyName, name, WriteBindings.STRING); graph.claim(ontology, L0.HasName, L0.NameOf, ontologyName); + graph.claim(ontology, L0.PartOf, L0.ConsistsOf, parent); return ontology; } else { diff --git a/bundles/org.simantics/src/org/simantics/SimanticsPlatform.java b/bundles/org.simantics/src/org/simantics/SimanticsPlatform.java index cf4928fe7..d5b223460 100644 --- a/bundles/org.simantics/src/org/simantics/SimanticsPlatform.java +++ b/bundles/org.simantics/src/org/simantics/SimanticsPlatform.java @@ -83,6 +83,7 @@ import org.simantics.graph.db.GraphDependencyAnalyzer; import org.simantics.graph.db.GraphDependencyAnalyzer.IU; import org.simantics.graph.db.GraphDependencyAnalyzer.IdentityNode; import org.simantics.graph.db.IImportAdvisor; +import org.simantics.graph.db.ImportResult; import org.simantics.graph.db.TransferableGraphs; import org.simantics.graph.diff.Diff; import org.simantics.graph.diff.TransferableGraphDelta1; @@ -107,6 +108,7 @@ import org.simantics.project.management.WorkspaceUtil; import org.simantics.utils.FileUtils; import org.simantics.utils.datastructures.Pair; import org.simantics.utils.logging.TimeLogger; +import org.simantics.utils.strings.EString; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -408,7 +410,10 @@ public class SimanticsPlatform implements LifecycleListener { // Install TG log.log(new Status(IStatus.INFO, Activator.PLUGIN_ID, "Installing "+tg.toString()+" - "+tg.getName())); - TransferableGraphs.importGraph1(session, new TGTransferableGraphSource(tg.getGraph()), advisor, null); + ImportResult result = TransferableGraphs.importGraph1(session, new TGTransferableGraphSource(tg.getGraph()), advisor, null); + if (!result.missingExternals.isEmpty()) { + log.log(new Status(IStatus.ERROR, Activator.PLUGIN_ID, "Import of " + tg.toString() + " was missing the following external entities:\n" + EString.implode(result.missingExternals))); + } } else { // Merge TG startTransaction(session, false);