1 /*******************************************************************************
2 * Copyright (c) 2012, 2017 Association for Decentralized Information Management
4 * All rights reserved. This program and the accompanying materials
5 * are made available under the terms of the Eclipse Public License v1.0
6 * which accompanies this distribution, and is available at
7 * http://www.eclipse.org/legal/epl-v10.html
10 * VTT Technical Research Centre of Finland - initial API and implementation
11 * Semantum Oy - e.g. #7016
12 *******************************************************************************/
13 package org.simantics.graph.db;
15 import java.io.DataInput;
16 import java.io.IOException;
17 import java.io.InputStream;
18 import java.util.ArrayList;
19 import java.util.Collections;
20 import java.util.HashMap;
21 import java.util.HashSet;
24 import java.util.TreeMap;
26 import org.simantics.databoard.Bindings;
27 import org.simantics.databoard.adapter.AdaptException;
28 import org.simantics.databoard.binding.Binding;
29 import org.simantics.databoard.binding.mutable.Variant;
30 import org.simantics.databoard.serialization.Serializer;
31 import org.simantics.databoard.type.Datatype;
32 import org.simantics.databoard.util.URIStringUtils;
33 import org.simantics.db.ReadGraph;
34 import org.simantics.db.Resource;
35 import org.simantics.db.Session;
36 import org.simantics.db.VirtualGraph;
37 import org.simantics.db.WriteOnlyGraph;
38 import org.simantics.db.common.WriteBindings;
39 import org.simantics.db.common.procedure.adapter.TransientCacheAsyncListener;
40 import org.simantics.db.common.request.PossibleIndexRoot;
41 import org.simantics.db.common.uri.UnescapedChildMapOfResource;
42 import org.simantics.db.common.utils.Logger;
43 import org.simantics.db.exception.DatabaseException;
44 import org.simantics.db.service.ClusterBuilder2;
45 import org.simantics.db.service.ClusterBuilderFactory;
46 import org.simantics.db.service.ClusteringSupport;
47 import org.simantics.db.service.SerialisationSupport;
48 import org.simantics.db.service.XSupport;
49 import org.simantics.graph.db.TransferableGraphSource.TransferableGraphSourceProcedure;
50 import org.simantics.graph.db.TransferableGraphSource.TransferableGraphSourceValueProcedure;
51 import org.simantics.graph.representation.Extensions;
52 import org.simantics.graph.representation.External;
53 import org.simantics.graph.representation.Identity;
54 import org.simantics.graph.representation.IdentityDefinition;
55 import org.simantics.graph.representation.Internal;
56 import org.simantics.graph.representation.Optional;
57 import org.simantics.graph.representation.Root;
58 import org.simantics.graph.representation.TransferableGraphUtils;
59 import org.simantics.graph.utils.TGResourceUtil;
60 import org.simantics.graph.utils.TGResourceUtil.LongAdapter;
61 import org.simantics.utils.datastructures.Pair;
62 import org.slf4j.LoggerFactory;
64 import gnu.trove.map.TIntObjectMap;
65 import gnu.trove.map.hash.TIntObjectHashMap;
67 public class StreamingTransferableGraphImportProcess implements TransferableGraphImporter {
69 private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(StreamingTransferableGraphImportProcess.class);
72 TransferableGraphSource tg;
74 IImportAdvisor2 advisor;
75 TGStatusMonitor monitor;
76 ClusterBuilder2 builder;
77 final TGResourceUtil resourceUtil = new TGResourceUtil();
81 Map<String,Integer> allMissingExternals = new HashMap<>();
82 Set<String> missingExternals = new HashSet<>();
83 Map<String,Resource> resolvedParents = new HashMap<>();
84 TIntObjectHashMap<Resource> existingInternalMap = new TIntObjectHashMap<>();
87 Identity[] identities;
88 TreeMap<String, Variant> extensions;
93 Resource ExternalEntity;
102 public StreamingTransferableGraphImportProcess(Session session, VirtualGraph vg, TransferableGraphSource tg, IImportAdvisor2 advisor) {
103 this(session, vg, tg, advisor, null);
106 public StreamingTransferableGraphImportProcess(Session session, VirtualGraph vg, TransferableGraphSource tg, IImportAdvisor2 advisor, TGStatusMonitor monitor) {
109 this.advisor = advisor;
110 this.monitor = monitor;
113 private int updatePercentage(int percentage, int done, int total) {
114 if (monitor != null && (done & 63) == 0) {
115 int current = 100*done / total;
116 if (current > percentage) {
117 percentage = current;
118 monitor.status(percentage);
124 public void readIdentities(ReadGraph g) throws Exception {
125 extensions = tg.getExtensions();
126 resourceCount = tg.getResourceCount();
127 identities = new Identity[tg.getIdentityCount()];
128 tg.forIdentities(g, new TransferableGraphSourceProcedure<Identity>() {
133 public void execute(Identity value) throws Exception {
134 identities[counter++] = value;
139 public void findBuiltins(WriteOnlyGraph g) throws DatabaseException {
140 RootLibrary = g.getBuiltin("http:/");
141 String = g.getBuiltin(CoreInitialization.LAYER0 + "String");
142 Library = g.getBuiltin(CoreInitialization.LAYER0 + "Library");
143 InstanceOf = g.getBuiltin(CoreInitialization.LAYER0 + "InstanceOf");
144 ConsistsOf = g.getBuiltin(CoreInitialization.LAYER0 + "ConsistsOf");
145 PartOf = g.getBuiltin(CoreInitialization.LAYER0 + "PartOf");
146 HasName = g.getBuiltin(CoreInitialization.LAYER0 + "HasName");
147 NameOf = g.getBuiltin(CoreInitialization.LAYER0 + "NameOf");
148 ExternalEntity = g.getBuiltin(CoreInitialization.LAYER0 + "ExternalEntity");
151 public void findBuiltins(ReadGraph g) throws DatabaseException {
152 RootLibrary = g.getBuiltin("http:/");
153 String = g.getBuiltin(CoreInitialization.LAYER0 + "String");
154 Library = g.getBuiltin(CoreInitialization.LAYER0 + "Library");
155 InstanceOf = g.getBuiltin(CoreInitialization.LAYER0 + "InstanceOf");
156 ConsistsOf = g.getBuiltin(CoreInitialization.LAYER0 + "ConsistsOf");
157 PartOf = g.getBuiltin(CoreInitialization.LAYER0 + "PartOf");
158 HasName = g.getBuiltin(CoreInitialization.LAYER0 + "HasName");
159 NameOf = g.getBuiltin(CoreInitialization.LAYER0 + "NameOf");
160 ExternalEntity = g.getBuiltin(CoreInitialization.LAYER0 + "ExternalEntity");
163 void addMissing(int handleIndex, String external) {
164 allMissingExternals.put(external, handleIndex);
165 Set<String> removals = new HashSet<>();
166 for(String ext : missingExternals) if(ext.startsWith(external + "/")) return;
167 for(String ext : missingExternals) if(external.startsWith(ext + "/")) removals.add(ext);
168 missingExternals.removeAll(removals);
169 missingExternals.add(external);
172 void prepare(ReadGraph graph) throws Exception {
174 Resource target = advisor.getTarget();
176 indexRoot = graph.syncRequest(new PossibleIndexRoot(target));
179 readIdentities(graph);
181 // System.err.println("ext: " + extensions);
182 // System.err.println("rc: " + resourceCount);
183 // System.err.println("ic: " + identities.length);
185 ClusterBuilderFactory factory = graph.getService(ClusterBuilderFactory.class);
186 ClusterBuilder2 builder = factory.create(vg, false);
188 this.handles = new int[resourceCount];
189 TIntObjectMap<Identity> identityMap = TransferableGraphUtils.mapIdentities(identities);
191 for(Identity identity : identities) {
192 IdentityDefinition definition = identity.definition;
193 if(definition instanceof External) {
194 External def = (External)definition;
195 if(def.parent == -1) {
196 handles[identity.resource] = builder.handle(RootLibrary);
198 if("@inverse".equals(def.name)) {
199 int parent = handles[def.parent];
200 int child = builder.handle(graph.getInverse(builder.resource(parent)));
201 handles[identity.resource] = child;
203 int handle = handles[def.parent];
204 Resource parent = handle != 0 ? builder.resource(handle) : null;
205 // TODO: escape should be removed when names become well-behaving
207 resolvedParents.put(graph.getURI(parent), parent);
208 Map<String,Resource> childMap = graph
209 .syncRequest(new UnescapedChildMapOfResource(parent),
210 TransientCacheAsyncListener.instance());
211 Resource child = childMap.get(def.name);
213 addMissing(identity.resource, graph.getURI(parent) + "/" + URIStringUtils.escape(def.name));
215 handles[identity.resource] = builder.handle(child);
218 addMissing(identity.resource, TransferableGraphUtils.getURI(resourceCount, identityMap, def.parent) + "/" + URIStringUtils.escape(def.name));
223 else if(definition instanceof Internal) {
224 String uri = TransferableGraphUtils.getURI(resourceCount, identityMap, identity.resource);
225 Resource existing = graph.getPossibleResource(uri);
226 if(existing != null) {
227 existingInternalMap.put(identity.resource, existing);
230 else if(definition instanceof Root) {
231 Root root = (Root)definition;
232 if(root.name.equals(""))
233 handles[identity.resource] = builder.handle(RootLibrary);
235 Resource existing = advisor.analyzeRoot(graph, root);
237 handles[identity.resource] = builder.handle(existing);
240 else if(definition instanceof Optional) {
241 External def = (External)definition;
242 Resource parent = builder.resource(handles[def.parent]);
244 handles[identity.resource] = builder.handle(graph.syncRequest(new UnescapedChildMapOfResource(parent)).get(def.name));
248 if (!missingExternals.isEmpty() && failOnMissingEntities())
249 throw new MissingDependencyException(this);
252 private boolean failOnMissingEntities() {
253 return "true".equalsIgnoreCase(
255 "org.simantics.tg.import.failOnMissingEntities",
260 public Resource createChild(WriteOnlyGraph graph, Resource parent, Resource child, String name) throws DatabaseException {
261 if(child == null) child = graph.newResource();
262 Resource nameResource = graph.newResource();
263 graph.claim(nameResource, InstanceOf, null, String);
264 graph.claimValue(nameResource, name, WriteBindings.STRING);
265 graph.claim(child, HasName, NameOf, nameResource);
269 int[] getClustering() {
270 Variant v = extensions.get(Extensions.CLUSTERING);
271 if(v == null) return null;
273 return (int[])v.getValue(Bindings.INT_ARRAY);
274 } catch (AdaptException e) {
275 Logger.defaultLogError(e);
280 int[] getClusterSets() {
281 Variant v = extensions.get(Extensions.CLUSTER_SETS);
282 if(v == null) return null;
284 return (int[])v.getValue(Bindings.INT_ARRAY);
285 } catch (AdaptException e) {
286 Logger.defaultLogError(e);
291 boolean needTranslation(Datatype type) {
292 return resourceUtil.mayHaveResource(type);
295 void findClusterSet(WriteOnlyGraph graph, Resource rootLibrary, int[] clustering, int[] clusterSets, long[] clusters, int id) throws DatabaseException {
296 ClusteringSupport support = graph.getService(ClusteringSupport.class);
297 if(id == Extensions.ROOT_LIBRARY_CLUSTER_SET || id == Extensions.INDEX_ROOT_CLUSTER_SET) return;
298 Resource indexRootClusterSetResource = rootLibrary;
299 if(indexRoot != null && support.isClusterSet(indexRoot)) {
300 indexRootClusterSetResource = indexRoot;
302 graph.setClusterSet4NewResource(rootLibrary);
303 graph.flushCluster();
305 int indexRootCsHandle = builder.handle(indexRootClusterSetResource);
306 for(int pos=0,index=0;index<clustering.length;index++) {
307 pos += clustering[index];
309 int cs = clusterSets[index];
310 if(handles[id] == 0) {
312 if(cs == Extensions.ROOT_LIBRARY_CLUSTER_SET) csHandle = builder.handle(rootLibrary);
313 else if(cs == Extensions.INDEX_ROOT_CLUSTER_SET) {
314 if(indexRoot == null) throw new DatabaseException("No index root was available in TG import.");
315 csHandle = indexRootCsHandle;
318 findClusterSet(graph, rootLibrary, clustering, clusterSets, clusters, cs);
319 csHandle = handles[cs];
322 if(clusters[index] != 0)
323 builder.selectCluster(clusters[index]);
325 builder.newCluster(csHandle);
327 handles[id] = builder.newResource(csHandle);
328 clusters[index] = support.getCluster(builder.resource(handles[id]));
330 builder.createClusterSet(handles[id]);
337 void createMissing(final WriteOnlyGraph graph) throws Exception {
339 if(allMissingExternals.isEmpty()) return;
341 XSupport xs = graph.getService(XSupport.class);
342 Pair<Boolean,Boolean> serviceMode = xs.getServiceMode();
343 xs.setServiceMode(true, false);
345 ArrayList<String> missing = new ArrayList<>(allMissingExternals.keySet());
346 Collections.sort(missing);
347 for(String uri : missing) {
348 String[] parts = URIStringUtils.splitURI(uri);
349 // URIStringUtils.splitURI returns root URI in non-standard format, so fix it manually as a workaround
350 if (parts[0].equals("http://")) {
354 Resource parent = resolvedParents.get(parts[0]);
355 // TODO: proper exception message
357 throw new IllegalStateException("Missing URI: " + uri);
360 Resource childResource = graph.newResource();
361 graph.claim(childResource, InstanceOf, null, ExternalEntity);
363 Resource nameResource = graph.newResource();
364 graph.claim(nameResource, InstanceOf, null, String);
365 graph.claimValue(nameResource, URIStringUtils.unescape(parts[1]), WriteBindings.STRING);
366 graph.claim(childResource, HasName, NameOf, nameResource);
368 graph.claim(parent, ConsistsOf, PartOf, childResource);
370 resolvedParents.put(uri, childResource);
372 handles[allMissingExternals.get(uri)] = builder.handle(childResource);
375 xs.setServiceMode(serviceMode.first, serviceMode.second);
379 void write(final WriteOnlyGraph graph) throws Exception {
381 final SerialisationSupport ss = graph.getService(SerialisationSupport.class);
383 ClusterBuilderFactory factory = graph.getService(ClusterBuilderFactory.class);
384 if(advisor instanceof IImportAdvisor2) {
385 boolean allowImmutable = ((IImportAdvisor2)advisor).allowImmutableModifications();
386 builder = factory.create(vg, allowImmutable);
388 builder = factory.create(vg, false);
391 createMissing(graph);
393 final int[] handles = this.handles;
395 int[] clustering = getClustering();
396 if(clustering != null) {
398 int[] clusterSets = getClusterSets();
399 if(clusterSets != null) {
401 assert(clustering.length == clusterSets.length);
403 long[] clusters = new long[clustering.length];
406 for(int i=0;i<clusterSets.length;i++) {
407 findClusterSet(graph, graph.getRootLibrary(), clustering, clusterSets, clusters, clusterSets[i]);
410 // Then create all resources
412 for(int j=0;j<clustering.length;j++) {
413 int c = clustering[j];
414 int s = clusterSets[j];
416 if(s == Extensions.ROOT_LIBRARY_CLUSTER_SET)
417 setHandle = builder.handle(graph.getRootLibrary());
418 else if(s == Extensions.INDEX_ROOT_CLUSTER_SET) {
419 if(indexRoot == null) throw new DatabaseException("No index root was available in TG import.");
420 setHandle = builder.handle(indexRoot);
422 else setHandle = handles[s];
423 // Preserve clustering only for internal resources
425 builder.selectCluster(clusters[j]);
427 builder.newCluster(setHandle);
428 for(int r=0;r<c;r++, i++)
429 if(handles[i] == 0) handles[i] = builder.newResource();
432 for(;i<handles.length;++i)
433 if(handles[i] == 0) handles[i] = builder.newResource();
438 for(int c : clustering) {
439 builder.newCluster();
440 for(int r=0;r<c;r++, i++)
441 if(handles[i] == 0) handles[i] = builder.newResource();
444 for(;i<handles.length;++i)
445 if(handles[i] == 0) handles[i] = builder.newResource();
451 // Create blank resources
452 for(int i=0;i<handles.length;++i)
453 if(handles[i] == 0) handles[i] = builder.newResource();
457 // Internal identities
458 for(Identity identity : identities) {
459 IdentityDefinition definition = identity.definition;
460 // if(handles[identity.resource] != 0)
462 if(definition instanceof External) {
463 // Already done everything
465 else if(definition instanceof Internal) {
466 Internal def = (Internal)definition;
468 Resource external = existingInternalMap.get(identity.resource);
469 if(external != null) {
470 handles[identity.resource] = builder.handle(external);
472 if(handles[identity.resource] != 0)
473 handles[identity.resource] = builder.handle(advisor.createChild(graph, this, builder.resource(handles[def.parent]), builder.resource(handles[identity.resource]), def.name));
475 handles[identity.resource] = builder.handle(advisor.createChild(graph, this, builder.resource(handles[def.parent]), null, def.name));
479 else if(definition instanceof Root) {
481 Root root = (Root)definition;
482 if(handles[identity.resource] != 0)
483 handles[identity.resource] = builder.handle(advisor.createRoot(graph, root, builder.resource(handles[identity.resource])));
485 handles[identity.resource] = builder.handle(advisor.createRoot(graph, root, null));
487 else if(definition instanceof Optional) {
488 Optional def = (Optional)definition;
489 if(handles[identity.resource] != 0) {
490 Resource child = advisor.createChild(graph, this, builder.resource(handles[def.parent]), builder.resource(handles[identity.resource]), def.name);
491 graph.claim(child, InstanceOf, null, Library); // ???
492 handles[identity.resource] = builder.handle(child);
494 Resource child = advisor.createChild(graph, this, builder.resource(handles[def.parent]), null, def.name);
495 graph.claim(child, InstanceOf, null, Library); // ???
496 handles[identity.resource] = builder.handle(child);
502 int[] percentage = { 0 };
504 int statementCount = tg.getStatementCount();
505 tg.forStatements(null, new TransferableGraphSourceProcedure<int[]>() {
508 public void execute(int[] value) throws Exception {
515 int subject = handles[sub];
516 int predicate = handles[pred];
517 int object = handles[obj];
519 builder.addStatement(graph, subject, predicate, object);
521 int inverse = handles[inv];
522 builder.addStatement(graph, object, inverse, subject);
525 // Count from 0% -> 50% => total = statementCount*2
526 percentage[0] = updatePercentage(percentage[0], done[0]++, statementCount*2);
532 int valueCount = tg.getValueCount();
535 class ValueProcedure extends InputStream implements TransferableGraphSourceValueProcedure {
537 private TGResourceUtil util = new TGResourceUtil();
538 private DataInput source;
541 public void execute(int _resource, Datatype type, DataInput stream) throws Exception {
545 //int file = _resource & 0x80000000;
546 int resource = _resource & 0x7FFFFFFF;
548 Binding binding = Bindings.getBinding(type);
549 Serializer s = Bindings.getSerializer(binding);
551 builder.beginValue(handles[resource]);
552 if(util.mayHaveResource(type)) {
553 Object value = s.deserialize(stream);
554 util.adaptValue( binding, value, new LongAdapter() {
556 public long adapt(long in) {
558 return ss.getRandomAccessId(handles[(int)in]);
559 } catch (DatabaseException e) {
560 throw new IllegalStateException(e);
564 byte[] bytes = s.serialize(value);
565 for(byte b : bytes) {
567 if(val < 0) val += 256;
568 builder.appendValue(val);
579 public int read() throws IOException {
580 int value = source.readUnsignedByte();
582 builder.appendValue(value);
583 } catch (DatabaseException e) {
584 LOGGER.error("Failed to write value into database", e);
590 public void rawCopy(int resource, int length, DataInput input) throws Exception {
591 builder.beginValue(handles[resource]);
592 for (int i = 0; i < length; ++i)
593 builder.appendValue(input.readUnsignedByte());
598 private void work() {
599 // Count from 50% -> 100% => [valueCount, valueCount*2)
600 percentage[0] = updatePercentage(percentage[0], valueCount + done[0]++, valueCount*2);
604 tg.forValues2(null, new ValueProcedure());
606 for(Resource r : existingInternalMap.valueCollection()) {
607 graph.deny(r, InstanceOf, null, ExternalEntity, null);
613 public long[] getResourceIds(SerialisationSupport serializer) throws DatabaseException {
614 final int count = handles.length;
615 long[] resourceIds = new long[count];
616 for(int i=0;i<count;++i)
617 resourceIds[i] = serializer.getRandomAccessId(handles[i]);