1 /*******************************************************************************
2 * Copyright (c) 2012, 2016 Association for Decentralized Information Management
4 * All rights reserved. This program and the accompanying materials
5 * are made available under the terms of the Eclipse Public License v1.0
6 * which accompanies this distribution, and is available at
7 * http://www.eclipse.org/legal/epl-v10.html
10 * VTT Technical Research Centre of Finland - initial API and implementation
12 *******************************************************************************/
13 package org.simantics.graph.db;
15 import java.io.DataInput;
16 import java.io.DataOutput;
17 import java.io.DataOutputStream;
18 import java.io.FileNotFoundException;
19 import java.io.FileOutputStream;
20 import java.io.IOException;
21 import java.io.InputStream;
22 import java.util.HashSet;
25 import java.util.TreeMap;
27 import org.simantics.databoard.Bindings;
28 import org.simantics.databoard.adapter.AdaptException;
29 import org.simantics.databoard.binding.Binding;
30 import org.simantics.databoard.binding.mutable.Variant;
31 import org.simantics.databoard.serialization.Serializer;
32 import org.simantics.databoard.type.Datatype;
33 import org.simantics.db.ReadGraph;
34 import org.simantics.db.Resource;
35 import org.simantics.db.Session;
36 import org.simantics.db.VirtualGraph;
37 import org.simantics.db.WriteOnlyGraph;
38 import org.simantics.db.common.WriteBindings;
39 import org.simantics.db.common.procedure.adapter.TransientCacheAsyncListener;
40 import org.simantics.db.common.request.PossibleIndexRoot;
41 import org.simantics.db.common.uri.UnescapedChildMapOfResource;
42 import org.simantics.db.common.utils.Logger;
43 import org.simantics.db.exception.DatabaseException;
44 import org.simantics.db.service.ClusterBuilder2;
45 import org.simantics.db.service.ClusterBuilderFactory;
46 import org.simantics.db.service.ClusteringSupport;
47 import org.simantics.db.service.SerialisationSupport;
48 import org.simantics.graph.db.TransferableGraphSource.TransferableGraphSourceProcedure;
49 import org.simantics.graph.db.TransferableGraphSource.TransferableGraphSourceValueProcedure;
50 import org.simantics.graph.representation.Extensions;
51 import org.simantics.graph.representation.External;
52 import org.simantics.graph.representation.Identity;
53 import org.simantics.graph.representation.IdentityDefinition;
54 import org.simantics.graph.representation.Internal;
55 import org.simantics.graph.representation.Optional;
56 import org.simantics.graph.representation.Root;
57 import org.simantics.graph.representation.TransferableGraphUtils;
58 import org.simantics.graph.utils.TGResourceUtil;
59 import org.simantics.graph.utils.TGResourceUtil.LongAdapter;
61 public class StreamingTransferableGraphImportProcess implements TransferableGraphImporter {
63 public static String LOG_FILE = "transferableGraphs.log";
64 final static private boolean LOG = false;
66 static DataOutput log;
73 FileOutputStream stream = new FileOutputStream(LOG_FILE);
74 log = new DataOutputStream(stream);
75 } catch (FileNotFoundException e) {
82 private static void log(String line) {
85 log.writeUTF(line + "\n");
86 } catch (IOException e) {
93 TransferableGraphSource tg;
95 IImportAdvisor2 advisor;
96 TGStatusMonitor monitor;
97 ClusterBuilder2 builder;
98 final TGResourceUtil resourceUtil = new TGResourceUtil();
102 Set<String> missingExternals = new HashSet<String>();
105 Identity[] identities;
106 TreeMap<String, Variant> extensions;
109 Resource RootLibrary;
119 public StreamingTransferableGraphImportProcess(Session session, VirtualGraph vg, TransferableGraphSource tg, IImportAdvisor2 advisor) {
120 this(session, vg, tg, advisor, null);
123 public StreamingTransferableGraphImportProcess(Session session, VirtualGraph vg, TransferableGraphSource tg, IImportAdvisor2 advisor, TGStatusMonitor monitor) {
126 this.advisor = advisor;
127 this.monitor = monitor;
130 private int updatePercentage(int percentage, int done, int total) {
131 if (monitor != null && (done & 63) == 0) {
132 int current = 100*done / total;
133 if (current > percentage) {
134 percentage = current;
135 monitor.status(percentage);
141 public void readIdentities(ReadGraph g) throws Exception {
142 extensions = tg.getExtensions();
143 resourceCount = tg.getResourceCount();
144 identities = new Identity[tg.getIdentityCount()];
145 tg.forIdentities(g, new TransferableGraphSourceProcedure<Identity>() {
150 public void execute(Identity value) throws Exception {
151 identities[counter++] = value;
156 public void findBuiltins(WriteOnlyGraph g) throws DatabaseException {
157 RootLibrary = g.getBuiltin("http:/");
158 String = g.getBuiltin(CoreInitialization.LAYER0 + "String");
159 Library = g.getBuiltin(CoreInitialization.LAYER0 + "Library");
160 InstanceOf = g.getBuiltin(CoreInitialization.LAYER0 + "InstanceOf");
161 ConsistsOf = g.getBuiltin(CoreInitialization.LAYER0 + "ConsistsOf");
162 PartOf = g.getBuiltin(CoreInitialization.LAYER0 + "PartOf");
163 HasName = g.getBuiltin(CoreInitialization.LAYER0 + "HasName");
164 NameOf = g.getBuiltin(CoreInitialization.LAYER0 + "NameOf");
167 public void findBuiltins(ReadGraph g) throws DatabaseException {
168 RootLibrary = g.getBuiltin("http:/");
169 String = g.getBuiltin(CoreInitialization.LAYER0 + "String");
170 Library = g.getBuiltin(CoreInitialization.LAYER0 + "Library");
171 InstanceOf = g.getBuiltin(CoreInitialization.LAYER0 + "InstanceOf");
172 ConsistsOf = g.getBuiltin(CoreInitialization.LAYER0 + "ConsistsOf");
173 PartOf = g.getBuiltin(CoreInitialization.LAYER0 + "PartOf");
174 HasName = g.getBuiltin(CoreInitialization.LAYER0 + "HasName");
175 NameOf = g.getBuiltin(CoreInitialization.LAYER0 + "NameOf");
178 // /* Preparation that is used when the core is empty.
180 // void initialPrepare(WriteOnlyGraph graph) throws DatabaseException {
181 // findBuiltins(graph);
183 // resources = new Resource[tg.resourceCount];
186 // int SimanticsDomain = -1;
189 // for(Identity identity : tg.identities) {
190 // if(identity.definition instanceof Internal) {
191 // Internal def = (Internal)identity.definition;
192 // Resource res = null;
193 // if(def.parent == Layer0) {
195 // res = graph.getBuiltin(CoreInitialization.LAYER0 + def.name);
196 // } catch(ResourceNotFoundException e) {
199 // else if(def.parent == SimanticsDomain) {
200 // if(def.name.equals("Layer0-1.0"))
201 // Layer0 = identity.resource;
203 // else if(def.parent == Root) {
204 // if(def.name.equals("www.simantics.org"))
205 // SimanticsDomain = identity.resource;
209 // res = createChild(graph, resources[def.parent], def.name);
211 // createChild(graph, res, resources[def.parent], def.name);
212 // resources[identity.resource] = res;
214 // else if(identity.definition instanceof Root) {
215 // Root = identity.resource;
216 // resources[identity.resource] = RootLibrary;
221 void addMissing(String external) {
222 Set<String> removals = new HashSet<String>();
223 for(String ext : missingExternals) if(ext.startsWith(external)) return;
224 for(String ext : missingExternals) if(external.startsWith(ext)) removals.add(ext);
225 missingExternals.removeAll(removals);
226 missingExternals.add(external);
229 void prepare(ReadGraph graph) throws Exception {
231 Resource target = advisor.getTarget();
233 indexRoot = graph.syncRequest(new PossibleIndexRoot(target));
236 readIdentities(graph);
238 // System.err.println("ext: " + extensions);
239 // System.err.println("rc: " + resourceCount);
240 // System.err.println("ic: " + identities.length);
242 ClusterBuilderFactory factory = graph.getService(ClusterBuilderFactory.class);
243 ClusterBuilder2 builder = factory.create(vg, false);
245 this.handles = new int[resourceCount];
247 for(Identity identity : identities) {
248 IdentityDefinition definition = identity.definition;
249 if(definition instanceof External) {
250 External def = (External)definition;
251 if(def.parent == -1) {
252 handles[identity.resource] = builder.handle(RootLibrary);
254 if("@inverse".equals(def.name)) {
255 int parent = handles[def.parent];
256 int child = builder.handle(graph.getInverse(builder.resource(parent)));
257 handles[identity.resource] = child;
259 int handle = handles[def.parent];
260 Resource parent = handle != 0 ? builder.resource(handle) : null;
261 // TODO: escape should be removed when names become well-behaving
263 Map<String,Resource> childMap = graph
264 .syncRequest(new UnescapedChildMapOfResource(parent),
265 new TransientCacheAsyncListener<Map<String, Resource>>());
266 Resource child = childMap.get(def.name);
268 addMissing(graph.getURI(parent) + "/" + def.name);
270 handles[identity.resource] = builder.handle(child);
273 addMissing(TransferableGraphUtils.getURI(resourceCount, identities, def.parent) + "/" + def.name);
278 else if(definition instanceof Internal) {
279 // Do not do anything for now
281 else if(definition instanceof Root) {
282 Root root = (Root)definition;
283 if(root.name.equals(""))
284 handles[identity.resource] = builder.handle(RootLibrary);
286 Resource existing = advisor.analyzeRoot(graph, root);
288 handles[identity.resource] = builder.handle(existing);
291 else if(definition instanceof Optional) {
292 External def = (External)definition;
293 Resource parent = builder.resource(handles[def.parent]);
295 handles[identity.resource] = builder.handle(graph.syncRequest(new UnescapedChildMapOfResource(parent)).get(def.name));
299 if(!missingExternals.isEmpty()) throw new MissingDependencyException(this);
304 public Resource createChild(WriteOnlyGraph graph, Resource parent, Resource child, String name) throws DatabaseException {
305 if(child == null) child = graph.newResource();
306 Resource nameResource = graph.newResource();
307 graph.claim(nameResource, InstanceOf, null, String);
308 graph.claimValue(nameResource, name, WriteBindings.STRING);
309 graph.claim(child, HasName, NameOf, nameResource);
313 int[] getClustering() {
314 Variant v = extensions.get(Extensions.CLUSTERING);
315 if(v == null) return null;
317 return (int[])v.getValue(Bindings.INT_ARRAY);
318 } catch (AdaptException e) {
319 Logger.defaultLogError(e);
324 int[] getClusterSets() {
325 Variant v = extensions.get(Extensions.CLUSTER_SETS);
326 if(v == null) return null;
328 return (int[])v.getValue(Bindings.INT_ARRAY);
329 } catch (AdaptException e) {
330 Logger.defaultLogError(e);
335 boolean needTranslation(Datatype type) {
336 return resourceUtil.mayHaveResource(type);
339 void findClusterSet(WriteOnlyGraph graph, Resource rootLibrary, int[] clustering, int[] clusterSets, long[] clusters, int id) throws DatabaseException {
340 ClusteringSupport support = graph.getService(ClusteringSupport.class);
341 if(id == Extensions.ROOT_LIBRARY_CLUSTER_SET || id == Extensions.INDEX_ROOT_CLUSTER_SET) return;
342 Resource indexRootClusterSetResource = rootLibrary;
343 if(support.isClusterSet(indexRoot)) {
344 indexRootClusterSetResource = indexRoot;
346 graph.setClusterSet4NewResource(rootLibrary);
347 graph.flushCluster();
349 int indexRootCsHandle = builder.handle(indexRootClusterSetResource);
350 for(int pos=0,index=0;index<clustering.length;index++) {
351 pos += clustering[index];
353 int cs = clusterSets[index];
354 if(handles[id] == 0) {
356 if(cs == Extensions.ROOT_LIBRARY_CLUSTER_SET) csHandle = builder.handle(rootLibrary);
357 else if(cs == Extensions.INDEX_ROOT_CLUSTER_SET) {
358 if(indexRoot == null) throw new DatabaseException("No index root was available in TG import.");
359 csHandle = indexRootCsHandle;
362 findClusterSet(graph, rootLibrary, clustering, clusterSets, clusters, cs);
363 csHandle = handles[cs];
366 if(clusters[index] != 0)
367 builder.selectCluster(clusters[index]);
369 builder.newCluster(csHandle);
371 handles[id] = builder.newResource(csHandle);
372 clusters[index] = support.getCluster(builder.resource(handles[id]));
374 builder.createClusterSet(handles[id]);
381 void write(final WriteOnlyGraph graph) throws Exception {
383 final SerialisationSupport ss = graph.getService(SerialisationSupport.class);
385 ClusterBuilderFactory factory = graph.getService(ClusterBuilderFactory.class);
386 if(advisor instanceof IImportAdvisor2) {
387 boolean allowImmutable = ((IImportAdvisor2)advisor).allowImmutableModifications();
388 builder = factory.create(vg, allowImmutable);
390 builder = factory.create(vg, false);
393 final int[] handles = this.handles;
395 int[] clustering = getClustering();
396 if(clustering != null) {
398 int[] clusterSets = getClusterSets();
399 if(clusterSets != null) {
401 assert(clustering.length == clusterSets.length);
403 long[] clusters = new long[clustering.length];
406 for(int i=0;i<clusterSets.length;i++) {
407 findClusterSet(graph, graph.getRootLibrary(), clustering, clusterSets, clusters, clusterSets[i]);
410 // Then create all resources
412 for(int j=0;j<clustering.length;j++) {
413 int c = clustering[j];
414 int s = clusterSets[j];
416 if(s == Extensions.ROOT_LIBRARY_CLUSTER_SET)
417 setHandle = builder.handle(graph.getRootLibrary());
418 else if(s == Extensions.INDEX_ROOT_CLUSTER_SET) {
419 if(indexRoot == null) throw new DatabaseException("No index root was available in TG import.");
420 setHandle = builder.handle(indexRoot);
422 else setHandle = handles[s];
423 // Preserve clustering only for internal resources
425 builder.selectCluster(clusters[j]);
427 builder.newCluster(setHandle);
428 for(int r=0;r<c;r++, i++)
429 if(handles[i] == 0) handles[i] = builder.newResource();
432 for(;i<handles.length;++i)
433 if(handles[i] == 0) handles[i] = builder.newResource();
438 for(int c : clustering) {
439 builder.newCluster();
440 for(int r=0;r<c;r++, i++)
441 if(handles[i] == 0) handles[i] = builder.newResource();
444 for(;i<handles.length;++i)
445 if(handles[i] == 0) handles[i] = builder.newResource();
451 // Create blank resources
452 for(int i=0;i<handles.length;++i)
453 if(handles[i] == 0) handles[i] = builder.newResource();
457 // Internal identities
458 for(Identity identity : identities) {
459 IdentityDefinition definition = identity.definition;
460 // if(handles[identity.resource] != 0)
462 if(definition instanceof External) {
463 // Already done everything
465 else if(definition instanceof Internal) {
466 Internal def = (Internal)definition;
467 if(handles[identity.resource] != 0)
468 handles[identity.resource] = builder.handle(advisor.createChild(graph, this, builder.resource(handles[def.parent]), builder.resource(handles[identity.resource]), def.name));
470 handles[identity.resource] = builder.handle(advisor.createChild(graph, this, builder.resource(handles[def.parent]), null, def.name));
472 else if(definition instanceof Root) {
474 Root root = (Root)definition;
475 if(handles[identity.resource] != 0)
476 handles[identity.resource] = builder.handle(advisor.createRoot(graph, root, builder.resource(handles[identity.resource])));
478 handles[identity.resource] = builder.handle(advisor.createRoot(graph, root, null));
480 else if(definition instanceof Optional) {
481 Optional def = (Optional)definition;
482 if(handles[identity.resource] != 0) {
483 Resource child = advisor.createChild(graph, this, builder.resource(handles[def.parent]), builder.resource(handles[identity.resource]), def.name);
484 graph.claim(child, InstanceOf, null, Library); // ???
485 handles[identity.resource] = builder.handle(child);
487 Resource child = advisor.createChild(graph, this, builder.resource(handles[def.parent]), null, def.name);
488 graph.claim(child, InstanceOf, null, Library); // ???
489 handles[identity.resource] = builder.handle(child);
495 int[] percentage = { 0 };
497 int statementCount = tg.getStatementCount();
498 tg.forStatements(null, new TransferableGraphSourceProcedure<int[]>() {
501 public void execute(int[] value) throws Exception {
508 int subject = handles[sub];
509 int predicate = handles[pred];
510 int object = handles[obj];
512 builder.addStatement(graph, subject, predicate, object);
514 int inverse = handles[inv];
515 builder.addStatement(graph, object, inverse, subject);
518 // Count from 0% -> 50% => total = statementCount*2
519 percentage[0] = updatePercentage(percentage[0], done[0]++, statementCount*2);
525 int valueCount = tg.getValueCount();
528 class ValueProcedure extends InputStream implements TransferableGraphSourceValueProcedure {
530 private TGResourceUtil util = new TGResourceUtil();
531 private DataInput source;
534 public void execute(int _resource, Datatype type, DataInput stream) throws Exception {
538 //int file = _resource & 0x80000000;
539 int resource = _resource & 0x7FFFFFFF;
541 Binding binding = Bindings.getBinding(type);
542 Serializer s = Bindings.getSerializer(binding);
544 builder.beginValue(handles[resource]);
545 if(util.mayHaveResource(type)) {
546 Object value = s.deserialize(stream);
547 util.adaptValue( binding, value, new LongAdapter() {
549 public long adapt(long in) {
551 return ss.getRandomAccessId(handles[(int)in]);
552 } catch (DatabaseException e) {
553 throw new IllegalStateException(e);
557 byte[] bytes = s.serialize(value);
558 for(byte b : bytes) {
560 if(val < 0) val += 256;
561 builder.appendValue(val);
572 public int read() throws IOException {
573 int value = source.readUnsignedByte();
575 builder.appendValue(value);
576 } catch (DatabaseException e) {
583 public void rawCopy(int resource, int length, DataInput input) throws Exception {
584 builder.beginValue(handles[resource]);
585 for (int i = 0; i < length; ++i)
586 builder.appendValue(input.readUnsignedByte());
591 private void worked() {
592 // Count from 50% -> 100% => [valueCount, valueCount*2)
593 percentage[0] = updatePercentage(percentage[0], valueCount + done[0]++, valueCount*2);
597 tg.forValues2(null, new ValueProcedure());
602 public long[] getResourceIds(SerialisationSupport serializer) throws DatabaseException {
603 final int count = handles.length;
604 long[] resourceIds = new long[count];
605 for(int i=0;i<count;++i)
606 resourceIds[i] = serializer.getRandomAccessId(handles[i]);