1 /*******************************************************************************
\r
2 * Copyright (c) 2012, 2016 Association for Decentralized Information Management
\r
3 * in Industry THTH ry.
\r
4 * All rights reserved. This program and the accompanying materials
\r
5 * are made available under the terms of the Eclipse Public License v1.0
\r
6 * which accompanies this distribution, and is available at
\r
7 * http://www.eclipse.org/legal/epl-v10.html
\r
10 * VTT Technical Research Centre of Finland - initial API and implementation
\r
12 *******************************************************************************/
\r
13 package org.simantics.graph.db;
\r
15 import java.io.DataInput;
\r
16 import java.io.DataOutput;
\r
17 import java.io.DataOutputStream;
\r
18 import java.io.FileNotFoundException;
\r
19 import java.io.FileOutputStream;
\r
20 import java.io.IOException;
\r
21 import java.io.InputStream;
\r
22 import java.util.HashSet;
\r
23 import java.util.Map;
\r
24 import java.util.Set;
\r
25 import java.util.TreeMap;
\r
27 import org.simantics.databoard.Bindings;
\r
28 import org.simantics.databoard.adapter.AdaptException;
\r
29 import org.simantics.databoard.binding.Binding;
\r
30 import org.simantics.databoard.binding.mutable.Variant;
\r
31 import org.simantics.databoard.serialization.Serializer;
\r
32 import org.simantics.databoard.type.Datatype;
\r
33 import org.simantics.db.ReadGraph;
\r
34 import org.simantics.db.Resource;
\r
35 import org.simantics.db.Session;
\r
36 import org.simantics.db.VirtualGraph;
\r
37 import org.simantics.db.WriteOnlyGraph;
\r
38 import org.simantics.db.common.WriteBindings;
\r
39 import org.simantics.db.common.procedure.adapter.TransientCacheAsyncListener;
\r
40 import org.simantics.db.common.uri.UnescapedChildMapOfResource;
\r
41 import org.simantics.db.common.utils.Logger;
\r
42 import org.simantics.db.exception.DatabaseException;
\r
43 import org.simantics.db.service.ClusterBuilder2;
\r
44 import org.simantics.db.service.ClusterBuilderFactory;
\r
45 import org.simantics.db.service.ClusteringSupport;
\r
46 import org.simantics.db.service.SerialisationSupport;
\r
47 import org.simantics.graph.db.TransferableGraphSource.TransferableGraphSourceProcedure;
\r
48 import org.simantics.graph.db.TransferableGraphSource.TransferableGraphSourceValueProcedure;
\r
49 import org.simantics.graph.representation.Extensions;
\r
50 import org.simantics.graph.representation.External;
\r
51 import org.simantics.graph.representation.Identity;
\r
52 import org.simantics.graph.representation.IdentityDefinition;
\r
53 import org.simantics.graph.representation.Internal;
\r
54 import org.simantics.graph.representation.Optional;
\r
55 import org.simantics.graph.representation.Root;
\r
56 import org.simantics.graph.representation.TransferableGraphUtils;
\r
57 import org.simantics.graph.utils.TGResourceUtil;
\r
58 import org.simantics.graph.utils.TGResourceUtil.LongAdapter;
\r
60 public class StreamingTransferableGraphImportProcess implements TransferableGraphImporter {
\r
62 public static String LOG_FILE = "transferableGraphs.log";
\r
63 final static private boolean LOG = false;
\r
65 static DataOutput log;
\r
72 FileOutputStream stream = new FileOutputStream(LOG_FILE);
\r
73 log = new DataOutputStream(stream);
\r
74 } catch (FileNotFoundException e) {
\r
75 e.printStackTrace();
\r
81 private static void log(String line) {
\r
84 log.writeUTF(line + "\n");
\r
85 } catch (IOException e) {
\r
86 e.printStackTrace();
\r
92 TransferableGraphSource tg;
\r
94 IImportAdvisor2 advisor;
\r
95 ClusterBuilder2 builder;
\r
96 final TGResourceUtil resourceUtil = new TGResourceUtil();
\r
100 Set<String> missingExternals = new HashSet<String>();
\r
103 Identity[] identities;
\r
104 TreeMap<String, Variant> extensions;
\r
107 Resource RootLibrary;
\r
111 Resource InstanceOf;
\r
112 Resource ConsistsOf;
\r
117 public StreamingTransferableGraphImportProcess(Session session, VirtualGraph vg, TransferableGraphSource tg, IImportAdvisor2 advisor) {
\r
120 this.advisor = advisor;
\r
123 public void readIdentities(ReadGraph g) throws Exception {
\r
124 extensions = tg.getExtensions();
\r
125 resourceCount = tg.getResourceCount();
\r
126 identities = new Identity[tg.getIdentityCount()];
\r
127 tg.forIdentities(g, new TransferableGraphSourceProcedure<Identity>() {
\r
132 public void execute(Identity value) throws Exception {
\r
133 identities[counter++] = value;
\r
138 public void findBuiltins(WriteOnlyGraph g) throws DatabaseException {
\r
139 RootLibrary = g.getBuiltin("http:/");
\r
140 String = g.getBuiltin(CoreInitialization.LAYER0 + "String");
\r
141 Library = g.getBuiltin(CoreInitialization.LAYER0 + "Library");
\r
142 InstanceOf = g.getBuiltin(CoreInitialization.LAYER0 + "InstanceOf");
\r
143 ConsistsOf = g.getBuiltin(CoreInitialization.LAYER0 + "ConsistsOf");
\r
144 PartOf = g.getBuiltin(CoreInitialization.LAYER0 + "PartOf");
\r
145 HasName = g.getBuiltin(CoreInitialization.LAYER0 + "HasName");
\r
146 NameOf = g.getBuiltin(CoreInitialization.LAYER0 + "NameOf");
\r
149 public void findBuiltins(ReadGraph g) throws DatabaseException {
\r
150 RootLibrary = g.getBuiltin("http:/");
\r
151 String = g.getBuiltin(CoreInitialization.LAYER0 + "String");
\r
152 Library = g.getBuiltin(CoreInitialization.LAYER0 + "Library");
\r
153 InstanceOf = g.getBuiltin(CoreInitialization.LAYER0 + "InstanceOf");
\r
154 ConsistsOf = g.getBuiltin(CoreInitialization.LAYER0 + "ConsistsOf");
\r
155 PartOf = g.getBuiltin(CoreInitialization.LAYER0 + "PartOf");
\r
156 HasName = g.getBuiltin(CoreInitialization.LAYER0 + "HasName");
\r
157 NameOf = g.getBuiltin(CoreInitialization.LAYER0 + "NameOf");
\r
160 // /* Preparation that is used when the core is empty.
\r
162 // void initialPrepare(WriteOnlyGraph graph) throws DatabaseException {
\r
163 // findBuiltins(graph);
\r
165 // resources = new Resource[tg.resourceCount];
\r
168 // int SimanticsDomain = -1;
\r
169 // int Layer0 = -1;
\r
171 // for(Identity identity : tg.identities) {
\r
172 // if(identity.definition instanceof Internal) {
\r
173 // Internal def = (Internal)identity.definition;
\r
174 // Resource res = null;
\r
175 // if(def.parent == Layer0) {
\r
177 // res = graph.getBuiltin(CoreInitialization.LAYER0 + def.name);
\r
178 // } catch(ResourceNotFoundException e) {
\r
181 // else if(def.parent == SimanticsDomain) {
\r
182 // if(def.name.equals("Layer0-1.0"))
\r
183 // Layer0 = identity.resource;
\r
185 // else if(def.parent == Root) {
\r
186 // if(def.name.equals("www.simantics.org"))
\r
187 // SimanticsDomain = identity.resource;
\r
191 // res = createChild(graph, resources[def.parent], def.name);
\r
193 // createChild(graph, res, resources[def.parent], def.name);
\r
194 // resources[identity.resource] = res;
\r
196 // else if(identity.definition instanceof Root) {
\r
197 // Root = identity.resource;
\r
198 // resources[identity.resource] = RootLibrary;
\r
203 void addMissing(String external) {
\r
204 Set<String> removals = new HashSet<String>();
\r
205 for(String ext : missingExternals) if(ext.startsWith(external)) return;
\r
206 for(String ext : missingExternals) if(external.startsWith(ext)) removals.add(ext);
\r
207 missingExternals.removeAll(removals);
\r
208 missingExternals.add(external);
\r
211 void prepare(ReadGraph graph) throws Exception {
\r
213 // indexRoot = graph.syncRequest(new PossibleIndexRoot(((IImportAdvisor2)advisor).getTarget()));
\r
215 findBuiltins(graph);
\r
216 readIdentities(graph);
\r
218 // System.err.println("ext: " + extensions);
\r
219 // System.err.println("rc: " + resourceCount);
\r
220 // System.err.println("ic: " + identities.length);
\r
222 ClusterBuilderFactory factory = graph.getService(ClusterBuilderFactory.class);
\r
223 ClusterBuilder2 builder = factory.create(vg, false);
\r
225 this.handles = new int[resourceCount];
\r
227 for(Identity identity : identities) {
\r
228 IdentityDefinition definition = identity.definition;
\r
229 if(definition instanceof External) {
\r
230 External def = (External)definition;
\r
231 if(def.parent == -1) {
\r
232 handles[identity.resource] = builder.handle(RootLibrary);
\r
234 if("@inverse".equals(def.name)) {
\r
235 int parent = handles[def.parent];
\r
236 int child = builder.handle(graph.getInverse(builder.resource(parent)));
\r
237 handles[identity.resource] = child;
\r
239 int handle = handles[def.parent];
\r
240 Resource parent = handle != 0 ? builder.resource(handle) : null;
\r
241 // TODO: escape should be removed when names become well-behaving
\r
242 if(parent != null) {
\r
243 Map<String,Resource> childMap = graph
\r
244 .syncRequest(new UnescapedChildMapOfResource(parent),
\r
245 new TransientCacheAsyncListener<Map<String, Resource>>());
\r
246 Resource child = childMap.get(def.name);
\r
247 if(child == null) {
\r
248 addMissing(graph.getURI(parent) + "/" + def.name);
\r
250 handles[identity.resource] = builder.handle(child);
\r
253 addMissing(TransferableGraphUtils.getURI(resourceCount, identities, def.parent) + "/" + def.name);
\r
258 else if(definition instanceof Internal) {
\r
259 // Do not do anything for now
\r
261 else if(definition instanceof Root) {
\r
262 Root root = (Root)definition;
\r
263 if(root.name.equals(""))
\r
264 handles[identity.resource] = builder.handle(RootLibrary);
\r
266 Resource existing = advisor.analyzeRoot(graph, root);
\r
267 if(existing != null)
\r
268 handles[identity.resource] = builder.handle(existing);
\r
271 else if(definition instanceof Optional) {
\r
272 External def = (External)definition;
\r
273 Resource parent = builder.resource(handles[def.parent]);
\r
275 handles[identity.resource] = builder.handle(graph.syncRequest(new UnescapedChildMapOfResource(parent)).get(def.name));
\r
279 if(!missingExternals.isEmpty()) throw new MissingDependencyException(this);
\r
284 public Resource createChild(WriteOnlyGraph graph, Resource parent, Resource child, String name) throws DatabaseException {
\r
285 if(child == null) child = graph.newResource();
\r
286 Resource nameResource = graph.newResource();
\r
287 graph.claim(nameResource, InstanceOf, null, String);
\r
288 graph.claimValue(nameResource, name, WriteBindings.STRING);
\r
289 graph.claim(child, HasName, NameOf, nameResource);
\r
293 int[] getClustering() {
\r
294 Variant v = extensions.get(Extensions.CLUSTERING);
\r
295 if(v == null) return null;
\r
297 return (int[])v.getValue(Bindings.INT_ARRAY);
\r
298 } catch (AdaptException e) {
\r
299 Logger.defaultLogError(e);
\r
304 int[] getClusterSets() {
\r
305 Variant v = extensions.get(Extensions.CLUSTER_SETS);
\r
306 if(v == null) return null;
\r
308 return (int[])v.getValue(Bindings.INT_ARRAY);
\r
309 } catch (AdaptException e) {
\r
310 Logger.defaultLogError(e);
\r
315 boolean needTranslation(Datatype type) {
\r
316 return resourceUtil.mayHaveResource(type);
\r
319 void findClusterSet(WriteOnlyGraph graph, Resource rootLibrary, Resource indexRoot, int[] clustering, int[] clusterSets, long[] clusters, int id) throws DatabaseException {
\r
320 ClusteringSupport support = graph.getService(ClusteringSupport.class);
\r
321 if(id == Extensions.ROOT_LIBRARY_CLUSTER_SET || id == Extensions.INDEX_ROOT_CLUSTER_SET) return;
\r
322 for(int pos=0,index=0;index<clustering.length;index++) {
\r
323 pos += clustering[index];
\r
325 int cs = clusterSets[index];
\r
326 if(handles[id] == 0) {
\r
328 if(cs == Extensions.ROOT_LIBRARY_CLUSTER_SET) csHandle = builder.handle(rootLibrary);
\r
329 else if(cs == Extensions.INDEX_ROOT_CLUSTER_SET) csHandle = builder.handle(indexRoot);
\r
331 findClusterSet(graph, rootLibrary, indexRoot, clustering, clusterSets, clusters, cs);
\r
332 csHandle = handles[cs];
\r
335 if(clusters[index] != 0)
\r
336 builder.selectCluster(clusters[index]);
\r
338 builder.newCluster(csHandle);
\r
340 handles[id] = builder.newResource(csHandle);
\r
341 clusters[index] = support.getCluster(builder.resource(handles[id]));
\r
343 builder.createClusterSet(handles[id]);
\r
350 void write(final WriteOnlyGraph graph) throws Exception {
\r
352 final SerialisationSupport ss = graph.getService(SerialisationSupport.class);
\r
354 ClusterBuilderFactory factory = graph.getService(ClusterBuilderFactory.class);
\r
355 if(advisor instanceof IImportAdvisor2) {
\r
356 boolean allowImmutable = ((IImportAdvisor2)advisor).allowImmutableModifications();
\r
357 builder = factory.create(vg, allowImmutable);
\r
359 builder = factory.create(vg, false);
\r
362 final int[] handles = this.handles;
\r
364 int[] clustering = getClustering();
\r
365 if(clustering != null) {
\r
367 int[] clusterSets = getClusterSets();
\r
368 if(clusterSets != null) {
\r
370 assert(clustering.length == clusterSets.length);
\r
372 long[] clusters = new long[clustering.length];
\r
374 // Create clustering
\r
375 for(int i=0;i<clusterSets.length;i++) {
\r
376 findClusterSet(graph, graph.getRootLibrary(), indexRoot, clustering, clusterSets, clusters, clusterSets[i]);
\r
379 // Then create all resources
\r
381 for(int j=0;j<clustering.length;j++) {
\r
382 int c = clustering[j];
\r
383 int s = clusterSets[j];
\r
385 if(s == Extensions.ROOT_LIBRARY_CLUSTER_SET)
\r
386 setHandle = builder.handle(graph.getRootLibrary());
\r
387 else if(s == Extensions.INDEX_ROOT_CLUSTER_SET)
\r
388 setHandle = builder.handle(indexRoot);
\r
389 else setHandle = handles[s];
\r
390 // Preserve clustering only for internal resources
\r
391 if(clusters[j] != 0)
\r
392 builder.selectCluster(clusters[j]);
\r
394 builder.newCluster(setHandle);
\r
395 for(int r=0;r<c;r++, i++)
\r
396 if(handles[i] == 0) handles[i] = builder.newResource();
\r
399 for(;i<handles.length;++i)
\r
400 if(handles[i] == 0) handles[i] = builder.newResource();
\r
405 for(int c : clustering) {
\r
406 builder.newCluster();
\r
407 for(int r=0;r<c;r++, i++)
\r
408 if(handles[i] == 0) handles[i] = builder.newResource();
\r
411 for(;i<handles.length;++i)
\r
412 if(handles[i] == 0) handles[i] = builder.newResource();
\r
418 // Create blank resources
\r
419 for(int i=0;i<handles.length;++i)
\r
420 if(handles[i] == 0) handles[i] = builder.newResource();
\r
424 // Internal identities
\r
425 for(Identity identity : identities) {
\r
426 IdentityDefinition definition = identity.definition;
\r
427 // if(handles[identity.resource] != 0)
\r
429 if(definition instanceof External) {
\r
430 // Already done everything
\r
432 else if(definition instanceof Internal) {
\r
433 Internal def = (Internal)definition;
\r
434 if(handles[identity.resource] != 0)
\r
435 handles[identity.resource] = builder.handle(advisor.createChild(graph, this, builder.resource(handles[def.parent]), builder.resource(handles[identity.resource]), def.name));
\r
437 handles[identity.resource] = builder.handle(advisor.createChild(graph, this, builder.resource(handles[def.parent]), null, def.name));
\r
439 else if(definition instanceof Root) {
\r
441 Root root = (Root)definition;
\r
442 if(handles[identity.resource] != 0)
\r
443 handles[identity.resource] = builder.handle(advisor.createRoot(graph, root, builder.resource(handles[identity.resource])));
\r
445 handles[identity.resource] = builder.handle(advisor.createRoot(graph, root, null));
\r
447 else if(definition instanceof Optional) {
\r
448 Optional def = (Optional)definition;
\r
449 if(handles[identity.resource] != 0) {
\r
450 Resource child = advisor.createChild(graph, this, builder.resource(handles[def.parent]), builder.resource(handles[identity.resource]), def.name);
\r
451 graph.claim(child, InstanceOf, null, Library); // ???
\r
452 handles[identity.resource] = builder.handle(child);
\r
454 Resource child = advisor.createChild(graph, this, builder.resource(handles[def.parent]), null, def.name);
\r
455 graph.claim(child, InstanceOf, null, Library); // ???
\r
456 handles[identity.resource] = builder.handle(child);
\r
461 tg.getStatementCount();
\r
462 tg.forStatements(null, new TransferableGraphSourceProcedure<int[]>() {
\r
465 public void execute(int[] value) throws Exception {
\r
467 int sub = value[0];
\r
468 int pred = value[1];
\r
469 int inv = value[2];
\r
470 int obj = value[3];
\r
472 int subject = handles[sub];
\r
473 int predicate = handles[pred];
\r
474 int object = handles[obj];
\r
476 builder.addStatement(graph, subject, predicate, object);
\r
478 int inverse = handles[inv];
\r
479 builder.addStatement(graph, object, inverse, subject);
\r
486 tg.getValueCount();
\r
488 class ValueProcedure extends InputStream implements TransferableGraphSourceValueProcedure {
\r
490 private TGResourceUtil util = new TGResourceUtil();
\r
491 private DataInput source;
\r
494 public void execute(int _resource, Datatype type, DataInput stream) throws Exception {
\r
498 //int file = _resource & 0x80000000;
\r
499 int resource = _resource & 0x7FFFFFFF;
\r
501 Binding binding = Bindings.getBinding(type);
\r
502 Serializer s = Bindings.getSerializer(binding);
\r
504 builder.beginValue(handles[resource]);
\r
505 if(util.mayHaveResource(type)) {
\r
506 Object value = s.deserialize(stream);
\r
507 util.adaptValue( binding, value, new LongAdapter() {
\r
509 public long adapt(long in) {
\r
511 return ss.getRandomAccessId(handles[(int)in]);
\r
512 } catch (DatabaseException e) {
\r
513 throw new IllegalStateException(e);
\r
517 byte[] bytes = s.serialize(value);
\r
518 for(byte b : bytes) {
\r
520 if(val < 0) val += 256;
\r
521 builder.appendValue(val);
\r
526 builder.endValue();
\r
531 public int read() throws IOException {
\r
532 int value = source.readUnsignedByte();
\r
534 builder.appendValue(value);
\r
535 } catch (DatabaseException e) {
\r
536 e.printStackTrace();
\r
542 public void rawCopy(int resource, int length, DataInput input) throws Exception {
\r
543 builder.beginValue(handles[resource]);
\r
544 for (int i = 0; i < length; ++i)
\r
545 builder.appendValue(input.readUnsignedByte());
\r
546 builder.endValue();
\r
551 tg.forValues2(null, new ValueProcedure());
\r
556 public long[] getResourceIds(SerialisationSupport serializer) throws DatabaseException {
\r
557 final int count = handles.length;
\r
558 long[] resourceIds = new long[count];
\r
559 for(int i=0;i<count;++i)
\r
560 resourceIds[i] = serializer.getRandomAccessId(handles[i]);
\r
561 return resourceIds;
\r