1 /*******************************************************************************
\r
2 * Copyright (c) 2012, 2016 Association for Decentralized Information Management
\r
3 * in Industry THTH ry.
\r
4 * All rights reserved. This program and the accompanying materials
\r
5 * are made available under the terms of the Eclipse Public License v1.0
\r
6 * which accompanies this distribution, and is available at
\r
7 * http://www.eclipse.org/legal/epl-v10.html
\r
10 * VTT Technical Research Centre of Finland - initial API and implementation
\r
12 *******************************************************************************/
\r
13 package org.simantics.graph.db;
\r
15 import java.io.DataInput;
\r
16 import java.io.DataOutput;
\r
17 import java.io.DataOutputStream;
\r
18 import java.io.FileNotFoundException;
\r
19 import java.io.FileOutputStream;
\r
20 import java.io.IOException;
\r
21 import java.io.InputStream;
\r
22 import java.util.HashSet;
\r
23 import java.util.Map;
\r
24 import java.util.Set;
\r
25 import java.util.TreeMap;
\r
27 import org.simantics.databoard.Bindings;
\r
28 import org.simantics.databoard.adapter.AdaptException;
\r
29 import org.simantics.databoard.binding.Binding;
\r
30 import org.simantics.databoard.binding.mutable.Variant;
\r
31 import org.simantics.databoard.serialization.Serializer;
\r
32 import org.simantics.databoard.type.Datatype;
\r
33 import org.simantics.db.ReadGraph;
\r
34 import org.simantics.db.Resource;
\r
35 import org.simantics.db.Session;
\r
36 import org.simantics.db.VirtualGraph;
\r
37 import org.simantics.db.WriteOnlyGraph;
\r
38 import org.simantics.db.common.WriteBindings;
\r
39 import org.simantics.db.common.procedure.adapter.TransientCacheAsyncListener;
\r
40 import org.simantics.db.common.request.PossibleIndexRoot;
\r
41 import org.simantics.db.common.uri.UnescapedChildMapOfResource;
\r
42 import org.simantics.db.common.utils.Logger;
\r
43 import org.simantics.db.exception.DatabaseException;
\r
44 import org.simantics.db.service.ClusterBuilder2;
\r
45 import org.simantics.db.service.ClusterBuilderFactory;
\r
46 import org.simantics.db.service.ClusteringSupport;
\r
47 import org.simantics.db.service.SerialisationSupport;
\r
48 import org.simantics.graph.db.TransferableGraphSource.TransferableGraphSourceProcedure;
\r
49 import org.simantics.graph.db.TransferableGraphSource.TransferableGraphSourceValueProcedure;
\r
50 import org.simantics.graph.representation.Extensions;
\r
51 import org.simantics.graph.representation.External;
\r
52 import org.simantics.graph.representation.Identity;
\r
53 import org.simantics.graph.representation.IdentityDefinition;
\r
54 import org.simantics.graph.representation.Internal;
\r
55 import org.simantics.graph.representation.Optional;
\r
56 import org.simantics.graph.representation.Root;
\r
57 import org.simantics.graph.representation.TransferableGraphUtils;
\r
58 import org.simantics.graph.utils.TGResourceUtil;
\r
59 import org.simantics.graph.utils.TGResourceUtil.LongAdapter;
\r
61 public class StreamingTransferableGraphImportProcess implements TransferableGraphImporter {
\r
63 public static String LOG_FILE = "transferableGraphs.log";
\r
64 final static private boolean LOG = false;
\r
66 static DataOutput log;
\r
73 FileOutputStream stream = new FileOutputStream(LOG_FILE);
\r
74 log = new DataOutputStream(stream);
\r
75 } catch (FileNotFoundException e) {
\r
76 e.printStackTrace();
\r
82 private static void log(String line) {
\r
85 log.writeUTF(line + "\n");
\r
86 } catch (IOException e) {
\r
87 e.printStackTrace();
\r
93 TransferableGraphSource tg;
\r
95 IImportAdvisor2 advisor;
\r
96 ClusterBuilder2 builder;
\r
97 final TGResourceUtil resourceUtil = new TGResourceUtil();
\r
101 Set<String> missingExternals = new HashSet<String>();
\r
104 Identity[] identities;
\r
105 TreeMap<String, Variant> extensions;
\r
108 Resource RootLibrary;
\r
112 Resource InstanceOf;
\r
113 Resource ConsistsOf;
\r
118 public StreamingTransferableGraphImportProcess(Session session, VirtualGraph vg, TransferableGraphSource tg, IImportAdvisor2 advisor) {
\r
121 this.advisor = advisor;
\r
124 public void readIdentities(ReadGraph g) throws Exception {
\r
125 extensions = tg.getExtensions();
\r
126 resourceCount = tg.getResourceCount();
\r
127 identities = new Identity[tg.getIdentityCount()];
\r
128 tg.forIdentities(g, new TransferableGraphSourceProcedure<Identity>() {
\r
133 public void execute(Identity value) throws Exception {
\r
134 identities[counter++] = value;
\r
139 public void findBuiltins(WriteOnlyGraph g) throws DatabaseException {
\r
140 RootLibrary = g.getBuiltin("http:/");
\r
141 String = g.getBuiltin(CoreInitialization.LAYER0 + "String");
\r
142 Library = g.getBuiltin(CoreInitialization.LAYER0 + "Library");
\r
143 InstanceOf = g.getBuiltin(CoreInitialization.LAYER0 + "InstanceOf");
\r
144 ConsistsOf = g.getBuiltin(CoreInitialization.LAYER0 + "ConsistsOf");
\r
145 PartOf = g.getBuiltin(CoreInitialization.LAYER0 + "PartOf");
\r
146 HasName = g.getBuiltin(CoreInitialization.LAYER0 + "HasName");
\r
147 NameOf = g.getBuiltin(CoreInitialization.LAYER0 + "NameOf");
\r
150 public void findBuiltins(ReadGraph g) throws DatabaseException {
\r
151 RootLibrary = g.getBuiltin("http:/");
\r
152 String = g.getBuiltin(CoreInitialization.LAYER0 + "String");
\r
153 Library = g.getBuiltin(CoreInitialization.LAYER0 + "Library");
\r
154 InstanceOf = g.getBuiltin(CoreInitialization.LAYER0 + "InstanceOf");
\r
155 ConsistsOf = g.getBuiltin(CoreInitialization.LAYER0 + "ConsistsOf");
\r
156 PartOf = g.getBuiltin(CoreInitialization.LAYER0 + "PartOf");
\r
157 HasName = g.getBuiltin(CoreInitialization.LAYER0 + "HasName");
\r
158 NameOf = g.getBuiltin(CoreInitialization.LAYER0 + "NameOf");
\r
161 // /* Preparation that is used when the core is empty.
\r
163 // void initialPrepare(WriteOnlyGraph graph) throws DatabaseException {
\r
164 // findBuiltins(graph);
\r
166 // resources = new Resource[tg.resourceCount];
\r
169 // int SimanticsDomain = -1;
\r
170 // int Layer0 = -1;
\r
172 // for(Identity identity : tg.identities) {
\r
173 // if(identity.definition instanceof Internal) {
\r
174 // Internal def = (Internal)identity.definition;
\r
175 // Resource res = null;
\r
176 // if(def.parent == Layer0) {
\r
178 // res = graph.getBuiltin(CoreInitialization.LAYER0 + def.name);
\r
179 // } catch(ResourceNotFoundException e) {
\r
182 // else if(def.parent == SimanticsDomain) {
\r
183 // if(def.name.equals("Layer0-1.0"))
\r
184 // Layer0 = identity.resource;
\r
186 // else if(def.parent == Root) {
\r
187 // if(def.name.equals("www.simantics.org"))
\r
188 // SimanticsDomain = identity.resource;
\r
192 // res = createChild(graph, resources[def.parent], def.name);
\r
194 // createChild(graph, res, resources[def.parent], def.name);
\r
195 // resources[identity.resource] = res;
\r
197 // else if(identity.definition instanceof Root) {
\r
198 // Root = identity.resource;
\r
199 // resources[identity.resource] = RootLibrary;
\r
204 void addMissing(String external) {
\r
205 Set<String> removals = new HashSet<String>();
\r
206 for(String ext : missingExternals) if(ext.startsWith(external)) return;
\r
207 for(String ext : missingExternals) if(external.startsWith(ext)) removals.add(ext);
\r
208 missingExternals.removeAll(removals);
\r
209 missingExternals.add(external);
\r
212 void prepare(ReadGraph graph) throws Exception {
\r
214 Resource target = advisor.getTarget();
\r
216 indexRoot = graph.syncRequest(new PossibleIndexRoot(target));
\r
218 findBuiltins(graph);
\r
219 readIdentities(graph);
\r
221 // System.err.println("ext: " + extensions);
\r
222 // System.err.println("rc: " + resourceCount);
\r
223 // System.err.println("ic: " + identities.length);
\r
225 ClusterBuilderFactory factory = graph.getService(ClusterBuilderFactory.class);
\r
226 ClusterBuilder2 builder = factory.create(vg, false);
\r
228 this.handles = new int[resourceCount];
\r
230 for(Identity identity : identities) {
\r
231 IdentityDefinition definition = identity.definition;
\r
232 if(definition instanceof External) {
\r
233 External def = (External)definition;
\r
234 if(def.parent == -1) {
\r
235 handles[identity.resource] = builder.handle(RootLibrary);
\r
237 if("@inverse".equals(def.name)) {
\r
238 int parent = handles[def.parent];
\r
239 int child = builder.handle(graph.getInverse(builder.resource(parent)));
\r
240 handles[identity.resource] = child;
\r
242 int handle = handles[def.parent];
\r
243 Resource parent = handle != 0 ? builder.resource(handle) : null;
\r
244 // TODO: escape should be removed when names become well-behaving
\r
245 if(parent != null) {
\r
246 Map<String,Resource> childMap = graph
\r
247 .syncRequest(new UnescapedChildMapOfResource(parent),
\r
248 new TransientCacheAsyncListener<Map<String, Resource>>());
\r
249 Resource child = childMap.get(def.name);
\r
250 if(child == null) {
\r
251 addMissing(graph.getURI(parent) + "/" + def.name);
\r
253 handles[identity.resource] = builder.handle(child);
\r
256 addMissing(TransferableGraphUtils.getURI(resourceCount, identities, def.parent) + "/" + def.name);
\r
261 else if(definition instanceof Internal) {
\r
262 // Do not do anything for now
\r
264 else if(definition instanceof Root) {
\r
265 Root root = (Root)definition;
\r
266 if(root.name.equals(""))
\r
267 handles[identity.resource] = builder.handle(RootLibrary);
\r
269 Resource existing = advisor.analyzeRoot(graph, root);
\r
270 if(existing != null)
\r
271 handles[identity.resource] = builder.handle(existing);
\r
274 else if(definition instanceof Optional) {
\r
275 External def = (External)definition;
\r
276 Resource parent = builder.resource(handles[def.parent]);
\r
278 handles[identity.resource] = builder.handle(graph.syncRequest(new UnescapedChildMapOfResource(parent)).get(def.name));
\r
282 if(!missingExternals.isEmpty()) throw new MissingDependencyException(this);
\r
287 public Resource createChild(WriteOnlyGraph graph, Resource parent, Resource child, String name) throws DatabaseException {
\r
288 if(child == null) child = graph.newResource();
\r
289 Resource nameResource = graph.newResource();
\r
290 graph.claim(nameResource, InstanceOf, null, String);
\r
291 graph.claimValue(nameResource, name, WriteBindings.STRING);
\r
292 graph.claim(child, HasName, NameOf, nameResource);
\r
296 int[] getClustering() {
\r
297 Variant v = extensions.get(Extensions.CLUSTERING);
\r
298 if(v == null) return null;
\r
300 return (int[])v.getValue(Bindings.INT_ARRAY);
\r
301 } catch (AdaptException e) {
\r
302 Logger.defaultLogError(e);
\r
307 int[] getClusterSets() {
\r
308 Variant v = extensions.get(Extensions.CLUSTER_SETS);
\r
309 if(v == null) return null;
\r
311 return (int[])v.getValue(Bindings.INT_ARRAY);
\r
312 } catch (AdaptException e) {
\r
313 Logger.defaultLogError(e);
\r
318 boolean needTranslation(Datatype type) {
\r
319 return resourceUtil.mayHaveResource(type);
\r
322 void findClusterSet(WriteOnlyGraph graph, Resource rootLibrary, int[] clustering, int[] clusterSets, long[] clusters, int id) throws DatabaseException {
\r
323 ClusteringSupport support = graph.getService(ClusteringSupport.class);
\r
324 if(id == Extensions.ROOT_LIBRARY_CLUSTER_SET || id == Extensions.INDEX_ROOT_CLUSTER_SET) return;
\r
325 for(int pos=0,index=0;index<clustering.length;index++) {
\r
326 pos += clustering[index];
\r
328 int cs = clusterSets[index];
\r
329 if(handles[id] == 0) {
\r
331 if(cs == Extensions.ROOT_LIBRARY_CLUSTER_SET) csHandle = builder.handle(rootLibrary);
\r
332 else if(cs == Extensions.INDEX_ROOT_CLUSTER_SET) {
\r
333 if(indexRoot == null) throw new DatabaseException("No index root was available in TG import.");
\r
334 csHandle = builder.handle(indexRoot);
\r
337 findClusterSet(graph, rootLibrary, clustering, clusterSets, clusters, cs);
\r
338 csHandle = handles[cs];
\r
341 if(clusters[index] != 0)
\r
342 builder.selectCluster(clusters[index]);
\r
344 builder.newCluster(csHandle);
\r
346 handles[id] = builder.newResource(csHandle);
\r
347 clusters[index] = support.getCluster(builder.resource(handles[id]));
\r
349 builder.createClusterSet(handles[id]);
\r
356 void write(final WriteOnlyGraph graph) throws Exception {
\r
358 final SerialisationSupport ss = graph.getService(SerialisationSupport.class);
\r
360 ClusterBuilderFactory factory = graph.getService(ClusterBuilderFactory.class);
\r
361 if(advisor instanceof IImportAdvisor2) {
\r
362 boolean allowImmutable = ((IImportAdvisor2)advisor).allowImmutableModifications();
\r
363 builder = factory.create(vg, allowImmutable);
\r
365 builder = factory.create(vg, false);
\r
368 final int[] handles = this.handles;
\r
370 int[] clustering = getClustering();
\r
371 if(clustering != null) {
\r
373 int[] clusterSets = getClusterSets();
\r
374 if(clusterSets != null) {
\r
376 assert(clustering.length == clusterSets.length);
\r
378 long[] clusters = new long[clustering.length];
\r
380 // Create clustering
\r
381 for(int i=0;i<clusterSets.length;i++) {
\r
382 findClusterSet(graph, graph.getRootLibrary(), clustering, clusterSets, clusters, clusterSets[i]);
\r
385 // Then create all resources
\r
387 for(int j=0;j<clustering.length;j++) {
\r
388 int c = clustering[j];
\r
389 int s = clusterSets[j];
\r
391 if(s == Extensions.ROOT_LIBRARY_CLUSTER_SET)
\r
392 setHandle = builder.handle(graph.getRootLibrary());
\r
393 else if(s == Extensions.INDEX_ROOT_CLUSTER_SET) {
\r
394 if(indexRoot == null) throw new DatabaseException("No index root was available in TG import.");
\r
395 setHandle = builder.handle(indexRoot);
\r
397 else setHandle = handles[s];
\r
398 // Preserve clustering only for internal resources
\r
399 if(clusters[j] != 0)
\r
400 builder.selectCluster(clusters[j]);
\r
402 builder.newCluster(setHandle);
\r
403 for(int r=0;r<c;r++, i++)
\r
404 if(handles[i] == 0) handles[i] = builder.newResource();
\r
407 for(;i<handles.length;++i)
\r
408 if(handles[i] == 0) handles[i] = builder.newResource();
\r
413 for(int c : clustering) {
\r
414 builder.newCluster();
\r
415 for(int r=0;r<c;r++, i++)
\r
416 if(handles[i] == 0) handles[i] = builder.newResource();
\r
419 for(;i<handles.length;++i)
\r
420 if(handles[i] == 0) handles[i] = builder.newResource();
\r
426 // Create blank resources
\r
427 for(int i=0;i<handles.length;++i)
\r
428 if(handles[i] == 0) handles[i] = builder.newResource();
\r
432 // Internal identities
\r
433 for(Identity identity : identities) {
\r
434 IdentityDefinition definition = identity.definition;
\r
435 // if(handles[identity.resource] != 0)
\r
437 if(definition instanceof External) {
\r
438 // Already done everything
\r
440 else if(definition instanceof Internal) {
\r
441 Internal def = (Internal)definition;
\r
442 if(handles[identity.resource] != 0)
\r
443 handles[identity.resource] = builder.handle(advisor.createChild(graph, this, builder.resource(handles[def.parent]), builder.resource(handles[identity.resource]), def.name));
\r
445 handles[identity.resource] = builder.handle(advisor.createChild(graph, this, builder.resource(handles[def.parent]), null, def.name));
\r
447 else if(definition instanceof Root) {
\r
449 Root root = (Root)definition;
\r
450 if(handles[identity.resource] != 0)
\r
451 handles[identity.resource] = builder.handle(advisor.createRoot(graph, root, builder.resource(handles[identity.resource])));
\r
453 handles[identity.resource] = builder.handle(advisor.createRoot(graph, root, null));
\r
455 else if(definition instanceof Optional) {
\r
456 Optional def = (Optional)definition;
\r
457 if(handles[identity.resource] != 0) {
\r
458 Resource child = advisor.createChild(graph, this, builder.resource(handles[def.parent]), builder.resource(handles[identity.resource]), def.name);
\r
459 graph.claim(child, InstanceOf, null, Library); // ???
\r
460 handles[identity.resource] = builder.handle(child);
\r
462 Resource child = advisor.createChild(graph, this, builder.resource(handles[def.parent]), null, def.name);
\r
463 graph.claim(child, InstanceOf, null, Library); // ???
\r
464 handles[identity.resource] = builder.handle(child);
\r
469 tg.getStatementCount();
\r
470 tg.forStatements(null, new TransferableGraphSourceProcedure<int[]>() {
\r
473 public void execute(int[] value) throws Exception {
\r
475 int sub = value[0];
\r
476 int pred = value[1];
\r
477 int inv = value[2];
\r
478 int obj = value[3];
\r
480 int subject = handles[sub];
\r
481 int predicate = handles[pred];
\r
482 int object = handles[obj];
\r
484 builder.addStatement(graph, subject, predicate, object);
\r
486 int inverse = handles[inv];
\r
487 builder.addStatement(graph, object, inverse, subject);
\r
494 tg.getValueCount();
\r
496 class ValueProcedure extends InputStream implements TransferableGraphSourceValueProcedure {
\r
498 private TGResourceUtil util = new TGResourceUtil();
\r
499 private DataInput source;
\r
502 public void execute(int _resource, Datatype type, DataInput stream) throws Exception {
\r
506 //int file = _resource & 0x80000000;
\r
507 int resource = _resource & 0x7FFFFFFF;
\r
509 Binding binding = Bindings.getBinding(type);
\r
510 Serializer s = Bindings.getSerializer(binding);
\r
512 builder.beginValue(handles[resource]);
\r
513 if(util.mayHaveResource(type)) {
\r
514 Object value = s.deserialize(stream);
\r
515 util.adaptValue( binding, value, new LongAdapter() {
\r
517 public long adapt(long in) {
\r
519 return ss.getRandomAccessId(handles[(int)in]);
\r
520 } catch (DatabaseException e) {
\r
521 throw new IllegalStateException(e);
\r
525 byte[] bytes = s.serialize(value);
\r
526 for(byte b : bytes) {
\r
528 if(val < 0) val += 256;
\r
529 builder.appendValue(val);
\r
534 builder.endValue();
\r
539 public int read() throws IOException {
\r
540 int value = source.readUnsignedByte();
\r
542 builder.appendValue(value);
\r
543 } catch (DatabaseException e) {
\r
544 e.printStackTrace();
\r
550 public void rawCopy(int resource, int length, DataInput input) throws Exception {
\r
551 builder.beginValue(handles[resource]);
\r
552 for (int i = 0; i < length; ++i)
\r
553 builder.appendValue(input.readUnsignedByte());
\r
554 builder.endValue();
\r
559 tg.forValues2(null, new ValueProcedure());
\r
564 public long[] getResourceIds(SerialisationSupport serializer) throws DatabaseException {
\r
565 final int count = handles.length;
\r
566 long[] resourceIds = new long[count];
\r
567 for(int i=0;i<count;++i)
\r
568 resourceIds[i] = serializer.getRandomAccessId(handles[i]);
\r
569 return resourceIds;
\r