]> gerrit.simantics Code Review - simantics/platform.git/blob
48cebf342c2cfe5566a3fb325bd33994437b20f7
[simantics/platform.git] /
1 /*******************************************************************************
2  * Copyright (c) 2012, 2016 Association for Decentralized Information Management
3  * in Industry THTH ry.
4  * All rights reserved. This program and the accompanying materials
5  * are made available under the terms of the Eclipse Public License v1.0
6  * which accompanies this distribution, and is available at
7  * http://www.eclipse.org/legal/epl-v10.html
8  *
9  * Contributors:
10  *     VTT Technical Research Centre of Finland - initial API and implementation
11  *     Semantum Oy
12  *******************************************************************************/
13 package org.simantics.graph.db;
14
15 import java.io.DataInput;
16 import java.io.DataOutput;
17 import java.io.DataOutputStream;
18 import java.io.FileNotFoundException;
19 import java.io.FileOutputStream;
20 import java.io.IOException;
21 import java.io.InputStream;
22 import java.util.HashSet;
23 import java.util.Map;
24 import java.util.Set;
25 import java.util.TreeMap;
26
27 import org.simantics.databoard.Bindings;
28 import org.simantics.databoard.adapter.AdaptException;
29 import org.simantics.databoard.binding.Binding;
30 import org.simantics.databoard.binding.mutable.Variant;
31 import org.simantics.databoard.serialization.Serializer;
32 import org.simantics.databoard.type.Datatype;
33 import org.simantics.db.ReadGraph;
34 import org.simantics.db.Resource;
35 import org.simantics.db.Session;
36 import org.simantics.db.VirtualGraph;
37 import org.simantics.db.WriteOnlyGraph;
38 import org.simantics.db.common.WriteBindings;
39 import org.simantics.db.common.procedure.adapter.TransientCacheAsyncListener;
40 import org.simantics.db.common.request.PossibleIndexRoot;
41 import org.simantics.db.common.uri.UnescapedChildMapOfResource;
42 import org.simantics.db.common.utils.Logger;
43 import org.simantics.db.exception.DatabaseException;
44 import org.simantics.db.service.ClusterBuilder2;
45 import org.simantics.db.service.ClusterBuilderFactory;
46 import org.simantics.db.service.ClusteringSupport;
47 import org.simantics.db.service.SerialisationSupport;
48 import org.simantics.graph.db.TransferableGraphSource.TransferableGraphSourceProcedure;
49 import org.simantics.graph.db.TransferableGraphSource.TransferableGraphSourceValueProcedure;
50 import org.simantics.graph.representation.Extensions;
51 import org.simantics.graph.representation.External;
52 import org.simantics.graph.representation.Identity;
53 import org.simantics.graph.representation.IdentityDefinition;
54 import org.simantics.graph.representation.Internal;
55 import org.simantics.graph.representation.Optional;
56 import org.simantics.graph.representation.Root;
57 import org.simantics.graph.representation.TransferableGraphUtils;
58 import org.simantics.graph.utils.TGResourceUtil;
59 import org.simantics.graph.utils.TGResourceUtil.LongAdapter;
60
61 public class StreamingTransferableGraphImportProcess implements TransferableGraphImporter {
62         
63         public static String LOG_FILE = "transferableGraphs.log";
64         final static private boolean LOG = false;
65         
66         static DataOutput log;
67
68         static {
69
70                 if (LOG) {
71                         
72                         try {
73                                 FileOutputStream stream = new FileOutputStream(LOG_FILE);
74                                 log = new DataOutputStream(stream);
75                         } catch (FileNotFoundException e) {
76                                 e.printStackTrace();
77                         }
78                 }
79
80         }
81         
82         private static void log(String line) {
83                 if (LOG) {
84                         try {
85                                 log.writeUTF(line + "\n");
86                         } catch (IOException e) {
87                                 e.printStackTrace();
88                         }
89                 }
90         }
91
92         Resource indexRoot;
93         TransferableGraphSource tg;
94         VirtualGraph vg;
95         IImportAdvisor2 advisor;
96         ClusterBuilder2 builder;
97         final TGResourceUtil resourceUtil = new TGResourceUtil();
98         
99         int[] handles;
100         
101         Set<String> missingExternals = new HashSet<String>(); 
102         
103         int resourceCount;
104         Identity[] identities;
105         TreeMap<String, Variant> extensions;
106         
107         // Builtins
108         Resource RootLibrary;
109         Resource String;
110         Resource Library;
111         
112         Resource InstanceOf;
113         Resource ConsistsOf;
114         Resource PartOf;
115         Resource HasName;
116         Resource NameOf;        
117                                 
118         public StreamingTransferableGraphImportProcess(Session session, VirtualGraph vg, TransferableGraphSource tg, IImportAdvisor2 advisor) {
119                 this.tg = tg;
120                 this.vg = vg;
121                 this.advisor = advisor;
122         }
123         
124         public void readIdentities(ReadGraph g) throws Exception {
125                 extensions = tg.getExtensions();
126                 resourceCount = tg.getResourceCount();
127                 identities = new Identity[tg.getIdentityCount()];
128                 tg.forIdentities(g, new TransferableGraphSourceProcedure<Identity>() {
129                         
130                         int counter = 0;
131                         
132                         @Override
133                         public void execute(Identity value) throws Exception {
134                                 identities[counter++] = value;
135                         }
136                 });
137         }
138         
139         public void findBuiltins(WriteOnlyGraph g) throws DatabaseException {
140                 RootLibrary = g.getBuiltin("http:/");
141                 String = g.getBuiltin(CoreInitialization.LAYER0 + "String");
142                 Library = g.getBuiltin(CoreInitialization.LAYER0 + "Library");
143                 InstanceOf = g.getBuiltin(CoreInitialization.LAYER0 + "InstanceOf");
144                 ConsistsOf = g.getBuiltin(CoreInitialization.LAYER0 + "ConsistsOf");
145                 PartOf = g.getBuiltin(CoreInitialization.LAYER0 + "PartOf");
146                 HasName = g.getBuiltin(CoreInitialization.LAYER0 + "HasName");
147                 NameOf = g.getBuiltin(CoreInitialization.LAYER0 + "NameOf");
148         }
149         
150         public void findBuiltins(ReadGraph g) throws DatabaseException {
151                 RootLibrary = g.getBuiltin("http:/");
152                 String = g.getBuiltin(CoreInitialization.LAYER0 + "String");
153                 Library = g.getBuiltin(CoreInitialization.LAYER0 + "Library");
154                 InstanceOf = g.getBuiltin(CoreInitialization.LAYER0 + "InstanceOf");
155                 ConsistsOf = g.getBuiltin(CoreInitialization.LAYER0 + "ConsistsOf");
156                 PartOf = g.getBuiltin(CoreInitialization.LAYER0 + "PartOf");
157                 HasName = g.getBuiltin(CoreInitialization.LAYER0 + "HasName");
158                 NameOf = g.getBuiltin(CoreInitialization.LAYER0 + "NameOf");
159         }
160
161 //      /* Preparation that is used when the core is empty. 
162 //       */
163 //      void initialPrepare(WriteOnlyGraph graph) throws DatabaseException {
164 //              findBuiltins(graph);
165 //              
166 //              resources = new Resource[tg.resourceCount];
167 //              
168 //              int Root = -1;
169 //              int SimanticsDomain = -1;
170 //              int Layer0 = -1;
171 //              
172 //              for(Identity identity : tg.identities) {
173 //                      if(identity.definition instanceof Internal) {
174 //                              Internal def = (Internal)identity.definition;
175 //                              Resource res = null;
176 //                              if(def.parent == Layer0) {
177 //                                      try {
178 //                                              res = graph.getBuiltin(CoreInitialization.LAYER0 + def.name);
179 //                                      } catch(ResourceNotFoundException e) {                                                                          
180 //                                      }
181 //                              }
182 //                              else if(def.parent == SimanticsDomain) {
183 //                                      if(def.name.equals("Layer0-1.0"))
184 //                                              Layer0 = identity.resource;
185 //                              }
186 //                              else if(def.parent == Root) {
187 //                                      if(def.name.equals("www.simantics.org"))
188 //                                              SimanticsDomain = identity.resource;
189 //                              }
190 //
191 //                              if(res == null)
192 //                                      res = createChild(graph, resources[def.parent], def.name);
193 //                              else
194 //                                      createChild(graph, res, resources[def.parent], def.name);
195 //                              resources[identity.resource] = res;
196 //                      }
197 //                      else if(identity.definition instanceof Root) {
198 //                              Root = identity.resource;
199 //                              resources[identity.resource] = RootLibrary;                             
200 //                      } 
201 //              }
202 //      }
203         
204         void addMissing(String external) {
205                 Set<String> removals = new HashSet<String>();
206                 for(String ext : missingExternals) if(ext.startsWith(external)) return;
207                 for(String ext : missingExternals) if(external.startsWith(ext)) removals.add(ext);
208                 missingExternals.removeAll(removals);
209                 missingExternals.add(external);
210         }
211         
212         void prepare(ReadGraph graph) throws Exception {
213
214                 Resource target = advisor.getTarget();
215                 if(target != null)
216                         indexRoot = graph.syncRequest(new PossibleIndexRoot(target));
217                 
218                 findBuiltins(graph);
219                 readIdentities(graph);
220                 
221 //              System.err.println("ext: " + extensions);
222 //              System.err.println("rc: " + resourceCount);
223 //              System.err.println("ic: " + identities.length);
224                 
225                 ClusterBuilderFactory factory = graph.getService(ClusterBuilderFactory.class);
226                 ClusterBuilder2 builder = factory.create(vg, false);
227                 
228         this.handles = new int[resourceCount];
229                 
230                 for(Identity identity : identities) {
231                         IdentityDefinition definition = identity.definition;
232                         if(definition instanceof External) {
233                                 External def = (External)definition;
234                                 if(def.parent == -1) {
235                                     handles[identity.resource] = builder.handle(RootLibrary);
236                                 } else {
237                                         if("@inverse".equals(def.name)) {
238                                                 int parent = handles[def.parent];
239                                                 int child = builder.handle(graph.getInverse(builder.resource(parent)));
240                                                 handles[identity.resource] = child;
241                                         } else {
242                                                 int handle = handles[def.parent];
243                                                 Resource parent = handle != 0 ? builder.resource(handle) : null;
244                                                 // TODO: escape should be removed when names become well-behaving
245                                                 if(parent != null) {
246                                                     Map<String,Resource> childMap = graph
247                                             .syncRequest(new UnescapedChildMapOfResource(parent),
248                                                     new TransientCacheAsyncListener<Map<String, Resource>>()); 
249                                                         Resource child = childMap.get(def.name); 
250                                                         if(child == null) {
251                                                                 addMissing(graph.getURI(parent) + "/" + def.name);
252                                                         } else {
253                                                                 handles[identity.resource] = builder.handle(child);
254                                                         }
255                                                 } else {
256                                                     addMissing(TransferableGraphUtils.getURI(resourceCount, identities, def.parent) + "/" + def.name);
257                                                 }
258                                         }
259                                 }
260                         }
261                         else if(definition instanceof Internal) {
262                                 // Do not do anything for now
263                         }
264                         else if(definition instanceof Root) {
265                                 Root root = (Root)definition;
266                                 if(root.name.equals(""))
267                                     handles[identity.resource] = builder.handle(RootLibrary);
268                                 else  {
269                                         Resource existing = advisor.analyzeRoot(graph, root);
270                                         if(existing != null)
271                                             handles[identity.resource] = builder.handle(existing);
272                                 }
273                         }
274                         else if(definition instanceof Optional) {
275                                 External def = (External)definition;
276                                 Resource parent = builder.resource(handles[def.parent]);
277                                 if(parent != null)
278                                         handles[identity.resource] = builder.handle(graph.syncRequest(new UnescapedChildMapOfResource(parent)).get(def.name));                          
279                         }
280                 }               
281                 
282                 if(!missingExternals.isEmpty()) throw new MissingDependencyException(this);
283                 
284         }
285
286         @Override
287         public Resource createChild(WriteOnlyGraph graph, Resource parent, Resource child, String name) throws DatabaseException {
288             if(child == null) child = graph.newResource();
289                 Resource nameResource = graph.newResource();
290                 graph.claim(nameResource, InstanceOf, null, String);
291                 graph.claimValue(nameResource, name, WriteBindings.STRING);
292                 graph.claim(child, HasName, NameOf, nameResource);
293                 return child;
294         }
295         
296         int[] getClustering() {
297                 Variant v = extensions.get(Extensions.CLUSTERING);
298                 if(v == null) return null;
299                 try {
300                         return (int[])v.getValue(Bindings.INT_ARRAY);
301                 } catch (AdaptException e) {
302                         Logger.defaultLogError(e);
303                         return null;
304                 }
305         }
306
307         int[] getClusterSets() {
308                 Variant v = extensions.get(Extensions.CLUSTER_SETS);
309                 if(v == null) return null;
310                 try {
311                         return (int[])v.getValue(Bindings.INT_ARRAY);
312                 } catch (AdaptException e) {
313                         Logger.defaultLogError(e);
314                         return null;
315                 }
316         }
317
318         boolean needTranslation(Datatype type) {
319             return resourceUtil.mayHaveResource(type);
320         }
321         
322         void findClusterSet(WriteOnlyGraph graph, Resource rootLibrary, int[] clustering, int[] clusterSets, long[] clusters, int id) throws DatabaseException {
323                 ClusteringSupport support = graph.getService(ClusteringSupport.class);
324                 if(id == Extensions.ROOT_LIBRARY_CLUSTER_SET || id == Extensions.INDEX_ROOT_CLUSTER_SET) return;
325                 for(int pos=0,index=0;index<clustering.length;index++) {
326                         pos += clustering[index];
327                         if(id < pos) {
328                                 int cs = clusterSets[index]; 
329                                 if(handles[id] == 0) {
330                                         int csHandle = 0;
331                                         if(cs == Extensions.ROOT_LIBRARY_CLUSTER_SET) csHandle = builder.handle(rootLibrary);
332                                         else if(cs == Extensions.INDEX_ROOT_CLUSTER_SET) {
333                                                 if(indexRoot == null) throw new DatabaseException("No index root was available in TG import.");
334                                                 csHandle = builder.handle(indexRoot);
335                                         }
336                                         else {
337                                                 findClusterSet(graph, rootLibrary, clustering, clusterSets, clusters, cs);
338                                                 csHandle = handles[cs];
339                                         }
340                                         
341                                         if(clusters[index] != 0)
342                                                 builder.selectCluster(clusters[index]);
343                                         else if(cs >= 0)
344                                                 builder.newCluster(csHandle);
345                                         
346                                         handles[id] = builder.newResource(csHandle);
347                                         clusters[index] = support.getCluster(builder.resource(handles[id]));
348                                                         
349                                         builder.createClusterSet(handles[id]);
350                                 }
351                                 return;
352                         }
353                 }
354         }
355         
356         void write(final WriteOnlyGraph graph) throws Exception {
357         
358         final SerialisationSupport ss = graph.getService(SerialisationSupport.class);
359             
360         ClusterBuilderFactory factory = graph.getService(ClusterBuilderFactory.class);
361         if(advisor instanceof IImportAdvisor2) {
362             boolean allowImmutable = ((IImportAdvisor2)advisor).allowImmutableModifications();
363             builder = factory.create(vg, allowImmutable);
364         } else {
365             builder = factory.create(vg, false);
366         }
367         
368                 final int[] handles = this.handles; 
369                 
370                 int[] clustering = getClustering();
371                 if(clustering != null) {
372                         
373                         int[] clusterSets = getClusterSets();
374                         if(clusterSets != null) {
375
376                                 assert(clustering.length == clusterSets.length);
377
378                                 long[] clusters = new long[clustering.length];
379                                 
380                                 // Create clustering
381                                 for(int i=0;i<clusterSets.length;i++) {
382                                         findClusterSet(graph, graph.getRootLibrary(), clustering, clusterSets, clusters, clusterSets[i]);
383                                 }
384                                 
385                                 // Then create all resources
386                                 int i=0;
387                             for(int j=0;j<clustering.length;j++) {
388                                 int c = clustering[j];
389                                 int s = clusterSets[j];
390                                 int setHandle = 0;
391                                         if(s == Extensions.ROOT_LIBRARY_CLUSTER_SET)
392                                                 setHandle = builder.handle(graph.getRootLibrary());
393                                         else if(s == Extensions.INDEX_ROOT_CLUSTER_SET) {
394                                                 if(indexRoot == null) throw new DatabaseException("No index root was available in TG import.");
395                                                 setHandle = builder.handle(indexRoot);
396                                         }
397                                         else setHandle = handles[s];
398                                         // Preserve clustering only for internal resources
399                                         if(clusters[j] != 0)
400                                                 builder.selectCluster(clusters[j]);
401                                         else if(s >= 0)
402                                                 builder.newCluster(setHandle);
403                                         for(int r=0;r<c;r++, i++)
404                                                 if(handles[i] == 0) handles[i] = builder.newResource();
405                                 }
406
407                                 for(;i<handles.length;++i)
408                                         if(handles[i] == 0) handles[i] = builder.newResource();
409                                 
410                         } else {
411
412                         int i = 0;
413                                 for(int c : clustering) {
414                                         builder.newCluster();
415                                         for(int r=0;r<c;r++, i++)
416                                                 if(handles[i] == 0) handles[i] = builder.newResource();
417                                 }
418
419                                 for(;i<handles.length;++i)
420                                         if(handles[i] == 0) handles[i] = builder.newResource();
421                                 
422                         }
423                         
424                 } else {
425                 
426                         // Create blank resources
427                         for(int i=0;i<handles.length;++i)
428                                 if(handles[i] == 0) handles[i] = builder.newResource();
429
430                 }
431                 
432                 // Internal identities          
433                 for(Identity identity : identities) {
434                         IdentityDefinition definition = identity.definition;
435 //                      if(handles[identity.resource] != 0)
436 //                              continue;
437                         if(definition instanceof External) {
438                                 // Already done everything
439                         }
440                         else if(definition instanceof Internal) {
441                                 Internal def = (Internal)definition;
442                                 if(handles[identity.resource] != 0)
443                                         handles[identity.resource] = builder.handle(advisor.createChild(graph, this, builder.resource(handles[def.parent]), builder.resource(handles[identity.resource]), def.name));
444                                 else
445                                         handles[identity.resource] = builder.handle(advisor.createChild(graph, this, builder.resource(handles[def.parent]), null, def.name));
446                         }
447                         else if(definition instanceof Root) {
448                                 
449                                 Root root = (Root)definition;
450                                 if(handles[identity.resource] != 0)
451                                         handles[identity.resource] = builder.handle(advisor.createRoot(graph, root, builder.resource(handles[identity.resource])));
452                                 else
453                                         handles[identity.resource] = builder.handle(advisor.createRoot(graph, root, null));
454                         }
455                         else if(definition instanceof Optional) {
456                                 Optional def = (Optional)definition;
457                                 if(handles[identity.resource] != 0) {
458                                         Resource child = advisor.createChild(graph, this, builder.resource(handles[def.parent]), builder.resource(handles[identity.resource]), def.name);
459                                         graph.claim(child, InstanceOf, null, Library); // ???
460                                         handles[identity.resource] = builder.handle(child);
461                                 } else {
462                                         Resource child = advisor.createChild(graph, this, builder.resource(handles[def.parent]), null, def.name);
463                                         graph.claim(child, InstanceOf, null, Library); // ???
464                                         handles[identity.resource] = builder.handle(child);
465                                 }
466                         }
467                 }               
468                 
469                 tg.getStatementCount();
470                 tg.forStatements(null, new TransferableGraphSourceProcedure<int[]>() {
471
472                         @Override
473                         public void execute(int[] value) throws Exception {
474                                 
475                                 int sub = value[0];
476                                 int pred = value[1];
477                                 int inv = value[2];
478                                 int obj = value[3];
479
480                                 int subject = handles[sub];
481                                 int predicate = handles[pred];
482                                 int object = handles[obj];
483
484                                 builder.addStatement(graph, subject, predicate, object);        
485                                 if(inv >= 0) {
486                                     int inverse = handles[inv];
487                                     builder.addStatement(graph, object, inverse, subject);    
488                                 }
489                                 
490                         }
491                         
492                 }); 
493                 
494                 tg.getValueCount();
495
496                 class ValueProcedure extends InputStream implements TransferableGraphSourceValueProcedure {
497
498             private TGResourceUtil util = new TGResourceUtil();
499                     private DataInput source;
500
501             @Override
502             public void execute(int _resource, Datatype type, DataInput stream) throws Exception {
503
504                 source = stream;
505
506                 //int file = _resource & 0x80000000;
507                 int resource = _resource & 0x7FFFFFFF;
508
509                 Binding binding = Bindings.getBinding(type);
510                 Serializer s = Bindings.getSerializer(binding);
511
512                 builder.beginValue(handles[resource]);
513                 if(util.mayHaveResource(type)) {
514                     Object value = s.deserialize(stream);
515                     util.adaptValue( binding,  value, new LongAdapter() {
516                                 @Override
517                                 public long adapt(long in) {
518                                     try {
519                                         return ss.getRandomAccessId(handles[(int)in]);
520                                     } catch (DatabaseException e) {
521                                         throw new IllegalStateException(e);
522                                     }
523                                 }
524                             });
525                     byte[] bytes = s.serialize(value);
526                     for(byte b : bytes) {
527                         int val = b;
528                         if(val < 0) val += 256;
529                         builder.appendValue(val);
530                     }
531                 } else {
532                     s.skip(this);
533                 }
534                 builder.endValue();
535                 
536             }
537
538             @Override
539             public int read() throws IOException {
540                 int value = source.readUnsignedByte();
541                 try {
542                     builder.appendValue(value);
543                 } catch (DatabaseException e) {
544                     e.printStackTrace();
545                 }
546                 return value;
547             }
548
549             @Override
550             public void rawCopy(int resource, int length, DataInput input) throws Exception {
551                 builder.beginValue(handles[resource]);
552                 for (int i = 0; i < length; ++i)
553                     builder.appendValue(input.readUnsignedByte());
554                 builder.endValue();
555             }
556
557                 };
558                 
559                 tg.forValues2(null, new ValueProcedure());
560                 
561         }
562         
563         @Override
564         public long[] getResourceIds(SerialisationSupport serializer) throws DatabaseException {
565                 final int count = handles.length;
566                 long[] resourceIds = new long[count];
567                 for(int i=0;i<count;++i)
568                     resourceIds[i] = serializer.getRandomAccessId(handles[i]);
569                 return resourceIds;
570         }
571 }