4ec7138660fc2dd8c585fefe07bceb96bc4b5c00
[simantics/platform.git] / bundles / org.simantics.graph.db / src / org / simantics / graph / db / TransferableGraphImportProcess.java
1 package org.simantics.graph.db;
2
3 import java.io.DataOutput;
4 import java.io.DataOutputStream;
5 import java.io.FileNotFoundException;
6 import java.io.FileOutputStream;
7 import java.io.IOException;
8 import java.util.HashSet;
9 import java.util.Map;
10 import java.util.Set;
11
12 import org.simantics.databoard.Bindings;
13 import org.simantics.databoard.accessor.error.AccessorException;
14 import org.simantics.databoard.adapter.AdaptException;
15 import org.simantics.databoard.binding.mutable.Variant;
16 import org.simantics.databoard.util.URIStringUtils;
17 import org.simantics.db.ReadGraph;
18 import org.simantics.db.Resource;
19 import org.simantics.db.WriteOnlyGraph;
20 import org.simantics.db.common.WriteBindings;
21 import org.simantics.db.common.procedure.adapter.TransientCacheAsyncListener;
22 import org.simantics.db.common.uri.UnescapedChildMapOfResource;
23 import org.simantics.db.common.utils.Logger;
24 import org.simantics.db.common.utils.NameUtils;
25 import org.simantics.db.exception.DatabaseException;
26 import org.simantics.db.exception.ResourceNotFoundException;
27 import org.simantics.db.service.ClusterBuilder;
28 import org.simantics.db.service.ClusterBuilder.ResourceHandle;
29 import org.simantics.db.service.SerialisationSupport;
30 import org.simantics.graph.representation.Extensions;
31 import org.simantics.graph.representation.External;
32 import org.simantics.graph.representation.Identity;
33 import org.simantics.graph.representation.IdentityDefinition;
34 import org.simantics.graph.representation.Internal;
35 import org.simantics.graph.representation.Optional;
36 import org.simantics.graph.representation.Root;
37 import org.simantics.graph.representation.TransferableGraph1;
38 import org.simantics.graph.representation.TransferableGraphUtils;
39 import org.simantics.graph.representation.Value;
40 import org.simantics.graph.utils.TGResourceUtil;
41 import org.simantics.graph.utils.TGResourceUtil.LongAdapter;
42
43 public class TransferableGraphImportProcess implements TransferableGraphImporter {
44         
45         public static String LOG_FILE = "transferableGraphs.log";
46         final static private boolean LOG = false;
47         
48         static DataOutput log;
49
50         static {
51
52                 if (LOG) {
53                         try {
54                                 FileOutputStream stream = new FileOutputStream(LOG_FILE);
55                                 log = new DataOutputStream(stream);
56                         } catch (FileNotFoundException e) {
57                                 e.printStackTrace();
58                         }
59                 }
60
61         }
62         
63         private static void log(String line) {
64                 if (LOG) {
65                         try {
66                                 log.writeUTF(line + "\n");
67                         } catch (IOException e) {
68                                 e.printStackTrace();
69                         }
70                 }
71         }
72         
73         TransferableGraph1 tg;
74         IImportAdvisor advisor;
75         TGStatusMonitor monitor;
76         final TGResourceUtil resourceUtil = new TGResourceUtil();
77         
78         Resource[] resources;
79         ResourceHandle[] handles;
80         
81         Set<String> missingExternals = new HashSet<String>(); 
82         
83         // Builtins
84         Resource RootLibrary;
85         Resource String;
86         Resource Library;
87         
88         Resource InstanceOf;
89         Resource ConsistsOf;
90         Resource PartOf;
91         Resource HasName;
92         Resource NameOf;        
93                                 
94         public TransferableGraphImportProcess(TransferableGraph1 tg, IImportAdvisor advisor, TGStatusMonitor monitor) {
95                 this.tg = tg;
96                 this.advisor = advisor;
97                 this.monitor = monitor;
98         }
99         
100     public TransferableGraphImportProcess(TransferableGraph1 tg, IImportAdvisor advisor) {
101         this(tg, advisor, null);
102     }
103
104         public void findBuiltins(WriteOnlyGraph g) throws DatabaseException {
105                 RootLibrary = g.getBuiltin("http:/");
106                 String = g.getBuiltin(CoreInitialization.LAYER0 + "String");
107                 Library = g.getBuiltin(CoreInitialization.LAYER0 + "Library");
108                 InstanceOf = g.getBuiltin(CoreInitialization.LAYER0 + "InstanceOf");
109                 ConsistsOf = g.getBuiltin(CoreInitialization.LAYER0 + "ConsistsOf");
110                 PartOf = g.getBuiltin(CoreInitialization.LAYER0 + "PartOf");
111                 HasName = g.getBuiltin(CoreInitialization.LAYER0 + "HasName");
112                 NameOf = g.getBuiltin(CoreInitialization.LAYER0 + "NameOf");
113         }
114         
115         public void findBuiltins(ReadGraph g) throws DatabaseException {
116                 RootLibrary = g.getBuiltin("http:/");
117                 String = g.getBuiltin(CoreInitialization.LAYER0 + "String");
118                 Library = g.getBuiltin(CoreInitialization.LAYER0 + "Library");
119                 InstanceOf = g.getBuiltin(CoreInitialization.LAYER0 + "InstanceOf");
120                 ConsistsOf = g.getBuiltin(CoreInitialization.LAYER0 + "ConsistsOf");
121                 PartOf = g.getBuiltin(CoreInitialization.LAYER0 + "PartOf");
122                 HasName = g.getBuiltin(CoreInitialization.LAYER0 + "HasName");
123                 NameOf = g.getBuiltin(CoreInitialization.LAYER0 + "NameOf");
124         }
125
126         /* Preparation that is used when the core is empty. 
127          */
128         void initialPrepare(WriteOnlyGraph graph) throws DatabaseException {
129                 findBuiltins(graph);
130                 
131                 resources = new Resource[tg.resourceCount];
132                 
133                 int Root = -1;
134                 int SimanticsDomain = -1;
135                 int Layer0 = -1;
136                 
137                 for(Identity identity : tg.identities) {
138                         if(identity.definition instanceof Internal) {
139                                 Internal def = (Internal)identity.definition;
140                                 Resource res = null;
141                                 if(def.parent == Layer0) {
142                                         try {
143                                                 res = graph.getBuiltin(CoreInitialization.LAYER0 + def.name);
144                                         } catch(ResourceNotFoundException e) {                                                                          
145                                         }
146                                 }
147                                 else if(def.parent == SimanticsDomain) {
148                                         if(def.name.equals("Layer0-1.1"))
149                                                 Layer0 = identity.resource;
150                                 }
151                                 else if(def.parent == Root) {
152                                         if(def.name.equals("www.simantics.org"))
153                                                 SimanticsDomain = identity.resource;
154                                 }
155
156                                 if(res == null)
157                                         res = createChild(graph, resources[def.parent], def.name);
158                                 else
159                                         createChild(graph, res, resources[def.parent], def.name);
160                                 resources[identity.resource] = res;
161                         }
162                         else if(identity.definition instanceof Root) {
163                                 Root = identity.resource;
164                                 resources[identity.resource] = RootLibrary;                             
165                         } 
166                 }
167         }
168         
169         void addMissing(String external) {
170                 Set<String> removals = new HashSet<String>();
171                 for(String ext : missingExternals) if(ext.startsWith(external)) return;
172                 for(String ext : missingExternals) if(external.startsWith(ext)) removals.add(ext);
173                 missingExternals.removeAll(removals);
174                 missingExternals.add(external);
175         }
176         
177         void prepare(ReadGraph graph) throws DatabaseException {
178                 findBuiltins(graph);
179                 
180                 Resource[] resources = new Resource[tg.resourceCount];
181                 
182                 for(Identity identity : tg.identities) {
183                         IdentityDefinition definition = identity.definition;
184                         if(definition instanceof External) {
185                                 External def = (External)definition;
186                                 if(def.parent == -1) {
187                                         resources[identity.resource] = RootLibrary;
188                                 } else {
189                                         if("@inverse".equals(def.name)) {
190                                                 Resource parent = resources[def.parent];
191                                                 Resource child = graph.getInverse(parent);
192                                                 resources[identity.resource] = child;
193                                         } else {
194                                                 Resource parent = resources[def.parent];
195                                                 // TODO: escape should be removed when names become well-behaving
196                                                 if(parent != null) {
197                                                         Resource child = graph
198                                                         .syncRequest(new UnescapedChildMapOfResource(parent),
199                                                                 new TransientCacheAsyncListener<Map<String, Resource>>())
200                                                                 .get(def.name);
201                                                         if(child == null) {
202                                                                 String uri = graph.getPossibleURI(parent);
203                                                                 if(uri == null) {
204                                                                         addMissing(URIStringUtils.escape(NameUtils.getSafeName(graph, parent)) + " /" + URIStringUtils.escape(def.name));
205                                                                 } else {
206                                                                         addMissing(graph.getURI(parent) + "/" + URIStringUtils.escape(def.name));
207                                                                 }
208                                                         }
209                                                         resources[identity.resource] = child;
210                                                 } else {
211                                                     addMissing(TransferableGraphUtils.getURI(tg, def.parent) + "/" + URIStringUtils.escape(def.name));
212                                                 }
213                                         }
214                                 }
215                         }
216                         else if(definition instanceof Internal) {
217                                 // Do not do anything for now
218                         }
219                         else if(definition instanceof Root) {
220                                 Root root = (Root)definition;
221                                 if(root.name.equals(""))
222                                         resources[identity.resource] = RootLibrary;
223                                 else {
224                                         Resource existing = advisor.analyzeRoot(graph, root);
225                                         if(existing != null)
226                                                 resources[identity.resource] = existing;
227                                 }
228                         }
229                         else if(definition instanceof Optional) {
230                                 External def = (External)definition;
231                                 Resource parent = resources[def.parent];
232                                 if(parent != null)
233                                         resources[identity.resource] = 
234                                                 graph.syncRequest(new UnescapedChildMapOfResource(parent)).get(def.name);                               
235                         }
236                 }               
237                 
238                 this.resources = resources;
239                 
240                 if(!missingExternals.isEmpty()) throw new MissingDependencyException(this);
241                 
242         }
243
244         Resource createChild(WriteOnlyGraph graph, Resource parent, String name) throws DatabaseException {
245                 Resource child = graph.newResource();
246                 //graph.claim(parent, ConsistsOf, PartOf, child);
247                 Resource nameResource = graph.newResource();
248                 graph.claim(nameResource, InstanceOf, null, String);
249                 graph.claimValue(nameResource, name, WriteBindings.STRING);
250                 graph.claim(child, HasName, NameOf, nameResource);
251                 return child;
252         }
253
254         @Override
255         public Resource createChild(WriteOnlyGraph graph, Resource child, Resource parent, String name) throws DatabaseException {
256                 graph.claim(parent, ConsistsOf, PartOf, child);
257                 Resource nameResource = graph.newResource();
258                 graph.claim(nameResource, InstanceOf, null, String);
259                 graph.claimValue(nameResource, name, WriteBindings.STRING);
260                 graph.claim(child, HasName, NameOf, nameResource);
261                 return child;
262         }
263         
264         int[] getClustering() {
265                 Variant v = tg.extensions.get(Extensions.CLUSTERING);
266                 if(v == null) return null;
267                 try {
268                         return (int[])v.getValue(Bindings.INT_ARRAY);
269                 } catch (AdaptException e) {
270                         Logger.defaultLogError(e);
271                         return null;
272                 }
273         }
274
275         class ResourceAdapter implements LongAdapter {
276             final SerialisationSupport ss;
277             final Value value;
278             Variant original;
279             ResourceAdapter(SerialisationSupport ss, Value value) {
280                 this.ss = ss;
281                 this.value = value;
282             }
283         @Override
284         public long adapt(long in) {
285             if(original == null) original = value.value.clone();
286             Resource res = handles[(int)in].resource(ss);
287             // Maybe throw?
288             if(res == null) return in;
289             return res.getResourceId();
290         }           
291         }
292         
293         Variant translate(SerialisationSupport ss, Value value, final ResourceHandle[] handles) throws DatabaseException {
294                 
295                 try {
296                     ResourceAdapter adapter = new ResourceAdapter(ss, value);
297                         resourceUtil.adaptValue( value.value.getBinding(), value.value.getValue(), adapter);
298                         Variant result = value.value;
299                         if(adapter.original != null) value.value = adapter.original;
300                         return result;
301                 } catch (AccessorException e) {
302                         e.printStackTrace();
303                 }               
304                 
305                 return value.value;
306                 
307         }
308         
309     class ResourceAdapter2 implements LongAdapter {
310         final Value value;
311         Variant original;
312         ResourceAdapter2(Value value) {
313             this.value = value;
314         }
315         @Override
316         public long adapt(long in) {
317             if(original == null) original = value.value.clone();
318             Resource res = resources[(int)in];
319             // Maybe throw?
320             if(res == null) return in;
321             return res.getResourceId();
322         }           
323     }
324
325         Variant translate2(Value value, final Resource[] resources) throws DatabaseException {
326                 
327         try {
328             ResourceAdapter2 adapter = new ResourceAdapter2(value);
329             resourceUtil.adaptValue( value.value.getBinding(), value.value.getValue(), adapter);
330             Variant result = value.value;
331             if(adapter.original != null) value.value = adapter.original;
332             return result;
333         } catch (AccessorException e) {
334             e.printStackTrace();
335         }       
336         
337         return value.value;
338                 
339         }
340
341         void write(WriteOnlyGraph graph) throws DatabaseException {
342
343                 Resource[] resources = this.resources;
344
345                 this.handles = new ResourceHandle[resources.length];
346                 
347                 ResourceHandle[] handles = this.handles; 
348
349                 int[] clustering = getClustering();
350
351                 // Internal identities          
352                 for(Identity identity : tg.identities) {
353                         IdentityDefinition definition = identity.definition;
354                         if(resources[identity.resource] != null)
355                                 continue;
356                         if(definition instanceof External) {
357                                 // Already done everything
358                         }
359                         else if(definition instanceof Internal) {
360                                 Internal def = (Internal)definition;
361                                 resources[identity.resource] = 
362                                         createChild(graph, resources[def.parent], def.name);
363                         }
364                         else if(definition instanceof Root) {
365                                 Root root = (Root)definition;                           
366                                 resources[identity.resource] = advisor.createRoot(graph, root);                                 
367                         }
368                         else if(definition instanceof Optional) {
369                                 Optional def = (Optional)definition;
370                                 Resource child = createChild(graph, resources[def.parent], def.name);
371                                 graph.claim(child, InstanceOf, null, Library); // ???
372                                 resources[identity.resource] = child;                                   
373                         }
374                 }               
375
376                 ClusterBuilder builder = graph.getService(ClusterBuilder.class);
377                 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
378
379                 if(clustering != null) {
380                         
381                 int i = 0;
382                         for(int c : clustering) {
383                                 builder.newCluster();
384                                 for(int r=0;r<c;r++, i++)
385                                         if(resources[i] == null)
386                                                 handles[i] = builder.newResource();
387                                         else 
388                                                 handles[i] = builder.resource(resources[i]);
389
390                         }
391
392                         for(;i<resources.length;++i)
393                                 if(resources[i] == null)
394                                         handles[i] = builder.newResource();
395                                 else 
396                                         handles[i] = builder.resource(resources[i]);
397                         
398                 } else {
399                 
400                         // Create blank resources
401                         for(int i=0;i<resources.length;++i)
402                                 if(resources[i] == null)
403                                         handles[i] = builder.newResource();
404                                 else 
405                                         handles[i] = builder.resource(resources[i]);
406
407                 }
408                 
409                 // Write statements
410                 int[] statements = tg.statements;
411                 
412 //              int internals = tg.resourceCount - tg.identities.length;
413                 
414                 for(int i=0;i<statements.length;i+=4) {
415
416                         int sub = statements[i];
417                         int pred = statements[i+1];
418                         int inv = statements[i+2];
419                         int obj = statements[i+3];
420
421                         ResourceHandle subject = handles[sub];
422                         ResourceHandle predicate = handles[pred];
423                         ResourceHandle object = handles[obj];
424
425                         if(resources[sub] == null) {
426                                 subject.addStatement(graph, predicate, object); 
427                         } else {
428                                 graph.claim(
429                                                 handles[sub].resource(ss),
430                                                 handles[pred].resource(ss),
431                                                 null, handles[obj].resource(ss));
432                         }
433                         
434                         if(inv >= 0) {
435                                 
436                                 if(resources[obj] == null) {
437                                         ResourceHandle inverse = handles[inv];
438                                         object.addStatement(graph, inverse, subject);   
439                                 } else {
440                                     Resource s = handles[obj].resource(ss);
441                                     if(!graph.isImmutable(s))
442                                         graph.claim(
443                                                         s,
444                                                         handles[inv].resource(ss),
445                                                         null, handles[sub].resource(ss));
446                                 }
447                                 
448                         }
449                         
450                         if(LOG) {
451                                 log("[STATEMENT] " + resources[statements[i]].getResourceId() + ", " + resources[statements[i+1]].getResourceId() + ", " + resources[statements[i+3]].getResourceId());
452                         }
453                         
454                 }
455                                 
456                 // Write values
457                 for(Value value : tg.values) {
458                         Variant variant = translate(ss, value, handles);
459                         int file = value.resource & 0x80000000;
460                         int resource = value.resource & 0x7FFFFFFF;
461                         if (file != 0) {
462                             throw new UnsupportedOperationException();
463                                 //graph.claimValue(handles[resource].resource(), value.value, Bindings.BYTE_ARRAY);
464                         } else {
465                                 graph.claimValue(handles[resource].resource(ss), variant.getValue(), variant.getBinding());
466                         }
467                 }
468         }
469
470         private int updatePercentage(int percentage, int done, int total) {
471             if(monitor != null && (done & 63) == 0) {
472                 int current = 100*done / total;
473                 if(current > percentage) {
474                     percentage = current;
475                     monitor.status(percentage);
476                 }
477             }
478         return percentage;
479         }
480         
481         void write2(WriteOnlyGraph graph) throws DatabaseException {
482                 Resource[] resources = this.resources;
483                 
484                 // Internal identities          
485                 for(Identity identity : tg.identities) {
486                         IdentityDefinition definition = identity.definition;
487                         if(resources[identity.resource] != null)
488                                 continue;
489                         if(definition instanceof External) {
490                                 // Already done everything
491                         }
492                         else if(definition instanceof Internal) {
493                                 Internal def = (Internal)definition;
494                                 resources[identity.resource] = 
495                                         createChild(graph, resources[def.parent], def.name);
496                         }
497                         else if(definition instanceof Root) {                           
498                                 Root root = (Root)definition;                           
499                                 resources[identity.resource] = advisor.createRoot(graph, root);                                 
500                         }
501                         else if(definition instanceof Optional) {
502                                 Optional def = (Optional)definition;
503                                 Resource child = createChild(graph, resources[def.parent], def.name);
504                                 graph.claim(child, InstanceOf, null, Library); // ???
505                                 resources[identity.resource] = child;                                   
506                         }
507                 }               
508                 
509                 // Create blank resources
510                 for(int i=0;i<resources.length;++i)
511                         if(resources[i] == null)
512                                 resources[i] = graph.newResource();
513                 
514                 int total = tg.statements.length + tg.values.length;
515                 int done = 0;
516                 int percentage = 0;
517                 
518                 // Write statements
519                 int[] statements = tg.statements;
520                 
521                 for(int i=0;i<statements.length;i+=4) {
522                         int inv = statements[i+2];
523                         graph.claim(
524                                         resources[statements[i]],
525                                         resources[statements[i+1]],
526                                         inv < 0 ? null : resources[inv],
527                                         resources[statements[i+3]]
528                                         );
529                         if(LOG) {
530                                 log("[STATEMENT] " + resources[statements[i]].getResourceId() + ", " + resources[statements[i+1]].getResourceId() + ", " + resources[statements[i+3]].getResourceId());
531                         }
532                         percentage = updatePercentage(percentage, done++, total);
533                 }
534
535                 // Write values
536                 for(Value value : tg.values) {
537                         Variant variant = translate2(value, resources);
538                         int file = value.resource & 0x80000000;
539                         int resource = value.resource & 0x7FFFFFFF;
540                         if (file != 0) {
541                             throw new UnsupportedOperationException();
542                                 //graph.claimValue(resources[resource], value.value, Bindings.BYTE_ARRAY);
543                         } else {
544                             graph.claimValue(resources[resource], variant.getValue(), variant.getBinding());
545                         }
546             percentage = updatePercentage(percentage, done++, total);
547                 }
548                 
549         }
550         
551         public long[] getResourceIds(SerialisationSupport serializer) throws DatabaseException {
552                 final int count = resources.length;
553                 long[] resourceIds = new long[count];
554                 if(handles != null) {
555                         for(int i=0;i<count;++i)
556                                 resourceIds[i] = serializer.getRandomAccessId(handles[i].resource(serializer));
557                 } else {
558                         for(int i=0;i<count;++i)
559                                 resourceIds[i] = serializer.getRandomAccessId(resources[i]);
560                 }
561                 return resourceIds;
562         }
563 }