Expose TransferableGraphImportProcess Resources table.
[simantics/platform.git] / bundles / org.simantics.graph.db / src / org / simantics / graph / db / TransferableGraphImportProcess.java
1 package org.simantics.graph.db;
2
3 import java.io.DataOutput;
4 import java.io.DataOutputStream;
5 import java.io.FileNotFoundException;
6 import java.io.FileOutputStream;
7 import java.io.IOException;
8 import java.util.HashSet;
9 import java.util.Map;
10 import java.util.Set;
11
12 import org.simantics.databoard.Bindings;
13 import org.simantics.databoard.accessor.error.AccessorException;
14 import org.simantics.databoard.adapter.AdaptException;
15 import org.simantics.databoard.binding.mutable.Variant;
16 import org.simantics.databoard.util.URIStringUtils;
17 import org.simantics.db.ReadGraph;
18 import org.simantics.db.Resource;
19 import org.simantics.db.WriteOnlyGraph;
20 import org.simantics.db.common.WriteBindings;
21 import org.simantics.db.common.procedure.adapter.TransientCacheAsyncListener;
22 import org.simantics.db.common.uri.UnescapedChildMapOfResource;
23 import org.simantics.db.common.utils.Logger;
24 import org.simantics.db.common.utils.NameUtils;
25 import org.simantics.db.exception.DatabaseException;
26 import org.simantics.db.exception.ResourceNotFoundException;
27 import org.simantics.db.service.ClusterBuilder;
28 import org.simantics.db.service.ClusterBuilder.ResourceHandle;
29 import org.simantics.db.service.SerialisationSupport;
30 import org.simantics.graph.representation.Extensions;
31 import org.simantics.graph.representation.External;
32 import org.simantics.graph.representation.Identity;
33 import org.simantics.graph.representation.IdentityDefinition;
34 import org.simantics.graph.representation.Internal;
35 import org.simantics.graph.representation.Optional;
36 import org.simantics.graph.representation.Root;
37 import org.simantics.graph.representation.TransferableGraph1;
38 import org.simantics.graph.representation.TransferableGraphUtils;
39 import org.simantics.graph.representation.Value;
40 import org.simantics.graph.utils.TGResourceUtil;
41 import org.simantics.graph.utils.TGResourceUtil.LongAdapter;
42
43 public class TransferableGraphImportProcess implements TransferableGraphImporter {
44         
45         public static String LOG_FILE = "transferableGraphs.log";
46         final static private boolean LOG = false;
47         
48         static DataOutput log;
49
50         static {
51
52                 if (LOG) {
53                         try {
54                                 FileOutputStream stream = new FileOutputStream(LOG_FILE);
55                                 log = new DataOutputStream(stream);
56                         } catch (FileNotFoundException e) {
57                                 e.printStackTrace();
58                         }
59                 }
60
61         }
62         
63         private static void log(String line) {
64                 if (LOG) {
65                         try {
66                                 log.writeUTF(line + "\n");
67                         } catch (IOException e) {
68                                 e.printStackTrace();
69                         }
70                 }
71         }
72         
73         TransferableGraph1 tg;
74         IImportAdvisor advisor;
75         TGStatusMonitor monitor;
76         final TGResourceUtil resourceUtil = new TGResourceUtil();
77         
78         Resource[] resources;
79         ResourceHandle[] handles;
80         
81         Set<String> missingExternals = new HashSet<String>(); 
82         
83         // Builtins
84         Resource RootLibrary;
85         Resource String;
86         Resource Library;
87         
88         Resource InstanceOf;
89         Resource ConsistsOf;
90         Resource PartOf;
91         Resource HasName;
92         Resource NameOf;        
93                                 
94         public TransferableGraphImportProcess(TransferableGraph1 tg, IImportAdvisor advisor, TGStatusMonitor monitor) {
95                 this.tg = tg;
96                 this.advisor = advisor;
97                 this.monitor = monitor;
98         }
99         
100     public TransferableGraphImportProcess(TransferableGraph1 tg, IImportAdvisor advisor) {
101         this(tg, advisor, null);
102     }
103
104         public void findBuiltins(WriteOnlyGraph g) throws DatabaseException {
105                 RootLibrary = g.getBuiltin("http:/");
106                 String = g.getBuiltin(CoreInitialization.LAYER0 + "String");
107                 Library = g.getBuiltin(CoreInitialization.LAYER0 + "Library");
108                 InstanceOf = g.getBuiltin(CoreInitialization.LAYER0 + "InstanceOf");
109                 ConsistsOf = g.getBuiltin(CoreInitialization.LAYER0 + "ConsistsOf");
110                 PartOf = g.getBuiltin(CoreInitialization.LAYER0 + "PartOf");
111                 HasName = g.getBuiltin(CoreInitialization.LAYER0 + "HasName");
112                 NameOf = g.getBuiltin(CoreInitialization.LAYER0 + "NameOf");
113         }
114         
115         public void findBuiltins(ReadGraph g) throws DatabaseException {
116                 RootLibrary = g.getBuiltin("http:/");
117                 String = g.getBuiltin(CoreInitialization.LAYER0 + "String");
118                 Library = g.getBuiltin(CoreInitialization.LAYER0 + "Library");
119                 InstanceOf = g.getBuiltin(CoreInitialization.LAYER0 + "InstanceOf");
120                 ConsistsOf = g.getBuiltin(CoreInitialization.LAYER0 + "ConsistsOf");
121                 PartOf = g.getBuiltin(CoreInitialization.LAYER0 + "PartOf");
122                 HasName = g.getBuiltin(CoreInitialization.LAYER0 + "HasName");
123                 NameOf = g.getBuiltin(CoreInitialization.LAYER0 + "NameOf");
124         }
125
126         /* Preparation that is used when the core is empty. 
127          */
128         void initialPrepare(WriteOnlyGraph graph) throws DatabaseException {
129                 findBuiltins(graph);
130                 
131                 resources = new Resource[tg.resourceCount];
132                 
133                 int Root = -1;
134                 int SimanticsDomain = -1;
135                 int Layer0 = -1;
136                 
137                 for(Identity identity : tg.identities) {
138                         if(identity.definition instanceof Internal) {
139                                 Internal def = (Internal)identity.definition;
140                                 Resource res = null;
141                                 if(def.parent == Layer0) {
142                                         try {
143                                                 res = graph.getBuiltin(CoreInitialization.LAYER0 + def.name);
144                                         } catch(ResourceNotFoundException e) {                                                                          
145                                         }
146                                 }
147                                 else if(def.parent == SimanticsDomain) {
148                                         if(def.name.equals("Layer0-1.1"))
149                                                 Layer0 = identity.resource;
150                                 }
151                                 else if(def.parent == Root) {
152                                         if(def.name.equals("www.simantics.org"))
153                                                 SimanticsDomain = identity.resource;
154                                 }
155
156                                 if(res == null)
157                                         res = createChild(graph, resources[def.parent], def.name);
158                                 else
159                                         createChild(graph, res, resources[def.parent], def.name);
160                                 resources[identity.resource] = res;
161                         }
162                         else if(identity.definition instanceof Root) {
163                                 Root = identity.resource;
164                                 resources[identity.resource] = RootLibrary;                             
165                         } 
166                 }
167         }
168         
169         void addMissing(String external) {
170                 Set<String> removals = new HashSet<String>();
171                 for(String ext : missingExternals) if(ext.startsWith(external)) return;
172                 for(String ext : missingExternals) if(external.startsWith(ext)) removals.add(ext);
173                 missingExternals.removeAll(removals);
174                 missingExternals.add(external);
175         }
176         
177         public Resource[] getResources() {
178                 return resources;
179         }
180         
181         public void prepare(ReadGraph graph) throws DatabaseException {
182                 findBuiltins(graph);
183                 
184                 Resource[] resources = new Resource[tg.resourceCount];
185                 
186                 for(Identity identity : tg.identities) {
187                         IdentityDefinition definition = identity.definition;
188                         if(definition instanceof External) {
189                                 External def = (External)definition;
190                                 if(def.parent == -1) {
191                                         resources[identity.resource] = RootLibrary;
192                                 } else {
193                                         if("@inverse".equals(def.name)) {
194                                                 Resource parent = resources[def.parent];
195                                                 Resource child = graph.getInverse(parent);
196                                                 resources[identity.resource] = child;
197                                         } else {
198                                                 Resource parent = resources[def.parent];
199                                                 // TODO: escape should be removed when names become well-behaving
200                                                 if(parent != null) {
201                                                         Resource child = graph
202                                                         .syncRequest(new UnescapedChildMapOfResource(parent),
203                                                                 new TransientCacheAsyncListener<Map<String, Resource>>())
204                                                                 .get(def.name);
205                                                         if(child == null) {
206                                                                 String uri = graph.getPossibleURI(parent);
207                                                                 if(uri == null) {
208                                                                         addMissing(URIStringUtils.escape(NameUtils.getSafeName(graph, parent)) + " /" + URIStringUtils.escape(def.name));
209                                                                 } else {
210                                                                         addMissing(graph.getURI(parent) + "/" + URIStringUtils.escape(def.name));
211                                                                 }
212                                                         }
213                                                         resources[identity.resource] = child;
214                                                 } else {
215                                                     addMissing(TransferableGraphUtils.getURI(tg, def.parent) + "/" + URIStringUtils.escape(def.name));
216                                                 }
217                                         }
218                                 }
219                         }
220                         else if(definition instanceof Internal) {
221                                 // Do not do anything for now
222                         }
223                         else if(definition instanceof Root) {
224                                 Root root = (Root)definition;
225                                 if(root.name.equals(""))
226                                         resources[identity.resource] = RootLibrary;
227                                 else {
228                                         Resource existing = advisor.analyzeRoot(graph, root);
229                                         if(existing != null)
230                                                 resources[identity.resource] = existing;
231                                 }
232                         }
233                         else if(definition instanceof Optional) {
234                                 External def = (External)definition;
235                                 Resource parent = resources[def.parent];
236                                 if(parent != null)
237                                         resources[identity.resource] = 
238                                                 graph.syncRequest(new UnescapedChildMapOfResource(parent)).get(def.name);                               
239                         }
240                 }               
241                 
242                 this.resources = resources;
243                 
244                 if(!missingExternals.isEmpty()) throw new MissingDependencyException(this);
245                 
246         }
247
248         Resource createChild(WriteOnlyGraph graph, Resource parent, String name) throws DatabaseException {
249                 Resource child = graph.newResource();
250                 //graph.claim(parent, ConsistsOf, PartOf, child);
251                 Resource nameResource = graph.newResource();
252                 graph.claim(nameResource, InstanceOf, null, String);
253                 graph.claimValue(nameResource, name, WriteBindings.STRING);
254                 graph.claim(child, HasName, NameOf, nameResource);
255                 return child;
256         }
257
258         @Override
259         public Resource createChild(WriteOnlyGraph graph, Resource child, Resource parent, String name) throws DatabaseException {
260                 graph.claim(parent, ConsistsOf, PartOf, child);
261                 Resource nameResource = graph.newResource();
262                 graph.claim(nameResource, InstanceOf, null, String);
263                 graph.claimValue(nameResource, name, WriteBindings.STRING);
264                 graph.claim(child, HasName, NameOf, nameResource);
265                 return child;
266         }
267         
268         int[] getClustering() {
269                 Variant v = tg.extensions.get(Extensions.CLUSTERING);
270                 if(v == null) return null;
271                 try {
272                         return (int[])v.getValue(Bindings.INT_ARRAY);
273                 } catch (AdaptException e) {
274                         Logger.defaultLogError(e);
275                         return null;
276                 }
277         }
278
279         class ResourceAdapter implements LongAdapter {
280             final SerialisationSupport ss;
281             final Value value;
282             Variant original;
283             ResourceAdapter(SerialisationSupport ss, Value value) {
284                 this.ss = ss;
285                 this.value = value;
286             }
287         @Override
288         public long adapt(long in) {
289             if(original == null) original = value.value.clone();
290             Resource res = handles[(int)in].resource(ss);
291             // Maybe throw?
292             if(res == null) return in;
293             return res.getResourceId();
294         }           
295         }
296         
297         Variant translate(SerialisationSupport ss, Value value, final ResourceHandle[] handles) throws DatabaseException {
298                 
299                 try {
300                     ResourceAdapter adapter = new ResourceAdapter(ss, value);
301                         resourceUtil.adaptValue( value.value.getBinding(), value.value.getValue(), adapter);
302                         Variant result = value.value;
303                         if(adapter.original != null) value.value = adapter.original;
304                         return result;
305                 } catch (AccessorException e) {
306                         e.printStackTrace();
307                 }               
308                 
309                 return value.value;
310                 
311         }
312         
313     class ResourceAdapter2 implements LongAdapter {
314         final Value value;
315         Variant original;
316         ResourceAdapter2(Value value) {
317             this.value = value;
318         }
319         @Override
320         public long adapt(long in) {
321             if(original == null) original = value.value.clone();
322             Resource res = resources[(int)in];
323             // Maybe throw?
324             if(res == null) return in;
325             return res.getResourceId();
326         }           
327     }
328
329         Variant translate2(Value value, final Resource[] resources) throws DatabaseException {
330                 
331         try {
332             ResourceAdapter2 adapter = new ResourceAdapter2(value);
333             resourceUtil.adaptValue( value.value.getBinding(), value.value.getValue(), adapter);
334             Variant result = value.value;
335             if(adapter.original != null) value.value = adapter.original;
336             return result;
337         } catch (AccessorException e) {
338             e.printStackTrace();
339         }       
340         
341         return value.value;
342                 
343         }
344
345         public void write(WriteOnlyGraph graph) throws DatabaseException {
346
347                 Resource[] resources = this.resources;
348
349                 this.handles = new ResourceHandle[resources.length];
350                 
351                 ResourceHandle[] handles = this.handles; 
352
353                 int[] clustering = getClustering();
354
355                 // Internal identities          
356                 for(Identity identity : tg.identities) {
357                         IdentityDefinition definition = identity.definition;
358                         if(resources[identity.resource] != null)
359                                 continue;
360                         if(definition instanceof External) {
361                                 // Already done everything
362                         }
363                         else if(definition instanceof Internal) {
364                                 Internal def = (Internal)definition;
365                                 resources[identity.resource] = 
366                                         createChild(graph, resources[def.parent], def.name);
367                         }
368                         else if(definition instanceof Root) {
369                                 Root root = (Root)definition;                           
370                                 resources[identity.resource] = advisor.createRoot(graph, root);                                 
371                         }
372                         else if(definition instanceof Optional) {
373                                 Optional def = (Optional)definition;
374                                 Resource child = createChild(graph, resources[def.parent], def.name);
375                                 graph.claim(child, InstanceOf, null, Library); // ???
376                                 resources[identity.resource] = child;                                   
377                         }
378                 }               
379
380                 ClusterBuilder builder = graph.getService(ClusterBuilder.class);
381                 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
382
383                 if(clustering != null) {
384                         
385                 int i = 0;
386                         for(int c : clustering) {
387                                 builder.newCluster();
388                                 for(int r=0;r<c;r++, i++)
389                                         if(resources[i] == null)
390                                                 handles[i] = builder.newResource();
391                                         else 
392                                                 handles[i] = builder.resource(resources[i]);
393
394                         }
395
396                         for(;i<resources.length;++i)
397                                 if(resources[i] == null)
398                                         handles[i] = builder.newResource();
399                                 else 
400                                         handles[i] = builder.resource(resources[i]);
401                         
402                 } else {
403                 
404                         // Create blank resources
405                         for(int i=0;i<resources.length;++i)
406                                 if(resources[i] == null)
407                                         handles[i] = builder.newResource();
408                                 else 
409                                         handles[i] = builder.resource(resources[i]);
410
411                 }
412                 
413                 // Write statements
414                 int[] statements = tg.statements;
415                 
416 //              int internals = tg.resourceCount - tg.identities.length;
417                 
418                 for(int i=0;i<statements.length;i+=4) {
419
420                         int sub = statements[i];
421                         int pred = statements[i+1];
422                         int inv = statements[i+2];
423                         int obj = statements[i+3];
424
425                         ResourceHandle subject = handles[sub];
426                         ResourceHandle predicate = handles[pred];
427                         ResourceHandle object = handles[obj];
428
429                         if(resources[sub] == null) {
430                                 subject.addStatement(graph, predicate, object); 
431                         } else {
432                                 graph.claim(
433                                                 handles[sub].resource(ss),
434                                                 handles[pred].resource(ss),
435                                                 null, handles[obj].resource(ss));
436                         }
437                         
438                         if(inv >= 0) {
439                                 
440                                 if(resources[obj] == null) {
441                                         ResourceHandle inverse = handles[inv];
442                                         object.addStatement(graph, inverse, subject);   
443                                 } else {
444                                     Resource s = handles[obj].resource(ss);
445                                     if(!graph.isImmutable(s))
446                                         graph.claim(
447                                                         s,
448                                                         handles[inv].resource(ss),
449                                                         null, handles[sub].resource(ss));
450                                 }
451                                 
452                         }
453                         
454                         if(LOG) {
455                                 log("[STATEMENT] " + resources[statements[i]].getResourceId() + ", " + resources[statements[i+1]].getResourceId() + ", " + resources[statements[i+3]].getResourceId());
456                         }
457                         
458                 }
459                                 
460                 // Write values
461                 for(Value value : tg.values) {
462                         Variant variant = translate(ss, value, handles);
463                         int file = value.resource & 0x80000000;
464                         int resource = value.resource & 0x7FFFFFFF;
465                         if (file != 0) {
466                             throw new UnsupportedOperationException();
467                                 //graph.claimValue(handles[resource].resource(), value.value, Bindings.BYTE_ARRAY);
468                         } else {
469                                 graph.claimValue(handles[resource].resource(ss), variant.getValue(), variant.getBinding());
470                         }
471                 }
472         }
473
474         private int updatePercentage(int percentage, int done, int total) {
475             if(monitor != null && (done & 63) == 0) {
476                 int current = 100*done / total;
477                 if(current > percentage) {
478                     percentage = current;
479                     monitor.status(percentage);
480                 }
481             }
482         return percentage;
483         }
484         
485         void write2(WriteOnlyGraph graph) throws DatabaseException {
486                 Resource[] resources = this.resources;
487                 
488                 // Internal identities          
489                 for(Identity identity : tg.identities) {
490                         IdentityDefinition definition = identity.definition;
491                         if(resources[identity.resource] != null)
492                                 continue;
493                         if(definition instanceof External) {
494                                 // Already done everything
495                         }
496                         else if(definition instanceof Internal) {
497                                 Internal def = (Internal)definition;
498                                 resources[identity.resource] = 
499                                         createChild(graph, resources[def.parent], def.name);
500                         }
501                         else if(definition instanceof Root) {                           
502                                 Root root = (Root)definition;                           
503                                 resources[identity.resource] = advisor.createRoot(graph, root);                                 
504                         }
505                         else if(definition instanceof Optional) {
506                                 Optional def = (Optional)definition;
507                                 Resource child = createChild(graph, resources[def.parent], def.name);
508                                 graph.claim(child, InstanceOf, null, Library); // ???
509                                 resources[identity.resource] = child;                                   
510                         }
511                 }               
512                 
513                 // Create blank resources
514                 for(int i=0;i<resources.length;++i)
515                         if(resources[i] == null)
516                                 resources[i] = graph.newResource();
517                 
518                 int total = tg.statements.length + tg.values.length;
519                 int done = 0;
520                 int percentage = 0;
521                 
522                 // Write statements
523                 int[] statements = tg.statements;
524                 
525                 for(int i=0;i<statements.length;i+=4) {
526                         int inv = statements[i+2];
527                         graph.claim(
528                                         resources[statements[i]],
529                                         resources[statements[i+1]],
530                                         inv < 0 ? null : resources[inv],
531                                         resources[statements[i+3]]
532                                         );
533                         if(LOG) {
534                                 log("[STATEMENT] " + resources[statements[i]].getResourceId() + ", " + resources[statements[i+1]].getResourceId() + ", " + resources[statements[i+3]].getResourceId());
535                         }
536                         percentage = updatePercentage(percentage, done++, total);
537                 }
538
539                 // Write values
540                 for(Value value : tg.values) {
541                         Variant variant = translate2(value, resources);
542                         int file = value.resource & 0x80000000;
543                         int resource = value.resource & 0x7FFFFFFF;
544                         if (file != 0) {
545                             throw new UnsupportedOperationException();
546                                 //graph.claimValue(resources[resource], value.value, Bindings.BYTE_ARRAY);
547                         } else {
548                             graph.claimValue(resources[resource], variant.getValue(), variant.getBinding());
549                         }
550             percentage = updatePercentage(percentage, done++, total);
551                 }
552                 
553         }
554         
555         public long[] getResourceIds(SerialisationSupport serializer) throws DatabaseException {
556                 final int count = resources.length;
557                 long[] resourceIds = new long[count];
558                 if(handles != null) {
559                         for(int i=0;i<count;++i)
560                                 resourceIds[i] = serializer.getRandomAccessId(handles[i].resource(serializer));
561                 } else {
562                         for(int i=0;i<count;++i)
563                                 resourceIds[i] = serializer.getRandomAccessId(resources[i]);
564                 }
565                 return resourceIds;
566         }
567 }