]> gerrit.simantics Code Review - simantics/platform.git/blob - bundles/org.simantics.graph.db/src/org/simantics/graph/db/TransferableGraphImportProcess.java
Collect model dependencies in reversed topological order
[simantics/platform.git] / bundles / org.simantics.graph.db / src / org / simantics / graph / db / TransferableGraphImportProcess.java
1 package org.simantics.graph.db;
2
3 import java.io.DataOutput;
4 import java.io.DataOutputStream;
5 import java.io.FileNotFoundException;
6 import java.io.FileOutputStream;
7 import java.io.IOException;
8 import java.util.HashSet;
9 import java.util.Map;
10 import java.util.Set;
11
12 import org.simantics.databoard.Bindings;
13 import org.simantics.databoard.accessor.error.AccessorException;
14 import org.simantics.databoard.adapter.AdaptException;
15 import org.simantics.databoard.binding.mutable.Variant;
16 import org.simantics.db.ReadGraph;
17 import org.simantics.db.Resource;
18 import org.simantics.db.WriteOnlyGraph;
19 import org.simantics.db.common.WriteBindings;
20 import org.simantics.db.common.procedure.adapter.TransientCacheAsyncListener;
21 import org.simantics.db.common.uri.UnescapedChildMapOfResource;
22 import org.simantics.db.common.utils.Logger;
23 import org.simantics.db.common.utils.NameUtils;
24 import org.simantics.db.exception.DatabaseException;
25 import org.simantics.db.exception.ResourceNotFoundException;
26 import org.simantics.db.service.ClusterBuilder;
27 import org.simantics.db.service.ClusterBuilder.ResourceHandle;
28 import org.simantics.db.service.SerialisationSupport;
29 import org.simantics.graph.representation.Extensions;
30 import org.simantics.graph.representation.External;
31 import org.simantics.graph.representation.Identity;
32 import org.simantics.graph.representation.IdentityDefinition;
33 import org.simantics.graph.representation.Internal;
34 import org.simantics.graph.representation.Optional;
35 import org.simantics.graph.representation.Root;
36 import org.simantics.graph.representation.TransferableGraph1;
37 import org.simantics.graph.representation.TransferableGraphUtils;
38 import org.simantics.graph.representation.Value;
39 import org.simantics.graph.utils.TGResourceUtil;
40 import org.simantics.graph.utils.TGResourceUtil.LongAdapter;
41
42 public class TransferableGraphImportProcess implements TransferableGraphImporter {
43         
44         public static String LOG_FILE = "transferableGraphs.log";
45         final static private boolean LOG = false;
46         
47         static DataOutput log;
48
49         static {
50
51                 if (LOG) {
52                         try {
53                                 FileOutputStream stream = new FileOutputStream(LOG_FILE);
54                                 log = new DataOutputStream(stream);
55                         } catch (FileNotFoundException e) {
56                                 e.printStackTrace();
57                         }
58                 }
59
60         }
61         
62         private static void log(String line) {
63                 if (LOG) {
64                         try {
65                                 log.writeUTF(line + "\n");
66                         } catch (IOException e) {
67                                 e.printStackTrace();
68                         }
69                 }
70         }
71         
72         TransferableGraph1 tg;
73         IImportAdvisor advisor;
74         TGStatusMonitor monitor;
75         final TGResourceUtil resourceUtil = new TGResourceUtil();
76         
77         Resource[] resources;
78         ResourceHandle[] handles;
79         
80         Set<String> missingExternals = new HashSet<String>(); 
81         
82         // Builtins
83         Resource RootLibrary;
84         Resource String;
85         Resource Library;
86         
87         Resource InstanceOf;
88         Resource ConsistsOf;
89         Resource PartOf;
90         Resource HasName;
91         Resource NameOf;        
92                                 
93         public TransferableGraphImportProcess(TransferableGraph1 tg, IImportAdvisor advisor, TGStatusMonitor monitor) {
94                 this.tg = tg;
95                 this.advisor = advisor;
96                 this.monitor = monitor;
97         }
98         
99     public TransferableGraphImportProcess(TransferableGraph1 tg, IImportAdvisor advisor) {
100         this(tg, advisor, null);
101     }
102
103         public void findBuiltins(WriteOnlyGraph g) throws DatabaseException {
104                 RootLibrary = g.getBuiltin("http:/");
105                 String = g.getBuiltin(CoreInitialization.LAYER0 + "String");
106                 Library = g.getBuiltin(CoreInitialization.LAYER0 + "Library");
107                 InstanceOf = g.getBuiltin(CoreInitialization.LAYER0 + "InstanceOf");
108                 ConsistsOf = g.getBuiltin(CoreInitialization.LAYER0 + "ConsistsOf");
109                 PartOf = g.getBuiltin(CoreInitialization.LAYER0 + "PartOf");
110                 HasName = g.getBuiltin(CoreInitialization.LAYER0 + "HasName");
111                 NameOf = g.getBuiltin(CoreInitialization.LAYER0 + "NameOf");
112         }
113         
114         public void findBuiltins(ReadGraph g) throws DatabaseException {
115                 RootLibrary = g.getBuiltin("http:/");
116                 String = g.getBuiltin(CoreInitialization.LAYER0 + "String");
117                 Library = g.getBuiltin(CoreInitialization.LAYER0 + "Library");
118                 InstanceOf = g.getBuiltin(CoreInitialization.LAYER0 + "InstanceOf");
119                 ConsistsOf = g.getBuiltin(CoreInitialization.LAYER0 + "ConsistsOf");
120                 PartOf = g.getBuiltin(CoreInitialization.LAYER0 + "PartOf");
121                 HasName = g.getBuiltin(CoreInitialization.LAYER0 + "HasName");
122                 NameOf = g.getBuiltin(CoreInitialization.LAYER0 + "NameOf");
123         }
124
125         /* Preparation that is used when the core is empty. 
126          */
127         void initialPrepare(WriteOnlyGraph graph) throws DatabaseException {
128                 findBuiltins(graph);
129                 
130                 resources = new Resource[tg.resourceCount];
131                 
132                 int Root = -1;
133                 int SimanticsDomain = -1;
134                 int Layer0 = -1;
135                 
136                 for(Identity identity : tg.identities) {
137                         if(identity.definition instanceof Internal) {
138                                 Internal def = (Internal)identity.definition;
139                                 Resource res = null;
140                                 if(def.parent == Layer0) {
141                                         try {
142                                                 res = graph.getBuiltin(CoreInitialization.LAYER0 + def.name);
143                                         } catch(ResourceNotFoundException e) {                                                                          
144                                         }
145                                 }
146                                 else if(def.parent == SimanticsDomain) {
147                                         if(def.name.equals("Layer0-1.1"))
148                                                 Layer0 = identity.resource;
149                                 }
150                                 else if(def.parent == Root) {
151                                         if(def.name.equals("www.simantics.org"))
152                                                 SimanticsDomain = identity.resource;
153                                 }
154
155                                 if(res == null)
156                                         res = createChild(graph, resources[def.parent], def.name);
157                                 else
158                                         createChild(graph, res, resources[def.parent], def.name);
159                                 resources[identity.resource] = res;
160                         }
161                         else if(identity.definition instanceof Root) {
162                                 Root = identity.resource;
163                                 resources[identity.resource] = RootLibrary;                             
164                         } 
165                 }
166         }
167         
168         void addMissing(String external) {
169                 Set<String> removals = new HashSet<String>();
170                 for(String ext : missingExternals) if(ext.startsWith(external)) return;
171                 for(String ext : missingExternals) if(external.startsWith(ext)) removals.add(ext);
172                 missingExternals.removeAll(removals);
173                 missingExternals.add(external);
174         }
175         
176         void prepare(ReadGraph graph) throws DatabaseException {
177                 findBuiltins(graph);
178                 
179                 Resource[] resources = new Resource[tg.resourceCount];
180                 
181                 for(Identity identity : tg.identities) {
182                         IdentityDefinition definition = identity.definition;
183                         if(definition instanceof External) {
184                                 External def = (External)definition;
185                                 if(def.parent == -1) {
186                                         resources[identity.resource] = RootLibrary;
187                                 } else {
188                                         if("@inverse".equals(def.name)) {
189                                                 Resource parent = resources[def.parent];
190                                                 Resource child = graph.getInverse(parent);
191                                                 resources[identity.resource] = child;
192                                         } else {
193                                                 Resource parent = resources[def.parent];
194                                                 // TODO: escape should be removed when names become well-behaving
195                                                 if(parent != null) {
196                                                         Resource child = graph
197                                                         .syncRequest(new UnescapedChildMapOfResource(parent),
198                                                                 new TransientCacheAsyncListener<Map<String, Resource>>())
199                                                                 .get(def.name);
200                                                         if(child == null) {
201                                                                 String uri = graph.getPossibleURI(parent);
202                                                                 if(uri == null) {
203                                                                         addMissing(NameUtils.getSafeName(graph, parent) + " /" + def.name);
204                                                                 } else {
205                                                                         addMissing(graph.getURI(parent) + "/" + def.name);
206                                                                 }
207                                                         }
208                                                         resources[identity.resource] = child;
209                                                 } else {
210                                                     addMissing(TransferableGraphUtils.getURI(tg, def.parent) + "/" + def.name);
211                                                 }
212                                         }
213                                 }
214                         }
215                         else if(definition instanceof Internal) {
216                                 // Do not do anything for now
217                         }
218                         else if(definition instanceof Root) {
219                                 Root root = (Root)definition;
220                                 if(root.name.equals(""))
221                                         resources[identity.resource] = RootLibrary;
222                                 else {
223                                         Resource existing = advisor.analyzeRoot(graph, root);
224                                         if(existing != null)
225                                                 resources[identity.resource] = existing;
226                                 }
227                         }
228                         else if(definition instanceof Optional) {
229                                 External def = (External)definition;
230                                 Resource parent = resources[def.parent];
231                                 if(parent != null)
232                                         resources[identity.resource] = 
233                                                 graph.syncRequest(new UnescapedChildMapOfResource(parent)).get(def.name);                               
234                         }
235                 }               
236                 
237                 this.resources = resources;
238                 
239                 if(!missingExternals.isEmpty()) throw new MissingDependencyException(this);
240                 
241         }
242
243         Resource createChild(WriteOnlyGraph graph, Resource parent, String name) throws DatabaseException {
244                 Resource child = graph.newResource();
245                 //graph.claim(parent, ConsistsOf, PartOf, child);
246                 Resource nameResource = graph.newResource();
247                 graph.claim(nameResource, InstanceOf, null, String);
248                 graph.claimValue(nameResource, name, WriteBindings.STRING);
249                 graph.claim(child, HasName, NameOf, nameResource);
250                 return child;
251         }
252
253         @Override
254         public Resource createChild(WriteOnlyGraph graph, Resource child, Resource parent, String name) throws DatabaseException {
255                 graph.claim(parent, ConsistsOf, PartOf, child);
256                 Resource nameResource = graph.newResource();
257                 graph.claim(nameResource, InstanceOf, null, String);
258                 graph.claimValue(nameResource, name, WriteBindings.STRING);
259                 graph.claim(child, HasName, NameOf, nameResource);
260                 return child;
261         }
262         
263         int[] getClustering() {
264                 Variant v = tg.extensions.get(Extensions.CLUSTERING);
265                 if(v == null) return null;
266                 try {
267                         return (int[])v.getValue(Bindings.INT_ARRAY);
268                 } catch (AdaptException e) {
269                         Logger.defaultLogError(e);
270                         return null;
271                 }
272         }
273
274         class ResourceAdapter implements LongAdapter {
275             final SerialisationSupport ss;
276             final Value value;
277             Variant original;
278             ResourceAdapter(SerialisationSupport ss, Value value) {
279                 this.ss = ss;
280                 this.value = value;
281             }
282         @Override
283         public long adapt(long in) {
284             if(original == null) original = value.value.clone();
285             Resource res = handles[(int)in].resource(ss);
286             // Maybe throw?
287             if(res == null) return in;
288             return res.getResourceId();
289         }           
290         }
291         
292         Variant translate(SerialisationSupport ss, Value value, final ResourceHandle[] handles) throws DatabaseException {
293                 
294                 try {
295                     ResourceAdapter adapter = new ResourceAdapter(ss, value);
296                         resourceUtil.adaptValue( value.value.getBinding(), value.value.getValue(), adapter);
297                         Variant result = value.value;
298                         if(adapter.original != null) value.value = adapter.original;
299                         return result;
300                 } catch (AccessorException e) {
301                         e.printStackTrace();
302                 }               
303                 
304                 return value.value;
305                 
306         }
307         
308     class ResourceAdapter2 implements LongAdapter {
309         final Value value;
310         Variant original;
311         ResourceAdapter2(Value value) {
312             this.value = value;
313         }
314         @Override
315         public long adapt(long in) {
316             if(original == null) original = value.value.clone();
317             Resource res = resources[(int)in];
318             // Maybe throw?
319             if(res == null) return in;
320             return res.getResourceId();
321         }           
322     }
323
324         Variant translate2(Value value, final Resource[] resources) throws DatabaseException {
325                 
326         try {
327             ResourceAdapter2 adapter = new ResourceAdapter2(value);
328             resourceUtil.adaptValue( value.value.getBinding(), value.value.getValue(), adapter);
329             Variant result = value.value;
330             if(adapter.original != null) value.value = adapter.original;
331             return result;
332         } catch (AccessorException e) {
333             e.printStackTrace();
334         }       
335         
336         return value.value;
337                 
338         }
339
340         void write(WriteOnlyGraph graph) throws DatabaseException {
341
342                 Resource[] resources = this.resources;
343
344                 this.handles = new ResourceHandle[resources.length];
345                 
346                 ResourceHandle[] handles = this.handles; 
347
348                 int[] clustering = getClustering();
349
350                 // Internal identities          
351                 for(Identity identity : tg.identities) {
352                         IdentityDefinition definition = identity.definition;
353                         if(resources[identity.resource] != null)
354                                 continue;
355                         if(definition instanceof External) {
356                                 // Already done everything
357                         }
358                         else if(definition instanceof Internal) {
359                                 Internal def = (Internal)definition;
360                                 resources[identity.resource] = 
361                                         createChild(graph, resources[def.parent], def.name);
362                         }
363                         else if(definition instanceof Root) {
364                                 Root root = (Root)definition;                           
365                                 resources[identity.resource] = advisor.createRoot(graph, root);                                 
366                         }
367                         else if(definition instanceof Optional) {
368                                 Optional def = (Optional)definition;
369                                 Resource child = createChild(graph, resources[def.parent], def.name);
370                                 graph.claim(child, InstanceOf, null, Library); // ???
371                                 resources[identity.resource] = child;                                   
372                         }
373                 }               
374
375                 ClusterBuilder builder = graph.getService(ClusterBuilder.class);
376                 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
377
378                 if(clustering != null) {
379                         
380                 int i = 0;
381                         for(int c : clustering) {
382                                 builder.newCluster();
383                                 for(int r=0;r<c;r++, i++)
384                                         if(resources[i] == null)
385                                                 handles[i] = builder.newResource();
386                                         else 
387                                                 handles[i] = builder.resource(resources[i]);
388
389                         }
390
391                         for(;i<resources.length;++i)
392                                 if(resources[i] == null)
393                                         handles[i] = builder.newResource();
394                                 else 
395                                         handles[i] = builder.resource(resources[i]);
396                         
397                 } else {
398                 
399                         // Create blank resources
400                         for(int i=0;i<resources.length;++i)
401                                 if(resources[i] == null)
402                                         handles[i] = builder.newResource();
403                                 else 
404                                         handles[i] = builder.resource(resources[i]);
405
406                 }
407                 
408                 // Write statements
409                 int[] statements = tg.statements;
410                 
411 //              int internals = tg.resourceCount - tg.identities.length;
412                 
413                 for(int i=0;i<statements.length;i+=4) {
414
415                         int sub = statements[i];
416                         int pred = statements[i+1];
417                         int inv = statements[i+2];
418                         int obj = statements[i+3];
419
420                         ResourceHandle subject = handles[sub];
421                         ResourceHandle predicate = handles[pred];
422                         ResourceHandle object = handles[obj];
423
424                         if(resources[sub] == null) {
425                                 subject.addStatement(graph, predicate, object); 
426                         } else {
427                                 graph.claim(
428                                                 handles[sub].resource(ss),
429                                                 handles[pred].resource(ss),
430                                                 null, handles[obj].resource(ss));
431                         }
432                         
433                         if(inv >= 0) {
434                                 
435                                 if(resources[obj] == null) {
436                                         ResourceHandle inverse = handles[inv];
437                                         object.addStatement(graph, inverse, subject);   
438                                 } else {
439                                     Resource s = handles[obj].resource(ss);
440                                     if(!graph.isImmutable(s))
441                                         graph.claim(
442                                                         s,
443                                                         handles[inv].resource(ss),
444                                                         null, handles[sub].resource(ss));
445                                 }
446                                 
447                         }
448                         
449                         if(LOG) {
450                                 log("[STATEMENT] " + resources[statements[i]].getResourceId() + ", " + resources[statements[i+1]].getResourceId() + ", " + resources[statements[i+3]].getResourceId());
451                         }
452                         
453                 }
454                                 
455                 // Write values
456                 for(Value value : tg.values) {
457                         Variant variant = translate(ss, value, handles);
458                         int file = value.resource & 0x80000000;
459                         int resource = value.resource & 0x7FFFFFFF;
460                         if (file != 0) {
461                             throw new UnsupportedOperationException();
462                                 //graph.claimValue(handles[resource].resource(), value.value, Bindings.BYTE_ARRAY);
463                         } else {
464                                 graph.claimValue(handles[resource].resource(ss), variant.getValue(), variant.getBinding());
465                         }
466                 }
467         }
468
469         private int updatePercentage(int percentage, int done, int total) {
470             if(monitor != null && (done & 63) == 0) {
471                 int current = 100*done / total;
472                 if(current > percentage) {
473                     percentage = current;
474                     monitor.status(percentage);
475                 }
476             }
477         return percentage;
478         }
479         
480         void write2(WriteOnlyGraph graph) throws DatabaseException {
481                 Resource[] resources = this.resources;
482                 
483                 // Internal identities          
484                 for(Identity identity : tg.identities) {
485                         IdentityDefinition definition = identity.definition;
486                         if(resources[identity.resource] != null)
487                                 continue;
488                         if(definition instanceof External) {
489                                 // Already done everything
490                         }
491                         else if(definition instanceof Internal) {
492                                 Internal def = (Internal)definition;
493                                 resources[identity.resource] = 
494                                         createChild(graph, resources[def.parent], def.name);
495                         }
496                         else if(definition instanceof Root) {                           
497                                 Root root = (Root)definition;                           
498                                 resources[identity.resource] = advisor.createRoot(graph, root);                                 
499                         }
500                         else if(definition instanceof Optional) {
501                                 Optional def = (Optional)definition;
502                                 Resource child = createChild(graph, resources[def.parent], def.name);
503                                 graph.claim(child, InstanceOf, null, Library); // ???
504                                 resources[identity.resource] = child;                                   
505                         }
506                 }               
507                 
508                 // Create blank resources
509                 for(int i=0;i<resources.length;++i)
510                         if(resources[i] == null)
511                                 resources[i] = graph.newResource();
512                 
513                 int total = tg.statements.length + tg.values.length;
514                 int done = 0;
515                 int percentage = 0;
516                 
517                 // Write statements
518                 int[] statements = tg.statements;
519                 
520                 for(int i=0;i<statements.length;i+=4) {
521                         int inv = statements[i+2];
522                         graph.claim(
523                                         resources[statements[i]],
524                                         resources[statements[i+1]],
525                                         inv < 0 ? null : resources[inv],
526                                         resources[statements[i+3]]
527                                         );
528                         if(LOG) {
529                                 log("[STATEMENT] " + resources[statements[i]].getResourceId() + ", " + resources[statements[i+1]].getResourceId() + ", " + resources[statements[i+3]].getResourceId());
530                         }
531                         percentage = updatePercentage(percentage, done++, total);
532                 }
533
534                 // Write values
535                 for(Value value : tg.values) {
536                         Variant variant = translate2(value, resources);
537                         int file = value.resource & 0x80000000;
538                         int resource = value.resource & 0x7FFFFFFF;
539                         if (file != 0) {
540                             throw new UnsupportedOperationException();
541                                 //graph.claimValue(resources[resource], value.value, Bindings.BYTE_ARRAY);
542                         } else {
543                             graph.claimValue(resources[resource], variant.getValue(), variant.getBinding());
544                         }
545             percentage = updatePercentage(percentage, done++, total);
546                 }
547                 
548         }
549         
550         public long[] getResourceIds(SerialisationSupport serializer) throws DatabaseException {
551                 final int count = resources.length;
552                 long[] resourceIds = new long[count];
553                 if(handles != null) {
554                         for(int i=0;i<count;++i)
555                                 resourceIds[i] = serializer.getRandomAccessId(handles[i].resource(serializer));
556                 } else {
557                         for(int i=0;i<count;++i)
558                                 resourceIds[i] = serializer.getRandomAccessId(resources[i]);
559                 }
560                 return resourceIds;
561         }
562 }