]> gerrit.simantics Code Review - simantics/platform.git/blob - bundles/org.simantics.db.layer0/src/org/simantics/db/layer0/util/ModelTransferableGraphSource.java
Merge "Better emptying of trash bin"
[simantics/platform.git] / bundles / org.simantics.db.layer0 / src / org / simantics / db / layer0 / util / ModelTransferableGraphSource.java
1 package org.simantics.db.layer0.util;
2
3 import java.io.Closeable;
4 import java.io.DataInput;
5 import java.io.File;
6 import java.io.IOException;
7 import java.nio.file.Files;
8 import java.util.ArrayList;
9 import java.util.Collection;
10 import java.util.Collections;
11 import java.util.HashSet;
12 import java.util.List;
13 import java.util.TreeMap;
14
15 import org.simantics.databoard.Bindings;
16 import org.simantics.databoard.Databoard;
17 import org.simantics.databoard.binding.Binding;
18 import org.simantics.databoard.binding.mutable.Variant;
19 import org.simantics.databoard.container.DataContainer;
20 import org.simantics.databoard.serialization.Serializer;
21 import org.simantics.databoard.type.Datatype;
22 import org.simantics.databoard.util.binary.ByteBufferReadable;
23 import org.simantics.db.ReadGraph;
24 import org.simantics.db.Resource;
25 import org.simantics.db.common.utils.NameUtils;
26 import org.simantics.db.exception.DatabaseException;
27 import org.simantics.db.exception.RuntimeDatabaseException;
28 import org.simantics.db.exception.ValidationException;
29 import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus;
30 import org.simantics.db.layer0.util.ConsistsOfProcess.ConsistsOfProcessEntry;
31 import org.simantics.db.layer0.util.TransferableGraphConfiguration2.SeedSpec;
32 import org.simantics.db.layer0.util.TransferableGraphConfiguration2.SeedSpec.SeedSpecType;
33 import org.simantics.db.service.SerialisationSupport;
34 import org.simantics.graph.db.TransferableGraphSource;
35 import org.simantics.graph.representation.External;
36 import org.simantics.graph.representation.Identity;
37 import org.simantics.graph.representation.Internal;
38 import org.simantics.graph.representation.Root;
39 import org.simantics.graph.representation.Value;
40 import org.simantics.layer0.Layer0;
41 import org.slf4j.Logger;
42 import org.slf4j.LoggerFactory;
43
44 import gnu.trove.list.array.TIntArrayList;
45 import gnu.trove.map.TIntObjectMap;
46 import gnu.trove.map.hash.TIntObjectHashMap;
47 import gnu.trove.procedure.TIntIntProcedure;
48
49 public class ModelTransferableGraphSource implements TransferableGraphSource {
50
51     private static final Logger                   LOGGER          = LoggerFactory
52             .getLogger(ModelTransferableGraphSource.class);
53
54     final private TransferableGraphConfiguration2 configuration;
55
56     final private DomainProcessorState            state;
57
58     final private int                             externalBase;
59
60     final private int                             resourceCount;
61
62     final private File[]                          files;
63
64     final private TGValueModifier                 valueModifier;
65
66     private volatile boolean                      closed          = false;
67
68     TIntArrayList                                 externalParents = new TIntArrayList();
69
70     ArrayList<String>                             externalNames   = new ArrayList<>();
71
72     TreeMap<String, String>                       downloads       = new TreeMap<String, String>();
73
74     public ModelTransferableGraphSource(final ReadGraph graph, TransferableGraphConfiguration2 configuration,
75             final DomainProcessorState state, File... fs) throws DatabaseException {
76
77         this.configuration = configuration;
78         this.state = state;
79         this.files = fs;
80         this.valueModifier = state.valueModifier;
81
82         SerialisationSupport ss = graph.getService(SerialisationSupport.class);
83
84         // At this point ids contains all internal resources. Now add roots and
85         // externals.
86
87         // Root Library
88         state.ids.put(ss.getTransientId(graph.getRootLibrary()), state.id++);
89
90         // External roots - internal roots were already processed as internal
91         // resources by domain processor
92         for (SeedSpec spec : configuration.seeds) {
93             if (SeedSpecType.SPECIAL_ROOT.equals(spec.specType)) {
94                 int resourceId = ss.getTransientId(spec.resource);
95                 state.ids.put(resourceId, state.id++);
96                 // The fixed roots have been seen as externals by domain
97                 // processor. Now remove them from external set.
98                 state.externals.remove(resourceId);
99             }
100         }
101
102         this.externalBase = state.id;
103
104         final Collection<String> errors = new HashSet<>();
105
106         // All resource considered as not internal by domain processor. Can also
107         // contain roots.
108         int[] externals = state.externals.toArray();
109
110         // Build up the state.externals, externalNames and externalParents
111         for (int i = 0; i < externals.length; i++) {
112             getId(graph, externals[i], errors);
113         }
114
115         state.inverses.forEachEntry(new TIntIntProcedure() {
116
117             @Override
118             public boolean execute(int predicate, int inverse) {
119                 try {
120                     getId(graph, predicate, errors);
121                     if (inverse != 0)
122                         getId(graph, inverse, errors);
123                 } catch (DatabaseException e) {
124                     throw new RuntimeDatabaseException(e);
125                 }
126                 return true;
127             }
128
129         });
130
131         if (!errors.isEmpty()) {
132             ArrayList<String> sorted = new ArrayList<>(errors);
133             Collections.sort(sorted);
134             StringBuilder message = new StringBuilder();
135             message.append("Errors in exported model:\n");
136             for (String error : sorted) {
137                 message.append(error);
138                 message.append("\n");
139             }
140             throw new DatabaseException(message.toString());
141         }
142
143         this.resourceCount = state.id;
144
145         state.extensions.put(ExternalDownloadBean.EXTENSION_KEY,
146                 new Variant(ExternalDownloadBean.BINDING, new ExternalDownloadBean(downloads)));
147
148     }
149
150     int indent = 0;
151
152     public boolean validateExternal(Resource ext) {
153         if (configuration.validate) {
154             ExtentStatus status = configuration.preStatus.get(ext);
155             if (status != null) {
156                 if (ExtentStatus.INTERNAL.equals(status))
157                     return false;
158                 else if (ExtentStatus.EXCLUDED.equals(status))
159                     return false;
160             }
161         }
162         return true;
163     }
164
165     private Resource getResource(ReadGraph graph, int r) throws DatabaseException {
166         SerialisationSupport ss = graph.getService(SerialisationSupport.class);
167         return ss.getResource(r);
168     }
169
170     final public int getExistingId(ReadGraph graph, int r) throws DatabaseException {
171
172         int ret = state.ids.get(r);
173         if (ret != -1) {
174             return ret;
175         } else {
176             SerialisationSupport ss = graph.getService(SerialisationSupport.class);
177             throw new DatabaseException(
178                     "Id has not been created for " + NameUtils.getSafeName(graph, ss.getResource(r)));
179         }
180
181     }
182
183     /*
184      * 
185      * @return -2 if r is not really external and the statement should be
186      * excluded
187      * 
188      */
189     public int getId(ReadGraph graph, int r, Collection<String> errors) throws DatabaseException {
190
191 //              // First external is root library
192 //              if(r == rootId) return internalCount;
193
194         SerialisationSupport ss = graph.getService(SerialisationSupport.class);
195         Layer0 L0 = Layer0.getInstance(graph);
196
197         if (state.ids.containsKey(r)) {
198             int ret = state.ids.get(r);
199             if (ret == -1) {
200                 for (int i = 0; i <= indent; ++i)
201                     System.out.print("  ");
202                 System.out.println("Cycle!!!"); // with " +
203                                                 // GraphUtils.getReadableName(g,
204                                                 // r));
205             }
206             return ret;
207         } else {
208             Resource res = getResource(graph, r);
209             if (!validateExternal(res)) {
210                 errors.add("Illegal reference to " + graph.getPossibleURI(getResource(graph, r)));
211                 return -2;
212             }
213             Collection<Resource> parents = graph.getObjects(res, L0.PartOf);
214             if (parents.size() != 1) {
215                 throw new ValidationException(
216                         "Reference to external resource " + NameUtils.getSafeName(graph, getResource(graph, r), true)
217                                 + " without unique uri (" + parents.size() + " parents).");
218             }
219             int pid = 0;
220             for (Resource p : parents) {
221                 ++indent;
222                 pid = getId(graph, ss.getTransientId(p), errors);
223                 if (pid == -2) {
224                     errors.add("Illegal reference to " + graph.getPossibleURI(getResource(graph, r)));
225                     return -2;
226                 }
227             }
228             --indent;
229             String name = graph.getRelatedValue(res, L0.HasName);
230             // Record the external entry
231             externalParents.add(pid);
232             externalNames.add(name);
233             state.ids.put(r, state.id);
234             // Ensure that this resource is included into the set of externals
235             // to maintain the total number of externals
236             state.externals.add(r);
237             String download = graph.getPossibleRelatedValue(res, L0.Ontology_download, Bindings.STRING);
238             if (download != null) {
239                 String uri = graph.getURI(res);
240                 downloads.put(uri, download);
241             }
242             return state.id++;
243         }
244     }
245
246     @Override
247     public DataContainer getHeader() throws Exception {
248         return null;
249     }
250
251     @Override
252     public int getResourceCount() {
253         return resourceCount;
254     }
255
256     private int countRootSeeds() {
257         int result = 0;
258         for (SeedSpec spec : configuration.seeds) {
259             if (SeedSpecType.INTERNAL.equals(spec.specType))
260                 continue;
261             result++;
262         }
263         return result;
264     }
265
266     @Override
267     public int getIdentityCount() {
268         return countRootSeeds() + state.externals.size() + state.internalEntries.size() + 1;
269     }
270
271     @Override
272     public int getStatementCount() {
273         return state.statementCount;
274     }
275
276     @Override
277     public int getValueCount() {
278         return state.valueCount;
279     }
280
281     @Override
282     public void forStatements(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
283
284         int[] value = new int[4];
285         long length = state.otherStatementsInput.length();
286         state.otherStatementsInput.position(0);
287
288         while (state.otherStatementsInput.position() < length && !state.monitor.isCanceled()) {
289
290             int s = state.otherStatementsInput.readInt();
291             int subjectId = state.ids.get(s);
292
293             boolean exclude = subjectId == -1;
294
295             int size = state.otherStatementsInput.readInt();
296             for (int i = 0; i < size; i++) {
297                 int p = state.otherStatementsInput.readInt();
298                 int o = state.otherStatementsInput.readInt();
299                 if (!exclude) {
300                     if (state.pending.contains(o)) {
301                         System.err.println("excluding garbage statement " + s + " " + p + " " + o
302                                 + ", object resource is garbage");
303                     } else if (state.excludedShared.contains(o)) {
304                         System.err.println("excluding shared " + s + " " + p + " " + o);
305                     } else {
306
307                         int objectId = getExistingId(graph, o);
308                         // The statement can be denied still
309                         if (objectId != -2) {
310                             value[0] = subjectId;
311                             value[1] = getExistingId(graph, p);
312                             int inverse = state.inverses.get(p);
313                             if (inverse != 0) {
314                                 value[2] = getExistingId(graph, inverse);
315                             } else {
316                                 value[2] = -1;
317                             }
318                             value[3] = objectId;
319
320                             try {
321                                 procedure.execute(value);
322                             } catch (Exception e) {
323                                 LOGGER.error("Error while processing a statement in transferable graph source", e);
324                             }
325
326                         } else {
327                             System.err.println("Denied (" + NameUtils.getSafeName(graph, getResource(graph, s)) + ", "
328                                     + NameUtils.getSafeName(graph, getResource(graph, p)) + ","
329                                     + NameUtils.getSafeName(graph, getResource(graph, o)) + ")");
330                         }
331
332                     }
333                 } else {
334                     System.err.println("excluding shared " + s);
335                 }
336             }
337         }
338     }
339
340     @Override
341     public void forValues(ReadGraph graph, TransferableGraphSourceProcedure<Value> procedure) throws Exception {
342
343         Serializer variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT);
344         List<Object> idContext = new ArrayList<>();
345         long length = state.valueInput.length();
346         state.valueInput.position(0);
347
348         while (state.valueInput.position() < length && !state.monitor.isCanceled()) {
349
350             // Ignore value type tag
351             int s = state.valueInput.readInt();
352             byte valueType = state.valueInput.readByte();
353             switch (valueType) {
354                 case TAG_RAW_COPY_VARIANT_VALUE:
355                     state.valueInput.readInt();
356                     // Intentional fallthrough.
357
358                 case TAG_POTENTIALLY_MODIFIED_VARIANT_VALUE: {
359                     idContext.clear();
360                     Variant variant = (Variant) variantSerializer.deserialize((DataInput) state.valueInput, idContext);
361                     if (valueModifier.mayNeedModification(variant.type())) {
362                         Object currentObject = variant.getValue();
363                         Object newObject = valueModifier.modify(state, variant.getBinding(), currentObject);
364                         if (newObject != currentObject)
365                             variant = new Variant(variant.getBinding(), newObject);
366                     }
367                     try {
368                         procedure.execute(new Value(state.ids.get(s), variant));
369                     } catch (Exception e) {
370                         LOGGER.error("Error while processing a value in transferable graph source", e);
371                     }
372                     break;
373                 }
374
375                 default:
376                     throw new IllegalArgumentException("Unrecognized variant value type encountered: " + valueType);
377             }
378         }
379     }
380
381     @Override
382     public void forValues2(ReadGraph graph, TransferableGraphSourceValueProcedure procedure) throws Exception {
383
384         Serializer datatypeSerializer = graph.getService(Databoard.class)
385                 .getSerializerUnchecked(Bindings.getBindingUnchecked(Datatype.class));
386         List<Object> idContext = new ArrayList<>();
387         long length = state.valueInput.length();
388         state.valueInput.position(0);
389
390         while (state.valueInput.position() < length && !state.monitor.isCanceled()) {
391
392             int s = state.valueInput.readInt();
393             byte valueType = state.valueInput.readByte();
394
395             switch (valueType) {
396                 case TAG_RAW_COPY_VARIANT_VALUE: {
397                     // Variant data could be copied raw, no need for
398                     // modifications
399                     // Note that the variant data is a concatenation of the
400                     // variant datatype serialization and the value
401                     // serialization.
402                     // variantLength contains both datatype and value data.
403                     int variantLength = state.valueInput.readInt();
404                     try {
405                         procedure.rawCopy(state.ids.get(s), variantLength, state.valueInput);
406                     } catch (Exception e) {
407                         LOGGER.error("Error while processing a raw value in transferable graph source", e);
408                     }
409                     break;
410                 }
411
412                 case TAG_POTENTIALLY_MODIFIED_VARIANT_VALUE: {
413                     // Variant data may need to be modified.
414                     // Cannot optimize this case with raw copying.
415                     idContext.clear();
416                     Datatype type = (Datatype) datatypeSerializer.deserialize((DataInput) state.valueInput, idContext);
417
418                     if (valueModifier.mayNeedModification(type)) {
419                         Binding binding = Bindings.getBinding(type);
420                         Serializer serializer = Bindings.getSerializerUnchecked(binding);
421                         Object value = serializer.deserialize((DataInput) state.valueInput);
422                         value = valueModifier.modify(state, binding, value);
423                         byte[] bytes = serializer.serialize(value);
424                         try {
425                             procedure.execute(state.ids.get(s), type, new ByteBufferReadable(bytes));
426                         } catch (Exception e) {
427                             LOGGER.error("Error while processing a data type in transferable graph source", e);
428                         }
429                     } else {
430                         try {
431                             procedure.execute(state.ids.get(s), type, state.valueInput);
432                         } catch (Exception e) {
433                             LOGGER.error("Error while processing a raw value in transferable graph source", e);
434                         }
435                     }
436                     break;
437                 }
438
439                 default:
440                     throw new IllegalArgumentException("Unrecognized variant value type encountered: " + valueType);
441             }
442         }
443     }
444
445     protected Identity getRootIdentity(DomainProcessorState state, SerialisationSupport support, Resource rootLibrary)
446             throws DatabaseException {
447         return new Identity(state.ids.get(support.getTransientId(rootLibrary)), new External(-1, ""));
448     }
449
450     @Override
451     public void forIdentities(ReadGraph graph, TransferableGraphSourceProcedure<Identity> procedure) throws Exception {
452
453         SerialisationSupport support = graph.getService(SerialisationSupport.class);
454         Layer0 L0 = Layer0.getInstance(graph);
455
456         // TODO: this should be Root with name ""
457         try {
458             procedure.execute(getRootIdentity(state, support, graph.getRootLibrary()));
459         } catch (Exception e) {
460             LOGGER.error("Error while processing a root identity in transferable graph source", e);
461         }
462
463         TIntObjectMap<Identity> internalMap = new TIntObjectHashMap<>(100, 0.5f, Integer.MIN_VALUE);
464
465         // Declare internal and external roots
466         for (SeedSpec r : configuration.seeds) {
467             if (SeedSpecType.INTERNAL.equals(r.specType))
468                 continue;
469             String typeId = r.type;
470             if (typeId == null) {
471                 Resource type = graph.getPossibleType(r.resource, L0.Entity);
472                 typeId = type != null ? graph.getURI(type) : Layer0.URIs.Entity;
473             }
474             int id = state.ids.get(support.getTransientId(r.resource));
475             Root root = new Root(r.name, typeId);
476             Identity rootId = new Identity(id, root);
477             internalMap.put(id, rootId);
478             try {
479                 procedure.execute(rootId);
480             } catch (Exception e) {
481                 LOGGER.error("Error while processing a root in transferable graph source", e);
482             }
483         }
484
485         for (int i = 0; i < state.externals.size(); i++) {
486             int parent = externalParents.get(i);
487             String name = externalNames.get(i);
488             try {
489                 procedure.execute(new Identity(externalBase + i, new External(parent, name)));
490             } catch (Exception e) {
491                 LOGGER.error("Error while processing a an external identity in transferable graph source", e);
492             }
493         }
494
495         if (state.internalEntries != null) {
496             for (ConsistsOfProcessEntry ie : state.internalEntries) {
497                 if (ie.parent != null) {
498                     if (ie.name != null) {
499                         try {
500                             procedure.execute(resolveInternal(graph, support, ie, internalMap));
501                         } catch (Exception e) {
502                             LOGGER.error("Error while processing an internal identity in transferable graph source", e);
503                         }
504                     } else {
505                         // In this case there is a child that has no HasName =>
506                         // this should be treated as a blank
507                     }
508                 } else {
509                     try {
510                         procedure.execute(resolveInternal(graph, support, ie, internalMap));
511                     } catch (Exception e) {
512                         LOGGER.error("Error while processing an internal identity in transferable graph source", e);
513                     }
514                 }
515             }
516         }
517
518     }
519
520     private Identity resolveInternal(ReadGraph graph, SerialisationSupport ss, ConsistsOfProcessEntry entry,
521             TIntObjectMap<Identity> internalMap) throws DatabaseException {
522         int id = state.ids.get(ss.getTransientId(entry.resource));
523         Identity existing = internalMap.get(id);
524         if (existing != null)
525             return existing;
526
527         if (entry.parent == null) {
528             Layer0 L0 = Layer0.getInstance(graph);
529             Resource possibleParent = graph.getPossibleObject(entry.resource, L0.PartOf);
530             if (possibleParent == null)
531                 throw new DatabaseException("Invalid root or internal parent path: " + entry.resource);
532             int externalId = state.ids.get(ss.getTransientId(possibleParent));
533             Identity result = new Identity(id, new Internal(externalId, entry.name));
534             internalMap.put(id, result);
535             return result;
536         } else {
537             Identity parent = resolveInternal(graph, ss, entry.parent, internalMap);
538             Identity result = new Identity(id, new Internal(parent.resource, entry.name));
539             internalMap.put(id, result);
540             return result;
541         }
542     }
543
544     @Override
545     public TreeMap<String, Variant> getExtensions() {
546         return state.extensions;
547     }
548
549     public File[] getFiles() {
550         return files;
551     }
552
553     private static <T> T tryClose(T c) throws IOException {
554         if (c != null && c instanceof Closeable)
555             ((Closeable) c).close();
556         return null;
557     }
558
559     public void closeStreams() throws IOException {
560         state.valueInput = tryClose(state.valueInput);
561         state.otherStatementsInput = tryClose(state.otherStatementsInput);
562         state.statementsOutput = tryClose(state.statementsOutput);
563         state.valueOutput = tryClose(state.valueOutput);
564     }
565
566     @Override
567     public void reset() throws Exception {
568         throw new UnsupportedOperationException();
569     }
570
571     public long[] getResourceArray(ReadGraph graph) throws DatabaseException {
572         final SerialisationSupport ss = graph.getService(SerialisationSupport.class);
573         final long[] result = new long[state.ids.size()];
574         state.ids.forEachEntry(new TIntIntProcedure() {
575
576             @Override
577             public boolean execute(int a, int b) {
578
579                 try {
580                     Resource r = ss.getResource(a);
581                     result[b] = r.getResourceId();
582                 } catch (DatabaseException e) {
583                     e.printStackTrace();
584                 }
585
586                 return true;
587
588             }
589         });
590         return result;
591     }
592
593     public DomainProcessorState getState() {
594         return state;
595     }
596
597     public void forResourceStatements(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure)
598             throws Exception {
599
600         int[] value = new int[4];
601         long length = state.otherStatementsInput.length();
602         state.otherStatementsInput.position(0);
603
604         while (state.otherStatementsInput.position() < length) {
605
606             int s = state.otherStatementsInput.readInt();
607             int subjectId = state.ids.get(s);
608
609             boolean exclude = subjectId == -1;
610
611             int size = state.otherStatementsInput.readInt();
612             for (int i = 0; i < size; i++) {
613                 int p = state.otherStatementsInput.readInt();
614                 int o = state.otherStatementsInput.readInt();
615                 if (!exclude) {
616                     if (state.excludedShared.contains(o)) {
617                         System.err.println("excluding shared " + s + " " + p + " " + o);
618                     } else {
619
620                         int objectId = getExistingId(graph, o);
621                         // The statement can be denied still
622                         if (objectId != -2) {
623                             value[0] = s;
624                             value[1] = p;
625                             int inverse = state.inverses.get(p);
626                             if (inverse != 0) {
627                                 value[2] = inverse;
628                             } else {
629                                 value[2] = -1;
630                             }
631                             value[3] = o;
632
633                             try {
634                                 procedure.execute(value);
635                             } catch (Exception e) {
636                                 LOGGER.error("Error while processing a statement in transferable graph source", e);
637                             }
638
639                         } else {
640                             System.err.println("Denied (" + NameUtils.getSafeName(graph, getResource(graph, s)) + ", "
641                                     + NameUtils.getSafeName(graph, getResource(graph, p)) + ","
642                                     + NameUtils.getSafeName(graph, getResource(graph, o)) + ")");
643                         }
644
645                     }
646                 } else {
647                     System.err.println("excluding shared " + s);
648                 }
649             }
650         }
651     }
652
653     public void forValueResources(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
654         int[] value = { 0 };
655         long length = state.valueInput.length();
656         while (state.valueInput.position() < length) {
657             value[0] = state.valueInput.readInt();
658             try {
659                 procedure.execute(value);
660             } catch (Exception e) {
661                 LOGGER.error("Error while processing a value in transferable graph source", e);
662             }
663         }
664     }
665
666     public TransferableGraphConfiguration2 getConfiguration() {
667         return configuration;
668     }
669
670     @Override
671     public void close() throws IOException {
672         synchronized (this) {
673             if (closed)
674                 return;
675             closed = true;
676         }
677         closeStreams();
678         if (files != null) {
679             for (File f : files) {
680                 Files.deleteIfExists(f.toPath());
681             }
682         }
683     }
684
685 }