1 package org.simantics.db.layer0.util;
\r
3 import java.io.Closeable;
\r
4 import java.io.DataInput;
\r
6 import java.io.IOException;
\r
7 import java.nio.file.Files;
\r
8 import java.util.ArrayList;
\r
9 import java.util.Collection;
\r
10 import java.util.Collections;
\r
11 import java.util.HashSet;
\r
12 import java.util.List;
\r
13 import java.util.TreeMap;
\r
15 import org.simantics.databoard.Bindings;
\r
16 import org.simantics.databoard.Databoard;
\r
17 import org.simantics.databoard.binding.Binding;
\r
18 import org.simantics.databoard.binding.mutable.Variant;
\r
19 import org.simantics.databoard.container.DataContainer;
\r
20 import org.simantics.databoard.serialization.Serializer;
\r
21 import org.simantics.databoard.type.Datatype;
\r
22 import org.simantics.databoard.util.binary.ByteBufferReadable;
\r
23 import org.simantics.db.ReadGraph;
\r
24 import org.simantics.db.Resource;
\r
25 import org.simantics.db.common.utils.NameUtils;
\r
26 import org.simantics.db.exception.DatabaseException;
\r
27 import org.simantics.db.exception.RuntimeDatabaseException;
\r
28 import org.simantics.db.exception.ValidationException;
\r
29 import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus;
\r
30 import org.simantics.db.layer0.util.TransferableGraphConfiguration2.RootSpec;
\r
31 import org.simantics.db.service.SerialisationSupport;
\r
32 import org.simantics.graph.db.TransferableGraphSource;
\r
33 import org.simantics.graph.representation.External;
\r
34 import org.simantics.graph.representation.Identity;
\r
35 import org.simantics.graph.representation.Root;
\r
36 import org.simantics.graph.representation.Value;
\r
37 import org.simantics.layer0.Layer0;
\r
39 import gnu.trove.list.array.TIntArrayList;
\r
40 import gnu.trove.procedure.TIntIntProcedure;
\r
42 public class ModelTransferableGraphSource implements TransferableGraphSource {
\r
44 final private TransferableGraphConfiguration2 configuration;
\r
45 final private DomainProcessorState state;
\r
46 final private int externalBase;
\r
47 final private int resourceCount;
\r
48 final private File[] files;
\r
49 final private TGValueModifier valueModifier;
\r
51 private volatile boolean closed = false;
\r
53 TIntArrayList externalParents = new TIntArrayList();
\r
54 ArrayList<String> externalNames = new ArrayList<String>();
\r
56 public ModelTransferableGraphSource(final ReadGraph graph, TransferableGraphConfiguration2 configuration, final DomainProcessorState state, File ... fs) throws DatabaseException {
\r
58 this.configuration = configuration;
\r
61 this.valueModifier = state.valueModifier;
\r
63 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
\r
65 // At this point ids contains all internal resources. Now add roots and externals.
\r
68 state.ids.put(ss.getTransientId(graph.getRootLibrary()), state.id++);
\r
70 // External roots - internal roots were already processed as internal resources by domain processor
\r
71 for(RootSpec rs : configuration.roots) {
\r
73 int resourceId = ss.getTransientId(rs.resource);
\r
74 state.ids.put(resourceId, state.id++);
\r
75 // The fixed roots have been seen as externals by domain processor. Now remove them from external set.
\r
76 state.externals.remove(resourceId);
\r
80 this.externalBase = state.id;
\r
82 final Collection<String> errors = new HashSet<String>();
\r
84 // All resource considered as not internal by domain processor. Can also contain roots.
\r
85 int[] externals = state.externals.toArray();
\r
87 // Build up the state.externals, externalNames and externalParents
\r
88 for(int i=0;i<externals.length;i++) {
\r
89 getId(graph, externals[i], errors);
\r
92 state.inverses.forEachEntry(new TIntIntProcedure() {
\r
95 public boolean execute(int predicate, int inverse) {
\r
97 getId(graph, predicate, errors);
\r
98 if(inverse != 0) getId(graph, inverse, errors);
\r
99 } catch (DatabaseException e) {
\r
100 throw new RuntimeDatabaseException(e);
\r
107 if(!errors.isEmpty()) {
\r
108 ArrayList<String> sorted = new ArrayList<String>(errors);
\r
109 Collections.sort(sorted);
\r
110 StringBuilder message = new StringBuilder();
\r
111 message.append("Errors in exported model:\n");
\r
112 for(String error : sorted) {
\r
113 message.append(error);
\r
114 message.append("\n");
\r
116 throw new DatabaseException(message.toString());
\r
119 this.resourceCount = state.id;
\r
125 public boolean validateExternal(Resource ext) {
\r
126 if(configuration.validate) {
\r
127 ExtentStatus status = configuration.preStatus.get(ext);
\r
128 if(status != null) {
\r
129 if(ExtentStatus.INTERNAL.equals(status)) return false;
\r
130 else if(ExtentStatus.EXCLUDED.equals(status)) return false;
\r
136 private Resource getResource(ReadGraph graph, int r) throws DatabaseException {
\r
137 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
\r
138 return ss.getResource(r);
\r
141 final public int getExistingId(ReadGraph graph, int r) throws DatabaseException {
\r
143 int ret = state.ids.get(r);
\r
147 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
\r
148 throw new DatabaseException("Id has not been created for " + NameUtils.getSafeName(graph, ss.getResource(r)));
\r
155 * @return -2 if r is not really external and the statement should be excluded
\r
158 public int getId(ReadGraph graph, int r, Collection<String> errors) throws DatabaseException {
\r
160 // // First external is root library
\r
161 // if(r == rootId) return internalCount;
\r
163 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
\r
164 Layer0 L0 = Layer0.getInstance(graph);
\r
166 if(state.ids.containsKey(r)) {
\r
167 int ret = state.ids.get(r);
\r
169 for(int i=0;i<=indent;++i)
\r
170 System.out.print(" ");
\r
171 System.out.println("Cycle!!!"); // with " + GraphUtils.getReadableName(g, r));
\r
176 Resource res = getResource(graph, r);
\r
177 if(!validateExternal(res)) {
\r
178 errors.add("Illegal reference to " + graph.getPossibleURI(getResource(graph, r)));
\r
181 Collection<Resource> parents = graph.getObjects(res, L0.PartOf);
\r
182 if(parents.size() != 1) {
\r
183 throw new ValidationException("Reference to external resource "
\r
184 + NameUtils.getSafeName(graph, getResource(graph, r), true) + " without unique uri (" + parents.size() + " parents).");
\r
187 for(Resource p : parents) {
\r
189 pid = getId(graph, ss.getTransientId(p), errors);
\r
191 errors.add("Illegal reference to " + graph.getPossibleURI(getResource(graph, r)));
\r
196 String name = graph.getRelatedValue(res, L0.HasName);
\r
197 // Record the external entry
\r
198 externalParents.add(pid);
\r
199 externalNames.add(name);
\r
200 state.ids.put(r, state.id);
\r
201 // Ensure that this resource is included into the set of externals to maintain the total number of externals
\r
202 state.externals.add(r);
\r
208 public DataContainer getHeader() throws Exception {
\r
213 public int getResourceCount() {
\r
214 return resourceCount;
\r
218 public int getIdentityCount() {
\r
219 return configuration.roots.size() + state.externals.size() + 1;
\r
223 public int getStatementCount() {
\r
224 return state.statementCount;
\r
228 public int getValueCount() {
\r
229 return state.valueCount;
\r
233 public void forStatements(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
\r
235 int[] value = new int[4];
\r
236 long length = state.otherStatementsInput.length();
\r
237 state.otherStatementsInput.position(0);
\r
239 while(state.otherStatementsInput.position() < length && !state.monitor.isCanceled()) {
\r
241 int s = state.otherStatementsInput.readInt();
\r
242 int subjectId = state.ids.get(s);
\r
244 boolean exclude = subjectId == -1;
\r
246 int size = state.otherStatementsInput.readInt();
\r
247 for(int i=0;i<size;i++) {
\r
248 int p = state.otherStatementsInput.readInt();
\r
249 int o = state.otherStatementsInput.readInt();
\r
251 if(state.pending.contains(o)) {
\r
252 System.err.println("excluding garbage statement " + s + " " + p + " " + o + ", object resource is garbage");
\r
253 } else if(state.excludedShared.contains(o)) {
\r
254 System.err.println("excluding shared " + s + " " + p + " " + o);
\r
257 int objectId = getExistingId(graph, o);
\r
258 // The statement can be denied still
\r
259 if(objectId != -2) {
\r
260 value[0] = subjectId;
\r
261 value[1] = getExistingId(graph, p);
\r
262 int inverse = state.inverses.get(p);
\r
264 value[2] = getExistingId(graph, inverse);
\r
268 value[3] = objectId;
\r
270 procedure.execute(value);
\r
273 System.err.println("Denied (" + NameUtils.getSafeName(graph, getResource(graph, s)) + ", " + NameUtils.getSafeName(graph, getResource(graph, p)) + "," + NameUtils.getSafeName(graph, getResource(graph, o)) + ")");
\r
278 System.err.println("excluding shared " + s);
\r
285 public void forValues(ReadGraph graph, TransferableGraphSourceProcedure<Value> procedure) throws Exception {
\r
287 Serializer variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT);
\r
288 List<Object> idContext = new ArrayList<>();
\r
289 long length = state.valueInput.length();
\r
290 state.valueInput.position(0);
\r
292 while(state.valueInput.position() < length && !state.monitor.isCanceled()) {
\r
294 // Ignore value type tag
\r
295 int s = state.valueInput.readInt();
\r
296 byte valueType = state.valueInput.readByte();
\r
297 switch (valueType) {
\r
298 case TAG_RAW_COPY_VARIANT_VALUE:
\r
299 state.valueInput.readInt();
\r
300 // Intentional fallthrough.
\r
302 case TAG_POTENTIALLY_MODIFIED_VARIANT_VALUE: {
\r
304 Variant variant = (Variant) variantSerializer.deserialize((DataInput)state.valueInput, idContext);
\r
305 if (valueModifier.mayNeedModification(variant.type())) {
\r
306 Object currentObject = variant.getValue();
\r
307 Object newObject = valueModifier.modify(state, variant.getBinding(), currentObject);
\r
308 if (newObject != currentObject)
\r
309 variant = new Variant(variant.getBinding(), newObject);
\r
311 procedure.execute(new Value(state.ids.get(s), variant));
\r
316 throw new IllegalArgumentException("Unrecognized variant value type encountered: " + valueType);
\r
322 public void forValues2(ReadGraph graph, TransferableGraphSourceValueProcedure procedure) throws Exception {
\r
324 Serializer datatypeSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.getBindingUnchecked(Datatype.class));
\r
325 List<Object> idContext = new ArrayList<>();
\r
326 long length = state.valueInput.length();
\r
327 state.valueInput.position(0);
\r
329 while(state.valueInput.position() < length && !state.monitor.isCanceled()) {
\r
331 int s = state.valueInput.readInt();
\r
332 byte valueType = state.valueInput.readByte();
\r
334 switch (valueType) {
\r
335 case TAG_RAW_COPY_VARIANT_VALUE: {
\r
336 // Variant data could be copied raw, no need for modifications
\r
337 // Note that the variant data is a concatenation of the
\r
338 // variant datatype serialization and the value serialization.
\r
339 // variantLength contains both datatype and value data.
\r
340 int variantLength = state.valueInput.readInt();
\r
341 procedure.rawCopy(state.ids.get(s), variantLength, state.valueInput);
\r
345 case TAG_POTENTIALLY_MODIFIED_VARIANT_VALUE: {
\r
346 // Variant data may need to be modified.
\r
347 // Cannot optimize this case with raw copying.
\r
349 Datatype type = (Datatype)datatypeSerializer.deserialize((DataInput)state.valueInput, idContext);
\r
351 if (valueModifier.mayNeedModification(type)) {
\r
352 Binding binding = Bindings.getBinding(type);
\r
353 Serializer serializer = Bindings.getSerializerUnchecked(binding);
\r
354 Object value = serializer.deserialize((DataInput)state.valueInput);
\r
355 value = valueModifier.modify(state, binding, value);
\r
356 byte[] bytes = serializer.serialize(value);
\r
357 procedure.execute(state.ids.get(s), type, new ByteBufferReadable(bytes));
\r
359 procedure.execute(state.ids.get(s), type, state.valueInput);
\r
365 throw new IllegalArgumentException("Unrecognized variant value type encountered: " + valueType);
\r
370 protected Identity getRootIdentity(DomainProcessorState state, SerialisationSupport support, Resource rootLibrary) throws DatabaseException {
\r
371 return new Identity(state.ids.get(support.getTransientId(rootLibrary)), new External(-1, ""));
\r
375 public void forIdentities(ReadGraph graph, TransferableGraphSourceProcedure<Identity> procedure) throws Exception {
\r
377 SerialisationSupport support = graph.getService(SerialisationSupport.class);
\r
378 Layer0 L0 = Layer0.getInstance(graph);
\r
380 // TODO: this should be Root with name ""
\r
381 procedure.execute(getRootIdentity(state, support, graph.getRootLibrary()));
\r
383 // Declare internal and external roots
\r
384 for(RootSpec r : configuration.roots) {
\r
385 Resource type = graph.getPossibleType(r.resource, L0.Entity);
\r
386 if(type == null) type = L0.Entity;
\r
387 procedure.execute(new Identity(
\r
388 state.ids.get(support.getTransientId(r.resource)),
\r
389 new Root(r.name, graph.getURI(type))));
\r
392 for(int i = 0; i < state.externals.size() ; i++) {
\r
393 int parent = externalParents.get(i);
\r
394 String name = externalNames.get(i);
\r
395 procedure.execute(new Identity(externalBase + i, new External(parent,name)));
\r
401 public TreeMap<String, Variant> getExtensions() {
\r
402 return state.extensions;
\r
405 public File[] getFiles() {
\r
409 private static <T> T tryClose(T c) throws IOException {
\r
410 if (c != null && c instanceof Closeable)
\r
411 ((Closeable) c).close();
\r
415 public void closeStreams() throws IOException {
\r
416 state.valueInput = tryClose(state.valueInput);
\r
417 state.otherStatementsInput = tryClose(state.otherStatementsInput);
\r
418 state.statementsOutput = tryClose(state.statementsOutput);
\r
419 state.valueOutput = tryClose(state.valueOutput);
\r
423 public void reset() throws Exception {
\r
424 throw new UnsupportedOperationException();
\r
427 public long[] getResourceArray(ReadGraph graph) throws DatabaseException {
\r
428 final SerialisationSupport ss = graph.getService(SerialisationSupport.class);
\r
429 final long[] result = new long[state.ids.size()];
\r
430 state.ids.forEachEntry(new TIntIntProcedure() {
\r
433 public boolean execute(int a, int b) {
\r
436 Resource r = ss.getResource(a);
\r
437 result[b] = r.getResourceId();
\r
438 } catch (DatabaseException e) {
\r
439 e.printStackTrace();
\r
449 public DomainProcessorState getState() {
\r
453 public void forResourceStatements(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
\r
455 int[] value = new int[4];
\r
456 long length = state.otherStatementsInput.length();
\r
457 state.otherStatementsInput.position(0);
\r
459 while(state.otherStatementsInput.position() < length) {
\r
461 int s = state.otherStatementsInput.readInt();
\r
462 int subjectId = state.ids.get(s);
\r
464 boolean exclude = subjectId == -1;
\r
466 int size = state.otherStatementsInput.readInt();
\r
467 for(int i=0;i<size;i++) {
\r
468 int p = state.otherStatementsInput.readInt();
\r
469 int o = state.otherStatementsInput.readInt();
\r
471 if(state.excludedShared.contains(o)) {
\r
472 System.err.println("excluding shared " + s + " " + p + " " + o);
\r
475 int objectId = getExistingId(graph, o);
\r
476 // The statement can be denied still
\r
477 if(objectId != -2) {
\r
480 int inverse = state.inverses.get(p);
\r
482 value[2] = inverse;
\r
488 procedure.execute(value);
\r
491 System.err.println("Denied (" + NameUtils.getSafeName(graph, getResource(graph, s)) + ", " + NameUtils.getSafeName(graph, getResource(graph, p)) + "," + NameUtils.getSafeName(graph, getResource(graph, o)) + ")");
\r
496 System.err.println("excluding shared " + s);
\r
502 public void forValueResources(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
\r
503 int[] value = { 0 };
\r
504 long length = state.valueInput.length();
\r
505 while (state.valueInput.position() < length) {
\r
506 value[0] = state.valueInput.readInt();
\r
507 procedure.execute(value);
\r
511 public TransferableGraphConfiguration2 getConfiguration() {
\r
512 return configuration;
\r
516 public void close() throws IOException {
\r
517 synchronized (this) {
\r
523 if (files != null) {
\r
524 for (File f : files) {
\r
525 Files.deleteIfExists(f.toPath());
\r