1 package org.simantics.db.layer0.util;
3 import java.io.Closeable;
4 import java.io.DataInput;
6 import java.io.IOException;
7 import java.nio.file.Files;
8 import java.util.ArrayList;
9 import java.util.Collection;
10 import java.util.Collections;
11 import java.util.HashSet;
12 import java.util.List;
13 import java.util.TreeMap;
15 import org.simantics.databoard.Bindings;
16 import org.simantics.databoard.Databoard;
17 import org.simantics.databoard.binding.Binding;
18 import org.simantics.databoard.binding.mutable.Variant;
19 import org.simantics.databoard.container.DataContainer;
20 import org.simantics.databoard.serialization.Serializer;
21 import org.simantics.databoard.type.Datatype;
22 import org.simantics.databoard.util.binary.ByteBufferReadable;
23 import org.simantics.db.ReadGraph;
24 import org.simantics.db.Resource;
25 import org.simantics.db.common.utils.NameUtils;
26 import org.simantics.db.exception.DatabaseException;
27 import org.simantics.db.exception.RuntimeDatabaseException;
28 import org.simantics.db.exception.ValidationException;
29 import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus;
30 import org.simantics.db.layer0.util.TransferableGraphConfiguration2.RootSpec;
31 import org.simantics.db.service.SerialisationSupport;
32 import org.simantics.graph.db.TransferableGraphSource;
33 import org.simantics.graph.representation.External;
34 import org.simantics.graph.representation.Identity;
35 import org.simantics.graph.representation.Root;
36 import org.simantics.graph.representation.Value;
37 import org.simantics.layer0.Layer0;
39 import gnu.trove.list.array.TIntArrayList;
40 import gnu.trove.procedure.TIntIntProcedure;
42 public class ModelTransferableGraphSource implements TransferableGraphSource {
44 final private TransferableGraphConfiguration2 configuration;
45 final private DomainProcessorState state;
46 final private int externalBase;
47 final private int resourceCount;
48 final private File[] files;
49 final private TGValueModifier valueModifier;
51 private volatile boolean closed = false;
53 TIntArrayList externalParents = new TIntArrayList();
54 ArrayList<String> externalNames = new ArrayList<String>();
56 public ModelTransferableGraphSource(final ReadGraph graph, TransferableGraphConfiguration2 configuration, final DomainProcessorState state, File ... fs) throws DatabaseException {
58 this.configuration = configuration;
61 this.valueModifier = state.valueModifier;
63 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
65 // At this point ids contains all internal resources. Now add roots and externals.
68 state.ids.put(ss.getTransientId(graph.getRootLibrary()), state.id++);
70 // External roots - internal roots were already processed as internal resources by domain processor
71 for(RootSpec rs : configuration.roots) {
73 int resourceId = ss.getTransientId(rs.resource);
74 state.ids.put(resourceId, state.id++);
75 // The fixed roots have been seen as externals by domain processor. Now remove them from external set.
76 state.externals.remove(resourceId);
80 this.externalBase = state.id;
82 final Collection<String> errors = new HashSet<String>();
84 // All resource considered as not internal by domain processor. Can also contain roots.
85 int[] externals = state.externals.toArray();
87 // Build up the state.externals, externalNames and externalParents
88 for(int i=0;i<externals.length;i++) {
89 getId(graph, externals[i], errors);
92 state.inverses.forEachEntry(new TIntIntProcedure() {
95 public boolean execute(int predicate, int inverse) {
97 getId(graph, predicate, errors);
98 if(inverse != 0) getId(graph, inverse, errors);
99 } catch (DatabaseException e) {
100 throw new RuntimeDatabaseException(e);
107 if(!errors.isEmpty()) {
108 ArrayList<String> sorted = new ArrayList<String>(errors);
109 Collections.sort(sorted);
110 StringBuilder message = new StringBuilder();
111 message.append("Errors in exported model:\n");
112 for(String error : sorted) {
113 message.append(error);
114 message.append("\n");
116 throw new DatabaseException(message.toString());
119 this.resourceCount = state.id;
125 public boolean validateExternal(Resource ext) {
126 if(configuration.validate) {
127 ExtentStatus status = configuration.preStatus.get(ext);
129 if(ExtentStatus.INTERNAL.equals(status)) return false;
130 else if(ExtentStatus.EXCLUDED.equals(status)) return false;
136 private Resource getResource(ReadGraph graph, int r) throws DatabaseException {
137 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
138 return ss.getResource(r);
141 final public int getExistingId(ReadGraph graph, int r) throws DatabaseException {
143 int ret = state.ids.get(r);
147 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
148 throw new DatabaseException("Id has not been created for " + NameUtils.getSafeName(graph, ss.getResource(r)));
155 * @return -2 if r is not really external and the statement should be excluded
158 public int getId(ReadGraph graph, int r, Collection<String> errors) throws DatabaseException {
160 // // First external is root library
161 // if(r == rootId) return internalCount;
163 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
164 Layer0 L0 = Layer0.getInstance(graph);
166 if(state.ids.containsKey(r)) {
167 int ret = state.ids.get(r);
169 for(int i=0;i<=indent;++i)
170 System.out.print(" ");
171 System.out.println("Cycle!!!"); // with " + GraphUtils.getReadableName(g, r));
176 Resource res = getResource(graph, r);
177 if(!validateExternal(res)) {
178 errors.add("Illegal reference to " + graph.getPossibleURI(getResource(graph, r)));
181 Collection<Resource> parents = graph.getObjects(res, L0.PartOf);
182 if(parents.size() != 1) {
183 throw new ValidationException("Reference to external resource "
184 + NameUtils.getSafeName(graph, getResource(graph, r), true) + " without unique uri (" + parents.size() + " parents).");
187 for(Resource p : parents) {
189 pid = getId(graph, ss.getTransientId(p), errors);
191 errors.add("Illegal reference to " + graph.getPossibleURI(getResource(graph, r)));
196 String name = graph.getRelatedValue(res, L0.HasName);
197 // Record the external entry
198 externalParents.add(pid);
199 externalNames.add(name);
200 state.ids.put(r, state.id);
201 // Ensure that this resource is included into the set of externals to maintain the total number of externals
202 state.externals.add(r);
208 public DataContainer getHeader() throws Exception {
213 public int getResourceCount() {
214 return resourceCount;
218 public int getIdentityCount() {
219 return configuration.roots.size() + state.externals.size() + 1;
223 public int getStatementCount() {
224 return state.statementCount;
228 public int getValueCount() {
229 return state.valueCount;
233 public void forStatements(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
235 int[] value = new int[4];
236 long length = state.otherStatementsInput.length();
237 state.otherStatementsInput.position(0);
239 while(state.otherStatementsInput.position() < length && !state.monitor.isCanceled()) {
241 int s = state.otherStatementsInput.readInt();
242 int subjectId = state.ids.get(s);
244 boolean exclude = subjectId == -1;
246 int size = state.otherStatementsInput.readInt();
247 for(int i=0;i<size;i++) {
248 int p = state.otherStatementsInput.readInt();
249 int o = state.otherStatementsInput.readInt();
251 if(state.pending.contains(o)) {
252 System.err.println("excluding garbage statement " + s + " " + p + " " + o + ", object resource is garbage");
253 } else if(state.excludedShared.contains(o)) {
254 System.err.println("excluding shared " + s + " " + p + " " + o);
257 int objectId = getExistingId(graph, o);
258 // The statement can be denied still
260 value[0] = subjectId;
261 value[1] = getExistingId(graph, p);
262 int inverse = state.inverses.get(p);
264 value[2] = getExistingId(graph, inverse);
270 procedure.execute(value);
273 System.err.println("Denied (" + NameUtils.getSafeName(graph, getResource(graph, s)) + ", " + NameUtils.getSafeName(graph, getResource(graph, p)) + "," + NameUtils.getSafeName(graph, getResource(graph, o)) + ")");
278 System.err.println("excluding shared " + s);
285 public void forValues(ReadGraph graph, TransferableGraphSourceProcedure<Value> procedure) throws Exception {
287 Serializer variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT);
288 List<Object> idContext = new ArrayList<>();
289 long length = state.valueInput.length();
290 state.valueInput.position(0);
292 while(state.valueInput.position() < length && !state.monitor.isCanceled()) {
294 // Ignore value type tag
295 int s = state.valueInput.readInt();
296 byte valueType = state.valueInput.readByte();
298 case TAG_RAW_COPY_VARIANT_VALUE:
299 state.valueInput.readInt();
300 // Intentional fallthrough.
302 case TAG_POTENTIALLY_MODIFIED_VARIANT_VALUE: {
304 Variant variant = (Variant) variantSerializer.deserialize((DataInput)state.valueInput, idContext);
305 if (valueModifier.mayNeedModification(variant.type())) {
306 Object currentObject = variant.getValue();
307 Object newObject = valueModifier.modify(state, variant.getBinding(), currentObject);
308 if (newObject != currentObject)
309 variant = new Variant(variant.getBinding(), newObject);
311 procedure.execute(new Value(state.ids.get(s), variant));
316 throw new IllegalArgumentException("Unrecognized variant value type encountered: " + valueType);
322 public void forValues2(ReadGraph graph, TransferableGraphSourceValueProcedure procedure) throws Exception {
324 Serializer datatypeSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.getBindingUnchecked(Datatype.class));
325 List<Object> idContext = new ArrayList<>();
326 long length = state.valueInput.length();
327 state.valueInput.position(0);
329 while(state.valueInput.position() < length && !state.monitor.isCanceled()) {
331 int s = state.valueInput.readInt();
332 byte valueType = state.valueInput.readByte();
335 case TAG_RAW_COPY_VARIANT_VALUE: {
336 // Variant data could be copied raw, no need for modifications
337 // Note that the variant data is a concatenation of the
338 // variant datatype serialization and the value serialization.
339 // variantLength contains both datatype and value data.
340 int variantLength = state.valueInput.readInt();
341 procedure.rawCopy(state.ids.get(s), variantLength, state.valueInput);
345 case TAG_POTENTIALLY_MODIFIED_VARIANT_VALUE: {
346 // Variant data may need to be modified.
347 // Cannot optimize this case with raw copying.
349 Datatype type = (Datatype)datatypeSerializer.deserialize((DataInput)state.valueInput, idContext);
351 if (valueModifier.mayNeedModification(type)) {
352 Binding binding = Bindings.getBinding(type);
353 Serializer serializer = Bindings.getSerializerUnchecked(binding);
354 Object value = serializer.deserialize((DataInput)state.valueInput);
355 value = valueModifier.modify(state, binding, value);
356 byte[] bytes = serializer.serialize(value);
357 procedure.execute(state.ids.get(s), type, new ByteBufferReadable(bytes));
359 procedure.execute(state.ids.get(s), type, state.valueInput);
365 throw new IllegalArgumentException("Unrecognized variant value type encountered: " + valueType);
370 protected Identity getRootIdentity(DomainProcessorState state, SerialisationSupport support, Resource rootLibrary) throws DatabaseException {
371 return new Identity(state.ids.get(support.getTransientId(rootLibrary)), new External(-1, ""));
375 public void forIdentities(ReadGraph graph, TransferableGraphSourceProcedure<Identity> procedure) throws Exception {
377 SerialisationSupport support = graph.getService(SerialisationSupport.class);
378 Layer0 L0 = Layer0.getInstance(graph);
380 // TODO: this should be Root with name ""
381 procedure.execute(getRootIdentity(state, support, graph.getRootLibrary()));
383 // Declare internal and external roots
384 for(RootSpec r : configuration.roots) {
385 Resource type = graph.getPossibleType(r.resource, L0.Entity);
386 if(type == null) type = L0.Entity;
387 procedure.execute(new Identity(
388 state.ids.get(support.getTransientId(r.resource)),
389 new Root(r.name, graph.getURI(type))));
392 for(int i = 0; i < state.externals.size() ; i++) {
393 int parent = externalParents.get(i);
394 String name = externalNames.get(i);
395 procedure.execute(new Identity(externalBase + i, new External(parent,name)));
401 public TreeMap<String, Variant> getExtensions() {
402 return state.extensions;
405 public File[] getFiles() {
409 private static <T> T tryClose(T c) throws IOException {
410 if (c != null && c instanceof Closeable)
411 ((Closeable) c).close();
415 public void closeStreams() throws IOException {
416 state.valueInput = tryClose(state.valueInput);
417 state.otherStatementsInput = tryClose(state.otherStatementsInput);
418 state.statementsOutput = tryClose(state.statementsOutput);
419 state.valueOutput = tryClose(state.valueOutput);
423 public void reset() throws Exception {
424 throw new UnsupportedOperationException();
427 public long[] getResourceArray(ReadGraph graph) throws DatabaseException {
428 final SerialisationSupport ss = graph.getService(SerialisationSupport.class);
429 final long[] result = new long[state.ids.size()];
430 state.ids.forEachEntry(new TIntIntProcedure() {
433 public boolean execute(int a, int b) {
436 Resource r = ss.getResource(a);
437 result[b] = r.getResourceId();
438 } catch (DatabaseException e) {
449 public DomainProcessorState getState() {
453 public void forResourceStatements(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
455 int[] value = new int[4];
456 long length = state.otherStatementsInput.length();
457 state.otherStatementsInput.position(0);
459 while(state.otherStatementsInput.position() < length) {
461 int s = state.otherStatementsInput.readInt();
462 int subjectId = state.ids.get(s);
464 boolean exclude = subjectId == -1;
466 int size = state.otherStatementsInput.readInt();
467 for(int i=0;i<size;i++) {
468 int p = state.otherStatementsInput.readInt();
469 int o = state.otherStatementsInput.readInt();
471 if(state.excludedShared.contains(o)) {
472 System.err.println("excluding shared " + s + " " + p + " " + o);
475 int objectId = getExistingId(graph, o);
476 // The statement can be denied still
480 int inverse = state.inverses.get(p);
488 procedure.execute(value);
491 System.err.println("Denied (" + NameUtils.getSafeName(graph, getResource(graph, s)) + ", " + NameUtils.getSafeName(graph, getResource(graph, p)) + "," + NameUtils.getSafeName(graph, getResource(graph, o)) + ")");
496 System.err.println("excluding shared " + s);
502 public void forValueResources(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
504 long length = state.valueInput.length();
505 while (state.valueInput.position() < length) {
506 value[0] = state.valueInput.readInt();
507 procedure.execute(value);
511 public TransferableGraphConfiguration2 getConfiguration() {
512 return configuration;
516 public void close() throws IOException {
517 synchronized (this) {
524 for (File f : files) {
525 Files.deleteIfExists(f.toPath());