1 package org.simantics.db.layer0.util;
3 import java.io.Closeable;
4 import java.io.DataInput;
6 import java.io.IOException;
7 import java.nio.file.Files;
8 import java.util.ArrayList;
9 import java.util.Collection;
10 import java.util.Collections;
11 import java.util.HashSet;
12 import java.util.List;
13 import java.util.TreeMap;
15 import org.simantics.databoard.Bindings;
16 import org.simantics.databoard.Databoard;
17 import org.simantics.databoard.binding.Binding;
18 import org.simantics.databoard.binding.mutable.Variant;
19 import org.simantics.databoard.container.DataContainer;
20 import org.simantics.databoard.serialization.Serializer;
21 import org.simantics.databoard.type.Datatype;
22 import org.simantics.databoard.util.binary.ByteBufferReadable;
23 import org.simantics.db.ReadGraph;
24 import org.simantics.db.Resource;
25 import org.simantics.db.common.utils.NameUtils;
26 import org.simantics.db.exception.DatabaseException;
27 import org.simantics.db.exception.RuntimeDatabaseException;
28 import org.simantics.db.exception.ValidationException;
29 import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus;
30 import org.simantics.db.layer0.util.ConsistsOfProcess.InternalEntry;
31 import org.simantics.db.layer0.util.TransferableGraphConfiguration2.RootSpec;
32 import org.simantics.db.service.SerialisationSupport;
33 import org.simantics.graph.db.TransferableGraphSource;
34 import org.simantics.graph.representation.External;
35 import org.simantics.graph.representation.Identity;
36 import org.simantics.graph.representation.Internal;
37 import org.simantics.graph.representation.Root;
38 import org.simantics.graph.representation.Value;
39 import org.simantics.layer0.Layer0;
41 import gnu.trove.list.array.TIntArrayList;
42 import gnu.trove.map.TIntObjectMap;
43 import gnu.trove.map.hash.TIntObjectHashMap;
44 import gnu.trove.procedure.TIntIntProcedure;
46 public class ModelTransferableGraphSource implements TransferableGraphSource {
48 final private TransferableGraphConfiguration2 configuration;
49 final private DomainProcessorState state;
50 final private int externalBase;
51 final private int resourceCount;
52 final private File[] files;
53 final private TGValueModifier valueModifier;
55 private volatile boolean closed = false;
57 TIntArrayList externalParents = new TIntArrayList();
58 ArrayList<String> externalNames = new ArrayList<>();
59 TreeMap<String,String> downloads = new TreeMap<String,String>();
61 public ModelTransferableGraphSource(final ReadGraph graph, TransferableGraphConfiguration2 configuration, final DomainProcessorState state, File ... fs) throws DatabaseException {
63 this.configuration = configuration;
66 this.valueModifier = state.valueModifier;
68 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
70 // At this point ids contains all internal resources. Now add roots and externals.
73 state.ids.put(ss.getTransientId(graph.getRootLibrary()), state.id++);
75 // External roots - internal roots were already processed as internal resources by domain processor
76 for(RootSpec rs : configuration.roots) {
78 int resourceId = ss.getTransientId(rs.resource);
79 state.ids.put(resourceId, state.id++);
80 // The fixed roots have been seen as externals by domain processor. Now remove them from external set.
81 state.externals.remove(resourceId);
85 this.externalBase = state.id;
87 final Collection<String> errors = new HashSet<>();
89 // All resource considered as not internal by domain processor. Can also contain roots.
90 int[] externals = state.externals.toArray();
92 // Build up the state.externals, externalNames and externalParents
93 for(int i=0;i<externals.length;i++) {
94 getId(graph, externals[i], errors);
97 state.inverses.forEachEntry(new TIntIntProcedure() {
100 public boolean execute(int predicate, int inverse) {
102 getId(graph, predicate, errors);
103 if(inverse != 0) getId(graph, inverse, errors);
104 } catch (DatabaseException e) {
105 throw new RuntimeDatabaseException(e);
112 if(!errors.isEmpty()) {
113 ArrayList<String> sorted = new ArrayList<>(errors);
114 Collections.sort(sorted);
115 StringBuilder message = new StringBuilder();
116 message.append("Errors in exported model:\n");
117 for(String error : sorted) {
118 message.append(error);
119 message.append("\n");
121 throw new DatabaseException(message.toString());
124 this.resourceCount = state.id;
126 state.extensions.put(ExternalDownloadBean.EXTENSION_KEY, new Variant(ExternalDownloadBean.BINDING, new ExternalDownloadBean(downloads)));
132 public boolean validateExternal(Resource ext) {
133 if(configuration.validate) {
134 ExtentStatus status = configuration.preStatus.get(ext);
136 if(ExtentStatus.INTERNAL.equals(status)) return false;
137 else if(ExtentStatus.EXCLUDED.equals(status)) return false;
143 private Resource getResource(ReadGraph graph, int r) throws DatabaseException {
144 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
145 return ss.getResource(r);
148 final public int getExistingId(ReadGraph graph, int r) throws DatabaseException {
150 int ret = state.ids.get(r);
154 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
155 throw new DatabaseException("Id has not been created for " + NameUtils.getSafeName(graph, ss.getResource(r)));
162 * @return -2 if r is not really external and the statement should be excluded
165 public int getId(ReadGraph graph, int r, Collection<String> errors) throws DatabaseException {
167 // // First external is root library
168 // if(r == rootId) return internalCount;
170 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
171 Layer0 L0 = Layer0.getInstance(graph);
173 if(state.ids.containsKey(r)) {
174 int ret = state.ids.get(r);
176 for(int i=0;i<=indent;++i)
177 System.out.print(" ");
178 System.out.println("Cycle!!!"); // with " + GraphUtils.getReadableName(g, r));
183 Resource res = getResource(graph, r);
184 if(!validateExternal(res)) {
185 errors.add("Illegal reference to " + graph.getPossibleURI(getResource(graph, r)));
188 Collection<Resource> parents = graph.getObjects(res, L0.PartOf);
189 if(parents.size() != 1) {
190 throw new ValidationException("Reference to external resource "
191 + NameUtils.getSafeName(graph, getResource(graph, r), true) + " without unique uri (" + parents.size() + " parents).");
194 for(Resource p : parents) {
196 pid = getId(graph, ss.getTransientId(p), errors);
198 errors.add("Illegal reference to " + graph.getPossibleURI(getResource(graph, r)));
203 String name = graph.getRelatedValue(res, L0.HasName);
204 // Record the external entry
205 externalParents.add(pid);
206 externalNames.add(name);
207 state.ids.put(r, state.id);
208 // Ensure that this resource is included into the set of externals to maintain the total number of externals
209 state.externals.add(r);
210 String download = graph.getPossibleRelatedValue(res, L0.Ontology_download, Bindings.STRING);
211 if(download != null) {
212 String uri = graph.getURI(res);
213 downloads.put(uri, download);
220 public DataContainer getHeader() throws Exception {
225 public int getResourceCount() {
226 return resourceCount;
230 public int getIdentityCount() {
231 return configuration.roots.size() + state.externals.size() + state.internalEntries.size() + 1;
235 public int getStatementCount() {
236 return state.statementCount;
240 public int getValueCount() {
241 return state.valueCount;
245 public void forStatements(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
247 int[] value = new int[4];
248 long length = state.otherStatementsInput.length();
249 state.otherStatementsInput.position(0);
251 while(state.otherStatementsInput.position() < length && !state.monitor.isCanceled()) {
253 int s = state.otherStatementsInput.readInt();
254 int subjectId = state.ids.get(s);
256 boolean exclude = subjectId == -1;
258 int size = state.otherStatementsInput.readInt();
259 for(int i=0;i<size;i++) {
260 int p = state.otherStatementsInput.readInt();
261 int o = state.otherStatementsInput.readInt();
263 if(state.pending.contains(o)) {
264 System.err.println("excluding garbage statement " + s + " " + p + " " + o + ", object resource is garbage");
265 } else if(state.excludedShared.contains(o)) {
266 System.err.println("excluding shared " + s + " " + p + " " + o);
269 int objectId = getExistingId(graph, o);
270 // The statement can be denied still
272 value[0] = subjectId;
273 value[1] = getExistingId(graph, p);
274 int inverse = state.inverses.get(p);
276 value[2] = getExistingId(graph, inverse);
282 procedure.execute(value);
285 System.err.println("Denied (" + NameUtils.getSafeName(graph, getResource(graph, s)) + ", " + NameUtils.getSafeName(graph, getResource(graph, p)) + "," + NameUtils.getSafeName(graph, getResource(graph, o)) + ")");
290 System.err.println("excluding shared " + s);
297 public void forValues(ReadGraph graph, TransferableGraphSourceProcedure<Value> procedure) throws Exception {
299 Serializer variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT);
300 List<Object> idContext = new ArrayList<>();
301 long length = state.valueInput.length();
302 state.valueInput.position(0);
304 while(state.valueInput.position() < length && !state.monitor.isCanceled()) {
306 // Ignore value type tag
307 int s = state.valueInput.readInt();
308 byte valueType = state.valueInput.readByte();
310 case TAG_RAW_COPY_VARIANT_VALUE:
311 state.valueInput.readInt();
312 // Intentional fallthrough.
314 case TAG_POTENTIALLY_MODIFIED_VARIANT_VALUE: {
316 Variant variant = (Variant) variantSerializer.deserialize((DataInput)state.valueInput, idContext);
317 if (valueModifier.mayNeedModification(variant.type())) {
318 Object currentObject = variant.getValue();
319 Object newObject = valueModifier.modify(state, variant.getBinding(), currentObject);
320 if (newObject != currentObject)
321 variant = new Variant(variant.getBinding(), newObject);
323 procedure.execute(new Value(state.ids.get(s), variant));
328 throw new IllegalArgumentException("Unrecognized variant value type encountered: " + valueType);
334 public void forValues2(ReadGraph graph, TransferableGraphSourceValueProcedure procedure) throws Exception {
336 Serializer datatypeSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.getBindingUnchecked(Datatype.class));
337 List<Object> idContext = new ArrayList<>();
338 long length = state.valueInput.length();
339 state.valueInput.position(0);
341 while(state.valueInput.position() < length && !state.monitor.isCanceled()) {
343 int s = state.valueInput.readInt();
344 byte valueType = state.valueInput.readByte();
347 case TAG_RAW_COPY_VARIANT_VALUE: {
348 // Variant data could be copied raw, no need for modifications
349 // Note that the variant data is a concatenation of the
350 // variant datatype serialization and the value serialization.
351 // variantLength contains both datatype and value data.
352 int variantLength = state.valueInput.readInt();
353 procedure.rawCopy(state.ids.get(s), variantLength, state.valueInput);
357 case TAG_POTENTIALLY_MODIFIED_VARIANT_VALUE: {
358 // Variant data may need to be modified.
359 // Cannot optimize this case with raw copying.
361 Datatype type = (Datatype)datatypeSerializer.deserialize((DataInput)state.valueInput, idContext);
363 if (valueModifier.mayNeedModification(type)) {
364 Binding binding = Bindings.getBinding(type);
365 Serializer serializer = Bindings.getSerializerUnchecked(binding);
366 Object value = serializer.deserialize((DataInput)state.valueInput);
367 value = valueModifier.modify(state, binding, value);
368 byte[] bytes = serializer.serialize(value);
369 procedure.execute(state.ids.get(s), type, new ByteBufferReadable(bytes));
371 procedure.execute(state.ids.get(s), type, state.valueInput);
377 throw new IllegalArgumentException("Unrecognized variant value type encountered: " + valueType);
382 protected Identity getRootIdentity(DomainProcessorState state, SerialisationSupport support, Resource rootLibrary) throws DatabaseException {
383 return new Identity(state.ids.get(support.getTransientId(rootLibrary)), new External(-1, ""));
387 public void forIdentities(ReadGraph graph, TransferableGraphSourceProcedure<Identity> procedure) throws Exception {
389 SerialisationSupport support = graph.getService(SerialisationSupport.class);
390 Layer0 L0 = Layer0.getInstance(graph);
392 // TODO: this should be Root with name ""
393 procedure.execute(getRootIdentity(state, support, graph.getRootLibrary()));
395 TIntObjectMap<Identity> internalMap = new TIntObjectHashMap<>(100, 0.5f, Integer.MIN_VALUE);
397 // Declare internal and external roots
398 for(RootSpec r : configuration.roots) {
399 Resource type = graph.getPossibleType(r.resource, L0.Entity);
400 if(type == null) type = L0.Entity;
401 int id = state.ids.get(support.getTransientId(r.resource));
402 Root root = new Root(r.name, graph.getURI(type));
403 Identity rootId = new Identity(id,root);
404 internalMap.put(id, rootId);
405 procedure.execute(rootId);
408 for(int i = 0; i < state.externals.size() ; i++) {
409 int parent = externalParents.get(i);
410 String name = externalNames.get(i);
411 procedure.execute(new Identity(externalBase + i, new External(parent,name)));
414 if(state.internalEntries != null) {
415 for(InternalEntry ie : state.internalEntries) {
416 if(ie.parent != null && ie.name != null) {
417 procedure.execute(resolveInternal(graph, support, ie, internalMap));
419 throw new DatabaseException("Invalid internal entry " + ie);
426 private Identity resolveInternal(ReadGraph graph, SerialisationSupport ss, InternalEntry entry, TIntObjectMap<Identity> internalMap) throws DatabaseException {
427 int id = state.ids.get(ss.getTransientId(entry.resource));
428 Identity existing = internalMap.get(id);
429 if(existing != null) return existing;
430 Identity parent = resolveInternal(graph, ss, entry.parent, internalMap);
431 Identity result = new Identity(id,
432 new Internal(parent.resource, entry.name));
433 internalMap.put(id, result);
438 public TreeMap<String, Variant> getExtensions() {
439 return state.extensions;
442 public File[] getFiles() {
446 private static <T> T tryClose(T c) throws IOException {
447 if (c != null && c instanceof Closeable)
448 ((Closeable) c).close();
452 public void closeStreams() throws IOException {
453 state.valueInput = tryClose(state.valueInput);
454 state.otherStatementsInput = tryClose(state.otherStatementsInput);
455 state.statementsOutput = tryClose(state.statementsOutput);
456 state.valueOutput = tryClose(state.valueOutput);
460 public void reset() throws Exception {
461 throw new UnsupportedOperationException();
464 public long[] getResourceArray(ReadGraph graph) throws DatabaseException {
465 final SerialisationSupport ss = graph.getService(SerialisationSupport.class);
466 final long[] result = new long[state.ids.size()];
467 state.ids.forEachEntry(new TIntIntProcedure() {
470 public boolean execute(int a, int b) {
473 Resource r = ss.getResource(a);
474 result[b] = r.getResourceId();
475 } catch (DatabaseException e) {
486 public DomainProcessorState getState() {
490 public void forResourceStatements(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
492 int[] value = new int[4];
493 long length = state.otherStatementsInput.length();
494 state.otherStatementsInput.position(0);
496 while(state.otherStatementsInput.position() < length) {
498 int s = state.otherStatementsInput.readInt();
499 int subjectId = state.ids.get(s);
501 boolean exclude = subjectId == -1;
503 int size = state.otherStatementsInput.readInt();
504 for(int i=0;i<size;i++) {
505 int p = state.otherStatementsInput.readInt();
506 int o = state.otherStatementsInput.readInt();
508 if(state.excludedShared.contains(o)) {
509 System.err.println("excluding shared " + s + " " + p + " " + o);
512 int objectId = getExistingId(graph, o);
513 // The statement can be denied still
517 int inverse = state.inverses.get(p);
525 procedure.execute(value);
528 System.err.println("Denied (" + NameUtils.getSafeName(graph, getResource(graph, s)) + ", " + NameUtils.getSafeName(graph, getResource(graph, p)) + "," + NameUtils.getSafeName(graph, getResource(graph, o)) + ")");
533 System.err.println("excluding shared " + s);
539 public void forValueResources(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
541 long length = state.valueInput.length();
542 while (state.valueInput.position() < length) {
543 value[0] = state.valueInput.readInt();
544 procedure.execute(value);
548 public TransferableGraphConfiguration2 getConfiguration() {
549 return configuration;
553 public void close() throws IOException {
554 synchronized (this) {
561 for (File f : files) {
562 Files.deleteIfExists(f.toPath());