1 package org.simantics.db.layer0.util;
3 import java.io.Closeable;
4 import java.io.DataInput;
6 import java.io.IOException;
7 import java.nio.file.Files;
8 import java.util.ArrayList;
9 import java.util.Collection;
10 import java.util.Collections;
11 import java.util.HashSet;
12 import java.util.List;
13 import java.util.TreeMap;
15 import org.simantics.databoard.Bindings;
16 import org.simantics.databoard.Databoard;
17 import org.simantics.databoard.binding.Binding;
18 import org.simantics.databoard.binding.mutable.Variant;
19 import org.simantics.databoard.container.DataContainer;
20 import org.simantics.databoard.serialization.Serializer;
21 import org.simantics.databoard.type.Datatype;
22 import org.simantics.databoard.util.binary.ByteBufferReadable;
23 import org.simantics.db.ReadGraph;
24 import org.simantics.db.Resource;
25 import org.simantics.db.common.utils.NameUtils;
26 import org.simantics.db.exception.DatabaseException;
27 import org.simantics.db.exception.RuntimeDatabaseException;
28 import org.simantics.db.exception.ValidationException;
29 import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus;
30 import org.simantics.db.layer0.util.ConsistsOfProcess.ConsistsOfProcessEntry;
31 import org.simantics.db.layer0.util.TransferableGraphConfiguration2.SeedSpec;
32 import org.simantics.db.layer0.util.TransferableGraphConfiguration2.SeedSpec.SeedSpecType;
33 import org.simantics.db.service.SerialisationSupport;
34 import org.simantics.graph.db.TransferableGraphSource;
35 import org.simantics.graph.representation.External;
36 import org.simantics.graph.representation.Identity;
37 import org.simantics.graph.representation.Internal;
38 import org.simantics.graph.representation.Root;
39 import org.simantics.graph.representation.Value;
40 import org.simantics.layer0.Layer0;
41 import org.slf4j.Logger;
42 import org.slf4j.LoggerFactory;
44 import gnu.trove.list.array.TIntArrayList;
45 import gnu.trove.map.TIntObjectMap;
46 import gnu.trove.map.hash.TIntObjectHashMap;
47 import gnu.trove.procedure.TIntIntProcedure;
49 public class ModelTransferableGraphSource implements TransferableGraphSource {
51 private static final Logger LOGGER = LoggerFactory
52 .getLogger(ModelTransferableGraphSource.class);
54 final private TransferableGraphConfiguration2 configuration;
56 final private DomainProcessorState state;
58 final private int externalBase;
60 final private int resourceCount;
62 final private File[] files;
64 final private TGValueModifier valueModifier;
66 private volatile boolean closed = false;
68 TIntArrayList externalParents = new TIntArrayList();
70 ArrayList<String> externalNames = new ArrayList<>();
72 TreeMap<String, String> downloads = new TreeMap<String, String>();
74 public ModelTransferableGraphSource(final ReadGraph graph, TransferableGraphConfiguration2 configuration,
75 final DomainProcessorState state, File... fs) throws DatabaseException {
77 this.configuration = configuration;
80 this.valueModifier = state.valueModifier;
82 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
84 // At this point ids contains all internal resources. Now add roots and
88 state.ids.put(ss.getTransientId(graph.getRootLibrary()), state.id++);
90 // External roots - internal roots were already processed as internal
91 // resources by domain processor
92 for (SeedSpec spec : configuration.seeds) {
93 if (SeedSpecType.SPECIAL_ROOT.equals(spec.specType)) {
94 int resourceId = ss.getTransientId(spec.resource);
95 state.ids.put(resourceId, state.id++);
96 // The fixed roots have been seen as externals by domain
97 // processor. Now remove them from external set.
98 state.externals.remove(resourceId);
102 this.externalBase = state.id;
104 final Collection<String> errors = new HashSet<>();
106 // All resource considered as not internal by domain processor. Can also
108 int[] externals = state.externals.toArray();
110 // Build up the state.externals, externalNames and externalParents
111 for (int i = 0; i < externals.length; i++) {
112 getId(graph, externals[i], errors);
115 state.inverses.forEachEntry(new TIntIntProcedure() {
118 public boolean execute(int predicate, int inverse) {
120 getId(graph, predicate, errors);
122 getId(graph, inverse, errors);
123 } catch (DatabaseException e) {
124 throw new RuntimeDatabaseException(e);
131 if (!errors.isEmpty()) {
132 ArrayList<String> sorted = new ArrayList<>(errors);
133 Collections.sort(sorted);
134 StringBuilder message = new StringBuilder();
135 message.append("Errors in exported model:\n");
136 for (String error : sorted) {
137 message.append(error);
138 message.append("\n");
140 throw new DatabaseException(message.toString());
143 this.resourceCount = state.id;
145 state.extensions.put(ExternalDownloadBean.EXTENSION_KEY,
146 new Variant(ExternalDownloadBean.BINDING, new ExternalDownloadBean(downloads)));
152 public boolean validateExternal(Resource ext) {
153 if (configuration.validate) {
154 ExtentStatus status = configuration.preStatus.get(ext);
155 if (status != null) {
156 if (ExtentStatus.INTERNAL.equals(status))
158 else if (ExtentStatus.EXCLUDED.equals(status))
165 private Resource getResource(ReadGraph graph, int r) throws DatabaseException {
166 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
167 return ss.getResource(r);
170 final public int getExistingId(ReadGraph graph, int r) throws DatabaseException {
172 int ret = state.ids.get(r);
176 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
177 throw new DatabaseException(
178 "Id has not been created for " + NameUtils.getSafeName(graph, ss.getResource(r)));
185 * @return -2 if r is not really external and the statement should be
189 public int getId(ReadGraph graph, int r, Collection<String> errors) throws DatabaseException {
191 // // First external is root library
192 // if(r == rootId) return internalCount;
194 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
195 Layer0 L0 = Layer0.getInstance(graph);
197 if (state.ids.containsKey(r)) {
198 int ret = state.ids.get(r);
200 for (int i = 0; i <= indent; ++i)
201 System.out.print(" ");
202 System.out.println("Cycle!!!"); // with " +
203 // GraphUtils.getReadableName(g,
208 Resource res = getResource(graph, r);
209 if (!validateExternal(res)) {
210 errors.add("Illegal reference to " + graph.getPossibleURI(getResource(graph, r)));
213 Collection<Resource> parents = graph.getObjects(res, L0.PartOf);
214 if (parents.size() != 1) {
215 throw new ValidationException(
216 "Reference to external resource " + NameUtils.getSafeName(graph, getResource(graph, r), true)
217 + " without unique uri (" + parents.size() + " parents).");
220 for (Resource p : parents) {
222 pid = getId(graph, ss.getTransientId(p), errors);
224 errors.add("Illegal reference to " + graph.getPossibleURI(getResource(graph, r)));
229 String name = graph.getRelatedValue(res, L0.HasName);
230 // Record the external entry
231 externalParents.add(pid);
232 externalNames.add(name);
233 state.ids.put(r, state.id);
234 // Ensure that this resource is included into the set of externals
235 // to maintain the total number of externals
236 state.externals.add(r);
237 String download = graph.getPossibleRelatedValue(res, L0.Ontology_download, Bindings.STRING);
238 if (download != null) {
239 String uri = graph.getURI(res);
240 downloads.put(uri, download);
247 public DataContainer getHeader() throws Exception {
252 public int getResourceCount() {
253 return resourceCount;
256 private int countRootSeeds() {
258 for (SeedSpec spec : configuration.seeds) {
259 if (SeedSpecType.INTERNAL.equals(spec.specType))
267 public int getIdentityCount() {
268 return countRootSeeds() + state.externals.size() + state.internalEntries.size() + 1;
272 public int getStatementCount() {
273 return state.statementCount;
277 public int getValueCount() {
278 return state.valueCount;
282 public void forStatements(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
284 int[] value = new int[4];
285 long length = state.otherStatementsInput.length();
286 state.otherStatementsInput.position(0);
288 while (state.otherStatementsInput.position() < length && !state.monitor.isCanceled()) {
290 int s = state.otherStatementsInput.readInt();
291 int subjectId = state.ids.get(s);
293 boolean exclude = subjectId == -1;
295 int size = state.otherStatementsInput.readInt();
296 for (int i = 0; i < size; i++) {
297 int p = state.otherStatementsInput.readInt();
298 int o = state.otherStatementsInput.readInt();
300 if (state.pending.contains(o)) {
301 System.err.println("excluding garbage statement " + s + " " + p + " " + o
302 + ", object resource is garbage");
303 } else if (state.excludedShared.contains(o)) {
304 System.err.println("excluding shared " + s + " " + p + " " + o);
307 int objectId = getExistingId(graph, o);
308 // The statement can be denied still
309 if (objectId != -2) {
310 value[0] = subjectId;
311 value[1] = getExistingId(graph, p);
312 int inverse = state.inverses.get(p);
314 value[2] = getExistingId(graph, inverse);
321 procedure.execute(value);
322 } catch (Exception e) {
323 LOGGER.error("Error while processing a statement in transferable graph source", e);
327 System.err.println("Denied (" + NameUtils.getSafeName(graph, getResource(graph, s)) + ", "
328 + NameUtils.getSafeName(graph, getResource(graph, p)) + ","
329 + NameUtils.getSafeName(graph, getResource(graph, o)) + ")");
334 System.err.println("excluding shared " + s);
341 public void forValues(ReadGraph graph, TransferableGraphSourceProcedure<Value> procedure) throws Exception {
343 Serializer variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT);
344 List<Object> idContext = new ArrayList<>();
345 long length = state.valueInput.length();
346 state.valueInput.position(0);
348 while (state.valueInput.position() < length && !state.monitor.isCanceled()) {
350 // Ignore value type tag
351 int s = state.valueInput.readInt();
352 byte valueType = state.valueInput.readByte();
354 case TAG_RAW_COPY_VARIANT_VALUE:
355 state.valueInput.readInt();
356 // Intentional fallthrough.
358 case TAG_POTENTIALLY_MODIFIED_VARIANT_VALUE: {
360 Variant variant = (Variant) variantSerializer.deserialize((DataInput) state.valueInput, idContext);
361 if (valueModifier.mayNeedModification(variant.type())) {
362 Object currentObject = variant.getValue();
363 Object newObject = valueModifier.modify(state, variant.getBinding(), currentObject);
364 if (newObject != currentObject)
365 variant = new Variant(variant.getBinding(), newObject);
368 procedure.execute(new Value(state.ids.get(s), variant));
369 } catch (Exception e) {
370 LOGGER.error("Error while processing a value in transferable graph source", e);
376 throw new IllegalArgumentException("Unrecognized variant value type encountered: " + valueType);
382 public void forValues2(ReadGraph graph, TransferableGraphSourceValueProcedure procedure) throws Exception {
384 Serializer datatypeSerializer = graph.getService(Databoard.class)
385 .getSerializerUnchecked(Bindings.getBindingUnchecked(Datatype.class));
386 List<Object> idContext = new ArrayList<>();
387 long length = state.valueInput.length();
388 state.valueInput.position(0);
390 while (state.valueInput.position() < length && !state.monitor.isCanceled()) {
392 int s = state.valueInput.readInt();
393 byte valueType = state.valueInput.readByte();
396 case TAG_RAW_COPY_VARIANT_VALUE: {
397 // Variant data could be copied raw, no need for
399 // Note that the variant data is a concatenation of the
400 // variant datatype serialization and the value
402 // variantLength contains both datatype and value data.
403 int variantLength = state.valueInput.readInt();
405 procedure.rawCopy(state.ids.get(s), variantLength, state.valueInput);
406 } catch (Exception e) {
407 LOGGER.error("Error while processing a raw value in transferable graph source", e);
412 case TAG_POTENTIALLY_MODIFIED_VARIANT_VALUE: {
413 // Variant data may need to be modified.
414 // Cannot optimize this case with raw copying.
416 Datatype type = (Datatype) datatypeSerializer.deserialize((DataInput) state.valueInput, idContext);
418 if (valueModifier.mayNeedModification(type)) {
419 Binding binding = Bindings.getBinding(type);
420 Serializer serializer = Bindings.getSerializerUnchecked(binding);
421 Object value = serializer.deserialize((DataInput) state.valueInput);
422 value = valueModifier.modify(state, binding, value);
423 byte[] bytes = serializer.serialize(value);
425 procedure.execute(state.ids.get(s), type, new ByteBufferReadable(bytes));
426 } catch (Exception e) {
427 LOGGER.error("Error while processing a data type in transferable graph source", e);
431 procedure.execute(state.ids.get(s), type, state.valueInput);
432 } catch (Exception e) {
433 LOGGER.error("Error while processing a raw value in transferable graph source", e);
440 throw new IllegalArgumentException("Unrecognized variant value type encountered: " + valueType);
445 protected Identity getRootIdentity(DomainProcessorState state, SerialisationSupport support, Resource rootLibrary)
446 throws DatabaseException {
447 return new Identity(state.ids.get(support.getTransientId(rootLibrary)), new External(-1, ""));
451 public void forIdentities(ReadGraph graph, TransferableGraphSourceProcedure<Identity> procedure) throws Exception {
453 SerialisationSupport support = graph.getService(SerialisationSupport.class);
454 Layer0 L0 = Layer0.getInstance(graph);
456 // TODO: this should be Root with name ""
458 procedure.execute(getRootIdentity(state, support, graph.getRootLibrary()));
459 } catch (Exception e) {
460 LOGGER.error("Error while processing a root identity in transferable graph source", e);
463 TIntObjectMap<Identity> internalMap = new TIntObjectHashMap<>(100, 0.5f, Integer.MIN_VALUE);
465 // Declare internal and external roots
466 for (SeedSpec r : configuration.seeds) {
467 if (SeedSpecType.INTERNAL.equals(r.specType))
469 String typeId = r.type;
470 if (typeId == null) {
471 Resource type = graph.getPossibleType(r.resource, L0.Entity);
472 typeId = type != null ? graph.getURI(type) : Layer0.URIs.Entity;
474 int id = state.ids.get(support.getTransientId(r.resource));
475 Root root = new Root(r.name, typeId);
476 Identity rootId = new Identity(id, root);
477 internalMap.put(id, rootId);
479 procedure.execute(rootId);
480 } catch (Exception e) {
481 LOGGER.error("Error while processing a root in transferable graph source", e);
485 for (int i = 0; i < state.externals.size(); i++) {
486 int parent = externalParents.get(i);
487 String name = externalNames.get(i);
489 procedure.execute(new Identity(externalBase + i, new External(parent, name)));
490 } catch (Exception e) {
491 LOGGER.error("Error while processing a an external identity in transferable graph source", e);
495 if (state.internalEntries != null) {
496 for (ConsistsOfProcessEntry ie : state.internalEntries) {
497 if (ie.parent != null) {
498 if (ie.name != null) {
500 procedure.execute(resolveInternal(graph, support, ie, internalMap));
501 } catch (Exception e) {
502 LOGGER.error("Error while processing an internal identity in transferable graph source", e);
505 // In this case there is a child that has no HasName =>
506 // this should be treated as a blank
510 procedure.execute(resolveInternal(graph, support, ie, internalMap));
511 } catch (Exception e) {
512 LOGGER.error("Error while processing an internal identity in transferable graph source", e);
520 private Identity resolveInternal(ReadGraph graph, SerialisationSupport ss, ConsistsOfProcessEntry entry,
521 TIntObjectMap<Identity> internalMap) throws DatabaseException {
522 int id = state.ids.get(ss.getTransientId(entry.resource));
523 Identity existing = internalMap.get(id);
524 if (existing != null)
527 if (entry.parent == null) {
528 Layer0 L0 = Layer0.getInstance(graph);
529 Resource possibleParent = graph.getPossibleObject(entry.resource, L0.PartOf);
530 if (possibleParent == null)
531 throw new DatabaseException("Invalid root or internal parent path: " + entry.resource);
532 int externalId = state.ids.get(ss.getTransientId(possibleParent));
533 Identity result = new Identity(id, new Internal(externalId, entry.name));
534 internalMap.put(id, result);
537 Identity parent = resolveInternal(graph, ss, entry.parent, internalMap);
538 Identity result = new Identity(id, new Internal(parent.resource, entry.name));
539 internalMap.put(id, result);
545 public TreeMap<String, Variant> getExtensions() {
546 return state.extensions;
549 public File[] getFiles() {
553 private static <T> T tryClose(T c) throws IOException {
554 if (c != null && c instanceof Closeable)
555 ((Closeable) c).close();
559 public void closeStreams() throws IOException {
560 state.valueInput = tryClose(state.valueInput);
561 state.otherStatementsInput = tryClose(state.otherStatementsInput);
562 state.statementsOutput = tryClose(state.statementsOutput);
563 state.valueOutput = tryClose(state.valueOutput);
567 public void reset() throws Exception {
568 throw new UnsupportedOperationException();
571 public long[] getResourceArray(ReadGraph graph) throws DatabaseException {
572 final SerialisationSupport ss = graph.getService(SerialisationSupport.class);
573 final long[] result = new long[state.ids.size()];
574 state.ids.forEachEntry(new TIntIntProcedure() {
577 public boolean execute(int a, int b) {
580 Resource r = ss.getResource(a);
581 result[b] = r.getResourceId();
582 } catch (DatabaseException e) {
593 public DomainProcessorState getState() {
597 public void forResourceStatements(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure)
600 int[] value = new int[4];
601 long length = state.otherStatementsInput.length();
602 state.otherStatementsInput.position(0);
604 while (state.otherStatementsInput.position() < length) {
606 int s = state.otherStatementsInput.readInt();
607 int subjectId = state.ids.get(s);
609 boolean exclude = subjectId == -1;
611 int size = state.otherStatementsInput.readInt();
612 for (int i = 0; i < size; i++) {
613 int p = state.otherStatementsInput.readInt();
614 int o = state.otherStatementsInput.readInt();
616 if (state.excludedShared.contains(o)) {
617 System.err.println("excluding shared " + s + " " + p + " " + o);
620 int objectId = getExistingId(graph, o);
621 // The statement can be denied still
622 if (objectId != -2) {
625 int inverse = state.inverses.get(p);
634 procedure.execute(value);
635 } catch (Exception e) {
636 LOGGER.error("Error while processing a statement in transferable graph source", e);
640 System.err.println("Denied (" + NameUtils.getSafeName(graph, getResource(graph, s)) + ", "
641 + NameUtils.getSafeName(graph, getResource(graph, p)) + ","
642 + NameUtils.getSafeName(graph, getResource(graph, o)) + ")");
647 System.err.println("excluding shared " + s);
653 public void forValueResources(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
655 long length = state.valueInput.length();
656 while (state.valueInput.position() < length) {
657 value[0] = state.valueInput.readInt();
659 procedure.execute(value);
660 } catch (Exception e) {
661 LOGGER.error("Error while processing a value in transferable graph source", e);
666 public TransferableGraphConfiguration2 getConfiguration() {
667 return configuration;
671 public void close() throws IOException {
672 synchronized (this) {
679 for (File f : files) {
680 Files.deleteIfExists(f.toPath());