1 package org.simantics.db.layer0.util;
3 import java.io.Closeable;
4 import java.io.DataInput;
6 import java.io.IOException;
7 import java.nio.file.Files;
8 import java.util.ArrayList;
9 import java.util.Collection;
10 import java.util.Collections;
11 import java.util.HashSet;
12 import java.util.List;
13 import java.util.TreeMap;
15 import org.simantics.databoard.Bindings;
16 import org.simantics.databoard.Databoard;
17 import org.simantics.databoard.binding.Binding;
18 import org.simantics.databoard.binding.mutable.Variant;
19 import org.simantics.databoard.container.DataContainer;
20 import org.simantics.databoard.serialization.Serializer;
21 import org.simantics.databoard.type.Datatype;
22 import org.simantics.databoard.util.binary.ByteBufferReadable;
23 import org.simantics.db.ReadGraph;
24 import org.simantics.db.Resource;
25 import org.simantics.db.common.utils.NameUtils;
26 import org.simantics.db.exception.DatabaseException;
27 import org.simantics.db.exception.RuntimeDatabaseException;
28 import org.simantics.db.exception.ValidationException;
29 import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus;
30 import org.simantics.db.layer0.util.ConsistsOfProcess.ConsistsOfProcessEntry;
31 import org.simantics.db.layer0.util.TransferableGraphConfiguration2.SeedSpec;
32 import org.simantics.db.layer0.util.TransferableGraphConfiguration2.SeedSpec.SeedSpecType;
33 import org.simantics.db.service.SerialisationSupport;
34 import org.simantics.graph.db.TransferableGraphSource;
35 import org.simantics.graph.representation.External;
36 import org.simantics.graph.representation.Identity;
37 import org.simantics.graph.representation.Internal;
38 import org.simantics.graph.representation.Root;
39 import org.simantics.graph.representation.Value;
40 import org.simantics.layer0.Layer0;
42 import gnu.trove.list.array.TIntArrayList;
43 import gnu.trove.map.TIntObjectMap;
44 import gnu.trove.map.hash.TIntObjectHashMap;
45 import gnu.trove.procedure.TIntIntProcedure;
47 public class ModelTransferableGraphSource implements TransferableGraphSource {
49 final private TransferableGraphConfiguration2 configuration;
50 final private DomainProcessorState state;
51 final private int externalBase;
52 final private int resourceCount;
53 final private File[] files;
54 final private TGValueModifier valueModifier;
56 private volatile boolean closed = false;
58 TIntArrayList externalParents = new TIntArrayList();
59 ArrayList<String> externalNames = new ArrayList<>();
60 TreeMap<String,String> downloads = new TreeMap<String,String>();
62 public ModelTransferableGraphSource(final ReadGraph graph, TransferableGraphConfiguration2 configuration, final DomainProcessorState state, File ... fs) throws DatabaseException {
64 this.configuration = configuration;
67 this.valueModifier = state.valueModifier;
69 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
71 // At this point ids contains all internal resources. Now add roots and externals.
74 state.ids.put(ss.getTransientId(graph.getRootLibrary()), state.id++);
76 // External roots - internal roots were already processed as internal resources by domain processor
77 for(SeedSpec spec : configuration.seeds) {
78 if(SeedSpecType.SPECIAL_ROOT.equals(spec.specType)) {
79 int resourceId = ss.getTransientId(spec.resource);
80 state.ids.put(resourceId, state.id++);
81 // The fixed roots have been seen as externals by domain processor. Now remove them from external set.
82 state.externals.remove(resourceId);
86 this.externalBase = state.id;
88 final Collection<String> errors = new HashSet<>();
90 // All resource considered as not internal by domain processor. Can also contain roots.
91 int[] externals = state.externals.toArray();
93 // Build up the state.externals, externalNames and externalParents
94 for(int i=0;i<externals.length;i++) {
95 getId(graph, externals[i], errors);
98 state.inverses.forEachEntry(new TIntIntProcedure() {
101 public boolean execute(int predicate, int inverse) {
103 getId(graph, predicate, errors);
104 if(inverse != 0) getId(graph, inverse, errors);
105 } catch (DatabaseException e) {
106 throw new RuntimeDatabaseException(e);
113 if(!errors.isEmpty()) {
114 ArrayList<String> sorted = new ArrayList<>(errors);
115 Collections.sort(sorted);
116 StringBuilder message = new StringBuilder();
117 message.append("Errors in exported model:\n");
118 for(String error : sorted) {
119 message.append(error);
120 message.append("\n");
122 throw new DatabaseException(message.toString());
125 this.resourceCount = state.id;
127 state.extensions.put(ExternalDownloadBean.EXTENSION_KEY, new Variant(ExternalDownloadBean.BINDING, new ExternalDownloadBean(downloads)));
133 public boolean validateExternal(Resource ext) {
134 if(configuration.validate) {
135 ExtentStatus status = configuration.preStatus.get(ext);
137 if(ExtentStatus.INTERNAL.equals(status)) return false;
138 else if(ExtentStatus.EXCLUDED.equals(status)) return false;
144 private Resource getResource(ReadGraph graph, int r) throws DatabaseException {
145 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
146 return ss.getResource(r);
149 final public int getExistingId(ReadGraph graph, int r) throws DatabaseException {
151 int ret = state.ids.get(r);
155 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
156 throw new DatabaseException("Id has not been created for " + NameUtils.getSafeName(graph, ss.getResource(r)));
163 * @return -2 if r is not really external and the statement should be excluded
166 public int getId(ReadGraph graph, int r, Collection<String> errors) throws DatabaseException {
168 // // First external is root library
169 // if(r == rootId) return internalCount;
171 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
172 Layer0 L0 = Layer0.getInstance(graph);
174 if(state.ids.containsKey(r)) {
175 int ret = state.ids.get(r);
177 for(int i=0;i<=indent;++i)
178 System.out.print(" ");
179 System.out.println("Cycle!!!"); // with " + GraphUtils.getReadableName(g, r));
184 Resource res = getResource(graph, r);
185 if(!validateExternal(res)) {
186 errors.add("Illegal reference to " + graph.getPossibleURI(getResource(graph, r)));
189 Collection<Resource> parents = graph.getObjects(res, L0.PartOf);
190 if(parents.size() != 1) {
191 throw new ValidationException("Reference to external resource "
192 + NameUtils.getSafeName(graph, getResource(graph, r), true) + " without unique uri (" + parents.size() + " parents).");
195 for(Resource p : parents) {
197 pid = getId(graph, ss.getTransientId(p), errors);
199 errors.add("Illegal reference to " + graph.getPossibleURI(getResource(graph, r)));
204 String name = graph.getRelatedValue(res, L0.HasName);
205 // Record the external entry
206 externalParents.add(pid);
207 externalNames.add(name);
208 state.ids.put(r, state.id);
209 // Ensure that this resource is included into the set of externals to maintain the total number of externals
210 state.externals.add(r);
211 String download = graph.getPossibleRelatedValue(res, L0.Ontology_download, Bindings.STRING);
212 if(download != null) {
213 String uri = graph.getURI(res);
214 downloads.put(uri, download);
221 public DataContainer getHeader() throws Exception {
226 public int getResourceCount() {
227 return resourceCount;
230 private int countRootSeeds() {
232 for(SeedSpec spec : configuration.seeds) {
233 if(SeedSpecType.INTERNAL.equals(spec.specType)) continue;
240 public int getIdentityCount() {
241 return countRootSeeds() + state.externals.size() + state.internalEntries.size() + 1;
245 public int getStatementCount() {
246 return state.statementCount;
250 public int getValueCount() {
251 return state.valueCount;
255 public void forStatements(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
257 int[] value = new int[4];
258 long length = state.otherStatementsInput.length();
259 state.otherStatementsInput.position(0);
261 while(state.otherStatementsInput.position() < length && !state.monitor.isCanceled()) {
263 int s = state.otherStatementsInput.readInt();
264 int subjectId = state.ids.get(s);
266 boolean exclude = subjectId == -1;
268 int size = state.otherStatementsInput.readInt();
269 for(int i=0;i<size;i++) {
270 int p = state.otherStatementsInput.readInt();
271 int o = state.otherStatementsInput.readInt();
273 if(state.pending.contains(o)) {
274 System.err.println("excluding garbage statement " + s + " " + p + " " + o + ", object resource is garbage");
275 } else if(state.excludedShared.contains(o)) {
276 System.err.println("excluding shared " + s + " " + p + " " + o);
279 int objectId = getExistingId(graph, o);
280 // The statement can be denied still
282 value[0] = subjectId;
283 value[1] = getExistingId(graph, p);
284 int inverse = state.inverses.get(p);
286 value[2] = getExistingId(graph, inverse);
292 procedure.execute(value);
295 System.err.println("Denied (" + NameUtils.getSafeName(graph, getResource(graph, s)) + ", " + NameUtils.getSafeName(graph, getResource(graph, p)) + "," + NameUtils.getSafeName(graph, getResource(graph, o)) + ")");
300 System.err.println("excluding shared " + s);
307 public void forValues(ReadGraph graph, TransferableGraphSourceProcedure<Value> procedure) throws Exception {
309 Serializer variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT);
310 List<Object> idContext = new ArrayList<>();
311 long length = state.valueInput.length();
312 state.valueInput.position(0);
314 while(state.valueInput.position() < length && !state.monitor.isCanceled()) {
316 // Ignore value type tag
317 int s = state.valueInput.readInt();
318 byte valueType = state.valueInput.readByte();
320 case TAG_RAW_COPY_VARIANT_VALUE:
321 state.valueInput.readInt();
322 // Intentional fallthrough.
324 case TAG_POTENTIALLY_MODIFIED_VARIANT_VALUE: {
326 Variant variant = (Variant) variantSerializer.deserialize((DataInput)state.valueInput, idContext);
327 if (valueModifier.mayNeedModification(variant.type())) {
328 Object currentObject = variant.getValue();
329 Object newObject = valueModifier.modify(state, variant.getBinding(), currentObject);
330 if (newObject != currentObject)
331 variant = new Variant(variant.getBinding(), newObject);
333 procedure.execute(new Value(state.ids.get(s), variant));
338 throw new IllegalArgumentException("Unrecognized variant value type encountered: " + valueType);
344 public void forValues2(ReadGraph graph, TransferableGraphSourceValueProcedure procedure) throws Exception {
346 Serializer datatypeSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.getBindingUnchecked(Datatype.class));
347 List<Object> idContext = new ArrayList<>();
348 long length = state.valueInput.length();
349 state.valueInput.position(0);
351 while(state.valueInput.position() < length && !state.monitor.isCanceled()) {
353 int s = state.valueInput.readInt();
354 byte valueType = state.valueInput.readByte();
357 case TAG_RAW_COPY_VARIANT_VALUE: {
358 // Variant data could be copied raw, no need for modifications
359 // Note that the variant data is a concatenation of the
360 // variant datatype serialization and the value serialization.
361 // variantLength contains both datatype and value data.
362 int variantLength = state.valueInput.readInt();
363 procedure.rawCopy(state.ids.get(s), variantLength, state.valueInput);
367 case TAG_POTENTIALLY_MODIFIED_VARIANT_VALUE: {
368 // Variant data may need to be modified.
369 // Cannot optimize this case with raw copying.
371 Datatype type = (Datatype)datatypeSerializer.deserialize((DataInput)state.valueInput, idContext);
373 if (valueModifier.mayNeedModification(type)) {
374 Binding binding = Bindings.getBinding(type);
375 Serializer serializer = Bindings.getSerializerUnchecked(binding);
376 Object value = serializer.deserialize((DataInput)state.valueInput);
377 value = valueModifier.modify(state, binding, value);
378 byte[] bytes = serializer.serialize(value);
379 procedure.execute(state.ids.get(s), type, new ByteBufferReadable(bytes));
381 procedure.execute(state.ids.get(s), type, state.valueInput);
387 throw new IllegalArgumentException("Unrecognized variant value type encountered: " + valueType);
392 protected Identity getRootIdentity(DomainProcessorState state, SerialisationSupport support, Resource rootLibrary) throws DatabaseException {
393 return new Identity(state.ids.get(support.getTransientId(rootLibrary)), new External(-1, ""));
397 public void forIdentities(ReadGraph graph, TransferableGraphSourceProcedure<Identity> procedure) throws Exception {
399 SerialisationSupport support = graph.getService(SerialisationSupport.class);
400 Layer0 L0 = Layer0.getInstance(graph);
402 // TODO: this should be Root with name ""
403 procedure.execute(getRootIdentity(state, support, graph.getRootLibrary()));
405 TIntObjectMap<Identity> internalMap = new TIntObjectHashMap<>(100, 0.5f, Integer.MIN_VALUE);
407 // Declare internal and external roots
408 for(SeedSpec r : configuration.seeds) {
409 if(SeedSpecType.INTERNAL.equals(r.specType)) continue;
410 String typeId = r.type;
411 if (typeId == null) {
412 Resource type = graph.getPossibleType(r.resource, L0.Entity);
413 typeId = type != null ? graph.getURI(type) : Layer0.URIs.Entity;
415 int id = state.ids.get(support.getTransientId(r.resource));
416 Root root = new Root(r.name, typeId);
417 Identity rootId = new Identity(id,root);
418 internalMap.put(id, rootId);
419 procedure.execute(rootId);
422 for(int i = 0; i < state.externals.size() ; i++) {
423 int parent = externalParents.get(i);
424 String name = externalNames.get(i);
425 procedure.execute(new Identity(externalBase + i, new External(parent,name)));
428 if(state.internalEntries != null) {
429 for(ConsistsOfProcessEntry ie : state.internalEntries) {
430 if(ie.parent != null) {
431 if(ie.name != null) {
432 procedure.execute(resolveInternal(graph, support, ie, internalMap));
434 // In this case there is a child that has no HasName => this should be treated as a blank
437 procedure.execute(resolveInternal(graph, support, ie, internalMap));
444 private Identity resolveInternal(ReadGraph graph, SerialisationSupport ss, ConsistsOfProcessEntry entry, TIntObjectMap<Identity> internalMap) throws DatabaseException {
445 int id = state.ids.get(ss.getTransientId(entry.resource));
446 Identity existing = internalMap.get(id);
447 if(existing != null) return existing;
449 if(entry.parent == null) {
450 Layer0 L0 = Layer0.getInstance(graph);
451 Resource possibleParent = graph.getPossibleObject(entry.resource, L0.PartOf);
452 if(possibleParent == null) throw new DatabaseException("Invalid root or internal parent path: " + entry.resource);
453 int externalId = state.ids.get(ss.getTransientId(possibleParent));
454 Identity result = new Identity(id,
455 new Internal(externalId, entry.name));
456 internalMap.put(id, result);
459 Identity parent = resolveInternal(graph, ss, entry.parent, internalMap);
460 Identity result = new Identity(id,
461 new Internal(parent.resource, entry.name));
462 internalMap.put(id, result);
468 public TreeMap<String, Variant> getExtensions() {
469 return state.extensions;
472 public File[] getFiles() {
476 private static <T> T tryClose(T c) throws IOException {
477 if (c != null && c instanceof Closeable)
478 ((Closeable) c).close();
482 public void closeStreams() throws IOException {
483 state.valueInput = tryClose(state.valueInput);
484 state.otherStatementsInput = tryClose(state.otherStatementsInput);
485 state.statementsOutput = tryClose(state.statementsOutput);
486 state.valueOutput = tryClose(state.valueOutput);
490 public void reset() throws Exception {
491 throw new UnsupportedOperationException();
494 public long[] getResourceArray(ReadGraph graph) throws DatabaseException {
495 final SerialisationSupport ss = graph.getService(SerialisationSupport.class);
496 final long[] result = new long[state.ids.size()];
497 state.ids.forEachEntry(new TIntIntProcedure() {
500 public boolean execute(int a, int b) {
503 Resource r = ss.getResource(a);
504 result[b] = r.getResourceId();
505 } catch (DatabaseException e) {
516 public DomainProcessorState getState() {
520 public void forResourceStatements(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
522 int[] value = new int[4];
523 long length = state.otherStatementsInput.length();
524 state.otherStatementsInput.position(0);
526 while(state.otherStatementsInput.position() < length) {
528 int s = state.otherStatementsInput.readInt();
529 int subjectId = state.ids.get(s);
531 boolean exclude = subjectId == -1;
533 int size = state.otherStatementsInput.readInt();
534 for(int i=0;i<size;i++) {
535 int p = state.otherStatementsInput.readInt();
536 int o = state.otherStatementsInput.readInt();
538 if(state.excludedShared.contains(o)) {
539 System.err.println("excluding shared " + s + " " + p + " " + o);
542 int objectId = getExistingId(graph, o);
543 // The statement can be denied still
547 int inverse = state.inverses.get(p);
555 procedure.execute(value);
558 System.err.println("Denied (" + NameUtils.getSafeName(graph, getResource(graph, s)) + ", " + NameUtils.getSafeName(graph, getResource(graph, p)) + "," + NameUtils.getSafeName(graph, getResource(graph, o)) + ")");
563 System.err.println("excluding shared " + s);
569 public void forValueResources(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
571 long length = state.valueInput.length();
572 while (state.valueInput.position() < length) {
573 value[0] = state.valueInput.readInt();
574 procedure.execute(value);
578 public TransferableGraphConfiguration2 getConfiguration() {
579 return configuration;
583 public void close() throws IOException {
584 synchronized (this) {
591 for (File f : files) {
592 Files.deleteIfExists(f.toPath());