1 package org.simantics.db.layer0.util;
3 import java.io.Closeable;
4 import java.io.DataInput;
6 import java.io.IOException;
7 import java.nio.file.Files;
8 import java.util.ArrayList;
9 import java.util.Collection;
10 import java.util.Collections;
11 import java.util.HashSet;
12 import java.util.List;
13 import java.util.TreeMap;
15 import org.simantics.databoard.Bindings;
16 import org.simantics.databoard.Databoard;
17 import org.simantics.databoard.binding.Binding;
18 import org.simantics.databoard.binding.mutable.Variant;
19 import org.simantics.databoard.container.DataContainer;
20 import org.simantics.databoard.serialization.Serializer;
21 import org.simantics.databoard.type.Datatype;
22 import org.simantics.databoard.util.binary.ByteBufferReadable;
23 import org.simantics.db.ReadGraph;
24 import org.simantics.db.Resource;
25 import org.simantics.db.common.utils.NameUtils;
26 import org.simantics.db.exception.DatabaseException;
27 import org.simantics.db.exception.RuntimeDatabaseException;
28 import org.simantics.db.exception.ValidationException;
29 import org.simantics.db.layer0.adapter.SubgraphExtent.ExtentStatus;
30 import org.simantics.db.layer0.util.TransferableGraphConfiguration2.RootSpec;
31 import org.simantics.db.service.SerialisationSupport;
32 import org.simantics.graph.db.TransferableGraphSource;
33 import org.simantics.graph.representation.External;
34 import org.simantics.graph.representation.Identity;
35 import org.simantics.graph.representation.Root;
36 import org.simantics.graph.representation.Value;
37 import org.simantics.layer0.Layer0;
39 import gnu.trove.list.array.TIntArrayList;
40 import gnu.trove.procedure.TIntIntProcedure;
42 public class ModelTransferableGraphSource implements TransferableGraphSource {
44 final private TransferableGraphConfiguration2 configuration;
45 final private DomainProcessorState state;
46 final private int externalBase;
47 final private int resourceCount;
48 final private File[] files;
49 final private TGValueModifier valueModifier;
51 private volatile boolean closed = false;
53 TIntArrayList externalParents = new TIntArrayList();
54 ArrayList<String> externalNames = new ArrayList<String>();
55 TreeMap<String,String> downloads = new TreeMap<String,String>();
57 public ModelTransferableGraphSource(final ReadGraph graph, TransferableGraphConfiguration2 configuration, final DomainProcessorState state, File ... fs) throws DatabaseException {
59 this.configuration = configuration;
62 this.valueModifier = state.valueModifier;
64 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
66 // At this point ids contains all internal resources. Now add roots and externals.
69 state.ids.put(ss.getTransientId(graph.getRootLibrary()), state.id++);
71 // External roots - internal roots were already processed as internal resources by domain processor
72 for(RootSpec rs : configuration.roots) {
74 int resourceId = ss.getTransientId(rs.resource);
75 state.ids.put(resourceId, state.id++);
76 // The fixed roots have been seen as externals by domain processor. Now remove them from external set.
77 state.externals.remove(resourceId);
81 this.externalBase = state.id;
83 final Collection<String> errors = new HashSet<String>();
85 // All resource considered as not internal by domain processor. Can also contain roots.
86 int[] externals = state.externals.toArray();
88 // Build up the state.externals, externalNames and externalParents
89 for(int i=0;i<externals.length;i++) {
90 getId(graph, externals[i], errors);
93 state.inverses.forEachEntry(new TIntIntProcedure() {
96 public boolean execute(int predicate, int inverse) {
98 getId(graph, predicate, errors);
99 if(inverse != 0) getId(graph, inverse, errors);
100 } catch (DatabaseException e) {
101 throw new RuntimeDatabaseException(e);
108 if(!errors.isEmpty()) {
109 ArrayList<String> sorted = new ArrayList<String>(errors);
110 Collections.sort(sorted);
111 StringBuilder message = new StringBuilder();
112 message.append("Errors in exported model:\n");
113 for(String error : sorted) {
114 message.append(error);
115 message.append("\n");
117 throw new DatabaseException(message.toString());
120 this.resourceCount = state.id;
122 state.extensions.put(ExternalDownloadBean.EXTENSION_KEY, new Variant(ExternalDownloadBean.BINDING, new ExternalDownloadBean(downloads)));
128 public boolean validateExternal(Resource ext) {
129 if(configuration.validate) {
130 ExtentStatus status = configuration.preStatus.get(ext);
132 if(ExtentStatus.INTERNAL.equals(status)) return false;
133 else if(ExtentStatus.EXCLUDED.equals(status)) return false;
139 private Resource getResource(ReadGraph graph, int r) throws DatabaseException {
140 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
141 return ss.getResource(r);
144 final public int getExistingId(ReadGraph graph, int r) throws DatabaseException {
146 int ret = state.ids.get(r);
150 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
151 throw new DatabaseException("Id has not been created for " + NameUtils.getSafeName(graph, ss.getResource(r)));
158 * @return -2 if r is not really external and the statement should be excluded
161 public int getId(ReadGraph graph, int r, Collection<String> errors) throws DatabaseException {
163 // // First external is root library
164 // if(r == rootId) return internalCount;
166 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
167 Layer0 L0 = Layer0.getInstance(graph);
169 if(state.ids.containsKey(r)) {
170 int ret = state.ids.get(r);
172 for(int i=0;i<=indent;++i)
173 System.out.print(" ");
174 System.out.println("Cycle!!!"); // with " + GraphUtils.getReadableName(g, r));
179 Resource res = getResource(graph, r);
180 if(!validateExternal(res)) {
181 errors.add("Illegal reference to " + graph.getPossibleURI(getResource(graph, r)));
184 Collection<Resource> parents = graph.getObjects(res, L0.PartOf);
185 if(parents.size() != 1) {
186 throw new ValidationException("Reference to external resource "
187 + NameUtils.getSafeName(graph, getResource(graph, r), true) + " without unique uri (" + parents.size() + " parents).");
190 for(Resource p : parents) {
192 pid = getId(graph, ss.getTransientId(p), errors);
194 errors.add("Illegal reference to " + graph.getPossibleURI(getResource(graph, r)));
199 String name = graph.getRelatedValue(res, L0.HasName);
200 // Record the external entry
201 externalParents.add(pid);
202 externalNames.add(name);
203 state.ids.put(r, state.id);
204 // Ensure that this resource is included into the set of externals to maintain the total number of externals
205 state.externals.add(r);
206 String download = graph.getPossibleRelatedValue(res, L0.Ontology_download, Bindings.STRING);
207 if(download != null) {
208 String uri = graph.getURI(res);
209 downloads.put(uri, download);
216 public DataContainer getHeader() throws Exception {
221 public int getResourceCount() {
222 return resourceCount;
226 public int getIdentityCount() {
227 return configuration.roots.size() + state.externals.size() + 1;
231 public int getStatementCount() {
232 return state.statementCount;
236 public int getValueCount() {
237 return state.valueCount;
241 public void forStatements(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
243 int[] value = new int[4];
244 long length = state.otherStatementsInput.length();
245 state.otherStatementsInput.position(0);
247 while(state.otherStatementsInput.position() < length && !state.monitor.isCanceled()) {
249 int s = state.otherStatementsInput.readInt();
250 int subjectId = state.ids.get(s);
252 boolean exclude = subjectId == -1;
254 int size = state.otherStatementsInput.readInt();
255 for(int i=0;i<size;i++) {
256 int p = state.otherStatementsInput.readInt();
257 int o = state.otherStatementsInput.readInt();
259 if(state.pending.contains(o)) {
260 System.err.println("excluding garbage statement " + s + " " + p + " " + o + ", object resource is garbage");
261 } else if(state.excludedShared.contains(o)) {
262 System.err.println("excluding shared " + s + " " + p + " " + o);
265 int objectId = getExistingId(graph, o);
266 // The statement can be denied still
268 value[0] = subjectId;
269 value[1] = getExistingId(graph, p);
270 int inverse = state.inverses.get(p);
272 value[2] = getExistingId(graph, inverse);
278 procedure.execute(value);
281 System.err.println("Denied (" + NameUtils.getSafeName(graph, getResource(graph, s)) + ", " + NameUtils.getSafeName(graph, getResource(graph, p)) + "," + NameUtils.getSafeName(graph, getResource(graph, o)) + ")");
286 System.err.println("excluding shared " + s);
293 public void forValues(ReadGraph graph, TransferableGraphSourceProcedure<Value> procedure) throws Exception {
295 Serializer variantSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.VARIANT);
296 List<Object> idContext = new ArrayList<>();
297 long length = state.valueInput.length();
298 state.valueInput.position(0);
300 while(state.valueInput.position() < length && !state.monitor.isCanceled()) {
302 // Ignore value type tag
303 int s = state.valueInput.readInt();
304 byte valueType = state.valueInput.readByte();
306 case TAG_RAW_COPY_VARIANT_VALUE:
307 state.valueInput.readInt();
308 // Intentional fallthrough.
310 case TAG_POTENTIALLY_MODIFIED_VARIANT_VALUE: {
312 Variant variant = (Variant) variantSerializer.deserialize((DataInput)state.valueInput, idContext);
313 if (valueModifier.mayNeedModification(variant.type())) {
314 Object currentObject = variant.getValue();
315 Object newObject = valueModifier.modify(state, variant.getBinding(), currentObject);
316 if (newObject != currentObject)
317 variant = new Variant(variant.getBinding(), newObject);
319 procedure.execute(new Value(state.ids.get(s), variant));
324 throw new IllegalArgumentException("Unrecognized variant value type encountered: " + valueType);
330 public void forValues2(ReadGraph graph, TransferableGraphSourceValueProcedure procedure) throws Exception {
332 Serializer datatypeSerializer = graph.getService(Databoard.class).getSerializerUnchecked(Bindings.getBindingUnchecked(Datatype.class));
333 List<Object> idContext = new ArrayList<>();
334 long length = state.valueInput.length();
335 state.valueInput.position(0);
337 while(state.valueInput.position() < length && !state.monitor.isCanceled()) {
339 int s = state.valueInput.readInt();
340 byte valueType = state.valueInput.readByte();
343 case TAG_RAW_COPY_VARIANT_VALUE: {
344 // Variant data could be copied raw, no need for modifications
345 // Note that the variant data is a concatenation of the
346 // variant datatype serialization and the value serialization.
347 // variantLength contains both datatype and value data.
348 int variantLength = state.valueInput.readInt();
349 procedure.rawCopy(state.ids.get(s), variantLength, state.valueInput);
353 case TAG_POTENTIALLY_MODIFIED_VARIANT_VALUE: {
354 // Variant data may need to be modified.
355 // Cannot optimize this case with raw copying.
357 Datatype type = (Datatype)datatypeSerializer.deserialize((DataInput)state.valueInput, idContext);
359 if (valueModifier.mayNeedModification(type)) {
360 Binding binding = Bindings.getBinding(type);
361 Serializer serializer = Bindings.getSerializerUnchecked(binding);
362 Object value = serializer.deserialize((DataInput)state.valueInput);
363 value = valueModifier.modify(state, binding, value);
364 byte[] bytes = serializer.serialize(value);
365 procedure.execute(state.ids.get(s), type, new ByteBufferReadable(bytes));
367 procedure.execute(state.ids.get(s), type, state.valueInput);
373 throw new IllegalArgumentException("Unrecognized variant value type encountered: " + valueType);
378 protected Identity getRootIdentity(DomainProcessorState state, SerialisationSupport support, Resource rootLibrary) throws DatabaseException {
379 return new Identity(state.ids.get(support.getTransientId(rootLibrary)), new External(-1, ""));
383 public void forIdentities(ReadGraph graph, TransferableGraphSourceProcedure<Identity> procedure) throws Exception {
385 SerialisationSupport support = graph.getService(SerialisationSupport.class);
386 Layer0 L0 = Layer0.getInstance(graph);
388 // TODO: this should be Root with name ""
389 procedure.execute(getRootIdentity(state, support, graph.getRootLibrary()));
391 // Declare internal and external roots
392 for(RootSpec r : configuration.roots) {
393 String typeId = r.type;
394 if (typeId == null) {
395 Resource type = graph.getPossibleType(r.resource, L0.Entity);
396 typeId = type != null ? graph.getURI(type) : Layer0.URIs.Entity;
398 procedure.execute(new Identity(
399 state.ids.get(support.getTransientId(r.resource)),
400 new Root(r.name, typeId)));
403 for(int i = 0; i < state.externals.size() ; i++) {
404 int parent = externalParents.get(i);
405 String name = externalNames.get(i);
406 procedure.execute(new Identity(externalBase + i, new External(parent,name)));
412 public TreeMap<String, Variant> getExtensions() {
413 return state.extensions;
416 public File[] getFiles() {
420 private static <T> T tryClose(T c) throws IOException {
421 if (c != null && c instanceof Closeable)
422 ((Closeable) c).close();
426 public void closeStreams() throws IOException {
427 state.valueInput = tryClose(state.valueInput);
428 state.otherStatementsInput = tryClose(state.otherStatementsInput);
429 state.statementsOutput = tryClose(state.statementsOutput);
430 state.valueOutput = tryClose(state.valueOutput);
434 public void reset() throws Exception {
435 throw new UnsupportedOperationException();
438 public long[] getResourceArray(ReadGraph graph) throws DatabaseException {
439 final SerialisationSupport ss = graph.getService(SerialisationSupport.class);
440 final long[] result = new long[state.ids.size()];
441 state.ids.forEachEntry(new TIntIntProcedure() {
444 public boolean execute(int a, int b) {
447 Resource r = ss.getResource(a);
448 result[b] = r.getResourceId();
449 } catch (DatabaseException e) {
460 public DomainProcessorState getState() {
464 public void forResourceStatements(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
466 int[] value = new int[4];
467 long length = state.otherStatementsInput.length();
468 state.otherStatementsInput.position(0);
470 while(state.otherStatementsInput.position() < length) {
472 int s = state.otherStatementsInput.readInt();
473 int subjectId = state.ids.get(s);
475 boolean exclude = subjectId == -1;
477 int size = state.otherStatementsInput.readInt();
478 for(int i=0;i<size;i++) {
479 int p = state.otherStatementsInput.readInt();
480 int o = state.otherStatementsInput.readInt();
482 if(state.excludedShared.contains(o)) {
483 System.err.println("excluding shared " + s + " " + p + " " + o);
486 int objectId = getExistingId(graph, o);
487 // The statement can be denied still
491 int inverse = state.inverses.get(p);
499 procedure.execute(value);
502 System.err.println("Denied (" + NameUtils.getSafeName(graph, getResource(graph, s)) + ", " + NameUtils.getSafeName(graph, getResource(graph, p)) + "," + NameUtils.getSafeName(graph, getResource(graph, o)) + ")");
507 System.err.println("excluding shared " + s);
513 public void forValueResources(ReadGraph graph, TransferableGraphSourceProcedure<int[]> procedure) throws Exception {
515 long length = state.valueInput.length();
516 while (state.valueInput.position() < length) {
517 value[0] = state.valueInput.readInt();
518 procedure.execute(value);
522 public TransferableGraphConfiguration2 getConfiguration() {
523 return configuration;
527 public void close() throws IOException {
528 synchronized (this) {
535 for (File f : files) {
536 Files.deleteIfExists(f.toPath());