bundles/org.simantics.graph.db/src/org/simantics/graph/db/TransferableGraphImportProcess.java

   1 package org.simantics.graph.db;
   2
   3 import java.io.DataOutput;
   4 import java.io.DataOutputStream;
   5 import java.io.FileNotFoundException;
   6 import java.io.FileOutputStream;
   7 import java.io.IOException;
   8 import java.util.HashSet;
   9 import java.util.Map;
  10 import java.util.Set;
  11
  12 import org.simantics.databoard.Bindings;
  13 import org.simantics.databoard.accessor.error.AccessorException;
  14 import org.simantics.databoard.adapter.AdaptException;
  15 import org.simantics.databoard.binding.mutable.Variant;
  16 import org.simantics.databoard.util.URIStringUtils;
  17 import org.simantics.db.ReadGraph;
  18 import org.simantics.db.Resource;
  19 import org.simantics.db.WriteOnlyGraph;
  20 import org.simantics.db.common.WriteBindings;
  21 import org.simantics.db.common.procedure.adapter.TransientCacheAsyncListener;
  22 import org.simantics.db.common.uri.UnescapedChildMapOfResource;
  23 import org.simantics.db.common.utils.Logger;
  24 import org.simantics.db.common.utils.NameUtils;
  25 import org.simantics.db.exception.DatabaseException;
  26 import org.simantics.db.exception.ResourceNotFoundException;
  27 import org.simantics.db.service.ClusterBuilder;
  28 import org.simantics.db.service.ClusterBuilder.ResourceHandle;
  29 import org.simantics.db.service.SerialisationSupport;
  30 import org.simantics.graph.representation.Extensions;
  31 import org.simantics.graph.representation.External;
  32 import org.simantics.graph.representation.Identity;
  33 import org.simantics.graph.representation.IdentityDefinition;
  34 import org.simantics.graph.representation.Internal;
  35 import org.simantics.graph.representation.Optional;
  36 import org.simantics.graph.representation.Root;
  37 import org.simantics.graph.representation.TransferableGraph1;
  38 import org.simantics.graph.representation.TransferableGraphUtils;
  39 import org.simantics.graph.representation.Value;
  40 import org.simantics.graph.utils.TGResourceUtil;
  41 import org.simantics.graph.utils.TGResourceUtil.LongAdapter;
  42
  43 public class TransferableGraphImportProcess implements TransferableGraphImporter {
  44
  45         public static String LOG_FILE = "transferableGraphs.log";
  46         final static private boolean LOG = false;
  47
  48         static DataOutput log;
  49
  50         static {
  51
  52                 if (LOG) {
  53                         try {
  54                                 FileOutputStream stream = new FileOutputStream(LOG_FILE);
  55                                 log = new DataOutputStream(stream);
  56                         } catch (FileNotFoundException e) {
  57                                 e.printStackTrace();
  58                         }
  59                 }
  60
  61         }
  62
  63         private static void log(String line) {
  64                 if (LOG) {
  65                         try {
  66                                 log.writeUTF(line + "\n");
  67                         } catch (IOException e) {
  68                                 e.printStackTrace();
  69                         }
  70                 }
  71         }
  72
  73         TransferableGraph1 tg;
  74         IImportAdvisor advisor;
  75         TGStatusMonitor monitor;
  76         final TGResourceUtil resourceUtil = new TGResourceUtil();
  77
  78         Resource[] resources;
  79         ResourceHandle[] handles;
  80
  81         Set<String> missingExternals = new HashSet<String>();
  82
  83         // Builtins
  84         Resource RootLibrary;
  85         Resource String;
  86         Resource Library;
  87
  88         Resource InstanceOf;
  89         Resource ConsistsOf;
  90         Resource PartOf;
  91         Resource HasName;
  92         Resource NameOf;
  93
  94         public TransferableGraphImportProcess(TransferableGraph1 tg, IImportAdvisor advisor, TGStatusMonitor monitor) {
  95                 this.tg = tg;
  96                 this.advisor = advisor;
  97                 this.monitor = monitor;
  98         }
  99
 100     public TransferableGraphImportProcess(TransferableGraph1 tg, IImportAdvisor advisor) {
 101         this(tg, advisor, null);
 102     }
 103
 104         public void findBuiltins(WriteOnlyGraph g) throws DatabaseException {
 105                 RootLibrary = g.getBuiltin("http:/");
 106                 String = g.getBuiltin(CoreInitialization.LAYER0 + "String");
 107                 Library = g.getBuiltin(CoreInitialization.LAYER0 + "Library");
 108                 InstanceOf = g.getBuiltin(CoreInitialization.LAYER0 + "InstanceOf");
 109                 ConsistsOf = g.getBuiltin(CoreInitialization.LAYER0 + "ConsistsOf");
 110                 PartOf = g.getBuiltin(CoreInitialization.LAYER0 + "PartOf");
 111                 HasName = g.getBuiltin(CoreInitialization.LAYER0 + "HasName");
 112                 NameOf = g.getBuiltin(CoreInitialization.LAYER0 + "NameOf");
 113         }
 114
 115         public void findBuiltins(ReadGraph g) throws DatabaseException {
 116                 RootLibrary = g.getBuiltin("http:/");
 117                 String = g.getBuiltin(CoreInitialization.LAYER0 + "String");
 118                 Library = g.getBuiltin(CoreInitialization.LAYER0 + "Library");
 119                 InstanceOf = g.getBuiltin(CoreInitialization.LAYER0 + "InstanceOf");
 120                 ConsistsOf = g.getBuiltin(CoreInitialization.LAYER0 + "ConsistsOf");
 121                 PartOf = g.getBuiltin(CoreInitialization.LAYER0 + "PartOf");
 122                 HasName = g.getBuiltin(CoreInitialization.LAYER0 + "HasName");
 123                 NameOf = g.getBuiltin(CoreInitialization.LAYER0 + "NameOf");
 124         }
 125
 126         /* Preparation that is used when the core is empty.
 127          */
 128         void initialPrepare(WriteOnlyGraph graph) throws DatabaseException {
 129                 findBuiltins(graph);
 130
 131                 resources = new Resource[tg.resourceCount];
 132
 133                 int Root = -1;
 134                 int SimanticsDomain = -1;
 135                 int Layer0 = -1;
 136
 137                 for(Identity identity : tg.identities) {
 138                         if(identity.definition instanceof Internal) {
 139                                 Internal def = (Internal)identity.definition;
 140                                 Resource res = null;
 141                                 if(def.parent == Layer0) {
 142                                         try {
 143                                                 res = graph.getBuiltin(CoreInitialization.LAYER0 + def.name);
 144                                         } catch(ResourceNotFoundException e) {
 145                                         }
 146                                 }
 147                                 else if(def.parent == SimanticsDomain) {
 148                                         if(def.name.equals("Layer0-1.1"))
 149                                                 Layer0 = identity.resource;
 150                                 }
 151                                 else if(def.parent == Root) {
 152                                         if(def.name.equals("www.simantics.org"))
 153                                                 SimanticsDomain = identity.resource;
 154                                 }
 155
 156                                 if(res == null)
 157                                         res = createChild(graph, resources[def.parent], def.name);
 158                                 else
 159                                         createChild(graph, res, resources[def.parent], def.name);
 160                                 resources[identity.resource] = res;
 161                         }
 162                         else if(identity.definition instanceof Root) {
 163                                 Root = identity.resource;
 164                                 resources[identity.resource] = RootLibrary;
 165                         }
 166                 }
 167         }
 168
 169         void addMissing(String external) {
 170                 Set<String> removals = new HashSet<String>();
 171                 for(String ext : missingExternals) if(ext.startsWith(external)) return;
 172                 for(String ext : missingExternals) if(external.startsWith(ext)) removals.add(ext);
 173                 missingExternals.removeAll(removals);
 174                 missingExternals.add(external);
 175         }
 176
 177         public Resource[] getResources() {
 178                 return resources;
 179         }
 180
 181         public void prepare(ReadGraph graph) throws DatabaseException {
 182                 findBuiltins(graph);
 183
 184                 Resource[] resources = new Resource[tg.resourceCount];
 185
 186                 for(Identity identity : tg.identities) {
 187                         IdentityDefinition definition = identity.definition;
 188                         if(definition instanceof External) {
 189                                 External def = (External)definition;
 190                                 if(def.parent == -1) {
 191                                         resources[identity.resource] = RootLibrary;
 192                                 } else {
 193                                         if("@inverse".equals(def.name)) {
 194                                                 Resource parent = resources[def.parent];
 195                                                 Resource child = graph.getInverse(parent);
 196                                                 resources[identity.resource] = child;
 197                                         } else {
 198                                                 Resource parent = resources[def.parent];
 199                                                 // TODO: escape should be removed when names become well-behaving
 200                                                 if(parent != null) {
 201                                                         Resource child = graph
 202                                                         .syncRequest(new UnescapedChildMapOfResource(parent),
 203                                                                 new TransientCacheAsyncListener<Map<String, Resource>>())
 204                                                                 .get(def.name);
 205                                                         if(child == null) {
 206                                                                 String uri = graph.getPossibleURI(parent);
 207                                                                 if(uri == null) {
 208                                                                         addMissing(URIStringUtils.escape(NameUtils.getSafeName(graph, parent)) + " /" + URIStringUtils.escape(def.name));
 209                                                                 } else {
 210                                                                         addMissing(graph.getURI(parent) + "/" + URIStringUtils.escape(def.name));
 211                                                                 }
 212                                                         }
 213                                                         resources[identity.resource] = child;
 214                                                 } else {
 215                                                     addMissing(TransferableGraphUtils.getURI(tg, def.parent) + "/" + URIStringUtils.escape(def.name));
 216                                                 }
 217                                         }
 218                                 }
 219                         }
 220                         else if(definition instanceof Internal) {
 221                                 // Do not do anything for now
 222                         }
 223                         else if(definition instanceof Root) {
 224                                 Root root = (Root)definition;
 225                                 if(root.name.equals(""))
 226                                         resources[identity.resource] = RootLibrary;
 227                                 else {
 228                                         Resource existing = advisor.analyzeRoot(graph, root);
 229                                         if(existing != null)
 230                                                 resources[identity.resource] = existing;
 231                                 }
 232                         }
 233                         else if(definition instanceof Optional) {
 234                                 External def = (External)definition;
 235                                 Resource parent = resources[def.parent];
 236                                 if(parent != null)
 237                                         resources[identity.resource] =
 238                                                 graph.syncRequest(new UnescapedChildMapOfResource(parent)).get(def.name);
 239                         }
 240                 }
 241
 242                 this.resources = resources;
 243
 244                 if(!missingExternals.isEmpty()) throw new MissingDependencyException(this);
 245
 246         }
 247
 248         Resource createChild(WriteOnlyGraph graph, Resource parent, String name) throws DatabaseException {
 249                 Resource child = graph.newResource();
 250                 //graph.claim(parent, ConsistsOf, PartOf, child);
 251                 Resource nameResource = graph.newResource();
 252                 graph.claim(nameResource, InstanceOf, null, String);
 253                 graph.claimValue(nameResource, name, WriteBindings.STRING);
 254                 graph.claim(child, HasName, NameOf, nameResource);
 255                 return child;
 256         }
 257
 258         @Override
 259         public Resource createChild(WriteOnlyGraph graph, Resource child, Resource parent, String name) throws DatabaseException {
 260                 graph.claim(parent, ConsistsOf, PartOf, child);
 261                 Resource nameResource = graph.newResource();
 262                 graph.claim(nameResource, InstanceOf, null, String);
 263                 graph.claimValue(nameResource, name, WriteBindings.STRING);
 264                 graph.claim(child, HasName, NameOf, nameResource);
 265                 return child;
 266         }
 267
 268         int[] getClustering() {
 269                 Variant v = tg.extensions.get(Extensions.CLUSTERING);
 270                 if(v == null) return null;
 271                 try {
 272                         return (int[])v.getValue(Bindings.INT_ARRAY);
 273                 } catch (AdaptException e) {
 274                         Logger.defaultLogError(e);
 275                         return null;
 276                 }
 277         }
 278
 279         class ResourceAdapter implements LongAdapter {
 280             final SerialisationSupport ss;
 281             final Value value;
 282             Variant original;
 283             ResourceAdapter(SerialisationSupport ss, Value value) {
 284                 this.ss = ss;
 285                 this.value = value;
 286             }
 287         @Override
 288         public long adapt(long in) {
 289             if(original == null) original = value.value.clone();
 290             Resource res = handles[(int)in].resource(ss);
 291             // Maybe throw?
 292             if(res == null) return in;
 293             return res.getResourceId();
 294         }
 295         }
 296
 297         Variant translate(SerialisationSupport ss, Value value, final ResourceHandle[] handles) throws DatabaseException {
 298
 299                 try {
 300                     ResourceAdapter adapter = new ResourceAdapter(ss, value);
 301                         resourceUtil.adaptValue( value.value.getBinding(), value.value.getValue(), adapter);
 302                         Variant result = value.value;
 303                         if(adapter.original != null) value.value = adapter.original;
 304                         return result;
 305                 } catch (AccessorException e) {
 306                         e.printStackTrace();
 307                 }
 308
 309                 return value.value;
 310
 311         }
 312
 313     class ResourceAdapter2 implements LongAdapter {
 314         final Value value;
 315         Variant original;
 316         ResourceAdapter2(Value value) {
 317             this.value = value;
 318         }
 319         @Override
 320         public long adapt(long in) {
 321             if(original == null) original = value.value.clone();
 322             Resource res = resources[(int)in];
 323             // Maybe throw?
 324             if(res == null) return in;
 325             return res.getResourceId();
 326         }
 327     }
 328
 329         Variant translate2(Value value, final Resource[] resources) throws DatabaseException {
 330
 331         try {
 332             ResourceAdapter2 adapter = new ResourceAdapter2(value);
 333             resourceUtil.adaptValue( value.value.getBinding(), value.value.getValue(), adapter);
 334             Variant result = value.value;
 335             if(adapter.original != null) value.value = adapter.original;
 336             return result;
 337         } catch (AccessorException e) {
 338             e.printStackTrace();
 339         }
 340
 341         return value.value;
 342
 343         }
 344
 345         public void write(WriteOnlyGraph graph) throws DatabaseException {
 346
 347                 Resource[] resources = this.resources;
 348
 349                 this.handles = new ResourceHandle[resources.length];
 350
 351                 ResourceHandle[] handles = this.handles;
 352
 353                 int[] clustering = getClustering();
 354
 355                 // Internal identities
 356                 for(Identity identity : tg.identities) {
 357                         IdentityDefinition definition = identity.definition;
 358                         if(resources[identity.resource] != null)
 359                                 continue;
 360                         if(definition instanceof External) {
 361                                 // Already done everything
 362                         }
 363                         else if(definition instanceof Internal) {
 364                                 Internal def = (Internal)definition;
 365                                 resources[identity.resource] =
 366                                         createChild(graph, resources[def.parent], def.name);
 367                         }
 368                         else if(definition instanceof Root) {
 369                                 Root root = (Root)definition;
 370                                 resources[identity.resource] = advisor.createRoot(graph, root);
 371                         }
 372                         else if(definition instanceof Optional) {
 373                                 Optional def = (Optional)definition;
 374                                 Resource child = createChild(graph, resources[def.parent], def.name);
 375                                 graph.claim(child, InstanceOf, null, Library); // ???
 376                                 resources[identity.resource] = child;
 377                         }
 378                 }
 379
 380                 ClusterBuilder builder = graph.getService(ClusterBuilder.class);
 381                 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
 382
 383                 if(clustering != null) {
 384
 385                 int i = 0;
 386                         for(int c : clustering) {
 387                                 builder.newCluster();
 388                                 for(int r=0;r<c;r++, i++)
 389                                         if(resources[i] == null)
 390                                                 handles[i] = builder.newResource();
 391                                         else
 392                                                 handles[i] = builder.resource(resources[i]);
 393
 394                         }
 395
 396                         for(;i<resources.length;++i)
 397                                 if(resources[i] == null)
 398                                         handles[i] = builder.newResource();
 399                                 else
 400                                         handles[i] = builder.resource(resources[i]);
 401
 402                 } else {
 403
 404                         // Create blank resources
 405                         for(int i=0;i<resources.length;++i)
 406                                 if(resources[i] == null)
 407                                         handles[i] = builder.newResource();
 408                                 else
 409                                         handles[i] = builder.resource(resources[i]);
 410
 411                 }
 412
 413                 // Write statements
 414                 int[] statements = tg.statements;
 415
 416 //              int internals = tg.resourceCount - tg.identities.length;
 417
 418                 for(int i=0;i<statements.length;i+=4) {
 419
 420                         int sub = statements[i];
 421                         int pred = statements[i+1];
 422                         int inv = statements[i+2];
 423                         int obj = statements[i+3];
 424
 425                         ResourceHandle subject = handles[sub];
 426                         ResourceHandle predicate = handles[pred];
 427                         ResourceHandle object = handles[obj];
 428
 429                         if(resources[sub] == null) {
 430                                 subject.addStatement(graph, predicate, object);
 431                         } else {
 432                                 graph.claim(
 433                                                 handles[sub].resource(ss),
 434                                                 handles[pred].resource(ss),
 435                                                 null, handles[obj].resource(ss));
 436                         }
 437
 438                         if(inv >= 0) {
 439
 440                                 if(resources[obj] == null) {
 441                                         ResourceHandle inverse = handles[inv];
 442                                         object.addStatement(graph, inverse, subject);
 443                                 } else {
 444                                     Resource s = handles[obj].resource(ss);
 445                                     if(!graph.isImmutable(s))
 446                                         graph.claim(
 447                                                         s,
 448                                                         handles[inv].resource(ss),
 449                                                         null, handles[sub].resource(ss));
 450                                 }
 451
 452                         }
 453
 454                         if(LOG) {
 455                                 log("[STATEMENT] " + resources[statements[i]].getResourceId() + ", " + resources[statements[i+1]].getResourceId() + ", " + resources[statements[i+3]].getResourceId());
 456                         }
 457
 458                 }
 459
 460                 // Write values
 461                 for(Value value : tg.values) {
 462                         Variant variant = translate(ss, value, handles);
 463                         int file = value.resource & 0x80000000;
 464                         int resource = value.resource & 0x7FFFFFFF;
 465                         if (file != 0) {
 466                             throw new UnsupportedOperationException();
 467                                 //graph.claimValue(handles[resource].resource(), value.value, Bindings.BYTE_ARRAY);
 468                         } else {
 469                                 graph.claimValue(handles[resource].resource(ss), variant.getValue(), variant.getBinding());
 470                         }
 471                 }
 472         }
 473
 474         private int updatePercentage(int percentage, int done, int total) {
 475             if(monitor != null && (done & 63) == 0) {
 476                 int current = 100*done / total;
 477                 if(current > percentage) {
 478                     percentage = current;
 479                     monitor.status(percentage);
 480                 }
 481             }
 482         return percentage;
 483         }
 484
 485         void write2(WriteOnlyGraph graph) throws DatabaseException {
 486                 Resource[] resources = this.resources;
 487
 488                 // Internal identities
 489                 for(Identity identity : tg.identities) {
 490                         IdentityDefinition definition = identity.definition;
 491                         if(resources[identity.resource] != null)
 492                                 continue;
 493                         if(definition instanceof External) {
 494                                 // Already done everything
 495                         }
 496                         else if(definition instanceof Internal) {
 497                                 Internal def = (Internal)definition;
 498                                 resources[identity.resource] =
 499                                         createChild(graph, resources[def.parent], def.name);
 500                         }
 501                         else if(definition instanceof Root) {
 502                                 Root root = (Root)definition;
 503                                 resources[identity.resource] = advisor.createRoot(graph, root);
 504                         }
 505                         else if(definition instanceof Optional) {
 506                                 Optional def = (Optional)definition;
 507                                 Resource child = createChild(graph, resources[def.parent], def.name);
 508                                 graph.claim(child, InstanceOf, null, Library); // ???
 509                                 resources[identity.resource] = child;
 510                         }
 511                 }
 512
 513                 // Create blank resources
 514                 for(int i=0;i<resources.length;++i)
 515                         if(resources[i] == null)
 516                                 resources[i] = graph.newResource();
 517
 518                 int total = tg.statements.length + tg.values.length;
 519                 int done = 0;
 520                 int percentage = 0;
 521
 522                 // Write statements
 523                 int[] statements = tg.statements;
 524
 525                 for(int i=0;i<statements.length;i+=4) {
 526                         int inv = statements[i+2];
 527                         graph.claim(
 528                                         resources[statements[i]],
 529                                         resources[statements[i+1]],
 530                                         inv < 0 ? null : resources[inv],
 531                                         resources[statements[i+3]]
 532                                         );
 533                         if(LOG) {
 534                                 log("[STATEMENT] " + resources[statements[i]].getResourceId() + ", " + resources[statements[i+1]].getResourceId() + ", " + resources[statements[i+3]].getResourceId());
 535                         }
 536                         percentage = updatePercentage(percentage, done++, total);
 537                 }
 538
 539                 // Write values
 540                 for(Value value : tg.values) {
 541                         Variant variant = translate2(value, resources);
 542                         int file = value.resource & 0x80000000;
 543                         int resource = value.resource & 0x7FFFFFFF;
 544                         if (file != 0) {
 545                             throw new UnsupportedOperationException();
 546                                 //graph.claimValue(resources[resource], value.value, Bindings.BYTE_ARRAY);
 547                         } else {
 548                             graph.claimValue(resources[resource], variant.getValue(), variant.getBinding());
 549                         }
 550             percentage = updatePercentage(percentage, done++, total);
 551                 }
 552
 553         }
 554
 555         public long[] getResourceIds(SerialisationSupport serializer) throws DatabaseException {
 556                 final int count = resources.length;
 557                 long[] resourceIds = new long[count];
 558                 if(handles != null) {
 559                         for(int i=0;i<count;++i)
 560                                 resourceIds[i] = serializer.getRandomAccessId(handles[i].resource(serializer));
 561                 } else {
 562                         for(int i=0;i<count;++i)
 563                                 resourceIds[i] = serializer.getRandomAccessId(resources[i]);
 564                 }
 565                 return resourceIds;
 566         }
 567 }