bundles/org.simantics.graph.db/src/org/simantics/graph/db/TransferableGraphImportProcess.java

   1 package org.simantics.graph.db;
   2
   3 import java.io.DataOutput;
   4 import java.io.DataOutputStream;
   5 import java.io.FileNotFoundException;
   6 import java.io.FileOutputStream;
   7 import java.io.IOException;
   8 import java.util.HashSet;
   9 import java.util.Map;
  10 import java.util.Set;
  11
  12 import org.simantics.databoard.Bindings;
  13 import org.simantics.databoard.accessor.error.AccessorException;
  14 import org.simantics.databoard.adapter.AdaptException;
  15 import org.simantics.databoard.binding.mutable.Variant;
  16 import org.simantics.databoard.util.URIStringUtils;
  17 import org.simantics.db.ReadGraph;
  18 import org.simantics.db.Resource;
  19 import org.simantics.db.WriteOnlyGraph;
  20 import org.simantics.db.common.WriteBindings;
  21 import org.simantics.db.common.procedure.adapter.TransientCacheAsyncListener;
  22 import org.simantics.db.common.uri.UnescapedChildMapOfResource;
  23 import org.simantics.db.common.utils.Logger;
  24 import org.simantics.db.common.utils.NameUtils;
  25 import org.simantics.db.exception.DatabaseException;
  26 import org.simantics.db.exception.ResourceNotFoundException;
  27 import org.simantics.db.service.ClusterBuilder;
  28 import org.simantics.db.service.ClusterBuilder.ResourceHandle;
  29 import org.simantics.db.service.SerialisationSupport;
  30 import org.simantics.graph.representation.Extensions;
  31 import org.simantics.graph.representation.External;
  32 import org.simantics.graph.representation.Identity;
  33 import org.simantics.graph.representation.IdentityDefinition;
  34 import org.simantics.graph.representation.Internal;
  35 import org.simantics.graph.representation.Optional;
  36 import org.simantics.graph.representation.Root;
  37 import org.simantics.graph.representation.TransferableGraph1;
  38 import org.simantics.graph.representation.TransferableGraphUtils;
  39 import org.simantics.graph.representation.Value;
  40 import org.simantics.graph.utils.TGResourceUtil;
  41 import org.simantics.graph.utils.TGResourceUtil.LongAdapter;
  42
  43 public class TransferableGraphImportProcess implements TransferableGraphImporter {
  44
  45         public static String LOG_FILE = "transferableGraphs.log";
  46         final static private boolean LOG = false;
  47
  48         static DataOutput log;
  49
  50         static {
  51
  52                 if (LOG) {
  53                         try {
  54                                 FileOutputStream stream = new FileOutputStream(LOG_FILE);
  55                                 log = new DataOutputStream(stream);
  56                         } catch (FileNotFoundException e) {
  57                                 e.printStackTrace();
  58                         }
  59                 }
  60
  61         }
  62
  63         private static void log(String line) {
  64                 if (LOG) {
  65                         try {
  66                                 log.writeUTF(line + "\n");
  67                         } catch (IOException e) {
  68                                 e.printStackTrace();
  69                         }
  70                 }
  71         }
  72
  73         TransferableGraph1 tg;
  74         IImportAdvisor advisor;
  75         TGStatusMonitor monitor;
  76         final TGResourceUtil resourceUtil = new TGResourceUtil();
  77
  78         Resource[] resources;
  79         ResourceHandle[] handles;
  80
  81         Set<String> missingExternals = new HashSet<String>();
  82
  83         // Builtins
  84         Resource RootLibrary;
  85         Resource String;
  86         Resource Library;
  87
  88         Resource InstanceOf;
  89         Resource ConsistsOf;
  90         Resource PartOf;
  91         Resource HasName;
  92         Resource NameOf;
  93
  94         public TransferableGraphImportProcess(TransferableGraph1 tg, IImportAdvisor advisor, TGStatusMonitor monitor) {
  95                 this.tg = tg;
  96                 this.advisor = advisor;
  97                 this.monitor = monitor;
  98         }
  99
 100     public TransferableGraphImportProcess(TransferableGraph1 tg, IImportAdvisor advisor) {
 101         this(tg, advisor, null);
 102     }
 103
 104         public void findBuiltins(WriteOnlyGraph g) throws DatabaseException {
 105                 RootLibrary = g.getBuiltin("http:/");
 106                 String = g.getBuiltin(CoreInitialization.LAYER0 + "String");
 107                 Library = g.getBuiltin(CoreInitialization.LAYER0 + "Library");
 108                 InstanceOf = g.getBuiltin(CoreInitialization.LAYER0 + "InstanceOf");
 109                 ConsistsOf = g.getBuiltin(CoreInitialization.LAYER0 + "ConsistsOf");
 110                 PartOf = g.getBuiltin(CoreInitialization.LAYER0 + "PartOf");
 111                 HasName = g.getBuiltin(CoreInitialization.LAYER0 + "HasName");
 112                 NameOf = g.getBuiltin(CoreInitialization.LAYER0 + "NameOf");
 113         }
 114
 115         public void findBuiltins(ReadGraph g) throws DatabaseException {
 116                 RootLibrary = g.getBuiltin("http:/");
 117                 String = g.getBuiltin(CoreInitialization.LAYER0 + "String");
 118                 Library = g.getBuiltin(CoreInitialization.LAYER0 + "Library");
 119                 InstanceOf = g.getBuiltin(CoreInitialization.LAYER0 + "InstanceOf");
 120                 ConsistsOf = g.getBuiltin(CoreInitialization.LAYER0 + "ConsistsOf");
 121                 PartOf = g.getBuiltin(CoreInitialization.LAYER0 + "PartOf");
 122                 HasName = g.getBuiltin(CoreInitialization.LAYER0 + "HasName");
 123                 NameOf = g.getBuiltin(CoreInitialization.LAYER0 + "NameOf");
 124         }
 125
 126         /* Preparation that is used when the core is empty.
 127          */
 128         void initialPrepare(WriteOnlyGraph graph) throws DatabaseException {
 129                 findBuiltins(graph);
 130
 131                 resources = new Resource[tg.resourceCount];
 132
 133                 int Root = -1;
 134                 int SimanticsDomain = -1;
 135                 int Layer0 = -1;
 136
 137                 for(Identity identity : tg.identities) {
 138                         if(identity.definition instanceof Internal) {
 139                                 Internal def = (Internal)identity.definition;
 140                                 Resource res = null;
 141                                 if(def.parent == Layer0) {
 142                                         try {
 143                                                 res = graph.getBuiltin(CoreInitialization.LAYER0 + def.name);
 144                                         } catch(ResourceNotFoundException e) {
 145                                         }
 146                                 }
 147                                 else if(def.parent == SimanticsDomain) {
 148                                         if(def.name.equals("Layer0-1.1"))
 149                                                 Layer0 = identity.resource;
 150                                 }
 151                                 else if(def.parent == Root) {
 152                                         if(def.name.equals("www.simantics.org"))
 153                                                 SimanticsDomain = identity.resource;
 154                                 }
 155
 156                                 if(res == null)
 157                                         res = createChild(graph, resources[def.parent], def.name);
 158                                 else
 159                                         createChild(graph, res, resources[def.parent], def.name);
 160                                 resources[identity.resource] = res;
 161                         }
 162                         else if(identity.definition instanceof Root) {
 163                                 Root = identity.resource;
 164                                 resources[identity.resource] = RootLibrary;
 165                         }
 166                 }
 167         }
 168
 169         void addMissing(String external) {
 170                 Set<String> removals = new HashSet<String>();
 171                 for(String ext : missingExternals) if(ext.startsWith(external)) return;
 172                 for(String ext : missingExternals) if(external.startsWith(ext)) removals.add(ext);
 173                 missingExternals.removeAll(removals);
 174                 missingExternals.add(external);
 175         }
 176
 177         void prepare(ReadGraph graph) throws DatabaseException {
 178                 findBuiltins(graph);
 179
 180                 Resource[] resources = new Resource[tg.resourceCount];
 181
 182                 for(Identity identity : tg.identities) {
 183                         IdentityDefinition definition = identity.definition;
 184                         if(definition instanceof External) {
 185                                 External def = (External)definition;
 186                                 if(def.parent == -1) {
 187                                         resources[identity.resource] = RootLibrary;
 188                                 } else {
 189                                         if("@inverse".equals(def.name)) {
 190                                                 Resource parent = resources[def.parent];
 191                                                 Resource child = graph.getInverse(parent);
 192                                                 resources[identity.resource] = child;
 193                                         } else {
 194                                                 Resource parent = resources[def.parent];
 195                                                 // TODO: escape should be removed when names become well-behaving
 196                                                 if(parent != null) {
 197                                                         Resource child = graph
 198                                                         .syncRequest(new UnescapedChildMapOfResource(parent),
 199                                                                 new TransientCacheAsyncListener<Map<String, Resource>>())
 200                                                                 .get(def.name);
 201                                                         if(child == null) {
 202                                                                 String uri = graph.getPossibleURI(parent);
 203                                                                 if(uri == null) {
 204                                                                         addMissing(URIStringUtils.escape(NameUtils.getSafeName(graph, parent)) + " /" + URIStringUtils.escape(def.name));
 205                                                                 } else {
 206                                                                         addMissing(graph.getURI(parent) + "/" + URIStringUtils.escape(def.name));
 207                                                                 }
 208                                                         }
 209                                                         resources[identity.resource] = child;
 210                                                 } else {
 211                                                     addMissing(TransferableGraphUtils.getURI(tg, def.parent) + "/" + URIStringUtils.escape(def.name));
 212                                                 }
 213                                         }
 214                                 }
 215                         }
 216                         else if(definition instanceof Internal) {
 217                                 // Do not do anything for now
 218                         }
 219                         else if(definition instanceof Root) {
 220                                 Root root = (Root)definition;
 221                                 if(root.name.equals(""))
 222                                         resources[identity.resource] = RootLibrary;
 223                                 else {
 224                                         Resource existing = advisor.analyzeRoot(graph, root);
 225                                         if(existing != null)
 226                                                 resources[identity.resource] = existing;
 227                                 }
 228                         }
 229                         else if(definition instanceof Optional) {
 230                                 External def = (External)definition;
 231                                 Resource parent = resources[def.parent];
 232                                 if(parent != null)
 233                                         resources[identity.resource] =
 234                                                 graph.syncRequest(new UnescapedChildMapOfResource(parent)).get(def.name);
 235                         }
 236                 }
 237
 238                 this.resources = resources;
 239
 240                 if(!missingExternals.isEmpty()) throw new MissingDependencyException(this);
 241
 242         }
 243
 244         Resource createChild(WriteOnlyGraph graph, Resource parent, String name) throws DatabaseException {
 245                 Resource child = graph.newResource();
 246                 //graph.claim(parent, ConsistsOf, PartOf, child);
 247                 Resource nameResource = graph.newResource();
 248                 graph.claim(nameResource, InstanceOf, null, String);
 249                 graph.claimValue(nameResource, name, WriteBindings.STRING);
 250                 graph.claim(child, HasName, NameOf, nameResource);
 251                 return child;
 252         }
 253
 254         @Override
 255         public Resource createChild(WriteOnlyGraph graph, Resource child, Resource parent, String name) throws DatabaseException {
 256                 graph.claim(parent, ConsistsOf, PartOf, child);
 257                 Resource nameResource = graph.newResource();
 258                 graph.claim(nameResource, InstanceOf, null, String);
 259                 graph.claimValue(nameResource, name, WriteBindings.STRING);
 260                 graph.claim(child, HasName, NameOf, nameResource);
 261                 return child;
 262         }
 263
 264         int[] getClustering() {
 265                 Variant v = tg.extensions.get(Extensions.CLUSTERING);
 266                 if(v == null) return null;
 267                 try {
 268                         return (int[])v.getValue(Bindings.INT_ARRAY);
 269                 } catch (AdaptException e) {
 270                         Logger.defaultLogError(e);
 271                         return null;
 272                 }
 273         }
 274
 275         class ResourceAdapter implements LongAdapter {
 276             final SerialisationSupport ss;
 277             final Value value;
 278             Variant original;
 279             ResourceAdapter(SerialisationSupport ss, Value value) {
 280                 this.ss = ss;
 281                 this.value = value;
 282             }
 283         @Override
 284         public long adapt(long in) {
 285             if(original == null) original = value.value.clone();
 286             Resource res = handles[(int)in].resource(ss);
 287             // Maybe throw?
 288             if(res == null) return in;
 289             return res.getResourceId();
 290         }
 291         }
 292
 293         Variant translate(SerialisationSupport ss, Value value, final ResourceHandle[] handles) throws DatabaseException {
 294
 295                 try {
 296                     ResourceAdapter adapter = new ResourceAdapter(ss, value);
 297                         resourceUtil.adaptValue( value.value.getBinding(), value.value.getValue(), adapter);
 298                         Variant result = value.value;
 299                         if(adapter.original != null) value.value = adapter.original;
 300                         return result;
 301                 } catch (AccessorException e) {
 302                         e.printStackTrace();
 303                 }
 304
 305                 return value.value;
 306
 307         }
 308
 309     class ResourceAdapter2 implements LongAdapter {
 310         final Value value;
 311         Variant original;
 312         ResourceAdapter2(Value value) {
 313             this.value = value;
 314         }
 315         @Override
 316         public long adapt(long in) {
 317             if(original == null) original = value.value.clone();
 318             Resource res = resources[(int)in];
 319             // Maybe throw?
 320             if(res == null) return in;
 321             return res.getResourceId();
 322         }
 323     }
 324
 325         Variant translate2(Value value, final Resource[] resources) throws DatabaseException {
 326
 327         try {
 328             ResourceAdapter2 adapter = new ResourceAdapter2(value);
 329             resourceUtil.adaptValue( value.value.getBinding(), value.value.getValue(), adapter);
 330             Variant result = value.value;
 331             if(adapter.original != null) value.value = adapter.original;
 332             return result;
 333         } catch (AccessorException e) {
 334             e.printStackTrace();
 335         }
 336
 337         return value.value;
 338
 339         }
 340
 341         void write(WriteOnlyGraph graph) throws DatabaseException {
 342
 343                 Resource[] resources = this.resources;
 344
 345                 this.handles = new ResourceHandle[resources.length];
 346
 347                 ResourceHandle[] handles = this.handles;
 348
 349                 int[] clustering = getClustering();
 350
 351                 // Internal identities
 352                 for(Identity identity : tg.identities) {
 353                         IdentityDefinition definition = identity.definition;
 354                         if(resources[identity.resource] != null)
 355                                 continue;
 356                         if(definition instanceof External) {
 357                                 // Already done everything
 358                         }
 359                         else if(definition instanceof Internal) {
 360                                 Internal def = (Internal)definition;
 361                                 resources[identity.resource] =
 362                                         createChild(graph, resources[def.parent], def.name);
 363                         }
 364                         else if(definition instanceof Root) {
 365                                 Root root = (Root)definition;
 366                                 resources[identity.resource] = advisor.createRoot(graph, root);
 367                         }
 368                         else if(definition instanceof Optional) {
 369                                 Optional def = (Optional)definition;
 370                                 Resource child = createChild(graph, resources[def.parent], def.name);
 371                                 graph.claim(child, InstanceOf, null, Library); // ???
 372                                 resources[identity.resource] = child;
 373                         }
 374                 }
 375
 376                 ClusterBuilder builder = graph.getService(ClusterBuilder.class);
 377                 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
 378
 379                 if(clustering != null) {
 380
 381                 int i = 0;
 382                         for(int c : clustering) {
 383                                 builder.newCluster();
 384                                 for(int r=0;r<c;r++, i++)
 385                                         if(resources[i] == null)
 386                                                 handles[i] = builder.newResource();
 387                                         else
 388                                                 handles[i] = builder.resource(resources[i]);
 389
 390                         }
 391
 392                         for(;i<resources.length;++i)
 393                                 if(resources[i] == null)
 394                                         handles[i] = builder.newResource();
 395                                 else
 396                                         handles[i] = builder.resource(resources[i]);
 397
 398                 } else {
 399
 400                         // Create blank resources
 401                         for(int i=0;i<resources.length;++i)
 402                                 if(resources[i] == null)
 403                                         handles[i] = builder.newResource();
 404                                 else
 405                                         handles[i] = builder.resource(resources[i]);
 406
 407                 }
 408
 409                 // Write statements
 410                 int[] statements = tg.statements;
 411
 412 //              int internals = tg.resourceCount - tg.identities.length;
 413
 414                 for(int i=0;i<statements.length;i+=4) {
 415
 416                         int sub = statements[i];
 417                         int pred = statements[i+1];
 418                         int inv = statements[i+2];
 419                         int obj = statements[i+3];
 420
 421                         ResourceHandle subject = handles[sub];
 422                         ResourceHandle predicate = handles[pred];
 423                         ResourceHandle object = handles[obj];
 424
 425                         if(resources[sub] == null) {
 426                                 subject.addStatement(graph, predicate, object);
 427                         } else {
 428                                 graph.claim(
 429                                                 handles[sub].resource(ss),
 430                                                 handles[pred].resource(ss),
 431                                                 null, handles[obj].resource(ss));
 432                         }
 433
 434                         if(inv >= 0) {
 435
 436                                 if(resources[obj] == null) {
 437                                         ResourceHandle inverse = handles[inv];
 438                                         object.addStatement(graph, inverse, subject);
 439                                 } else {
 440                                     Resource s = handles[obj].resource(ss);
 441                                     if(!graph.isImmutable(s))
 442                                         graph.claim(
 443                                                         s,
 444                                                         handles[inv].resource(ss),
 445                                                         null, handles[sub].resource(ss));
 446                                 }
 447
 448                         }
 449
 450                         if(LOG) {
 451                                 log("[STATEMENT] " + resources[statements[i]].getResourceId() + ", " + resources[statements[i+1]].getResourceId() + ", " + resources[statements[i+3]].getResourceId());
 452                         }
 453
 454                 }
 455
 456                 // Write values
 457                 for(Value value : tg.values) {
 458                         Variant variant = translate(ss, value, handles);
 459                         int file = value.resource & 0x80000000;
 460                         int resource = value.resource & 0x7FFFFFFF;
 461                         if (file != 0) {
 462                             throw new UnsupportedOperationException();
 463                                 //graph.claimValue(handles[resource].resource(), value.value, Bindings.BYTE_ARRAY);
 464                         } else {
 465                                 graph.claimValue(handles[resource].resource(ss), variant.getValue(), variant.getBinding());
 466                         }
 467                 }
 468         }
 469
 470         private int updatePercentage(int percentage, int done, int total) {
 471             if(monitor != null && (done & 63) == 0) {
 472                 int current = 100*done / total;
 473                 if(current > percentage) {
 474                     percentage = current;
 475                     monitor.status(percentage);
 476                 }
 477             }
 478         return percentage;
 479         }
 480
 481         void write2(WriteOnlyGraph graph) throws DatabaseException {
 482                 Resource[] resources = this.resources;
 483
 484                 // Internal identities
 485                 for(Identity identity : tg.identities) {
 486                         IdentityDefinition definition = identity.definition;
 487                         if(resources[identity.resource] != null)
 488                                 continue;
 489                         if(definition instanceof External) {
 490                                 // Already done everything
 491                         }
 492                         else if(definition instanceof Internal) {
 493                                 Internal def = (Internal)definition;
 494                                 resources[identity.resource] =
 495                                         createChild(graph, resources[def.parent], def.name);
 496                         }
 497                         else if(definition instanceof Root) {
 498                                 Root root = (Root)definition;
 499                                 resources[identity.resource] = advisor.createRoot(graph, root);
 500                         }
 501                         else if(definition instanceof Optional) {
 502                                 Optional def = (Optional)definition;
 503                                 Resource child = createChild(graph, resources[def.parent], def.name);
 504                                 graph.claim(child, InstanceOf, null, Library); // ???
 505                                 resources[identity.resource] = child;
 506                         }
 507                 }
 508
 509                 // Create blank resources
 510                 for(int i=0;i<resources.length;++i)
 511                         if(resources[i] == null)
 512                                 resources[i] = graph.newResource();
 513
 514                 int total = tg.statements.length + tg.values.length;
 515                 int done = 0;
 516                 int percentage = 0;
 517
 518                 // Write statements
 519                 int[] statements = tg.statements;
 520
 521                 for(int i=0;i<statements.length;i+=4) {
 522                         int inv = statements[i+2];
 523                         graph.claim(
 524                                         resources[statements[i]],
 525                                         resources[statements[i+1]],
 526                                         inv < 0 ? null : resources[inv],
 527                                         resources[statements[i+3]]
 528                                         );
 529                         if(LOG) {
 530                                 log("[STATEMENT] " + resources[statements[i]].getResourceId() + ", " + resources[statements[i+1]].getResourceId() + ", " + resources[statements[i+3]].getResourceId());
 531                         }
 532                         percentage = updatePercentage(percentage, done++, total);
 533                 }
 534
 535                 // Write values
 536                 for(Value value : tg.values) {
 537                         Variant variant = translate2(value, resources);
 538                         int file = value.resource & 0x80000000;
 539                         int resource = value.resource & 0x7FFFFFFF;
 540                         if (file != 0) {
 541                             throw new UnsupportedOperationException();
 542                                 //graph.claimValue(resources[resource], value.value, Bindings.BYTE_ARRAY);
 543                         } else {
 544                             graph.claimValue(resources[resource], variant.getValue(), variant.getBinding());
 545                         }
 546             percentage = updatePercentage(percentage, done++, total);
 547                 }
 548
 549         }
 550
 551         public long[] getResourceIds(SerialisationSupport serializer) throws DatabaseException {
 552                 final int count = resources.length;
 553                 long[] resourceIds = new long[count];
 554                 if(handles != null) {
 555                         for(int i=0;i<count;++i)
 556                                 resourceIds[i] = serializer.getRandomAccessId(handles[i].resource(serializer));
 557                 } else {
 558                         for(int i=0;i<count;++i)
 559                                 resourceIds[i] = serializer.getRandomAccessId(resources[i]);
 560                 }
 561                 return resourceIds;
 562         }
 563 }