bundles/org.simantics.graph.db/src/org/simantics/graph/db/TransferableGraphImportProcess.java

   1 package org.simantics.graph.db;
   2
   3 import java.io.DataOutput;
   4 import java.io.DataOutputStream;
   5 import java.io.FileNotFoundException;
   6 import java.io.FileOutputStream;
   7 import java.io.IOException;
   8 import java.util.HashSet;
   9 import java.util.Map;
  10 import java.util.Set;
  11
  12 import org.simantics.databoard.Bindings;
  13 import org.simantics.databoard.accessor.error.AccessorException;
  14 import org.simantics.databoard.adapter.AdaptException;
  15 import org.simantics.databoard.binding.mutable.Variant;
  16 import org.simantics.db.ReadGraph;
  17 import org.simantics.db.Resource;
  18 import org.simantics.db.WriteOnlyGraph;
  19 import org.simantics.db.common.WriteBindings;
  20 import org.simantics.db.common.procedure.adapter.TransientCacheAsyncListener;
  21 import org.simantics.db.common.uri.UnescapedChildMapOfResource;
  22 import org.simantics.db.common.utils.Logger;
  23 import org.simantics.db.common.utils.NameUtils;
  24 import org.simantics.db.exception.DatabaseException;
  25 import org.simantics.db.exception.ResourceNotFoundException;
  26 import org.simantics.db.service.ClusterBuilder;
  27 import org.simantics.db.service.ClusterBuilder.ResourceHandle;
  28 import org.simantics.db.service.SerialisationSupport;
  29 import org.simantics.graph.representation.Extensions;
  30 import org.simantics.graph.representation.External;
  31 import org.simantics.graph.representation.Identity;
  32 import org.simantics.graph.representation.IdentityDefinition;
  33 import org.simantics.graph.representation.Internal;
  34 import org.simantics.graph.representation.Optional;
  35 import org.simantics.graph.representation.Root;
  36 import org.simantics.graph.representation.TransferableGraph1;
  37 import org.simantics.graph.representation.TransferableGraphUtils;
  38 import org.simantics.graph.representation.Value;
  39 import org.simantics.graph.utils.TGResourceUtil;
  40 import org.simantics.graph.utils.TGResourceUtil.LongAdapter;
  41
  42 public class TransferableGraphImportProcess implements TransferableGraphImporter {
  43
  44         public static String LOG_FILE = "transferableGraphs.log";
  45         final static private boolean LOG = false;
  46
  47         static DataOutput log;
  48
  49         static {
  50
  51                 if (LOG) {
  52                         try {
  53                                 FileOutputStream stream = new FileOutputStream(LOG_FILE);
  54                                 log = new DataOutputStream(stream);
  55                         } catch (FileNotFoundException e) {
  56                                 e.printStackTrace();
  57                         }
  58                 }
  59
  60         }
  61
  62         private static void log(String line) {
  63                 if (LOG) {
  64                         try {
  65                                 log.writeUTF(line + "\n");
  66                         } catch (IOException e) {
  67                                 e.printStackTrace();
  68                         }
  69                 }
  70         }
  71
  72         TransferableGraph1 tg;
  73         IImportAdvisor advisor;
  74         TGStatusMonitor monitor;
  75         final TGResourceUtil resourceUtil = new TGResourceUtil();
  76
  77         Resource[] resources;
  78         ResourceHandle[] handles;
  79
  80         Set<String> missingExternals = new HashSet<String>();
  81
  82         // Builtins
  83         Resource RootLibrary;
  84         Resource String;
  85         Resource Library;
  86
  87         Resource InstanceOf;
  88         Resource ConsistsOf;
  89         Resource PartOf;
  90         Resource HasName;
  91         Resource NameOf;
  92
  93         public TransferableGraphImportProcess(TransferableGraph1 tg, IImportAdvisor advisor, TGStatusMonitor monitor) {
  94                 this.tg = tg;
  95                 this.advisor = advisor;
  96                 this.monitor = monitor;
  97         }
  98
  99     public TransferableGraphImportProcess(TransferableGraph1 tg, IImportAdvisor advisor) {
 100         this(tg, advisor, null);
 101     }
 102
 103         public void findBuiltins(WriteOnlyGraph g) throws DatabaseException {
 104                 RootLibrary = g.getBuiltin("http:/");
 105                 String = g.getBuiltin(CoreInitialization.LAYER0 + "String");
 106                 Library = g.getBuiltin(CoreInitialization.LAYER0 + "Library");
 107                 InstanceOf = g.getBuiltin(CoreInitialization.LAYER0 + "InstanceOf");
 108                 ConsistsOf = g.getBuiltin(CoreInitialization.LAYER0 + "ConsistsOf");
 109                 PartOf = g.getBuiltin(CoreInitialization.LAYER0 + "PartOf");
 110                 HasName = g.getBuiltin(CoreInitialization.LAYER0 + "HasName");
 111                 NameOf = g.getBuiltin(CoreInitialization.LAYER0 + "NameOf");
 112         }
 113
 114         public void findBuiltins(ReadGraph g) throws DatabaseException {
 115                 RootLibrary = g.getBuiltin("http:/");
 116                 String = g.getBuiltin(CoreInitialization.LAYER0 + "String");
 117                 Library = g.getBuiltin(CoreInitialization.LAYER0 + "Library");
 118                 InstanceOf = g.getBuiltin(CoreInitialization.LAYER0 + "InstanceOf");
 119                 ConsistsOf = g.getBuiltin(CoreInitialization.LAYER0 + "ConsistsOf");
 120                 PartOf = g.getBuiltin(CoreInitialization.LAYER0 + "PartOf");
 121                 HasName = g.getBuiltin(CoreInitialization.LAYER0 + "HasName");
 122                 NameOf = g.getBuiltin(CoreInitialization.LAYER0 + "NameOf");
 123         }
 124
 125         /* Preparation that is used when the core is empty.
 126          */
 127         void initialPrepare(WriteOnlyGraph graph) throws DatabaseException {
 128                 findBuiltins(graph);
 129
 130                 resources = new Resource[tg.resourceCount];
 131
 132                 int Root = -1;
 133                 int SimanticsDomain = -1;
 134                 int Layer0 = -1;
 135
 136                 for(Identity identity : tg.identities) {
 137                         if(identity.definition instanceof Internal) {
 138                                 Internal def = (Internal)identity.definition;
 139                                 Resource res = null;
 140                                 if(def.parent == Layer0) {
 141                                         try {
 142                                                 res = graph.getBuiltin(CoreInitialization.LAYER0 + def.name);
 143                                         } catch(ResourceNotFoundException e) {
 144                                         }
 145                                 }
 146                                 else if(def.parent == SimanticsDomain) {
 147                                         if(def.name.equals("Layer0-1.1"))
 148                                                 Layer0 = identity.resource;
 149                                 }
 150                                 else if(def.parent == Root) {
 151                                         if(def.name.equals("www.simantics.org"))
 152                                                 SimanticsDomain = identity.resource;
 153                                 }
 154
 155                                 if(res == null)
 156                                         res = createChild(graph, resources[def.parent], def.name);
 157                                 else
 158                                         createChild(graph, res, resources[def.parent], def.name);
 159                                 resources[identity.resource] = res;
 160                         }
 161                         else if(identity.definition instanceof Root) {
 162                                 Root = identity.resource;
 163                                 resources[identity.resource] = RootLibrary;
 164                         }
 165                 }
 166         }
 167
 168         void addMissing(String external) {
 169                 Set<String> removals = new HashSet<String>();
 170                 for(String ext : missingExternals) if(ext.startsWith(external)) return;
 171                 for(String ext : missingExternals) if(external.startsWith(ext)) removals.add(ext);
 172                 missingExternals.removeAll(removals);
 173                 missingExternals.add(external);
 174         }
 175
 176         void prepare(ReadGraph graph) throws DatabaseException {
 177                 findBuiltins(graph);
 178
 179                 Resource[] resources = new Resource[tg.resourceCount];
 180
 181                 for(Identity identity : tg.identities) {
 182                         IdentityDefinition definition = identity.definition;
 183                         if(definition instanceof External) {
 184                                 External def = (External)definition;
 185                                 if(def.parent == -1) {
 186                                         resources[identity.resource] = RootLibrary;
 187                                 } else {
 188                                         if("@inverse".equals(def.name)) {
 189                                                 Resource parent = resources[def.parent];
 190                                                 Resource child = graph.getInverse(parent);
 191                                                 resources[identity.resource] = child;
 192                                         } else {
 193                                                 Resource parent = resources[def.parent];
 194                                                 // TODO: escape should be removed when names become well-behaving
 195                                                 if(parent != null) {
 196                                                         Resource child = graph
 197                                                         .syncRequest(new UnescapedChildMapOfResource(parent),
 198                                                                 new TransientCacheAsyncListener<Map<String, Resource>>())
 199                                                                 .get(def.name);
 200                                                         if(child == null) {
 201                                                                 String uri = graph.getPossibleURI(parent);
 202                                                                 if(uri == null) {
 203                                                                         addMissing(NameUtils.getSafeName(graph, parent) + " /" + def.name);
 204                                                                 } else {
 205                                                                         addMissing(graph.getURI(parent) + "/" + def.name);
 206                                                                 }
 207                                                         }
 208                                                         resources[identity.resource] = child;
 209                                                 } else {
 210                                                     addMissing(TransferableGraphUtils.getURI(tg, def.parent) + "/" + def.name);
 211                                                 }
 212                                         }
 213                                 }
 214                         }
 215                         else if(definition instanceof Internal) {
 216                                 // Do not do anything for now
 217                         }
 218                         else if(definition instanceof Root) {
 219                                 Root root = (Root)definition;
 220                                 if(root.name.equals(""))
 221                                         resources[identity.resource] = RootLibrary;
 222                                 else {
 223                                         Resource existing = advisor.analyzeRoot(graph, root);
 224                                         if(existing != null)
 225                                                 resources[identity.resource] = existing;
 226                                 }
 227                         }
 228                         else if(definition instanceof Optional) {
 229                                 External def = (External)definition;
 230                                 Resource parent = resources[def.parent];
 231                                 if(parent != null)
 232                                         resources[identity.resource] =
 233                                                 graph.syncRequest(new UnescapedChildMapOfResource(parent)).get(def.name);
 234                         }
 235                 }
 236
 237                 this.resources = resources;
 238
 239                 if(!missingExternals.isEmpty()) throw new MissingDependencyException(this);
 240
 241         }
 242
 243         Resource createChild(WriteOnlyGraph graph, Resource parent, String name) throws DatabaseException {
 244                 Resource child = graph.newResource();
 245                 //graph.claim(parent, ConsistsOf, PartOf, child);
 246                 Resource nameResource = graph.newResource();
 247                 graph.claim(nameResource, InstanceOf, null, String);
 248                 graph.claimValue(nameResource, name, WriteBindings.STRING);
 249                 graph.claim(child, HasName, NameOf, nameResource);
 250                 return child;
 251         }
 252
 253         @Override
 254         public Resource createChild(WriteOnlyGraph graph, Resource child, Resource parent, String name) throws DatabaseException {
 255                 graph.claim(parent, ConsistsOf, PartOf, child);
 256                 Resource nameResource = graph.newResource();
 257                 graph.claim(nameResource, InstanceOf, null, String);
 258                 graph.claimValue(nameResource, name, WriteBindings.STRING);
 259                 graph.claim(child, HasName, NameOf, nameResource);
 260                 return child;
 261         }
 262
 263         int[] getClustering() {
 264                 Variant v = tg.extensions.get(Extensions.CLUSTERING);
 265                 if(v == null) return null;
 266                 try {
 267                         return (int[])v.getValue(Bindings.INT_ARRAY);
 268                 } catch (AdaptException e) {
 269                         Logger.defaultLogError(e);
 270                         return null;
 271                 }
 272         }
 273
 274         class ResourceAdapter implements LongAdapter {
 275             final SerialisationSupport ss;
 276             final Value value;
 277             Variant original;
 278             ResourceAdapter(SerialisationSupport ss, Value value) {
 279                 this.ss = ss;
 280                 this.value = value;
 281             }
 282         @Override
 283         public long adapt(long in) {
 284             if(original == null) original = value.value.clone();
 285             Resource res = handles[(int)in].resource(ss);
 286             // Maybe throw?
 287             if(res == null) return in;
 288             return res.getResourceId();
 289         }
 290         }
 291
 292         Variant translate(SerialisationSupport ss, Value value, final ResourceHandle[] handles) throws DatabaseException {
 293
 294                 try {
 295                     ResourceAdapter adapter = new ResourceAdapter(ss, value);
 296                         resourceUtil.adaptValue( value.value.getBinding(), value.value.getValue(), adapter);
 297                         Variant result = value.value;
 298                         if(adapter.original != null) value.value = adapter.original;
 299                         return result;
 300                 } catch (AccessorException e) {
 301                         e.printStackTrace();
 302                 }
 303
 304                 return value.value;
 305
 306         }
 307
 308     class ResourceAdapter2 implements LongAdapter {
 309         final Value value;
 310         Variant original;
 311         ResourceAdapter2(Value value) {
 312             this.value = value;
 313         }
 314         @Override
 315         public long adapt(long in) {
 316             if(original == null) original = value.value.clone();
 317             Resource res = resources[(int)in];
 318             // Maybe throw?
 319             if(res == null) return in;
 320             return res.getResourceId();
 321         }
 322     }
 323
 324         Variant translate2(Value value, final Resource[] resources) throws DatabaseException {
 325
 326         try {
 327             ResourceAdapter2 adapter = new ResourceAdapter2(value);
 328             resourceUtil.adaptValue( value.value.getBinding(), value.value.getValue(), adapter);
 329             Variant result = value.value;
 330             if(adapter.original != null) value.value = adapter.original;
 331             return result;
 332         } catch (AccessorException e) {
 333             e.printStackTrace();
 334         }
 335
 336         return value.value;
 337
 338         }
 339
 340         void write(WriteOnlyGraph graph) throws DatabaseException {
 341
 342                 Resource[] resources = this.resources;
 343
 344                 this.handles = new ResourceHandle[resources.length];
 345
 346                 ResourceHandle[] handles = this.handles;
 347
 348                 int[] clustering = getClustering();
 349
 350                 // Internal identities
 351                 for(Identity identity : tg.identities) {
 352                         IdentityDefinition definition = identity.definition;
 353                         if(resources[identity.resource] != null)
 354                                 continue;
 355                         if(definition instanceof External) {
 356                                 // Already done everything
 357                         }
 358                         else if(definition instanceof Internal) {
 359                                 Internal def = (Internal)definition;
 360                                 resources[identity.resource] =
 361                                         createChild(graph, resources[def.parent], def.name);
 362                         }
 363                         else if(definition instanceof Root) {
 364                                 Root root = (Root)definition;
 365                                 resources[identity.resource] = advisor.createRoot(graph, root);
 366                         }
 367                         else if(definition instanceof Optional) {
 368                                 Optional def = (Optional)definition;
 369                                 Resource child = createChild(graph, resources[def.parent], def.name);
 370                                 graph.claim(child, InstanceOf, null, Library); // ???
 371                                 resources[identity.resource] = child;
 372                         }
 373                 }
 374
 375                 ClusterBuilder builder = graph.getService(ClusterBuilder.class);
 376                 SerialisationSupport ss = graph.getService(SerialisationSupport.class);
 377
 378                 if(clustering != null) {
 379
 380                 int i = 0;
 381                         for(int c : clustering) {
 382                                 builder.newCluster();
 383                                 for(int r=0;r<c;r++, i++)
 384                                         if(resources[i] == null)
 385                                                 handles[i] = builder.newResource();
 386                                         else
 387                                                 handles[i] = builder.resource(resources[i]);
 388
 389                         }
 390
 391                         for(;i<resources.length;++i)
 392                                 if(resources[i] == null)
 393                                         handles[i] = builder.newResource();
 394                                 else
 395                                         handles[i] = builder.resource(resources[i]);
 396
 397                 } else {
 398
 399                         // Create blank resources
 400                         for(int i=0;i<resources.length;++i)
 401                                 if(resources[i] == null)
 402                                         handles[i] = builder.newResource();
 403                                 else
 404                                         handles[i] = builder.resource(resources[i]);
 405
 406                 }
 407
 408                 // Write statements
 409                 int[] statements = tg.statements;
 410
 411 //              int internals = tg.resourceCount - tg.identities.length;
 412
 413                 for(int i=0;i<statements.length;i+=4) {
 414
 415                         int sub = statements[i];
 416                         int pred = statements[i+1];
 417                         int inv = statements[i+2];
 418                         int obj = statements[i+3];
 419
 420                         ResourceHandle subject = handles[sub];
 421                         ResourceHandle predicate = handles[pred];
 422                         ResourceHandle object = handles[obj];
 423
 424                         if(resources[sub] == null) {
 425                                 subject.addStatement(graph, predicate, object);
 426                         } else {
 427                                 graph.claim(
 428                                                 handles[sub].resource(ss),
 429                                                 handles[pred].resource(ss),
 430                                                 null, handles[obj].resource(ss));
 431                         }
 432
 433                         if(inv >= 0) {
 434
 435                                 if(resources[obj] == null) {
 436                                         ResourceHandle inverse = handles[inv];
 437                                         object.addStatement(graph, inverse, subject);
 438                                 } else {
 439                                     Resource s = handles[obj].resource(ss);
 440                                     if(!graph.isImmutable(s))
 441                                         graph.claim(
 442                                                         s,
 443                                                         handles[inv].resource(ss),
 444                                                         null, handles[sub].resource(ss));
 445                                 }
 446
 447                         }
 448
 449                         if(LOG) {
 450                                 log("[STATEMENT] " + resources[statements[i]].getResourceId() + ", " + resources[statements[i+1]].getResourceId() + ", " + resources[statements[i+3]].getResourceId());
 451                         }
 452
 453                 }
 454
 455                 // Write values
 456                 for(Value value : tg.values) {
 457                         Variant variant = translate(ss, value, handles);
 458                         int file = value.resource & 0x80000000;
 459                         int resource = value.resource & 0x7FFFFFFF;
 460                         if (file != 0) {
 461                             throw new UnsupportedOperationException();
 462                                 //graph.claimValue(handles[resource].resource(), value.value, Bindings.BYTE_ARRAY);
 463                         } else {
 464                                 graph.claimValue(handles[resource].resource(ss), variant.getValue(), variant.getBinding());
 465                         }
 466                 }
 467         }
 468
 469         private int updatePercentage(int percentage, int done, int total) {
 470             if(monitor != null && (done & 63) == 0) {
 471                 int current = 100*done / total;
 472                 if(current > percentage) {
 473                     percentage = current;
 474                     monitor.status(percentage);
 475                 }
 476             }
 477         return percentage;
 478         }
 479
 480         void write2(WriteOnlyGraph graph) throws DatabaseException {
 481                 Resource[] resources = this.resources;
 482
 483                 // Internal identities
 484                 for(Identity identity : tg.identities) {
 485                         IdentityDefinition definition = identity.definition;
 486                         if(resources[identity.resource] != null)
 487                                 continue;
 488                         if(definition instanceof External) {
 489                                 // Already done everything
 490                         }
 491                         else if(definition instanceof Internal) {
 492                                 Internal def = (Internal)definition;
 493                                 resources[identity.resource] =
 494                                         createChild(graph, resources[def.parent], def.name);
 495                         }
 496                         else if(definition instanceof Root) {
 497                                 Root root = (Root)definition;
 498                                 resources[identity.resource] = advisor.createRoot(graph, root);
 499                         }
 500                         else if(definition instanceof Optional) {
 501                                 Optional def = (Optional)definition;
 502                                 Resource child = createChild(graph, resources[def.parent], def.name);
 503                                 graph.claim(child, InstanceOf, null, Library); // ???
 504                                 resources[identity.resource] = child;
 505                         }
 506                 }
 507
 508                 // Create blank resources
 509                 for(int i=0;i<resources.length;++i)
 510                         if(resources[i] == null)
 511                                 resources[i] = graph.newResource();
 512
 513                 int total = tg.statements.length + tg.values.length;
 514                 int done = 0;
 515                 int percentage = 0;
 516
 517                 // Write statements
 518                 int[] statements = tg.statements;
 519
 520                 for(int i=0;i<statements.length;i+=4) {
 521                         int inv = statements[i+2];
 522                         graph.claim(
 523                                         resources[statements[i]],
 524                                         resources[statements[i+1]],
 525                                         inv < 0 ? null : resources[inv],
 526                                         resources[statements[i+3]]
 527                                         );
 528                         if(LOG) {
 529                                 log("[STATEMENT] " + resources[statements[i]].getResourceId() + ", " + resources[statements[i+1]].getResourceId() + ", " + resources[statements[i+3]].getResourceId());
 530                         }
 531                         percentage = updatePercentage(percentage, done++, total);
 532                 }
 533
 534                 // Write values
 535                 for(Value value : tg.values) {
 536                         Variant variant = translate2(value, resources);
 537                         int file = value.resource & 0x80000000;
 538                         int resource = value.resource & 0x7FFFFFFF;
 539                         if (file != 0) {
 540                             throw new UnsupportedOperationException();
 541                                 //graph.claimValue(resources[resource], value.value, Bindings.BYTE_ARRAY);
 542                         } else {
 543                             graph.claimValue(resources[resource], variant.getValue(), variant.getBinding());
 544                         }
 545             percentage = updatePercentage(percentage, done++, total);
 546                 }
 547
 548         }
 549
 550         public long[] getResourceIds(SerialisationSupport serializer) throws DatabaseException {
 551                 final int count = resources.length;
 552                 long[] resourceIds = new long[count];
 553                 if(handles != null) {
 554                         for(int i=0;i<count;++i)
 555                                 resourceIds[i] = serializer.getRandomAccessId(handles[i].resource(serializer));
 556                 } else {
 557                         for(int i=0;i<count;++i)
 558                                 resourceIds[i] = serializer.getRandomAccessId(resources[i]);
 559                 }
 560                 return resourceIds;
 561         }
 562 }