bundles/org.simantics.layer0.utils/src/org/simantics/layer0/utils/writer/DelayedGraphWriter.java

   1 /*******************************************************************************
   2  * Copyright (c) 2007, 2010 Association for Decentralized Information Management
   3  * in Industry THTH ry.
   4  * All rights reserved. This program and the accompanying materials
   5  * are made available under the terms of the Eclipse Public License v1.0
   6  * which accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *     VTT Technical Research Centre of Finland - initial API and implementation
  11  *******************************************************************************/
  12 package org.simantics.layer0.utils.writer;
  13
  14
  15 import gnu.trove.map.hash.TIntIntHashMap;
  16 import gnu.trove.map.hash.TIntLongHashMap;
  17
  18 import java.io.BufferedInputStream;
  19 import java.io.BufferedOutputStream;
  20 import java.io.File;
  21 import java.io.FileInputStream;
  22 import java.io.FileOutputStream;
  23 import java.io.IOException;
  24 import java.io.ObjectInputStream;
  25 import java.io.ObjectOutputStream;
  26 import java.nio.channels.FileChannel;
  27
  28 import org.eclipse.core.runtime.IProgressMonitor;
  29 import org.simantics.db.ReadGraph;
  30 import org.simantics.db.Resource;
  31 import org.simantics.db.WriteOnlyGraph;
  32 import org.simantics.db.exception.DatabaseException;
  33
  34 public class DelayedGraphWriter extends AbstractDelayedGraphWriter {
  35
  36         protected File file;
  37         protected ObjectOutputStream s;
  38         TIntIntHashMap clusterHints = new TIntIntHashMap();
  39         TIntLongHashMap clusterIds = new TIntLongHashMap();
  40
  41         public DelayedGraphWriter(ReadGraph graph) {
  42                 super(graph);
  43                 try {
  44                         file = File.createTempFile("graph", ".tmp");
  45                         System.out.println("Temp file: " + file.getAbsolutePath());
  46                         s = new ObjectOutputStream(new BufferedOutputStream(new FileOutputStream(file), 1024*1024));
  47 //                      s = new ObjectOutputStream(backup);
  48
  49                         file.deleteOnExit();
  50                 } catch (IOException e) {
  51                         throw new RuntimeException("Opening output stream to temporary file failed", e);
  52                 }
  53         }
  54
  55         private void writeRef(int ref) throws IOException {
  56                 if(ref > 0)
  57                         timestamps.set(ref-1, time++);
  58                 s.writeInt(ref);
  59         }
  60
  61         @Override
  62         public GraphWriter let(Resource p, Resource o) throws DatabaseException {
  63                 assert(p != null);
  64                 assert(o != null);
  65                 try {
  66                         s.writeByte(0);
  67                         writeRef(current);
  68                         writeRef(getPredicateId(p));
  69                         writeRef(getId(o));
  70                 } catch (IOException e) {
  71                         throw new RuntimeException("Writing statement failed.", e);
  72                 }
  73                 return this;
  74         }
  75
  76         @Override
  77         public GraphWriter let(Resource p, Object value, Resource dataType) throws DatabaseException {
  78                 assert(p != null);
  79                 assert(value != null);
  80                 assert(dataType != null);
  81                 try {
  82                         ++internalCount;
  83                         timestamps.add(0);
  84
  85                         s.writeByte(1);
  86                         writeRef(internalCount);
  87                         s.writeUnshared(value);
  88                         writeRef(getId(dataType));
  89
  90                         s.writeByte(0);
  91                         writeRef(current);
  92                         writeRef(getPredicateId(p));
  93                         writeRef(internalCount);
  94                 } catch (IOException e) {
  95                         throw new RuntimeException("Writing statement failed.", e);
  96                 }
  97                 return this;
  98         }
  99
 100         @Override
 101         public GraphWriter let(int clusterHint, Resource p, Object value,
 102                 Resource dataType) throws DatabaseException {
 103                 let(p, value, dataType);
 104                 clusterHints.put(internalCount, clusterHint);
 105                 return this;
 106         }
 107
 108         @Override
 109     public GraphWriter createLiteral(Object value, Resource dataType) {
 110                 assert(value != null);
 111                 assert(dataType != null);
 112         try {
 113             ++internalCount;
 114             timestamps.add(0);
 115
 116             s.writeByte(1);
 117             writeRef(internalCount);
 118             s.writeUnshared(value);
 119             writeRef(getId(dataType));
 120
 121             current = internalCount;
 122         } catch (IOException e) {
 123             throw new RuntimeException("Writing statement failed.", e);
 124         }
 125         return this;
 126     }
 127
 128         @Override
 129         public GraphWriter createLiteral(int clusterHint, Object value,
 130                 Resource dataType) {
 131                 createLiteral(value, dataType);
 132
 133                 clusterHints.put(current, clusterHint);
 134                 return this;
 135         }
 136
 137     @Override
 138     public GraphWriter createInverse(int cluster, Resource r) {
 139         assert(r != null);
 140         create(cluster);
 141         try {
 142             s.writeByte(5);
 143             writeRef(current);
 144             writeRef(getId(r));
 145         } catch (IOException e) {
 146             throw new RuntimeException("Writing statement failed.", e);
 147         }
 148         return this;
 149     }
 150
 151     @Override
 152     public GraphWriter createInverse(Resource r) {
 153         assert(r != null);
 154         create();
 155         try {
 156             s.writeByte(5);
 157             writeRef(current);
 158             writeRef(getId(r));
 159         } catch (IOException e) {
 160             throw new RuntimeException("Writing statement failed.", e);
 161         }
 162         return this;
 163     }
 164
 165     protected Resource getResource(WriteOnlyGraph wg, Resource[] internals, long[] resourceIds, int id) {
 166                 if(id > 0) {
 167                         Resource ret = internals[id-1];
 168                         if(ret == null) {
 169                             throw new Error("Error");
 170 //                              ret = wg.newResource(types.get(id));
 171 //                              internals[id-1] = ret;
 172 //                              resourceIds[id-1] = ret.getResourceId();
 173                         }
 174                         if(timestamps.getQuick(id-1)==time)
 175                                 internals[id-1] = null;
 176                         ++time;
 177                         return ret;
 178                 }
 179                 else if(id < 0) {
 180                         return externals.get(-1-id);
 181                 }
 182                 else
 183                         return null;
 184         }
 185
 186         @Override
 187         public void commit(IProgressMonitor monitor, WriteOnlyGraph wg) throws DatabaseException {
 188
 189                 try {
 190                     if (s != null) {
 191                         s.close();
 192                         s = null;
 193                     }
 194                         externalsInv = null;
 195
 196                         monitor.beginTask("", 100);
 197
 198                         int lastPercentageDone = 0;
 199                         long fileLength = file.length();
 200
 201 //                      System.out.println("size of commit file: " + fileLength);
 202
 203                         Resource[] internals = new Resource[internalCount];
 204                         resourceIds = new long[internalCount];
 205
 206
 207                         FileInputStream fis = new FileInputStream(file);
 208                         FileChannel fc = fis.getChannel();
 209                         ObjectInputStream is = new ObjectInputStream(new BufferedInputStream(fis, 1024*1024));
 210             int resourceCounter = 0;
 211                         int statementCounter = 0;
 212             int valueCounter = 0;
 213                         time = 0;
 214                         loop: while(true) {
 215                                 switch(is.read()) {
 216                                 case -1:
 217                                         break loop;
 218                                 case 0: {
 219                                         int s = is.readInt();
 220                                         int p = is.readInt();
 221                                         int o = is.readInt();
 222                                         Resource rs = getResource(wg, internals, resourceIds, s);
 223                                         Resource rp = getResource(wg, internals, resourceIds, p);
 224                                         Resource ro = getResource(wg, internals, resourceIds, o);
 225                                         Resource rpInv = inverses.get(p);
 226                                         wg.claim(rs, rp, rpInv, ro);
 227                                         statementCounter += 2;
 228                                 } break;
 229                                 case 1: {
 230                                     int id = is.readInt();
 231                                         Object value = is.readUnshared();
 232                                         int type = is.readInt();
 233
 234                                         Resource r = newResource(wg, internals, id);
 235                                         wg.claim(r, l0.InstanceOf, null, getResource(wg, internals, resourceIds, type));
 236                                         wg.claimValue(r, value);
 237                     statementCounter ++;
 238                     resourceCounter ++;
 239                     valueCounter ++;
 240
 241                                 } break;
 242                                 case 2: {
 243                                     wg.flushCluster();
 244                 } break;
 245                                 case 3: {
 246
 247                     int s = is.readInt();
 248                     int t = is.readInt();
 249
 250                     Resource type = getResource(wg, internals, resourceIds, t);
 251                     wg.claim(newResource(wg, internals, s), l0.InstanceOf, null, type);
 252                     statementCounter ++;
 253                     resourceCounter ++;
 254
 255                                 } break;
 256                                 case 4: {
 257
 258                     int s = is.readInt();
 259                     newResource(wg, internals, s);
 260                     resourceCounter ++;
 261
 262                                 } break;
 263                                 case 5: { // InverseOf
 264
 265                     int r1 = is.readInt();
 266                     int r2 = is.readInt();
 267
 268                     Resource rr1 = getResource(wg, internals, resourceIds, r1);
 269                     Resource rr2 = getResource(wg, internals, resourceIds, r2);
 270                     wg.claim(rr1, l0.InverseOf, l0.InverseOf, rr2);
 271                     statementCounter += 2;
 272
 273                     inverses.put(r1, rr2);
 274                     inverses.put(r2, rr1);
 275
 276                 } break;
 277                                 }
 278
 279 //                if((counter % 200000) == 0) {
 280 //                    System.out.println("Written " + counter + " statements.");
 281 //                }
 282
 283                 double percentageDone = 100.0 * (double) fc.position() / (double) fileLength;
 284                 int newPercentageDone = (int) Math.round(percentageDone);
 285                 if(newPercentageDone > lastPercentageDone) {
 286                     monitor.setTaskName("Writing database (" + newPercentageDone + "%)");
 287                     monitor.worked(newPercentageDone - lastPercentageDone);
 288                     lastPercentageDone = newPercentageDone;
 289                 }
 290                         }
 291
 292                         System.out.println("clusterIds.size() = " + clusterIds.size());
 293
 294                         System.out.println("Wrote " + resourceCounter + " resources, " + statementCounter + " statements and " + valueCounter + " values.");
 295
 296                 } catch (IOException e) {
 297                         throw new RuntimeException("Commiting delayed graph writings failed.", e);
 298                 } catch (ClassNotFoundException e) {
 299                         throw new RuntimeException("Commiting delayed graph writings failed.", e);
 300                 } finally {
 301                         file.delete();
 302                         monitor.done();
 303                 }
 304         }
 305
 306         private Resource newResource(WriteOnlyGraph wg, Resource[] internals, int s) throws DatabaseException {
 307                 int clusterHint = clusterHints.get(s);
 308
 309                 Resource r;
 310                 if(clusterHint == 0)
 311                         r = wg.newResource();
 312                 else {
 313                         long clusterId = clusterIds.get(clusterHint);
 314                         if(clusterId == 0) {
 315                                 wg.flushCluster();
 316                                 r = wg.newResource();
 317                                 clusterIds.put(clusterHint, clustering.getCluster(r));
 318                         }
 319                         else
 320                                 r = wg.newResource(clusterId);
 321                 }
 322                 internals[s-1] = r;
 323                 resourceIds[s-1] = r.getResourceId();
 324                 return r;
 325         }
 326
 327 //    public ValueDescriptor createDescriptor(Object obj) {
 328 //
 329 //        if(obj instanceof String[])
 330 //            return new ValueDescriptor(ValueTrait.StaticValue, (String[])obj);
 331 //        else if(obj instanceof int[])
 332 //            return new ValueDescriptor(ValueTrait.StaticValue, (int[])obj);
 333 //        else if(obj instanceof double[])
 334 //            return new ValueDescriptor(ValueTrait.StaticValue, (double[])obj);
 335 //        else if(obj instanceof float[])
 336 //            return new ValueDescriptor(ValueTrait.StaticValue, (float[])obj);
 337 //        else if(obj instanceof boolean[])
 338 //            return new ValueDescriptor(ValueTrait.StaticValue, (boolean[])obj);
 339 //        else if(obj instanceof long[])
 340 //            return new ValueDescriptor(ValueTrait.StaticValue, (long[])obj);
 341 //        else if(obj instanceof byte[])
 342 //            return new ValueDescriptor(ValueTrait.StaticValue, (byte[])obj);
 343 //
 344 //        else if(obj instanceof String)
 345 //            return new ValueDescriptor(ValueTrait.StaticValue, new String[] {(String)obj});
 346 //        else if(obj instanceof Double)
 347 //            return new ValueDescriptor(ValueTrait.StaticValue, new double[] {(Double)obj});
 348 //        else if(obj instanceof Float)
 349 //            return new ValueDescriptor(ValueTrait.StaticValue, new float[] {(Float)obj});
 350 //        else if(obj instanceof Integer)
 351 //            return new ValueDescriptor(ValueTrait.StaticValue, new int[] {(Integer)obj});
 352 //        else if(obj instanceof Boolean)
 353 //            return new ValueDescriptor(ValueTrait.StaticValue, new boolean[] {(Boolean)obj});
 354 //        else if(obj instanceof Byte)
 355 //            return new ValueDescriptor(ValueTrait.StaticValue, new byte[] {(Byte)obj});
 356 //        else if(obj instanceof Long)
 357 //            return new ValueDescriptor(ValueTrait.StaticValue, new long[] {(Long)obj});
 358 //
 359 //        throw new Error("Wrong type!");
 360 //
 361 //    }
 362
 363     @Override
 364     public GraphWriter create() {
 365
 366         current = ++internalCount;
 367         timestamps.add(0);
 368
 369         try {
 370             s.writeByte(4);
 371             s.writeInt(current);
 372         } catch (IOException e) {
 373             throw new RuntimeException("Writing statement failed.", e);
 374         }
 375
 376         return this;
 377
 378     }
 379
 380     @Override
 381     public GraphWriter create(int clusterHint) {
 382
 383         create();
 384         clusterHints.put(current, clusterHint);
 385         return this;
 386
 387     }
 388
 389         @Override
 390         public GraphWriter create(Resource type) {
 391                 assert(type != null);
 392
 393             current = ++internalCount;
 394         timestamps.add(0);
 395
 396         try {
 397             s.writeByte(3);
 398             s.writeInt(current);
 399             s.writeInt(getId(type));
 400         } catch (IOException e) {
 401             throw new RuntimeException("Writing statement failed.", e);
 402         }
 403
 404                 return this;
 405
 406         }
 407
 408         @Override
 409     public GraphWriter create(int clusterHint, Resource type) {
 410
 411                 create(type);
 412         clusterHints.put(current, clusterHint);
 413         return this;
 414
 415     }
 416
 417         @Override
 418         public Resource get() {
 419                 if(current > 0)
 420                         return new InternalResource(current);
 421                 else if(current < 0)
 422                         return externals.get(-1-current);
 423                 else
 424                         return null;
 425         }
 426
 427         @Override
 428         public GraphWriter handle(Resource s) {
 429                 assert(s != null);
 430
 431                 current = getId(s);
 432                 return this;
 433         }
 434
 435         public GraphWriter flush() {
 436         try {
 437             s.writeByte(2);
 438         } catch(IOException e) {
 439             throw new RuntimeException("Writing flush failed.", e);
 440         }
 441         return this;
 442         }
 443
 444 }