bundles/org.simantics.layer0.utils/src/org/simantics/layer0/utils/writer/DelayedGraphWriter.java

   1 /*******************************************************************************\r
   2  * Copyright (c) 2007, 2010 Association for Decentralized Information Management\r
   3  * in Industry THTH ry.\r
   4  * All rights reserved. This program and the accompanying materials\r
   5  * are made available under the terms of the Eclipse Public License v1.0\r
   6  * which accompanies this distribution, and is available at\r
   7  * http://www.eclipse.org/legal/epl-v10.html\r
   8  *\r
   9  * Contributors:\r
  10  *     VTT Technical Research Centre of Finland - initial API and implementation\r
  11  *******************************************************************************/\r
  12 package org.simantics.layer0.utils.writer;\r
  13 \r
  14 \r
  15 import gnu.trove.map.hash.TIntIntHashMap;\r
  16 import gnu.trove.map.hash.TIntLongHashMap;\r
  17 \r
  18 import java.io.BufferedInputStream;\r
  19 import java.io.BufferedOutputStream;\r
  20 import java.io.File;\r
  21 import java.io.FileInputStream;\r
  22 import java.io.FileOutputStream;\r
  23 import java.io.IOException;\r
  24 import java.io.ObjectInputStream;\r
  25 import java.io.ObjectOutputStream;\r
  26 import java.nio.channels.FileChannel;\r
  27 \r
  28 import org.eclipse.core.runtime.IProgressMonitor;\r
  29 import org.simantics.db.ReadGraph;\r
  30 import org.simantics.db.Resource;\r
  31 import org.simantics.db.WriteOnlyGraph;\r
  32 import org.simantics.db.exception.DatabaseException;\r
  33 \r
  34 public class DelayedGraphWriter extends AbstractDelayedGraphWriter {\r
  35 \r
  36         protected File file;\r
  37         protected ObjectOutputStream s;\r
  38         TIntIntHashMap clusterHints = new TIntIntHashMap();\r
  39         TIntLongHashMap clusterIds = new TIntLongHashMap();\r
  40         \r
  41         public DelayedGraphWriter(ReadGraph graph) {\r
  42                 super(graph);\r
  43                 try {\r
  44                         file = File.createTempFile("graph", ".tmp");\r
  45                         System.out.println("Temp file: " + file.getAbsolutePath());\r
  46                         s = new ObjectOutputStream(new BufferedOutputStream(new FileOutputStream(file), 1024*1024));\r
  47 //                      s = new ObjectOutputStream(backup);\r
  48                         \r
  49                         file.deleteOnExit();\r
  50                 } catch (IOException e) {\r
  51                         throw new RuntimeException("Opening output stream to temporary file failed", e);\r
  52                 }               \r
  53         }\r
  54         \r
  55         private void writeRef(int ref) throws IOException {\r
  56                 if(ref > 0) \r
  57                         timestamps.set(ref-1, time++);\r
  58                 s.writeInt(ref);\r
  59         }\r
  60         \r
  61         @Override\r
  62         public GraphWriter let(Resource p, Resource o) throws DatabaseException {\r
  63                 assert(p != null);\r
  64                 assert(o != null);\r
  65                 try {\r
  66                         s.writeByte(0);\r
  67                         writeRef(current);\r
  68                         writeRef(getPredicateId(p));\r
  69                         writeRef(getId(o));\r
  70                 } catch (IOException e) {\r
  71                         throw new RuntimeException("Writing statement failed.", e);\r
  72                 }\r
  73                 return this;\r
  74         }\r
  75 \r
  76         @Override\r
  77         public GraphWriter let(Resource p, Object value, Resource dataType) throws DatabaseException {\r
  78                 assert(p != null);\r
  79                 assert(value != null);\r
  80                 assert(dataType != null);\r
  81                 try {\r
  82                         ++internalCount;                \r
  83                         timestamps.add(0);\r
  84                         \r
  85                         s.writeByte(1);                 \r
  86                         writeRef(internalCount);                        \r
  87                         s.writeUnshared(value);\r
  88                         writeRef(getId(dataType));                                      \r
  89                         \r
  90                         s.writeByte(0);\r
  91                         writeRef(current);\r
  92                         writeRef(getPredicateId(p));\r
  93                         writeRef(internalCount);\r
  94                 } catch (IOException e) {\r
  95                         throw new RuntimeException("Writing statement failed.", e);\r
  96                 }\r
  97                 return this;\r
  98         }\r
  99         \r
 100         @Override\r
 101         public GraphWriter let(int clusterHint, Resource p, Object value,\r
 102                 Resource dataType) throws DatabaseException {\r
 103                 let(p, value, dataType);\r
 104                 clusterHints.put(internalCount, clusterHint);\r
 105                 return this;\r
 106         }\r
 107         \r
 108         @Override\r
 109     public GraphWriter createLiteral(Object value, Resource dataType) {\r
 110                 assert(value != null);\r
 111                 assert(dataType != null);\r
 112         try {\r
 113             ++internalCount;        \r
 114             timestamps.add(0);\r
 115             \r
 116             s.writeByte(1);         \r
 117             writeRef(internalCount);            \r
 118             s.writeUnshared(value);\r
 119             writeRef(getId(dataType));                  \r
 120     \r
 121             current = internalCount;\r
 122         } catch (IOException e) {\r
 123             throw new RuntimeException("Writing statement failed.", e);\r
 124         }\r
 125         return this;\r
 126     }\r
 127         \r
 128         @Override\r
 129         public GraphWriter createLiteral(int clusterHint, Object value,\r
 130                 Resource dataType) {\r
 131                 createLiteral(value, dataType);\r
 132 \r
 133                 clusterHints.put(current, clusterHint);\r
 134                 return this;\r
 135         }\r
 136             \r
 137     @Override\r
 138     public GraphWriter createInverse(int cluster, Resource r) {\r
 139         assert(r != null);\r
 140         create(cluster);\r
 141         try {\r
 142             s.writeByte(5);\r
 143             writeRef(current);\r
 144             writeRef(getId(r));\r
 145         } catch (IOException e) {\r
 146             throw new RuntimeException("Writing statement failed.", e);\r
 147         }\r
 148         return this;\r
 149     }    \r
 150         \r
 151     @Override\r
 152     public GraphWriter createInverse(Resource r) {\r
 153         assert(r != null);\r
 154         create();\r
 155         try {\r
 156             s.writeByte(5);\r
 157             writeRef(current);\r
 158             writeRef(getId(r));\r
 159         } catch (IOException e) {\r
 160             throw new RuntimeException("Writing statement failed.", e);\r
 161         }\r
 162         return this;\r
 163     }    \r
 164 \r
 165     protected Resource getResource(WriteOnlyGraph wg, Resource[] internals, long[] resourceIds, int id) {\r
 166                 if(id > 0) {\r
 167                         Resource ret = internals[id-1];\r
 168                         if(ret == null) {\r
 169                             throw new Error("Error");\r
 170 //                              ret = wg.newResource(types.get(id));\r
 171 //                              internals[id-1] = ret;\r
 172 //                              resourceIds[id-1] = ret.getResourceId();\r
 173                         }\r
 174                         if(timestamps.getQuick(id-1)==time)\r
 175                                 internals[id-1] = null;\r
 176                         ++time;\r
 177                         return ret;\r
 178                 }\r
 179                 else if(id < 0) {\r
 180                         return externals.get(-1-id);\r
 181                 }\r
 182                 else\r
 183                         return null;\r
 184         }\r
 185         \r
 186         @Override\r
 187         public void commit(IProgressMonitor monitor, WriteOnlyGraph wg) throws DatabaseException {\r
 188                 \r
 189                 try {\r
 190                     if (s != null) {\r
 191                         s.close();\r
 192                         s = null;\r
 193                     }\r
 194                         externalsInv = null;\r
 195 \r
 196                         monitor.beginTask("", 100);\r
 197 \r
 198                         int lastPercentageDone = 0;\r
 199                         long fileLength = file.length();\r
 200                         \r
 201 //                      System.out.println("size of commit file: " + fileLength);\r
 202 \r
 203                         Resource[] internals = new Resource[internalCount];\r
 204                         resourceIds = new long[internalCount];\r
 205                         \r
 206                         \r
 207                         FileInputStream fis = new FileInputStream(file);\r
 208                         FileChannel fc = fis.getChannel();\r
 209                         ObjectInputStream is = new ObjectInputStream(new BufferedInputStream(fis, 1024*1024));\r
 210             int resourceCounter = 0;\r
 211                         int statementCounter = 0;\r
 212             int valueCounter = 0;\r
 213                         time = 0;\r
 214                         loop: while(true) {\r
 215                                 switch(is.read()) {\r
 216                                 case -1:\r
 217                                         break loop;\r
 218                                 case 0: {\r
 219                                         int s = is.readInt();\r
 220                                         int p = is.readInt();\r
 221                                         int o = is.readInt();\r
 222                                         Resource rs = getResource(wg, internals, resourceIds, s);\r
 223                                         Resource rp = getResource(wg, internals, resourceIds, p);\r
 224                                         Resource ro = getResource(wg, internals, resourceIds, o);\r
 225                                         Resource rpInv = inverses.get(p);\r
 226                                         wg.claim(rs, rp, rpInv, ro);\r
 227                                         statementCounter += 2;\r
 228                                 } break;                                        \r
 229                                 case 1: {\r
 230                                     int id = is.readInt();\r
 231                                         Object value = is.readUnshared();\r
 232                                         int type = is.readInt();\r
 233                                         \r
 234                                         Resource r = newResource(wg, internals, id);\r
 235                                         wg.claim(r, l0.InstanceOf, null, getResource(wg, internals, resourceIds, type));\r
 236                                         wg.claimValue(r, value);\r
 237                     statementCounter ++;\r
 238                     resourceCounter ++;\r
 239                     valueCounter ++;\r
 240                                         \r
 241                                 } break;\r
 242                                 case 2: {\r
 243                                     wg.flushCluster();\r
 244                 } break; \r
 245                                 case 3: {\r
 246 \r
 247                     int s = is.readInt();\r
 248                     int t = is.readInt();\r
 249                                     \r
 250                     Resource type = getResource(wg, internals, resourceIds, t);\r
 251                     wg.claim(newResource(wg, internals, s), l0.InstanceOf, null, type);\r
 252                     statementCounter ++;\r
 253                     resourceCounter ++;\r
 254                                     \r
 255                                 } break;\r
 256                                 case 4: {\r
 257 \r
 258                     int s = is.readInt();\r
 259                     newResource(wg, internals, s);\r
 260                     resourceCounter ++;\r
 261                     \r
 262                                 } break;\r
 263                                 case 5: { // InverseOf\r
 264 \r
 265                     int r1 = is.readInt();\r
 266                     int r2 = is.readInt();\r
 267                     \r
 268                     Resource rr1 = getResource(wg, internals, resourceIds, r1);\r
 269                     Resource rr2 = getResource(wg, internals, resourceIds, r2);\r
 270                     wg.claim(rr1, l0.InverseOf, l0.InverseOf, rr2);\r
 271                     statementCounter += 2;\r
 272                     \r
 273                     inverses.put(r1, rr2);\r
 274                     inverses.put(r2, rr1);\r
 275                     \r
 276                 } break;\r
 277                                 }\r
 278 \r
 279 //                if((counter % 200000) == 0) {\r
 280 //                    System.out.println("Written " + counter + " statements.");\r
 281 //                }\r
 282 \r
 283                 double percentageDone = 100.0 * (double) fc.position() / (double) fileLength;\r
 284                 int newPercentageDone = (int) Math.round(percentageDone);\r
 285                 if(newPercentageDone > lastPercentageDone) {\r
 286                     monitor.setTaskName("Writing database (" + newPercentageDone + "%)");\r
 287                     monitor.worked(newPercentageDone - lastPercentageDone);\r
 288                     lastPercentageDone = newPercentageDone;\r
 289                 }\r
 290                         }\r
 291                         \r
 292                         System.out.println("clusterIds.size() = " + clusterIds.size());\r
 293                         \r
 294                         System.out.println("Wrote " + resourceCounter + " resources, " + statementCounter + " statements and " + valueCounter + " values.");\r
 295                         \r
 296                 } catch (IOException e) {\r
 297                         throw new RuntimeException("Commiting delayed graph writings failed.", e);\r
 298                 } catch (ClassNotFoundException e) {\r
 299                         throw new RuntimeException("Commiting delayed graph writings failed.", e);\r
 300                 } finally {             \r
 301                         file.delete();\r
 302                         monitor.done();\r
 303                 }\r
 304         }\r
 305         \r
 306         private Resource newResource(WriteOnlyGraph wg, Resource[] internals, int s) throws DatabaseException {\r
 307                 int clusterHint = clusterHints.get(s);\r
 308                 \r
 309                 Resource r;\r
 310                 if(clusterHint == 0)\r
 311                         r = wg.newResource();\r
 312                 else {\r
 313                         long clusterId = clusterIds.get(clusterHint);\r
 314                         if(clusterId == 0) {\r
 315                                 wg.flushCluster();\r
 316                                 r = wg.newResource();\r
 317                                 clusterIds.put(clusterHint, clustering.getCluster(r));\r
 318                         }\r
 319                         else\r
 320                                 r = wg.newResource(clusterId);\r
 321                 }\r
 322                 internals[s-1] = r;\r
 323                 resourceIds[s-1] = r.getResourceId();\r
 324                 return r;\r
 325         }\r
 326         \r
 327 //    public ValueDescriptor createDescriptor(Object obj) {\r
 328 //        \r
 329 //        if(obj instanceof String[])\r
 330 //            return new ValueDescriptor(ValueTrait.StaticValue, (String[])obj);\r
 331 //        else if(obj instanceof int[])\r
 332 //            return new ValueDescriptor(ValueTrait.StaticValue, (int[])obj);\r
 333 //        else if(obj instanceof double[])\r
 334 //            return new ValueDescriptor(ValueTrait.StaticValue, (double[])obj);\r
 335 //        else if(obj instanceof float[])\r
 336 //            return new ValueDescriptor(ValueTrait.StaticValue, (float[])obj);\r
 337 //        else if(obj instanceof boolean[])\r
 338 //            return new ValueDescriptor(ValueTrait.StaticValue, (boolean[])obj);\r
 339 //        else if(obj instanceof long[])\r
 340 //            return new ValueDescriptor(ValueTrait.StaticValue, (long[])obj);\r
 341 //        else if(obj instanceof byte[])\r
 342 //            return new ValueDescriptor(ValueTrait.StaticValue, (byte[])obj);\r
 343 //        \r
 344 //        else if(obj instanceof String)\r
 345 //            return new ValueDescriptor(ValueTrait.StaticValue, new String[] {(String)obj});\r
 346 //        else if(obj instanceof Double)\r
 347 //            return new ValueDescriptor(ValueTrait.StaticValue, new double[] {(Double)obj});\r
 348 //        else if(obj instanceof Float)\r
 349 //            return new ValueDescriptor(ValueTrait.StaticValue, new float[] {(Float)obj});\r
 350 //        else if(obj instanceof Integer)\r
 351 //            return new ValueDescriptor(ValueTrait.StaticValue, new int[] {(Integer)obj});\r
 352 //        else if(obj instanceof Boolean)\r
 353 //            return new ValueDescriptor(ValueTrait.StaticValue, new boolean[] {(Boolean)obj});\r
 354 //        else if(obj instanceof Byte)\r
 355 //            return new ValueDescriptor(ValueTrait.StaticValue, new byte[] {(Byte)obj});\r
 356 //        else if(obj instanceof Long)\r
 357 //            return new ValueDescriptor(ValueTrait.StaticValue, new long[] {(Long)obj});\r
 358 //        \r
 359 //        throw new Error("Wrong type!");\r
 360 //        \r
 361 //    }\r
 362 \r
 363     @Override\r
 364     public GraphWriter create() {\r
 365         \r
 366         current = ++internalCount;\r
 367         timestamps.add(0);\r
 368 \r
 369         try {\r
 370             s.writeByte(4);\r
 371             s.writeInt(current);\r
 372         } catch (IOException e) {\r
 373             throw new RuntimeException("Writing statement failed.", e);\r
 374         }\r
 375 \r
 376         return this;\r
 377         \r
 378     }\r
 379 \r
 380     @Override\r
 381     public GraphWriter create(int clusterHint) {\r
 382         \r
 383         create();       \r
 384         clusterHints.put(current, clusterHint);\r
 385         return this;\r
 386         \r
 387     }\r
 388     \r
 389         @Override\r
 390         public GraphWriter create(Resource type) {\r
 391                 assert(type != null);\r
 392                 \r
 393             current = ++internalCount;\r
 394         timestamps.add(0);\r
 395 \r
 396         try {\r
 397             s.writeByte(3);\r
 398             s.writeInt(current);\r
 399             s.writeInt(getId(type));\r
 400         } catch (IOException e) {\r
 401             throw new RuntimeException("Writing statement failed.", e);\r
 402         }\r
 403 \r
 404                 return this;\r
 405                 \r
 406         }\r
 407         \r
 408         @Override\r
 409     public GraphWriter create(int clusterHint, Resource type) {\r
 410                 \r
 411                 create(type);           \r
 412         clusterHints.put(current, clusterHint);\r
 413         return this;\r
 414         \r
 415     }\r
 416 \r
 417         @Override\r
 418         public Resource get() {\r
 419                 if(current > 0)\r
 420                         return new InternalResource(current);\r
 421                 else if(current < 0)\r
 422                         return externals.get(-1-current);\r
 423                 else\r
 424                         return null;\r
 425         }\r
 426         \r
 427         @Override\r
 428         public GraphWriter handle(Resource s) {\r
 429                 assert(s != null);\r
 430                 \r
 431                 current = getId(s);\r
 432                 return this;\r
 433         }\r
 434 \r
 435         public GraphWriter flush() {       \r
 436         try {\r
 437             s.writeByte(2);\r
 438         } catch(IOException e) {\r
 439             throw new RuntimeException("Writing flush failed.", e);\r
 440         }\r
 441         return this;\r
 442         }\r
 443 \r
 444 }\r