bundles/org.simantics.databoard/src/org/simantics/databoard/file/RandomAccessBinaryList.java

   1 /*******************************************************************************
   2  *  Copyright (c) 2010 Association for Decentralized Information Management in
   3  *  Industry THTH ry.
   4  *  All rights reserved. This program and the accompanying materials
   5  *  are made available under the terms of the Eclipse Public License v1.0
   6  *  which accompanies this distribution, and is available at
   7  *  http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  *  Contributors:
  10  *      VTT Technical Research Centre of Finland - initial API and implementation
  11  *******************************************************************************/
  12 package org.simantics.databoard.file;
  13
  14 import gnu.trove.list.array.TLongArrayList;
  15 import gnu.trove.map.hash.TObjectIntHashMap;
  16
  17 import java.io.File;
  18 import java.io.IOException;
  19 import java.util.AbstractList;
  20 import java.util.ArrayList;
  21 import java.util.Collection;
  22 import java.util.List;
  23 import java.util.RandomAccess;
  24
  25 import org.simantics.databoard.binding.Binding;
  26 import org.simantics.databoard.binding.error.RuntimeBindingException;
  27 import org.simantics.databoard.serialization.SerializationException;
  28 import org.simantics.databoard.serialization.Serializer;
  29 import org.simantics.databoard.serialization.SerializerConstructionException;
  30 import org.simantics.databoard.serialization.SerializerScheme;
  31 import org.simantics.databoard.util.binary.RandomAccessBinary;
  32 import org.simantics.databoard.util.binary.RandomAccessBinary.ByteSide;
  33
  34 /**
  35  * BlobList is a {@link RandomAccessBinary} backend implementation of a List
  36  * collection. add() and get() operations serialize and deserialize objects
  37  * from binary format.
  38  *
  39  * Set, remove, insert and add operations flush() modifications to before return.<p>
  40  *
  41  * Each operation may throw {@link RuntimeIOException}, if there is IOException
  42  * in the {@link RandomAccessBinary}<p>
  43  *
  44  * Entry position index is on open if the file has variable width
  45  * data type (eg. String). The entire file is scanned through.<p>
  46  *
  47  * TODO lazy index. Append alone (add()) doesn't require scan.
  48  *
  49  * @see FileList File based implementation
  50  * @author Toni Kalajainen <toni.kalajainen@vtt.fi>
  51  */
  52 public class RandomAccessBinaryList<T> extends AbstractList<T> implements IFileList<T>, RandomAccess {
  53
  54         /** Reader */
  55         RandomAccessBinary blob;
  56
  57         /** Format */
  58         SerializerScheme format;
  59
  60         /** Binding */
  61         Binding binding;
  62
  63         /** Serializer */
  64         Serializer serializer;
  65
  66         /** Offset table */
  67         Index table;
  68
  69         /** identities */
  70         List<Object> identities = new ArrayList<Object>();
  71
  72         /** identities */
  73         TObjectIntHashMap<Object> identities2 = new TObjectIntHashMap<Object>();
  74
  75         /** Status */
  76         boolean closed = false;
  77
  78         /**
  79          * Create new random access list backed by a file
  80          *
  81          * @param blob blob
  82          * @param binding
  83          * @param startPos The position of the first sample in file
  84          * @param format serialization format
  85          * @throws IOException
  86          * @throws SerializerConstructionException could not create serializer, never thrown with BinarySerializationFormat
  87          * @throws SerializationException Error with the file, could not build entry index
  88          */
  89         public RandomAccessBinaryList(RandomAccessBinary blob, Binding binding, long startPos, SerializerScheme format)
  90         throws IOException, SerializerConstructionException, SerializationException
  91         {
  92                 this.format = format;
  93                 this.blob = blob;
  94                 this.binding = binding;
  95                 serializer = format.getSerializer(binding);
  96
  97                 Integer sampleSize = serializer.getConstantSize();
  98
  99                 // Variable width sample
 100                 if (sampleSize==null)
 101                 {
 102                         table = new Table(startPos);
 103                         blob.position(startPos);
 104                         long length = blob.length();
 105                         long pos = startPos;
 106                         while (pos<length)
 107                         {
 108                                 serializer.skip(blob);
 109                                 pos = blob.position();
 110                                 table.add(pos);
 111                         }
 112                 } else
 113                 // Fixed width sample
 114                 {
 115                         long fileSize = blob.length();
 116                         long count = (fileSize - startPos) / sampleSize;
 117                         if (count>Integer.MAX_VALUE) throw new IllegalArgumentException("The blob is too large");
 118                         table = new Constant(startPos, sampleSize, (int) count);
 119                 }
 120         }
 121
 122
 123         @Override
 124         public int size() throws RuntimeIOException
 125         {
 126                 return table.size()-1;
 127         }
 128
 129         public boolean isOpen()
 130         {
 131                 return !closed;
 132         }
 133
 134         /**
 135          * Flushes the caches and closes the file handle.
 136          */
 137         public void close()
 138         {
 139                 synchronized(this) {
 140                         if (closed) return;
 141                         closed = true;
 142                 }
 143
 144                 try {
 145                         blob.flush();
 146                         blob.close();
 147                 } catch (IOException ignored) {
 148                 }
 149         }
 150
 151         @Override
 152         public Binding getBinding() {
 153                 return binding;
 154         }
 155
 156         @SuppressWarnings("unchecked")
 157         @Override
 158         public T get(int index) throws RuntimeIOException {
 159                 if (index<0 || index>=size())
 160                         throw new IndexOutOfBoundsException();
 161                 try {
 162                         blob.position( table.get(index) );
 163                         identities.clear();
 164                         return (T) serializer.deserialize(blob, identities);
 165                 } catch (IOException e) {
 166                         throw new RuntimeIOException(e);
 167                 }
 168         }
 169
 170         public void add(int index, T element) throws RuntimeIOException, RuntimeBindingException {
 171                 if (index<0 || index>size())
 172                         throw new IndexOutOfBoundsException();
 173
 174                 // Append
 175                 try {
 176                         if (index==size()) {
 177                                 blob.position( table.get(index) );
 178                                 identities2.clear();
 179                                 serializer.serialize(blob, identities2, element);
 180                                 table.add(blob.position());
 181                                 modCount++;
 182                                 blob.flush();
 183                         } else
 184                         // Insert
 185                         {
 186                                 // Make some room in-file
 187                                 identities2.clear();
 188                                 long len = serializer.getSize(element, identities2);
 189                                 long pos = table.get(index);
 190                                 blob.flush();
 191                                 blob.position(pos);
 192                                 blob.insertBytes(len, ByteSide.Left);
 193
 194                                 // Write
 195                                 identities2.clear();
 196                                 blob.position(pos);
 197                                 serializer.serialize(blob, identities2, element);
 198
 199                                 assert(pos+len == blob.position());
 200                                 blob.flush();
 201
 202                                 // Update table
 203                                 table.insert(index, pos);
 204                                 table.adjust(index+1, table.size(), len);
 205                                 modCount++;
 206                         }
 207                         blob.flush();
 208                 } catch (IOException e) {
 209                         throw new RuntimeIOException(e);
 210                 }
 211         };
 212
 213         /**
 214          * Replace the whole content with the content of another collection
 215          *
 216          * @param c collection
 217          * @throws RuntimeIOException
 218          */
 219         public void setAll(Collection<? extends T> c) throws RuntimeIOException
 220         {
 221                 try {
 222                         blob.flush();
 223                         if (table.size()>1)
 224                                 table.remove(1, table.size()-1);
 225                         long zerosize = table.get(0);
 226                         blob.position(zerosize);
 227                         blob.setLength(zerosize);
 228                         addAll(c);
 229                         blob.flush();
 230                 } catch (IOException e) {
 231                         throw new RuntimeIOException(e);
 232                 }
 233         }
 234
 235     @SuppressWarnings("unchecked")
 236         public T set(int index, T element) throws RuntimeIOException, RuntimeBindingException {
 237                 if (index<0 || index>=size())
 238                         throw new IndexOutOfBoundsException();
 239
 240                 try {
 241                         long startPos = table.get(index);
 242                         long oldEndPos = table.get(index+1);
 243                         long oldSize = oldEndPos - startPos;
 244
 245                         // Read old
 246                         blob.position( startPos );
 247                         identities.clear();
 248                         T result = (T) serializer.deserialize(blob, identities);
 249                         assert(blob.position() == oldEndPos);
 250
 251                         // Calc size of new
 252                         identities2.clear();
 253                         long newSize = serializer.getSize(element, identities2);
 254
 255                         long diff = newSize - oldSize;
 256                         if (diff>0) {
 257                                 blob.flush();
 258                                 blob.position(startPos);
 259                                 blob.insertBytes(diff, ByteSide.Left);
 260                         } else if (diff<0) {
 261                                 blob.flush();
 262                                 blob.position(startPos);
 263                                 blob.insertBytes(-diff, ByteSide.Left);
 264                         }
 265
 266                         // Write new
 267                         identities2.clear();
 268                         blob.position( startPos );
 269                         serializer.serialize(blob, identities2, element);
 270                         assert(startPos+newSize == blob.position());
 271                         blob.flush();
 272
 273                         // Update table
 274                         table.adjust(index+1, table.size(), diff);
 275
 276                         if (diff!=0)
 277                                 modCount++;
 278                         blob.flush();
 279                         return result;
 280                 } catch (IOException e) {
 281                         throw new RuntimeIOException(e);
 282                 }
 283     }
 284
 285         @Override
 286         public void removeRange(int fromIndex, int toIndex) throws RuntimeIOException {
 287                 if (fromIndex<0 || toIndex<0 || fromIndex>toIndex || toIndex>size())
 288                         throw new IndexOutOfBoundsException();
 289                 try {
 290                         int count = toIndex - fromIndex;
 291                         if (count==0) return;
 292                         long startPos = table.get(fromIndex);
 293                         long endPos = table.get(toIndex);
 294                         long length = endPos - startPos;
 295
 296                         blob.position(startPos);
 297                         blob.removeBytes(length, ByteSide.Left);
 298                         table.remove(fromIndex+1, count);
 299                         table.adjust(fromIndex+1, table.size(), -length);
 300                         modCount++;
 301                         blob.flush();
 302                 } catch (IOException e) {
 303                         throw new RuntimeIOException(e);
 304                 }
 305         }
 306
 307         @SuppressWarnings("unchecked")
 308         @Override
 309         public T remove(int index) throws RuntimeIOException {
 310                 if (index<0 || index>=size())
 311                         throw new IndexOutOfBoundsException();
 312                 try {
 313                         long startPos = table.get(index);
 314                         long endPos = table.get(index+1);
 315                         long length = endPos - startPos;
 316
 317                         blob.position(startPos);
 318                         identities.clear();
 319                         T result = (T) serializer.deserialize(blob, identities);
 320                         blob.position(startPos);
 321                         blob.removeBytes(length, ByteSide.Left);
 322
 323                         table.remove(index+1, 1);
 324                         table.adjust(index+1, table.size(), -length);
 325                         modCount++;
 326                         blob.flush();
 327                         return result;
 328                 } catch (IOException e) {
 329                         throw new RuntimeIOException(e);
 330                 }
 331         }
 332
 333         @Override
 334         public boolean addAll(Collection<? extends T> c) throws RuntimeIOException {
 335                 return addAll(0, c);
 336         }
 337
 338
 339         @Override
 340         public boolean addAll(int index, Collection<? extends T> c) throws RuntimeIOException, RuntimeBindingException {
 341                 if (index<0 || index>size())
 342                         throw new IndexOutOfBoundsException();
 343
 344                 // Append
 345                 try {
 346                         if (index==size()) {
 347                                 blob.position( table.get(index) );
 348                                 identities2.clear();
 349                                 for (T element : c) {
 350                                         serializer.serialize(blob, identities2, element);
 351                                         table.add(blob.position());
 352                                 }
 353                                 blob.flush();
 354                                 modCount++;
 355                         } else
 356                         // Insert
 357                         {
 358                                 // Calc sizes
 359                                 long startPos = table.get(index);
 360                                 long endPos = startPos;
 361                                 int i=0;
 362                                 for (T element : c) {
 363                                         identities2.clear();
 364                                         long len = serializer.getSize(element, identities2);
 365                                         endPos += len;
 366                                         i++;
 367                                         table.insert(index+i, endPos);
 368                                 }
 369
 370                                 // Make room
 371                                 blob.flush();
 372                                 blob.position(startPos);
 373                                 blob.insertBytes(endPos-startPos, ByteSide.Left);
 374
 375                                 // Write
 376                                 blob.position(startPos);
 377                                 for (T element : c) {
 378                                         identities2.clear();
 379                                         serializer.serialize(blob, identities2, element);
 380                                 }
 381                                 blob.flush();
 382                                 modCount++;
 383                         }
 384                         blob.flush();
 385                         return !c.isEmpty();
 386                 } catch (IOException e) {
 387                         throw new RuntimeIOException(e);
 388                 }
 389         }
 390
 391         interface Index {
 392                 long get(int index);
 393                 /**
 394                  * Get the number of position entries. (= sample count + 1)
 395                  * @return
 396                  */
 397                 int size();
 398                 void add(long position);
 399                 void set(int index, long position);
 400                 void insert(int index, long position);
 401                 void remove(int index, int count);
 402
 403                 /**
 404                  * Adjust positions
 405                  *
 406                  * @param fromIndex
 407                  * @param toIndex end index (exclusive)
 408                  * @param diff position adjustment
 409                  */
 410                 void adjust(int fromIndex, int toIndex, long diff);
 411         }
 412
 413         private static class Table implements Index {
 414
 415                 /** Table of file positions of each index. There are size() + 1 entries in the table. if null sample size is fixed */
 416                 TLongArrayList table = new TLongArrayList(32);
 417
 418                 Table(long start) {
 419                         table.add(start);
 420                 }
 421
 422                 @Override
 423                 public void add(long position) {
 424                         table.add(position);
 425                 }
 426
 427                 @Override
 428                 public void insert(int index, long position) {
 429                         table.insert(index, position);
 430                 }
 431
 432                 @Override
 433                 public void set(int index, long position) {
 434                         table.set(index, position);
 435                 }
 436
 437                 @Override
 438                 public void remove(int index, int length) {
 439                         table.remove(index, length);
 440                 }
 441
 442                 @Override
 443                 public void adjust(int fromIndex, int toIndex, long diff) {
 444                         if (diff==0) return;
 445                         for (int index = fromIndex; index<toIndex; index++)
 446                                 table.set(index, table.get(index) + diff);
 447                 }
 448
 449                 @Override
 450                 public long get(int index) {
 451                         return table.get(index);
 452                 }
 453
 454                 @Override
 455                 public int size() {
 456                         return table.size();
 457                 }
 458
 459         }
 460
 461         private static class Constant implements Index {
 462                 long start;
 463                 long sampleSize;
 464                 int count;
 465                 Constant(long start, int sampleSize, int count) {
 466                         this.start = start;
 467                         this.sampleSize = sampleSize;
 468                         this.count = count;
 469                 }
 470                 @Override
 471                 public void add(long position) {
 472                         assert( (position-start) % sampleSize == 0);
 473                         count++;
 474                 }
 475                 @Override
 476                 public void set(int index, long position) {
 477                         assert( position == start + index * sampleSize);
 478                 }
 479                 @Override
 480                 public void adjust(int fromIndex, int toIndex, long diff) {
 481                 }
 482                 @Override
 483                 public long get(int index) {
 484                         return start + index*sampleSize;
 485                 }
 486                 @Override
 487                 public void insert(int index, long position) {
 488                         assert( position == start + index * sampleSize );
 489                         count++;
 490                 }
 491                 @Override
 492                 public void remove(int index, int count) {
 493                         this.count -= count;
 494                 }
 495                 @Override
 496                 public int size() {
 497                         return count+1;
 498                 }
 499         }
 500
 501         @Override
 502         public File getFile() {
 503                 return null;
 504         }
 505
 506
 507 }
 508
 509