bundles/org.simantics.history/src/org/simantics/history/impl/FileHistory.java

   1 /*******************************************************************************
   2  * Copyright (c) 2007, 2011 Association for Decentralized Information Management in
   3  * Industry THTH ry.
   4  * All rights reserved. This program and the accompanying materials
   5  * are made available under the terms of the Eclipse Public License v1.0
   6  * which accompanies this distribution, and is available at
   7  * http://www.eclipse.org/legal/epl-v10.html
   8  *
   9  * Contributors:
  10  *     VTT Technical Research Centre of Finland - initial API and implementation
  11  *******************************************************************************/
  12 package org.simantics.history.impl;
  13
  14 import java.io.DataInput;
  15 import java.io.DataOutput;
  16 import java.io.File;
  17 import java.io.FilenameFilter;
  18 import java.io.IOException;
  19 import java.nio.BufferUnderflowException;
  20 import java.nio.ByteBuffer;
  21 import java.util.ArrayList;
  22 import java.util.List;
  23 import java.util.logging.Logger;
  24
  25 import org.simantics.databoard.Accessors;
  26 import org.simantics.databoard.Bindings;
  27 import org.simantics.databoard.accessor.ArrayAccessor;
  28 import org.simantics.databoard.accessor.StreamAccessor;
  29 import org.simantics.databoard.accessor.error.AccessorConstructionException;
  30 import org.simantics.databoard.accessor.error.AccessorException;
  31 import org.simantics.databoard.adapter.AdaptException;
  32 import org.simantics.databoard.adapter.Adapter;
  33 import org.simantics.databoard.adapter.AdapterConstructionException;
  34 import org.simantics.databoard.binding.Binding;
  35 import org.simantics.databoard.binding.error.BindingException;
  36 import org.simantics.databoard.binding.error.RuntimeBindingConstructionException;
  37 import org.simantics.databoard.serialization.Serializer;
  38 import org.simantics.databoard.serialization.SerializerConstructionException;
  39 import org.simantics.databoard.type.ArrayType;
  40 import org.simantics.databoard.type.Component;
  41 import org.simantics.databoard.type.Datatype;
  42 import org.simantics.databoard.type.NumberType;
  43 import org.simantics.databoard.type.RecordType;
  44 import org.simantics.databoard.util.Bean;
  45 import org.simantics.databoard.util.ObjectUtils;
  46 import org.simantics.databoard.util.URIUtil;
  47 import org.simantics.databoard.util.binary.BinaryMemory;
  48 import org.simantics.databoard.util.binary.ByteBufferWriteable;
  49 import org.simantics.history.HistoryException;
  50 import org.simantics.history.HistoryManager;
  51 import org.simantics.utils.FileUtils;
  52
  53 /**
  54  * File history uses workarea (directory) to manage items.
  55  * There are two files for every item: stream file and metadata file.
  56  * Metadata file is JSON ascii file.
  57  *   itemname.data
  58  *   itemname.txt
  59  *
  60  * @author toni.kalajainen
  61  *
  62  */
  63 public class FileHistory implements HistoryManager {
  64
  65         private static final boolean PROFILE = false;
  66         private static final boolean DEBUG = false;
  67
  68         /** Logger */
  69         static Logger logger = Logger.getLogger(FileHistory.class.getName());
  70
  71         static FilenameFilter txtFilter;
  72
  73         static ArrayType INDEX_TYPE = Bindings.LONG_ARRAY.type();
  74
  75         File workarea;
  76
  77         /** Support async usage of the data */
  78         public boolean asyncUsage = true;
  79
  80         public FileHistory(File workarea) {
  81                 this.workarea = workarea;
  82         }
  83
  84     public File getWorkarea() {
  85         return workarea;
  86     }
  87
  88         @Override
  89         public void create(Bean... items) throws HistoryException {
  90
  91                 try {
  92                         for (Bean item : items) {
  93                                 if (DEBUG)
  94                                         System.out.println("create(" + item + ")");
  95 //                              if ( !item.getFieldBinding("format").type().equals( Bindings.getBindingUnchecked(Datatype.class).type() ) )
  96 //                                      System.err.println("Error");
  97
  98                                 // Write meta data
  99                                 writeMetadata( item );
 100
 101                                 // Create stream file
 102                                 String id = (String) item.getField("id");
 103                                 File dataFile = toDataFile( id );
 104                                 dataFile.createNewFile();
 105
 106                                 Datatype type = (Datatype) item.getField("format");
 107                                 if ( isVariableWidth(type) ) {
 108                                         File indexFile = toIndexFile( id );
 109                                         indexFile.createNewFile();
 110                                 }
 111 //                              if ( !dataFile.createNewFile() ) {
 112 //                                      throw new HistoryException("Could not create file "+dataFile);
 113 //                              }
 114                         }
 115                 } catch (BindingException e) {
 116                         throw new HistoryException(e);
 117                 } catch (IOException e) {
 118                         throw new HistoryException(e);
 119                 }
 120         }
 121
 122         @Override
 123         public void delete(String... itemIds) throws HistoryException {
 124                 for (String itemId : itemIds ) {
 125                         File meta = toMetaFile( itemId );
 126                         File data = toDataFile( itemId );
 127                         File index = toIndexFile( itemId );
 128                         if ( meta.exists() ) {
 129                                 if ( !meta.delete() ) {
 130                                         throw new HistoryException("Failed to delete "+meta);
 131                                 }
 132                         }
 133                         if ( data.exists() ) {
 134                                 if ( !data.delete() ) {
 135                                         throw new HistoryException("Failed to delete "+data);
 136                                 }
 137                         }
 138                         if ( index.exists() ) {
 139                                 if ( !index.delete() );
 140                         }
 141                 }
 142         }
 143
 144         @Override
 145         public void modify(Bean... items) throws HistoryException {
 146
 147                 try {
 148                         for ( Bean item : items ) {
 149                                 if (DEBUG)
 150                                         System.out.println("modify(" + item + ")");
 151 //                              if ( !item.getFieldBinding("format").type().equals( Bindings.getBindingUnchecked(Datatype.class).type() ) )
 152 //                                      System.err.println("Error");
 153
 154                                 String id = (String) item.getField("id");
 155                                 File metaFile = toMetaFile( id );
 156                                 if ( !metaFile.exists() ) {
 157                                         create( item );
 158                                 } else {
 159                                         Bean oldItem = getItem( id );
 160                                         File dataFile = toDataFile( id );
 161                                         if ( dataFile.exists() ) {
 162                                                 boolean enabled = item.hasField("enabled") ? (Boolean) item.getFieldUnchecked("enabled") : true;
 163                                                 Datatype oldFormat = (Datatype) oldItem.getField( "format" );
 164                                                 Datatype newFormat = (Datatype) item.getField( "format" );
 165                                                 if (DEBUG)
 166                                                         System.out.println("formats: " + oldFormat +  " -> " + newFormat);
 167                                                 Datatype unitStrippedOldFormat = stripUnitAnnotations(oldFormat);
 168                                                 Datatype unitStrippedNewFormat = stripUnitAnnotations(newFormat);
 169                                                 if (DEBUG)
 170                                                         System.out.println("formats after unit strip: " + unitStrippedOldFormat +  " -> " + unitStrippedNewFormat);
 171                                                 if ( enabled && !unitStrippedOldFormat.equals(unitStrippedNewFormat) ) {
 172                                                         try {
 173                                                                 Binding oldBinding = Bindings.getBeanBinding(unitStrippedOldFormat);
 174                                                                 Binding newBinding = Bindings.getBeanBinding(unitStrippedNewFormat);
 175                                                                 Serializer oldS = Bindings.getSerializer(oldBinding);
 176                                                                 Serializer newS = Bindings.getSerializer(newBinding);
 177                                                                 if (oldS.getConstantSize()==null || newS.getConstantSize()==null || oldS.getConstantSize()!=newS.getConstantSize())
 178                                                                         throw new HistoryException("Changing of file format is not supported to: "+dataFile);
 179                                                                 Adapter adapter = Bindings.getAdapter(oldBinding, newBinding);
 180                                                                 Object oldSample = oldBinding.createDefault();
 181                                                                 Object newSample = newBinding.createDefault();
 182                                                                 StreamAccessor sa = openStream(id, "rw");
 183                                                                 try {
 184                                                                         int c = sa.size();
 185                                                                         for (int i=0; i<c; i++) {
 186                                                                                 sa.get(i, oldBinding, oldSample);
 187                                                                                 newSample = adapter.adapt(oldSample);
 188                                                                                 sa.set(i, newBinding, newSample);
 189                                                                         }
 190                                                                 } finally {
 191                                                                         sa.close();
 192                                                                 }
 193                                                         } catch (AdapterConstructionException e) {
 194                                                                 throw new HistoryException("Changing of file format is not supported to: "+id);
 195                                                         } catch (SerializerConstructionException e) {
 196                                                                 throw new HistoryException("Changing of file format is not supported to: "+id);
 197                                                         } catch (AccessorException e) {
 198                                                                 throw new HistoryException("Changing of file format failed to: "+id);
 199                                                         } catch (AdaptException e) {
 200                                                                 throw new HistoryException("Changing of file format failed to: "+id);
 201                                                         }
 202                                                 }
 203                                         } else {
 204                                                 dataFile.createNewFile();
 205                                         }
 206
 207                                         // Write new meta-data if necessary
 208                                         if (!equalsWithoutState(item, oldItem))
 209                                                 writeMetadata( item );
 210                                 }
 211                         }
 212                 } catch (BindingException e) {
 213                         throw new HistoryException( e );
 214                 } catch (IOException e) {
 215                         throw new HistoryException( e );
 216                 }
 217         }
 218
 219         @Override
 220         public Bean getItem(String itemId) throws HistoryException {
 221                 return getItem( toMetaFile( itemId ) );
 222         }
 223
 224         void writeMetadata( Bean item ) throws HistoryException {
 225
 226 //              long s = System.nanoTime();
 227                 try {
 228                         String id = (String) item.getField("id");
 229                         String idEnc  = URIUtil.encodeURI(id);
 230                         File metaFile = new File(workarea, idEnc + ".txt");
 231                         Serializer typeSerializer = Bindings.getSerializer( Bindings.getBindingUnchecked(Datatype.class) );
 232                         Serializer beanSerializer = Bindings.getSerializer( item.getBinding() );
 233                         int size = typeSerializer.getSize(item.getBinding().type()) +
 234                                         beanSerializer.getSize(item);
 235                         byte data[] = new byte[size];
 236                         DataOutput out = new ByteBufferWriteable( ByteBuffer.wrap(data) );
 237
 238                         if ( metaFile.exists() && asyncUsage ) {
 239                                 if (DEBUG)
 240                                         System.out.println("WARNING: FileHistory.writeMetadata: on SLOW path for " + item);
 241                                 File tmpFile = new File(workarea, idEnc + ".tmp");
 242                                 File tmp2File = new File(workarea, idEnc + ".tmp2");
 243                                 tmpFile.delete();
 244
 245                                 typeSerializer.serialize(out, item.getBinding().type());
 246                                 beanSerializer.serialize(out, item);
 247                                 FileUtils.writeFile(tmpFile, data);
 248                                 metaFile.renameTo(tmp2File);
 249                                 tmpFile.renameTo(metaFile);
 250                                 tmp2File.delete();
 251                         } else {
 252                                 typeSerializer.serialize(out, item.getBinding().type());
 253                                 beanSerializer.serialize(out, item);
 254                                 FileUtils.writeFile(metaFile, data);
 255                         }
 256
 257 //                      if (PROFILE)
 258 //                              System.out.println("PROFILE: FileHistory.writeMetadata( " + metaFile.getName() + " ) in " + ((System.nanoTime() - s)*1e-6) + " ms");
 259                 } catch (BindingException e) {
 260                         throw new HistoryException(e);
 261                 } catch (IOException e) {
 262                         throw new HistoryException(e);
 263                 } catch (SerializerConstructionException e) {
 264                         throw new HistoryException(e);
 265                 }
 266         }
 267
 268         Bean getItem(File file) throws HistoryException {
 269 //              FileInputStream fis;
 270 //              try {
 271 //                      fis = new FileInputStream(file);
 272 //              } catch (FileNotFoundException e1) {
 273 //                      throw new HistoryException(e1);
 274 //              }
 275                 try {
 276                         byte[] data = FileUtils.readFile(file);
 277                         DataInput in = new BinaryMemory(data);
 278                         Serializer typeSerializer = Bindings.getSerializer( Bindings.getBindingUnchecked(Datatype.class) );
 279                         Datatype type = (Datatype) typeSerializer.deserialize(in);
 280                         Binding beanBinding = Bindings.getBeanBinding( type );
 281                         Serializer s = Bindings.getSerializer( beanBinding );
 282                         Bean bean = (Bean) s.deserialize(in);
 283                         /*
 284                         DataInput in = new InputStreamReadable( fis, file.length() );
 285                         Serializer typeSerializer = Bindings.getSerializer( Bindings.getBindingUnchecked(Datatype.class) );
 286                         Datatype type = (Datatype) typeSerializer.deserialize(in);
 287                         Binding beanBinding = Bindings.getBeanBinding( type );
 288                         Serializer s = Bindings.getSerializer( beanBinding );
 289                         Bean bean = (Bean) s.deserialize(in);
 290                         */
 291                         return bean;
 292 /*                      String txt = new String(data, UTF8.CHARSET);
 293                         DataValueRepository repo = new DataValueRepository();
 294                         String name = repo.addValueDefinition(txt);
 295                         MutableVariant value = repo.get(name);
 296                         Binding beanBinding = Bindings.getBeanBinding( value.type() );
 297                         return (Bean) value.getValue(beanBinding);*/
 298                 } catch(BufferUnderflowException e) {
 299                         throw new HistoryException( e );
 300                 } catch(IOException e) {
 301                         throw new HistoryException( e );
 302 //              } catch (DataTypeSyntaxError e) {
 303 //                      throw new HistoryException( e );
 304                 } catch (RuntimeBindingConstructionException e) {
 305                         throw new HistoryException( e );
 306                 } catch (SerializerConstructionException e) {
 307                         throw new HistoryException( e );
 308                 } finally {
 309 //                      try {
 310 //                              fis.close();
 311 //                      } catch (IOException e) {
 312 //                      }
 313                 }
 314         }
 315
 316         File toMetaFile(String itemId)
 317         {
 318                 String name = URIUtil.encodeURI(itemId) + ".txt";
 319                 File f = new File(workarea, name);
 320                 return f;
 321         }
 322
 323         File toDataFile(String itemId)
 324         {
 325                 String name = URIUtil.encodeURI(itemId) + ".data";
 326                 File f = new File(workarea, name);
 327                 return f;
 328         }
 329
 330         File toIndexFile(String itemId)
 331         {
 332                 String name = URIUtil.encodeURI(itemId) + ".index";
 333                 File f = new File(workarea, name);
 334                 return f;
 335         }
 336
 337         boolean isVariableWidth(Datatype type)
 338         throws HistoryException {
 339                 try {
 340                         Binding beanBinding = Bindings.getBeanBinding(type);
 341                         Serializer s = Bindings.getSerializer( beanBinding );
 342                         return s.getConstantSize() == null;
 343                 } catch (SerializerConstructionException e) {
 344                         throw new HistoryException(e);
 345                 }
 346         }
 347
 348         @Override
 349         public Bean[] getItems() throws HistoryException {
 350                 List<Bean> result = new ArrayList<Bean>();
 351                 File[] files = workarea.listFiles(txtFilter);
 352                 if ( files != null ) {
 353                         for (File file : files) {
 354                                 result.add( getItem(file) );
 355                         }
 356                 }
 357                 return result.toArray( new Bean[ result.size() ] );
 358         }
 359
 360         @Override
 361         public void close() {
 362                 // Nothing to do.
 363         }
 364
 365         @Override
 366         public StreamAccessor openStream(String itemId, String mode) throws HistoryException {
 367                 try {
 368                         Bean bean = getItem(itemId);
 369                         Datatype format = (Datatype) bean.getField("format");
 370                         ArrayType arrayType = new ArrayType(format);
 371                         File dataFile = toDataFile( itemId );
 372                         if ( isVariableWidth(format) ) {
 373                                 File indexFile = toIndexFile( itemId );
 374                                 ArrayAccessor index = Accessors.openStream(indexFile, INDEX_TYPE, mode);
 375                                 return (StreamAccessor) Accessors.openStream(dataFile, arrayType, mode, index);
 376                         } else {
 377                                 return (StreamAccessor) Accessors.openStream(dataFile, arrayType, mode);
 378                         }
 379                 } catch (AccessorConstructionException e) {
 380                         throw new HistoryException(e);
 381                 } catch (BindingException e) {
 382                         throw new HistoryException(e);
 383                 }
 384         }
 385
 386         @Override
 387         public boolean exists(String itemId) throws HistoryException {
 388                 return toMetaFile(itemId).exists();
 389         }
 390
 391         @Override
 392         public int hashCode() {
 393                 return workarea.hashCode();
 394         }
 395
 396         @Override
 397         public boolean equals(Object obj) {
 398                 if ( obj==null ) return false;
 399                 if ( obj instanceof FileHistory == false ) return false;
 400                 FileHistory other = (FileHistory) obj;
 401                 return other.workarea.equals(workarea);
 402         }
 403
 404         @Override
 405         public String toString() {
 406                 return "FileHistory: "+workarea;
 407         }
 408
 409         private boolean equalsWithoutState(Bean i1, Bean i2) {
 410                 Component[] components1 = i1.getBinding().type().getComponents();
 411                 Component[] components2 = i2.getBinding().type().getComponents();
 412                 int components = Math.min(components1.length, components2.length);
 413                 for (int c = 0; c < components; ++c) {
 414                         Object o1 = i1.getFieldUnchecked(c);
 415                         Object o2 = i2.getFieldUnchecked(c);
 416                         if ("collectorState".equals(components1[c].name) && (o1 == null || o2 == null))
 417                                 continue;
 418                         if (!ObjectUtils.objectEquals(o1, o2))
 419                                 return false;
 420                 }
 421                 return true;
 422         }
 423
 424         static {
 425                 txtFilter = new FilenameFilter() {
 426                         public boolean accept(File dir, String name) {
 427                                 return name.toLowerCase().endsWith(".txt");
 428                         }
 429                 };
 430         }
 431
 432         private static Datatype stripUnitAnnotations(Datatype datatype) {
 433                 if (datatype instanceof NumberType) {
 434                         NumberType nt = (NumberType) datatype;
 435                         if (nt.getUnit() != null) {
 436                                 Binding dtb = Bindings.getBindingUnchecked(Datatype.class);
 437                                 datatype = nt = (NumberType) Bindings.cloneUnchecked(datatype, dtb, dtb);
 438                                 nt.setUnit(null);
 439                         }
 440                 } else if (datatype instanceof ArrayType) {
 441                         ArrayType at = (ArrayType) datatype;
 442                         Datatype ct = at.componentType();
 443                         Datatype component = stripUnitAnnotations(ct);
 444                         if (component != ct) {
 445                                 Binding dtb = Bindings.getBindingUnchecked(Datatype.class);
 446                                 datatype = at = (ArrayType) Bindings.cloneUnchecked(datatype, dtb, dtb);
 447                                 at.setComponentType(component);
 448                         }
 449                 } else if (datatype instanceof RecordType) {
 450                         RecordType rt = (RecordType) datatype;
 451                         int componentCount = rt.getComponentCount();
 452                         Component[] newComponents = new Component[componentCount];
 453                         for (int i = 0; i < componentCount; ++i) {
 454                                 Component c = rt.getComponent(i);
 455                                 Datatype ct = c.type;
 456                                 Datatype sct = stripUnitAnnotations(ct);
 457                                 newComponents[i] = new Component(c.name, sct);
 458                         }
 459                         return new RecordType(rt.isReferable(), newComponents);
 460                 }
 461                 return datatype;
 462         }
 463
 464 }