bundles/org.simantics.history/src/org/simantics/history/impl/FileHistory.java

   1 /*******************************************************************************\r
   2  * Copyright (c) 2007, 2011 Association for Decentralized Information Management in\r
   3  * Industry THTH ry.\r
   4  * All rights reserved. This program and the accompanying materials\r
   5  * are made available under the terms of the Eclipse Public License v1.0\r
   6  * which accompanies this distribution, and is available at\r
   7  * http://www.eclipse.org/legal/epl-v10.html\r
   8  *\r
   9  * Contributors:\r
  10  *     VTT Technical Research Centre of Finland - initial API and implementation\r
  11  *******************************************************************************/\r
  12 package org.simantics.history.impl;\r
  13 \r
  14 import java.io.DataInput;\r
  15 import java.io.DataOutput;\r
  16 import java.io.File;\r
  17 import java.io.FilenameFilter;\r
  18 import java.io.IOException;\r
  19 import java.nio.BufferUnderflowException;\r
  20 import java.nio.ByteBuffer;\r
  21 import java.util.ArrayList;\r
  22 import java.util.List;\r
  23 import java.util.logging.Logger;\r
  24 \r
  25 import org.simantics.databoard.Accessors;\r
  26 import org.simantics.databoard.Bindings;\r
  27 import org.simantics.databoard.accessor.ArrayAccessor;\r
  28 import org.simantics.databoard.accessor.StreamAccessor;\r
  29 import org.simantics.databoard.accessor.error.AccessorConstructionException;\r
  30 import org.simantics.databoard.accessor.error.AccessorException;\r
  31 import org.simantics.databoard.adapter.AdaptException;\r
  32 import org.simantics.databoard.adapter.Adapter;\r
  33 import org.simantics.databoard.adapter.AdapterConstructionException;\r
  34 import org.simantics.databoard.binding.Binding;\r
  35 import org.simantics.databoard.binding.error.BindingException;\r
  36 import org.simantics.databoard.binding.error.RuntimeBindingConstructionException;\r
  37 import org.simantics.databoard.serialization.Serializer;\r
  38 import org.simantics.databoard.serialization.SerializerConstructionException;\r
  39 import org.simantics.databoard.type.ArrayType;\r
  40 import org.simantics.databoard.type.Component;\r
  41 import org.simantics.databoard.type.Datatype;\r
  42 import org.simantics.databoard.type.NumberType;\r
  43 import org.simantics.databoard.type.RecordType;\r
  44 import org.simantics.databoard.util.Bean;\r
  45 import org.simantics.databoard.util.ObjectUtils;\r
  46 import org.simantics.databoard.util.URIUtil;\r
  47 import org.simantics.databoard.util.binary.BinaryMemory;\r
  48 import org.simantics.databoard.util.binary.ByteBufferWriteable;\r
  49 import org.simantics.history.HistoryException;\r
  50 import org.simantics.history.HistoryManager;\r
  51 import org.simantics.utils.FileUtils;\r
  52 \r
  53 /**\r
  54  * File history uses workarea (directory) to manage items.\r
  55  * There are two files for every item: stream file and metadata file.\r
  56  * Metadata file is JSON ascii file.  \r
  57  *   itemname.data\r
  58  *   itemname.txt\r
  59  *  \r
  60  * @author toni.kalajainen\r
  61  *\r
  62  */\r
  63 public class FileHistory implements HistoryManager {\r
  64 \r
  65         private static final boolean PROFILE = false;\r
  66         private static final boolean DEBUG = false;\r
  67 \r
  68         /** Logger */\r
  69         static Logger logger = Logger.getLogger(FileHistory.class.getName());\r
  70         \r
  71         static FilenameFilter txtFilter;\r
  72         \r
  73         static ArrayType INDEX_TYPE = Bindings.LONG_ARRAY.type();\r
  74         \r
  75         File workarea;\r
  76         \r
  77         /** Support async usage of the data */\r
  78         public boolean asyncUsage = true;\r
  79 \r
  80         public FileHistory(File workarea) {\r
  81                 this.workarea = workarea;\r
  82         }\r
  83         \r
  84     public File getWorkarea() {\r
  85         return workarea;\r
  86     }\r
  87         \r
  88         @Override\r
  89         public void create(Bean... items) throws HistoryException {\r
  90                 \r
  91                 try {\r
  92                         for (Bean item : items) {\r
  93                                 if (DEBUG)\r
  94                                         System.out.println("create(" + item + ")");\r
  95 //                              if ( !item.getFieldBinding("format").type().equals( Bindings.getBindingUnchecked(Datatype.class).type() ) )\r
  96 //                                      System.err.println("Error");\r
  97                                 \r
  98                                 // Write meta data\r
  99                                 writeMetadata( item );\r
 100                                 \r
 101                                 // Create stream file\r
 102                                 String id = (String) item.getField("id");\r
 103                                 File dataFile = toDataFile( id );\r
 104                                 dataFile.createNewFile();\r
 105                                 \r
 106                                 Datatype type = (Datatype) item.getField("format");\r
 107                                 if ( isVariableWidth(type) ) {\r
 108                                         File indexFile = toIndexFile( id );\r
 109                                         indexFile.createNewFile();\r
 110                                 }\r
 111 //                              if ( !dataFile.createNewFile() ) {\r
 112 //                                      throw new HistoryException("Could not create file "+dataFile);\r
 113 //                              }\r
 114                         }               \r
 115                 } catch (BindingException e) {\r
 116                         throw new HistoryException(e);\r
 117                 } catch (IOException e) {\r
 118                         throw new HistoryException(e);\r
 119                 }\r
 120         }\r
 121 \r
 122         @Override\r
 123         public void delete(String... itemIds) throws HistoryException {\r
 124                 for (String itemId : itemIds ) {\r
 125                         File meta = toMetaFile( itemId );\r
 126                         File data = toDataFile( itemId );\r
 127                         File index = toIndexFile( itemId );\r
 128                         if ( meta.exists() ) {\r
 129                                 if ( !meta.delete() ) {\r
 130                                         throw new HistoryException("Failed to delete "+meta);\r
 131                                 }\r
 132                         }\r
 133                         if ( data.exists() ) {\r
 134                                 if ( !data.delete() ) {\r
 135                                         throw new HistoryException("Failed to delete "+data);\r
 136                                 }\r
 137                         }\r
 138                         if ( index.exists() ) {\r
 139                                 if ( !index.delete() );\r
 140                         }\r
 141                 }\r
 142         }\r
 143 \r
 144         @Override\r
 145         public void modify(Bean... items) throws HistoryException {\r
 146                 \r
 147                 try {\r
 148                         for ( Bean item : items ) {\r
 149                                 if (DEBUG)\r
 150                                         System.out.println("modify(" + item + ")");\r
 151 //                              if ( !item.getFieldBinding("format").type().equals( Bindings.getBindingUnchecked(Datatype.class).type() ) )\r
 152 //                                      System.err.println("Error");\r
 153 \r
 154                                 String id = (String) item.getField("id");\r
 155                                 File metaFile = toMetaFile( id );\r
 156                                 if ( !metaFile.exists() ) {\r
 157                                         create( item );\r
 158                                 } else {\r
 159                                         Bean oldItem = getItem( id );\r
 160                                         File dataFile = toDataFile( id );\r
 161                                         if ( dataFile.exists() ) {\r
 162                                                 boolean enabled = item.hasField("enabled") ? (Boolean) item.getFieldUnchecked("enabled") : true;\r
 163                                                 Datatype oldFormat = (Datatype) oldItem.getField( "format" );\r
 164                                                 Datatype newFormat = (Datatype) item.getField( "format" );\r
 165                                                 if (DEBUG)\r
 166                                                         System.out.println("formats: " + oldFormat +  " -> " + newFormat);\r
 167                                                 Datatype unitStrippedOldFormat = stripUnitAnnotations(oldFormat);\r
 168                                                 Datatype unitStrippedNewFormat = stripUnitAnnotations(newFormat);\r
 169                                                 if (DEBUG)\r
 170                                                         System.out.println("formats after unit strip: " + unitStrippedOldFormat +  " -> " + unitStrippedNewFormat);\r
 171                                                 if ( enabled && !unitStrippedOldFormat.equals(unitStrippedNewFormat) ) {\r
 172                                                         try {\r
 173                                                                 Binding oldBinding = Bindings.getBeanBinding(unitStrippedOldFormat);\r
 174                                                                 Binding newBinding = Bindings.getBeanBinding(unitStrippedNewFormat);\r
 175                                                                 Serializer oldS = Bindings.getSerializer(oldBinding);\r
 176                                                                 Serializer newS = Bindings.getSerializer(newBinding);\r
 177                                                                 if (oldS.getConstantSize()==null || newS.getConstantSize()==null || oldS.getConstantSize()!=newS.getConstantSize())\r
 178                                                                         throw new HistoryException("Changing of file format is not supported to: "+dataFile);\r
 179                                                                 Adapter adapter = Bindings.getAdapter(oldBinding, newBinding);\r
 180                                                                 Object oldSample = oldBinding.createDefault();\r
 181                                                                 Object newSample = newBinding.createDefault();\r
 182                                                                 StreamAccessor sa = openStream(id, "rw");\r
 183                                                                 try {\r
 184                                                                         int c = sa.size();\r
 185                                                                         for (int i=0; i<c; i++) {\r
 186                                                                                 sa.get(i, oldBinding, oldSample);\r
 187                                                                                 newSample = adapter.adapt(oldSample);\r
 188                                                                                 sa.set(i, newBinding, newSample);\r
 189                                                                         }\r
 190                                                                 } finally {\r
 191                                                                         sa.close();\r
 192                                                                 }\r
 193                                                         } catch (AdapterConstructionException e) {\r
 194                                                                 throw new HistoryException("Changing of file format is not supported to: "+id);\r
 195                                                         } catch (SerializerConstructionException e) {\r
 196                                                                 throw new HistoryException("Changing of file format is not supported to: "+id);\r
 197                                                         } catch (AccessorException e) {\r
 198                                                                 throw new HistoryException("Changing of file format failed to: "+id);\r
 199                                                         } catch (AdaptException e) {\r
 200                                                                 throw new HistoryException("Changing of file format failed to: "+id);\r
 201                                                         }\r
 202                                                 }\r
 203                                         } else {\r
 204                                                 dataFile.createNewFile();\r
 205                                         }\r
 206 \r
 207                                         // Write new meta-data if necessary\r
 208                                         if (!equalsWithoutState(item, oldItem))\r
 209                                                 writeMetadata( item );\r
 210                                 }\r
 211                         }\r
 212                 } catch (BindingException e) {\r
 213                         throw new HistoryException( e );\r
 214                 } catch (IOException e) {\r
 215                         throw new HistoryException( e );\r
 216                 }\r
 217         }\r
 218 \r
 219         @Override\r
 220         public Bean getItem(String itemId) throws HistoryException {\r
 221                 return getItem( toMetaFile( itemId ) );\r
 222         }\r
 223         \r
 224         void writeMetadata( Bean item ) throws HistoryException {\r
 225                 \r
 226 //              long s = System.nanoTime();\r
 227                 try {\r
 228                         String id = (String) item.getField("id");\r
 229                         String idEnc  = URIUtil.encodeURI(id);\r
 230                         File metaFile = new File(workarea, idEnc + ".txt");\r
 231                         Serializer typeSerializer = Bindings.getSerializer( Bindings.getBindingUnchecked(Datatype.class) );\r
 232                         Serializer beanSerializer = Bindings.getSerializer( item.getBinding() );\r
 233                         int size = typeSerializer.getSize(item.getBinding().type()) +\r
 234                                         beanSerializer.getSize(item);\r
 235                         byte data[] = new byte[size];\r
 236                         DataOutput out = new ByteBufferWriteable( ByteBuffer.wrap(data) );\r
 237                         \r
 238                         if ( metaFile.exists() && asyncUsage ) {\r
 239                                 if (DEBUG)\r
 240                                         System.out.println("WARNING: FileHistory.writeMetadata: on SLOW path for " + item);\r
 241                                 File tmpFile = new File(workarea, idEnc + ".tmp");\r
 242                                 File tmp2File = new File(workarea, idEnc + ".tmp2");\r
 243                                 tmpFile.delete();\r
 244         \r
 245                                 typeSerializer.serialize(out, item.getBinding().type());\r
 246                                 beanSerializer.serialize(out, item);\r
 247                                 FileUtils.writeFile(tmpFile, data);\r
 248                                 metaFile.renameTo(tmp2File);\r
 249                                 tmpFile.renameTo(metaFile);\r
 250                                 tmp2File.delete();\r
 251                         } else {\r
 252                                 typeSerializer.serialize(out, item.getBinding().type());\r
 253                                 beanSerializer.serialize(out, item);\r
 254                                 FileUtils.writeFile(metaFile, data);\r
 255                         }\r
 256 \r
 257 //                      if (PROFILE)\r
 258 //                              System.out.println("PROFILE: FileHistory.writeMetadata( " + metaFile.getName() + " ) in " + ((System.nanoTime() - s)*1e-6) + " ms");\r
 259                 } catch (BindingException e) {\r
 260                         throw new HistoryException(e);\r
 261                 } catch (IOException e) {\r
 262                         throw new HistoryException(e);\r
 263                 } catch (SerializerConstructionException e) {\r
 264                         throw new HistoryException(e);\r
 265                 }\r
 266         }\r
 267 \r
 268         Bean getItem(File file) throws HistoryException {\r
 269 //              FileInputStream fis;\r
 270 //              try {\r
 271 //                      fis = new FileInputStream(file);\r
 272 //              } catch (FileNotFoundException e1) {\r
 273 //                      throw new HistoryException(e1);\r
 274 //              }\r
 275                 try {\r
 276                         byte[] data = FileUtils.readFile(file);\r
 277                         DataInput in = new BinaryMemory(data);  \r
 278                         Serializer typeSerializer = Bindings.getSerializer( Bindings.getBindingUnchecked(Datatype.class) );                     \r
 279                         Datatype type = (Datatype) typeSerializer.deserialize(in);\r
 280                         Binding beanBinding = Bindings.getBeanBinding( type );\r
 281                         Serializer s = Bindings.getSerializer( beanBinding );\r
 282                         Bean bean = (Bean) s.deserialize(in);\r
 283                         /*\r
 284                         DataInput in = new InputStreamReadable( fis, file.length() );\r
 285                         Serializer typeSerializer = Bindings.getSerializer( Bindings.getBindingUnchecked(Datatype.class) );                     \r
 286                         Datatype type = (Datatype) typeSerializer.deserialize(in);\r
 287                         Binding beanBinding = Bindings.getBeanBinding( type );\r
 288                         Serializer s = Bindings.getSerializer( beanBinding );\r
 289                         Bean bean = (Bean) s.deserialize(in);\r
 290                         */\r
 291                         return bean;\r
 292 /*                      String txt = new String(data, UTF8.CHARSET);                    \r
 293                         DataValueRepository repo = new DataValueRepository();\r
 294                         String name = repo.addValueDefinition(txt);\r
 295                         MutableVariant value = repo.get(name);\r
 296                         Binding beanBinding = Bindings.getBeanBinding( value.type() );\r
 297                         return (Bean) value.getValue(beanBinding);*/\r
 298                 } catch(BufferUnderflowException e) {\r
 299                         throw new HistoryException( e );\r
 300                 } catch(IOException e) {\r
 301                         throw new HistoryException( e );\r
 302 //              } catch (DataTypeSyntaxError e) {\r
 303 //                      throw new HistoryException( e );\r
 304                 } catch (RuntimeBindingConstructionException e) {\r
 305                         throw new HistoryException( e );\r
 306                 } catch (SerializerConstructionException e) {\r
 307                         throw new HistoryException( e );\r
 308                 } finally {\r
 309 //                      try {\r
 310 //                              fis.close();\r
 311 //                      } catch (IOException e) {\r
 312 //                      }\r
 313                 }\r
 314         }\r
 315         \r
 316         File toMetaFile(String itemId)\r
 317         {\r
 318                 String name = URIUtil.encodeURI(itemId) + ".txt";\r
 319                 File f = new File(workarea, name);\r
 320                 return f;\r
 321         }\r
 322 \r
 323         File toDataFile(String itemId)\r
 324         {\r
 325                 String name = URIUtil.encodeURI(itemId) + ".data";\r
 326                 File f = new File(workarea, name);\r
 327                 return f;\r
 328         }\r
 329         \r
 330         File toIndexFile(String itemId)\r
 331         {\r
 332                 String name = URIUtil.encodeURI(itemId) + ".index";\r
 333                 File f = new File(workarea, name);\r
 334                 return f;\r
 335         }\r
 336         \r
 337         boolean isVariableWidth(Datatype type) \r
 338         throws HistoryException {\r
 339                 try {\r
 340                         Binding beanBinding = Bindings.getBeanBinding(type);\r
 341                         Serializer s = Bindings.getSerializer( beanBinding );\r
 342                         return s.getConstantSize() == null;\r
 343                 } catch (SerializerConstructionException e) {\r
 344                         throw new HistoryException(e);\r
 345                 }\r
 346         }\r
 347 \r
 348         @Override\r
 349         public Bean[] getItems() throws HistoryException {\r
 350                 List<Bean> result = new ArrayList<Bean>();\r
 351                 File[] files = workarea.listFiles(txtFilter);\r
 352                 if ( files != null ) {\r
 353                         for (File file : files) {\r
 354                                 result.add( getItem(file) );\r
 355                         }\r
 356                 }\r
 357                 return result.toArray( new Bean[ result.size() ] );\r
 358         }\r
 359 \r
 360         @Override\r
 361         public void close() {\r
 362                 // Nothing to do.\r
 363         }\r
 364 \r
 365         @Override\r
 366         public StreamAccessor openStream(String itemId, String mode) throws HistoryException {\r
 367                 try {\r
 368                         Bean bean = getItem(itemId);\r
 369                         Datatype format = (Datatype) bean.getField("format");\r
 370                         ArrayType arrayType = new ArrayType(format);\r
 371                         File dataFile = toDataFile( itemId );\r
 372                         if ( isVariableWidth(format) ) {\r
 373                                 File indexFile = toIndexFile( itemId );\r
 374                                 ArrayAccessor index = Accessors.openStream(indexFile, INDEX_TYPE, mode);\r
 375                                 return (StreamAccessor) Accessors.openStream(dataFile, arrayType, mode, index);\r
 376                         } else {\r
 377                                 return (StreamAccessor) Accessors.openStream(dataFile, arrayType, mode);\r
 378                         }\r
 379                 } catch (AccessorConstructionException e) {\r
 380                         throw new HistoryException(e);\r
 381                 } catch (BindingException e) {\r
 382                         throw new HistoryException(e);\r
 383                 }\r
 384         }\r
 385 \r
 386         @Override\r
 387         public boolean exists(String itemId) throws HistoryException {\r
 388                 return toMetaFile(itemId).exists();\r
 389         }\r
 390         \r
 391         @Override\r
 392         public int hashCode() {\r
 393                 return workarea.hashCode();\r
 394         }\r
 395         \r
 396         @Override\r
 397         public boolean equals(Object obj) {\r
 398                 if ( obj==null ) return false;\r
 399                 if ( obj instanceof FileHistory == false ) return false;\r
 400                 FileHistory other = (FileHistory) obj;          \r
 401                 return other.workarea.equals(workarea);\r
 402         }\r
 403         \r
 404         @Override\r
 405         public String toString() {\r
 406                 return "FileHistory: "+workarea;\r
 407         }\r
 408 \r
 409         private boolean equalsWithoutState(Bean i1, Bean i2) {\r
 410                 Component[] components1 = i1.getBinding().type().getComponents();\r
 411                 Component[] components2 = i2.getBinding().type().getComponents();\r
 412                 int components = Math.min(components1.length, components2.length);\r
 413                 for (int c = 0; c < components; ++c) {\r
 414                         Object o1 = i1.getFieldUnchecked(c);\r
 415                         Object o2 = i2.getFieldUnchecked(c);\r
 416                         if ("collectorState".equals(components1[c].name) && (o1 == null || o2 == null))\r
 417                                 continue;\r
 418                         if (!ObjectUtils.objectEquals(o1, o2))\r
 419                                 return false;\r
 420                 }\r
 421                 return true;\r
 422         }\r
 423 \r
 424         static {\r
 425                 txtFilter = new FilenameFilter() {\r
 426                         public boolean accept(File dir, String name) {\r
 427                                 return name.toLowerCase().endsWith(".txt");\r
 428                         }\r
 429                 };\r
 430         }\r
 431 \r
 432         private static Datatype stripUnitAnnotations(Datatype datatype) {\r
 433                 if (datatype instanceof NumberType) {\r
 434                         NumberType nt = (NumberType) datatype;\r
 435                         if (nt.getUnit() != null) {\r
 436                                 Binding dtb = Bindings.getBindingUnchecked(Datatype.class);\r
 437                                 datatype = nt = (NumberType) Bindings.cloneUnchecked(datatype, dtb, dtb);\r
 438                                 nt.setUnit(null);\r
 439                         }\r
 440                 } else if (datatype instanceof ArrayType) {\r
 441                         ArrayType at = (ArrayType) datatype;\r
 442                         Datatype ct = at.componentType();\r
 443                         Datatype component = stripUnitAnnotations(ct);\r
 444                         if (component != ct) {\r
 445                                 Binding dtb = Bindings.getBindingUnchecked(Datatype.class);\r
 446                                 datatype = at = (ArrayType) Bindings.cloneUnchecked(datatype, dtb, dtb);\r
 447                                 at.setComponentType(component);\r
 448                         }\r
 449                 } else if (datatype instanceof RecordType) {\r
 450                         RecordType rt = (RecordType) datatype;\r
 451                         int componentCount = rt.getComponentCount();\r
 452                         Component[] newComponents = new Component[componentCount];\r
 453                         for (int i = 0; i < componentCount; ++i) {\r
 454                                 Component c = rt.getComponent(i);\r
 455                                 Datatype ct = c.type;\r
 456                                 Datatype sct = stripUnitAnnotations(ct);\r
 457                                 newComponents[i] = new Component(c.name, sct);\r
 458                         }\r
 459                         return new RecordType(rt.isReferable(), newComponents);\r
 460                 }\r
 461                 return datatype;\r
 462         }\r
 463 \r
 464 }\r