1 /*******************************************************************************
2 * Copyright (c) 2007, 2011 Association for Decentralized Information Management in
4 * All rights reserved. This program and the accompanying materials
5 * are made available under the terms of the Eclipse Public License v1.0
6 * which accompanies this distribution, and is available at
7 * http://www.eclipse.org/legal/epl-v10.html
10 * VTT Technical Research Centre of Finland - initial API and implementation
11 *******************************************************************************/
12 package org.simantics.history.impl;
14 import java.io.DataInput;
15 import java.io.DataOutput;
17 import java.io.FilenameFilter;
18 import java.io.IOException;
19 import java.nio.BufferUnderflowException;
20 import java.nio.ByteBuffer;
21 import java.util.ArrayList;
22 import java.util.List;
23 import java.util.logging.Logger;
25 import org.simantics.databoard.Accessors;
26 import org.simantics.databoard.Bindings;
27 import org.simantics.databoard.accessor.ArrayAccessor;
28 import org.simantics.databoard.accessor.StreamAccessor;
29 import org.simantics.databoard.accessor.error.AccessorConstructionException;
30 import org.simantics.databoard.accessor.error.AccessorException;
31 import org.simantics.databoard.adapter.AdaptException;
32 import org.simantics.databoard.adapter.Adapter;
33 import org.simantics.databoard.adapter.AdapterConstructionException;
34 import org.simantics.databoard.binding.Binding;
35 import org.simantics.databoard.binding.error.BindingException;
36 import org.simantics.databoard.binding.error.RuntimeBindingConstructionException;
37 import org.simantics.databoard.serialization.Serializer;
38 import org.simantics.databoard.serialization.SerializerConstructionException;
39 import org.simantics.databoard.type.ArrayType;
40 import org.simantics.databoard.type.Component;
41 import org.simantics.databoard.type.Datatype;
42 import org.simantics.databoard.type.NumberType;
43 import org.simantics.databoard.type.RecordType;
44 import org.simantics.databoard.util.Bean;
45 import org.simantics.databoard.util.ObjectUtils;
46 import org.simantics.databoard.util.URIUtil;
47 import org.simantics.databoard.util.binary.BinaryMemory;
48 import org.simantics.databoard.util.binary.ByteBufferWriteable;
49 import org.simantics.history.HistoryException;
50 import org.simantics.history.HistoryManager;
51 import org.simantics.utils.FileUtils;
54 * File history uses workarea (directory) to manage items.
55 * There are two files for every item: stream file and metadata file.
56 * Metadata file is JSON ascii file.
60 * @author toni.kalajainen
63 public class FileHistory implements HistoryManager {
65 private static final boolean PROFILE = false;
66 private static final boolean DEBUG = false;
69 static Logger logger = Logger.getLogger(FileHistory.class.getName());
71 static FilenameFilter txtFilter;
73 static ArrayType INDEX_TYPE = Bindings.LONG_ARRAY.type();
77 /** Support async usage of the data */
78 public boolean asyncUsage = true;
80 public FileHistory(File workarea) {
81 this.workarea = workarea;
84 public File getWorkarea() {
89 public void create(Bean... items) throws HistoryException {
92 for (Bean item : items) {
94 System.out.println("create(" + item + ")");
95 // if ( !item.getFieldBinding("format").type().equals( Bindings.getBindingUnchecked(Datatype.class).type() ) )
96 // System.err.println("Error");
99 writeMetadata( item );
101 // Create stream file
102 String id = (String) item.getField("id");
103 File dataFile = toDataFile( id );
104 dataFile.createNewFile();
106 Datatype type = (Datatype) item.getField("format");
107 if ( isVariableWidth(type) ) {
108 File indexFile = toIndexFile( id );
109 indexFile.createNewFile();
111 // if ( !dataFile.createNewFile() ) {
112 // throw new HistoryException("Could not create file "+dataFile);
115 } catch (BindingException e) {
116 throw new HistoryException(e);
117 } catch (IOException e) {
118 throw new HistoryException(e);
123 public void delete(String... itemIds) throws HistoryException {
124 for (String itemId : itemIds ) {
125 File meta = toMetaFile( itemId );
126 File data = toDataFile( itemId );
127 File index = toIndexFile( itemId );
128 if ( meta.exists() ) {
129 if ( !meta.delete() ) {
130 throw new HistoryException("Failed to delete "+meta);
133 if ( data.exists() ) {
134 if ( !data.delete() ) {
135 throw new HistoryException("Failed to delete "+data);
138 if ( index.exists() ) {
139 if ( !index.delete() );
145 public void modify(Bean... items) throws HistoryException {
148 for ( Bean item : items ) {
150 System.out.println("modify(" + item + ")");
151 // if ( !item.getFieldBinding("format").type().equals( Bindings.getBindingUnchecked(Datatype.class).type() ) )
152 // System.err.println("Error");
154 String id = (String) item.getField("id");
155 File metaFile = toMetaFile( id );
156 if ( !metaFile.exists() ) {
159 Bean oldItem = getItem( id );
160 File dataFile = toDataFile( id );
161 if ( dataFile.exists() ) {
162 boolean enabled = item.hasField("enabled") ? (Boolean) item.getFieldUnchecked("enabled") : true;
163 Datatype oldFormat = (Datatype) oldItem.getField( "format" );
164 Datatype newFormat = (Datatype) item.getField( "format" );
166 System.out.println("formats: " + oldFormat + " -> " + newFormat);
167 Datatype unitStrippedOldFormat = stripUnitAnnotations(oldFormat);
168 Datatype unitStrippedNewFormat = stripUnitAnnotations(newFormat);
170 System.out.println("formats after unit strip: " + unitStrippedOldFormat + " -> " + unitStrippedNewFormat);
171 if ( enabled && !unitStrippedOldFormat.equals(unitStrippedNewFormat) ) {
173 Binding oldBinding = Bindings.getBeanBinding(unitStrippedOldFormat);
174 Binding newBinding = Bindings.getBeanBinding(unitStrippedNewFormat);
175 Serializer oldS = Bindings.getSerializer(oldBinding);
176 Serializer newS = Bindings.getSerializer(newBinding);
177 if (oldS.getConstantSize()==null || newS.getConstantSize()==null || oldS.getConstantSize()!=newS.getConstantSize())
178 throw new HistoryException("Changing of file format is not supported to: "+dataFile);
179 Adapter adapter = Bindings.getAdapter(oldBinding, newBinding);
180 Object oldSample = oldBinding.createDefault();
181 Object newSample = newBinding.createDefault();
182 StreamAccessor sa = openStream(id, "rw");
185 for (int i=0; i<c; i++) {
186 sa.get(i, oldBinding, oldSample);
187 newSample = adapter.adapt(oldSample);
188 sa.set(i, newBinding, newSample);
193 } catch (AdapterConstructionException e) {
194 throw new HistoryException("Changing of file format is not supported to: "+id);
195 } catch (SerializerConstructionException e) {
196 throw new HistoryException("Changing of file format is not supported to: "+id);
197 } catch (AccessorException e) {
198 throw new HistoryException("Changing of file format failed to: "+id);
199 } catch (AdaptException e) {
200 throw new HistoryException("Changing of file format failed to: "+id);
204 dataFile.createNewFile();
207 // Write new meta-data if necessary
208 if (!equalsWithoutState(item, oldItem))
209 writeMetadata( item );
212 } catch (BindingException e) {
213 throw new HistoryException( e );
214 } catch (IOException e) {
215 throw new HistoryException( e );
220 public Bean getItem(String itemId) throws HistoryException {
221 return getItem( toMetaFile( itemId ) );
224 void writeMetadata( Bean item ) throws HistoryException {
226 // long s = System.nanoTime();
228 String id = (String) item.getField("id");
229 String idEnc = URIUtil.encodeURI(id);
230 File metaFile = new File(workarea, idEnc + ".txt");
231 Serializer typeSerializer = Bindings.getSerializer( Bindings.getBindingUnchecked(Datatype.class) );
232 Serializer beanSerializer = Bindings.getSerializer( item.getBinding() );
233 int size = typeSerializer.getSize(item.getBinding().type()) +
234 beanSerializer.getSize(item);
235 byte data[] = new byte[size];
236 DataOutput out = new ByteBufferWriteable( ByteBuffer.wrap(data) );
238 if ( metaFile.exists() && asyncUsage ) {
240 System.out.println("WARNING: FileHistory.writeMetadata: on SLOW path for " + item);
241 File tmpFile = new File(workarea, idEnc + ".tmp");
242 File tmp2File = new File(workarea, idEnc + ".tmp2");
245 typeSerializer.serialize(out, item.getBinding().type());
246 beanSerializer.serialize(out, item);
247 FileUtils.writeFile(tmpFile, data);
248 metaFile.renameTo(tmp2File);
249 tmpFile.renameTo(metaFile);
252 typeSerializer.serialize(out, item.getBinding().type());
253 beanSerializer.serialize(out, item);
254 FileUtils.writeFile(metaFile, data);
258 // System.out.println("PROFILE: FileHistory.writeMetadata( " + metaFile.getName() + " ) in " + ((System.nanoTime() - s)*1e-6) + " ms");
259 } catch (BindingException e) {
260 throw new HistoryException(e);
261 } catch (IOException e) {
262 throw new HistoryException(e);
263 } catch (SerializerConstructionException e) {
264 throw new HistoryException(e);
268 Bean getItem(File file) throws HistoryException {
269 // FileInputStream fis;
271 // fis = new FileInputStream(file);
272 // } catch (FileNotFoundException e1) {
273 // throw new HistoryException(e1);
276 byte[] data = FileUtils.readFile(file);
277 DataInput in = new BinaryMemory(data);
278 Serializer typeSerializer = Bindings.getSerializer( Bindings.getBindingUnchecked(Datatype.class) );
279 Datatype type = (Datatype) typeSerializer.deserialize(in);
280 Binding beanBinding = Bindings.getBeanBinding( type );
281 Serializer s = Bindings.getSerializer( beanBinding );
282 Bean bean = (Bean) s.deserialize(in);
284 DataInput in = new InputStreamReadable( fis, file.length() );
285 Serializer typeSerializer = Bindings.getSerializer( Bindings.getBindingUnchecked(Datatype.class) );
286 Datatype type = (Datatype) typeSerializer.deserialize(in);
287 Binding beanBinding = Bindings.getBeanBinding( type );
288 Serializer s = Bindings.getSerializer( beanBinding );
289 Bean bean = (Bean) s.deserialize(in);
292 /* String txt = new String(data, UTF8.CHARSET);
293 DataValueRepository repo = new DataValueRepository();
294 String name = repo.addValueDefinition(txt);
295 MutableVariant value = repo.get(name);
296 Binding beanBinding = Bindings.getBeanBinding( value.type() );
297 return (Bean) value.getValue(beanBinding);*/
298 } catch(BufferUnderflowException e) {
299 throw new HistoryException( e );
300 } catch(IOException e) {
301 throw new HistoryException( e );
302 // } catch (DataTypeSyntaxError e) {
303 // throw new HistoryException( e );
304 } catch (RuntimeBindingConstructionException e) {
305 throw new HistoryException( e );
306 } catch (SerializerConstructionException e) {
307 throw new HistoryException( e );
311 // } catch (IOException e) {
316 File toMetaFile(String itemId)
318 String name = URIUtil.encodeURI(itemId) + ".txt";
319 File f = new File(workarea, name);
323 File toDataFile(String itemId)
325 String name = URIUtil.encodeURI(itemId) + ".data";
326 File f = new File(workarea, name);
330 File toIndexFile(String itemId)
332 String name = URIUtil.encodeURI(itemId) + ".index";
333 File f = new File(workarea, name);
337 boolean isVariableWidth(Datatype type)
338 throws HistoryException {
340 Binding beanBinding = Bindings.getBeanBinding(type);
341 Serializer s = Bindings.getSerializer( beanBinding );
342 return s.getConstantSize() == null;
343 } catch (SerializerConstructionException e) {
344 throw new HistoryException(e);
349 public Bean[] getItems() throws HistoryException {
350 List<Bean> result = new ArrayList<Bean>();
351 File[] files = workarea.listFiles(txtFilter);
352 if ( files != null ) {
353 for (File file : files) {
354 result.add( getItem(file) );
357 return result.toArray( new Bean[ result.size() ] );
361 public void close() {
366 public StreamAccessor openStream(String itemId, String mode) throws HistoryException {
368 Bean bean = getItem(itemId);
369 Datatype format = (Datatype) bean.getField("format");
370 ArrayType arrayType = new ArrayType(format);
371 File dataFile = toDataFile( itemId );
372 if ( isVariableWidth(format) ) {
373 File indexFile = toIndexFile( itemId );
374 ArrayAccessor index = Accessors.openStream(indexFile, INDEX_TYPE, mode);
375 return (StreamAccessor) Accessors.openStream(dataFile, arrayType, mode, index);
377 return (StreamAccessor) Accessors.openStream(dataFile, arrayType, mode);
379 } catch (AccessorConstructionException e) {
380 throw new HistoryException(e);
381 } catch (BindingException e) {
382 throw new HistoryException(e);
387 public boolean exists(String itemId) throws HistoryException {
388 return toMetaFile(itemId).exists();
392 public int hashCode() {
393 return workarea.hashCode();
397 public boolean equals(Object obj) {
398 if ( obj==null ) return false;
399 if ( obj instanceof FileHistory == false ) return false;
400 FileHistory other = (FileHistory) obj;
401 return other.workarea.equals(workarea);
405 public String toString() {
406 return "FileHistory: "+workarea;
409 private boolean equalsWithoutState(Bean i1, Bean i2) {
410 Component[] components1 = i1.getBinding().type().getComponents();
411 Component[] components2 = i2.getBinding().type().getComponents();
412 int components = Math.min(components1.length, components2.length);
413 for (int c = 0; c < components; ++c) {
414 Object o1 = i1.getFieldUnchecked(c);
415 Object o2 = i2.getFieldUnchecked(c);
416 if ("collectorState".equals(components1[c].name) && (o1 == null || o2 == null))
418 if (!ObjectUtils.objectEquals(o1, o2))
425 txtFilter = new FilenameFilter() {
426 public boolean accept(File dir, String name) {
427 return name.toLowerCase().endsWith(".txt");
432 private static Datatype stripUnitAnnotations(Datatype datatype) {
433 if (datatype instanceof NumberType) {
434 NumberType nt = (NumberType) datatype;
435 if (nt.getUnit() != null) {
436 Binding dtb = Bindings.getBindingUnchecked(Datatype.class);
437 datatype = nt = (NumberType) Bindings.cloneUnchecked(datatype, dtb, dtb);
440 } else if (datatype instanceof ArrayType) {
441 ArrayType at = (ArrayType) datatype;
442 Datatype ct = at.componentType();
443 Datatype component = stripUnitAnnotations(ct);
444 if (component != ct) {
445 Binding dtb = Bindings.getBindingUnchecked(Datatype.class);
446 datatype = at = (ArrayType) Bindings.cloneUnchecked(datatype, dtb, dtb);
447 at.setComponentType(component);
449 } else if (datatype instanceof RecordType) {
450 RecordType rt = (RecordType) datatype;
451 int componentCount = rt.getComponentCount();
452 Component[] newComponents = new Component[componentCount];
453 for (int i = 0; i < componentCount; ++i) {
454 Component c = rt.getComponent(i);
455 Datatype ct = c.type;
456 Datatype sct = stripUnitAnnotations(ct);
457 newComponents[i] = new Component(c.name, sct);
459 return new RecordType(rt.isReferable(), newComponents);