1 /*******************************************************************************
\r
2 * Copyright (c) 2007, 2011 Association for Decentralized Information Management in
\r
4 * All rights reserved. This program and the accompanying materials
\r
5 * are made available under the terms of the Eclipse Public License v1.0
\r
6 * which accompanies this distribution, and is available at
\r
7 * http://www.eclipse.org/legal/epl-v10.html
\r
10 * VTT Technical Research Centre of Finland - initial API and implementation
\r
11 *******************************************************************************/
\r
12 package org.simantics.history.impl;
\r
14 import java.io.DataInput;
\r
15 import java.io.DataOutput;
\r
16 import java.io.File;
\r
17 import java.io.FilenameFilter;
\r
18 import java.io.IOException;
\r
19 import java.nio.BufferUnderflowException;
\r
20 import java.nio.ByteBuffer;
\r
21 import java.util.ArrayList;
\r
22 import java.util.List;
\r
23 import java.util.logging.Logger;
\r
25 import org.simantics.databoard.Accessors;
\r
26 import org.simantics.databoard.Bindings;
\r
27 import org.simantics.databoard.accessor.ArrayAccessor;
\r
28 import org.simantics.databoard.accessor.StreamAccessor;
\r
29 import org.simantics.databoard.accessor.error.AccessorConstructionException;
\r
30 import org.simantics.databoard.accessor.error.AccessorException;
\r
31 import org.simantics.databoard.adapter.AdaptException;
\r
32 import org.simantics.databoard.adapter.Adapter;
\r
33 import org.simantics.databoard.adapter.AdapterConstructionException;
\r
34 import org.simantics.databoard.binding.Binding;
\r
35 import org.simantics.databoard.binding.error.BindingException;
\r
36 import org.simantics.databoard.binding.error.RuntimeBindingConstructionException;
\r
37 import org.simantics.databoard.serialization.Serializer;
\r
38 import org.simantics.databoard.serialization.SerializerConstructionException;
\r
39 import org.simantics.databoard.type.ArrayType;
\r
40 import org.simantics.databoard.type.Component;
\r
41 import org.simantics.databoard.type.Datatype;
\r
42 import org.simantics.databoard.type.NumberType;
\r
43 import org.simantics.databoard.type.RecordType;
\r
44 import org.simantics.databoard.util.Bean;
\r
45 import org.simantics.databoard.util.ObjectUtils;
\r
46 import org.simantics.databoard.util.URIUtil;
\r
47 import org.simantics.databoard.util.binary.BinaryMemory;
\r
48 import org.simantics.databoard.util.binary.ByteBufferWriteable;
\r
49 import org.simantics.history.HistoryException;
\r
50 import org.simantics.history.HistoryManager;
\r
51 import org.simantics.utils.FileUtils;
\r
54 * File history uses workarea (directory) to manage items.
\r
55 * There are two files for every item: stream file and metadata file.
\r
56 * Metadata file is JSON ascii file.
\r
60 * @author toni.kalajainen
\r
63 public class FileHistory implements HistoryManager {
\r
65 private static final boolean PROFILE = false;
\r
66 private static final boolean DEBUG = false;
\r
69 static Logger logger = Logger.getLogger(FileHistory.class.getName());
\r
71 static FilenameFilter txtFilter;
\r
73 static ArrayType INDEX_TYPE = Bindings.LONG_ARRAY.type();
\r
77 /** Support async usage of the data */
\r
78 public boolean asyncUsage = true;
\r
80 public FileHistory(File workarea) {
\r
81 this.workarea = workarea;
\r
84 public File getWorkarea() {
\r
89 public void create(Bean... items) throws HistoryException {
\r
92 for (Bean item : items) {
\r
94 System.out.println("create(" + item + ")");
\r
95 // if ( !item.getFieldBinding("format").type().equals( Bindings.getBindingUnchecked(Datatype.class).type() ) )
\r
96 // System.err.println("Error");
\r
99 writeMetadata( item );
\r
101 // Create stream file
\r
102 String id = (String) item.getField("id");
\r
103 File dataFile = toDataFile( id );
\r
104 dataFile.createNewFile();
\r
106 Datatype type = (Datatype) item.getField("format");
\r
107 if ( isVariableWidth(type) ) {
\r
108 File indexFile = toIndexFile( id );
\r
109 indexFile.createNewFile();
\r
111 // if ( !dataFile.createNewFile() ) {
\r
112 // throw new HistoryException("Could not create file "+dataFile);
\r
115 } catch (BindingException e) {
\r
116 throw new HistoryException(e);
\r
117 } catch (IOException e) {
\r
118 throw new HistoryException(e);
\r
123 public void delete(String... itemIds) throws HistoryException {
\r
124 for (String itemId : itemIds ) {
\r
125 File meta = toMetaFile( itemId );
\r
126 File data = toDataFile( itemId );
\r
127 File index = toIndexFile( itemId );
\r
128 if ( meta.exists() ) {
\r
129 if ( !meta.delete() ) {
\r
130 throw new HistoryException("Failed to delete "+meta);
\r
133 if ( data.exists() ) {
\r
134 if ( !data.delete() ) {
\r
135 throw new HistoryException("Failed to delete "+data);
\r
138 if ( index.exists() ) {
\r
139 if ( !index.delete() );
\r
145 public void modify(Bean... items) throws HistoryException {
\r
148 for ( Bean item : items ) {
\r
150 System.out.println("modify(" + item + ")");
\r
151 // if ( !item.getFieldBinding("format").type().equals( Bindings.getBindingUnchecked(Datatype.class).type() ) )
\r
152 // System.err.println("Error");
\r
154 String id = (String) item.getField("id");
\r
155 File metaFile = toMetaFile( id );
\r
156 if ( !metaFile.exists() ) {
\r
159 Bean oldItem = getItem( id );
\r
160 File dataFile = toDataFile( id );
\r
161 if ( dataFile.exists() ) {
\r
162 boolean enabled = item.hasField("enabled") ? (Boolean) item.getFieldUnchecked("enabled") : true;
\r
163 Datatype oldFormat = (Datatype) oldItem.getField( "format" );
\r
164 Datatype newFormat = (Datatype) item.getField( "format" );
\r
166 System.out.println("formats: " + oldFormat + " -> " + newFormat);
\r
167 Datatype unitStrippedOldFormat = stripUnitAnnotations(oldFormat);
\r
168 Datatype unitStrippedNewFormat = stripUnitAnnotations(newFormat);
\r
170 System.out.println("formats after unit strip: " + unitStrippedOldFormat + " -> " + unitStrippedNewFormat);
\r
171 if ( enabled && !unitStrippedOldFormat.equals(unitStrippedNewFormat) ) {
\r
173 Binding oldBinding = Bindings.getBeanBinding(unitStrippedOldFormat);
\r
174 Binding newBinding = Bindings.getBeanBinding(unitStrippedNewFormat);
\r
175 Serializer oldS = Bindings.getSerializer(oldBinding);
\r
176 Serializer newS = Bindings.getSerializer(newBinding);
\r
177 if (oldS.getConstantSize()==null || newS.getConstantSize()==null || oldS.getConstantSize()!=newS.getConstantSize())
\r
178 throw new HistoryException("Changing of file format is not supported to: "+dataFile);
\r
179 Adapter adapter = Bindings.getAdapter(oldBinding, newBinding);
\r
180 Object oldSample = oldBinding.createDefault();
\r
181 Object newSample = newBinding.createDefault();
\r
182 StreamAccessor sa = openStream(id, "rw");
\r
185 for (int i=0; i<c; i++) {
\r
186 sa.get(i, oldBinding, oldSample);
\r
187 newSample = adapter.adapt(oldSample);
\r
188 sa.set(i, newBinding, newSample);
\r
193 } catch (AdapterConstructionException e) {
\r
194 throw new HistoryException("Changing of file format is not supported to: "+id);
\r
195 } catch (SerializerConstructionException e) {
\r
196 throw new HistoryException("Changing of file format is not supported to: "+id);
\r
197 } catch (AccessorException e) {
\r
198 throw new HistoryException("Changing of file format failed to: "+id);
\r
199 } catch (AdaptException e) {
\r
200 throw new HistoryException("Changing of file format failed to: "+id);
\r
204 dataFile.createNewFile();
\r
207 // Write new meta-data if necessary
\r
208 if (!equalsWithoutState(item, oldItem))
\r
209 writeMetadata( item );
\r
212 } catch (BindingException e) {
\r
213 throw new HistoryException( e );
\r
214 } catch (IOException e) {
\r
215 throw new HistoryException( e );
\r
220 public Bean getItem(String itemId) throws HistoryException {
\r
221 return getItem( toMetaFile( itemId ) );
\r
224 void writeMetadata( Bean item ) throws HistoryException {
\r
226 // long s = System.nanoTime();
\r
228 String id = (String) item.getField("id");
\r
229 String idEnc = URIUtil.encodeURI(id);
\r
230 File metaFile = new File(workarea, idEnc + ".txt");
\r
231 Serializer typeSerializer = Bindings.getSerializer( Bindings.getBindingUnchecked(Datatype.class) );
\r
232 Serializer beanSerializer = Bindings.getSerializer( item.getBinding() );
\r
233 int size = typeSerializer.getSize(item.getBinding().type()) +
\r
234 beanSerializer.getSize(item);
\r
235 byte data[] = new byte[size];
\r
236 DataOutput out = new ByteBufferWriteable( ByteBuffer.wrap(data) );
\r
238 if ( metaFile.exists() && asyncUsage ) {
\r
240 System.out.println("WARNING: FileHistory.writeMetadata: on SLOW path for " + item);
\r
241 File tmpFile = new File(workarea, idEnc + ".tmp");
\r
242 File tmp2File = new File(workarea, idEnc + ".tmp2");
\r
245 typeSerializer.serialize(out, item.getBinding().type());
\r
246 beanSerializer.serialize(out, item);
\r
247 FileUtils.writeFile(tmpFile, data);
\r
248 metaFile.renameTo(tmp2File);
\r
249 tmpFile.renameTo(metaFile);
\r
252 typeSerializer.serialize(out, item.getBinding().type());
\r
253 beanSerializer.serialize(out, item);
\r
254 FileUtils.writeFile(metaFile, data);
\r
258 // System.out.println("PROFILE: FileHistory.writeMetadata( " + metaFile.getName() + " ) in " + ((System.nanoTime() - s)*1e-6) + " ms");
\r
259 } catch (BindingException e) {
\r
260 throw new HistoryException(e);
\r
261 } catch (IOException e) {
\r
262 throw new HistoryException(e);
\r
263 } catch (SerializerConstructionException e) {
\r
264 throw new HistoryException(e);
\r
268 Bean getItem(File file) throws HistoryException {
\r
269 // FileInputStream fis;
\r
271 // fis = new FileInputStream(file);
\r
272 // } catch (FileNotFoundException e1) {
\r
273 // throw new HistoryException(e1);
\r
276 byte[] data = FileUtils.readFile(file);
\r
277 DataInput in = new BinaryMemory(data);
\r
278 Serializer typeSerializer = Bindings.getSerializer( Bindings.getBindingUnchecked(Datatype.class) );
\r
279 Datatype type = (Datatype) typeSerializer.deserialize(in);
\r
280 Binding beanBinding = Bindings.getBeanBinding( type );
\r
281 Serializer s = Bindings.getSerializer( beanBinding );
\r
282 Bean bean = (Bean) s.deserialize(in);
\r
284 DataInput in = new InputStreamReadable( fis, file.length() );
\r
285 Serializer typeSerializer = Bindings.getSerializer( Bindings.getBindingUnchecked(Datatype.class) );
\r
286 Datatype type = (Datatype) typeSerializer.deserialize(in);
\r
287 Binding beanBinding = Bindings.getBeanBinding( type );
\r
288 Serializer s = Bindings.getSerializer( beanBinding );
\r
289 Bean bean = (Bean) s.deserialize(in);
\r
292 /* String txt = new String(data, UTF8.CHARSET);
\r
293 DataValueRepository repo = new DataValueRepository();
\r
294 String name = repo.addValueDefinition(txt);
\r
295 MutableVariant value = repo.get(name);
\r
296 Binding beanBinding = Bindings.getBeanBinding( value.type() );
\r
297 return (Bean) value.getValue(beanBinding);*/
\r
298 } catch(BufferUnderflowException e) {
\r
299 throw new HistoryException( e );
\r
300 } catch(IOException e) {
\r
301 throw new HistoryException( e );
\r
302 // } catch (DataTypeSyntaxError e) {
\r
303 // throw new HistoryException( e );
\r
304 } catch (RuntimeBindingConstructionException e) {
\r
305 throw new HistoryException( e );
\r
306 } catch (SerializerConstructionException e) {
\r
307 throw new HistoryException( e );
\r
311 // } catch (IOException e) {
\r
316 File toMetaFile(String itemId)
\r
318 String name = URIUtil.encodeURI(itemId) + ".txt";
\r
319 File f = new File(workarea, name);
\r
323 File toDataFile(String itemId)
\r
325 String name = URIUtil.encodeURI(itemId) + ".data";
\r
326 File f = new File(workarea, name);
\r
330 File toIndexFile(String itemId)
\r
332 String name = URIUtil.encodeURI(itemId) + ".index";
\r
333 File f = new File(workarea, name);
\r
337 boolean isVariableWidth(Datatype type)
\r
338 throws HistoryException {
\r
340 Binding beanBinding = Bindings.getBeanBinding(type);
\r
341 Serializer s = Bindings.getSerializer( beanBinding );
\r
342 return s.getConstantSize() == null;
\r
343 } catch (SerializerConstructionException e) {
\r
344 throw new HistoryException(e);
\r
349 public Bean[] getItems() throws HistoryException {
\r
350 List<Bean> result = new ArrayList<Bean>();
\r
351 File[] files = workarea.listFiles(txtFilter);
\r
352 if ( files != null ) {
\r
353 for (File file : files) {
\r
354 result.add( getItem(file) );
\r
357 return result.toArray( new Bean[ result.size() ] );
\r
361 public void close() {
\r
366 public StreamAccessor openStream(String itemId, String mode) throws HistoryException {
\r
368 Bean bean = getItem(itemId);
\r
369 Datatype format = (Datatype) bean.getField("format");
\r
370 ArrayType arrayType = new ArrayType(format);
\r
371 File dataFile = toDataFile( itemId );
\r
372 if ( isVariableWidth(format) ) {
\r
373 File indexFile = toIndexFile( itemId );
\r
374 ArrayAccessor index = Accessors.openStream(indexFile, INDEX_TYPE, mode);
\r
375 return (StreamAccessor) Accessors.openStream(dataFile, arrayType, mode, index);
\r
377 return (StreamAccessor) Accessors.openStream(dataFile, arrayType, mode);
\r
379 } catch (AccessorConstructionException e) {
\r
380 throw new HistoryException(e);
\r
381 } catch (BindingException e) {
\r
382 throw new HistoryException(e);
\r
387 public boolean exists(String itemId) throws HistoryException {
\r
388 return toMetaFile(itemId).exists();
\r
392 public int hashCode() {
\r
393 return workarea.hashCode();
\r
397 public boolean equals(Object obj) {
\r
398 if ( obj==null ) return false;
\r
399 if ( obj instanceof FileHistory == false ) return false;
\r
400 FileHistory other = (FileHistory) obj;
\r
401 return other.workarea.equals(workarea);
\r
405 public String toString() {
\r
406 return "FileHistory: "+workarea;
\r
409 private boolean equalsWithoutState(Bean i1, Bean i2) {
\r
410 Component[] components1 = i1.getBinding().type().getComponents();
\r
411 Component[] components2 = i2.getBinding().type().getComponents();
\r
412 int components = Math.min(components1.length, components2.length);
\r
413 for (int c = 0; c < components; ++c) {
\r
414 Object o1 = i1.getFieldUnchecked(c);
\r
415 Object o2 = i2.getFieldUnchecked(c);
\r
416 if ("collectorState".equals(components1[c].name) && (o1 == null || o2 == null))
\r
418 if (!ObjectUtils.objectEquals(o1, o2))
\r
425 txtFilter = new FilenameFilter() {
\r
426 public boolean accept(File dir, String name) {
\r
427 return name.toLowerCase().endsWith(".txt");
\r
432 private static Datatype stripUnitAnnotations(Datatype datatype) {
\r
433 if (datatype instanceof NumberType) {
\r
434 NumberType nt = (NumberType) datatype;
\r
435 if (nt.getUnit() != null) {
\r
436 Binding dtb = Bindings.getBindingUnchecked(Datatype.class);
\r
437 datatype = nt = (NumberType) Bindings.cloneUnchecked(datatype, dtb, dtb);
\r
440 } else if (datatype instanceof ArrayType) {
\r
441 ArrayType at = (ArrayType) datatype;
\r
442 Datatype ct = at.componentType();
\r
443 Datatype component = stripUnitAnnotations(ct);
\r
444 if (component != ct) {
\r
445 Binding dtb = Bindings.getBindingUnchecked(Datatype.class);
\r
446 datatype = at = (ArrayType) Bindings.cloneUnchecked(datatype, dtb, dtb);
\r
447 at.setComponentType(component);
\r
449 } else if (datatype instanceof RecordType) {
\r
450 RecordType rt = (RecordType) datatype;
\r
451 int componentCount = rt.getComponentCount();
\r
452 Component[] newComponents = new Component[componentCount];
\r
453 for (int i = 0; i < componentCount; ++i) {
\r
454 Component c = rt.getComponent(i);
\r
455 Datatype ct = c.type;
\r
456 Datatype sct = stripUnitAnnotations(ct);
\r
457 newComponents[i] = new Component(c.name, sct);
\r
459 return new RecordType(rt.isReferable(), newComponents);
\r