1 /*******************************************************************************
2 * Copyright (c) 2010, 2016 Association for Decentralized Information Management
4 * All rights reserved. This program and the accompanying materials
5 * are made available under the terms of the Eclipse Public License v1.0
6 * which accompanies this distribution, and is available at
7 * http://www.eclipse.org/legal/epl-v10.html
10 * VTT Technical Research Centre of Finland - initial API and implementation
11 *******************************************************************************/
12 package org.simantics.databoard.util.binary;
14 import java.io.DataInputStream;
15 import java.io.EOFException;
17 import java.io.IOException;
18 import java.io.RandomAccessFile;
19 import java.nio.ByteBuffer;
22 * This class is a Random Access File implementation to RandomAccessBinary.
23 * The implementation is buffered. The implementation is not multi-thread
26 * There is a buffer used for reading and writing. It has a buffer read and
27 * write position. When read, the buffer is filled. When written, the buffer
28 * is written. When the file pointer is moved, the file size changed or flushed
29 * the buffers are cleared. If there were unwritten bytes, they are flushed
32 * There is internal pointer variable. The actual file pointer is moved on
33 * disc read and write operations.
35 * Primitive number writes (int, short, char, double, float and long) and
36 * written in big endian (network) byte order. Use {@link Endian}
37 * to make little endian operations.
39 * @author Toni Kalajainen <toni.kalajainen@vtt.fi>
41 public class BinaryFile implements RandomAccessBinary, BinaryReadable, BinaryWriteable {
46 /** The internal read and write buffer */
49 /** The number of valid bytes in the buffer */
50 int readable_bytes_count;
52 /** The number written but unflushed bytes in buf starting from buf[0]*/
55 /** The pointer of buf[0] in the file. The value is valid only if value >=0 */
58 /** File Pointer of current position */
61 /** Transient size of the file */
65 public BinaryFile(RandomAccessFile file) throws IOException
68 virtualLength = diskLength = raf.length();
73 public BinaryFile(File file) throws IOException
75 this.raf = new RandomAccessFile(file, "rw");
76 virtualLength = diskLength = raf.length();
82 public BinaryFile(File file, String mode) throws IOException
84 this.raf = new RandomAccessFile(file, mode);
85 virtualLength = diskLength = raf.length();
91 public BinaryFile(RandomAccessFile file, int bufSize) throws IOException
94 virtualLength = diskLength = raf.length();
96 buf = new byte[bufSize];
99 public BinaryFile(File file, int bufSize) throws IOException
101 this.raf = new RandomAccessFile(file, "rw");
102 virtualLength = diskLength = raf.length();
105 buf = new byte[bufSize];
108 public static BinaryFile tempFile(long size) throws IOException
110 File tmpFile = File.createTempFile("Temp", ".file");
111 tmpFile.deleteOnExit();
112 BinaryFile file = new BinaryFile(tmpFile);
113 file.setLength( size );
118 * Closes the object. Note, this will close the input random access file.
119 * This method may be called several times.
121 * @throws IOException
123 public synchronized void close() throws IOException {
124 if (raf==null) return;
132 public synchronized boolean isOpen() {
140 public RandomAccessFile getRandomAccessFile() {
145 * Get the number of readable bytes in buffer
147 * @return readable bytes or 0
149 private long readableBytesInBuffer() {
150 long posInBuf = pointer - buf_pos;
151 if (posInBuf<0) return 0;
152 long bytesLeft = readable_bytes_count - posInBuf;
153 return bytesLeft < 0 ? 0 : bytesLeft;
157 * Get the valid position of pointer in buf[]
161 private long positionInReadBuffer() {
162 long posInBuf = pointer - buf_pos;
163 if (posInBuf<0 || posInBuf>readable_bytes_count) return -1;
170 * @throws IOException
172 int _get() throws IOException
176 int posInBuf = (int) (pointer - buf_pos);
177 int result = buf[posInBuf] & 0xFF;
183 * Get next 4 bytes as an int value. This is an optimization.
185 * @throws IOException
187 int _getInt() throws IOException
191 int posInBuf = (int) (pointer - buf_pos);
193 (((int) (buf[posInBuf + 3] & 0xFF)) |
194 (((int) (buf[posInBuf + 2] & 0xFF)) << 8) |
195 (((int) (buf[posInBuf + 1] & 0xFF)) << 16) |
196 (((int) (buf[posInBuf] & 0xFF)) << 24));
202 * Get next 8 bytes as a long value. This is an optimization.
204 * @throws IOException
206 long _getLong() throws IOException
210 int posInBuf = (int) (pointer - buf_pos);
212 (((long) (buf[posInBuf + 7] & 0xFF)) |
213 (((long) (buf[posInBuf + 6] & 0xFF)) << 8) |
214 (((long) (buf[posInBuf + 5] & 0xFF)) << 16) |
215 (((long) (buf[posInBuf + 4] & 0xFF)) << 24) |
216 (((long) (buf[posInBuf + 3] & 0xFF)) << 32) |
217 (((long) (buf[posInBuf + 2] & 0xFF)) << 40) |
218 (((long) (buf[posInBuf + 1] & 0xFF)) << 48) |
219 (((long) (buf[posInBuf] & 0xFF)) << 56));
226 * @return 0..255 or -1 on end of file
227 * @throws IOException
229 int _read() throws IOException
231 if (readableBytesInBuffer()<1) {
233 if (readableBytesInBuffer()==0) return -1;
236 int posInBuf = (int) (pointer - buf_pos);
237 int result = buf[posInBuf] & 0xFF;
242 void _get(byte[] dst, int offset, int length) throws IOException {
244 long n = Math.min( readableBytesInBuffer(), length );
245 int posInBuf = (int) (pointer - buf_pos);
247 System.arraycopy(buf, (int) posInBuf, dst, offset, (int) n);
255 if (readableBytesInBuffer()==0) {
256 throw new EOFException();
263 * Flushes bytes 0..write_buf_count to file at buf_pos.
264 * Sets write_buf_count to 0.
266 * @throws IOException
268 private void writeFlush() throws IOException {
269 if (write_buf_count>0 && buf_pos>=0) {
271 raf.write(buf, 0, write_buf_count);
272 if (buf_pos+write_buf_count>diskLength) diskLength = buf_pos+write_buf_count;
275 if (diskLength != virtualLength) {
276 raf.setLength(virtualLength);
277 diskLength = virtualLength;
282 * This method ensures that write buffer is ready for writing
283 * at the pointer. Old buffer is flushed if necessary.
285 * @param requested number of bytes, up to 4096 guaranteed
286 * @return the number of bytes that can be written to buf
287 * @throws IOException
289 private int prepareForWrite(int bytes) throws IOException {
290 // Ensure pointer is within buf[0..readable_bytes_count]
291 int posInBuf = (int) (pointer - buf_pos);
294 readable_bytes_count = 0;
298 if (posInBuf>readable_bytes_count) {
299 // Pointer is at wrong place, flush data
303 int bytesToKeep = readable_bytes_count - posInBuf;
304 if (bytesToKeep<0) bytesToKeep = 0;
306 System.arraycopy(buf, posInBuf, buf, 0, bytesToKeep);
307 readable_bytes_count = bytesToKeep;
309 readable_bytes_count = 0;
314 // Ensure buf[] has room for bytes
315 if (buf.length - posInBuf < bytes) {
317 int bytesToKeep = readable_bytes_count - posInBuf;
318 if (bytesToKeep<0) bytesToKeep = 0;
320 System.arraycopy(buf, posInBuf, buf, 0, bytesToKeep);
321 readable_bytes_count = bytesToKeep;
323 readable_bytes_count = 0;
330 return (int) ( buf.length - posInBuf );
334 * Assert there is a set number of bytes in read buffer.
335 * bytes should not be larger than read buffer size (buf.length, 4K)
338 * @throws IOException
340 private void assertReadable(int bytes) throws IOException
342 if (readableBytesInBuffer()<bytes) {
344 if (readableBytesInBuffer()<bytes) throw new EOFException();
349 * Read the buffer at file position pointer.
351 * @throws IOException
353 private void fill() throws IOException
357 // Reuse previous cache, if possible
358 long old_buf_start = buf_pos;
359 long old_buf_end = buf_pos + readable_bytes_count;
360 int old_buf_length = readable_bytes_count;
361 long new_buf_start = pointer;
362 long new_buf_end = Math.min(pointer + buf.length, virtualLength);
363 int new_buf_length = (int) (new_buf_end - new_buf_start);
365 boolean old_buf_start_in_new_buf = (old_buf_start >= new_buf_start) && (old_buf_start < new_buf_end);
366 boolean old_buf_end_in_new_buf = (old_buf_end >= new_buf_start) && (old_buf_end < new_buf_end);
368 // Scenario 1, end of old buf in the new, but the whole old buffer doesnt fit in the new
369 if (old_buf_end_in_new_buf && !old_buf_start_in_new_buf)
371 int bytesToPreserve = (int) (old_buf_end - new_buf_start);
372 int bytesToRead = (int) (new_buf_end - old_buf_end);
373 System.arraycopy(buf, old_buf_length - bytesToPreserve, buf, 0, bytesToPreserve);
375 raf.seek(new_buf_start + bytesToPreserve);
376 raf.readFully(buf, bytesToPreserve, bytesToRead);
378 readable_bytes_count = new_buf_length;
382 // Scenario 2, start of old buf is in new, but not completely
383 if (old_buf_start_in_new_buf && !old_buf_end_in_new_buf) {
384 int bytesToPreserve = (int) (new_buf_end-old_buf_start);
385 int bytesToRead = (int) (old_buf_start-new_buf_start);
386 System.arraycopy(buf, 0, buf, bytesToRead, bytesToPreserve);
387 raf.seek(new_buf_start);
388 raf.readFully(buf, 0, bytesToRead);
390 readable_bytes_count = new_buf_length;
394 // Scenario 3, old buf is partially in new buf (Omited)
396 // Scenario 4, old and new buf do not intersect
398 int bytesToRead = new_buf_length;
399 raf.seek(new_buf_start);
400 raf.readFully(buf, 0, bytesToRead);
402 readable_bytes_count = new_buf_length;
407 public byte readByte() throws IOException {
408 return (byte) _get();
412 public char readChar() throws IOException {
413 return (char)((_get() << 8) | _get());
417 public int readUnsignedByte() throws IOException {
418 return _get() & 0x000000ff;
422 public boolean readBoolean() throws IOException {
427 public void readFully(byte[] dst, int offset, int length) throws IOException {
428 _get(dst, offset, length);
432 public void readFully(byte[] dst) throws IOException {
433 _get(dst, 0, dst.length);
437 public void readFully(ByteBuffer buf) throws IOException {
438 readFully(buf, buf.remaining());
442 public void readFully(ByteBuffer buf, int length) throws IOException {
444 assertReadable( Math.min(this.buf.length, length) );
445 long n = Math.min(readableBytesInBuffer(), length);
446 if (n==0) throw new EOFException();
447 long posInBuf = positionInReadBuffer();
448 if (n>0 && posInBuf>=0) {
449 buf.put(this.buf, (int)posInBuf, (int)n);
455 if (readableBytesInBuffer()==0) {
456 throw new EOFException();
463 public double readDouble() throws IOException {
464 return Double.longBitsToDouble(readLong());
468 public float readFloat() throws IOException {
469 return Float.intBitsToFloat(readInt());
473 public int readInt() throws IOException {
483 public long readLong() throws IOException {
485 // ( ((long)_get()) << 56) |
486 // ( ((long)_get()) << 48) |
487 // ( ((long)_get()) << 40) |
488 // ( ((long)_get()) << 32) |
489 // ( ((long)_get()) << 24) |
490 // ( ((long)_get()) << 16) |
491 // ( ((long)_get()) << 8) |
492 // ( ((long)_get()) );
497 public short readShort() throws IOException {
498 return (short) ( (_get() << 8) | _get() ) ;
502 public int readUnsignedShort() throws IOException {
503 return (int) ( (_get() << 8) | _get() ) ;
506 public final String readLine() throws IOException {
507 StringBuffer input = new StringBuffer();
512 switch (c = _read()) {
519 long cur = position();
520 if ((_read()) != '\n') {
525 input.append((char)c);
530 if ((c == -1) && (input.length() == 0)) {
533 return input.toString();
536 public final String readUTF() throws IOException {
537 return DataInputStream.readUTF(this);
541 public long position() {
545 public void position(long newPosition) {
546 pointer = newPosition;
550 * Flushes internal buffer
552 public void flush() throws IOException {
557 * Clears read&write buffer. The file can be modified elsewere after this.
559 * @throws IOException
561 public void reset() throws IOException {
563 readable_bytes_count = 0;
564 virtualLength = diskLength = raf.length();
568 public long skipBytes(long bytes) throws IOException {
574 public int skipBytes(int bytes) throws IOException {
582 void _putLong(long value) throws IOException
585 int posInBuf = (int) (pointer - buf_pos);
586 buf[posInBuf] = (byte) (value >>> 56);
587 buf[posInBuf+1] = (byte) (value >>> 48);
588 buf[posInBuf+2] = (byte) (value >>> 40);
589 buf[posInBuf+3] = (byte) (value >>> 32);
590 buf[posInBuf+4] = (byte) (value >>> 24);
591 buf[posInBuf+5] = (byte) (value >>> 16);
592 buf[posInBuf+6] = (byte) (value >>> 8);
593 buf[posInBuf+7] = (byte) value;
596 if (write_buf_count<posInBuf) write_buf_count = posInBuf;
597 if (readable_bytes_count<write_buf_count) readable_bytes_count=write_buf_count;
598 if (virtualLength<pointer) virtualLength=pointer;
601 void _putInt(int value) throws IOException
604 int posInBuf = (int) (pointer - buf_pos);
605 buf[posInBuf] = (byte) (value >>> 24);
606 buf[posInBuf+1] = (byte) (value >>> 16);
607 buf[posInBuf+2] = (byte) (value >>> 8);
608 buf[posInBuf+3] = (byte) value;
611 if (write_buf_count<posInBuf) write_buf_count = posInBuf;
612 if (readable_bytes_count<write_buf_count) readable_bytes_count=write_buf_count;
613 if (virtualLength<pointer) virtualLength=pointer;
616 void _put(int value) throws IOException
619 int posInBuf = (int) (pointer - buf_pos);
620 buf[posInBuf] = (byte) value;
623 if (write_buf_count<posInBuf) write_buf_count = posInBuf;
624 if (readable_bytes_count<write_buf_count) readable_bytes_count=write_buf_count;
625 if (virtualLength<pointer) virtualLength=pointer;
628 void _put(byte[] src, int offset, int length) throws IOException {
630 int n = Math.min(prepareForWrite(length), length);
631 int posInBuf = (int) (pointer - buf_pos);
632 System.arraycopy(src, offset, buf, posInBuf, n);
637 if (write_buf_count<posInBuf) write_buf_count = posInBuf;
638 if (readable_bytes_count<write_buf_count) readable_bytes_count=write_buf_count;
639 if (virtualLength<pointer) virtualLength=pointer;
644 public void write(int b) throws IOException {
649 public void writeByte(int b) throws IOException {
654 public void writeBoolean(boolean v) throws IOException {
659 public void writeFully(ByteBuffer src) throws IOException {
660 if (src.hasArray()) {
661 byte array[] = src.array();
662 _put(array, src.position(), src.remaining());
663 src.position(src.limit());
665 for (;src.hasRemaining();)
670 public void writeFully(ByteBuffer src, int length) throws IOException {
671 if (src.hasArray()) {
672 byte array[] = src.array();
673 _put(array, src.position(), length);
674 src.position(length);
676 for (int i=0; i<length; i++)
682 public void write(byte[] src, int offset, int length) throws IOException {
683 _put(src, offset, length);
687 public void write(byte[] src) throws IOException {
688 _put(src, 0, src.length);
692 public void writeDouble(double value) throws IOException {
693 writeLong(Double.doubleToLongBits(value));
697 public void writeFloat(float value) throws IOException {
698 writeInt(Float.floatToIntBits(value));
702 public void writeInt(int value) throws IOException {
707 public void writeLong(long value) throws IOException {
712 public void writeShort(int value) throws IOException {
718 public void writeChar(int value) throws IOException {
724 public void writeBytes(String s) throws IOException {
725 int len = s.length();
726 for (int i = 0 ; i < len ; i++) {
727 _put((byte)s.charAt(i));
732 public void writeChars(String s) throws IOException {
733 int len = s.length();
734 for (int i = 0 ; i < len ; i++) {
736 _put((v >>> 8) & 0xFF);
737 _put((v >>> 0) & 0xFF);
742 public void writeUTF(String s) throws IOException {
743 int len = UTF8.getModifiedUTF8EncodingByteLength(s);
745 UTF8.writeModifiedUTF(this, s);
749 public void insertBytes(long bytes, ByteSide side) throws IOException {
750 if (pointer>=virtualLength) {
751 setLength(pointer + bytes);
755 // insertion to buffer window
756 if (pointer>=buf_pos && (pointer<=buf_pos+readable_bytes_count))
758 // buffer window convers the end of the file & there is enough space in buffer to do the operation
759 if (buf_pos+readable_bytes_count >= virtualLength && readable_bytes_count + bytes < buf.length) {
761 int posInBuf = (int) (pointer - buf_pos);
762 System.arraycopy(buf, posInBuf, buf, (int) (posInBuf+bytes), (int) readable_bytes_count - posInBuf);
763 readable_bytes_count += bytes;
764 write_buf_count = readable_bytes_count;
765 virtualLength += bytes;
775 insertBytes(raf, pointer, bytes);
776 virtualLength += bytes;
780 if (buf_pos>pointer) {
786 public void removeBytes(long bytes, ByteSide side) throws IOException {
787 if (pointer+bytes>virtualLength || pointer<0) {
788 throw new IOException("Pointer outside file");
791 if (pointer+bytes==virtualLength) {
792 setLength(virtualLength - bytes);
796 // removal intersects buffer window
797 if (pointer+bytes>=buf_pos && (pointer<=buf_pos+readable_bytes_count))
799 // buffer window covers the ending
800 if (buf_pos+readable_bytes_count >= virtualLength) {
802 // Scenario 1 : Pointer before buffer
803 if (pointer<buf_pos) {
804 int cut_end_InBuf = (int) (pointer+bytes - buf_pos);
805 System.arraycopy(buf, cut_end_InBuf, buf, 0, readable_bytes_count-cut_end_InBuf);
806 readable_bytes_count -= cut_end_InBuf;
807 write_buf_count = readable_bytes_count;
808 virtualLength -= bytes;
813 // Scenario 2 : Pointer within buffer
814 if (pointer>=buf_pos) {
815 int posInBuf = (int) (pointer - buf_pos);
816 System.arraycopy(buf, (int) (posInBuf+bytes), buf, posInBuf, (int) (readable_bytes_count - posInBuf - bytes) );
817 readable_bytes_count -= bytes;
818 write_buf_count = readable_bytes_count;
819 virtualLength -= bytes;
830 removeBytes(raf, pointer, bytes);
831 virtualLength -= bytes;
835 if (buf_pos>pointer) {
841 public long length() throws IOException {
842 return virtualLength;
846 public void setLength(long newLength) throws IOException {
847 virtualLength = newLength;
848 if (buf_pos + readable_bytes_count > virtualLength) {
849 readable_bytes_count = (int) Math.max(virtualLength - buf_pos, 0L);
851 if (buf_pos + write_buf_count > virtualLength) {
852 write_buf_count = (int) Math.max(virtualLength - buf_pos, 0L);
856 private final static int bufferSize = 1024*32;
859 * Inserts bytes into a middle of a file.
864 * @throws IOException
866 public static void insertBytes(RandomAccessFile file, long position, long bytes)
869 if (position<0) throw new IndexOutOfBoundsException("position cannot be below 0");
870 if (bytes<0) throw new IndexOutOfBoundsException("bytes cannot be below 0");
871 if (bytes==0) return;
872 long length = file.length();
873 if (position>=length) {
874 file.setLength(position+bytes);
878 long bytesOnRight = length - position;
880 // Create buffer - the buffer is cyclic window
881 int bufLength = (int) Math.min(bytesOnRight, bufferSize);
882 byte[] buf = new byte[bufLength];
884 // Bytes transferred from right
886 while (n<bytesOnRight) {
887 // Bytes that have content
888 int count = (int) Math.min(bufLength, bytesOnRight-n);
889 file.seek( length-count-n );
890 file.readFully(buf, 0, count);
891 file.seek( length-count-n+bytes );
892 file.write(buf, 0, count);
898 * Remove bytes from a file
903 * @throws IOException
905 public static void removeBytes(RandomAccessFile file, long position, long bytes) throws IOException
907 if (position<0) throw new IndexOutOfBoundsException("position cannot be below 0");
908 if (bytes<0) throw new IndexOutOfBoundsException("bytes cannot be below 0");
909 if (bytes==0) return;
910 long length = file.length();
911 if (position>=length) return;
913 if (position+bytes>=length) {
914 file.setLength(position);
917 long bytesOnRight = length - position - bytes;
918 int bufLength = (int) Math.min(bytesOnRight, bufferSize);
919 byte[] buf = new byte[bufLength];
922 while (n<bytesOnRight) {
923 int count = (int) Math.min(bufLength, bytesOnRight-n);
924 file.seek( position + bytes + n );
925 file.readFully( buf, 0, count );
926 file.seek( position + n );
927 file.write(buf, 0, count);
930 file.setLength(position + bytesOnRight);
934 public String toString() {
936 return "File(file="+file.getName()+", size="+length()+")";
937 } catch (IOException e) {