/******************************************************************************* * Copyright (c) 2010, 2016 Association for Decentralized Information Management * in Industry THTH ry. * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Public License v1.0 * which accompanies this distribution, and is available at * http://www.eclipse.org/legal/epl-v10.html * * Contributors: * VTT Technical Research Centre of Finland - initial API and implementation *******************************************************************************/ package org.simantics.databoard.util.binary; import java.io.DataInputStream; import java.io.EOFException; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; import java.nio.ByteBuffer; /** * This class is a Random Access File implementation to RandomAccessBinary. * The implementation is buffered. The implementation is not multi-thread * safe. *

* There is a buffer used for reading and writing. It has a buffer read and * write position. When read, the buffer is filled. When written, the buffer * is written. When the file pointer is moved, the file size changed or flushed * the buffers are cleared. If there were unwritten bytes, they are flushed * to disc. *

* There is internal pointer variable. The actual file pointer is moved on * disc read and write operations. *

* Primitive number writes (int, short, char, double, float and long) and * written in big endian (network) byte order. Use {@link Endian} * to make little endian operations. * * @author Toni Kalajainen */ public class BinaryFile implements RandomAccessBinary, BinaryReadable, BinaryWriteable { RandomAccessFile raf; File file; /** The internal read and write buffer */ byte buf[]; /** The number of valid bytes in the buffer */ int readable_bytes_count; /** The number written but unflushed bytes in buf starting from buf[0]*/ int write_buf_count; /** The pointer of buf[0] in the file. The value is valid only if value >=0 */ long buf_pos; /** File Pointer of current position */ long pointer; /** Transient size of the file */ long virtualLength; long diskLength; public BinaryFile(RandomAccessFile file) throws IOException { this.raf = file; virtualLength = diskLength = raf.length(); pointer = 0; buf = new byte[4096]; } public BinaryFile(File file) throws IOException { this.raf = new RandomAccessFile(file, "rw"); virtualLength = diskLength = raf.length(); this.file = file; pointer = 0; buf = new byte[4096]; } public BinaryFile(File file, String mode) throws IOException { this.raf = new RandomAccessFile(file, mode); virtualLength = diskLength = raf.length(); this.file = file; pointer = 0; buf = new byte[4096]; } public BinaryFile(RandomAccessFile file, int bufSize) throws IOException { this.raf = file; virtualLength = diskLength = raf.length(); pointer = 0; buf = new byte[bufSize]; } public BinaryFile(File file, int bufSize) throws IOException { this.raf = new RandomAccessFile(file, "rw"); virtualLength = diskLength = raf.length(); this.file = file; pointer = 0; buf = new byte[bufSize]; } public static BinaryFile tempFile(long size) throws IOException { File tmpFile = File.createTempFile("Temp", ".file"); tmpFile.deleteOnExit(); BinaryFile file = new BinaryFile(tmpFile); file.setLength( size ); return file; } /** * Closes the object. Note, this will close the input random access file. * This method may be called several times. * * @throws IOException */ public synchronized void close() throws IOException { if (raf==null) return; flush(); pointer = -1; raf.close(); raf = null; buf = null; } public synchronized boolean isOpen() { return buf!=null; } public File file() { return file; } public RandomAccessFile getRandomAccessFile() { return raf; } /** * Get the number of readable bytes in buffer * * @return readable bytes or 0 */ private long readableBytesInBuffer() { long posInBuf = pointer - buf_pos; if (posInBuf<0) return 0; long bytesLeft = readable_bytes_count - posInBuf; return bytesLeft < 0 ? 0 : bytesLeft; } /** * Get the valid position of pointer in buf[] * * @return pos or -1 */ private long positionInReadBuffer() { long posInBuf = pointer - buf_pos; if (posInBuf<0 || posInBuf>readable_bytes_count) return -1; return posInBuf; } /** * Get next byte * @return 0..255 * @throws IOException */ int _get() throws IOException { assertReadable(1); int posInBuf = (int) (pointer - buf_pos); int result = buf[posInBuf] & 0xFF; pointer++; return result; } /** * Get next 4 bytes as an int value. This is an optimization. * @return long value * @throws IOException */ int _getInt() throws IOException { assertReadable(4); int posInBuf = (int) (pointer - buf_pos); int result = (((int) (buf[posInBuf + 3] & 0xFF)) | (((int) (buf[posInBuf + 2] & 0xFF)) << 8) | (((int) (buf[posInBuf + 1] & 0xFF)) << 16) | (((int) (buf[posInBuf] & 0xFF)) << 24)); pointer+=4; return result; } /** * Get next 8 bytes as a long value. This is an optimization. * @return long value * @throws IOException */ long _getLong() throws IOException { assertReadable(8); int posInBuf = (int) (pointer - buf_pos); long result = (((long) (buf[posInBuf + 7] & 0xFF)) | (((long) (buf[posInBuf + 6] & 0xFF)) << 8) | (((long) (buf[posInBuf + 5] & 0xFF)) << 16) | (((long) (buf[posInBuf + 4] & 0xFF)) << 24) | (((long) (buf[posInBuf + 3] & 0xFF)) << 32) | (((long) (buf[posInBuf + 2] & 0xFF)) << 40) | (((long) (buf[posInBuf + 1] & 0xFF)) << 48) | (((long) (buf[posInBuf] & 0xFF)) << 56)); pointer+=8; return result; } /** * Get next byte * @return 0..255 or -1 on end of file * @throws IOException */ int _read() throws IOException { if (readableBytesInBuffer()<1) { fill(); if (readableBytesInBuffer()==0) return -1; } int posInBuf = (int) (pointer - buf_pos); int result = buf[posInBuf] & 0xFF; pointer++; return result; } void _get(byte[] dst, int offset, int length) throws IOException { while (length>0) { long n = Math.min( readableBytesInBuffer(), length ); int posInBuf = (int) (pointer - buf_pos); if (n>0) { System.arraycopy(buf, (int) posInBuf, dst, offset, (int) n); offset += n; length -= n; pointer += n; } if (length>0) { fill(); if (readableBytesInBuffer()==0) { throw new EOFException(); } } } } /** * Flushes bytes 0..write_buf_count to file at buf_pos. * Sets write_buf_count to 0. * * @throws IOException */ private void writeFlush() throws IOException { if (write_buf_count>0 && buf_pos>=0) { raf.seek(buf_pos); raf.write(buf, 0, write_buf_count); if (buf_pos+write_buf_count>diskLength) diskLength = buf_pos+write_buf_count; write_buf_count = 0; } if (diskLength != virtualLength) { raf.setLength(virtualLength); diskLength = virtualLength; } } /** * This method ensures that write buffer is ready for writing * at the pointer. Old buffer is flushed if necessary. * * @param requested number of bytes, up to 4096 guaranteed * @return the number of bytes that can be written to buf * @throws IOException */ private int prepareForWrite(int bytes) throws IOException { // Ensure pointer is within buf[0..readable_bytes_count] int posInBuf = (int) (pointer - buf_pos); if (posInBuf<0) { writeFlush(); readable_bytes_count = 0; buf_pos = pointer; posInBuf = 0; } else if (posInBuf>readable_bytes_count) { // Pointer is at wrong place, flush data writeFlush(); // Move pointer int bytesToKeep = readable_bytes_count - posInBuf; if (bytesToKeep<0) bytesToKeep = 0; if (bytesToKeep>0) { System.arraycopy(buf, posInBuf, buf, 0, bytesToKeep); readable_bytes_count = bytesToKeep; } else { readable_bytes_count = 0; } buf_pos = pointer; posInBuf = 0; } // Ensure buf[] has room for bytes if (buf.length - posInBuf < bytes) { writeFlush(); int bytesToKeep = readable_bytes_count - posInBuf; if (bytesToKeep<0) bytesToKeep = 0; if (bytesToKeep>0) { System.arraycopy(buf, posInBuf, buf, 0, bytesToKeep); readable_bytes_count = bytesToKeep; } else { readable_bytes_count = 0; } buf_pos = pointer; posInBuf = 0; return buf.length; } return (int) ( buf.length - posInBuf ); } /** * Assert there is a set number of bytes in read buffer. * bytes should not be larger than read buffer size (buf.length, 4K) * * @param bytes * @throws IOException */ private void assertReadable(int bytes) throws IOException { if (readableBytesInBuffer()= new_buf_start) && (old_buf_start < new_buf_end); boolean old_buf_end_in_new_buf = (old_buf_end >= new_buf_start) && (old_buf_end < new_buf_end); // Scenario 1, end of old buf in the new, but the whole old buffer doesnt fit in the new if (old_buf_end_in_new_buf && !old_buf_start_in_new_buf) { int bytesToPreserve = (int) (old_buf_end - new_buf_start); int bytesToRead = (int) (new_buf_end - old_buf_end); System.arraycopy(buf, old_buf_length - bytesToPreserve, buf, 0, bytesToPreserve); // Read more raf.seek(new_buf_start + bytesToPreserve); raf.readFully(buf, bytesToPreserve, bytesToRead); buf_pos = pointer; readable_bytes_count = new_buf_length; return; } // Scenario 2, start of old buf is in new, but not completely if (old_buf_start_in_new_buf && !old_buf_end_in_new_buf) { int bytesToPreserve = (int) (new_buf_end-old_buf_start); int bytesToRead = (int) (old_buf_start-new_buf_start); System.arraycopy(buf, 0, buf, bytesToRead, bytesToPreserve); raf.seek(new_buf_start); raf.readFully(buf, 0, bytesToRead); buf_pos = pointer; readable_bytes_count = new_buf_length; return; } // Scenario 3, old buf is partially in new buf (Omited) // Scenario 4, old and new buf do not intersect { int bytesToRead = new_buf_length; raf.seek(new_buf_start); raf.readFully(buf, 0, bytesToRead); buf_pos = pointer; readable_bytes_count = new_buf_length; } } @Override public byte readByte() throws IOException { return (byte) _get(); } @Override public char readChar() throws IOException { return (char)((_get() << 8) | _get()); } @Override public int readUnsignedByte() throws IOException { return _get() & 0x000000ff; } @Override public boolean readBoolean() throws IOException { return _get()!=0; } @Override public void readFully(byte[] dst, int offset, int length) throws IOException { _get(dst, offset, length); } @Override public void readFully(byte[] dst) throws IOException { _get(dst, 0, dst.length); } @Override public void readFully(ByteBuffer buf) throws IOException { readFully(buf, buf.remaining()); } @Override public void readFully(ByteBuffer buf, int length) throws IOException { while (length>0) { assertReadable( Math.min(this.buf.length, length) ); long n = Math.min(readableBytesInBuffer(), length); if (n==0) throw new EOFException(); long posInBuf = positionInReadBuffer(); if (n>0 && posInBuf>=0) { buf.put(this.buf, (int)posInBuf, (int)n); length -= n; pointer += n; } if (length>0) { fill(); if (readableBytesInBuffer()==0) { throw new EOFException(); } } } } @Override public double readDouble() throws IOException { return Double.longBitsToDouble(readLong()); } @Override public float readFloat() throws IOException { return Float.intBitsToFloat(readInt()); } @Override public int readInt() throws IOException { return // ( _get() << 24) | // ( _get() << 16) | // ( _get() << 8) | // ( _get() ); _getInt(); } @Override public long readLong() throws IOException { return // ( ((long)_get()) << 56) | // ( ((long)_get()) << 48) | // ( ((long)_get()) << 40) | // ( ((long)_get()) << 32) | // ( ((long)_get()) << 24) | // ( ((long)_get()) << 16) | // ( ((long)_get()) << 8) | // ( ((long)_get()) ); _getLong(); } @Override public short readShort() throws IOException { return (short) ( (_get() << 8) | _get() ) ; } @Override public int readUnsignedShort() throws IOException { return (int) ( (_get() << 8) | _get() ) ; } public final String readLine() throws IOException { StringBuffer input = new StringBuffer(); int c = -1; boolean eol = false; while (!eol) { switch (c = _read()) { case -1: case '\n': eol = true; break; case '\r': eol = true; long cur = position(); if ((_read()) != '\n') { position(cur); } break; default: input.append((char)c); break; } } if ((c == -1) && (input.length() == 0)) { return null; } return input.toString(); } public final String readUTF() throws IOException { return DataInputStream.readUTF(this); } @Override public long position() { return pointer; } public void position(long newPosition) { pointer = newPosition; } /** * Flushes internal buffer */ public void flush() throws IOException { writeFlush(); } /** * Clears read&write buffer. The file can be modified elsewere after this. * * @throws IOException */ public void reset() throws IOException { writeFlush(); readable_bytes_count = 0; virtualLength = diskLength = raf.length(); } @Override public long skipBytes(long bytes) throws IOException { pointer += bytes; return bytes; } @Override public int skipBytes(int bytes) throws IOException { pointer += bytes; return bytes; } // WRITE void _putLong(long value) throws IOException { prepareForWrite(8); int posInBuf = (int) (pointer - buf_pos); buf[posInBuf] = (byte) (value >>> 56); buf[posInBuf+1] = (byte) (value >>> 48); buf[posInBuf+2] = (byte) (value >>> 40); buf[posInBuf+3] = (byte) (value >>> 32); buf[posInBuf+4] = (byte) (value >>> 24); buf[posInBuf+5] = (byte) (value >>> 16); buf[posInBuf+6] = (byte) (value >>> 8); buf[posInBuf+7] = (byte) value; posInBuf += 8; pointer += 8; if (write_buf_count>> 24); buf[posInBuf+1] = (byte) (value >>> 16); buf[posInBuf+2] = (byte) (value >>> 8); buf[posInBuf+3] = (byte) value; posInBuf += 4; pointer += 4; if (write_buf_count0) { int n = Math.min(prepareForWrite(length), length); int posInBuf = (int) (pointer - buf_pos); System.arraycopy(src, offset, buf, posInBuf, n); pointer += n; posInBuf += n; offset += n; length -= n; if (write_buf_count> 8); _put(value); } @Override public void writeChar(int value) throws IOException { _put(value >> 8); _put(value); } @Override public void writeBytes(String s) throws IOException { int len = s.length(); for (int i = 0 ; i < len ; i++) { _put((byte)s.charAt(i)); } } @Override public void writeChars(String s) throws IOException { int len = s.length(); for (int i = 0 ; i < len ; i++) { int v = s.charAt(i); _put((v >>> 8) & 0xFF); _put((v >>> 0) & 0xFF); } } @Override public void writeUTF(String s) throws IOException { int len = UTF8.getModifiedUTF8EncodingByteLength(s); writeShort(len); UTF8.writeModifiedUTF(this, s); } @Override public void insertBytes(long bytes, ByteSide side) throws IOException { if (pointer>=virtualLength) { setLength(pointer + bytes); return; } // insertion to buffer window if (pointer>=buf_pos && (pointer<=buf_pos+readable_bytes_count)) { // buffer window convers the end of the file & there is enough space in buffer to do the operation if (buf_pos+readable_bytes_count >= virtualLength && readable_bytes_count + bytes < buf.length) { // Move right siade int posInBuf = (int) (pointer - buf_pos); System.arraycopy(buf, posInBuf, buf, (int) (posInBuf+bytes), (int) readable_bytes_count - posInBuf); readable_bytes_count += bytes; write_buf_count = readable_bytes_count; virtualLength += bytes; return; } writeFlush(); reset(); } writeFlush(); reset(); insertBytes(raf, pointer, bytes); virtualLength += bytes; diskLength += bytes; // Move buffer if (buf_pos>pointer) { buf_pos += bytes; } } @Override public void removeBytes(long bytes, ByteSide side) throws IOException { if (pointer+bytes>virtualLength || pointer<0) { throw new IOException("Pointer outside file"); } if (pointer+bytes==virtualLength) { setLength(virtualLength - bytes); return; } // removal intersects buffer window if (pointer+bytes>=buf_pos && (pointer<=buf_pos+readable_bytes_count)) { // buffer window covers the ending if (buf_pos+readable_bytes_count >= virtualLength) { // Scenario 1 : Pointer before buffer if (pointer=buf_pos) { int posInBuf = (int) (pointer - buf_pos); System.arraycopy(buf, (int) (posInBuf+bytes), buf, posInBuf, (int) (readable_bytes_count - posInBuf - bytes) ); readable_bytes_count -= bytes; write_buf_count = readable_bytes_count; virtualLength -= bytes; return; } } writeFlush(); reset(); } writeFlush(); reset(); removeBytes(raf, pointer, bytes); virtualLength -= bytes; diskLength -= bytes; // Move buffer if (buf_pos>pointer) { buf_pos -= bytes; } } @Override public long length() throws IOException { return virtualLength; } @Override public void setLength(long newLength) throws IOException { virtualLength = newLength; if (buf_pos + readable_bytes_count > virtualLength) { readable_bytes_count = (int) Math.max(virtualLength - buf_pos, 0L); } if (buf_pos + write_buf_count > virtualLength) { write_buf_count = (int) Math.max(virtualLength - buf_pos, 0L); } } private final static int bufferSize = 1024*32; /** * Inserts bytes into a middle of a file. * * @param file file * @param position * @param bytes * @throws IOException */ public static void insertBytes(RandomAccessFile file, long position, long bytes) throws IOException { if (position<0) throw new IndexOutOfBoundsException("position cannot be below 0"); if (bytes<0) throw new IndexOutOfBoundsException("bytes cannot be below 0"); if (bytes==0) return; long length = file.length(); if (position>=length) { file.setLength(position+bytes); return; } long bytesOnRight = length - position; // Create buffer - the buffer is cyclic window int bufLength = (int) Math.min(bytesOnRight, bufferSize); byte[] buf = new byte[bufLength]; // Bytes transferred from right long n = 0; while (n=length) return; // Truncate if (position+bytes>=length) { file.setLength(position); return; } long bytesOnRight = length - position - bytes; int bufLength = (int) Math.min(bytesOnRight, bufferSize); byte[] buf = new byte[bufLength]; long n = 0; while (n