/*******************************************************************************
* Copyright (c) 2010, 2016 Association for Decentralized Information Management
* in Industry THTH ry.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* VTT Technical Research Centre of Finland - initial API and implementation
*******************************************************************************/
package org.simantics.databoard.util.binary;
import java.io.DataInputStream;
import java.io.EOFException;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
/**
* This class is a Random Access File implementation to RandomAccessBinary.
* The implementation is buffered. The implementation is not multi-thread
* safe.
*
* There is a buffer used for reading and writing. It has a buffer read and
* write position. When read, the buffer is filled. When written, the buffer
* is written. When the file pointer is moved, the file size changed or flushed
* the buffers are cleared. If there were unwritten bytes, they are flushed
* to disc.
*
* There is internal pointer variable. The actual file pointer is moved on
* disc read and write operations.
*
* Primitive number writes (int, short, char, double, float and long) and
* written in big endian (network) byte order. Use {@link Endian}
* to make little endian operations.
*
* @author Toni Kalajainen
*/
public class BinaryFile implements RandomAccessBinary, BinaryReadable, BinaryWriteable {
RandomAccessFile raf;
File file;
/** The internal read and write buffer */
byte buf[];
/** The number of valid bytes in the buffer */
int readable_bytes_count;
/** The number written but unflushed bytes in buf starting from buf[0]*/
int write_buf_count;
/** The pointer of buf[0] in the file. The value is valid only if value >=0 */
long buf_pos;
/** File Pointer of current position */
long pointer;
/** Transient size of the file */
long virtualLength;
long diskLength;
public BinaryFile(RandomAccessFile file) throws IOException
{
this.raf = file;
virtualLength = diskLength = raf.length();
pointer = 0;
buf = new byte[4096];
}
public BinaryFile(File file) throws IOException
{
this.raf = new RandomAccessFile(file, "rw");
virtualLength = diskLength = raf.length();
this.file = file;
pointer = 0;
buf = new byte[4096];
}
public BinaryFile(File file, String mode) throws IOException
{
this.raf = new RandomAccessFile(file, mode);
virtualLength = diskLength = raf.length();
this.file = file;
pointer = 0;
buf = new byte[4096];
}
public BinaryFile(RandomAccessFile file, int bufSize) throws IOException
{
this.raf = file;
virtualLength = diskLength = raf.length();
pointer = 0;
buf = new byte[bufSize];
}
public BinaryFile(File file, int bufSize) throws IOException
{
this.raf = new RandomAccessFile(file, "rw");
virtualLength = diskLength = raf.length();
this.file = file;
pointer = 0;
buf = new byte[bufSize];
}
public static BinaryFile tempFile(long size) throws IOException
{
File tmpFile = File.createTempFile("Temp", ".file");
tmpFile.deleteOnExit();
BinaryFile file = new BinaryFile(tmpFile);
file.setLength( size );
return file;
}
/**
* Closes the object. Note, this will close the input random access file.
* This method may be called several times.
*
* @throws IOException
*/
public synchronized void close() throws IOException {
if (raf==null) return;
flush();
pointer = -1;
raf.close();
raf = null;
buf = null;
}
public synchronized boolean isOpen() {
return buf!=null;
}
public File file() {
return file;
}
public RandomAccessFile getRandomAccessFile() {
return raf;
}
/**
* Get the number of readable bytes in buffer
*
* @return readable bytes or 0
*/
private long readableBytesInBuffer() {
long posInBuf = pointer - buf_pos;
if (posInBuf<0) return 0;
long bytesLeft = readable_bytes_count - posInBuf;
return bytesLeft < 0 ? 0 : bytesLeft;
}
/**
* Get the valid position of pointer in buf[]
*
* @return pos or -1
*/
private long positionInReadBuffer() {
long posInBuf = pointer - buf_pos;
if (posInBuf<0 || posInBuf>readable_bytes_count) return -1;
return posInBuf;
}
/**
* Get next byte
* @return 0..255
* @throws IOException
*/
int _get() throws IOException
{
assertReadable(1);
int posInBuf = (int) (pointer - buf_pos);
int result = buf[posInBuf] & 0xFF;
pointer++;
return result;
}
/**
* Get next 4 bytes as an int value. This is an optimization.
* @return long value
* @throws IOException
*/
int _getInt() throws IOException
{
assertReadable(4);
int posInBuf = (int) (pointer - buf_pos);
int result =
(((int) (buf[posInBuf + 3] & 0xFF)) |
(((int) (buf[posInBuf + 2] & 0xFF)) << 8) |
(((int) (buf[posInBuf + 1] & 0xFF)) << 16) |
(((int) (buf[posInBuf] & 0xFF)) << 24));
pointer+=4;
return result;
}
/**
* Get next 8 bytes as a long value. This is an optimization.
* @return long value
* @throws IOException
*/
long _getLong() throws IOException
{
assertReadable(8);
int posInBuf = (int) (pointer - buf_pos);
long result =
(((long) (buf[posInBuf + 7] & 0xFF)) |
(((long) (buf[posInBuf + 6] & 0xFF)) << 8) |
(((long) (buf[posInBuf + 5] & 0xFF)) << 16) |
(((long) (buf[posInBuf + 4] & 0xFF)) << 24) |
(((long) (buf[posInBuf + 3] & 0xFF)) << 32) |
(((long) (buf[posInBuf + 2] & 0xFF)) << 40) |
(((long) (buf[posInBuf + 1] & 0xFF)) << 48) |
(((long) (buf[posInBuf] & 0xFF)) << 56));
pointer+=8;
return result;
}
/**
* Get next byte
* @return 0..255 or -1 on end of file
* @throws IOException
*/
int _read() throws IOException
{
if (readableBytesInBuffer()<1) {
fill();
if (readableBytesInBuffer()==0) return -1;
}
int posInBuf = (int) (pointer - buf_pos);
int result = buf[posInBuf] & 0xFF;
pointer++;
return result;
}
void _get(byte[] dst, int offset, int length) throws IOException {
while (length>0) {
long n = Math.min( readableBytesInBuffer(), length );
int posInBuf = (int) (pointer - buf_pos);
if (n>0) {
System.arraycopy(buf, (int) posInBuf, dst, offset, (int) n);
offset += n;
length -= n;
pointer += n;
}
if (length>0) {
fill();
if (readableBytesInBuffer()==0) {
throw new EOFException();
}
}
}
}
/**
* Flushes bytes 0..write_buf_count to file at buf_pos.
* Sets write_buf_count to 0.
*
* @throws IOException
*/
private void writeFlush() throws IOException {
if (write_buf_count>0 && buf_pos>=0) {
raf.seek(buf_pos);
raf.write(buf, 0, write_buf_count);
if (buf_pos+write_buf_count>diskLength) diskLength = buf_pos+write_buf_count;
write_buf_count = 0;
}
if (diskLength != virtualLength) {
raf.setLength(virtualLength);
diskLength = virtualLength;
}
}
/**
* This method ensures that write buffer is ready for writing
* at the pointer. Old buffer is flushed if necessary.
*
* @param requested number of bytes, up to 4096 guaranteed
* @return the number of bytes that can be written to buf
* @throws IOException
*/
private int prepareForWrite(int bytes) throws IOException {
// Ensure pointer is within buf[0..readable_bytes_count]
int posInBuf = (int) (pointer - buf_pos);
if (posInBuf<0) {
writeFlush();
readable_bytes_count = 0;
buf_pos = pointer;
posInBuf = 0;
} else
if (posInBuf>readable_bytes_count) {
// Pointer is at wrong place, flush data
writeFlush();
// Move pointer
int bytesToKeep = readable_bytes_count - posInBuf;
if (bytesToKeep<0) bytesToKeep = 0;
if (bytesToKeep>0) {
System.arraycopy(buf, posInBuf, buf, 0, bytesToKeep);
readable_bytes_count = bytesToKeep;
} else {
readable_bytes_count = 0;
}
buf_pos = pointer;
posInBuf = 0;
}
// Ensure buf[] has room for bytes
if (buf.length - posInBuf < bytes) {
writeFlush();
int bytesToKeep = readable_bytes_count - posInBuf;
if (bytesToKeep<0) bytesToKeep = 0;
if (bytesToKeep>0) {
System.arraycopy(buf, posInBuf, buf, 0, bytesToKeep);
readable_bytes_count = bytesToKeep;
} else {
readable_bytes_count = 0;
}
buf_pos = pointer;
posInBuf = 0;
return buf.length;
}
return (int) ( buf.length - posInBuf );
}
/**
* Assert there is a set number of bytes in read buffer.
* bytes should not be larger than read buffer size (buf.length, 4K)
*
* @param bytes
* @throws IOException
*/
private void assertReadable(int bytes) throws IOException
{
if (readableBytesInBuffer()= new_buf_start) && (old_buf_start < new_buf_end);
boolean old_buf_end_in_new_buf = (old_buf_end >= new_buf_start) && (old_buf_end < new_buf_end);
// Scenario 1, end of old buf in the new, but the whole old buffer doesnt fit in the new
if (old_buf_end_in_new_buf && !old_buf_start_in_new_buf)
{
int bytesToPreserve = (int) (old_buf_end - new_buf_start);
int bytesToRead = (int) (new_buf_end - old_buf_end);
System.arraycopy(buf, old_buf_length - bytesToPreserve, buf, 0, bytesToPreserve);
// Read more
raf.seek(new_buf_start + bytesToPreserve);
raf.readFully(buf, bytesToPreserve, bytesToRead);
buf_pos = pointer;
readable_bytes_count = new_buf_length;
return;
}
// Scenario 2, start of old buf is in new, but not completely
if (old_buf_start_in_new_buf && !old_buf_end_in_new_buf) {
int bytesToPreserve = (int) (new_buf_end-old_buf_start);
int bytesToRead = (int) (old_buf_start-new_buf_start);
System.arraycopy(buf, 0, buf, bytesToRead, bytesToPreserve);
raf.seek(new_buf_start);
raf.readFully(buf, 0, bytesToRead);
buf_pos = pointer;
readable_bytes_count = new_buf_length;
return;
}
// Scenario 3, old buf is partially in new buf (Omited)
// Scenario 4, old and new buf do not intersect
{
int bytesToRead = new_buf_length;
raf.seek(new_buf_start);
raf.readFully(buf, 0, bytesToRead);
buf_pos = pointer;
readable_bytes_count = new_buf_length;
}
}
@Override
public byte readByte() throws IOException {
return (byte) _get();
}
@Override
public char readChar() throws IOException {
return (char)((_get() << 8) | _get());
}
@Override
public int readUnsignedByte() throws IOException {
return _get() & 0x000000ff;
}
@Override
public boolean readBoolean() throws IOException {
return _get()!=0;
}
@Override
public void readFully(byte[] dst, int offset, int length) throws IOException {
_get(dst, offset, length);
}
@Override
public void readFully(byte[] dst) throws IOException {
_get(dst, 0, dst.length);
}
@Override
public void readFully(ByteBuffer buf) throws IOException {
readFully(buf, buf.remaining());
}
@Override
public void readFully(ByteBuffer buf, int length) throws IOException {
while (length>0) {
assertReadable( Math.min(this.buf.length, length) );
long n = Math.min(readableBytesInBuffer(), length);
if (n==0) throw new EOFException();
long posInBuf = positionInReadBuffer();
if (n>0 && posInBuf>=0) {
buf.put(this.buf, (int)posInBuf, (int)n);
length -= n;
pointer += n;
}
if (length>0) {
fill();
if (readableBytesInBuffer()==0) {
throw new EOFException();
}
}
}
}
@Override
public double readDouble() throws IOException {
return Double.longBitsToDouble(readLong());
}
@Override
public float readFloat() throws IOException {
return Float.intBitsToFloat(readInt());
}
@Override
public int readInt() throws IOException {
return
// ( _get() << 24) |
// ( _get() << 16) |
// ( _get() << 8) |
// ( _get() );
_getInt();
}
@Override
public long readLong() throws IOException {
return
// ( ((long)_get()) << 56) |
// ( ((long)_get()) << 48) |
// ( ((long)_get()) << 40) |
// ( ((long)_get()) << 32) |
// ( ((long)_get()) << 24) |
// ( ((long)_get()) << 16) |
// ( ((long)_get()) << 8) |
// ( ((long)_get()) );
_getLong();
}
@Override
public short readShort() throws IOException {
return (short) ( (_get() << 8) | _get() ) ;
}
@Override
public int readUnsignedShort() throws IOException {
return (int) ( (_get() << 8) | _get() ) ;
}
public final String readLine() throws IOException {
StringBuffer input = new StringBuffer();
int c = -1;
boolean eol = false;
while (!eol) {
switch (c = _read()) {
case -1:
case '\n':
eol = true;
break;
case '\r':
eol = true;
long cur = position();
if ((_read()) != '\n') {
position(cur);
}
break;
default:
input.append((char)c);
break;
}
}
if ((c == -1) && (input.length() == 0)) {
return null;
}
return input.toString();
}
public final String readUTF() throws IOException {
return DataInputStream.readUTF(this);
}
@Override
public long position() {
return pointer;
}
public void position(long newPosition) {
pointer = newPosition;
}
/**
* Flushes internal buffer
*/
public void flush() throws IOException {
writeFlush();
}
/**
* Clears read&write buffer. The file can be modified elsewere after this.
*
* @throws IOException
*/
public void reset() throws IOException {
writeFlush();
readable_bytes_count = 0;
virtualLength = diskLength = raf.length();
}
@Override
public long skipBytes(long bytes) throws IOException {
pointer += bytes;
return bytes;
}
@Override
public int skipBytes(int bytes) throws IOException {
pointer += bytes;
return bytes;
}
// WRITE
void _putLong(long value) throws IOException
{
prepareForWrite(8);
int posInBuf = (int) (pointer - buf_pos);
buf[posInBuf] = (byte) (value >>> 56);
buf[posInBuf+1] = (byte) (value >>> 48);
buf[posInBuf+2] = (byte) (value >>> 40);
buf[posInBuf+3] = (byte) (value >>> 32);
buf[posInBuf+4] = (byte) (value >>> 24);
buf[posInBuf+5] = (byte) (value >>> 16);
buf[posInBuf+6] = (byte) (value >>> 8);
buf[posInBuf+7] = (byte) value;
posInBuf += 8;
pointer += 8;
if (write_buf_count>> 24);
buf[posInBuf+1] = (byte) (value >>> 16);
buf[posInBuf+2] = (byte) (value >>> 8);
buf[posInBuf+3] = (byte) value;
posInBuf += 4;
pointer += 4;
if (write_buf_count0) {
int n = Math.min(prepareForWrite(length), length);
int posInBuf = (int) (pointer - buf_pos);
System.arraycopy(src, offset, buf, posInBuf, n);
pointer += n;
posInBuf += n;
offset += n;
length -= n;
if (write_buf_count> 8);
_put(value);
}
@Override
public void writeChar(int value) throws IOException {
_put(value >> 8);
_put(value);
}
@Override
public void writeBytes(String s) throws IOException {
int len = s.length();
for (int i = 0 ; i < len ; i++) {
_put((byte)s.charAt(i));
}
}
@Override
public void writeChars(String s) throws IOException {
int len = s.length();
for (int i = 0 ; i < len ; i++) {
int v = s.charAt(i);
_put((v >>> 8) & 0xFF);
_put((v >>> 0) & 0xFF);
}
}
@Override
public void writeUTF(String s) throws IOException {
int len = UTF8.getModifiedUTF8EncodingByteLength(s);
writeShort(len);
UTF8.writeModifiedUTF(this, s);
}
@Override
public void insertBytes(long bytes, ByteSide side) throws IOException {
if (pointer>=virtualLength) {
setLength(pointer + bytes);
return;
}
// insertion to buffer window
if (pointer>=buf_pos && (pointer<=buf_pos+readable_bytes_count))
{
// buffer window convers the end of the file & there is enough space in buffer to do the operation
if (buf_pos+readable_bytes_count >= virtualLength && readable_bytes_count + bytes < buf.length) {
// Move right siade
int posInBuf = (int) (pointer - buf_pos);
System.arraycopy(buf, posInBuf, buf, (int) (posInBuf+bytes), (int) readable_bytes_count - posInBuf);
readable_bytes_count += bytes;
write_buf_count = readable_bytes_count;
virtualLength += bytes;
return;
}
writeFlush();
reset();
}
writeFlush();
reset();
insertBytes(raf, pointer, bytes);
virtualLength += bytes;
diskLength += bytes;
// Move buffer
if (buf_pos>pointer) {
buf_pos += bytes;
}
}
@Override
public void removeBytes(long bytes, ByteSide side) throws IOException {
if (pointer+bytes>virtualLength || pointer<0) {
throw new IOException("Pointer outside file");
}
if (pointer+bytes==virtualLength) {
setLength(virtualLength - bytes);
return;
}
// removal intersects buffer window
if (pointer+bytes>=buf_pos && (pointer<=buf_pos+readable_bytes_count))
{
// buffer window covers the ending
if (buf_pos+readable_bytes_count >= virtualLength) {
// Scenario 1 : Pointer before buffer
if (pointer=buf_pos) {
int posInBuf = (int) (pointer - buf_pos);
System.arraycopy(buf, (int) (posInBuf+bytes), buf, posInBuf, (int) (readable_bytes_count - posInBuf - bytes) );
readable_bytes_count -= bytes;
write_buf_count = readable_bytes_count;
virtualLength -= bytes;
return;
}
}
writeFlush();
reset();
}
writeFlush();
reset();
removeBytes(raf, pointer, bytes);
virtualLength -= bytes;
diskLength -= bytes;
// Move buffer
if (buf_pos>pointer) {
buf_pos -= bytes;
}
}
@Override
public long length() throws IOException {
return virtualLength;
}
@Override
public void setLength(long newLength) throws IOException {
virtualLength = newLength;
if (buf_pos + readable_bytes_count > virtualLength) {
readable_bytes_count = (int) Math.max(virtualLength - buf_pos, 0L);
}
if (buf_pos + write_buf_count > virtualLength) {
write_buf_count = (int) Math.max(virtualLength - buf_pos, 0L);
}
}
private final static int bufferSize = 1024*32;
/**
* Inserts bytes into a middle of a file.
*
* @param file file
* @param position
* @param bytes
* @throws IOException
*/
public static void insertBytes(RandomAccessFile file, long position, long bytes)
throws IOException
{
if (position<0) throw new IndexOutOfBoundsException("position cannot be below 0");
if (bytes<0) throw new IndexOutOfBoundsException("bytes cannot be below 0");
if (bytes==0) return;
long length = file.length();
if (position>=length) {
file.setLength(position+bytes);
return;
}
long bytesOnRight = length - position;
// Create buffer - the buffer is cyclic window
int bufLength = (int) Math.min(bytesOnRight, bufferSize);
byte[] buf = new byte[bufLength];
// Bytes transferred from right
long n = 0;
while (n=length) return;
// Truncate
if (position+bytes>=length) {
file.setLength(position);
return;
}
long bytesOnRight = length - position - bytes;
int bufLength = (int) Math.min(bytesOnRight, bufferSize);
byte[] buf = new byte[bufLength];
long n = 0;
while (n