1 /*******************************************************************************
\r
2 t * Copyright (c) 2007, 2010 Association for Decentralized Information Management
\r
3 * in Industry THTH ry.
\r
4 * All rights reserved. This program and the accompanying materials
\r
5 * are made available under the terms of the Eclipse Public License v1.0
\r
6 * which accompanies this distribution, and is available at
\r
7 * http://www.eclipse.org/legal/epl-v10.html
\r
10 * VTT Technical Research Centre of Finland - initial API and implementation
\r
11 *******************************************************************************/
\r
12 package org.simantics.fastlz;
14 import static org.junit.Assert.fail;
\r
16 import java.io.BufferedInputStream;
\r
17 import java.io.BufferedOutputStream;
\r
18 import java.io.File;
\r
19 import java.io.FileInputStream;
\r
20 import java.io.FileOutputStream;
\r
21 import java.io.IOException;
\r
22 import java.io.InputStream;
\r
23 import java.io.OutputStream;
\r
24 import java.util.Arrays;
\r
26 import junit.framework.Assert;
\r
28 import org.junit.BeforeClass;
\r
29 import org.junit.Test;
\r
32 * @author Tuukka Lehtonen
\r
34 @SuppressWarnings("deprecation")
\r
35 public class FastLZBasicTests {
\r
37 static File testData1;
\r
38 static File testData2;
\r
39 static File testDataPrime;
\r
42 public static void initialize() throws IOException {
\r
43 FastLZ.initialize(null);
\r
45 testData1 = new File("uncompressed.data");
\r
46 writeTestData(testData1, 10000000);
\r
47 testData2 = new File("uncompressed-small.data");
\r
48 writeTestData(testData2, 100000);
\r
49 testDataPrime = new File("uncompressed-prime.data");
\r
50 writeTestData(testDataPrime, 1000);
\r
52 System.out.println("test data directory: " + testData1.getAbsolutePath());
\r
54 // File javaFlz = new File("compressed-prime.data.native.flz");
\r
55 // for (int i = 0; i < 100; ++i) {
\r
56 // compressFlzJava(testDataPrime, javaFlz);
\r
61 public void validateCompress() throws IOException {
\r
62 validateCompress(testData1);
\r
63 validateCompress(new File("grades.snp"));
\r
66 private void validateCompress(File testData) throws IOException {
\r
67 System.out.println("==== validateCompress(" + testData.getName() + ") ====");
\r
69 File nativeFlz = new File("compressed.data.native.flz");
\r
70 long nativeCompressedSize = compressFlzNative(testData, nativeFlz);
\r
71 System.out.println("native compressed size: " + nativeCompressedSize);
\r
73 // Need to prime JVM JIT by performing multiple passes
\r
74 File javaFlz = new File("compressed.data.java.flz");
\r
75 long javaCompressedSize = compressFlzJava(testData, javaFlz);
\r
76 System.out.println("java compressed size: " + javaCompressedSize);
\r
78 Assert.assertEquals(nativeCompressedSize, javaCompressedSize);
\r
80 System.out.println("Comparing compressed outputs...");
\r
81 compareFiles(nativeFlz, javaFlz);
\r
82 System.out.println("Compressed outputs match.");
\r
84 File decompressedNativeFlz = new File("decompressed.data.native.flz");
\r
85 File decompressedJavaFlz = new File("decompressed.data.java.flz");
\r
86 decompressFlzNative(nativeFlz, decompressedNativeFlz);
\r
87 decompressFlzNative(javaFlz, decompressedJavaFlz);
\r
88 compareFiles(decompressedJavaFlz, decompressedNativeFlz);
\r
89 compareFiles(decompressedNativeFlz, testData);
\r
93 // public void validateCompressSmall() throws IOException {
\r
94 // File nativeFlz = new File("compressed-small.data.native.flz");
\r
95 // long nativeCompressedSize = compressFlzNative(testData2, nativeFlz);
\r
96 // System.out.println("native compressed size: " + nativeCompressedSize);
\r
98 // // Need to prime JVM JIT by performing multiple passes
\r
99 // File javaFlz = new File("compressed-small.data.java.flz");
\r
100 // long javaCompressedSize = compressFlzJava(testData2, javaFlz);
\r
101 // System.out.println("java compressed size: " + javaCompressedSize);
\r
103 // Assert.assertEquals(nativeCompressedSize, javaCompressedSize);
\r
105 // System.out.println("Comparing compressed outputs...");
\r
106 // compareFiles(nativeFlz, javaFlz);
\r
107 // System.out.println("Compressed outputs match.");
\r
111 // public void testNativeCompressPerformance() throws IOException {
\r
112 // File javaFlz = new File("compressed.data.native.flz");
\r
113 // for (int i = 0; i < 5; ++i) {
\r
114 // compressFlzNative(testData1, javaFlz);
\r
119 // public void testJavaCompressPerformance() throws IOException {
\r
120 // // Need to prime JVM JIT by performing multiple passes
\r
121 // File javaFlz = new File("compressed.data.java.flz");
\r
122 // for (int i = 0; i < 5; ++i) {
\r
123 // compressFlzJava(testData1, javaFlz);
\r
128 // public void testCompressNative() throws IOException {
\r
129 // File flz = new File("compressed.data.native.flz");
\r
130 // for (int i = 0; i < 5; ++i) {
\r
131 // long compressedSize = compressFlzNative(testData1, flz);
\r
132 // System.out.println("native compressed size: " + compressedSize);
\r
137 // public void testCompressionJava() throws IOException {
\r
138 // // Need to prime JVM JIT by performing multiple passes
\r
139 // File flz = new File("compressed.data.java.flz");
\r
140 // for (int i = 0; i < 5; ++i) {
\r
141 // long compressedSize = compressFlzJava(testData1, flz);
\r
142 // System.out.println("java compressed size: " + compressedSize);
\r
146 @SuppressWarnings("unused")
\r
148 public void testDecompress() throws IOException {
\r
149 File nativeFlz = new File("compressed.data.native.flz");
\r
150 long nativeCompressedSize = compressFlzNative(testData1, nativeFlz);
\r
151 System.out.println("native compressed size: " + nativeCompressedSize);
\r
153 // Need to prime JVM JIT by performing multiple passes
\r
154 File javaFlz = new File("compressed.data.java.flz");
\r
155 long javaCompressedSize = compressFlzJava(testData1, javaFlz);
\r
156 System.out.println("java compressed size: " + javaCompressedSize);
\r
158 Assert.assertEquals(nativeCompressedSize, javaCompressedSize);
\r
160 System.out.println("Comparing compressed outputs...");
\r
161 compareFiles(nativeFlz, javaFlz);
\r
162 System.out.println("Compressed outputs match.");
\r
164 File java1 = new File("java-compressed.data.decompressed-with-native");
\r
165 long nativeDecompressedSize = decompressFlzNative(javaFlz, java1);
\r
166 System.out.println("Comparing native-decompressed output...");
\r
167 compareFiles(testData1, java1);
\r
168 System.out.println("Native-decompressed output matches original.");
\r
170 File java2 = new File("java-compressed.data.decompressed-with-java");
\r
171 long javaDecompressedSize = decompressFlzJava(javaFlz, java2);
\r
172 System.out.println("Comparing java-decompressed output...");
\r
173 compareFiles(testData1, java2);
\r
174 System.out.println("Java-decompressed output matches original.");
\r
176 for (int i = 0; i < 5; ++i)
\r
177 decompressFlz(javaFlz, FastLZ.read(javaFlz), java1, NullOutputStream.INSTANCE);
\r
178 for (int i = 0; i < 5; ++i)
\r
179 decompressFlz(javaFlz, FastLZJava.read(javaFlz), java2, NullOutputStream.INSTANCE);
\r
183 public void testDecompressCluster() {
184 fail("Not yet implemented");
187 static void compareFiles(File file1, File file2) throws IOException {
\r
188 InputStream in1 = new BufferedInputStream(new FileInputStream(file1));
\r
189 InputStream in2 = new BufferedInputStream(new FileInputStream(file2));
\r
193 int b1 = in1.read();
\r
194 int b2 = in2.read();
\r
195 if (b1 == -1 && b2 == -1)
\r
198 fail("EOF reached in file1 " + file1.getName() + " but not in file2 " + file2.getName());
\r
200 fail("EOF reached in file1 " + file1.getName() + " but not in file2 " + file2.getName());
\r
202 fail("bytes at offset " + offset + " do not match: " + b1 + " vs. " + b2);
\r
211 public static void writeTestData(File file, int rows) throws IOException {
\r
215 System.out.println("writing test data...");
\r
216 OutputStream stream = new FileOutputStream(file);
\r
217 for (int i = 0; i < rows; ++i)
\r
218 stream.write((Integer.toString(i) + "\n").getBytes());
\r
220 System.out.println("wrote " + file.length() + " bytes of test data.");
\r
226 * @return compressed size in bytes
\r
227 * @throws IOException
\r
229 static long compressFlzJava(File source, File flz) throws IOException {
\r
230 return compressFlz(source, flz, FastLZJava.write(flz));
\r
236 * @return compressed size in bytes
\r
237 * @throws IOException
\r
239 static long compressFlzNative(File source, File flz) throws IOException {
\r
240 return compressFlz(source, flz, FastLZ.write(flz));
\r
246 * @return compressed size in bytes
\r
247 * @throws IOException
\r
249 static long compressFlz(File source, File flz, OutputStream flzOutput) throws IOException {
\r
250 System.out.println("compressFlz(" + source + ", " + flz + ")");
\r
251 InputStream input = new BufferedInputStream(new FileInputStream(source));
\r
252 copy(input, flzOutput);
\r
255 long compressed = flz.length();
\r
256 System.out.println("Wrote " + compressed + " compressed bytes");
\r
263 * @return compressed size in bytes
\r
264 * @throws IOException
\r
266 static long decompressFlzJava(File flz, File dest) throws IOException {
\r
267 return decompressFlz(flz, FastLZJava.read(flz), dest);
\r
273 * @return compressed size in bytes
\r
274 * @throws IOException
\r
276 static long decompressFlzNative(File flz, File dest) throws IOException {
\r
277 return decompressFlz(flz, FastLZ.read(flz), dest);
\r
283 * @return compressed size in bytes
\r
284 * @throws IOException
\r
286 static long decompressFlz(File source, InputStream flzInput, File dest) throws IOException {
\r
287 System.out.println("decompressFlz(" + source + ", " + dest + ")");
\r
288 OutputStream output = new BufferedOutputStream(new FileOutputStream(dest));
\r
289 copy(flzInput, output);
\r
292 long decompressed = dest.length();
\r
293 System.out.println("Wrote " + decompressed + " decompressed bytes");
\r
294 return decompressed;
\r
300 * @return compressed size in bytes
\r
301 * @throws IOException
\r
303 static long decompressFlz(File source, InputStream flzInput, File dest, OutputStream destStream) throws IOException {
\r
304 System.out.println("decompressFlz(" + source + ", " + dest + ")");
\r
305 copy(flzInput, destStream);
\r
307 destStream.close();
\r
308 long decompressed = dest.length();
\r
309 System.out.println("Wrote " + decompressed + " decompressed bytes");
\r
310 return decompressed;
\r
314 * Copy the content of the input stream into the output stream, using a temporary
\r
315 * byte array buffer whose size is defined by {@link #IO_BUFFER_SIZE}.
\r
317 * @param in The input stream to copy from.
\r
318 * @param out The output stream to copy to.
\r
320 * @throws IOException If any error occurs during the copy.
\r
322 private static final int IO_BUFFER_SIZE = 128 * 1024;
\r
324 public static long copy(InputStream in, OutputStream out) throws IOException {
\r
325 byte[] b = new byte[IO_BUFFER_SIZE];
\r
328 long start = System.nanoTime();
\r
334 //System.out.println("read " + read + " bytes, " + total + " bytes in total");
\r
335 out.write(b, 0, read);
\r
338 long end = System.nanoTime();
\r
339 double totalmb = total/(1024.0*1024.0);
\r
340 double time = (end-start)*1e-9;
\r
341 double rate = totalmb / time;
\r
342 System.out.format("Transferred %d bytes (%.3f Mbytes) in %f seconds (%.3f MB/s)\n", total, totalmb, time, rate);
\r
347 @SuppressWarnings("unused")
\r
348 private boolean checksumsEqual(File f1, File f2) throws IOException {
\r
349 byte[] s1 = ChecksumUtil.computeSum(f1);
\r
350 byte[] s2 = ChecksumUtil.computeSum(f2);
\r
351 return Arrays.equals(s1, s2);
\r