1 /*******************************************************************************
2 t * Copyright (c) 2007, 2010 Association for Decentralized Information Management
4 * All rights reserved. This program and the accompanying materials
5 * are made available under the terms of the Eclipse Public License v1.0
6 * which accompanies this distribution, and is available at
7 * http://www.eclipse.org/legal/epl-v10.html
10 * VTT Technical Research Centre of Finland - initial API and implementation
11 *******************************************************************************/
12 package org.simantics.fastlz;
14 import static org.junit.Assert.fail;
16 import java.io.BufferedInputStream;
17 import java.io.BufferedOutputStream;
19 import java.io.FileInputStream;
20 import java.io.FileOutputStream;
21 import java.io.IOException;
22 import java.io.InputStream;
23 import java.io.OutputStream;
24 import java.util.Arrays;
26 import junit.framework.Assert;
28 import org.junit.BeforeClass;
29 import org.junit.Test;
32 * @author Tuukka Lehtonen
34 @SuppressWarnings("deprecation")
35 public class FastLZBasicTests {
37 static File testData1;
38 static File testData2;
39 static File testDataPrime;
42 public static void initialize() throws IOException {
43 FastLZ.initialize(null);
45 testData1 = new File("uncompressed.data");
46 writeTestData(testData1, 10000000);
47 testData2 = new File("uncompressed-small.data");
48 writeTestData(testData2, 100000);
49 testDataPrime = new File("uncompressed-prime.data");
50 writeTestData(testDataPrime, 1000);
52 System.out.println("test data directory: " + testData1.getAbsolutePath());
54 // File javaFlz = new File("compressed-prime.data.native.flz");
55 // for (int i = 0; i < 100; ++i) {
56 // compressFlzJava(testDataPrime, javaFlz);
61 public void validateCompress() throws IOException {
62 validateCompress(testData1);
65 private void validateCompress(File testData) throws IOException {
66 System.out.println("==== validateCompress(" + testData.getName() + ") ====");
68 File nativeFlz = new File("compressed.data.native.flz");
69 long nativeCompressedSize = compressFlzNative(testData, nativeFlz);
70 System.out.println("native compressed size: " + nativeCompressedSize);
72 // Need to prime JVM JIT by performing multiple passes
73 File javaFlz = new File("compressed.data.java.flz");
74 long javaCompressedSize = compressFlzJava(testData, javaFlz);
75 System.out.println("java compressed size: " + javaCompressedSize);
77 Assert.assertEquals(nativeCompressedSize, javaCompressedSize);
79 System.out.println("Comparing compressed outputs...");
80 compareFiles(nativeFlz, javaFlz);
81 System.out.println("Compressed outputs match.");
83 File decompressedNativeFlz = new File("decompressed.data.native.flz");
84 File decompressedJavaFlz = new File("decompressed.data.java.flz");
85 decompressFlzNative(nativeFlz, decompressedNativeFlz);
86 decompressFlzNative(javaFlz, decompressedJavaFlz);
87 compareFiles(decompressedJavaFlz, decompressedNativeFlz);
88 compareFiles(decompressedNativeFlz, testData);
92 // public void validateCompressSmall() throws IOException {
93 // File nativeFlz = new File("compressed-small.data.native.flz");
94 // long nativeCompressedSize = compressFlzNative(testData2, nativeFlz);
95 // System.out.println("native compressed size: " + nativeCompressedSize);
97 // // Need to prime JVM JIT by performing multiple passes
98 // File javaFlz = new File("compressed-small.data.java.flz");
99 // long javaCompressedSize = compressFlzJava(testData2, javaFlz);
100 // System.out.println("java compressed size: " + javaCompressedSize);
102 // Assert.assertEquals(nativeCompressedSize, javaCompressedSize);
104 // System.out.println("Comparing compressed outputs...");
105 // compareFiles(nativeFlz, javaFlz);
106 // System.out.println("Compressed outputs match.");
110 // public void testNativeCompressPerformance() throws IOException {
111 // File javaFlz = new File("compressed.data.native.flz");
112 // for (int i = 0; i < 5; ++i) {
113 // compressFlzNative(testData1, javaFlz);
118 // public void testJavaCompressPerformance() throws IOException {
119 // // Need to prime JVM JIT by performing multiple passes
120 // File javaFlz = new File("compressed.data.java.flz");
121 // for (int i = 0; i < 5; ++i) {
122 // compressFlzJava(testData1, javaFlz);
127 // public void testCompressNative() throws IOException {
128 // File flz = new File("compressed.data.native.flz");
129 // for (int i = 0; i < 5; ++i) {
130 // long compressedSize = compressFlzNative(testData1, flz);
131 // System.out.println("native compressed size: " + compressedSize);
136 // public void testCompressionJava() throws IOException {
137 // // Need to prime JVM JIT by performing multiple passes
138 // File flz = new File("compressed.data.java.flz");
139 // for (int i = 0; i < 5; ++i) {
140 // long compressedSize = compressFlzJava(testData1, flz);
141 // System.out.println("java compressed size: " + compressedSize);
145 @SuppressWarnings("unused")
147 public void testDecompress() throws IOException {
148 File nativeFlz = new File("compressed.data.native.flz");
149 long nativeCompressedSize = compressFlzNative(testData1, nativeFlz);
150 System.out.println("native compressed size: " + nativeCompressedSize);
152 // Need to prime JVM JIT by performing multiple passes
153 File javaFlz = new File("compressed.data.java.flz");
154 long javaCompressedSize = compressFlzJava(testData1, javaFlz);
155 System.out.println("java compressed size: " + javaCompressedSize);
157 Assert.assertEquals(nativeCompressedSize, javaCompressedSize);
159 System.out.println("Comparing compressed outputs...");
160 compareFiles(nativeFlz, javaFlz);
161 System.out.println("Compressed outputs match.");
163 File java1 = new File("java-compressed.data.decompressed-with-native");
164 long nativeDecompressedSize = decompressFlzNative(javaFlz, java1);
165 System.out.println("Comparing native-decompressed output...");
166 compareFiles(testData1, java1);
167 System.out.println("Native-decompressed output matches original.");
169 File java2 = new File("java-compressed.data.decompressed-with-java");
170 long javaDecompressedSize = decompressFlzJava(javaFlz, java2);
171 System.out.println("Comparing java-decompressed output...");
172 compareFiles(testData1, java2);
173 System.out.println("Java-decompressed output matches original.");
175 for (int i = 0; i < 5; ++i)
176 decompressFlz(javaFlz, FastLZ.read(javaFlz), java1, NullOutputStream.INSTANCE);
177 for (int i = 0; i < 5; ++i)
178 decompressFlz(javaFlz, FastLZJava.read(javaFlz), java2, NullOutputStream.INSTANCE);
182 public void testDecompressCluster() {
183 fail("Not yet implemented");
186 static void compareFiles(File file1, File file2) throws IOException {
187 InputStream in1 = new BufferedInputStream(new FileInputStream(file1));
188 InputStream in2 = new BufferedInputStream(new FileInputStream(file2));
194 if (b1 == -1 && b2 == -1)
197 fail("EOF reached in file1 " + file1.getName() + " but not in file2 " + file2.getName());
199 fail("EOF reached in file1 " + file1.getName() + " but not in file2 " + file2.getName());
201 fail("bytes at offset " + offset + " do not match: " + b1 + " vs. " + b2);
210 public static void writeTestData(File file, int rows) throws IOException {
214 System.out.println("writing test data...");
215 OutputStream stream = new FileOutputStream(file);
216 for (int i = 0; i < rows; ++i)
217 stream.write((Integer.toString(i) + "\n").getBytes());
219 System.out.println("wrote " + file.length() + " bytes of test data.");
225 * @return compressed size in bytes
226 * @throws IOException
228 static long compressFlzJava(File source, File flz) throws IOException {
229 return compressFlz(source, flz, FastLZJava.write(flz));
235 * @return compressed size in bytes
236 * @throws IOException
238 static long compressFlzNative(File source, File flz) throws IOException {
239 return compressFlz(source, flz, FastLZ.write(flz));
245 * @return compressed size in bytes
246 * @throws IOException
248 static long compressFlz(File source, File flz, OutputStream flzOutput) throws IOException {
249 System.out.println("compressFlz(" + source + ", " + flz + ")");
250 InputStream input = new BufferedInputStream(new FileInputStream(source));
251 copy(input, flzOutput);
254 long compressed = flz.length();
255 System.out.println("Wrote " + compressed + " compressed bytes");
262 * @return compressed size in bytes
263 * @throws IOException
265 static long decompressFlzJava(File flz, File dest) throws IOException {
266 return decompressFlz(flz, FastLZJava.read(flz), dest);
272 * @return compressed size in bytes
273 * @throws IOException
275 static long decompressFlzNative(File flz, File dest) throws IOException {
276 return decompressFlz(flz, FastLZ.read(flz), dest);
282 * @return compressed size in bytes
283 * @throws IOException
285 static long decompressFlz(File source, InputStream flzInput, File dest) throws IOException {
286 System.out.println("decompressFlz(" + source + ", " + dest + ")");
287 OutputStream output = new BufferedOutputStream(new FileOutputStream(dest));
288 copy(flzInput, output);
291 long decompressed = dest.length();
292 System.out.println("Wrote " + decompressed + " decompressed bytes");
299 * @return compressed size in bytes
300 * @throws IOException
302 static long decompressFlz(File source, InputStream flzInput, File dest, OutputStream destStream) throws IOException {
303 System.out.println("decompressFlz(" + source + ", " + dest + ")");
304 copy(flzInput, destStream);
307 long decompressed = dest.length();
308 System.out.println("Wrote " + decompressed + " decompressed bytes");
313 * Copy the content of the input stream into the output stream, using a temporary
314 * byte array buffer whose size is defined by {@link #IO_BUFFER_SIZE}.
316 * @param in The input stream to copy from.
317 * @param out The output stream to copy to.
319 * @throws IOException If any error occurs during the copy.
321 private static final int IO_BUFFER_SIZE = 128 * 1024;
323 public static long copy(InputStream in, OutputStream out) throws IOException {
324 byte[] b = new byte[IO_BUFFER_SIZE];
327 long start = System.nanoTime();
333 //System.out.println("read " + read + " bytes, " + total + " bytes in total");
334 out.write(b, 0, read);
337 long end = System.nanoTime();
338 double totalmb = total/(1024.0*1024.0);
339 double time = (end-start)*1e-9;
340 double rate = totalmb / time;
341 System.out.format("Transferred %d bytes (%.3f Mbytes) in %f seconds (%.3f MB/s)\n", total, totalmb, time, rate);
346 @SuppressWarnings("unused")
347 private boolean checksumsEqual(File f1, File f2) throws IOException {
348 byte[] s1 = ChecksumUtil.computeSum(f1);
349 byte[] s2 = ChecksumUtil.computeSum(f2);
350 return Arrays.equals(s1, s2);