/******************************************************************************* * Copyright (c) 2007, 2010 Association for Decentralized Information Management * in Industry THTH ry. * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Public License v1.0 * which accompanies this distribution, and is available at * http://www.eclipse.org/legal/epl-v10.html * * Contributors: * VTT Technical Research Centre of Finland - initial API and implementation *******************************************************************************/ package org.simantics.utils; import java.io.BufferedReader; import java.io.Closeable; import java.io.File; import java.io.FileFilter; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.OutputStream; import java.io.RandomAccessFile; import java.io.UnsupportedEncodingException; import java.net.URI; import java.net.URL; import java.nio.channels.FileChannel; import java.nio.charset.Charset; import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.SimpleFileVisitor; import java.nio.file.StandardCopyOption; import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.Arrays; import java.util.Deque; import java.util.LinkedList; import java.util.Random; import java.util.zip.DataFormatException; import java.util.zip.Deflater; import java.util.zip.Inflater; import java.util.zip.ZipEntry; import java.util.zip.ZipException; import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; import org.simantics.databoard.Bindings; import org.simantics.databoard.adapter.AdaptException; import org.simantics.databoard.adapter.Adapter; import org.simantics.databoard.adapter.AdapterConstructionException; import org.simantics.databoard.binding.Binding; import org.simantics.databoard.type.Datatype; import org.simantics.utils.bytes.LEInt; import org.simantics.utils.strings.FileNameUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Utilities for common file operations. * * see StreamUtil in databoard for Input/OutputStream reading/writing utils * @see FileNameUtils for more utils * * @author Toni Kalajainen * @author Tuukka Lehtonen */ public class FileUtils { private static final Logger LOGGER = LoggerFactory.getLogger(FileUtils.class); /** * Create escaped filename * * @param name any string * @return file compatible string */ public static String escapeFileName(String name) { try { return java.net.URLEncoder.encode(name, "UTF-8"); } catch (UnsupportedEncodingException e) { // never expected throw new RuntimeException(e); } } /** * Unescape filename into string * * @param filename * @return any string */ public static String unescapeFileName(String filename) { try { return java.net.URLDecoder.decode(filename, "UTF-8"); } catch (UnsupportedEncodingException e) { // never expected throw new RuntimeException(e); } } public static File ensureParentDirectoryExists(String path) throws IOException { return ensureParentDirectoryExists(new File(path)); } /** * Ensures the parent directory pointed by the specified file path exists as a * directory. * * @param path the directory whose existence is to be ensured * @return the requested directory * @throws IOException if the specified directory cannot be created or * already exists as a file */ public static File ensureParentDirectoryExists(File path) throws IOException { return ensureDirectoryExists(path.getParentFile()); } public static File ensureDirectoryExists(String path) throws IOException { return ensureDirectoryExists(new File(path)); } /** * Ensures the directory pointed by the specified file path exists as a * directory. * * @param path the directory whose existence is to be ensured * @return the requested directory * @throws IOException if the specified directory cannot be created or * already exists as a file */ public static File ensureDirectoryExists(File path) throws IOException { if (path.isDirectory()) // Already exists, everything OK. return path; if (path.exists()) // Path is not a directory but it exists, fail! throw new IOException("file '" + path + "', already exists but it is not a directory"); path.mkdirs(); if (!path.exists()) // Path is not a directory but it exists, fail! throw new IOException("could not create directory '" + path + "' for an unknown reason"); // Directory created OK. return path; } /** * Copies the contents of a source file to the destination file. * * @param sourceFile the source file descriptor * @param destFile the destination file descriptor * @throws IOException when anything IO-related goes wrong during the copy */ public static void copyFile(File sourceFile, File destFile) throws IOException { if (!destFile.exists()) { destFile.createNewFile(); } FileInputStream fis = null; FileOutputStream fos = null; try { fis = new FileInputStream(sourceFile); fos = new FileOutputStream(destFile); FileChannel source = fis.getChannel(); FileChannel destination = fos.getChannel(); long count = 0; long size = source.size(); while (count < size) { count += destination.transferFrom(source, count, size - count); } } finally { if (fis != null) { uncheckedClose(fis); } if (fos != null) { uncheckedClose(fos); } } } /** * Copy file or dir recursively. * * @param src * @param dst * @throws IOException */ public static void copy(File src, File dst) throws IOException { if (src.isDirectory()) { if(!dst.exists()) dst.mkdir(); for (String file : src.list()) { File srcFile = new File(src, file); File dstFile = new File(dst, file); copy(srcFile,dstFile); } } else { copyFile(src, dst); } } // public static File createTempFileFromResource(URL sourceUrl) throws IOException { // sourceUrl = FileLocator.resolve(sourceUrl); // File sourceFile; // try { // if (sourceUrl.getProtocol().equalsIgnoreCase("file")) // sourceFile = new File(sourceUrl.getPath()); // else // sourceFile = new File(sourceUrl.toURI()); // } catch (URISyntaxException e) { // throw new RuntimeException(e); // } // // File tempFile = File.createTempFile("tmp", ".tmp"); // FileUtils.copyFile(sourceFile, tempFile); // return tempFile; // } /** * Reads entire binary file * @param file file * @return contents of binary file * @throws IOException on i/o problems */ public static byte[] readFile(File file) throws IOException { try (FileInputStream fis = new FileInputStream(file)) { long size = file.length(); if (size>Integer.MAX_VALUE) throw new IOException("File too big"); int len = (int) size; byte data [] = new byte[len]; int pos = 0; while (pos filter) throws IOException { if (dir.isFile()) { if (!filter.contains(dir.getAbsolutePath())) { dir.delete(); return; } } if (dir.isDirectory()) { File[] fs = dir.listFiles((FileFilter) null); if (fs == null) return; for (File f : fs) { if (f.isDirectory()) { deleteAllWithFilter(f, filter); } else { if (!filter.contains(f.getAbsolutePath())) { if (!f.delete()) { throw new IOException("Could not delete file: " + f.getAbsolutePath()); } } } } if (!filter.contains(dir.getAbsolutePath())) { if (!dir.delete()) { throw new IOException("Could not delete directory: " + dir.getAbsolutePath()); } } } else if (dir.exists()) { if (filter.contains(dir.getAbsolutePath())) { if (!dir.delete()) { throw new IOException("Could not delete file: " + dir.getAbsolutePath()); } } } } public static ArrayList createFileFilter(File dir, ArrayList filter) { if (filter == null) filter = new ArrayList(); if (dir.isFile()) { filter.add(dir.getAbsolutePath()); return filter; } if (dir.isDirectory()) { File[] fs = dir.listFiles((FileFilter) null); if (fs == null) return filter; for (File f : fs) { if (f.isDirectory()) { createFileFilter(f, filter); } else { filter.add(f.getAbsolutePath()); } } filter.add(dir.getAbsolutePath()); } else if (dir.exists()) { filter.add(dir.getAbsolutePath()); } return filter; } /** * Delete a directory incl. all files and sub-directories by best effort. * Does not throw exceptions if deletion fails, simply tries to delete the * provided directory to the best of Java File API's abilities. * * @param dir * directory to delete recursively * @boolean true if all files were successfully deleted, * false if some or all failed to be deleted */ public static boolean deleteDir(File dir) { if (LOGGER.isDebugEnabled()) LOGGER.debug("Deleting directory "+dir); boolean result = true; if (!dir.isDirectory()) return false; File[] fs = dir.listFiles(); if (fs != null) { for (File f : fs) { if (f.isDirectory()) result &= deleteDir(f); if (f.isFile()) result &= f.delete(); } } boolean ok = dir.delete(); // if (!ok) dir.deleteOnExit(); result &= ok; return result; } public static String deleteDirs(File dir) { if (LOGGER.isDebugEnabled()) LOGGER.debug("Deleting directory "+dir); boolean result = true; if (!dir.isDirectory()) return dir + " is not a directory!"; File[] fs = dir.listFiles(); if (fs != null) { for (File f : fs) { //System.out.println("file f :" + f); if (f.isDirectory()) { boolean a = deleteDir(f); result &= a; } if (f.isFile()) { boolean a = f.delete(); result &= a; } } } boolean ok = dir.delete(); // if (!ok) dir.deleteOnExit(); result &= ok; return "deleteDirs succesful for " + dir + " : " + result; } /** * Closes a stream and ignores any resulting exception. This is useful * when doing stream cleanup in a finally block where secondary exceptions * are not worth logging. */ public static void uncheckedClose(Closeable closeable) { try { if (closeable != null) closeable.close(); } catch (IOException e) { //ignore } } /** * Extracts the specified source file in the specified bundle into the * specified local directory. * * @param url the source URL to stream the resource from * @param targetFile the target file to write the resource to * @param deleteOnExit true to use {@link File#deleteOnExit()} * on the resulting file. Note that this does not guarantee that the * file is deleted when the JVM exits * @return the resulting file * @throws FileNotFoundException */ public static File copyResource(URL url, File targetFile, boolean deleteOnExit) throws IOException, FileNotFoundException { if (url == null) throw new IllegalArgumentException("null url"); FileOutputStream os = null; InputStream is = null; try { if (targetFile.exists()) targetFile.delete(); is = url.openStream(); int read; byte [] buffer = new byte [16384]; os = new FileOutputStream (targetFile); while ((read = is.read (buffer)) != -1) { os.write(buffer, 0, read); } os.close (); is.close (); // Request removal of the extracted files on JVM exit. if (deleteOnExit) targetFile.deleteOnExit(); return targetFile; } finally { FileUtils.uncheckedClose(os); FileUtils.uncheckedClose(is); } } /** * Creates the requested location under the system's temporary directories * for temporary data storage. * * @param pathSegments the path segments that are appended to * java.io.tmpdir. The path segments musn't contain path separators. * @return the resulting temporary directory as a File object * @throws IOException if anything goes wrong */ public static File getOrCreateTemporaryDirectory(boolean requireEmptyDirectory, String... pathSegments) throws IOException { String separator = System.getProperty("file.separator"); String tempDir = System.getProperty("java.io.tmpdir"); if (tempDir == null) throw new IllegalStateException("'java.io.tmpdir' property is not defined, is the system TMP environment variable defined?"); if (!hasTrailingSeparator(tempDir)) tempDir = tempDir + separator; for (int i = 0; i < pathSegments.length-1; ++i) { if (containsSeparator(pathSegments[i])) throw new IllegalArgumentException("path segments contain path separators: " + Arrays.toString(pathSegments)); tempDir = pathSegments[i] + separator; } // Find name for an empty or non-existing temporary directory if (pathSegments.length > 0) { String lastSegment = pathSegments[pathSegments.length - 1]; String suffix = ""; int counter = 0; // This loop will not exit until it succeeds or some error occurs. while (true) { File temp = new File(tempDir + lastSegment + suffix); if (!temp.exists()) { // Found non-existent temporary directory! if (temp.mkdirs()) { // Ok, everything seems to be fine. return temp; } // For some reason the directory could not be created, lets // try another name. } else { // Found an existing temporary file. if (temp.isDirectory() && temp.canWrite()) { if (requireEmptyDirectory) { String[] files = temp.list(); if (files != null) { if (files.length == 0) { // Ok, the directory is empty and writable. return temp; } } } else { return temp; } // Either the directory was empty or it could not be // checked. In any case, don't use the directory. } } // Try another directory name if no success with this one. suffix = Integer.toHexString(counter); ++counter; } } return new File(tempDir); } private static boolean containsSeparator(String s) { return s.contains("/") || s.contains("\\"); } private static boolean hasTrailingSeparator(String s) { return s.endsWith("/") || s.endsWith("\\"); } /** * Very simple compression using ZLib * @param input the uncompressed data * @return the compressed data */ public static byte[] deflate(byte[] input) { // Compress the bytes Deflater compresser = new Deflater(Deflater.BEST_COMPRESSION); compresser.setInput(input); compresser.finish(); int bufferSize = input.length<16 ? 16 : input.length; byte buffer[] = new byte[bufferSize]; int bufferPos = 0; do { int compressedDataLength = compresser.deflate(buffer, bufferPos, buffer.length-bufferPos); bufferPos += compressedDataLength; if (!compresser.finished()) buffer = Arrays.copyOf(buffer, buffer.length + bufferSize); } while (!compresser.finished()); byte[] result = new byte[bufferPos+4]; System.arraycopy(buffer, 0, result, 4, bufferPos); byte sizeData[] = LEInt.toBytes(input.length); System.arraycopy(sizeData, 0, result, 0, 4); return result; } /** * Very simple decompression using ZLib. * @param input the compressed input * @return the uncompressed data * @throws DataFormatException */ public static byte[] inflate(byte[] input) throws DataFormatException { // Decompress the bytes int inflatedSize = LEInt.toInt(input); Inflater decompresser = new Inflater(); decompresser.setInput(input, 4, input.length - 4); byte[] result = new byte[inflatedSize]; int resultLength = decompresser.inflate(result); assert(resultLength == inflatedSize); decompresser.end(); return result; } public static boolean isValidFileName(String name) { File f = new File(name) ; if (!f.exists()) { try { boolean ok = f.createNewFile(); if (ok) f.delete(); return ok; } catch (IOException ioe) { return false; } } return true; } /** * Create a temporary directory * * @return temporary directory */ public static File createTmpDir() { String tmp = System.getenv("tmp"); if (tmp==null) tmp = "c:/temp"; Random r = new Random(); String randomName = "simantics-tmp-"+(r.nextInt(10000)+10000); File tmpDir = new File(tmp+"/"+randomName); tmpDir.deleteOnExit(); Boolean ok = tmpDir.mkdirs(); if (!ok) throw new RuntimeException("tmp dir "+tmpDir+" was not created"); return tmpDir; } public static void compressZip(String sourcePath, String zipDir) throws IOException { if (LOGGER.isDebugEnabled()) LOGGER.debug("Compressing file " + sourcePath + " to zip " + zipDir + "."); File filesource = new File(sourcePath); URI base = filesource.toURI(); Deque queue = new LinkedList(); queue.push(filesource); try (ZipOutputStream zout = new ZipOutputStream(new FileOutputStream(zipDir))) { while (!queue.isEmpty()) { filesource = queue.pop(); for (File child : filesource.listFiles()) { String name = base.relativize(child.toURI()).getPath(); if (child.isDirectory()) { queue.push(child); name = name.endsWith("/") ? name : name + "/"; zout.putNextEntry(new ZipEntry(name)); } else { zout.putNextEntry(new ZipEntry(name)); copy(child, zout); zout.closeEntry(); } } } } finally { LOGGER.debug("Filecompression done."); } } private static void copy(File file, ZipOutputStream zout) throws IOException { try (InputStream in = new FileInputStream(file)) { copy(in, zout); } } /** * Extract a zip file into a directory * * @param zipFile * @param dst * @throws IOException */ public static void extractZip(File zipFile, File dst) throws IOException { if (LOGGER.isDebugEnabled()) LOGGER.debug("Extracting zip "+zipFile); try (FileInputStream fis = new FileInputStream(zipFile)) { extractZip(fis, dst); } } /** * Extract a zip file into a directory * * @param zipInput * @param dst directory * @throws IOException */ public static void extractZip(InputStream zipInput, File dst) throws IOException { byte[] buf = new byte[8192]; ZipInputStream zis = new ZipInputStream(zipInput); ZipEntry entry; entry = zis.getNextEntry(); while (entry != null) { // for each entry to be extracted String name = entry.getName(); if (LOGGER.isDebugEnabled()) LOGGER.debug("Extracting "+name); File file = new File(dst, name); if (entry.isDirectory()) { if ( !file.exists() ) file.mkdirs(); } else { File parent = file.getParentFile(); if (!parent.exists()) parent.mkdirs(); if (!file.exists()) file.createNewFile(); FileOutputStream fileoutputstream = new FileOutputStream(file); try { int n = 0; while ((n = zis.read(buf, 0, buf.length)) > -1) fileoutputstream.write(buf, 0, n); } catch (ZipException e) { throw new IOException("Failed to extract '" + name + "'.", e); } finally { fileoutputstream.close(); } } zis.closeEntry(); entry = zis.getNextEntry(); }// while zis.close(); } // Test inflate & deflate public static void main(String[] args) { try { // Test compression String str = "abcdefghijklmnopqrstuvwxyz"; byte data[] = str.getBytes(); System.out.println("Data:\t"+data.length); byte deflated[] = deflate(data); System.out.println("Deflated:\t"+deflated.length); byte inflated[] = inflate(deflated); System.out.println("Inflated:\t"+inflated.length); String str2 = new String(inflated); System.out.println("Strings are equal: "+str.endsWith(str2)); } catch (DataFormatException e) { e.printStackTrace(); } } public static File[] listFilesByExtension(File directory, final String extension) { return directory.listFiles(new ExtensionFilter(extension)); } /** * Copy the content of the input stream into the output stream, using a temporary * byte array buffer whose size is defined by {@link #IO_BUFFER_SIZE}. * * @param in The input stream to copy from. * @param out The output stream to copy to. * * @throws IOException If any error occurs during the copy. */ private static final int IO_BUFFER_SIZE = 64 * 1024; public static void copy(InputStream in, OutputStream out) throws IOException { byte[] b = new byte[IO_BUFFER_SIZE]; int read; while (true) { read = in.read(b); if (read < 0) break; out.write(b, 0, read); } } /** * @param in * @param out * @param maxBytesToCopy the maximum amount of bytes to copy * @return the amount of bytes copied * @throws IOException */ public static long copy(InputStream in, OutputStream out, long maxBytesToCopy) throws IOException { byte[] b = new byte[IO_BUFFER_SIZE]; int read = 0; while (read < maxBytesToCopy) { int l = (int) Math.min((long) IO_BUFFER_SIZE, maxBytesToCopy-read); int r = in.read(b, 0, l); if (r < 0) break; out.write(b, 0, r); read += r; } return read; } public static void delete(Path databaseLocation) throws IOException { Files.walkFileTree(databaseLocation, new DeleteDirectoriesVisitor()); } public static void copy(Path from, Path to) throws IOException { Files.walkFileTree(from, new CopyDirectoriesVisitor(from, to)); } public static class DeleteDirectoriesVisitor extends SimpleFileVisitor { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { Files.delete(file); if (Files.exists(file)) throw new IOException("Could not delete file " + file.toAbsolutePath().toString()); return FileVisitResult.CONTINUE; } @Override public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { if (exc != null) throw exc; Files.delete(dir); if (Files.exists(dir)) throw new IOException("Could not delete file " + dir.toAbsolutePath().toString()); return FileVisitResult.CONTINUE; } } public static class CopyDirectoriesVisitor extends SimpleFileVisitor { private final Path fromPath; private final Path toPath; public CopyDirectoriesVisitor(Path fromPath, Path toPath) { this.fromPath = fromPath; this.toPath = toPath; } @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { Path targetPath = toPath.resolve(fromPath.relativize(dir)); if(!Files.exists(targetPath)){ Files.createDirectory(targetPath); } return FileVisitResult.CONTINUE; } @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { Files.copy(file, toPath.resolve(fromPath.relativize(file)), StandardCopyOption.REPLACE_EXISTING); return FileVisitResult.CONTINUE; } } }