X-Git-Url: https://gerrit.simantics.org/r/gitweb?p=simantics%2Fplatform.git;a=blobdiff_plain;f=bundles%2Forg.simantics.utils%2Fsrc%2Forg%2Fsimantics%2Futils%2FFileUtils.java;h=44a3dfcf5731b9ff2c9334959a66e3463416f283;hp=51449d57bf4814a03e788b66c20c9e7f66996a25;hb=ad8fc537d4cde0d8891cf1cd39862055ca7f03cb;hpb=c125a1755cc7c4a6241c3c5bf841c3db0ff2d658 diff --git a/bundles/org.simantics.utils/src/org/simantics/utils/FileUtils.java b/bundles/org.simantics.utils/src/org/simantics/utils/FileUtils.java index 51449d57b..44a3dfcf5 100644 --- a/bundles/org.simantics.utils/src/org/simantics/utils/FileUtils.java +++ b/bundles/org.simantics.utils/src/org/simantics/utils/FileUtils.java @@ -1,1016 +1,1081 @@ -/******************************************************************************* - * Copyright (c) 2007, 2010 Association for Decentralized Information Management - * in Industry THTH ry. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Eclipse Public License v1.0 - * which accompanies this distribution, and is available at - * http://www.eclipse.org/legal/epl-v10.html - * - * Contributors: - * VTT Technical Research Centre of Finland - initial API and implementation - *******************************************************************************/ -package org.simantics.utils; - -import java.io.BufferedReader; -import java.io.Closeable; -import java.io.File; -import java.io.FileFilter; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.io.RandomAccessFile; -import java.io.UnsupportedEncodingException; -import java.net.URI; -import java.net.URL; -import java.nio.channels.FileChannel; -import java.nio.charset.Charset; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.StandardCopyOption; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Deque; -import java.util.LinkedList; -import java.util.Random; -import java.util.zip.DataFormatException; -import java.util.zip.Deflater; -import java.util.zip.Inflater; -import java.util.zip.ZipEntry; -import java.util.zip.ZipException; -import java.util.zip.ZipInputStream; -import java.util.zip.ZipOutputStream; - -import org.simantics.databoard.Bindings; -import org.simantics.databoard.adapter.AdaptException; -import org.simantics.databoard.adapter.Adapter; -import org.simantics.databoard.adapter.AdapterConstructionException; -import org.simantics.databoard.binding.Binding; -import org.simantics.databoard.type.Datatype; -import org.simantics.utils.bytes.LEInt; -import org.simantics.utils.strings.FileNameUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Utilities for common file operations. - * - * see StreamUtil in databoard for Input/OutputStream reading/writing utils - * @see FileNameUtils for more utils - * - * @author Toni Kalajainen - * @author Tuukka Lehtonen - */ -public class FileUtils { - - private static final Logger LOGGER = LoggerFactory.getLogger(FileUtils.class); - - /** - * Create escaped filename - * - * @param name any string - * @return file compatible string - */ - public static String escapeFileName(String name) { - try { - return java.net.URLEncoder.encode(name, "UTF-8"); - } catch (UnsupportedEncodingException e) { - // never expected - throw new RuntimeException(e); - } - } - - /** - * Unescape filename into string - * - * @param filename - * @return any string - */ - public static String unescapeFileName(String filename) { - try { - return java.net.URLDecoder.decode(filename, "UTF-8"); - } catch (UnsupportedEncodingException e) { - // never expected - throw new RuntimeException(e); - } - } - - public static File ensureParentDirectoryExists(String path) throws IOException { - return ensureParentDirectoryExists(new File(path)); - } - - /** - * Ensures the parent directory pointed by the specified file path exists as a - * directory. - * - * @param path the directory whose existence is to be ensured - * @return the requested directory - * @throws IOException if the specified directory cannot be created or - * already exists as a file - */ - public static File ensureParentDirectoryExists(File path) throws IOException { - return ensureDirectoryExists(path.getParentFile()); - } - - - public static File ensureDirectoryExists(String path) throws IOException { - return ensureDirectoryExists(new File(path)); - } - - /** - * Ensures the directory pointed by the specified file path exists as a - * directory. - * - * @param path the directory whose existence is to be ensured - * @return the requested directory - * @throws IOException if the specified directory cannot be created or - * already exists as a file - */ - public static File ensureDirectoryExists(File path) throws IOException { - if (path.isDirectory()) - // Already exists, everything OK. - return path; - - if (path.exists()) - // Path is not a directory but it exists, fail! - throw new IOException("file '" + path + "', already exists but it is not a directory"); - - path.mkdirs(); - if (!path.exists()) - // Path is not a directory but it exists, fail! - throw new IOException("could not create directory '" + path + "' for an unknown reason"); - - // Directory created OK. - return path; - } - - /** - * Copies the contents of a source file to the destination file. - * - * @param sourceFile the source file descriptor - * @param destFile the destination file descriptor - * @throws IOException when anything IO-related goes wrong during the copy - */ - public static void copyFile(File sourceFile, File destFile) throws IOException { - if (!destFile.exists()) { - destFile.createNewFile(); - } - - FileInputStream fis = null; - FileOutputStream fos = null; - try { - fis = new FileInputStream(sourceFile); - fos = new FileOutputStream(destFile); - FileChannel source = fis.getChannel(); - FileChannel destination = fos.getChannel(); - - long count = 0; - long size = source.size(); - while (count < size) { - count += destination.transferFrom(source, count, size - count); - } - } finally { - if (fis != null) { - uncheckedClose(fis); - } - if (fos != null) { - uncheckedClose(fos); - } - } - } - - /** - * Copy file or dir recursively. - * - * @param src - * @param dst - * @throws IOException - */ - public static void copy(File src, File dst) throws IOException - { - if (src.isDirectory()) { - if(!dst.exists()) dst.mkdir(); - - for (String file : src.list()) { - File srcFile = new File(src, file); - File dstFile = new File(dst, file); - copy(srcFile,dstFile); - } - } else { - copyFile(src, dst); - } - } - - -// public static File createTempFileFromResource(URL sourceUrl) throws IOException { -// sourceUrl = FileLocator.resolve(sourceUrl); -// File sourceFile; -// try { -// if (sourceUrl.getProtocol().equalsIgnoreCase("file")) -// sourceFile = new File(sourceUrl.getPath()); -// else -// sourceFile = new File(sourceUrl.toURI()); -// } catch (URISyntaxException e) { -// throw new RuntimeException(e); -// } -// -// File tempFile = File.createTempFile("tmp", ".tmp"); -// FileUtils.copyFile(sourceFile, tempFile); -// return tempFile; -// } - - /** - * Reads entire binary file - * @param file file - * @return contents of binary file - * @throws IOException on i/o problems - */ - public static byte[] readFile(File file) - throws IOException - { - try (FileInputStream fis = new FileInputStream(file)) { - long size = file.length(); - if (size>Integer.MAX_VALUE) - throw new IOException("File too big"); - int len = (int) size; - byte data [] = new byte[len]; - int pos = 0; - - while (pos filter) throws IOException { - if (dir.isFile()) { - if (!filter.contains(dir.getAbsolutePath())) { - dir.delete(); - return; - } - } - - if (dir.isDirectory()) { - File[] fs = dir.listFiles((FileFilter) null); - if (fs == null) - return; - - for (File f : fs) { - if (f.isDirectory()) { - deleteAllWithFilter(f, filter); - } else { - if (!filter.contains(f.getAbsolutePath())) { - if (!f.delete()) { - throw new IOException("Could not delete file: " + f.getAbsolutePath()); - } - } - } - } - if (!filter.contains(dir.getAbsolutePath())) { - if (!dir.delete()) { - throw new IOException("Could not delete directory: " + dir.getAbsolutePath()); - } - } - } else if (dir.exists()) { - if (filter.contains(dir.getAbsolutePath())) { - if (!dir.delete()) { - throw new IOException("Could not delete file: " + dir.getAbsolutePath()); - } - } - } - } - - public static ArrayList createFileFilter(File dir, ArrayList filter) { - if (filter == null) - filter = new ArrayList(); - if (dir.isFile()) { - filter.add(dir.getAbsolutePath()); - return filter; - } - - if (dir.isDirectory()) { - File[] fs = dir.listFiles((FileFilter) null); - if (fs == null) - return filter; - - for (File f : fs) { - if (f.isDirectory()) { - createFileFilter(f, filter); - } else { - filter.add(f.getAbsolutePath()); - } - } - filter.add(dir.getAbsolutePath()); - } else if (dir.exists()) { - filter.add(dir.getAbsolutePath()); - } - return filter; - } - - - /** - * Delete a directory incl. all files and sub-directories by best effort. - * Does not throw exceptions if deletion fails, simply tries to delete the - * provided directory to the best of Java File API's abilities. - * - * @param dir - * directory to delete recursively - * @boolean true if all files were successfully deleted, - * false if some or all failed to be deleted - */ - public static boolean deleteDir(File dir) { - if (LOGGER.isDebugEnabled()) - LOGGER.debug("Deleting directory "+dir); - boolean result = true; - - if (!dir.isDirectory()) return false; - File[] fs = dir.listFiles(); - if (fs != null) { - for (File f : fs) { - if (f.isDirectory()) result &= deleteDir(f); - if (f.isFile()) result &= f.delete(); - } - } - boolean ok = dir.delete(); -// if (!ok) dir.deleteOnExit(); - result &= ok; - return result; - } - - public static String deleteDirs(File dir) { - if (LOGGER.isDebugEnabled()) - LOGGER.debug("Deleting directory "+dir); - boolean result = true; - - if (!dir.isDirectory()) - return dir + " is not a directory!"; - File[] fs = dir.listFiles(); - if (fs != null) { - for (File f : fs) { - //System.out.println("file f :" + f); - if (f.isDirectory()) { - boolean a = deleteDir(f); - result &= a; - } - if (f.isFile()) { - boolean a = f.delete(); - result &= a; - } - - } - } - boolean ok = dir.delete(); -// if (!ok) dir.deleteOnExit(); - result &= ok; - return "deleteDirs succesful for " + dir + " : " + result; - } - - /** - * Closes a stream and ignores any resulting exception. This is useful - * when doing stream cleanup in a finally block where secondary exceptions - * are not worth logging. - */ - public static void uncheckedClose(Closeable closeable) { - try { - if (closeable != null) - closeable.close(); - } catch (IOException e) { - //ignore - } - } - - /** - * Extracts the specified source file in the specified bundle into the - * specified local directory. - * - * @param url the source URL to stream the resource from - * @param targetFile the target file to write the resource to - * @param deleteOnExit true to use {@link File#deleteOnExit()} - * on the resulting file. Note that this does not guarantee that the - * file is deleted when the JVM exits - * @return the resulting file - * @throws FileNotFoundException - */ - public static File copyResource(URL url, File targetFile, boolean deleteOnExit) throws IOException, FileNotFoundException { - if (url == null) - throw new IllegalArgumentException("null url"); - - FileOutputStream os = null; - InputStream is = null; - try { - if (targetFile.exists()) - targetFile.delete(); - - is = url.openStream(); - int read; - byte [] buffer = new byte [16384]; - os = new FileOutputStream (targetFile); - while ((read = is.read (buffer)) != -1) { - os.write(buffer, 0, read); - } - os.close (); - is.close (); - - // Request removal of the extracted files on JVM exit. - if (deleteOnExit) - targetFile.deleteOnExit(); - return targetFile; - } finally { - FileUtils.uncheckedClose(os); - FileUtils.uncheckedClose(is); - } - } - - /** - * Creates the requested location under the system's temporary directories - * for temporary data storage. - * - * @param pathSegments the path segments that are appended to - * java.io.tmpdir. The path segments musn't contain path separators. - * @return the resulting temporary directory as a File object - * @throws IOException if anything goes wrong - */ - public static File getOrCreateTemporaryDirectory(boolean requireEmptyDirectory, String... pathSegments) throws IOException { - String separator = System.getProperty("file.separator"); - String tempDir = System.getProperty("java.io.tmpdir"); - if (tempDir == null) - throw new IllegalStateException("'java.io.tmpdir' property is not defined, is the system TMP environment variable defined?"); - - if (!hasTrailingSeparator(tempDir)) - tempDir = tempDir + separator; - - for (int i = 0; i < pathSegments.length-1; ++i) { - if (containsSeparator(pathSegments[i])) - throw new IllegalArgumentException("path segments contain path separators: " + Arrays.toString(pathSegments)); - tempDir = pathSegments[i] + separator; - } - - // Find name for an empty or non-existing temporary directory - if (pathSegments.length > 0) { - String lastSegment = pathSegments[pathSegments.length - 1]; - String suffix = ""; - int counter = 0; - // This loop will not exit until it succeeds or some error occurs. - while (true) { - File temp = new File(tempDir + lastSegment + suffix); - - if (!temp.exists()) { - // Found non-existent temporary directory! - if (temp.mkdirs()) { - // Ok, everything seems to be fine. - return temp; - } - // For some reason the directory could not be created, lets - // try another name. - } else { - // Found an existing temporary file. - if (temp.isDirectory() && temp.canWrite()) { - if (requireEmptyDirectory) { - String[] files = temp.list(); - if (files != null) { - if (files.length == 0) { - // Ok, the directory is empty and writable. - return temp; - } - } - } else { - return temp; - } - // Either the directory was empty or it could not be - // checked. In any case, don't use the directory. - } - } - - // Try another directory name if no success with this one. - suffix = Integer.toHexString(counter); - ++counter; - } - } - return new File(tempDir); - } - - private static boolean containsSeparator(String s) { - return s.contains("/") || s.contains("\\"); - } - - private static boolean hasTrailingSeparator(String s) { - return s.endsWith("/") || s.endsWith("\\"); - } - - /** - * Very simple compression using ZLib - * @param input the uncompressed data - * @return the compressed data - */ - public static byte[] deflate(byte[] input) { - // Compress the bytes - Deflater compresser = new Deflater(Deflater.BEST_COMPRESSION); - compresser.setInput(input); - compresser.finish(); - - int bufferSize = input.length<16 ? 16 : input.length; - byte buffer[] = new byte[bufferSize]; - int bufferPos = 0; - do { - int compressedDataLength = compresser.deflate(buffer, bufferPos, buffer.length-bufferPos); - bufferPos += compressedDataLength; - if (!compresser.finished()) - buffer = Arrays.copyOf(buffer, buffer.length + bufferSize); - } while (!compresser.finished()); - - byte[] result = new byte[bufferPos+4]; - System.arraycopy(buffer, 0, result, 4, bufferPos); - byte sizeData[] = LEInt.toBytes(input.length); - System.arraycopy(sizeData, 0, result, 0, 4); - return result; - } - - /** - * Very simple decompression using ZLib. - * @param input the compressed input - * @return the uncompressed data - * @throws DataFormatException - */ - public static byte[] inflate(byte[] input) throws DataFormatException { - // Decompress the bytes - int inflatedSize = LEInt.toInt(input); - Inflater decompresser = new Inflater(); - decompresser.setInput(input, 4, input.length - 4); - byte[] result = new byte[inflatedSize]; - int resultLength = decompresser.inflate(result); - assert(resultLength == inflatedSize); - decompresser.end(); - return result; - } - - - public static boolean isValidFileName(String name) { - File f = new File(name) ; - if (!f.exists()) { - try { - boolean ok = f.createNewFile(); - if (ok) - f.delete(); - return ok; - } catch (IOException ioe) { - return false; - } - } - return true; - } - - /** - * Create a temporary directory - * - * @return temporary directory - */ - public static File createTmpDir() - { - String tmp = System.getenv("tmp"); - if (tmp==null) tmp = "c:/temp"; - Random r = new Random(); - String randomName = "simantics-tmp-"+(r.nextInt(10000)+10000); - File tmpDir = new File(tmp+"/"+randomName); - tmpDir.deleteOnExit(); - Boolean ok = tmpDir.mkdirs(); - if (!ok) throw new RuntimeException("tmp dir "+tmpDir+" was not created"); - return tmpDir; - } - - public static void compressZip(String sourcePath, String zipDir) throws IOException { - if (LOGGER.isDebugEnabled()) - LOGGER.debug("Compressing file " + sourcePath + " to zip " + zipDir + "."); - - File filesource = new File(sourcePath); - URI base = filesource.toURI(); - Deque queue = new LinkedList(); - queue.push(filesource); - try (ZipOutputStream zout = new ZipOutputStream(new FileOutputStream(zipDir))) { - while (!queue.isEmpty()) { - filesource = queue.pop(); - for (File child : filesource.listFiles()) { - String name = base.relativize(child.toURI()).getPath(); - if (child.isDirectory()) { - queue.push(child); - name = name.endsWith("/") ? name : name + "/"; - zout.putNextEntry(new ZipEntry(name)); - } else { - zout.putNextEntry(new ZipEntry(name)); - copy(child, zout); - zout.closeEntry(); - } - } - } - } finally { - LOGGER.debug("Filecompression done."); - } - } - - private static void copy(File file, ZipOutputStream zout) throws IOException { - try (InputStream in = new FileInputStream(file)) { - copy(in, zout); - } - } - - /** - * Extract a zip file into a directory - * - * @param zipFile - * @param dst - * @throws IOException - */ - public static void extractZip(File zipFile, File dst) throws IOException { - if (LOGGER.isDebugEnabled()) - LOGGER.debug("Extracting zip "+zipFile); - try (FileInputStream fis = new FileInputStream(zipFile)) { - extractZip(fis, dst); - } - } - - /** - * Extract a zip file into a directory - * - * @param zipInput - * @param dst directory - * @throws IOException - */ - public static void extractZip(InputStream zipInput, File dst) throws IOException { - byte[] buf = new byte[8192]; - ZipInputStream zis = new ZipInputStream(zipInput); - ZipEntry entry; - - entry = zis.getNextEntry(); - while (entry != null) { - // for each entry to be extracted - String name = entry.getName(); - if (LOGGER.isDebugEnabled()) - LOGGER.debug("Extracting "+name); - File file = new File(dst, name); - - if (entry.isDirectory()) - { - if ( !file.exists() ) file.mkdirs(); - } else { - File parent = file.getParentFile(); - if (!parent.exists()) parent.mkdirs(); - if (!file.exists()) file.createNewFile(); - - FileOutputStream fileoutputstream = new FileOutputStream(file); - try { - int n = 0; - while ((n = zis.read(buf, 0, buf.length)) > -1) - fileoutputstream.write(buf, 0, n); - } catch (ZipException e) { - throw new IOException("Failed to extract '" + name + "'.", e); - } finally { - fileoutputstream.close(); - } - } - - zis.closeEntry(); - entry = zis.getNextEntry(); - }// while - - zis.close(); - } - - // Test inflate & deflate - public static void main(String[] args) { - try { - // Test compression - String str = "abcdefghijklmnopqrstuvwxyz"; - byte data[] = str.getBytes(); - System.out.println("Data:\t"+data.length); - byte deflated[] = deflate(data); - System.out.println("Deflated:\t"+deflated.length); - byte inflated[] = inflate(deflated); - System.out.println("Inflated:\t"+inflated.length); - String str2 = new String(inflated); - System.out.println("Strings are equal: "+str.endsWith(str2)); - } catch (DataFormatException e) { - e.printStackTrace(); - } - } - - public static File[] listFilesByExtension(File directory, final String extension) { - return directory.listFiles(new ExtensionFilter(extension)); - } - - /** - * Copy the content of the input stream into the output stream, using a temporary - * byte array buffer whose size is defined by {@link #IO_BUFFER_SIZE}. - * - * @param in The input stream to copy from. - * @param out The output stream to copy to. - * - * @throws IOException If any error occurs during the copy. - */ - private static final int IO_BUFFER_SIZE = 64 * 1024; - - public static void copy(InputStream in, OutputStream out) throws IOException { - byte[] b = new byte[IO_BUFFER_SIZE]; - int read; - while (true) { - read = in.read(b); - if (read < 0) - break; - out.write(b, 0, read); - } - } - - /** - * @param in - * @param out - * @param maxBytesToCopy the maximum amount of bytes to copy - * @return the amount of bytes copied - * @throws IOException - */ - public static long copy(InputStream in, OutputStream out, long maxBytesToCopy) throws IOException { - byte[] b = new byte[IO_BUFFER_SIZE]; - int read = 0; - while (read < maxBytesToCopy) { - int l = (int) Math.min((long) IO_BUFFER_SIZE, maxBytesToCopy-read); - int r = in.read(b, 0, l); - if (r < 0) - break; - out.write(b, 0, r); - read += r; - } - return read; - } - - public static void delete(Path databaseLocation) throws IOException { - Files.walkFileTree(databaseLocation, new DeleteDirectoriesVisitor()); - } - - public static void copy(Path from, Path to) throws IOException { - Files.walkFileTree(from, new CopyDirectoriesVisitor(from, to)); - } - - public static class DeleteDirectoriesVisitor extends SimpleFileVisitor { - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - Files.delete(file); - if (Files.exists(file)) - throw new IOException("Could not delete file " + file.toAbsolutePath().toString()); - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { - if (exc != null) - throw exc; - Files.delete(dir); - if (Files.exists(dir)) - throw new IOException("Could not delete file " + dir.toAbsolutePath().toString()); - return FileVisitResult.CONTINUE; - } - } - - public static class CopyDirectoriesVisitor extends SimpleFileVisitor { - - private final Path fromPath; - private final Path toPath; - - public CopyDirectoriesVisitor(Path fromPath, Path toPath) { - this.fromPath = fromPath; - this.toPath = toPath; - } - - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - Path targetPath = toPath.resolve(fromPath.relativize(dir)); - if(!Files.exists(targetPath)){ - Files.createDirectory(targetPath); - } - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - Files.copy(file, toPath.resolve(fromPath.relativize(file)), StandardCopyOption.REPLACE_EXISTING); - return FileVisitResult.CONTINUE; - } - } -} +/******************************************************************************* + * Copyright (c) 2007, 2010 Association for Decentralized Information Management + * in Industry THTH ry. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * VTT Technical Research Centre of Finland - initial API and implementation + *******************************************************************************/ +package org.simantics.utils; + +import java.io.BufferedReader; +import java.io.Closeable; +import java.io.File; +import java.io.FileFilter; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.RandomAccessFile; +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URL; +import java.nio.channels.FileChannel; +import java.nio.charset.Charset; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Deque; +import java.util.LinkedList; +import java.util.Random; +import java.util.function.Predicate; +import java.util.zip.DataFormatException; +import java.util.zip.Deflater; +import java.util.zip.Inflater; +import java.util.zip.ZipEntry; +import java.util.zip.ZipException; +import java.util.zip.ZipInputStream; +import java.util.zip.ZipOutputStream; + +import org.simantics.databoard.Bindings; +import org.simantics.databoard.adapter.AdaptException; +import org.simantics.databoard.adapter.Adapter; +import org.simantics.databoard.adapter.AdapterConstructionException; +import org.simantics.databoard.binding.Binding; +import org.simantics.databoard.type.Datatype; +import org.simantics.utils.bytes.LEInt; +import org.simantics.utils.strings.FileNameUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utilities for common file operations. + * + * see StreamUtil in databoard for Input/OutputStream reading/writing utils + * @see FileNameUtils for more utils + * + * @author Toni Kalajainen + * @author Tuukka Lehtonen + */ +public class FileUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(FileUtils.class); + + /** + * Create escaped filename + * + * @param name any string + * @return file compatible string + */ + public static String escapeFileName(String name) { + try { + return java.net.URLEncoder.encode(name, "UTF-8"); + } catch (UnsupportedEncodingException e) { + // never expected + throw new RuntimeException(e); + } + } + + /** + * Unescape filename into string + * + * @param filename + * @return any string + */ + public static String unescapeFileName(String filename) { + try { + return java.net.URLDecoder.decode(filename, "UTF-8"); + } catch (UnsupportedEncodingException e) { + // never expected + throw new RuntimeException(e); + } + } + + public static File ensureParentDirectoryExists(String path) throws IOException { + return ensureParentDirectoryExists(new File(path)); + } + + /** + * Ensures the parent directory pointed by the specified file path exists as a + * directory. + * + * @param path the directory whose existence is to be ensured + * @return the requested directory + * @throws IOException if the specified directory cannot be created or + * already exists as a file + */ + public static File ensureParentDirectoryExists(File path) throws IOException { + return ensureDirectoryExists(path.getParentFile()); + } + + + public static File ensureDirectoryExists(String path) throws IOException { + return ensureDirectoryExists(new File(path)); + } + + /** + * Ensures the directory pointed by the specified file path exists as a + * directory. + * + * @param path the directory whose existence is to be ensured + * @return the requested directory + * @throws IOException if the specified directory cannot be created or + * already exists as a file + */ + public static File ensureDirectoryExists(File path) throws IOException { + if (path.isDirectory()) + // Already exists, everything OK. + return path; + + if (path.exists()) + // Path is not a directory but it exists, fail! + throw new IOException("file '" + path + "', already exists but it is not a directory"); + + path.mkdirs(); + if (!path.exists()) + // Path is not a directory but it exists, fail! + throw new IOException("could not create directory '" + path + "' for an unknown reason"); + + // Directory created OK. + return path; + } + + /** + * Copies the contents of a source file to the destination file. + * + * @param sourceFile the source file descriptor + * @param destFile the destination file descriptor + * @throws IOException when anything IO-related goes wrong during the copy + */ + public static void copyFile(File sourceFile, File destFile) throws IOException { + if (!destFile.exists()) { + destFile.createNewFile(); + } + + FileInputStream fis = null; + FileOutputStream fos = null; + try { + fis = new FileInputStream(sourceFile); + fos = new FileOutputStream(destFile); + FileChannel source = fis.getChannel(); + FileChannel destination = fos.getChannel(); + + long count = 0; + long size = source.size(); + while (count < size) { + count += destination.transferFrom(source, count, size - count); + } + } finally { + if (fis != null) { + uncheckedClose(fis); + } + if (fos != null) { + uncheckedClose(fos); + } + } + } + + /** + * Copy file or dir recursively. + * + * @param src + * @param dst + * @throws IOException + */ + public static void copy(File src, File dst) throws IOException + { + if (src.isDirectory()) { + if(!dst.exists()) dst.mkdir(); + + for (String file : src.list()) { + File srcFile = new File(src, file); + File dstFile = new File(dst, file); + copy(srcFile,dstFile); + } + } else { + copyFile(src, dst); + } + } + + +// public static File createTempFileFromResource(URL sourceUrl) throws IOException { +// sourceUrl = FileLocator.resolve(sourceUrl); +// File sourceFile; +// try { +// if (sourceUrl.getProtocol().equalsIgnoreCase("file")) +// sourceFile = new File(sourceUrl.getPath()); +// else +// sourceFile = new File(sourceUrl.toURI()); +// } catch (URISyntaxException e) { +// throw new RuntimeException(e); +// } +// +// File tempFile = File.createTempFile("tmp", ".tmp"); +// FileUtils.copyFile(sourceFile, tempFile); +// return tempFile; +// } + + /** + * Reads entire binary file + * @param file file + * @return contents of binary file + * @throws IOException on i/o problems + */ + public static byte[] readFile(File file) + throws IOException + { + try (FileInputStream fis = new FileInputStream(file)) { + long size = file.length(); + if (size>Integer.MAX_VALUE) + throw new IOException("File too big"); + int len = (int) size; + byte data [] = new byte[len]; + int pos = 0; + + while (pos filter) throws IOException { + if (dir.isFile()) { + if (!filter.contains(dir.getAbsolutePath())) { + dir.delete(); + return; + } + } + + if (dir.isDirectory()) { + File[] fs = dir.listFiles((FileFilter) null); + if (fs == null) + return; + + for (File f : fs) { + if (f.isDirectory()) { + deleteAllWithFilter(f, filter); + } else { + if (!filter.contains(f.getAbsolutePath())) { + if (!f.delete()) { + throw new IOException("Could not delete file: " + f.getAbsolutePath()); + } + } + } + } + if (!filter.contains(dir.getAbsolutePath())) { + if (!dir.delete()) { + throw new IOException("Could not delete directory: " + dir.getAbsolutePath()); + } + } + } else if (dir.exists()) { + if (!filter.contains(dir.getAbsolutePath())) { + if (!dir.delete()) { + throw new IOException("Could not delete file: " + dir.getAbsolutePath()); + } + } + } + } + + public static ArrayList createFileFilter(File dir, ArrayList filter) { + if (filter == null) + filter = new ArrayList(); + if (dir.isFile()) { + filter.add(dir.getAbsolutePath()); + return filter; + } + + if (dir.isDirectory()) { + File[] fs = dir.listFiles((FileFilter) null); + if (fs == null) + return filter; + + for (File f : fs) { + if (f.isDirectory()) { + createFileFilter(f, filter); + } else { + filter.add(f.getAbsolutePath()); + } + } + filter.add(dir.getAbsolutePath()); + } else if (dir.exists()) { + filter.add(dir.getAbsolutePath()); + } + return filter; + } + + + /** + * Delete a directory incl. all files and sub-directories by best effort. + * Does not throw exceptions if deletion fails, simply tries to delete the + * provided directory to the best of Java File API's abilities. + * + * @param dir + * directory to delete recursively + * @boolean true if all files were successfully deleted, + * false if some or all failed to be deleted + */ + public static boolean deleteDir(File dir) { + if (LOGGER.isDebugEnabled()) + LOGGER.debug("Deleting directory "+dir); + boolean result = true; + + if (!dir.isDirectory()) return false; + File[] fs = dir.listFiles(); + if (fs != null) { + for (File f : fs) { + if (f.isDirectory()) result &= deleteDir(f); + if (f.isFile()) result &= f.delete(); + } + } + boolean ok = dir.delete(); +// if (!ok) dir.deleteOnExit(); + result &= ok; + return result; + } + + public static String deleteDirs(File dir) { + if (LOGGER.isDebugEnabled()) + LOGGER.debug("Deleting directory "+dir); + boolean result = true; + + if (!dir.isDirectory()) + return dir + " is not a directory!"; + File[] fs = dir.listFiles(); + if (fs != null) { + for (File f : fs) { + //System.out.println("file f :" + f); + if (f.isDirectory()) { + boolean a = deleteDir(f); + result &= a; + } + if (f.isFile()) { + boolean a = f.delete(); + result &= a; + } + + } + } + boolean ok = dir.delete(); +// if (!ok) dir.deleteOnExit(); + result &= ok; + return "deleteDirs succesful for " + dir + " : " + result; + } + + /** + * Closes a stream and ignores any resulting exception. This is useful + * when doing stream cleanup in a finally block where secondary exceptions + * are not worth logging. + */ + public static void uncheckedClose(Closeable closeable) { + try { + if (closeable != null) + closeable.close(); + } catch (IOException e) { + //ignore + } + } + + /** + * Extracts the specified source file in the specified bundle into the + * specified local directory. + * + * @param url the source URL to stream the resource from + * @param targetFile the target file to write the resource to + * @param deleteOnExit true to use {@link File#deleteOnExit()} + * on the resulting file. Note that this does not guarantee that the + * file is deleted when the JVM exits + * @return the resulting file + * @throws FileNotFoundException + */ + public static File copyResource(URL url, File targetFile, boolean deleteOnExit) throws IOException, FileNotFoundException { + if (url == null) + throw new IllegalArgumentException("null url"); + + FileOutputStream os = null; + InputStream is = null; + try { + if (targetFile.exists()) + targetFile.delete(); + + is = url.openStream(); + int read; + byte [] buffer = new byte [16384]; + os = new FileOutputStream (targetFile); + while ((read = is.read (buffer)) != -1) { + os.write(buffer, 0, read); + } + os.close (); + is.close (); + + // Request removal of the extracted files on JVM exit. + if (deleteOnExit) + targetFile.deleteOnExit(); + return targetFile; + } finally { + FileUtils.uncheckedClose(os); + FileUtils.uncheckedClose(is); + } + } + + /** + * Creates the requested location under the system's temporary directories + * for temporary data storage. + * + * @param pathSegments the path segments that are appended to + * java.io.tmpdir. The path segments musn't contain path separators. + * @return the resulting temporary directory as a File object + * @throws IOException if anything goes wrong + */ + public static File getOrCreateTemporaryDirectory(boolean requireEmptyDirectory, String... pathSegments) throws IOException { + String separator = System.getProperty("file.separator"); + String tempDir = System.getProperty("java.io.tmpdir"); + if (tempDir == null) + throw new IllegalStateException("'java.io.tmpdir' property is not defined, is the system TMP environment variable defined?"); + + if (!hasTrailingSeparator(tempDir)) + tempDir = tempDir + separator; + + for (int i = 0; i < pathSegments.length-1; ++i) { + if (containsSeparator(pathSegments[i])) + throw new IllegalArgumentException("path segments contain path separators: " + Arrays.toString(pathSegments)); + tempDir = pathSegments[i] + separator; + } + + // Find name for an empty or non-existing temporary directory + if (pathSegments.length > 0) { + String lastSegment = pathSegments[pathSegments.length - 1]; + String suffix = ""; + int counter = 0; + // This loop will not exit until it succeeds or some error occurs. + while (true) { + File temp = new File(tempDir + lastSegment + suffix); + + if (!temp.exists()) { + // Found non-existent temporary directory! + if (temp.mkdirs()) { + // Ok, everything seems to be fine. + return temp; + } + // For some reason the directory could not be created, lets + // try another name. + } else { + // Found an existing temporary file. + if (temp.isDirectory() && temp.canWrite()) { + if (requireEmptyDirectory) { + String[] files = temp.list(); + if (files != null) { + if (files.length == 0) { + // Ok, the directory is empty and writable. + return temp; + } + } + } else { + return temp; + } + // Either the directory was empty or it could not be + // checked. In any case, don't use the directory. + } + } + + // Try another directory name if no success with this one. + suffix = Integer.toHexString(counter); + ++counter; + } + } + return new File(tempDir); + } + + private static boolean containsSeparator(String s) { + return s.contains("/") || s.contains("\\"); + } + + private static boolean hasTrailingSeparator(String s) { + return s.endsWith("/") || s.endsWith("\\"); + } + + /** + * Very simple compression using ZLib + * @param input the uncompressed data + * @return the compressed data + */ + public static byte[] deflate(byte[] input) { + // Compress the bytes + Deflater compresser = new Deflater(Deflater.BEST_COMPRESSION); + compresser.setInput(input); + compresser.finish(); + + int bufferSize = input.length<16 ? 16 : input.length; + byte buffer[] = new byte[bufferSize]; + int bufferPos = 0; + do { + int compressedDataLength = compresser.deflate(buffer, bufferPos, buffer.length-bufferPos); + bufferPos += compressedDataLength; + if (!compresser.finished()) + buffer = Arrays.copyOf(buffer, buffer.length + bufferSize); + } while (!compresser.finished()); + + byte[] result = new byte[bufferPos+4]; + System.arraycopy(buffer, 0, result, 4, bufferPos); + byte sizeData[] = LEInt.toBytes(input.length); + System.arraycopy(sizeData, 0, result, 0, 4); + return result; + } + + /** + * Very simple decompression using ZLib. + * @param input the compressed input + * @return the uncompressed data + * @throws DataFormatException + */ + public static byte[] inflate(byte[] input) throws DataFormatException { + // Decompress the bytes + int inflatedSize = LEInt.toInt(input); + Inflater decompresser = new Inflater(); + decompresser.setInput(input, 4, input.length - 4); + byte[] result = new byte[inflatedSize]; + int resultLength = decompresser.inflate(result); + assert(resultLength == inflatedSize); + decompresser.end(); + return result; + } + + + public static boolean isValidFileName(String name) { + File f = new File(name) ; + if (!f.exists()) { + try { + boolean ok = f.createNewFile(); + if (ok) + f.delete(); + return ok; + } catch (IOException ioe) { + return false; + } + } + return true; + } + + /** + * Create a temporary directory + * + * @return temporary directory + */ + public static File createTmpDir() + { + String tmp = System.getenv("tmp"); + if (tmp==null) tmp = "c:/temp"; + Random r = new Random(); + String randomName = "simantics-tmp-"+(r.nextInt(10000)+10000); + File tmpDir = new File(tmp+"/"+randomName); + tmpDir.deleteOnExit(); + Boolean ok = tmpDir.mkdirs(); + if (!ok) throw new RuntimeException("tmp dir "+tmpDir+" was not created"); + return tmpDir; + } + + public static void compressZip(String sourcePath, String zipDir) throws IOException { + if (LOGGER.isDebugEnabled()) + LOGGER.debug("Compressing file " + sourcePath + " to zip " + zipDir + "."); + + File filesource = new File(sourcePath); + URI base = filesource.toURI(); + Deque queue = new LinkedList(); + queue.push(filesource); + try (ZipOutputStream zout = new ZipOutputStream(new FileOutputStream(zipDir))) { + while (!queue.isEmpty()) { + filesource = queue.pop(); + for (File child : filesource.listFiles()) { + String name = base.relativize(child.toURI()).getPath(); + if (child.isDirectory()) { + queue.push(child); + name = name.endsWith("/") ? name : name + "/"; + zout.putNextEntry(new ZipEntry(name)); + } else { + zout.putNextEntry(new ZipEntry(name)); + copy(child, zout); + zout.closeEntry(); + } + } + } + } finally { + LOGGER.debug("Filecompression done."); + } + } + + public static void copy(File file, OutputStream out) throws IOException { + try (InputStream in = new FileInputStream(file)) { + copy(in, out); + } + } + + /** + * Extract a zip file into a directory + * + * @param zipFile + * @param dst + * @throws IOException + */ + public static void extractZip(File zipFile, File dst) throws IOException { + if (LOGGER.isTraceEnabled()) + LOGGER.trace("Extracting zip "+zipFile); + try (FileInputStream fis = new FileInputStream(zipFile)) { + extractZip(fis, dst); + } + } + + /** + * Extract a zip file into a directory + * + * @param zipInput + * @param dst directory + * @throws IOException + */ + public static void extractZip(InputStream zipInput, File dst) throws IOException { + byte[] buf = new byte[8192]; + ZipInputStream zis = new ZipInputStream(zipInput); + ZipEntry entry; + + entry = zis.getNextEntry(); + while (entry != null) { + // for each entry to be extracted + String name = entry.getName(); + if (LOGGER.isTraceEnabled()) + LOGGER.trace("Extracting "+name); + File file = new File(dst, name); + + if (entry.isDirectory()) + { + if ( !file.exists() ) file.mkdirs(); + } else { + File parent = file.getParentFile(); + if (!parent.exists()) parent.mkdirs(); + if (!file.exists()) file.createNewFile(); + + FileOutputStream fileoutputstream = new FileOutputStream(file); + try { + int n = 0; + while ((n = zis.read(buf, 0, buf.length)) > -1) + fileoutputstream.write(buf, 0, n); + } catch (ZipException e) { + throw new IOException("Failed to extract '" + name + "'.", e); + } finally { + fileoutputstream.close(); + } + } + + zis.closeEntry(); + entry = zis.getNextEntry(); + }// while + + zis.close(); + } + + // Test inflate & deflate + public static void main(String[] args) { + try { + // Test compression + String str = "abcdefghijklmnopqrstuvwxyz"; + byte data[] = str.getBytes(); + System.out.println("Data:\t"+data.length); + byte deflated[] = deflate(data); + System.out.println("Deflated:\t"+deflated.length); + byte inflated[] = inflate(deflated); + System.out.println("Inflated:\t"+inflated.length); + String str2 = new String(inflated); + System.out.println("Strings are equal: "+str.endsWith(str2)); + } catch (DataFormatException e) { + e.printStackTrace(); + } + } + + public static File[] listFilesByExtension(File directory, final String extension) { + return directory.listFiles(new ExtensionFilter(extension)); + } + + /** + * Copy the content of the input stream into the output stream, using a temporary + * byte array buffer whose size is defined by {@link #IO_BUFFER_SIZE}. + * + * @param in The input stream to copy from. + * @param out The output stream to copy to. + * + * @throws IOException If any error occurs during the copy. + */ + private static final int IO_BUFFER_SIZE = 64 * 1024; + + public static void copy(InputStream in, OutputStream out) throws IOException { + byte[] b = new byte[IO_BUFFER_SIZE]; + int read; + while (true) { + read = in.read(b); + if (read < 0) + break; + out.write(b, 0, read); + } + } + + /** + * @param in + * @param out + * @param maxBytesToCopy the maximum amount of bytes to copy + * @return the amount of bytes copied + * @throws IOException + */ + public static long copy(InputStream in, OutputStream out, long maxBytesToCopy) throws IOException { + byte[] b = new byte[IO_BUFFER_SIZE]; + int read = 0; + while (read < maxBytesToCopy) { + int l = (int) Math.min((long) IO_BUFFER_SIZE, maxBytesToCopy-read); + int r = in.read(b, 0, l); + if (r < 0) + break; + out.write(b, 0, r); + read += r; + } + return read; + } + + public static void deleteWithFilter(Path path, Predicate filter) throws IOException { + if (Files.exists(path)) + Files.walkFileTree(path, new DeleteDirectoriesVisitor(filter)); + } + + public static void delete(Path path) throws IOException { + deleteWithFilter(path, null); + } + + /** + * Empties the specified directory but does not delete the directory itself. + * If a non-directory path is given, it is simply deleted. + * + * @param path + * @throws IOException + */ + public static void emptyDirectory(Path path) throws IOException { + if (Files.isDirectory(path)) + Files.walkFileTree(path, new EmptyDirectoryVisitor()); + else + Files.deleteIfExists(path); + } + + public static void copy(Path from, Path to) throws IOException { + Files.walkFileTree(from, new CopyDirectoriesVisitor(from, to)); + } + + public static class DeleteDirectoriesVisitor extends SimpleFileVisitor { + + private Predicate filter; + + public DeleteDirectoriesVisitor(Predicate filter) { + this.filter = filter; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + if (filter != null && !filter.test(file)) { + return FileVisitResult.CONTINUE; + } + Files.deleteIfExists(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + if (exc != null) + throw exc; + if (filter != null && !filter.test(dir)) { + return FileVisitResult.CONTINUE; + } + Files.deleteIfExists(dir); + return FileVisitResult.CONTINUE; + } + } + + private static class EmptyDirectoryVisitor extends SimpleFileVisitor { + int depth = 0; + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.deleteIfExists(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + ++depth; + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + if (exc != null) + throw exc; + if (--depth > 0) + Files.deleteIfExists(dir); + return FileVisitResult.CONTINUE; + } + } + + public static class CopyDirectoriesVisitor extends SimpleFileVisitor { + + private final Path fromPath; + private final Path toPath; + + public CopyDirectoriesVisitor(Path fromPath, Path toPath) { + this.fromPath = fromPath; + this.toPath = toPath; + } + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + Path targetPath = toPath.resolve(fromPath.relativize(dir)); + if(!Files.exists(targetPath)){ + Files.createDirectory(targetPath); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.copy(file, toPath.resolve(fromPath.relativize(file)), StandardCopyOption.REPLACE_EXISTING); + return FileVisitResult.CONTINUE; + } + } + + public static void syncFile(File file) throws IOException { + try (RandomAccessFile raf = new RandomAccessFile(file, "rw")) { + raf.getFD().sync(); + } + } + + public static void sync(Path path) throws IOException { + try (InputStream stream = Files.newInputStream(path, StandardOpenOption.SYNC)) { + } + } +}