1 /*******************************************************************************
2 * Copyright (c) 2017 Association for Decentralized Information Management
4 * All rights reserved. This program and the accompanying materials
5 * are made available under the terms of the Eclipse Public License v1.0
6 * which accompanies this distribution, and is available at
7 * http://www.eclipse.org/legal/epl-v10.html
10 * Semantum Oy - #7330 - initial API and implementation
11 * Semantum Oy - #7551 - improved robustness
12 *******************************************************************************/
13 package org.simantics.utils.datastructures.file;
15 import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE;
16 import static java.nio.file.StandardWatchEventKinds.ENTRY_DELETE;
17 import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY;
18 import static java.nio.file.StandardWatchEventKinds.OVERFLOW;
20 import java.io.Closeable;
21 import java.io.IOException;
22 import java.nio.file.AccessDeniedException;
23 import java.nio.file.FileSystems;
24 import java.nio.file.FileVisitResult;
25 import java.nio.file.FileVisitor;
26 import java.nio.file.Files;
27 import java.nio.file.NoSuchFileException;
28 import java.nio.file.Path;
29 import java.nio.file.SimpleFileVisitor;
30 import java.nio.file.WatchEvent;
31 import java.nio.file.WatchEvent.Kind;
32 import java.nio.file.WatchKey;
33 import java.nio.file.WatchService;
34 import java.nio.file.attribute.BasicFileAttributes;
36 import java.util.concurrent.atomic.AtomicInteger;
37 import java.util.function.LongConsumer;
39 import org.simantics.databoard.util.BijectionMap;
40 import org.simantics.utils.FileUtils;
41 import org.simantics.utils.datastructures.MapSet;
42 import org.slf4j.Logger;
43 import org.slf4j.LoggerFactory;
45 import gnu.trove.map.TObjectLongMap;
46 import gnu.trove.map.hash.TObjectLongHashMap;
49 * @author Tuukka Lehtonen
52 public class DirectorySizeTracker implements Runnable, Closeable {
54 private static final Logger LOGGER = LoggerFactory.getLogger(DirectorySizeTracker.class);
56 private static final int TRACE_NONE = 0;
57 @SuppressWarnings("unused")
58 private static final int TRACE_DIRS = 1;
59 @SuppressWarnings("unused")
60 private static final int TRACE_EVENTS = 2;
61 @SuppressWarnings("unused")
62 private static final int TRACE_FILES = 3;
64 private static final Kind<?>[] ALL_EVENTS = { ENTRY_CREATE, ENTRY_DELETE, ENTRY_MODIFY };
66 private static AtomicInteger threadCounter = new AtomicInteger(1);
68 private LongConsumer consumer;
69 private Kind<?>[] events;
71 private WatchService watcher;
73 private Object lock = new Object();
75 // For watched directory tracking
76 private BijectionMap<WatchKey, Path> trackedDirs = new BijectionMap<>();
77 private MapSet<Path, Path> subdirs = new MapSet.Hash<Path, Path>();
78 private MapSet<Path, Path> files = new MapSet.Hash<Path, Path>();
80 // For directory and file size tracking
81 private TObjectLongMap<Path> entrySizes = new TObjectLongHashMap<>(1024, 0.5f, -1L);
82 private TObjectLongMap<Path> dirSizes = new TObjectLongHashMap<>(512, 0.5f, -1L);
83 private long totalSize = 0L;
85 private int traceLevel = TRACE_NONE;
86 private boolean running = true;
87 private Thread thread;
89 public static DirectorySizeTracker startTracker(LongConsumer sizeChangeListener) throws IOException {
90 DirectorySizeTracker watcher = new DirectorySizeTracker(sizeChangeListener, ALL_EVENTS);
91 watcher.thread = new Thread(watcher, DirectorySizeTracker.class.getSimpleName() + "-" + threadCounter.get());
92 watcher.thread.start();
96 private DirectorySizeTracker(LongConsumer sizeChangeListener, Kind<?>[] events) throws IOException {
97 this.consumer = sizeChangeListener;
99 this.watcher = FileSystems.getDefault().newWatchService();
102 public void close(boolean joinThread) throws InterruptedException {
110 public void close() throws IOException {
113 } catch (InterruptedException e) {
114 throw new IOException(e);
119 * Register the given directory with the WatchService to listen for the
120 * default set of events provided to the constructor.
122 public void track(Path dir) throws IOException {
123 synchronized (lock) {
124 if (trackedDirs.containsRight(dir))
127 LOGGER.info("Starting to track entire directory " + dir + " with a total of " + entrySizes.size() + " files and " + trackedDirs.size() + " dirs with a total size of " + totalSize);
128 Files.walkFileTree(dir, REGISTER);
130 LOGGER.info("Now tracking entire directory " + dir + " with a total of " + entrySizes.size() + " files and " + trackedDirs.size() + " dirs with a total size of " + totalSize);
134 private FileVisitor<Path> REGISTER = new SimpleFileVisitor<Path>() {
135 private Path currentDir;
137 public java.nio.file.FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
140 LOGGER.info("Set current dir to " + currentDir);
141 return FileVisitResult.CONTINUE;
144 public java.nio.file.FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
145 registerFile(currentDir, file, attrs);
146 return FileVisitResult.CONTINUE;
149 public java.nio.file.FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
150 LOGGER.warn("Failed to visit file " + file, exc);
151 return FileVisitResult.CONTINUE;
154 public java.nio.file.FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
155 // Start tracking directories only after registering all entries
157 Path parent = dir.getParent();
159 subdirs.add(parent, dir);
161 return FileVisitResult.CONTINUE;
166 * Register the given directory with the WatchService to listen for the
167 * specified set of events.
169 private boolean trackDir(Path dir) throws IOException {
170 if (trackedDirs.containsRight(dir))
173 WatchKey key = dir.register(watcher, events);
175 if (traceLevel > 1) {
176 Path prev = trackedDirs.getRight(key);
178 LOGGER.info("Tracking new directory {}\n", dir);
180 if (!dir.equals(prev)) {
181 LOGGER.info("Tracked directory update: {} -> {}\n", prev, dir);
186 trackedDirs.map(key, dir);
190 private boolean registerFile(Path dir, Path file, BasicFileAttributes attrs) {
191 if (files.add(dir, file)) {
192 long entrySize = attrs.size();
193 entrySizes.put(file, entrySize);
194 dirSizes.adjustOrPutValue(dir, entrySize, entrySize);
195 totalSize += entrySize;
197 LOGGER.info("Registered file " + file + " size " + entrySize + " for total size " + totalSize);
200 long size = attrs.size();
201 long oldSize = entrySizes.put(file, size);
202 long sizeDelta = oldSize >= 0 ? size - oldSize : size;
203 totalSize += sizeDelta;
204 dirSizes.adjustOrPutValue(dir, sizeDelta, sizeDelta);
206 LOGGER.info("Modified " + file + " size from " + oldSize + " to " + size
207 + " with delta " + sizeDelta + ", dir size = " + dirSizes.get(dir)
208 + ", total size = " + totalSize);
213 private boolean unregisterFile(Path dir, Path file) {
214 long fileSize = entrySizes.remove(file);
216 totalSize -= fileSize;
217 if (files.remove(dir, file))
218 dirSizes.adjustValue(dir, -fileSize);
220 LOGGER.info("Unregistered file " + file + " of size " + fileSize + ", dirSize = " + dirSizes.get(dir) + ", totalSize = " + totalSize);
226 public void untrack(Path dir) {
227 synchronized (lock) {
228 if (!trackedDirs.containsRight(dir))
231 LOGGER.info("Starting to untrack entire directory " + dir + " with total tracked size " + totalSize);
233 subdirs.remove(dir.getParent(), dir);
235 LOGGER.info("Done untracking entire directory " + dir + " with total tracked size " + totalSize);
239 private void untrackTree(Path dir) {
240 Set<Path> subdirs = this.subdirs.removeValues(dir);
241 for (Path subdir : subdirs)
246 private void untrackDir(Path dir) {
248 LOGGER.info("Untrack directory " + dir + " with total tracked size " + totalSize);
249 WatchKey key = trackedDirs.removeWithRight(dir);
253 Set<Path> registeredFiles = files.removeValues(dir);
254 for (Path file : registeredFiles)
255 unregisterFile(dir, file);
256 dirSizes.remove(dir);
259 @SuppressWarnings("unchecked")
260 static <T> WatchEvent<T> cast(WatchEvent<?> event) {
261 return (WatchEvent<T>) event;
265 * Process all events for keys queued to the watcher
267 void processEvents() {
269 // wait for key to be signaled
272 key = watcher.take();
273 } catch (InterruptedException x) {
277 Path dir = trackedDirs.getRight(key);
279 LOGGER.error("WatchKey not registered: " + key);
280 for (WatchEvent<?> event : key.pollEvents()) {
281 WatchEvent.Kind<?> kind = event.kind();
282 // TBD - provide example of how OVERFLOW event is handled
283 if (kind == OVERFLOW)
285 // Context for directory entry event is the file name of entry
286 WatchEvent<Path> evt = cast(event);
287 Path name = evt.context();
288 LOGGER.error(String.format("MISSED EVENT: %s: %s", event.kind().name(), name));
290 boolean valid = key.reset();
291 LOGGER.error("RESET KEY RETURNED: " + valid);
295 synchronized (lock) {
296 for (WatchEvent<?> event : key.pollEvents()) {
297 WatchEvent.Kind<?> kind = event.kind();
299 // TBD - provide example of how OVERFLOW event is handled
300 if (kind == OVERFLOW)
303 // Context for directory entry event is the file name of entry
304 WatchEvent<Path> evt = cast(event);
305 Path name = evt.context();
306 Path child = dir.resolve(name);
309 LOGGER.info(String.format("%s: %s", event.kind().name(), child));
311 if (kind == ENTRY_CREATE) {
313 BasicFileAttributes attrs = Files.readAttributes(child, BasicFileAttributes.class);
314 if (attrs.isDirectory())
316 else if (attrs.isRegularFile()) {
317 registerFile(dir, child, attrs);
319 } catch (NoSuchFileException ioe) {
320 // Intentionally ignore.
321 // The file was removed before we even had a chance to process it.
322 } catch (IOException ioe) {
323 LOGGER.error("Failed to read attribute for path " + child, ioe);
325 } else if (kind == ENTRY_MODIFY) {
327 BasicFileAttributes attrs = Files.readAttributes(child, BasicFileAttributes.class);
328 if (attrs.isRegularFile()) {
329 registerFile(dir, child, attrs);
331 } catch (NoSuchFileException ioe) {
332 // It is possible that child is a directory that has been removed.
333 // In this case, just untrack the whole tree of child.
334 if (!entrySizes.containsKey(child)) {
337 } catch (AccessDeniedException e) {
338 // Ignore. This can happen e.g. with temporary locked Lucene index files.
339 } catch (IOException ioe) {
340 LOGGER.error("Failed to read attribute for path " + child, ioe);
342 } else if (kind == ENTRY_DELETE) {
343 if (!unregisterFile(dir, child)) {
344 // This must have been a directory since it isn't registered as a file.
350 // reset key and remove from set if directory no longer accessible
351 boolean valid = key.reset();
354 LOGGER.info("WatchKey for dir " + dir + " is no longer valid. Untracking it.");
356 if (trackedDirs.isEmpty()) {
357 // all directories are inaccessible
364 LOGGER.info("STATUS: Tracking a total of " + entrySizes.size() + " files and " + trackedDirs.size() + " dirs with a total size of " + totalSize);
365 if (consumer != null)
366 consumer.accept(totalSize);
376 // It seems that for some reason cancelling all watch keys
377 // helps the watcher close itself. Otherwise closing the
378 // watch service seemed to get stuck easily.
380 FileUtils.uncheckedClose(watcher);
384 private void cancelWatchKeys() {
385 synchronized (lock) {
386 for (WatchKey key : trackedDirs.getLeftSet()) {
394 * @return total size of the tracked directories in bytes
396 public long getTotalSize() {
400 // public static void main(String[] args) {
401 // try (DirectorySizeTracker tracker = DirectorySizeTracker.startTracker(null)) {
402 // tracker.track(Paths.get("d:/temp dir"));
403 // LOGGER.info("AFTER TRACK: Total size from " + tracker.entrySizes.size() + " files and " + tracker.trackedDirs.size() + " directories is " + tracker.getTotalSize());
404 // Thread.sleep(2000);
405 // //tracker.untrack(Paths.get("d:/track-test"));
406 // //LOGGER.info("AFTER UNTRACK: Total size from " + tracker.entrySizes.size() + " files and " + tracker.trackedDirs.size() + " directories is " + tracker.getTotalSize());
407 // } catch (IOException | InterruptedException e) {
408 // LOGGER.error("test failed", e);