1 package org.simantics.db.server.internal;
3 import static java.nio.file.StandardCopyOption.COPY_ATTRIBUTES;
4 import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
6 import java.io.EOFException;
8 import java.io.FilenameFilter;
9 import java.io.IOException;
10 import java.io.OutputStream;
11 import java.io.RandomAccessFile;
12 import java.nio.file.DirectoryStream;
13 import java.nio.file.FileSystems;
14 import java.nio.file.FileVisitOption;
15 import java.nio.file.FileVisitResult;
16 import java.nio.file.Files;
17 import java.nio.file.Path;
18 import java.nio.file.PathMatcher;
19 import java.nio.file.Paths;
20 import java.nio.file.SimpleFileVisitor;
21 import java.nio.file.attribute.BasicFileAttributes;
22 import java.util.ArrayList;
23 import java.util.EnumSet;
24 import java.util.HashMap;
25 import java.util.List;
26 import java.util.Properties;
27 import java.util.TreeMap;
28 import java.util.UUID;
30 import org.simantics.databoard.Bindings;
31 import org.simantics.databoard.binding.impl.TreeMapBinding;
32 import org.simantics.databoard.serialization.Serializer;
33 import org.simantics.db.Database;
34 import org.simantics.db.DatabaseUserAgent;
35 import org.simantics.db.ServiceLocator;
36 import org.simantics.db.common.CommentMetadata;
37 import org.simantics.db.common.CommitMetadata;
38 import org.simantics.db.common.utils.Logger;
39 import org.simantics.db.common.utils.MetadataUtil;
40 import org.simantics.db.exception.SDBException;
41 import org.simantics.db.server.DatabaseLastExitException;
42 import org.simantics.db.server.ProCoreException;
43 import org.simantics.db.server.internal.ProCoreServer.TailFile;
44 import org.simantics.db.server.protocol.MessageNumber;
45 import org.simantics.db.server.protocol.MessageText;
47 public class DatabaseI implements Database {
48 private static String NL = System.getProperty("line.separator");
49 private static String TEMP_PREFIX = "db.temp.";
50 public static DatabaseI newDatabaseI(File dbFolder) {
51 return new DatabaseI(dbFolder);
53 private final SessionManager sessionManager = new SessionManager();
54 private final Server server;
55 private final JournalI journal;
56 private DatabaseUserAgent dbUserAgent = null;
57 private DatabaseI(File dbFolder) {
58 server = new Server(dbFolder);
59 journal = new JournalI(dbFolder);
61 public boolean isSubFolder(Path base, Path sub)
63 if (null == base || null == sub)
65 return isSubFolder(base.toFile(), sub.toFile());
67 public boolean isSubFolder(File base, File sub)
69 if (null == base || null == sub)
71 Path basePath = base.getCanonicalFile().toPath();
72 Path subPath = sub.getCanonicalFile().toPath();
73 if (subPath.startsWith(basePath))
78 public Path saveDatabase()
79 throws ProCoreException {
80 Path dbFolder = getFolder().toPath();
81 Path parent = getFolder().getParentFile().toPath();
82 Path temp = createTempFolder(parent, "Could not create temporary directory for saving database.");
83 copyTree(dbFolder, temp, FileOption.IF_NEW);
86 public Path saveDatabase(Path parent)
87 throws ProCoreException {
88 Path dbFolder = getFolder().toPath();
90 boolean yes = isSubFolder(dbFolder, parent);
92 throw new ProCoreException("Parent must not be subdirectory of database directory."
93 + NL + "database=" + dbFolder
94 + NL + "parent=" + parent);
95 } catch (IOException e) {
96 throw new ProCoreException("Failed to save database to parent."
97 + NL + "database=" + dbFolder
98 + NL + "parent=" + parent);
100 Path temp = createTempFolder(parent, "Could not create temporary directory for saving database.");
101 copyTree(dbFolder, temp, FileOption.IF_NEW);
104 public void deleteDatabaseFiles() throws ProCoreException {
105 if (server.isAlive()) {
107 if (server.isAlive())
108 throw new ProCoreException("Server must be dead to delete database.");
110 Path db = getFolder().toPath();
111 deleteDatabaseFiles(db); // Redundant, but tests the method.
114 Path copyDatabaseFiles(Path from, Path to) throws ProCoreException {
115 copyTree(from, to, ProCoreServer.BRANCH_DIR, FileOption.IF_EXIST);
116 if (Files.exists(from.resolve(ProCoreServer.TAIL_DIR)))
117 copyTree(from, to, ProCoreServer.TAIL_DIR, FileOption.IF_EXIST);
118 copyPath(from, to, ProCoreServer.CONFIG_FILE, FileOption.IF_EXIST);
119 copyPath(from, to, ProCoreServer.GUARD_FILE, FileOption.IF_EXIST);
120 copyPath(from, to, ProCoreServer.JOURNAL_FILE, FileOption.IF_EXIST);
121 copyPath(from, to, ProCoreServer.LOG_FILE, FileOption.IF_EXIST);
122 copyFiles(from, to, ProCoreServer.PAGE_FILE_PATTERN, FileOption.IF_EXIST);
125 public static void deleteDatabaseFiles(Path from) throws ProCoreException {
126 deleteFile(from, ProCoreServer.DCS_FILE, FileOption.IF_EXIST);
127 deleteFile(from, ProCoreServer.RECOVERY_NEEDED_FILE, FileOption.IF_EXIST);
128 deleteFile(from, ProCoreServer.RECOVERY_IGNORED_FILE, FileOption.IF_EXIST);
129 deleteFile(from, ProCoreServer.PROTOCOL_IGNORED_FILE, FileOption.IF_EXIST);
130 deleteFile(from, ProCoreServer.LOG_FILE, FileOption.IF_EXIST);
131 deleteFile(from, ProCoreServer.GUARD_FILE, FileOption.IF_EXIST);
132 deleteFile(from, ProCoreServer.CONFIG_FILE, FileOption.IF_EXIST);
133 deleteFiles(from, ProCoreServer.PAGE_FILE_PATTERN);
134 deleteTree(from.resolve(ProCoreServer.BRANCH_DIR));
135 deleteFile(from, ProCoreServer.JOURNAL_FILE, FileOption.IF_EXIST);
136 deleteTree(from.resolve(ProCoreServer.TAIL_DIR));
138 Path moveDatabaseFiles(Path from, Path to) throws ProCoreException {
139 moveFolder(from, to, ProCoreServer.BRANCH_DIR, FileOption.IF_NEW);
140 if (Files.exists(from.resolve(ProCoreServer.TAIL_DIR)))
141 moveFolder(from, to, ProCoreServer.TAIL_DIR, FileOption.IF_NEW);
142 movePath(from, to, ProCoreServer.CONFIG_FILE, FileOption.IF_NEW);
143 movePath(from, to, ProCoreServer.GUARD_FILE, FileOption.IF_NEW);
144 movePath(from, to, ProCoreServer.JOURNAL_FILE, FileOption.IF_NEW);
145 movePath(from, to, ProCoreServer.LOG_FILE, FileOption.IF_NEW);
146 moveFiles(from, to, ProCoreServer.PAGE_FILE_PATTERN, FileOption.IF_NEW);
147 movePath(from, to, ProCoreServer.DCS_FILE, FileOption.IF_NEW);
148 moveFile(from, to, ProCoreServer.RECOVERY_NEEDED_FILE, FileOption.IF_NEW);
149 moveFile(from, to, ProCoreServer.RECOVERY_IGNORED_FILE, FileOption.IF_NEW);
150 moveFile(from, to, ProCoreServer.PROTOCOL_IGNORED_FILE, FileOption.IF_NEW);
153 private void moveFile(Path from, Path to, String file, FileOption fileOption) throws ProCoreException {
154 if (Files.exists(from.resolve(file)))
155 movePath(from, to, file, fileOption);
157 public void serverCreateConfiguration() throws ProCoreException {
158 server.createConfiguration();
160 public boolean ignoreExit() throws ProCoreException {
162 throw new ProCoreException("Folder must be valid database folder to ignore exit." );
163 server.deleteGuard();
164 File folder = server.getFolder();
165 Path db = Paths.get(folder.getAbsolutePath());
166 Path ri = db.resolve(ProCoreServer.RECOVERY_IGNORED_FILE);
167 if (!Files.exists(ri))
168 try (OutputStream os = Files.newOutputStream(ri)) {
169 } catch (IOException e) {
170 throw new ProCoreException("Could not create file: " + ri, e);
173 DatabaseUserAgent dbu = dbUserAgent; // Save current user agent.
175 dbUserAgent = null; // Recursion not supported. Disable user agent.
177 ok = server.isActive();
180 dbUserAgent = dbu; // Restore used user agent.
184 Files.deleteIfExists(ri);
185 } catch (IOException e) {
186 Logger.defaultLogError("Could not delete file: " + ri, e);
188 Path rn = db.resolve(ProCoreServer.RECOVERY_NEEDED_FILE);
190 Files.deleteIfExists(rn);
191 } catch (IOException e) {
192 Logger.defaultLogError("Could not delete file: " + rn, e);
198 public boolean ignoreProtocol() throws ProCoreException {
200 throw new ProCoreException("Folder must be valid database folder to ignore exit." );
201 File folder = server.getFolder();
202 Path db = Paths.get(folder.getAbsolutePath());
203 Path ri = db.resolve(ProCoreServer.PROTOCOL_IGNORED_FILE);
204 if (!Files.exists(ri))
205 try (OutputStream os = Files.newOutputStream(ri)) {
206 } catch (IOException e) {
207 throw new ProCoreException("Could not create file: " + ri, e);
212 ok = server.isActive();
218 private long preJournalCheck() throws ProCoreException {
219 File folder = server.getFolder();
220 if (!folder.isDirectory())
221 throw new ProCoreException("Database folder does not exist." + NL + "folder=" + folder);
222 File file = new File(folder, ProCoreServer.JOURNAL_FILE);
224 throw new ProCoreException("Journal file does not exist." + NL + "file=" + file);
225 else if (!file.canRead())
226 throw new ProCoreException("Journal file must be readale to create database from journal." + NL + "file=" + file);
227 else if (server.isAlive())
228 throw new ProCoreException("Server must be dead to create database from journal file." + NL + "file=" + file);
229 return getNextClusterId(folder.toPath());
231 private void postJournalFix(long nextFreeId) throws SDBException {
232 Session s = newSession(null);
233 long current = s.reserveIds(0);
234 if (current < nextFreeId)
235 s.reserveIds((int)(nextFreeId - current));
238 private Path createTempFolder(Path parent, String fail) throws ProCoreException {
240 return Files.createTempDirectory(parent, TEMP_PREFIX);
241 } catch (IOException e) {
242 throw new ProCoreException(fail, e);
245 public void replaceFromJournal() throws SDBException {
246 long nextFreeId = preJournalCheck();
247 Path db = Paths.get(server.getFolder().getAbsolutePath());
248 Path temp = createTempFolder(db, "Could not create temporary directory for database to be created from journal.");
249 movePath(db, temp, ProCoreServer.CONFIG_FILE, FileOption.IF_NEW);
250 movePath(db, temp, ProCoreServer.JOURNAL_FILE, FileOption.IF_NEW);
251 deleteFile(db, ProCoreServer.GUARD_FILE, FileOption.IF_EXIST);
252 deleteFiles(db, ProCoreServer.PAGE_FILE_PATTERN);
253 Path dbHead = db.resolve(ProCoreServer.BRANCH_DIR);
254 deleteFiles(dbHead, ProCoreServer.VALUE_PATTERN);
255 deleteFiles(dbHead, ProCoreServer.DATA_PATTERN);
256 deleteFiles(dbHead, ProCoreServer.INDEX_PATTERN);
257 Path dbHeadTailFile = dbHead.resolve(ProCoreServer.TAIL_FILE);
258 boolean headTailFileExisted = Files.exists(dbHeadTailFile);
259 final long NEXT_REVISION = 1;
260 if (!headTailFileExisted) // Number of change sets and database id not known at this point. Fortunately they are not used by recovery.
261 TailFile.createTailFile(dbHeadTailFile.toFile(), NEXT_REVISION, nextFreeId, UUID.randomUUID().toString());
262 movePath(dbHead, temp, ProCoreServer.TAIL_FILE, FileOption.IF_NEW);
263 Path dbTail = db.resolve(ProCoreServer.TAIL_DIR);
264 boolean tailExisted = Files.isDirectory(dbTail);
266 copyPath(dbTail, dbHead, ProCoreServer.TAIL_FILE, FileOption.IF_NEW);
269 Files.createDirectory(dbTail);
270 } catch (IOException e) {
271 throw new ProCoreException("Failed to create directory: " + dbTail, e);
273 copyPath(temp, dbTail, ProCoreServer.TAIL_FILE, FileOption.IF_NEW);
275 server.createConfiguration();
278 String t = server.execute("journalRead " + temp.getFileName());
280 throw new ProCoreException("Could not read journal. reply=" + t);
281 postJournalFix(nextFreeId); // Not the right way for implementing this, but better than incorrect recovery.
285 movePath(temp, db, ProCoreServer.CONFIG_FILE, FileOption.OVERWRITE);
290 public boolean canPurge() throws ProCoreException {
291 File folder = server.getFolder();
292 Path db = Paths.get(folder.getAbsolutePath());
293 Path dbHead = db.resolve(ProCoreServer.BRANCH_DIR);
294 if (!Files.exists(dbHead))
295 return false; // Already clean.
296 boolean empty = isFolderEmpty(dbHead);
298 return false; // Already clean.
299 final boolean[] found = new boolean[1];
301 dbHead.toFile().listFiles(new FilenameFilter() {
303 public boolean accept(File dir, String name) {
306 else if (!name.equals(ProCoreServer.TAIL_FILE)) {
316 * Removes old history i.e. change sets.
317 * Note that also cleans last exit status and removes guard file.
318 * This means that all information about last exit status is also lost
320 * @throws ProCoreException
322 public void purge() throws SDBException {
323 synchronized (server.proCoreServer.getProCoreClient()) {
324 boolean wasAlive = server.isAlive();
327 Path db = Paths.get(server.getFolder().getAbsolutePath());
328 Path dbHead = db.resolve(ProCoreServer.BRANCH_DIR);
329 Path dbTail = db.resolve(ProCoreServer.TAIL_DIR);
330 if (!Files.isDirectory(dbTail)) {
332 Files.createDirectory(dbTail);
333 } catch (IOException e) {
334 throw new ProCoreException("Failed to create directory: " + dbTail, e);
337 long nextFreeId = getNextClusterId(db);
338 boolean cleanHead = Files.isDirectory(dbHead) && !isFolderEmpty(dbHead);
339 Logger.defaultLog("Purging old history and exit information. folder=" + db);
341 deleteClusters(dbHead, dbTail);
342 movePath(dbHead, dbTail, ProCoreServer.TAIL_FILE, FileOption.OVERWRITE);
343 moveFiles(dbHead, dbTail, ProCoreServer.VALUE_PATTERN, FileOption.IF_NEW);
344 moveFiles(dbHead, dbTail, ProCoreServer.DATA_PATTERN, FileOption.OVERWRITE);
345 deleteFiles(dbHead, ProCoreServer.INDEX_PATTERN);
347 deleteFiles(db, ProCoreServer.PAGE_FILE_PATTERN);
348 deleteFile(db, ProCoreServer.JOURNAL_FILE, FileOption.IF_EXIST);
349 deleteFile(db, ProCoreServer.LOG_FILE, FileOption.IF_EXIST);
350 deleteFile(db, ProCoreServer.DCS_FILE, FileOption.IF_EXIST);
351 deleteFile(db, ProCoreServer.RECOVERY_NEEDED_FILE, FileOption.IF_EXIST);
352 deleteFile(db, ProCoreServer.RECOVERY_IGNORED_FILE, FileOption.IF_EXIST);
353 deleteFile(db, ProCoreServer.PROTOCOL_IGNORED_FILE, FileOption.IF_EXIST);
354 deleteFile(db, ProCoreServer.GUARD_FILE, FileOption.IF_EXIST);
357 Session s = newSession(null);
358 long current = s.reserveIds(0);
359 if (current < nextFreeId)
360 s.reserveIds((int)(nextFreeId - current));
366 private void copyPath(Path fromFolder, Path toFolder, String file, FileOption fileOption) throws ProCoreException {
367 Path from = fromFolder.resolve(file);
368 Path to = toFolder.resolve(file);
370 if (FileOption.IF_EXIST.equals(fileOption)) {
371 if (!Files.exists(from))
374 if (FileOption.OVERWRITE.equals(fileOption))
375 Files.copy(from, to, COPY_ATTRIBUTES, REPLACE_EXISTING);
377 Files.copy(from, to, COPY_ATTRIBUTES);
378 } catch (IOException e) {
379 throw new ProCoreException("Could not copy " + from + " to " + to, e);
382 private static void deleteFile(Path from, String file, FileOption fileOption) throws ProCoreException {
383 Path path = from.resolve(file);
384 deletePath(path, fileOption);
386 private static void deletePath(Path path, FileOption fileOption) throws ProCoreException {
388 if (FileOption.IF_EXIST.equals(fileOption))
389 Files.deleteIfExists(path);
392 } catch (IOException e) {
393 throw new ProCoreException("Could not delete " + path, e);
396 private boolean isFolderEmpty(final Path folder) { // True if folder exists and is empty.
397 if (!Files.isDirectory(folder))
399 try(DirectoryStream<Path> folderStream = Files.newDirectoryStream(folder)) {
400 return !folderStream.iterator().hasNext();
401 } catch (IOException e) {
402 Logger.defaultLogError("Failed to open folder stream. folder=" + folder, e);
406 private void movePath(Path fromPath, Path toPath, String file, FileOption fileOption) throws ProCoreException {
407 Path from = fromPath.resolve(file);
408 Path to = toPath.resolve(file);
410 if (FileOption.OVERWRITE.equals(fileOption))
411 Files.move(from, to, REPLACE_EXISTING);
413 Files.move(from, to);
414 } catch (IOException e) {
415 throw new ProCoreException("Could not move " + from + " to " + to, e);
418 private static void copyTree(Path from, Path to, String path, final FileOption fileOption) throws ProCoreException {
419 copyTree(from.resolve(path), to.resolve(path), fileOption);
421 private static void copyTree(Path from, Path to, final FileOption fileOption) throws ProCoreException {
422 class Visitor extends SimpleFileVisitor<Path> {
423 private Path fromPath;
425 Visitor(Path from, Path to) {
430 public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
431 Path targetPath = toPath.resolve(fromPath.relativize(dir));
432 if (!Files.exists(targetPath)) {
433 Files.createDirectory(targetPath);
435 return FileVisitResult.CONTINUE;
438 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
439 if (FileOption.OVERWRITE.equals(fileOption))
440 Files.copy(file, toPath.resolve(fromPath.relativize(file)), COPY_ATTRIBUTES, REPLACE_EXISTING);
442 Files.copy(file, toPath.resolve(fromPath.relativize(file)), COPY_ATTRIBUTES);
443 return FileVisitResult.CONTINUE;
447 Visitor v = new Visitor(from, to);
448 EnumSet<FileVisitOption> opts = EnumSet.noneOf(FileVisitOption.class);
449 Files.walkFileTree(from, opts, Integer.MAX_VALUE, v);
450 } catch (IOException e) {
451 throw new ProCoreException("Could not copy " + from + " to " + to, e);
454 private static void deleteTree(Path path) throws ProCoreException {
455 if (!Files.exists(path))
457 class Visitor extends SimpleFileVisitor<Path> {
459 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
462 } catch (IOException ioe) {
463 ioe.printStackTrace();
466 return FileVisitResult.CONTINUE;
469 public FileVisitResult postVisitDirectory(Path dir, IOException e) throws IOException {
473 } catch (IOException ioe) {
474 ioe.printStackTrace();
477 return FileVisitResult.CONTINUE;
483 Visitor v = new Visitor();
484 EnumSet<FileVisitOption> opts = EnumSet.noneOf(FileVisitOption.class);
485 Files.walkFileTree(path, opts, Integer.MAX_VALUE, v);
486 } catch (IOException e) {
487 throw new ProCoreException("Could not delete " + path, e);
490 private static void moveFolder(Path fromPath, Path toPath, String folder, FileOption fileOption) throws ProCoreException {
491 Path from = fromPath.resolve(folder);
492 Path to = toPath.resolve(folder);
493 copyTree(from, to, fileOption);
501 private void copyFiles(final Path from, final Path to, String pattern, final FileOption fileOption) throws ProCoreException {
502 class Visitor extends SimpleFileVisitor<Path> {
503 private final PathMatcher matcher;
504 Visitor(String pattern) {
505 matcher = FileSystems.getDefault().getPathMatcher("glob:" + pattern);
508 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
509 Path name = file.getFileName();
511 return FileVisitResult.CONTINUE;
512 else if (!matcher.matches(name))
513 return FileVisitResult.CONTINUE;
514 if (FileOption.OVERWRITE.equals(fileOption))
515 Files.copy(file, to.resolve(name), REPLACE_EXISTING);
517 Files.copy(file, to.resolve(name));
518 return FileVisitResult.CONTINUE;
521 public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
522 if (dir.equals(from))
523 return FileVisitResult.CONTINUE;
525 return FileVisitResult.SKIP_SUBTREE;
529 Visitor v = new Visitor(pattern);
530 EnumSet<FileVisitOption> opts = EnumSet.noneOf(FileVisitOption.class);
531 Files.walkFileTree(from, opts, Integer.MAX_VALUE, v);
532 } catch (IOException e) {
533 throw new ProCoreException("Could not copy " + from + " to " + to, e);
536 private void moveFiles(final Path from, final Path to, String pattern, final FileOption fileOption) throws ProCoreException {
537 class Visitor extends SimpleFileVisitor<Path> {
538 private final PathMatcher matcher;
539 Visitor(String pattern) {
540 matcher = FileSystems.getDefault().getPathMatcher("glob:" + pattern);
543 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
544 Path name = file.getFileName();
546 return FileVisitResult.CONTINUE;
547 else if (!matcher.matches(name))
548 return FileVisitResult.CONTINUE;
549 if (FileOption.OVERWRITE.equals(fileOption))
550 Files.move(file, to.resolve(name), REPLACE_EXISTING);
552 Files.move(file, to.resolve(name));
553 return FileVisitResult.CONTINUE;
556 public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
557 if (dir.equals(from))
558 return FileVisitResult.CONTINUE;
560 return FileVisitResult.SKIP_SUBTREE;
564 Visitor v = new Visitor(pattern);
565 EnumSet<FileVisitOption> opts = EnumSet.noneOf(FileVisitOption.class);
566 Files.walkFileTree(from, opts, Integer.MAX_VALUE, v);
567 } catch (IOException e) {
568 throw new ProCoreException("Could not move " + from + " to " + to, e);
571 private void deleteClusters(final Path head, final Path tail) throws ProCoreException {
572 class Visitor extends SimpleFileVisitor<Path> {
573 private final PathMatcher matcher;
574 Visitor(String pattern) {
575 matcher = FileSystems.getDefault().getPathMatcher("glob:" + ProCoreServer.DELETED_PATTERN);
578 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
579 Path name = file.getFileName();
581 return FileVisitResult.CONTINUE;
582 else if (!matcher.matches(name))
583 return FileVisitResult.CONTINUE;
584 String deletedStr = name.toString();
585 String indexName = deletedStr.replaceFirst(ProCoreServer.DELETED_PREFIX, ProCoreServer.INDEX_PREFIX);
586 String dataName = deletedStr.replaceFirst(ProCoreServer.DELETED_PREFIX, ProCoreServer.DATA_PREFIX);
587 String[] ss = deletedStr.split("\\.");
588 String valuePattern = ProCoreServer.VALUE_PREFIX + ss[1] + "." + ss[2] + "*";
590 Files.delete(head.resolve(indexName));
591 Files.delete(head.resolve(dataName));
593 deleteFiles(head, valuePattern);
594 if (Files.isDirectory(tail)) {
595 Files.deleteIfExists(tail.resolve(dataName));
596 deleteFiles(tail, valuePattern);
598 } catch (ProCoreException e) {
599 throw new IOException("Delete values failed.", e);
601 return FileVisitResult.CONTINUE;
604 public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
605 if (dir.equals(head))
606 return FileVisitResult.CONTINUE;
608 return FileVisitResult.SKIP_SUBTREE;
612 Visitor v = new Visitor(ProCoreServer.DELETED_PREFIX);
613 EnumSet<FileVisitOption> opts = EnumSet.noneOf(FileVisitOption.class);
614 Files.walkFileTree(head, opts, Integer.MAX_VALUE, v);
615 } catch (IOException e) {
616 throw new ProCoreException("Could not delete cluster(s).\nhead=" + head + "\ntail=" + tail);
619 private static long convertHexStringToLong(String hex) {
620 if (hex.length() < 16)
621 return Long.parseLong(hex, 16);
622 long f = Long.parseLong(hex.substring(0,1), 16) << 60;
623 long l = Long.parseLong(hex.substring(1,16), 16);
626 private void purgeValues(final Path folder) throws ProCoreException {
628 long first; // First part of cluster id.
629 long second; // Secon part of cluster id.
630 int index; // Resource index of the value.
631 transient long cs; // Change set of the value.
632 public Value(long first, long second, int index, long cs) {
634 this.second = second;
639 return ProCoreServer.VALUE_PREFIX + toString() + ProCoreServer.VALUE_SUFFIX;
642 public String toString() {
643 return String.format("%x.%x.%d.%d", first, second, index, cs);
646 public boolean equals(Object o) {
649 else if (!(o instanceof Value))
652 return first == x.first && second == x.second && index == x.index;
655 public int hashCode() {
657 int f = (int)(first ^ (first >>> 32));
658 result = 31 * result + f;
659 int s = (int)(second ^ (second >>> 32));
660 result = 31 * result + s;
661 return result + index;
664 File[] files = folder.toFile().listFiles(new FilenameFilter() {
666 public boolean accept(File dir, String name) {
667 return name.startsWith(ProCoreServer.VALUE_PREFIX);
670 HashMap<Value, Value> values = new HashMap<Value, Value>();
671 for (int i = 0; i < files.length; ++i) {
672 String s = files[i].getName();
673 String[] ss = s.split("\\.");
674 if (ss.length != 6) {
675 Logger.defaultLogError("Illegal external value file name. name=" + s);
678 long first = convertHexStringToLong(ss[1]);
679 long second = convertHexStringToLong(ss[2]);
680 int ri = Integer.parseInt(ss[3]);
681 long cs = Long.parseLong(ss[4]);
682 Value nv = new Value(first, second, ri, cs);
683 Value ov = values.get(nv);
686 else if (ov.cs < nv.cs) {
687 deleteFile(folder, ov.getName(), FileOption.IF_EXIST);
690 deleteFile(folder, nv.getName(), FileOption.IF_EXIST);
693 private long getNextClusterId(final Path db) throws ProCoreException {
695 Path dbHead = db.resolve(ProCoreServer.BRANCH_DIR);
696 File[] files = dbHead.toFile().listFiles(new FilenameFilter() {
698 public boolean accept(File dir, String name) {
699 return name.startsWith(ProCoreServer.DATA_PREFIX);
702 for (int i = 0; i < files.length; ++i) {
703 String s = files[i].getName();
704 String[] ss = s.split("\\.", 4);
705 if (ss.length != 4) {
706 Logger.defaultLogError("Illegal cluster file name. name=" + s);
709 long id = convertHexStringToLong(ss[2]);
713 final Path dbTail = db.resolve(ProCoreServer.TAIL_DIR);
714 if (!Files.exists(dbTail))
715 return clusterId + 1;
716 class Visitor extends SimpleFileVisitor<Path> {
717 private final PathMatcher matcher;
719 Visitor(String pattern, long clusterId) {
720 this.clusterId = clusterId;
721 matcher = FileSystems.getDefault().getPathMatcher("glob:" + pattern);
724 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
725 Path name = file.getFileName();
727 return FileVisitResult.CONTINUE;
728 else if (!matcher.matches(name))
729 return FileVisitResult.CONTINUE;
730 String s = name.toString();
731 String[] ss = s.split("\\.", 4);
732 if (ss.length != 4) {
733 Logger.defaultLogError("Illegal cluster file name. name=" + s);
734 return FileVisitResult.CONTINUE;
736 long id = convertHexStringToLong(ss[2]);
739 return FileVisitResult.CONTINUE;
742 public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
743 if (dir.equals(dbTail))
744 return FileVisitResult.CONTINUE;
746 return FileVisitResult.SKIP_SUBTREE;
750 Visitor v = new Visitor(ProCoreServer.DATA_PATTERN, clusterId);
751 EnumSet<FileVisitOption> opts = EnumSet.noneOf(FileVisitOption.class);
752 Files.walkFileTree(dbTail, opts, Integer.MAX_VALUE, v);
753 return v.clusterId + 1;
754 } catch (IOException e) {
755 throw new ProCoreException("Could not get next free cluster id for " + db, e);
758 private static void deleteFiles(final Path folder, String pattern) throws ProCoreException {
759 if (!Files.exists(folder))
761 class Visitor extends SimpleFileVisitor<Path> {
762 private final PathMatcher matcher;
763 Visitor(String pattern) {
764 matcher = FileSystems.getDefault().getPathMatcher("glob:" + pattern);
767 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
768 Path name = file.getFileName();
770 return FileVisitResult.CONTINUE;
771 else if (!matcher.matches(name))
772 return FileVisitResult.CONTINUE;
774 return FileVisitResult.CONTINUE;
777 public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
778 if (dir.equals(folder))
779 return FileVisitResult.CONTINUE;
781 return FileVisitResult.SKIP_SUBTREE;
785 Visitor v = new Visitor(pattern);
786 EnumSet<FileVisitOption> opts = EnumSet.noneOf(FileVisitOption.class);
787 Files.walkFileTree(folder, opts, Integer.MAX_VALUE, v);
788 } catch (IOException e) {
789 throw new ProCoreException("Could not delete " + folder + " with " + pattern, e);
792 public void copy(File to) throws ProCoreException {
793 copyDatabaseFiles(server.getFolder().toPath(), to.toPath());
795 public boolean isServerAlive() {
796 return server.isAlive();
798 Client newClient() throws ProCoreException {
799 return server.newClient();
801 public void createFolder() throws ProCoreException {
802 if (server.folder.exists())
804 boolean created = server.folder.mkdirs();
806 throw new ProCoreException("Could not create folder=" + server.folder);
809 * @throws ProCoreException if could not stop server.
811 public boolean serverTryToStop() throws ProCoreException {
812 return server.tryToStop();
816 void Do(ProCoreServer proCoreServer) {
821 private final File folder; // Always non null.
822 private final ProCoreServer proCoreServer;
823 void start() throws ProCoreException {
825 proCoreServer.start();
826 } catch (DatabaseLastExitException e) {
827 if (null == dbUserAgent)
829 if (!dbUserAgent.handleStart(e))
830 throw new ProCoreException(folder, "Failed to handle start exception.", e);
831 proCoreServer.start();
834 void stop() throws ProCoreException {
836 proCoreServer.stop();
837 } catch (InterruptedException e) {
838 if (proCoreServer.isAlive())
839 throw new ProCoreException("ProCoreServer stop was interrupted.", e);
844 return proCoreServer.isAlive();
845 } catch (ProCoreException e) {
846 Logger.defaultLogError(e);
850 Server(File aFolder) {
852 throw new RuntimeException("Database folder can not be null.");
854 folder = aFolder.getCanonicalFile();
855 } catch (IOException e) {
856 String t = "Could not get canonical path. file=" + aFolder;
857 Logger.defaultLogError(t);
858 throw new RuntimeException(t);
861 File serverFolder = org.simantics.db.server.internal.Activator.getServerFolder();
862 if (!folder.isDirectory())
864 proCoreServer = ProCoreServer.getProCoreServer(serverFolder, folder);
866 } catch (Throwable t) {
867 Logger.defaultLogError(t);
868 throw new RuntimeException(t);
871 void createConfiguration() throws ProCoreException {
872 if (!folder.isDirectory())
873 throw new ProCoreException("Can't create configuration because folder is not ok. folder=" + folder.getAbsolutePath());
874 File file = new File(folder, ProCoreServer.CONFIG_FILE);
876 file.createNewFile();
877 } catch (IOException e) {
878 throw new ProCoreException("Can't create configuration file. file=" + file.getAbsolutePath());
881 void deleteGuard() throws ProCoreException {
882 if (server.isActive())
883 throw new ProCoreException("Will not delete guard file when server is alive.");
884 if (!server.isFolderOk(server.getFolder()))
885 throw new ProCoreException("Will not delete guard file when server folder is not ok. folder=" + server.getFolder());
886 File file = new File(folder, ProCoreServer.GUARD_FILE);
888 throw new ProCoreException("Guard file does not exist. file=" + file.getAbsolutePath());
889 boolean deleted = file.delete();
891 throw new ProCoreException("Failed to delete file=" + file.getAbsolutePath());
893 String execute(String aCommand) throws ProCoreException {
895 return proCoreServer.execute(aCommand);
896 } catch (InterruptedException e) {
897 throw new ProCoreException("Execute call was interrupted.", e);
901 Status status = Status.NoDatabase;
904 path = folder.getCanonicalPath();
905 } catch (IOException e) {
906 Util.logError("Could not get canonical path for folder. folder=" + folder.getAbsolutePath(), e);
909 if (!isFolderOk(folder))
910 status = Status.NoDatabase;
912 status = Status.NotRunning;
913 else if (isConnected()) {
915 status = Status.Local;
917 status = Status.Remote;
919 status = Status.Standalone;
920 status.message = status.getString() + "@" + path;
928 return proCoreServer.isActive();
929 } catch (ProCoreException e) {
930 Logger.defaultLogError("IsActive failed.", e);
934 boolean isConnected() {
936 return proCoreServer.isConnected();
937 } catch (ProCoreException e) {
938 Logger.defaultLogError("IsConnected failed.", e);
944 return proCoreServer.isLocal();
945 } catch (ProCoreException e) {
946 Logger.defaultLogError("IsLocal faailed.", e);
950 void connect() throws ProCoreException, InterruptedException {
951 proCoreServer.connect();
955 proCoreServer.disconnect();
956 } catch (ProCoreException e) {
957 Logger.defaultLogError("Could not disconnect.", e);
962 * @return true if given folder contains database journal file or
963 * configuration file.
965 boolean isFolderOk(File aFolder) {
966 if (!aFolder.isDirectory())
968 File config = new File(aFolder, ProCoreServer.CONFIG_FILE);
971 File journal = new File(aFolder, ProCoreServer.JOURNAL_FILE);
972 if (journal.exists())
977 * @throws ProCoreException if could not stop server.
979 boolean tryToStop() throws ProCoreException {
980 return proCoreServer.tryToStop();
982 Client newClient() throws ProCoreException {
983 return proCoreServer.newClient();
986 // From interface Database
988 public void initFolder(Properties properties) throws ProCoreException {
990 serverCreateConfiguration();
993 public void deleteFiles() throws ProCoreException {
994 deleteDatabaseFiles();
997 public void start() throws ProCoreException {
1001 public boolean isRunning() throws ProCoreException {
1002 return isServerAlive();
1005 public boolean tryToStop() throws ProCoreException {
1006 return serverTryToStop();
1009 public String execute(String aCommand) throws ProCoreException {
1010 return server.execute(aCommand);
1013 public void purgeDatabase() throws SDBException {
1014 synchronized (server.proCoreServer.getProCoreClient()) {
1015 if (!server.isLocal())
1016 throw new ProCoreException("Purge is allowed only for local server.");
1017 List<Client> clients = sessionManager.disconnect(this);
1020 sessionManager.connect(this, clients);
1021 } catch (InterruptedException e) {
1022 throw new ProCoreException("Failed to connect after purge.", e);
1027 public Session newSession(ServiceLocator locator) throws ProCoreException {
1028 return sessionManager.newSession(this);
1031 public Path createFromChangeSets(int revision) throws ProCoreException {
1033 throw new ProCoreException("Folder must be valid database folder to create database from journal." );
1034 File folder = server.getFolder();
1035 File file = new File(folder, ProCoreServer.DCS_FILE);
1036 if (!file.isFile() && !file.canRead())
1037 throw new ProCoreException("Dump file must be readable. file=" + file.getAbsolutePath());
1038 Path db = Paths.get(folder.getAbsolutePath());
1039 Path temp = createTempFolder(db, "Could not create temporary directory for database to be created from change sets.");
1040 Server s = new Server(temp.toFile());
1041 s.createConfiguration();
1043 String t = s.execute("loadChangeSets .. " + revision);
1045 throw new ProCoreException("Could not read journal. reply=" + t);
1048 int cs = Integer.parseInt(t);
1051 throw new ProCoreException("Could not load change sets. reply=" + t);
1052 } catch (NumberFormatException e) {
1053 throw new ProCoreException("Could not load change sets. reply=" + t);
1057 public void clone(File to, int revision, boolean saveHistory) {
1058 String history = saveHistory ? "with history." : "without history.";
1059 String message = "Clone to " + to.getAbsolutePath() + "@" + revision + " " + history;
1060 Util.trace(message);
1063 public DatabaseUserAgent getUserAgent() {
1067 public void setUserAgent(DatabaseUserAgent dbUserAgent) {
1068 this.dbUserAgent = dbUserAgent;
1071 public Status getStatus() {
1072 return server.getStatus();
1075 public File getFolder() {
1076 return server.getFolder();
1079 public boolean isFolderOk() {
1080 return server.isFolderOk(server.getFolder());
1083 public boolean isFolderOk(File folder) {
1084 return server.isFolderOk(folder);
1086 public boolean isFolderEmpty() {
1087 return isFolderEmpty(server.getFolder());
1090 public boolean isFolderEmpty(File folder) {
1091 return isFolderEmpty(folder.toPath());
1094 public void deleteGuard() throws ProCoreException {
1095 server.deleteGuard();
1098 public boolean isConnected() throws ProCoreException {
1099 return server.isConnected();
1102 public void connect() throws ProCoreException {
1104 throw new ProCoreException("Could not connect to " + getFolder());
1105 if (!server.isAlive())
1111 } catch (InterruptedException e) {
1112 Util.logError("Server connect was interrupted.", e);
1114 if (server.isActive())
1116 throw new ProCoreException("Could not connect to " + getFolder());
1119 public void disconnect() throws ProCoreException {
1120 server.disconnect();
1123 public Path dumpChangeSets() throws ProCoreException {
1125 throw new ProCoreException("Folder must be set to dump change sets.");
1126 if (!server.isActive())
1127 throw new ProCoreException("Server must be responsive to dump change sets.");
1128 String t = server.execute("dumpChangeSets").replaceAll("\n", "");
1130 int ncs = Integer.parseInt(t);
1133 File file = new File(getFolder(), ProCoreServer.DCS_FILE);
1134 return file.toPath();
1135 } catch (NumberFormatException e) {
1136 throw new ProCoreException("Could not dump change sets.", e);
1140 public long serverGetTailChangeSetId() throws ProCoreException {
1142 return server.proCoreServer.getTailData().nextChangeSetId;
1143 } catch (TailReadException e) {
1148 public JournalI getJournal() throws ProCoreException {
1151 class JournalI implements Database.Journal {
1153 private final File journalFile;
1154 private long lastModified;
1155 private RandomAccessFile file; // Journal file.
1156 private ArrayList<Long> offsets = new ArrayList<Long>(); // Offsets to commands in journal file.
1157 private Line lines[] = new Line[256];
1158 int firstLine = 0; // Index of the first element in lines table.
1159 Analyzed(File dbFolder) {
1160 journalFile = new File(dbFolder, ProCoreServer.JOURNAL_FILE);
1161 for (int i=0; i<lines.length; ++i)
1162 lines[i] = new Line();
1164 private boolean canRead() {
1165 return journalFile.isFile() && journalFile.canRead();
1167 private int count() {
1169 return readOffsets().size();
1170 } catch (ProCoreException e) {
1171 Util.logError("Failed to read or analyze journal. file=" + journalFile, e);
1176 private void clear() {
1180 private void close() {
1184 } catch (Throwable e) {
1185 Logger.defaultLogError("Close file threw an exception.", e);
1190 private void open() throws ProCoreException {
1192 file = new RandomAccessFile(journalFile, "r");
1193 } catch (Throwable e) {
1195 throw new ProCoreException("Failed to open journal file.", e);
1198 private String getComment(byte[] bytes) throws ProCoreException {
1199 Serializer METADATA_SERIALIZER = Bindings.getSerializerUnchecked(new TreeMapBinding(Bindings.STRING, Bindings.BYTE_ARRAY));
1201 @SuppressWarnings("unchecked")
1202 TreeMap<String, byte[]> metadata = (TreeMap<String, byte[]>) METADATA_SERIALIZER.deserialize(bytes);
1203 CommitMetadata commit = MetadataUtil.getMetadata(metadata, CommitMetadata.class);
1204 String date = "<metadata does not contain date>";
1205 if (null != commit && null != commit.date)
1206 date = commit.date.toString();
1207 CommentMetadata comment = MetadataUtil.getMetadata(metadata, CommentMetadata.class);
1208 String comments = "<metadata does not contain comment>";
1209 if (null != comment)
1210 comments = comment.toString();
1211 return date + " " + comments;
1212 } catch (IOException e) {
1213 throw new ProCoreException("Failed to interpret metadata.", e);
1216 private Line getRow(int index, Line line) throws ProCoreException {
1217 if (index < 0 || index >= offsets.size())
1218 return null; // Index out of range.
1219 if (index < firstLine || index >= firstLine + lines.length) {
1221 if (index < firstLine)
1222 return null; // Index out of range.
1224 int offset = index - firstLine;
1225 line.status = lines[offset].status;
1226 line.request = lines[offset].request;
1227 line.comment = lines[offset].comment;
1230 private void readLines(int first) throws ProCoreException {
1233 for (int i=0, index = first; i<lines.length; ++i, ++index)
1234 readLine(index, lines[i]);
1240 private void readLine(int index, Line line) throws ProCoreException {
1241 if (index >= offsets.size()) {
1242 line.status = false;
1243 line.request = "<Illegal request.>";
1244 line.comment = "<Illegal request.>";
1247 long offset = offsets.get(index);
1250 int i = file.readInt();
1251 int length = Integer.reverseBytes(i);
1252 byte b = file.readByte();
1253 boolean ok = b != 0;
1254 b = file.readByte();
1255 // boolean littleEndian = b != 0;
1257 int type = Integer.reverseBytes(i);
1258 String comment = "";
1260 throw new ProCoreException("Journal file corrupted at" + file.getFilePointer() + ".");
1261 else if (length > 6) {
1263 file.skipBytes(length - 6);
1264 else if (length > 22){
1267 int size = Integer.reverseBytes(i);
1268 if (size != length - 26)
1269 throw new ProCoreException("Metadata corrupted at" + file.getFilePointer() + ".");
1270 byte[] bytes = new byte[size];
1271 file.readFully(bytes);
1272 comment = getComment(bytes);
1273 if (null == comment)
1274 comment = "<metadata does not contain comment>";
1278 int length2 = Integer.reverseBytes(i);
1279 if (length != length2)
1280 throw new ProCoreException("Journal file corrupted at" + file.getFilePointer() + ".");
1281 String command = MessageText.get(type);
1283 line.request = command;
1284 line.comment = comment;
1285 } catch (IOException e) {
1286 throw new ProCoreException("Journal file corrupted.");
1289 private ArrayList<Long> readOffsets() throws ProCoreException {
1296 long modified = journalFile.lastModified();
1297 if (lastModified != 0 && lastModified == modified)
1298 return offsets; // Offsets already up to date.
1305 int i = file.readInt();
1306 int version = Integer.reverseBytes(i);
1308 throw new ProCoreException("Unsupported journal file version. expected=1 got=" + version);
1310 int major = Integer.reverseBytes(i);
1311 if (major != MessageNumber.ProtocolVersionMajor)
1312 throw new ProCoreException("Unsupported journal request major version. expected=" + MessageNumber.ProtocolVersionMajor + " got=" + major);
1314 int minor = Integer.reverseBytes(i);
1315 if (minor > MessageNumber.ProtocolVersionMinor)
1316 throw new ProCoreException("Unsupported journal request minor version. expected=" + MessageNumber.ProtocolVersionMinor + " got=" + minor);
1318 int length = Integer.reverseBytes(i);
1319 while (length > 0) { // Not supporting unsigned integers.
1321 throw new ProCoreException("Journal file corrupted at" + file.getFilePointer() + ".");
1322 file.skipBytes(length);
1324 int length2 = Integer.reverseBytes(i);
1325 if (length != length2)
1326 throw new ProCoreException("Journal file corrupted at " + file.getFilePointer() + ".");
1327 long offset = file.getFilePointer() - length - 8;
1328 offsets.add(offset);
1330 length = Integer.reverseBytes(i);
1332 } catch (EOFException e) {
1333 } catch (IOException e) {
1334 throw new ProCoreException("Failed to get command count.", e);
1338 lastModified = modified;
1343 private final Analyzed analyzed;
1344 JournalI(File dbFolder) {
1345 this.analyzed = new Analyzed(dbFolder);
1348 public boolean canRead() {
1349 return analyzed.canRead() && !isServerAlive();
1352 public int count() {
1353 return analyzed.count();
1356 public int read(int index, Line line) throws ProCoreException {
1357 int count = analyzed.count();
1358 analyzed.getRow(index, line);
1363 public String getCompression() {