1 package org.simantics.db.server.internal;
3 import static java.nio.file.StandardCopyOption.COPY_ATTRIBUTES;
4 import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
6 import java.io.EOFException;
8 import java.io.FilenameFilter;
9 import java.io.IOException;
10 import java.io.OutputStream;
11 import java.io.RandomAccessFile;
12 import java.nio.file.DirectoryStream;
13 import java.nio.file.FileSystems;
14 import java.nio.file.FileVisitOption;
15 import java.nio.file.FileVisitResult;
16 import java.nio.file.Files;
17 import java.nio.file.Path;
18 import java.nio.file.PathMatcher;
19 import java.nio.file.Paths;
20 import java.nio.file.SimpleFileVisitor;
21 import java.nio.file.attribute.BasicFileAttributes;
22 import java.util.ArrayList;
23 import java.util.EnumSet;
24 import java.util.HashMap;
25 import java.util.List;
26 import java.util.Properties;
27 import java.util.TreeMap;
28 import java.util.UUID;
30 import org.simantics.databoard.Bindings;
31 import org.simantics.databoard.binding.impl.TreeMapBinding;
32 import org.simantics.databoard.serialization.Serializer;
33 import org.simantics.db.Database;
34 import org.simantics.db.DatabaseUserAgent;
35 import org.simantics.db.ServiceLocator;
36 import org.simantics.db.common.CommentMetadata;
37 import org.simantics.db.common.CommitMetadata;
38 import org.simantics.db.common.utils.Logger;
39 import org.simantics.db.common.utils.MetadataUtil;
40 import org.simantics.db.exception.SDBException;
41 import org.simantics.db.server.DatabaseLastExitException;
42 import org.simantics.db.server.ProCoreException;
43 import org.simantics.db.server.internal.ProCoreServer.TailFile;
44 import org.simantics.db.server.protocol.MessageNumber;
45 import org.simantics.db.server.protocol.MessageText;
46 import org.slf4j.LoggerFactory;
48 public class DatabaseI implements Database {
50 private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(DatabaseI.class);
52 private static String NL = System.getProperty("line.separator");
53 private static String TEMP_PREFIX = "db.temp.";
54 public static DatabaseI newDatabaseI(File dbFolder) {
55 return new DatabaseI(dbFolder);
57 private final SessionManager sessionManager = new SessionManager();
58 private final Server server;
59 private final JournalI journal;
60 private DatabaseUserAgent dbUserAgent = null;
61 private DatabaseI(File dbFolder) {
62 server = new Server(dbFolder);
63 journal = new JournalI(dbFolder);
65 public boolean isSubFolder(Path base, Path sub)
67 if (null == base || null == sub)
69 return isSubFolder(base.toFile(), sub.toFile());
71 public boolean isSubFolder(File base, File sub)
73 if (null == base || null == sub)
75 Path basePath = base.getCanonicalFile().toPath();
76 Path subPath = sub.getCanonicalFile().toPath();
77 if (subPath.startsWith(basePath))
82 public Path saveDatabase()
83 throws ProCoreException {
84 Path dbFolder = getFolder().toPath();
85 Path parent = getFolder().getParentFile().toPath();
86 Path temp = createTempFolder(parent, "Could not create temporary directory for saving database.");
87 copyTree(dbFolder, temp, FileOption.IF_NEW);
90 public Path saveDatabase(Path parent)
91 throws ProCoreException {
92 Path dbFolder = getFolder().toPath();
94 boolean yes = isSubFolder(dbFolder, parent);
96 throw new ProCoreException("Parent must not be subdirectory of database directory."
97 + NL + "database=" + dbFolder
98 + NL + "parent=" + parent);
99 } catch (IOException e) {
100 throw new ProCoreException("Failed to save database to parent."
101 + NL + "database=" + dbFolder
102 + NL + "parent=" + parent);
104 Path temp = createTempFolder(parent, "Could not create temporary directory for saving database.");
105 copyTree(dbFolder, temp, FileOption.IF_NEW);
108 public void deleteDatabaseFiles() throws ProCoreException {
109 if (server.isAlive()) {
111 if (server.isAlive())
112 throw new ProCoreException("Server must be dead to delete database.");
114 Path db = getFolder().toPath();
115 deleteDatabaseFiles(db); // Redundant, but tests the method.
118 Path copyDatabaseFiles(Path from, Path to) throws ProCoreException {
119 copyTree(from, to, ProCoreServer.BRANCH_DIR, FileOption.IF_EXIST);
120 if (Files.exists(from.resolve(ProCoreServer.TAIL_DIR)))
121 copyTree(from, to, ProCoreServer.TAIL_DIR, FileOption.IF_EXIST);
122 copyPath(from, to, ProCoreServer.CONFIG_FILE, FileOption.IF_EXIST);
123 copyPath(from, to, ProCoreServer.GUARD_FILE, FileOption.IF_EXIST);
124 copyPath(from, to, ProCoreServer.JOURNAL_FILE, FileOption.IF_EXIST);
125 copyPath(from, to, ProCoreServer.LOG_FILE, FileOption.IF_EXIST);
126 copyFiles(from, to, ProCoreServer.PAGE_FILE_PATTERN, FileOption.IF_EXIST);
129 public static void deleteDatabaseFiles(Path from) throws ProCoreException {
130 deleteFile(from, ProCoreServer.DCS_FILE, FileOption.IF_EXIST);
131 deleteFile(from, ProCoreServer.RECOVERY_NEEDED_FILE, FileOption.IF_EXIST);
132 deleteFile(from, ProCoreServer.RECOVERY_IGNORED_FILE, FileOption.IF_EXIST);
133 deleteFile(from, ProCoreServer.PROTOCOL_IGNORED_FILE, FileOption.IF_EXIST);
134 deleteFile(from, ProCoreServer.LOG_FILE, FileOption.IF_EXIST);
135 deleteFile(from, ProCoreServer.GUARD_FILE, FileOption.IF_EXIST);
136 deleteFile(from, ProCoreServer.CONFIG_FILE, FileOption.IF_EXIST);
137 deleteFiles(from, ProCoreServer.PAGE_FILE_PATTERN);
138 deleteTree(from.resolve(ProCoreServer.BRANCH_DIR));
139 deleteFile(from, ProCoreServer.JOURNAL_FILE, FileOption.IF_EXIST);
140 deleteTree(from.resolve(ProCoreServer.TAIL_DIR));
142 Path moveDatabaseFiles(Path from, Path to) throws ProCoreException {
143 moveFolder(from, to, ProCoreServer.BRANCH_DIR, FileOption.IF_NEW);
144 if (Files.exists(from.resolve(ProCoreServer.TAIL_DIR)))
145 moveFolder(from, to, ProCoreServer.TAIL_DIR, FileOption.IF_NEW);
146 movePath(from, to, ProCoreServer.CONFIG_FILE, FileOption.IF_NEW);
147 movePath(from, to, ProCoreServer.GUARD_FILE, FileOption.IF_NEW);
148 movePath(from, to, ProCoreServer.JOURNAL_FILE, FileOption.IF_NEW);
149 movePath(from, to, ProCoreServer.LOG_FILE, FileOption.IF_NEW);
150 moveFiles(from, to, ProCoreServer.PAGE_FILE_PATTERN, FileOption.IF_NEW);
151 movePath(from, to, ProCoreServer.DCS_FILE, FileOption.IF_NEW);
152 moveFile(from, to, ProCoreServer.RECOVERY_NEEDED_FILE, FileOption.IF_NEW);
153 moveFile(from, to, ProCoreServer.RECOVERY_IGNORED_FILE, FileOption.IF_NEW);
154 moveFile(from, to, ProCoreServer.PROTOCOL_IGNORED_FILE, FileOption.IF_NEW);
157 private void moveFile(Path from, Path to, String file, FileOption fileOption) throws ProCoreException {
158 if (Files.exists(from.resolve(file)))
159 movePath(from, to, file, fileOption);
161 public void serverCreateConfiguration() throws ProCoreException {
162 server.createConfiguration();
164 public boolean ignoreExit() throws ProCoreException {
166 throw new ProCoreException("Folder must be valid database folder to ignore exit." );
167 server.deleteGuard();
168 File folder = server.getFolder();
169 Path db = Paths.get(folder.getAbsolutePath());
170 Path ri = db.resolve(ProCoreServer.RECOVERY_IGNORED_FILE);
171 if (!Files.exists(ri))
172 try (OutputStream os = Files.newOutputStream(ri)) {
173 } catch (IOException e) {
174 throw new ProCoreException("Could not create file: " + ri, e);
177 DatabaseUserAgent dbu = dbUserAgent; // Save current user agent.
179 dbUserAgent = null; // Recursion not supported. Disable user agent.
181 ok = server.isActive();
184 dbUserAgent = dbu; // Restore used user agent.
188 Files.deleteIfExists(ri);
189 } catch (IOException e) {
190 Logger.defaultLogError("Could not delete file: " + ri, e);
192 Path rn = db.resolve(ProCoreServer.RECOVERY_NEEDED_FILE);
194 Files.deleteIfExists(rn);
195 } catch (IOException e) {
196 Logger.defaultLogError("Could not delete file: " + rn, e);
202 public boolean ignoreProtocol() throws ProCoreException {
204 throw new ProCoreException("Folder must be valid database folder to ignore exit." );
205 File folder = server.getFolder();
206 Path db = Paths.get(folder.getAbsolutePath());
207 Path ri = db.resolve(ProCoreServer.PROTOCOL_IGNORED_FILE);
208 if (!Files.exists(ri))
209 try (OutputStream os = Files.newOutputStream(ri)) {
210 } catch (IOException e) {
211 throw new ProCoreException("Could not create file: " + ri, e);
216 ok = server.isActive();
222 private long preJournalCheck() throws ProCoreException {
223 File folder = server.getFolder();
224 if (!folder.isDirectory())
225 throw new ProCoreException("Database folder does not exist." + NL + "folder=" + folder);
226 File file = new File(folder, ProCoreServer.JOURNAL_FILE);
228 throw new ProCoreException("Journal file does not exist." + NL + "file=" + file);
229 else if (!file.canRead())
230 throw new ProCoreException("Journal file must be readale to create database from journal." + NL + "file=" + file);
231 else if (server.isAlive())
232 throw new ProCoreException("Server must be dead to create database from journal file." + NL + "file=" + file);
233 return getNextClusterId(folder.toPath());
235 private void postJournalFix(long nextFreeId) throws SDBException {
236 Session s = newSession(null);
237 long current = s.reserveIds(0);
238 if (current < nextFreeId)
239 s.reserveIds((int)(nextFreeId - current));
242 private Path createTempFolder(Path parent, String fail) throws ProCoreException {
244 return Files.createTempDirectory(parent, TEMP_PREFIX);
245 } catch (IOException e) {
246 throw new ProCoreException(fail, e);
249 public void replaceFromJournal() throws SDBException {
250 long nextFreeId = preJournalCheck();
251 Path db = Paths.get(server.getFolder().getAbsolutePath());
252 Path temp = createTempFolder(db, "Could not create temporary directory for database to be created from journal.");
253 movePath(db, temp, ProCoreServer.CONFIG_FILE, FileOption.IF_NEW);
254 movePath(db, temp, ProCoreServer.JOURNAL_FILE, FileOption.IF_NEW);
255 deleteFile(db, ProCoreServer.GUARD_FILE, FileOption.IF_EXIST);
256 deleteFiles(db, ProCoreServer.PAGE_FILE_PATTERN);
257 Path dbHead = db.resolve(ProCoreServer.BRANCH_DIR);
258 deleteFiles(dbHead, ProCoreServer.VALUE_PATTERN);
259 deleteFiles(dbHead, ProCoreServer.DATA_PATTERN);
260 deleteFiles(dbHead, ProCoreServer.INDEX_PATTERN);
261 Path dbHeadTailFile = dbHead.resolve(ProCoreServer.TAIL_FILE);
262 boolean headTailFileExisted = Files.exists(dbHeadTailFile);
263 final long NEXT_REVISION = 1;
264 if (!headTailFileExisted) // Number of change sets and database id not known at this point. Fortunately they are not used by recovery.
265 TailFile.createTailFile(dbHeadTailFile.toFile(), NEXT_REVISION, nextFreeId, UUID.randomUUID().toString());
266 movePath(dbHead, temp, ProCoreServer.TAIL_FILE, FileOption.IF_NEW);
267 Path dbTail = db.resolve(ProCoreServer.TAIL_DIR);
268 boolean tailExisted = Files.isDirectory(dbTail);
270 copyPath(dbTail, dbHead, ProCoreServer.TAIL_FILE, FileOption.IF_NEW);
273 Files.createDirectory(dbTail);
274 } catch (IOException e) {
275 throw new ProCoreException("Failed to create directory: " + dbTail, e);
277 copyPath(temp, dbTail, ProCoreServer.TAIL_FILE, FileOption.IF_NEW);
279 server.createConfiguration();
282 String t = server.execute("journalRead " + temp.getFileName());
284 throw new ProCoreException("Could not read journal. reply=" + t);
285 postJournalFix(nextFreeId); // Not the right way for implementing this, but better than incorrect recovery.
289 movePath(temp, db, ProCoreServer.CONFIG_FILE, FileOption.OVERWRITE);
294 public boolean canPurge() throws ProCoreException {
295 File folder = server.getFolder();
296 Path db = Paths.get(folder.getAbsolutePath());
297 Path dbHead = db.resolve(ProCoreServer.BRANCH_DIR);
298 if (!Files.exists(dbHead))
299 return false; // Already clean.
300 boolean empty = isFolderEmpty(dbHead);
302 return false; // Already clean.
303 final boolean[] found = new boolean[1];
305 dbHead.toFile().listFiles(new FilenameFilter() {
307 public boolean accept(File dir, String name) {
310 else if (!name.equals(ProCoreServer.TAIL_FILE)) {
320 * Removes old history i.e. change sets.
321 * Note that also cleans last exit status and removes guard file.
322 * This means that all information about last exit status is also lost
324 * @throws ProCoreException
326 public void purge() throws SDBException {
327 synchronized (server.proCoreServer.getProCoreClient()) {
328 boolean wasAlive = server.isAlive();
331 Path db = Paths.get(server.getFolder().getAbsolutePath());
332 Path dbHead = db.resolve(ProCoreServer.BRANCH_DIR);
333 Path dbTail = db.resolve(ProCoreServer.TAIL_DIR);
334 if (!Files.isDirectory(dbTail)) {
336 Files.createDirectory(dbTail);
337 } catch (IOException e) {
338 throw new ProCoreException("Failed to create directory: " + dbTail, e);
341 long nextFreeId = getNextClusterId(db);
342 boolean cleanHead = Files.isDirectory(dbHead) && !isFolderEmpty(dbHead);
343 LOGGER.info("Purging old history and exit information. folder=" + db);
345 deleteClusters(dbHead, dbTail);
346 movePath(dbHead, dbTail, ProCoreServer.TAIL_FILE, FileOption.OVERWRITE);
347 moveFiles(dbHead, dbTail, ProCoreServer.VALUE_PATTERN, FileOption.IF_NEW);
348 moveFiles(dbHead, dbTail, ProCoreServer.DATA_PATTERN, FileOption.OVERWRITE);
349 deleteFiles(dbHead, ProCoreServer.INDEX_PATTERN);
351 deleteFiles(db, ProCoreServer.PAGE_FILE_PATTERN);
352 deleteFile(db, ProCoreServer.JOURNAL_FILE, FileOption.IF_EXIST);
353 deleteFile(db, ProCoreServer.LOG_FILE, FileOption.IF_EXIST);
354 deleteFile(db, ProCoreServer.DCS_FILE, FileOption.IF_EXIST);
355 deleteFile(db, ProCoreServer.RECOVERY_NEEDED_FILE, FileOption.IF_EXIST);
356 deleteFile(db, ProCoreServer.RECOVERY_IGNORED_FILE, FileOption.IF_EXIST);
357 deleteFile(db, ProCoreServer.PROTOCOL_IGNORED_FILE, FileOption.IF_EXIST);
358 deleteFile(db, ProCoreServer.GUARD_FILE, FileOption.IF_EXIST);
361 Session s = newSession(null);
362 long current = s.reserveIds(0);
363 if (current < nextFreeId)
364 s.reserveIds((int)(nextFreeId - current));
370 private void copyPath(Path fromFolder, Path toFolder, String file, FileOption fileOption) throws ProCoreException {
371 Path from = fromFolder.resolve(file);
372 Path to = toFolder.resolve(file);
374 if (FileOption.IF_EXIST.equals(fileOption)) {
375 if (!Files.exists(from))
378 if (FileOption.OVERWRITE.equals(fileOption))
379 Files.copy(from, to, COPY_ATTRIBUTES, REPLACE_EXISTING);
381 Files.copy(from, to, COPY_ATTRIBUTES);
382 } catch (IOException e) {
383 throw new ProCoreException("Could not copy " + from + " to " + to, e);
386 private static void deleteFile(Path from, String file, FileOption fileOption) throws ProCoreException {
387 Path path = from.resolve(file);
388 deletePath(path, fileOption);
390 private static void deletePath(Path path, FileOption fileOption) throws ProCoreException {
392 if (FileOption.IF_EXIST.equals(fileOption))
393 Files.deleteIfExists(path);
396 } catch (IOException e) {
397 throw new ProCoreException("Could not delete " + path, e);
400 private boolean isFolderEmpty(final Path folder) { // True if folder exists and is empty.
401 if (!Files.isDirectory(folder))
403 try(DirectoryStream<Path> folderStream = Files.newDirectoryStream(folder)) {
404 return !folderStream.iterator().hasNext();
405 } catch (IOException e) {
406 Logger.defaultLogError("Failed to open folder stream. folder=" + folder, e);
410 private void movePath(Path fromPath, Path toPath, String file, FileOption fileOption) throws ProCoreException {
411 Path from = fromPath.resolve(file);
412 Path to = toPath.resolve(file);
414 if (FileOption.OVERWRITE.equals(fileOption))
415 Files.move(from, to, REPLACE_EXISTING);
417 Files.move(from, to);
418 } catch (IOException e) {
419 throw new ProCoreException("Could not move " + from + " to " + to, e);
422 private static void copyTree(Path from, Path to, String path, final FileOption fileOption) throws ProCoreException {
423 copyTree(from.resolve(path), to.resolve(path), fileOption);
425 private static void copyTree(Path from, Path to, final FileOption fileOption) throws ProCoreException {
426 class Visitor extends SimpleFileVisitor<Path> {
427 private Path fromPath;
429 Visitor(Path from, Path to) {
434 public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
435 Path targetPath = toPath.resolve(fromPath.relativize(dir));
436 if (!Files.exists(targetPath)) {
437 Files.createDirectory(targetPath);
439 return FileVisitResult.CONTINUE;
442 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
443 if (FileOption.OVERWRITE.equals(fileOption))
444 Files.copy(file, toPath.resolve(fromPath.relativize(file)), COPY_ATTRIBUTES, REPLACE_EXISTING);
446 Files.copy(file, toPath.resolve(fromPath.relativize(file)), COPY_ATTRIBUTES);
447 return FileVisitResult.CONTINUE;
451 Visitor v = new Visitor(from, to);
452 EnumSet<FileVisitOption> opts = EnumSet.noneOf(FileVisitOption.class);
453 Files.walkFileTree(from, opts, Integer.MAX_VALUE, v);
454 } catch (IOException e) {
455 throw new ProCoreException("Could not copy " + from + " to " + to, e);
458 private static void deleteTree(Path path) throws ProCoreException {
459 if (!Files.exists(path))
461 class Visitor extends SimpleFileVisitor<Path> {
463 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
466 } catch (IOException ioe) {
467 ioe.printStackTrace();
470 return FileVisitResult.CONTINUE;
473 public FileVisitResult postVisitDirectory(Path dir, IOException e) throws IOException {
477 } catch (IOException ioe) {
478 ioe.printStackTrace();
481 return FileVisitResult.CONTINUE;
487 Visitor v = new Visitor();
488 EnumSet<FileVisitOption> opts = EnumSet.noneOf(FileVisitOption.class);
489 Files.walkFileTree(path, opts, Integer.MAX_VALUE, v);
490 } catch (IOException e) {
491 throw new ProCoreException("Could not delete " + path, e);
494 private static void moveFolder(Path fromPath, Path toPath, String folder, FileOption fileOption) throws ProCoreException {
495 Path from = fromPath.resolve(folder);
496 Path to = toPath.resolve(folder);
497 copyTree(from, to, fileOption);
505 private void copyFiles(final Path from, final Path to, String pattern, final FileOption fileOption) throws ProCoreException {
506 class Visitor extends SimpleFileVisitor<Path> {
507 private final PathMatcher matcher;
508 Visitor(String pattern) {
509 matcher = FileSystems.getDefault().getPathMatcher("glob:" + pattern);
512 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
513 Path name = file.getFileName();
515 return FileVisitResult.CONTINUE;
516 else if (!matcher.matches(name))
517 return FileVisitResult.CONTINUE;
518 if (FileOption.OVERWRITE.equals(fileOption))
519 Files.copy(file, to.resolve(name), REPLACE_EXISTING);
521 Files.copy(file, to.resolve(name));
522 return FileVisitResult.CONTINUE;
525 public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
526 if (dir.equals(from))
527 return FileVisitResult.CONTINUE;
529 return FileVisitResult.SKIP_SUBTREE;
533 Visitor v = new Visitor(pattern);
534 EnumSet<FileVisitOption> opts = EnumSet.noneOf(FileVisitOption.class);
535 Files.walkFileTree(from, opts, Integer.MAX_VALUE, v);
536 } catch (IOException e) {
537 throw new ProCoreException("Could not copy " + from + " to " + to, e);
540 private void moveFiles(final Path from, final Path to, String pattern, final FileOption fileOption) throws ProCoreException {
541 class Visitor extends SimpleFileVisitor<Path> {
542 private final PathMatcher matcher;
543 Visitor(String pattern) {
544 matcher = FileSystems.getDefault().getPathMatcher("glob:" + pattern);
547 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
548 Path name = file.getFileName();
550 return FileVisitResult.CONTINUE;
551 else if (!matcher.matches(name))
552 return FileVisitResult.CONTINUE;
553 if (FileOption.OVERWRITE.equals(fileOption))
554 Files.move(file, to.resolve(name), REPLACE_EXISTING);
556 Files.move(file, to.resolve(name));
557 return FileVisitResult.CONTINUE;
560 public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
561 if (dir.equals(from))
562 return FileVisitResult.CONTINUE;
564 return FileVisitResult.SKIP_SUBTREE;
568 Visitor v = new Visitor(pattern);
569 EnumSet<FileVisitOption> opts = EnumSet.noneOf(FileVisitOption.class);
570 Files.walkFileTree(from, opts, Integer.MAX_VALUE, v);
571 } catch (IOException e) {
572 throw new ProCoreException("Could not move " + from + " to " + to, e);
575 private void deleteClusters(final Path head, final Path tail) throws ProCoreException {
576 class Visitor extends SimpleFileVisitor<Path> {
577 private final PathMatcher matcher;
578 Visitor(String pattern) {
579 matcher = FileSystems.getDefault().getPathMatcher("glob:" + ProCoreServer.DELETED_PATTERN);
582 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
583 Path name = file.getFileName();
585 return FileVisitResult.CONTINUE;
586 else if (!matcher.matches(name))
587 return FileVisitResult.CONTINUE;
588 String deletedStr = name.toString();
589 String indexName = deletedStr.replaceFirst(ProCoreServer.DELETED_PREFIX, ProCoreServer.INDEX_PREFIX);
590 String dataName = deletedStr.replaceFirst(ProCoreServer.DELETED_PREFIX, ProCoreServer.DATA_PREFIX);
591 String[] ss = deletedStr.split("\\.");
592 String valuePattern = ProCoreServer.VALUE_PREFIX + ss[1] + "." + ss[2] + "*";
594 Files.delete(head.resolve(indexName));
595 Files.delete(head.resolve(dataName));
597 deleteFiles(head, valuePattern);
598 if (Files.isDirectory(tail)) {
599 Files.deleteIfExists(tail.resolve(dataName));
600 deleteFiles(tail, valuePattern);
602 } catch (ProCoreException e) {
603 throw new IOException("Delete values failed.", e);
605 return FileVisitResult.CONTINUE;
608 public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
609 if (dir.equals(head))
610 return FileVisitResult.CONTINUE;
612 return FileVisitResult.SKIP_SUBTREE;
616 Visitor v = new Visitor(ProCoreServer.DELETED_PREFIX);
617 EnumSet<FileVisitOption> opts = EnumSet.noneOf(FileVisitOption.class);
618 Files.walkFileTree(head, opts, Integer.MAX_VALUE, v);
619 } catch (IOException e) {
620 throw new ProCoreException("Could not delete cluster(s).\nhead=" + head + "\ntail=" + tail);
623 private static long convertHexStringToLong(String hex) {
624 if (hex.length() < 16)
625 return Long.parseLong(hex, 16);
626 long f = Long.parseLong(hex.substring(0,1), 16) << 60;
627 long l = Long.parseLong(hex.substring(1,16), 16);
630 private void purgeValues(final Path folder) throws ProCoreException {
632 long first; // First part of cluster id.
633 long second; // Secon part of cluster id.
634 int index; // Resource index of the value.
635 transient long cs; // Change set of the value.
636 public Value(long first, long second, int index, long cs) {
638 this.second = second;
643 return ProCoreServer.VALUE_PREFIX + toString() + ProCoreServer.VALUE_SUFFIX;
646 public String toString() {
647 return String.format("%x.%x.%d.%d", first, second, index, cs);
650 public boolean equals(Object o) {
653 else if (!(o instanceof Value))
656 return first == x.first && second == x.second && index == x.index;
659 public int hashCode() {
661 int f = (int)(first ^ (first >>> 32));
662 result = 31 * result + f;
663 int s = (int)(second ^ (second >>> 32));
664 result = 31 * result + s;
665 return result + index;
668 File[] files = folder.toFile().listFiles(new FilenameFilter() {
670 public boolean accept(File dir, String name) {
671 return name.startsWith(ProCoreServer.VALUE_PREFIX);
674 HashMap<Value, Value> values = new HashMap<Value, Value>();
675 for (int i = 0; i < files.length; ++i) {
676 String s = files[i].getName();
677 String[] ss = s.split("\\.");
678 if (ss.length != 6) {
679 Logger.defaultLogError("Illegal external value file name. name=" + s);
682 long first = convertHexStringToLong(ss[1]);
683 long second = convertHexStringToLong(ss[2]);
684 int ri = Integer.parseInt(ss[3]);
685 long cs = Long.parseLong(ss[4]);
686 Value nv = new Value(first, second, ri, cs);
687 Value ov = values.get(nv);
690 else if (ov.cs < nv.cs) {
691 deleteFile(folder, ov.getName(), FileOption.IF_EXIST);
694 deleteFile(folder, nv.getName(), FileOption.IF_EXIST);
697 private long getNextClusterId(final Path db) throws ProCoreException {
699 Path dbHead = db.resolve(ProCoreServer.BRANCH_DIR);
700 File[] files = dbHead.toFile().listFiles(new FilenameFilter() {
702 public boolean accept(File dir, String name) {
703 return name.startsWith(ProCoreServer.DATA_PREFIX);
706 for (int i = 0; i < files.length; ++i) {
707 String s = files[i].getName();
708 String[] ss = s.split("\\.", 4);
709 if (ss.length != 4) {
710 Logger.defaultLogError("Illegal cluster file name. name=" + s);
713 long id = convertHexStringToLong(ss[2]);
717 final Path dbTail = db.resolve(ProCoreServer.TAIL_DIR);
718 if (!Files.exists(dbTail))
719 return clusterId + 1;
720 class Visitor extends SimpleFileVisitor<Path> {
721 private final PathMatcher matcher;
723 Visitor(String pattern, long clusterId) {
724 this.clusterId = clusterId;
725 matcher = FileSystems.getDefault().getPathMatcher("glob:" + pattern);
728 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
729 Path name = file.getFileName();
731 return FileVisitResult.CONTINUE;
732 else if (!matcher.matches(name))
733 return FileVisitResult.CONTINUE;
734 String s = name.toString();
735 String[] ss = s.split("\\.", 4);
736 if (ss.length != 4) {
737 Logger.defaultLogError("Illegal cluster file name. name=" + s);
738 return FileVisitResult.CONTINUE;
740 long id = convertHexStringToLong(ss[2]);
743 return FileVisitResult.CONTINUE;
746 public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
747 if (dir.equals(dbTail))
748 return FileVisitResult.CONTINUE;
750 return FileVisitResult.SKIP_SUBTREE;
754 Visitor v = new Visitor(ProCoreServer.DATA_PATTERN, clusterId);
755 EnumSet<FileVisitOption> opts = EnumSet.noneOf(FileVisitOption.class);
756 Files.walkFileTree(dbTail, opts, Integer.MAX_VALUE, v);
757 return v.clusterId + 1;
758 } catch (IOException e) {
759 throw new ProCoreException("Could not get next free cluster id for " + db, e);
762 private static void deleteFiles(final Path folder, String pattern) throws ProCoreException {
763 if (!Files.exists(folder))
765 class Visitor extends SimpleFileVisitor<Path> {
766 private final PathMatcher matcher;
767 Visitor(String pattern) {
768 matcher = FileSystems.getDefault().getPathMatcher("glob:" + pattern);
771 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
772 Path name = file.getFileName();
774 return FileVisitResult.CONTINUE;
775 else if (!matcher.matches(name))
776 return FileVisitResult.CONTINUE;
778 return FileVisitResult.CONTINUE;
781 public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
782 if (dir.equals(folder))
783 return FileVisitResult.CONTINUE;
785 return FileVisitResult.SKIP_SUBTREE;
789 Visitor v = new Visitor(pattern);
790 EnumSet<FileVisitOption> opts = EnumSet.noneOf(FileVisitOption.class);
791 Files.walkFileTree(folder, opts, Integer.MAX_VALUE, v);
792 } catch (IOException e) {
793 throw new ProCoreException("Could not delete " + folder + " with " + pattern, e);
796 public void copy(File to) throws ProCoreException {
797 copyDatabaseFiles(server.getFolder().toPath(), to.toPath());
799 public boolean isServerAlive() {
800 return server.isAlive();
802 Client newClient() throws ProCoreException {
803 return server.newClient();
805 public void createFolder() throws ProCoreException {
806 if (server.folder.exists())
808 boolean created = server.folder.mkdirs();
810 throw new ProCoreException("Could not create folder=" + server.folder);
813 * @throws ProCoreException if could not stop server.
815 public boolean serverTryToStop() throws ProCoreException {
816 return server.tryToStop();
820 void Do(ProCoreServer proCoreServer) {
825 private final File folder; // Always non null.
826 private final ProCoreServer proCoreServer;
827 void start() throws ProCoreException {
829 proCoreServer.start();
830 } catch (DatabaseLastExitException e) {
831 if (null == dbUserAgent)
833 if (!dbUserAgent.handleStart(e))
834 throw new ProCoreException(folder, "Failed to handle start exception.", e);
835 proCoreServer.start();
838 void stop() throws ProCoreException {
840 proCoreServer.stop();
841 } catch (InterruptedException e) {
842 if (proCoreServer.isAlive())
843 throw new ProCoreException("ProCoreServer stop was interrupted.", e);
848 return proCoreServer.isAlive();
849 } catch (ProCoreException e) {
850 Logger.defaultLogError(e);
854 Server(File aFolder) {
856 throw new RuntimeException("Database folder can not be null.");
858 folder = aFolder.getCanonicalFile();
859 } catch (IOException e) {
860 String t = "Could not get canonical path. file=" + aFolder;
861 Logger.defaultLogError(t);
862 throw new RuntimeException(t);
865 File serverFolder = org.simantics.db.server.internal.Activator.getServerFolder();
866 if (!folder.isDirectory())
868 proCoreServer = ProCoreServer.getProCoreServer(serverFolder, folder);
870 } catch (Throwable t) {
871 Logger.defaultLogError(t);
872 throw new RuntimeException(t);
875 void createConfiguration() throws ProCoreException {
876 if (!folder.isDirectory())
877 throw new ProCoreException("Can't create configuration because folder is not ok. folder=" + folder.getAbsolutePath());
878 File file = new File(folder, ProCoreServer.CONFIG_FILE);
880 file.createNewFile();
881 } catch (IOException e) {
882 throw new ProCoreException("Can't create configuration file. file=" + file.getAbsolutePath());
885 void deleteGuard() throws ProCoreException {
886 if (server.isActive())
887 throw new ProCoreException("Will not delete guard file when server is alive.");
888 if (!server.isFolderOk(server.getFolder()))
889 throw new ProCoreException("Will not delete guard file when server folder is not ok. folder=" + server.getFolder());
890 File file = new File(folder, ProCoreServer.GUARD_FILE);
892 throw new ProCoreException("Guard file does not exist. file=" + file.getAbsolutePath());
893 boolean deleted = file.delete();
895 throw new ProCoreException("Failed to delete file=" + file.getAbsolutePath());
897 String execute(String aCommand) throws ProCoreException {
899 return proCoreServer.execute(aCommand);
900 } catch (InterruptedException e) {
901 throw new ProCoreException("Execute call was interrupted.", e);
905 Status status = Status.NoDatabase;
908 path = folder.getCanonicalPath();
909 } catch (IOException e) {
910 Util.logError("Could not get canonical path for folder. folder=" + folder.getAbsolutePath(), e);
913 if (!isFolderOk(folder))
914 status = Status.NoDatabase;
916 status = Status.NotRunning;
917 else if (isConnected()) {
919 status = Status.Local;
921 status = Status.Remote;
923 status = Status.Standalone;
924 status.message = status.getString() + "@" + path;
932 return proCoreServer.isActive();
933 } catch (ProCoreException e) {
934 Logger.defaultLogError("IsActive failed.", e);
938 boolean isConnected() {
940 return proCoreServer.isConnected();
941 } catch (ProCoreException e) {
942 Logger.defaultLogError("IsConnected failed.", e);
948 return proCoreServer.isLocal();
949 } catch (ProCoreException e) {
950 Logger.defaultLogError("IsLocal faailed.", e);
954 void connect() throws ProCoreException, InterruptedException {
955 proCoreServer.connect();
959 proCoreServer.disconnect();
960 } catch (ProCoreException e) {
961 Logger.defaultLogError("Could not disconnect.", e);
966 * @return true if given folder contains database journal file or
967 * configuration file.
969 boolean isFolderOk(File aFolder) {
970 if (!aFolder.isDirectory())
972 File config = new File(aFolder, ProCoreServer.CONFIG_FILE);
975 File journal = new File(aFolder, ProCoreServer.JOURNAL_FILE);
976 if (journal.exists())
981 * @throws ProCoreException if could not stop server.
983 boolean tryToStop() throws ProCoreException {
984 return proCoreServer.tryToStop();
986 Client newClient() throws ProCoreException {
987 return proCoreServer.newClient();
990 // From interface Database
992 public void initFolder(Properties properties) throws ProCoreException {
994 serverCreateConfiguration();
997 public void deleteFiles() throws ProCoreException {
998 deleteDatabaseFiles();
1001 public void start() throws ProCoreException {
1005 public boolean isRunning() throws ProCoreException {
1006 return isServerAlive();
1009 public boolean tryToStop() throws ProCoreException {
1010 return serverTryToStop();
1013 public String execute(String aCommand) throws ProCoreException {
1014 return server.execute(aCommand);
1017 public void purgeDatabase() throws SDBException {
1018 synchronized (server.proCoreServer.getProCoreClient()) {
1019 if (!server.isLocal())
1020 throw new ProCoreException("Purge is allowed only for local server.");
1021 List<Client> clients = sessionManager.disconnect(this);
1024 sessionManager.connect(this, clients);
1025 } catch (InterruptedException e) {
1026 throw new ProCoreException("Failed to connect after purge.", e);
1031 public Session newSession(ServiceLocator locator) throws ProCoreException {
1032 return sessionManager.newSession(this);
1035 public Path createFromChangeSets(int revision) throws ProCoreException {
1037 throw new ProCoreException("Folder must be valid database folder to create database from journal." );
1038 File folder = server.getFolder();
1039 File file = new File(folder, ProCoreServer.DCS_FILE);
1040 if (!file.isFile() && !file.canRead())
1041 throw new ProCoreException("Dump file must be readable. file=" + file.getAbsolutePath());
1042 Path db = Paths.get(folder.getAbsolutePath());
1043 Path temp = createTempFolder(db, "Could not create temporary directory for database to be created from change sets.");
1044 Server s = new Server(temp.toFile());
1045 s.createConfiguration();
1047 String t = s.execute("loadChangeSets .. " + revision);
1049 throw new ProCoreException("Could not read journal. reply=" + t);
1052 int cs = Integer.parseInt(t);
1055 throw new ProCoreException("Could not load change sets. reply=" + t);
1056 } catch (NumberFormatException e) {
1057 throw new ProCoreException("Could not load change sets. reply=" + t);
1061 public void clone(File to, int revision, boolean saveHistory) {
1062 String history = saveHistory ? "with history." : "without history.";
1063 String message = "Clone to " + to.getAbsolutePath() + "@" + revision + " " + history;
1064 Util.trace(message);
1067 public DatabaseUserAgent getUserAgent() {
1071 public void setUserAgent(DatabaseUserAgent dbUserAgent) {
1072 this.dbUserAgent = dbUserAgent;
1075 public Status getStatus() {
1076 return server.getStatus();
1079 public File getFolder() {
1080 return server.getFolder();
1083 public boolean isFolderOk() {
1084 return server.isFolderOk(server.getFolder());
1087 public boolean isFolderOk(File folder) {
1088 return server.isFolderOk(folder);
1090 public boolean isFolderEmpty() {
1091 return isFolderEmpty(server.getFolder());
1094 public boolean isFolderEmpty(File folder) {
1095 return isFolderEmpty(folder.toPath());
1098 public void deleteGuard() throws ProCoreException {
1099 server.deleteGuard();
1102 public boolean isConnected() throws ProCoreException {
1103 return server.isConnected();
1106 public void connect() throws ProCoreException {
1108 throw new ProCoreException("Could not connect to " + getFolder());
1109 if (!server.isAlive())
1115 } catch (InterruptedException e) {
1116 Util.logError("Server connect was interrupted.", e);
1118 if (server.isActive())
1120 throw new ProCoreException("Could not connect to " + getFolder());
1123 public void disconnect() throws ProCoreException {
1124 server.disconnect();
1127 public Path dumpChangeSets() throws ProCoreException {
1129 throw new ProCoreException("Folder must be set to dump change sets.");
1130 if (!server.isActive())
1131 throw new ProCoreException("Server must be responsive to dump change sets.");
1132 String t = server.execute("dumpChangeSets").replaceAll("\n", "");
1134 int ncs = Integer.parseInt(t);
1137 File file = new File(getFolder(), ProCoreServer.DCS_FILE);
1138 return file.toPath();
1139 } catch (NumberFormatException e) {
1140 throw new ProCoreException("Could not dump change sets.", e);
1144 public long serverGetTailChangeSetId() throws ProCoreException {
1146 return server.proCoreServer.getTailData().nextChangeSetId;
1147 } catch (TailReadException e) {
1152 public JournalI getJournal() throws ProCoreException {
1155 class JournalI implements Database.Journal {
1157 private final File journalFile;
1158 private long lastModified;
1159 private RandomAccessFile file; // Journal file.
1160 private ArrayList<Long> offsets = new ArrayList<Long>(); // Offsets to commands in journal file.
1161 private Line lines[] = new Line[256];
1162 int firstLine = 0; // Index of the first element in lines table.
1163 Analyzed(File dbFolder) {
1164 journalFile = new File(dbFolder, ProCoreServer.JOURNAL_FILE);
1165 for (int i=0; i<lines.length; ++i)
1166 lines[i] = new Line();
1168 private boolean canRead() {
1169 return journalFile.isFile() && journalFile.canRead();
1171 private int count() {
1173 return readOffsets().size();
1174 } catch (ProCoreException e) {
1175 Util.logError("Failed to read or analyze journal. file=" + journalFile, e);
1180 private void clear() {
1184 private void close() {
1188 } catch (Throwable e) {
1189 Logger.defaultLogError("Close file threw an exception.", e);
1194 private void open() throws ProCoreException {
1196 file = new RandomAccessFile(journalFile, "r");
1197 } catch (Throwable e) {
1199 throw new ProCoreException("Failed to open journal file.", e);
1202 private String getComment(byte[] bytes) throws ProCoreException {
1203 Serializer METADATA_SERIALIZER = Bindings.getSerializerUnchecked(new TreeMapBinding(Bindings.STRING, Bindings.BYTE_ARRAY));
1205 @SuppressWarnings("unchecked")
1206 TreeMap<String, byte[]> metadata = (TreeMap<String, byte[]>) METADATA_SERIALIZER.deserialize(bytes);
1207 CommitMetadata commit = MetadataUtil.getMetadata(metadata, CommitMetadata.class);
1208 String date = "<metadata does not contain date>";
1209 if (null != commit && null != commit.date)
1210 date = commit.date.toString();
1211 CommentMetadata comment = MetadataUtil.getMetadata(metadata, CommentMetadata.class);
1212 String comments = "<metadata does not contain comment>";
1213 if (null != comment)
1214 comments = comment.toString();
1215 return date + " " + comments;
1216 } catch (IOException e) {
1217 throw new ProCoreException("Failed to interpret metadata.", e);
1220 private Line getRow(int index, Line line) throws ProCoreException {
1221 if (index < 0 || index >= offsets.size())
1222 return null; // Index out of range.
1223 if (index < firstLine || index >= firstLine + lines.length) {
1225 if (index < firstLine)
1226 return null; // Index out of range.
1228 int offset = index - firstLine;
1229 line.status = lines[offset].status;
1230 line.request = lines[offset].request;
1231 line.comment = lines[offset].comment;
1234 private void readLines(int first) throws ProCoreException {
1237 for (int i=0, index = first; i<lines.length; ++i, ++index)
1238 readLine(index, lines[i]);
1244 private void readLine(int index, Line line) throws ProCoreException {
1245 if (index >= offsets.size()) {
1246 line.status = false;
1247 line.request = "<Illegal request.>";
1248 line.comment = "<Illegal request.>";
1251 long offset = offsets.get(index);
1254 int i = file.readInt();
1255 int length = Integer.reverseBytes(i);
1256 byte b = file.readByte();
1257 boolean ok = b != 0;
1258 b = file.readByte();
1259 // boolean littleEndian = b != 0;
1261 int type = Integer.reverseBytes(i);
1262 String comment = "";
1264 throw new ProCoreException("Journal file corrupted at" + file.getFilePointer() + ".");
1265 else if (length > 6) {
1267 file.skipBytes(length - 6);
1268 else if (length > 22){
1271 int size = Integer.reverseBytes(i);
1272 if (size != length - 26)
1273 throw new ProCoreException("Metadata corrupted at" + file.getFilePointer() + ".");
1274 byte[] bytes = new byte[size];
1275 file.readFully(bytes);
1276 comment = getComment(bytes);
1277 if (null == comment)
1278 comment = "<metadata does not contain comment>";
1282 int length2 = Integer.reverseBytes(i);
1283 if (length != length2)
1284 throw new ProCoreException("Journal file corrupted at" + file.getFilePointer() + ".");
1285 String command = MessageText.get(type);
1287 line.request = command;
1288 line.comment = comment;
1289 } catch (IOException e) {
1290 throw new ProCoreException("Journal file corrupted.");
1293 private ArrayList<Long> readOffsets() throws ProCoreException {
1300 long modified = journalFile.lastModified();
1301 if (lastModified != 0 && lastModified == modified)
1302 return offsets; // Offsets already up to date.
1309 int i = file.readInt();
1310 int version = Integer.reverseBytes(i);
1312 throw new ProCoreException("Unsupported journal file version. expected=1 got=" + version);
1314 int major = Integer.reverseBytes(i);
1315 if (major != MessageNumber.ProtocolVersionMajor)
1316 throw new ProCoreException("Unsupported journal request major version. expected=" + MessageNumber.ProtocolVersionMajor + " got=" + major);
1318 int minor = Integer.reverseBytes(i);
1319 if (minor > MessageNumber.ProtocolVersionMinor)
1320 throw new ProCoreException("Unsupported journal request minor version. expected=" + MessageNumber.ProtocolVersionMinor + " got=" + minor);
1322 int length = Integer.reverseBytes(i);
1323 while (length > 0) { // Not supporting unsigned integers.
1325 throw new ProCoreException("Journal file corrupted at" + file.getFilePointer() + ".");
1326 file.skipBytes(length);
1328 int length2 = Integer.reverseBytes(i);
1329 if (length != length2)
1330 throw new ProCoreException("Journal file corrupted at " + file.getFilePointer() + ".");
1331 long offset = file.getFilePointer() - length - 8;
1332 offsets.add(offset);
1334 length = Integer.reverseBytes(i);
1336 } catch (EOFException e) {
1337 } catch (IOException e) {
1338 throw new ProCoreException("Failed to get command count.", e);
1342 lastModified = modified;
1347 private final Analyzed analyzed;
1348 JournalI(File dbFolder) {
1349 this.analyzed = new Analyzed(dbFolder);
1352 public boolean canRead() {
1353 return analyzed.canRead() && !isServerAlive();
1356 public int count() {
1357 return analyzed.count();
1360 public int read(int index, Line line) throws ProCoreException {
1361 int count = analyzed.count();
1362 analyzed.getRow(index, line);
1367 public String getCompression() {