From: Tuukka Lehtonen Date: Fri, 30 Sep 2016 08:06:06 +0000 (+0300) Subject: Merge commit 'bf75fd9' X-Git-Tag: v1.25.0~80 X-Git-Url: https://gerrit.simantics.org/r/gitweb?p=simantics%2Fplatform.git;a=commitdiff_plain;h=fdbe8762;hp=bf75fd9740858140eac90c18f0bca0aea3893248 Merge commit 'bf75fd9' Resolved conflicts: bundles/org.simantics.charts/src/org/simantics/charts/Charts.java refs #6724 refs #6723 refs #6704 refs #6495 --- diff --git a/.gitignore b/.gitignore index 0d0631b56..d61db6ef2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,6 @@ /**/bin/ +/features/*/target/ +/bundles/*/target/ +/releng/**/target/ +/tests/*/target/ /**/.polyglot.build.properties diff --git a/bundles/org.apache.batik/.classpath b/bundles/org.apache.batik/.classpath index 4a8d60f3c..0f23270ef 100644 --- a/bundles/org.apache.batik/.classpath +++ b/bundles/org.apache.batik/.classpath @@ -1,5 +1,6 @@ + @@ -11,8 +12,6 @@ - - diff --git a/bundles/org.apache.batik/META-INF/MANIFEST.MF b/bundles/org.apache.batik/META-INF/MANIFEST.MF index aebf12801..22d86a06d 100644 --- a/bundles/org.apache.batik/META-INF/MANIFEST.MF +++ b/bundles/org.apache.batik/META-INF/MANIFEST.MF @@ -3,22 +3,8 @@ Bundle-ManifestVersion: 2 Bundle-Name: Batik Bundle-SymbolicName: org.apache.batik;singleton:=true Bundle-Version: 1.8.0.qualifier -Bundle-Activator: org.apache.batik.Activator -Require-Bundle: org.eclipse.ui, - org.eclipse.core.runtime Bundle-RequiredExecutionEnvironment: JavaSE-1.8 -Bundle-ActivationPolicy: lazy Export-Package: java_cup.runtime, - javax.xml, - javax.xml.datatype, - javax.xml.namespace, - javax.xml.parsers, - javax.xml.transform, - javax.xml.transform.dom, - javax.xml.transform.sax, - javax.xml.transform.stream, - javax.xml.validation, - javax.xml.xpath, org.apache.avalon.framework, org.apache.avalon.framework.activity, org.apache.avalon.framework.configuration, @@ -108,12 +94,17 @@ Export-Package: java_cup.runtime, org.apache.fop, org.apache.fop.accessibility, org.apache.fop.apps, + org.apache.fop.apps.io, + org.apache.fop.complexscripts.bidi, org.apache.fop.complexscripts.fonts, + org.apache.fop.complexscripts.util, + org.apache.fop.events, org.apache.fop.fo, org.apache.fop.fonts, org.apache.fop.fonts.apps, org.apache.fop.fonts.autodetect, org.apache.fop.fonts.base14, + org.apache.fop.fonts.cff, org.apache.fop.fonts.substitute, org.apache.fop.fonts.truetype, org.apache.fop.fonts.type1, @@ -121,13 +112,18 @@ Export-Package: java_cup.runtime, org.apache.fop.pdf, org.apache.fop.pdf.xref, org.apache.fop.render, + org.apache.fop.render.gradient, org.apache.fop.render.intermediate, org.apache.fop.render.pdf, org.apache.fop.render.pdf.extensions, org.apache.fop.render.ps, org.apache.fop.render.ps.extensions, org.apache.fop.render.ps.fonts, + org.apache.fop.render.ps.svg, org.apache.fop.svg, + org.apache.fop.svg.font, + org.apache.fop.svg.text, + org.apache.fop.traits, org.apache.fop.util, org.apache.html.dom, org.apache.regexp, @@ -195,7 +191,6 @@ Export-Package: java_cup.runtime, org.apache.xml.serialize, org.apache.xml.utils, org.apache.xml.utils.res, - org.apache.xmlcommons, org.apache.xmlgraphics.fonts, org.apache.xmlgraphics.image, org.apache.xmlgraphics.image.codec.png, @@ -260,21 +255,11 @@ Export-Package: java_cup.runtime, org.w3c.css.sac, org.w3c.css.sac.helpers, org.w3c.dom, - org.w3c.dom.bootstrap, - org.w3c.dom.css, org.w3c.dom.events, org.w3c.dom.html, org.w3c.dom.ls, - org.w3c.dom.ranges, org.w3c.dom.smil, - org.w3c.dom.stylesheets, - org.w3c.dom.svg, - org.w3c.dom.traversal, - org.w3c.dom.views, - org.w3c.dom.xpath, - org.xml.sax, - org.xml.sax.ext, - org.xml.sax.helpers + org.w3c.dom.svg Bundle-ClassPath: lib/batik-awt-util-1.8.jar, lib/batik-dom-1.8.jar, lib/batik-ext-1.8.jar, @@ -298,6 +283,6 @@ Bundle-ClassPath: lib/batik-awt-util-1.8.jar, lib/js.jar, lib/xalan-2.7.0.jar, lib/xerces_2_5_0.jar, - lib/xml-apis-1.3.04.jar, - lib/xml-apis-ext-1.3.04.jar, - lib/fop-transcoder-allinone-svn-trunk.jar + lib/fop-transcoder-allinone-svn-trunk.jar, + lib/xml-apis-ext-1.3.04.jar +Require-Bundle: javax.xml;bundle-version="1.3.4" diff --git a/bundles/org.apache.batik/build.properties b/bundles/org.apache.batik/build.properties index 91aa3c77a..d435895c2 100644 --- a/bundles/org.apache.batik/build.properties +++ b/bundles/org.apache.batik/build.properties @@ -25,6 +25,5 @@ bin.includes = META-INF/,\ lib/js.jar,\ lib/xalan-2.7.0.jar,\ lib/xerces_2_5_0.jar,\ - lib/xml-apis-1.3.04.jar,\ - lib/xml-apis-ext-1.3.04.jar,\ - lib/fop-transcoder-allinone-svn-trunk.jar + lib/fop-transcoder-allinone-svn-trunk.jar,\ + lib/xml-apis-ext-1.3.04.jar diff --git a/bundles/org.apache.batik/src/.keep b/bundles/org.apache.batik/src/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/bundles/org.apache.batik/src/org/apache/batik/Activator.java b/bundles/org.apache.batik/src/org/apache/batik/Activator.java deleted file mode 100644 index 0023b84fa..000000000 --- a/bundles/org.apache.batik/src/org/apache/batik/Activator.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.apache.batik; - -import org.eclipse.ui.plugin.AbstractUIPlugin; -import org.osgi.framework.BundleContext; - -/** - * The activator class controls the plug-in life cycle - */ -public class Activator extends AbstractUIPlugin { - - // The plug-in ID - public static final String PLUGIN_ID = "org.apache.batik"; //$NON-NLS-1$ - - // The shared instance - private static Activator plugin; - - /** - * The constructor - */ - public Activator() { - } - - /* - * (non-Javadoc) - * @see org.eclipse.ui.plugin.AbstractUIPlugin#start(org.osgi.framework.BundleContext) - */ - public void start(BundleContext context) throws Exception { - super.start(context); - plugin = this; - } - - /* - * (non-Javadoc) - * @see org.eclipse.ui.plugin.AbstractUIPlugin#stop(org.osgi.framework.BundleContext) - */ - public void stop(BundleContext context) throws Exception { - plugin = null; - super.stop(context); - } - - /** - * Returns the shared instance - * - * @return the shared instance - */ - public static Activator getDefault() { - return plugin; - } - -} diff --git a/bundles/org.simantics.acorn/.classpath b/bundles/org.simantics.acorn/.classpath new file mode 100644 index 000000000..22f30643c --- /dev/null +++ b/bundles/org.simantics.acorn/.classpath @@ -0,0 +1,7 @@ + + + + + + + diff --git a/bundles/org.simantics.acorn/.project b/bundles/org.simantics.acorn/.project new file mode 100644 index 000000000..9726c0b94 --- /dev/null +++ b/bundles/org.simantics.acorn/.project @@ -0,0 +1,33 @@ + + + org.simantics.acorn + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.pde.ManifestBuilder + + + + + org.eclipse.pde.SchemaBuilder + + + + + org.eclipse.pde.ds.core.builder + + + + + + org.eclipse.pde.PluginNature + org.eclipse.jdt.core.javanature + + diff --git a/bundles/org.simantics.acorn/.settings/org.eclipse.jdt.core.prefs b/bundles/org.simantics.acorn/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 000000000..0c68a61dc --- /dev/null +++ b/bundles/org.simantics.acorn/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,7 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.problem.assertIdentifier=error +org.eclipse.jdt.core.compiler.problem.enumIdentifier=error +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/bundles/org.simantics.acorn/.svn/wc.db b/bundles/org.simantics.acorn/.svn/wc.db new file mode 100644 index 000000000..9defa9058 Binary files /dev/null and b/bundles/org.simantics.acorn/.svn/wc.db differ diff --git a/bundles/org.simantics.acorn/META-INF/MANIFEST.MF b/bundles/org.simantics.acorn/META-INF/MANIFEST.MF new file mode 100644 index 000000000..9152acafd --- /dev/null +++ b/bundles/org.simantics.acorn/META-INF/MANIFEST.MF @@ -0,0 +1,18 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: Acorn Database for Simantics +Bundle-SymbolicName: org.simantics.acorn +Bundle-Version: 1.1.2.qualifier +Bundle-Vendor: Semantum Oy +Require-Bundle: gnu.trove3;bundle-version="3.0.0", + gnu.trove2;bundle-version="2.0.4", + org.simantics.db.impl;bundle-version="0.8.0", + org.simantics.db.server;bundle-version="1.0.0", + org.simantics.compressions;bundle-version="1.0.0", + org.simantics.backup, + org.eclipse.core.runtime;bundle-version="3.11.1", + org.simantics.db.procore +Bundle-RequiredExecutionEnvironment: JavaSE-1.8 +Bundle-ActivationPolicy: lazy +Bundle-Activator: org.simantics.acorn.internal.Activator +Service-Component: OSGI-INF/component.xml, OSGI-INF/org.simantics.acorn.AcornDriver.xml diff --git a/bundles/org.simantics.acorn/OSGI-INF/component.xml b/bundles/org.simantics.acorn/OSGI-INF/component.xml new file mode 100644 index 000000000..5b88ac3c0 --- /dev/null +++ b/bundles/org.simantics.acorn/OSGI-INF/component.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/bundles/org.simantics.acorn/OSGI-INF/org.simantics.acorn.AcornDriver.xml b/bundles/org.simantics.acorn/OSGI-INF/org.simantics.acorn.AcornDriver.xml new file mode 100644 index 000000000..f1a97d175 --- /dev/null +++ b/bundles/org.simantics.acorn/OSGI-INF/org.simantics.acorn.AcornDriver.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/bundles/org.simantics.acorn/build.properties b/bundles/org.simantics.acorn/build.properties new file mode 100644 index 000000000..40374cc74 --- /dev/null +++ b/bundles/org.simantics.acorn/build.properties @@ -0,0 +1,17 @@ +############################################################################### +# Copyright (c) 2007, 2010 Association for Decentralized Information Management +# in Industry THTH ry. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +# +# Contributors: +# VTT Technical Research Centre of Finland - initial API and implementation +############################################################################### +output.. = bin/ +bin.includes = META-INF/,\ + .,\ + log4j.properties,\ + OSGI-INF/ +source.. = src/ diff --git a/bundles/org.simantics.acorn/log4j.properties b/bundles/org.simantics.acorn/log4j.properties new file mode 100644 index 000000000..6fecb6d25 --- /dev/null +++ b/bundles/org.simantics.acorn/log4j.properties @@ -0,0 +1,63 @@ +############################################################################### +# Copyright (c) 2007, 2010 Association for Decentralized Information Management +# in Industry THTH ry. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +# +# Contributors: +# VTT Technical Research Centre of Finland - initial API and implementation +############################################################################### +# For the general syntax of property based configuration files see the +# documentation of org.apache.log4j.PropertyConfigurator. + +# The root category uses the appender called rolling. If no priority is +# specified, the root category assumes the default priority for root +# which is DEBUG in log4j. The root category is the only category that +# has a default priority. All other categories need not be assigned a +# priority in which case they inherit their priority from the +# hierarchy. + +# This will provide console output on log4j configuration loading +#log4j.debug=true + +log4j.rootCategory=warn, stdout +#log4j.rootCategory=warn + +# BEGIN APPENDER: CONSOLE APPENDER (stdout) +# first: type of appender (fully qualified class name) +log4j.appender.stdout=org.apache.log4j.ConsoleAppender + +# second: Any configuration information needed for that appender. +# Many appenders require a layout. +log4j.appender.stdout.layout=org.apache.log4j.TTCCLayout +# log4j.appender.stdout.layout=org.apache.log4j.SimpleLayout + +# Possible information overload? +# log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +# additionally, some layouts can take additional information -- +# like the ConversionPattern for the PatternLayout. +# log4j.appender.stdout.layout.ConversionPattern=%d %-5p %-17c{2} (%30F:%L) %3x - %m%n +# END APPENDER: CONSOLE APPENDER (stdout) + +# BEGIN APPENDER: ROLLING FILE APPENDER (rolling) +#log4j.appender.rolling=com.tools.logging.PluginFileAppender +#log4j.appender.rolling=org.apache.log4j.FileAppender +log4j.appender.rolling=org.apache.log4j.RollingFileAppender +log4j.appender.rolling.File=procore.log +log4j.appender.rolling.append=true +log4j.appender.rolling.MaxFileSize=8192KB +# Keep one backup file +log4j.appender.rolling.MaxBackupIndex=1 +log4j.appender.rolling.layout=org.apache.log4j.PatternLayout +#log4j.appender.rolling.layout.ConversionPattern=%p %t %c - %m%n +log4j.appender.rolling.layout.ConversionPattern=%-6r [%15.15t] %-5p %30.30c - %m%n +# END APPENDER: ROLLING FILE APPENDER (rolling) + +# BEGIN APPENDER: PLUG-IN LOG APPENDER (plugin) +log4j.appender.plugin=com.tools.logging.PluginLogAppender +log4j.appender.plugin.layout=org.apache.log4j.PatternLayout +#log4j.appender.plugin.layout.ConversionPattern=%p %t %c - %m%n +log4j.appender.plugin.layout.ConversionPattern=%-6r [%15.15t] %-5p %30.30c - %m%n +# END APPENDER: PLUG-IN LOG APPENDER (plugin) diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/AcornDatabaseManager.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/AcornDatabaseManager.java new file mode 100644 index 000000000..db2c16763 --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/AcornDatabaseManager.java @@ -0,0 +1,40 @@ +package org.simantics.acorn; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; + +import org.simantics.acorn.internal.AcornDatabase; +import org.simantics.db.Database; +import org.simantics.db.server.ProCoreException; + +/** + * @author Tuukka Lehtonen + */ +public class AcornDatabaseManager { + + private static Map dbs = new HashMap(); + + public static synchronized Database getDatabase(Path folder) throws ProCoreException { + Path canonical; + try { + if (!Files.exists(folder)) + Files.createDirectories(folder); + canonical = folder.toRealPath(); + } catch (IOException e) { + throw new ProCoreException("Could not get canonical path.", e); + } + + String canonicalPath = canonical.toString(); + Database db = dbs.get(canonicalPath); + if (null != db) + return db; + + db = new AcornDatabase(canonical); + dbs.put(canonicalPath, db); + return db; + } + +} \ No newline at end of file diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/AcornDriver.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/AcornDriver.java new file mode 100644 index 000000000..536c35c74 --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/AcornDriver.java @@ -0,0 +1,131 @@ +package org.simantics.acorn; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import org.simantics.db.Database; +import org.simantics.db.DatabaseUserAgent; +import org.simantics.db.Driver; +import org.simantics.db.ServerI; +import org.simantics.db.ServerReference; +import org.simantics.db.Session; +import org.simantics.db.SessionReference; +import org.simantics.db.exception.DatabaseException; + +public class AcornDriver implements Driver { + + public static final String AcornDriverName = "acorn"; + + private Map servers = new HashMap<>(); + private Map managements = new HashMap<>(); + + @Override + public String getName() { + return AcornDriverName; + } + + @Override + public DatabaseUserAgent getDatabaseUserAgent(String address) throws DatabaseException { + return AcornDatabaseManager.getDatabase(Paths.get(address)).getUserAgent(); + } + + @Override + public void setDatabaseUserAgent(String address, DatabaseUserAgent dbUserAgent) throws DatabaseException { + AcornDatabaseManager.getDatabase(Paths.get(address)).setUserAgent(dbUserAgent); + } + + @Override + public Session getSession(String address, Properties properties) throws DatabaseException { + Path dbFolder = Paths.get(address); + Session session = AcornSessionManagerImpl.getInstance().createSession(new SessionReference() { + + @Override + public ServerReference getServerReference() { + return new ServerReference() { + + @Override + public Path getDBFolder() { + return dbFolder; + } + }; + } + + @Override + public long getSessionId() { + return 0L; + } + }, null); + if (!properties.containsKey("clientId")) + properties.put("clientId", dbFolder.toAbsolutePath().toString()); + session.registerService(Properties.class, properties); + Session s = session.peekService(Session.class); + if (null == s) + session.registerService(Session.class, session); + return session; + } + + @Override + public ServerI getServer(String address, Properties properties) throws DatabaseException { + ServerI server = servers.get(address); + if (server == null) { + server = new AcornServerI(AcornDatabaseManager.getDatabase(Paths.get(address)), address); + servers.put(address, server); + } + return server; + } + + @Override + public Management getManagement(String address, Properties properties) throws DatabaseException { + Management mgmt = managements.get(address); + if (mgmt == null) { + mgmt = new AcornManagement(AcornDatabaseManager.getDatabase(Paths.get(address)), properties); + managements.put(address, mgmt); + } + return mgmt; + } + + private static class AcornServerI implements ServerI { + + private Database database; + private String address; + + public AcornServerI(Database db, String address) { + this.database = db; + this.address = address; + } + + @Override + public void stop() throws DatabaseException { + database.tryToStop(); + } + + @Override + public void start() throws DatabaseException { + database.start(); + } + + @Override + public boolean isActive() throws DatabaseException { + return database.isRunning(); + } + + @Override + public String getAddress() throws DatabaseException { + return address; + } + + @Override + public String executeAndDisconnect(String command) throws DatabaseException { + return ""; + } + + @Override + public String execute(String command) throws DatabaseException { + return ""; + } + } + +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/AcornManagement.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/AcornManagement.java new file mode 100644 index 000000000..c21491210 --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/AcornManagement.java @@ -0,0 +1,51 @@ +package org.simantics.acorn; + +import java.nio.file.Path; +import java.util.Properties; + +import org.simantics.db.Database; +import org.simantics.db.Driver.Management; +import org.simantics.db.exception.DatabaseException; +import org.simantics.db.server.ProCoreException; + +public class AcornManagement implements Management { + + private final Database db; + private final Properties properties; + + AcornManagement(Database db, Properties properties) throws ProCoreException { + this.db = db; + this.properties = properties; + } + + @Override + public boolean exist() throws DatabaseException { + return db.isFolderOk(); + } + + @Override + public void delete() throws DatabaseException { + db.deleteFiles(); + if (exist()) + throw new DatabaseException("Failed to delete database. folder=" + db.getFolder()); + } + + @Override + public void create() throws DatabaseException { + db.initFolder(properties); + if (!exist()) + throw new DatabaseException("Failed to create Acorn database. folder=" + db.getFolder()); + } + + @Override + public void purge() throws DatabaseException { + db.purgeDatabase(); + } + + @Override + public void shutdown() throws DatabaseException { + db.tryToStop(); + db.disconnect(); + } + +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/AcornSessionManagerImpl.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/AcornSessionManagerImpl.java new file mode 100644 index 000000000..f67a4aa7c --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/AcornSessionManagerImpl.java @@ -0,0 +1,132 @@ +package org.simantics.acorn; + +import java.nio.file.Path; +import java.util.concurrent.ConcurrentHashMap; + +import org.simantics.db.Database; +import org.simantics.db.Session; +import org.simantics.db.SessionErrorHandler; +import org.simantics.db.SessionManager; +import org.simantics.db.SessionReference; +import org.simantics.db.authentication.UserAuthenticationAgent; +import org.simantics.db.common.utils.Logger; +import org.simantics.db.event.SessionEvent; +import org.simantics.db.event.SessionListener; +import org.simantics.db.exception.DatabaseException; +import org.simantics.db.exception.RuntimeDatabaseException; +import org.simantics.db.service.LifecycleSupport; +import org.simantics.utils.datastructures.ListenerList; + +import fi.vtt.simantics.procore.internal.SessionImplDb; +import fi.vtt.simantics.procore.internal.SessionImplSocket; + +public class AcornSessionManagerImpl implements SessionManager { + + private static AcornSessionManagerImpl INSTANCE; + + private ConcurrentHashMap sessionMap = new ConcurrentHashMap<>(); + private ListenerList sessionListeners = new ListenerList<>(SessionListener.class); + private SessionErrorHandler errorHandler; + + private Database database; + + private AcornSessionManagerImpl() {} + + void finish() { + sessionMap = null; + sessionListeners = null; + } + + @Override + public void addSessionListener(SessionListener listener) { + sessionListeners.add(listener); + } + + @Override + public Session createSession(SessionReference sessionReference, UserAuthenticationAgent authAgent) + throws DatabaseException { + SessionImplDb sessionImpl = new SessionImplDb(this, authAgent); + boolean ok = false; + try { + Path dbFolder = sessionReference.getServerReference().getDBFolder(); + database = AcornDatabaseManager.getDatabase(dbFolder); + Database.Session dbSession = database.newSession(sessionImpl); + sessionImpl.connect(sessionReference, dbSession); + sessionMap.put(sessionImpl, dbSession); + fireSessionOpened(sessionImpl); + ok = true; + } catch (Throwable e) { + Logger.defaultLogError("Connection failed. See exception for details.", e); + try { + fireSessionClosed(sessionImpl, e); + sessionMap.remove(sessionImpl); + sessionImpl = null; + } catch (Throwable t) { + } + throw new DatabaseException(e); + } finally { + if (!ok && null != sessionImpl) + sessionImpl.getService(LifecycleSupport.class).close(); + } + return sessionImpl; + } + + @Override + public void removeSessionListener(SessionListener listener) { + sessionListeners.remove(listener); + } + + private void fireSessionOpened(SessionImplSocket session) { + SessionEvent se = new SessionEvent(session, null); + for (SessionListener listener : sessionListeners.getListeners()) { + listener.sessionOpened(se); + } + } + + private void fireSessionClosed(SessionImplSocket session, Throwable cause) { + SessionEvent se = new SessionEvent(session, cause); + for (SessionListener listener : sessionListeners.getListeners()) { + listener.sessionClosed(se); + } + } + + @Override + public void shutdown(Session s, Throwable cause) { + SessionImplSocket sis = (SessionImplSocket) s; + if (null == sis) + return; + try { + fireSessionClosed(sis, cause); + } finally { + sessionMap.remove(s); + } + } + + @Override + public SessionErrorHandler getErrorHandler() { + return errorHandler; + } + + @Override + public void setErrorHandler(SessionErrorHandler errorHandler) { + this.errorHandler = errorHandler; + } + + public synchronized static AcornSessionManagerImpl getInstance() { + if (INSTANCE == null) + INSTANCE = new AcornSessionManagerImpl(); + return INSTANCE; + } + + @Override + public Database getDatabase() { + return database; + } + + public GraphClientImpl2 getClient() { + if (sessionMap.values().size() > 1) + throw new RuntimeDatabaseException("Currently only one GraphClientImpl2 per session is supported!"); + org.simantics.db.Database.Session client = sessionMap.values().iterator().next(); + return (GraphClientImpl2) client; + } +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/ClusterManager.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/ClusterManager.java new file mode 100644 index 000000000..51db52efc --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/ClusterManager.java @@ -0,0 +1,582 @@ +package org.simantics.acorn; + +import java.io.IOException; +import java.math.BigInteger; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.simantics.acorn.cluster.ClusterImpl; +import org.simantics.acorn.exception.AcornAccessVerificationException; +import org.simantics.acorn.exception.IllegalAcornStateException; +import org.simantics.acorn.exception.InvalidHeadStateException; +import org.simantics.acorn.internal.ClusterSupport2; +import org.simantics.acorn.lru.ChangeSetInfo; +import org.simantics.acorn.lru.ClusterInfo; +import org.simantics.acorn.lru.ClusterLRU; +import org.simantics.acorn.lru.ClusterStreamChunk; +import org.simantics.acorn.lru.FileInfo; +import org.simantics.acorn.lru.LRU; +import org.simantics.db.ClusterCreator; +import org.simantics.db.Database.Session.ClusterIds; +import org.simantics.db.Database.Session.ResourceSegment; +import org.simantics.db.ServiceLocator; +import org.simantics.db.exception.DatabaseException; +import org.simantics.db.impl.ClusterBase; +import org.simantics.db.impl.ClusterI; +import org.simantics.db.impl.ClusterSupport; +import org.simantics.db.procore.cluster.ClusterTraits; +import org.simantics.db.service.ClusterSetsSupport; +import org.simantics.db.service.ClusterUID; +import org.simantics.utils.threads.logger.ITask; +import org.simantics.utils.threads.logger.ThreadLogger; + +public class ClusterManager { + + private ArrayList currentChanges = new ArrayList(); + + public final Path dbFolder; + public Path lastSessionDirectory; + public Path workingDirectory; + + public LRU streamLRU; + public LRU csLRU; + public ClusterLRU clusterLRU; + public LRU fileLRU; + + public MainState mainState; + public HeadState state; + + private long lastSnapshot = System.nanoTime(); + + final public ClusterSupport2 support = new ClusterSupport2(this); + + /* + * Public interface + * + */ + + public ClusterManager(Path dbFolder) { + this.dbFolder = dbFolder; + } + + public ArrayList getChanges(long changeSetId) throws AcornAccessVerificationException, IllegalAcornStateException { + ChangeSetInfo info = csLRU.getWithoutMutex(changeSetId); + info.acquireMutex(); + try { + info.makeResident(); + return info.getCSSIds(); + } finally { + info.releaseMutex(); + } + } + + public ClusterBase getClusterByClusterKey(int clusterKey) throws DatabaseException, AcornAccessVerificationException, IllegalAcornStateException { + return clusterLRU.getClusterByClusterKey(clusterKey); + } + + public ClusterBase getClusterByClusterUIDOrMake(ClusterUID clusterUID) throws DatabaseException, AcornAccessVerificationException, IllegalAcornStateException { + return clusterLRU.getClusterByClusterUIDOrMake(clusterUID); + } + + public ClusterImpl getClusterByClusterUIDOrMakeProxy(ClusterUID clusterUID) throws DatabaseException, AcornAccessVerificationException, IllegalAcornStateException { + return clusterLRU.getClusterByClusterUIDOrMakeProxy(clusterUID); + } + + public int getClusterKeyByClusterUIDOrMake(ClusterUID clusterUID) throws AcornAccessVerificationException { + return clusterLRU.getClusterKeyByClusterUIDOrMake(clusterUID); + } + + public int getClusterKeyByClusterUIDOrMakeWithoutMutex(ClusterUID clusterUID) throws IllegalAcornStateException, AcornAccessVerificationException { + return clusterLRU.getClusterKeyByClusterUIDOrMakeWithoutMutex(clusterUID); + } + + public int getClusterKeyByUID(long id1, long id2) throws DatabaseException, IllegalAcornStateException { + return clusterLRU.getClusterKeyByUIDWithoutMutex(id1, id2); + } + + public T getClusterProxyByResourceKey(int resourceKey) throws DatabaseException, AcornAccessVerificationException, IllegalAcornStateException { + return clusterLRU.getClusterProxyByResourceKey(resourceKey); + } + + public ClusterUID getClusterUIDByResourceKey(int resourceKey) throws DatabaseException, AcornAccessVerificationException { + return clusterLRU.getClusterUIDByResourceKey(resourceKey); + } + + public ClusterUID getClusterUIDByResourceKeyWithoutMutex(int resourceKey) throws DatabaseException, IllegalAcornStateException, AcornAccessVerificationException { + return clusterLRU.getClusterUIDByResourceKeyWithoutMutex(resourceKey); + } + + /* + * Private implementation + * + */ + + private static long countFiles(Path directory) throws IOException { + try (DirectoryStream ds = Files.newDirectoryStream(directory)) { + int count = 0; + for (@SuppressWarnings("unused") Path p : ds) + ++count; + return count; + } + } + + // Add check to make sure if it safe to make snapshot (used with cancel which is not yet supported and may cause corrupted head.state writing) + private AtomicBoolean safeToMakeSnapshot = new AtomicBoolean(true); + private IllegalAcornStateException cause; + + public synchronized boolean makeSnapshot(ServiceLocator locator, boolean fullSave) throws IllegalAcornStateException { + try { + if (!safeToMakeSnapshot.get()) + throw cause; + // Maximum autosave frequency is per 60s + if(!fullSave && System.nanoTime() - lastSnapshot < 10*1000000000L) { + // System.err.println("lastSnapshot too early"); + return false; + } + + // Cluster files are always there + // Nothing has been written => no need to do anything + long amountOfFiles = countFiles(workingDirectory); + if(!fullSave && amountOfFiles < 3) { + // System.err.println("amountOfFiles < 3"); + return false; + } + + System.err.println("makeSnapshot"); + + // Schedule writing of all data to disk + refreshHeadState(); + + // Wait for all files to be written + clusterLRU.shutdown(); + fileLRU.shutdown(); + streamLRU.shutdown(); + csLRU.shutdown(); + + // Lets check if it is still safe to make a snapshot + if (!safeToMakeSnapshot.get()) + throw cause; + + persistHeadState(); + + if (fullSave) + mainState.save(dbFolder); + + ClusterSetsSupport cssi = locator.getService(ClusterSetsSupport.class); + cssi.save(); + + amountOfFiles = countFiles(workingDirectory); + + System.err.println(" -finished: amount of files is " + amountOfFiles); + + workingDirectory = dbFolder.resolve(Integer.toString(mainState.headDir)); + if (!Files.exists(workingDirectory)) { + Files.createDirectories(workingDirectory); + } + + cssi.updateWriteDirectory(workingDirectory); + + clusterLRU.setWriteDir(workingDirectory); + fileLRU.setWriteDir(workingDirectory); + streamLRU.setWriteDir(workingDirectory); + csLRU.setWriteDir(workingDirectory); + + clusterLRU.resume(); + fileLRU.resume(); + streamLRU.resume(); + csLRU.resume(); + + lastSnapshot = System.nanoTime(); + + return true; + } catch (IllegalAcornStateException e) { + notSafeToMakeSnapshot(e); + throw e; + } catch (IOException e) { + IllegalAcornStateException e1 = new IllegalAcornStateException(e); + notSafeToMakeSnapshot(e1); + throw e1; + } + } + + private void refreshHeadState() throws IOException, IllegalAcornStateException { + state.clusters.clear(); + state.files.clear(); + state.stream.clear(); + state.cs.clear(); + + clusterLRU.persist(state.clusters); + fileLRU.persist(state.files); + streamLRU.persist(state.stream); + csLRU.persist(state.cs); + } + + private void persistHeadState() throws IOException { + // Sync current working directory + Files.walk(workingDirectory, 1).filter(Files::isRegularFile).forEach(FileIO::uncheckedSyncPath); + state.save(workingDirectory); + mainState.headDir++; + } + + +// public void save() throws IOException { +// +// refreshHeadState(); +// +// clusterLRU.shutdown(); +// fileLRU.shutdown(); +// streamLRU.shutdown(); +// csLRU.shutdown(); +// +// persistHeadState(); +// +// mainState.save(getBaseDirectory()); + +// try { +// ThreadLogVisualizer visualizer = new ThreadLogVisualizer(); +// visualizer.read(new DataInputStream(new FileInputStream( +// ThreadLogger.LOG_FILE))); +// visualizer.visualize3(new PrintStream(ThreadLogger.LOG_FILE +// + ".svg")); +// } catch (FileNotFoundException e) { +// // TODO Auto-generated catch block +// e.printStackTrace(); +// } + + // System.err.println("-- load statistics --"); + // for(Pair entry : + // CollectionUtils.valueSortedEntries(histogram)) { + // System.err.println(" " + entry.second + " " + entry.first); + // } + +// } + + private void acquireAll() throws IllegalAcornStateException { + clusterLRU.acquireMutex(); + fileLRU.acquireMutex(); + streamLRU.acquireMutex(); + csLRU.acquireMutex(); + } + + private void releaseAll() { + csLRU.releaseMutex(); + streamLRU.releaseMutex(); + fileLRU.releaseMutex(); + clusterLRU.releaseMutex(); + } + + private AtomicBoolean rollback = new AtomicBoolean(false); + + boolean rolledback() { + return rollback.get(); + } + + public void load() throws IOException { + + // Main state + mainState = MainState.load(dbFolder, t -> rollback.set(true)); + + lastSessionDirectory = dbFolder.resolve(Integer.toString(mainState.headDir - 1)); + + // Head State + try { + state = HeadState.load(lastSessionDirectory); + } catch (InvalidHeadStateException e) { + // For backwards compatibility only! + Throwable cause = e.getCause(); + if (cause instanceof Throwable) { + try { + org.simantics.db.javacore.HeadState oldState = org.simantics.db.javacore.HeadState.load(lastSessionDirectory); + + HeadState newState = new HeadState(); + newState.clusters = oldState.clusters; + newState.cs = oldState.cs; + newState.files = oldState.files; + newState.stream = oldState.stream; + newState.headChangeSetId = oldState.headChangeSetId; + newState.reservedIds = oldState.reservedIds; + newState.transactionId = oldState.transactionId; + state = newState; + } catch (InvalidHeadStateException e1) { + throw new IOException("Could not load HeadState due to corruption", e1); + } + } else { + // This should never happen as MainState.load() checks the integrity + // of head.state files and rolls back in cases of corruption until a + // consistent state is found (could be case 0 - initial db state) + // IF this does happen something is completely wrong + throw new IOException("Could not load HeadState due to corruption", e); + } + } + try { + workingDirectory = dbFolder.resolve(Integer.toString(mainState.headDir)); + Files.createDirectories(workingDirectory); + + csLRU = new LRU(this, "Change Set", workingDirectory); + streamLRU = new LRU(this, "Cluster Stream", workingDirectory); + clusterLRU = new ClusterLRU(this, "Cluster", workingDirectory); + fileLRU = new LRU(this, "External Value", workingDirectory); + + acquireAll(); + + // Clusters + for (String clusterKey : state.clusters) { + String[] parts1 = clusterKey.split("#"); + String[] parts = parts1[0].split("\\."); + long first = new BigInteger(parts[0], 16).longValue(); + long second = new BigInteger(parts[1], 16).longValue(); + ClusterUID uuid = ClusterUID.make(first, second); + Path readDir = dbFolder.resolve(parts1[1]); + int offset = Integer.parseInt(parts1[2]); + int length = Integer.parseInt(parts1[3]); + clusterLRU.map(new ClusterInfo(this, clusterLRU, readDir, uuid, offset, length)); + } + // Files + for (String fileKey : state.files) { + // System.err.println("loadFile: " + fileKey); + String[] parts = fileKey.split("#"); + Path readDir = dbFolder.resolve(parts[1]); + int offset = Integer.parseInt(parts[2]); + int length = Integer.parseInt(parts[3]); + FileInfo info = new FileInfo(fileLRU, readDir, parts[0], offset, length); + fileLRU.map(info); + } + // Update chunks + for (String fileKey : state.stream) { + // System.err.println("loadStream: " + fileKey); + String[] parts = fileKey.split("#"); + Path readDir = dbFolder.resolve(parts[1]); + int offset = Integer.parseInt(parts[2]); + int length = Integer.parseInt(parts[3]); + ClusterStreamChunk info = new ClusterStreamChunk(this, + streamLRU, readDir, parts[0], offset, length); + streamLRU.map(info); + } + // Change sets + for (String fileKey : state.cs) { + String[] parts = fileKey.split("#"); + Path readDir = dbFolder.resolve(parts[1]); + Long revisionId = Long.parseLong(parts[0]); + int offset = Integer.parseInt(parts[2]); + int length = Integer.parseInt(parts[3]); + ChangeSetInfo info = new ChangeSetInfo(csLRU, readDir, revisionId, offset, length); + csLRU.map(info); + } + + releaseAll(); + } catch (IllegalAcornStateException | AcornAccessVerificationException e) { + // ROLLBACK ONE DIR UNTIL WE ARE FINE! + throw new IOException(e); + } + } + + public T clone(ClusterUID uid, ClusterCreator creator) throws DatabaseException, AcornAccessVerificationException, IllegalAcornStateException, IOException { + + clusterLRU.ensureUpdates(uid); + + ClusterInfo info = clusterLRU.getWithoutMutex(uid); + return info.clone(uid, creator); + } + + //private int loadCounter = 0; + + public static void startLog(String msg) { + tasks.put(msg, ThreadLogger.getInstance().begin(msg)); + } + + public static void endLog(String msg) { + ITask task = tasks.get(msg); + if (task != null) + task.finish(); + } + + static Map tasks = new HashMap(); + + public void update(ClusterUID uid, ClusterImpl clu) throws AcornAccessVerificationException, IllegalAcornStateException { + ClusterInfo info = clusterLRU.getWithoutMutex(uid); + info.acquireMutex(); + try { + info.update(clu); + } finally { + info.releaseMutex(); + } + } + + public long getClusterIdOrCreate(ClusterUID clusterUID) { + return 1; + } + + public int getResourceKey(ClusterUID uid, int index) throws AcornAccessVerificationException { + return clusterLRU.getResourceKey(uid, index); + } + + public int getResourceKeyWitoutMutex(ClusterUID uid, int index) throws IllegalAcornStateException { + return clusterLRU.getResourceKeyWithoutMutex(uid, index); + } + + public ClusterIds getClusterIds() throws IllegalAcornStateException { + clusterLRU.acquireMutex(); + + try { + Collection infos = clusterLRU.values(); + final int status = infos.size(); + final long[] firsts = new long[status]; + final long[] seconds = new long[status]; + + int index = 0; + for (ClusterInfo info : infos) { + firsts[index] = 0; + seconds[index] = info.getKey().second; + index++; + } + + return new ClusterIds() { + + @Override + public int getStatus() { + return status; + } + + @Override + public long[] getFirst() { + return firsts; + } + + @Override + public long[] getSecond() { + return seconds; + } + + }; + + } catch (Throwable t) { + throw new IllegalAcornStateException(t); + } finally { + clusterLRU.releaseMutex(); + } + } + + public void addIntoCurrentChangeSet(String ccs) throws IllegalAcornStateException { + csLRU.acquireMutex(); + + try { + currentChanges.add(ccs); + } catch (Throwable t) { + throw new IllegalAcornStateException(t); + } finally { + csLRU.releaseMutex(); + } + } + + public void commitChangeSet(long changeSetId, byte[] data) throws IllegalAcornStateException { + csLRU.acquireMutex(); + try { + ArrayList csids = new ArrayList(currentChanges); + currentChanges = new ArrayList(); + new ChangeSetInfo(csLRU, changeSetId, data, csids); + } catch (Throwable t) { + throw new IllegalAcornStateException(t); + } finally { + csLRU.releaseMutex(); + } + } + + public byte[] getMetadata(long changeSetId) throws AcornAccessVerificationException, IllegalAcornStateException { + + ChangeSetInfo info = csLRU.getWithoutMutex(changeSetId); + if (info == null) return null; + info.acquireMutex(); + try { + return info.getMetadataBytes(); + } catch (IllegalAcornStateException | AcornAccessVerificationException e) { + throw e; + } catch (Throwable t) { + throw new IllegalAcornStateException(t); + } finally { + info.releaseMutex(); + } + } + + public byte[] getResourceFile(final byte[] clusterUID, final int resourceIndex) throws AcornAccessVerificationException, IllegalAcornStateException { + + ClusterUID uid = ClusterUID.make(clusterUID, 0); + String key = uid.toString() + "_" + resourceIndex; + FileInfo info = fileLRU.getWithoutMutex(key); + if(info == null) return null; + info.acquireMutex(); + try { + return info.getResourceFile(); + } catch (IllegalAcornStateException | AcornAccessVerificationException e) { + throw e; + } catch (Throwable t) { + throw new IllegalAcornStateException(t); + } finally { + info.releaseMutex(); + } + } + + public ResourceSegment getResourceSegment(final byte[] clusterUID, final int resourceIndex, final long segmentOffset, short segmentSize) throws AcornAccessVerificationException, IllegalAcornStateException { + ClusterUID uid = ClusterUID.make(clusterUID, 0); + + String key = uid.toString() + "_" + resourceIndex; + FileInfo info = fileLRU.getWithoutMutex(key); + if(info == null) return null; + info.acquireMutex(); + try { + return info.getResourceSegment(clusterUID, resourceIndex, segmentOffset, segmentSize); + } catch (Throwable t) { + throw new IllegalAcornStateException(t); + } finally { + info.releaseMutex(); + } + } + + public void modiFileEx(ClusterUID uid, int resourceKey, long offset, long size, byte[] bytes, long pos, ClusterSupport support) throws IllegalAcornStateException { + try { + String key = uid.toString() + "_" + ClusterTraits.getResourceIndexFromResourceKey(resourceKey); + + FileInfo info = null; + fileLRU.acquireMutex(); + try { + info = fileLRU.get(key); + if (info == null) { + info = new FileInfo(fileLRU, key, (int) (offset + size)); + } + } catch (Throwable t) { + throw new IllegalAcornStateException(t); + } finally { + fileLRU.releaseMutex(); + } + + info.acquireMutex(); + try { + info.updateData(bytes, offset, pos, size); + } catch (Throwable t) { + throw new IllegalAcornStateException(t); + } finally { + info.releaseMutex(); + } + } catch (DatabaseException e) { + e.printStackTrace(); + } + } + + public void shutdown() { + clusterLRU.shutdown(); + fileLRU.shutdown(); + streamLRU.shutdown(); + csLRU.shutdown(); + } + + public void notSafeToMakeSnapshot(IllegalAcornStateException t) { + this.safeToMakeSnapshot.compareAndSet(true, false); + this.cause = t; + } + +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/ExternalizableExample.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/ExternalizableExample.java new file mode 100644 index 000000000..8d0bac29f --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/ExternalizableExample.java @@ -0,0 +1,43 @@ +package org.simantics.acorn; + +import java.io.Externalizable; +import java.io.IOException; +import java.io.ObjectInput; +import java.io.ObjectOutput; +import java.io.ObjectOutputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; + +public class ExternalizableExample implements Externalizable { + + public int first; + private long second; + + public ExternalizableExample(int first, long second) { + this.first = first; + this.second = second; + } + + @Override + public void writeExternal(ObjectOutput out) throws IOException { + out.writeInt(first); + out.writeLong(second); + } + + @Override + public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + + } + + + public static void main(String[] args) { + Externalizable test = new ExternalizableExample(123, 3456); + + try (ObjectOutputStream stream = new ObjectOutputStream(Files.newOutputStream(Paths.get("C:/Users/Jani Simomaa/Desktop/test"), StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING))) { + stream.writeObject(test); + } catch (IOException e) { + e.printStackTrace(); + } + } +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/FileIO.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/FileIO.java new file mode 100644 index 000000000..c5480d86e --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/FileIO.java @@ -0,0 +1,144 @@ +package org.simantics.acorn; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.Files; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.FileAttribute; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.simantics.databoard.file.RuntimeIOException; + +public class FileIO { + + private static final FileAttribute[] NO_ATTRIBUTES = new FileAttribute[0]; + + private static final Set CREATE_OPTIONS = new HashSet<>(2); + private static final Set APPEND_OPTIONS = new HashSet<>(1); + + static { + CREATE_OPTIONS.add(StandardOpenOption.WRITE); + CREATE_OPTIONS.add(StandardOpenOption.CREATE); + + APPEND_OPTIONS.add(StandardOpenOption.APPEND); + } + + private Path path; + private int writePosition = 0; + + private FileIO(Path path) { + this.path = path; + } + + private static Map map = new HashMap(); + + public static FileIO get(Path path) { + synchronized(map) { + FileIO existing = map.get(path); + if(existing == null) { + existing = new FileIO(path); + map.put(path, existing); + } + return existing; + } + } + + //private static final boolean TRACE_SWAP = false; + private static final boolean TRACE_PERF = false; + + public synchronized int saveBytes(byte[] bytes, int length, boolean overwrite) throws IOException { + if(overwrite) writePosition = 0; + int result = writePosition; + long start = System.nanoTime(); + Set options = writePosition == 0 ? CREATE_OPTIONS : APPEND_OPTIONS; + + ByteBuffer bb = ByteBuffer.wrap(bytes, 0, length); + try (FileChannel fc = FileChannel.open(path, options, NO_ATTRIBUTES)) { + fc.write(bb); + + writePosition += length; + if(TRACE_PERF) { + long duration = System.nanoTime()-start; + double ds = 1e-9*duration; + System.err.println("Wrote " + bytes.length + " bytes @ " + 1e-6*bytes.length / ds + "MB/s"); + } + return result; + } catch (Throwable t) { + throw new IOException("An error occured file saving bytes for file " + path.toAbsolutePath().toString(), t); + } + } + + public synchronized byte[] readBytes(int offset, int length) throws IOException { + long start = System.nanoTime(); + try (SeekableByteChannel channel = Files.newByteChannel(path)) { + channel.position(offset); + ByteBuffer buf = ByteBuffer.allocate(length); + int read = 0; + while (read < length) { + read += channel.read(buf); + } + byte[] result = buf.array(); + if (result.length != length) + System.err.println("faa"); + if (TRACE_PERF) { + long duration = System.nanoTime() - start; + double ds = 1e-9 * duration; + System.err.println("Read " + result.length + " bytes @ " + 1e-6 * result.length / ds + "MB/s"); + } + return result; + } + } + + public static void syncPath(Path f) throws IOException { + // Does not seem to need 's' according to unit test in Windows + try (RandomAccessFile raf = new RandomAccessFile(f.toFile(), "rw")) { + raf.getFD().sync(); + } + } + + static void uncheckedSyncPath(Path f) { + try { + syncPath(f); + } catch (IOException e) { + throw new RuntimeIOException(e); + } + } + + public static void main(String[] args) throws Exception { + + byte[] buf = new byte[1024*1024]; + + long s = System.nanoTime(); + + Path test = Paths.get("e:/work/test.dat"); + OutputStream fs = Files.newOutputStream(test); + OutputStream os = new BufferedOutputStream(fs, 128*1024); + + for(int i=0;i<40;i++) { + os.write(buf); + } + + os.flush(); + //fs.getFD().sync(); + os.close(); + + syncPath(test); + + long duration = System.nanoTime()-s; + System.err.println("Took " + 1e-6*duration + "ms."); + + + } + +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/GraphClientImpl2.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/GraphClientImpl2.java new file mode 100644 index 000000000..05f9c8de0 --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/GraphClientImpl2.java @@ -0,0 +1,730 @@ +/******************************************************************************* + * Copyright (c) 2007, 2010 Association for Decentralized Information Management + * in Industry THTH ry. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * VTT Technical Research Centre of Finland - initial API and implementation + *******************************************************************************/ +package org.simantics.acorn; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Semaphore; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; + +import org.simantics.acorn.exception.AcornAccessVerificationException; +import org.simantics.acorn.exception.IllegalAcornStateException; +import org.simantics.acorn.internal.ClusterChange; +import org.simantics.acorn.internal.ClusterUpdateProcessorBase; +import org.simantics.acorn.internal.UndoClusterUpdateProcessor; +import org.simantics.acorn.lru.ClusterInfo; +import org.simantics.acorn.lru.ClusterStreamChunk; +import org.simantics.acorn.lru.ClusterUpdateOperation; +import org.simantics.acorn.lru.ClusterChangeSet.Entry; +import org.simantics.db.ClusterCreator; +import org.simantics.db.Database; +import org.simantics.db.ServiceLocator; +import org.simantics.db.common.utils.Logger; +import org.simantics.db.exception.DatabaseException; +import org.simantics.db.exception.SDBException; +import org.simantics.db.server.ProCoreException; +import org.simantics.db.service.ClusterSetsSupport; +import org.simantics.db.service.ClusterUID; +import org.simantics.db.service.LifecycleSupport; +import org.simantics.utils.datastructures.Pair; +import org.simantics.utils.logging.TimeLogger; + +import gnu.trove.map.hash.TLongObjectHashMap; + +public class GraphClientImpl2 implements Database.Session { + + public static final boolean DEBUG = false; + + public final ClusterManager clusters; + + private TransactionManager transactionManager = new TransactionManager(); + private ExecutorService executor = Executors.newSingleThreadExecutor(new ClientThreadFactory("Core Main Program", false)); + private ExecutorService saver = Executors.newSingleThreadExecutor(new ClientThreadFactory("Core Snapshot Saver", true)); + + private Path dbFolder; + private final Database database; + private ServiceLocator locator; + private MainProgram mainProgram; + + static class ClientThreadFactory implements ThreadFactory { + + final String name; + final boolean daemon; + + public ClientThreadFactory(String name, boolean daemon) { + this.name = name; + this.daemon = daemon; + } + + @Override + public Thread newThread(Runnable r) { + Thread thread = new Thread(r, name); + thread.setDaemon(daemon); + return thread; + } + } + + public GraphClientImpl2(Database database, Path dbFolder, ServiceLocator locator) throws IOException { + this.database = database; + this.dbFolder = dbFolder; + this.locator = locator; + this.clusters = new ClusterManager(dbFolder); + load(); + ClusterSetsSupport cssi = locator.getService(ClusterSetsSupport.class); + cssi.setReadDirectory(clusters.lastSessionDirectory); + mainProgram = new MainProgram(this, clusters); + executor.execute(mainProgram); + } + + public Path getDbFolder() { + return dbFolder; + } + + public void tryMakeSnapshot() throws IOException { + + if (isClosing || unexpectedClose) + return; + + saver.execute(new Runnable() { + + @Override + public void run() { + Transaction tr = null; + try { + // First take a write transaction + tr = askWriteTransaction(-1); + // Then make sure that MainProgram is idling + mainProgram.mutex.acquire(); + try { + synchronized(mainProgram) { + if(mainProgram.operations.isEmpty()) { + makeSnapshot(false); + } else { + // MainProgram is becoming busy again - delay snapshotting + return; + } + } + } finally { + mainProgram.mutex.release(); + } + } catch (IllegalAcornStateException | ProCoreException e) { + Logger.defaultLogError(e); + unexpectedClose = true; + } catch (InterruptedException e) { + Logger.defaultLogError(e); + } finally { + try { + if(tr != null) + endTransaction(tr.getTransactionId()); + if (unexpectedClose) { + LifecycleSupport support = getServiceLocator().getService(LifecycleSupport.class); + try { + support.close(); + } catch (DatabaseException e1) { + Logger.defaultLogError(e1); + } + } + } catch (ProCoreException e) { + Logger.defaultLogError(e); + } + } + } + }); + } + + public void makeSnapshot(boolean fullSave) throws IllegalAcornStateException { + clusters.makeSnapshot(locator, fullSave); + } + + public T clone(ClusterUID uid, ClusterCreator creator) throws DatabaseException { + try { + return clusters.clone(uid, creator); + } catch (AcornAccessVerificationException | IllegalAcornStateException | IOException e) { + unexpectedClose = true; + throw new DatabaseException(e); + } + } + +// private void save() throws IOException { +// clusters.save(); +// } + + public void load() throws IOException { + clusters.load(); + } + +// public void modiFileEx(ClusterUID uid, int resourceKey, long offset, long size, byte[] bytes, long pos, ClusterSupport support) { +// clusters.modiFileEx(uid, resourceKey, offset, size, bytes, pos, support); +// } + + @Override + public Database getDatabase() { + return database; + } + + private boolean closed = false; + private boolean isClosing = false; + private boolean unexpectedClose = false; + + @Override + public void close() throws ProCoreException { + System.err.println("Closing " + this + " and mainProgram " + mainProgram); + if(!closed && !isClosing) { + isClosing = true; + try { + if (!unexpectedClose) + makeSnapshot(true); + + mainProgram.close(); + clusters.shutdown(); + executor.shutdown(); + saver.shutdown(); + boolean executorTerminated = executor.awaitTermination(500, TimeUnit.MILLISECONDS); + boolean saverTerminated = saver.awaitTermination(500, TimeUnit.MILLISECONDS); + + System.err.println("executorTerminated=" + executorTerminated + ", saverTerminated=" + saverTerminated); + + mainProgram = null; + executor = null; + saver = null; + + } catch (IllegalAcornStateException | InterruptedException e) { + throw new ProCoreException(e); + } + closed = true; + } + //impl.close(); + } + + @Override + public void open() throws ProCoreException { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isClosed() throws ProCoreException { + return closed; + } + + @Override + public void acceptCommit(long transactionId, long changeSetId, byte[] metadata) throws ProCoreException { + clusters.state.headChangeSetId++; + long committedChangeSetId = changeSetId + 1; + try { + clusters.commitChangeSet(committedChangeSetId, metadata); + + clusters.state.transactionId = transactionId; + + mainProgram.committed(); + + TimeLogger.log("Accepted commit"); + } catch (IllegalAcornStateException e) { + throw new ProCoreException(e); + } + } + + @Override + public long cancelCommit(long transactionId, long changeSetId, byte[] metadata, OnChangeSetUpdate onChangeSetUpdate) throws ProCoreException { + UnsupportedOperationException e = new UnsupportedOperationException("org.simantics.acorn.GraphClientImpl2.cancelCommit() is not supported operation! Closing down to prevent further havoc"); + clusters.notSafeToMakeSnapshot(new IllegalAcornStateException(e)); + throw e; +// System.err.println("GraphClientImpl2.cancelCommit() called!! this is experimental and might cause havoc!"); +// try { +// undo(new long[] {changeSetId}, onChangeSetUpdate); +// } catch (SDBException e) { +// e.printStackTrace(); +// throw new ProCoreException(e); +// } +// clusters.state.headChangeSetId++; +// return clusters.state.headChangeSetId; + } + + @Override + public Transaction askReadTransaction() throws ProCoreException { + return transactionManager.askReadTransaction(); + } + + enum TransactionState { + IDLE,WRITE,READ + } + + class TransactionRequest { + public TransactionState state; + public Semaphore semaphore; + public TransactionRequest(TransactionState state, Semaphore semaphore) { + this.state = state; + this.semaphore = semaphore; + } + } + + class TransactionManager { + + private TransactionState currentTransactionState = TransactionState.IDLE; + + private int reads = 0; + + LinkedList requests = new LinkedList(); + + TLongObjectHashMap requestMap = new TLongObjectHashMap(); + + private synchronized Transaction makeTransaction(TransactionRequest req) { + + final int csId = clusters.state.headChangeSetId; + final long trId = clusters.state.transactionId+1; + requestMap.put(trId, req); + return new Transaction() { + + @Override + public long getTransactionId() { + return trId; + } + + @Override + public long getHeadChangeSetId() { + return csId; + } + }; + } + + /* + * This method cannot be synchronized since it waits and must support multiple entries + * by query thread(s) and internal transactions such as snapshot saver + */ + public Transaction askReadTransaction() throws ProCoreException { + + Semaphore semaphore = new Semaphore(0); + + TransactionRequest req = queue(TransactionState.READ, semaphore); + + try { + semaphore.acquire(); + } catch (InterruptedException e) { + throw new ProCoreException(e); + } + + return makeTransaction(req); + + } + + private synchronized void dispatch() { + TransactionRequest r = requests.removeFirst(); + if(r.state == TransactionState.READ) reads++; + r.semaphore.release(); + } + + private synchronized void processRequests() { + + while(true) { + + if(requests.isEmpty()) return; + TransactionRequest req = requests.peek(); + + if(currentTransactionState == TransactionState.IDLE) { + + // Accept anything while IDLE + currentTransactionState = req.state; + dispatch(); + + } else if (currentTransactionState == TransactionState.READ) { + + if(req.state == currentTransactionState) { + + // Allow other reads + dispatch(); + + } else { + + // Wait + return; + + } + + } else if (currentTransactionState == TransactionState.WRITE) { + + // Wait + return; + + } + + } + + } + + private synchronized TransactionRequest queue(TransactionState state, Semaphore semaphore) { + TransactionRequest req = new TransactionRequest(state, semaphore); + requests.addLast(req); + processRequests(); + return req; + } + + /* + * This method cannot be synchronized since it waits and must support multiple entries + * by query thread(s) and internal transactions such as snapshot saver + */ + public Transaction askWriteTransaction() throws IllegalAcornStateException { + + Semaphore semaphore = new Semaphore(0); + TransactionRequest req = queue(TransactionState.WRITE, semaphore); + + try { + semaphore.acquire(); + } catch (InterruptedException e) { + throw new IllegalAcornStateException(e); + } + mainProgram.startTransaction(clusters.state.headChangeSetId+1); + return makeTransaction(req); + } + + public synchronized long endTransaction(long transactionId) throws ProCoreException { + + TransactionRequest req = requestMap.remove(transactionId); + if(req.state == TransactionState.WRITE) { + currentTransactionState = TransactionState.IDLE; + processRequests(); + } else { + reads--; + if(reads == 0) { + currentTransactionState = TransactionState.IDLE; + processRequests(); + } + } + return clusters.state.transactionId; + } + + } + + @Override + public Transaction askWriteTransaction(final long transactionId) throws ProCoreException { + try { + if (isClosing || unexpectedClose || closed) { + throw new ProCoreException("GraphClientImpl2 is already closing so no more write transactions allowed!"); + } + return transactionManager.askWriteTransaction(); + } catch (IllegalAcornStateException e) { + throw new ProCoreException(e); + } + } + + @Override + public long endTransaction(long transactionId) throws ProCoreException { + return transactionManager.endTransaction(transactionId); + } + + @Override + public String execute(String command) throws ProCoreException { + // This is called only by WriteGraphImpl.commitAccessorChanges + // We can ignore this in Acorn + return ""; + } + + @Override + public byte[] getChangeSetMetadata(long changeSetId) throws ProCoreException { + try { + return clusters.getMetadata(changeSetId); + } catch (AcornAccessVerificationException | IllegalAcornStateException e) { + throw new ProCoreException(e); + } + } + + @Override + public ChangeSetData getChangeSetData(long minChangeSetId, + long maxChangeSetId, OnChangeSetUpdate onChangeSetupate) + throws ProCoreException { + + new Exception("GetChangeSetDataFunction " + minChangeSetId + " " + maxChangeSetId).printStackTrace();; + return null; + + } + + @Override + public ChangeSetIds getChangeSetIds() throws ProCoreException { + throw new UnsupportedOperationException(); + } + + @Override + public Cluster getCluster(byte[] clusterId) throws ProCoreException { + throw new UnsupportedOperationException(); + } + + @Override + public ClusterChanges getClusterChanges(long changeSetId, byte[] clusterId) + throws ProCoreException { + throw new UnsupportedOperationException(); + } + + @Override + public ClusterIds getClusterIds() throws ProCoreException { + try { + return clusters.getClusterIds(); + } catch (IllegalAcornStateException e) { + throw new ProCoreException(e); + } + } + + @Override + public Information getInformation() throws ProCoreException { + return new Information() { + + @Override + public String getServerId() { + return "server"; + } + + @Override + public String getProtocolId() { + return ""; + } + + @Override + public String getDatabaseId() { + return "database"; + } + + @Override + public long getFirstChangeSetId() { + return 0; + } + + }; + } + + @Override + public Refresh getRefresh(long changeSetId) throws ProCoreException { + + final ClusterIds ids = getClusterIds(); + + return new Refresh() { + + @Override + public long getHeadChangeSetId() { + return clusters.state.headChangeSetId; + } + + @Override + public long[] getFirst() { + return ids.getFirst(); + } + + @Override + public long[] getSecond() { + return ids.getSecond(); + } + + }; + + } + + public byte[] getResourceFile(final byte[] clusterUID, final int resourceIndex) throws ProCoreException, AcornAccessVerificationException, IllegalAcornStateException { + return clusters.getResourceFile(clusterUID, resourceIndex); + } + + @Override + public ResourceSegment getResourceSegment(final byte[] clusterUID, final int resourceIndex, final long segmentOffset, short segmentSize) throws ProCoreException { + try { + return clusters.getResourceSegment(clusterUID, resourceIndex, segmentOffset, segmentSize); + } catch (AcornAccessVerificationException | IllegalAcornStateException e) { + throw new ProCoreException(e); + } + } + + @Override + public long reserveIds(int count) throws ProCoreException { + return clusters.state.reservedIds++; + } + + @Override + public void updateCluster(byte[] operations) throws ProCoreException { + ClusterInfo info = null; + try { + ClusterUpdateOperation operation = new ClusterUpdateOperation(clusters, operations); + info = clusters.clusterLRU.getOrCreate(operation.uid, true); + if(info == null) + throw new IllegalAcornStateException("info == null for operation " + operation); + info.acquireMutex(); + info.scheduleUpdate(); + mainProgram.schedule(operation); + } catch (IllegalAcornStateException | AcornAccessVerificationException e) { + throw new ProCoreException(e); + } finally { + if (info != null) + info.releaseMutex(); + } + } + + private UndoClusterUpdateProcessor getUndoCSS(String ccsId) throws DatabaseException, AcornAccessVerificationException, IllegalAcornStateException { + + String[] ss = ccsId.split("\\."); + String chunkKey = ss[0]; + int chunkOffset = Integer.parseInt(ss[1]); + ClusterStreamChunk chunk = clusters.streamLRU.getWithoutMutex(chunkKey); + if(chunk == null) throw new IllegalAcornStateException("Cluster Stream Chunk " + chunkKey + " was not found."); + chunk.acquireMutex(); + try { + return chunk.getUndoProcessor(clusters, chunkOffset, ccsId); + } catch (DatabaseException e) { + throw e; + } catch (Throwable t) { + throw new IllegalStateException(t); + } finally { + chunk.releaseMutex(); + } + } + + private void performUndo(String ccsId, ArrayList> clusterChanges, UndoClusterSupport support) throws ProCoreException, DatabaseException, IllegalAcornStateException, AcornAccessVerificationException { + UndoClusterUpdateProcessor proc = getUndoCSS(ccsId); + + int clusterKey = clusters.getClusterKeyByClusterUIDOrMakeWithoutMutex(proc.getClusterUID()); + + clusters.clusterLRU.acquireMutex(); + try { + + ClusterChange cs = new ClusterChange(clusterChanges, proc.getClusterUID()); + for(int i=0;i> clusterChanges = new ArrayList>(); + + UndoClusterSupport support = new UndoClusterSupport(clusters); + + final int changeSetId = clusters.state.headChangeSetId; + + if(ClusterUpdateProcessorBase.DEBUG) + System.err.println(" === BEGIN UNDO ==="); + + for(int i=0;i ccss = clusters.getChanges(id); + + for(int j=0;j pair = clusterChanges.get(i); + + final ClusterUID cuid = pair.first; + final byte[] data = pair.second; + + onChangeSetUpdate.onChangeSetUpdate(new ChangeSetUpdate() { + + @Override + public long getChangeSetId() { + return changeSetId; + } + + @Override + public int getChangeSetIndex() { + return 0; + } + + @Override + public int getNumberOfClusterChangeSets() { + return clusterChanges.size(); + } + + @Override + public int getIndexOfClusterChangeSet() { + return changeSetIndex; + } + + @Override + public byte[] getClusterId() { + return cuid.asBytes(); + } + + @Override + public boolean getNewCluster() { + return false; + } + + @Override + public byte[] getData() { + return data; + } + + }); + } + } catch (AcornAccessVerificationException | IllegalAcornStateException e1) { + throw new ProCoreException(e1); + } + return false; + } + + public ServiceLocator getServiceLocator() { + return locator; + } + + @Override + public boolean refreshEnabled() { + return false; + } + + @Override + public boolean rolledback() { + return clusters.rolledback(); + } + + + + + + + + + + + + //////////////////////// + + + + + + + + + + + + +} + diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/HeadState.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/HeadState.java new file mode 100644 index 000000000..dd8703c1f --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/HeadState.java @@ -0,0 +1,107 @@ +package org.simantics.acorn; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; + +import org.simantics.acorn.exception.InvalidHeadStateException; +import org.simantics.databoard.Bindings; +import org.simantics.databoard.binding.mutable.MutableVariant; +import org.simantics.databoard.serialization.Serializer; +import org.simantics.databoard.util.binary.BinaryMemory; + +public class HeadState { + + public static final String HEAD_STATE = "head.state"; + public static final String SHA_1 = "SHA-1"; + + public int headChangeSetId = 0; + public long transactionId = 1; + public long reservedIds = 3; + + public ArrayList clusters = new ArrayList<>(); + public ArrayList files = new ArrayList<>(); + public ArrayList stream = new ArrayList<>(); + public ArrayList cs = new ArrayList<>(); +// public ArrayList ccs = new ArrayList(); + + public static HeadState load(Path directory) throws InvalidHeadStateException { + Path f = directory.resolve(HEAD_STATE); + + try { + byte[] bytes = Files.readAllBytes(f); + MessageDigest sha1 = MessageDigest.getInstance(SHA_1); + int digestLength = sha1.getDigestLength(); + + sha1.update(bytes, digestLength, bytes.length - digestLength); + byte[] newChecksum = sha1.digest(); + if (!Arrays.equals(newChecksum, Arrays.copyOfRange(bytes, 0, digestLength))) { + throw new InvalidHeadStateException( + "Checksum " + Arrays.toString(newChecksum) + " does not match excpected " + + Arrays.toString(Arrays.copyOfRange(bytes, 0, digestLength)) + " for " + f.toAbsolutePath()); + } + try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes, digestLength, bytes.length - digestLength)) { + HeadState object = (HeadState) org.simantics.databoard.Files.readFile(bais, Bindings.getBindingUnchecked(HeadState.class)); + return object; + } + } catch (IOException i) { + return new HeadState(); +// throw new InvalidHeadStateException(i); + } catch (NoSuchAlgorithmException e) { + throw new Error("SHA-1 Algorithm not found", e); + } catch (Throwable t) { + throw new InvalidHeadStateException(t); + } + } + + public void save(Path directory) throws IOException { + Path f = directory.resolve(HEAD_STATE); + try { + BinaryMemory rf = new BinaryMemory(4096); + try { + MutableVariant v = new MutableVariant(Bindings.getBindingUnchecked(HeadState.class), this); + Serializer s = Bindings.getSerializerUnchecked( Bindings.VARIANT ); + s.serialize(rf, v); + } finally { + rf.close(); + } + + byte[] bytes = rf.toByteBuffer().array(); + + MessageDigest sha1 = MessageDigest.getInstance(SHA_1); + sha1.update(bytes); + byte[] checksum = sha1.digest(); + + try (OutputStream out = Files.newOutputStream(f)) { + out.write(checksum); + out.write(bytes); + } + FileIO.syncPath(f); + } catch (NoSuchAlgorithmException e) { + throw new Error("SHA-1 digest not found, should not happen", e); + } + } + + public static void validateHeadStateIntegrity(Path headState) throws InvalidHeadStateException, IOException { + try { + byte[] bytes = Files.readAllBytes(headState); + MessageDigest sha1 = MessageDigest.getInstance(SHA_1); + int digestLength = sha1.getDigestLength(); + sha1.update(bytes, digestLength, bytes.length - digestLength); + byte[] newChecksum = sha1.digest(); + if (!Arrays.equals(newChecksum, Arrays.copyOfRange(bytes, 0, digestLength))) { + throw new InvalidHeadStateException( + "Checksum " + Arrays.toString(newChecksum) + " does not match excpected " + + Arrays.toString(Arrays.copyOfRange(bytes, 0, digestLength)) + " for " + headState.toAbsolutePath()); + } + } catch (NoSuchAlgorithmException e) { + throw new Error("SHA-1 digest not found, should not happen", e); + } + } +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/MainProgram.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/MainProgram.java new file mode 100644 index 000000000..78ff9e899 --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/MainProgram.java @@ -0,0 +1,346 @@ +package org.simantics.acorn; + +import java.io.Closeable; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Semaphore; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; + +import org.simantics.acorn.exception.AcornAccessVerificationException; +import org.simantics.acorn.exception.IllegalAcornStateException; +import org.simantics.acorn.lru.ClusterStreamChunk; +import org.simantics.acorn.lru.ClusterUpdateOperation; +import org.simantics.db.service.ClusterUID; +import org.simantics.utils.logging.TimeLogger; + +public class MainProgram implements Runnable, Closeable { + + private static final int CLUSTER_THREADS = 4; + private static final int CHUNK_CACHE_SIZE = 100; + + private final GraphClientImpl2 client; + private final ClusterManager clusters; + private final ExecutorService[] clusterUpdateThreads; + private final List[] updateSchedules; + + private int residentOperationBytes = 0; + private long currentChangeSetId = -1; + private int nextChunkId = 0; + private boolean alive = true; + private Semaphore deathBarrier = new Semaphore(0); + + final Semaphore mutex = new Semaphore(1); + final LinkedList operations = new LinkedList<>(); + + static class ClusterThreadFactory implements ThreadFactory { + + final String name; + final boolean daemon; + + public ClusterThreadFactory(String name, boolean daemon) { + this.name = name; + this.daemon = daemon; + } + + @Override + public Thread newThread(Runnable r) { + Thread thread = new Thread(r, name); + thread.setDaemon(daemon); + return thread; + } + } + + public MainProgram(GraphClientImpl2 client, ClusterManager clusters) { + + this.client = client; + this.clusters = clusters; + this.clusterUpdateThreads = new ExecutorService[CLUSTER_THREADS]; + this.updateSchedules = new ArrayList[CLUSTER_THREADS]; + for(int i=0;i(); + } + } + + public void startTransaction(long id) { + currentChangeSetId = id; + nextChunkId = 0; + } + + private static Comparator clusterComparator = new Comparator() { + + @Override + public int compare(ClusterUID o1, ClusterUID o2) { + return Long.compare(o1.second, o2.second); + } + }; + + @Override + public void run() { + try { + + mutex.acquire(); + main: + while(alive) { + + TreeMap> updates = new TreeMap>(clusterComparator); + + synchronized(MainProgram.this) { + + while(!operations.isEmpty() && updates.size() < 100) { + + ClusterStreamChunk chunk = operations.pollFirst(); + + for(int i=chunk.nextToProcess;i ops = updates.get(uid); + if(ops == null) { + ops = new ArrayList(); + updates.put(uid, ops); + } + ops.add(o); + } + + chunk.nextToProcess = chunk.operations.size(); + + if(!chunk.isCommitted()) { + assert(operations.isEmpty()); + operations.add(chunk); + break; + } + + } + + if(updates.isEmpty()) { + try { + long start = System.nanoTime(); + mutex.release(); + MainProgram.this.wait(5000); + mutex.acquire(); + if (!alive) + break main; + long duration = System.nanoTime()-start; + if(duration > 4000000000L) { + + // Was this a time-out or a new stream request? + if(operations.isEmpty()) { + + /* + * We are idling here. + * Flush all caches gradually + */ + + // Write pending cs to disk + boolean written = clusters.csLRU.swapForced(); + while(written) { + if(!updates.isEmpty()) break; + written = clusters.csLRU.swapForced(); + } + // Write pending chunks to disk + written = clusters.streamLRU.swapForced(); + while(written) { + if(!updates.isEmpty()) break; + written = clusters.streamLRU.swapForced(); + } + // Write pending files to disk + written = clusters.fileLRU.swapForced(); + while(written) { + if(!updates.isEmpty()) break; + written = clusters.fileLRU.swapForced(); + } + // Write pending clusters to disk + written = clusters.clusterLRU.swapForced(); + while(written) { + if(!updates.isEmpty()) break; + written = clusters.clusterLRU.swapForced(); + } + + client.tryMakeSnapshot(); + } + } + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + +// long sss = System.nanoTime(); + + for(int i=0;i> entry : updates.entrySet()) { + ClusterUID key = entry.getKey(); + int hash = key.hashCode() & (clusterUpdateThreads.length-1); + updateSchedules[hash].addAll(entry.getValue()); + } + + // final AtomicLong elapsed = new AtomicLong(0); + int acquireAmount = 0; + for(int i=0;i ops = updateSchedules[i]; + if (!ops.isEmpty()) { + acquireAmount++; + clusterUpdateThreads[i].submit(new Callable() { + + @Override + public Object call() throws Exception { + //long st = System.nanoTime(); + try { + for(ClusterUpdateOperation op : ops) { + op.run(); + } + } finally { + s.release(); + } + return null; + + // long duration = System.nanoTime()-st; + // elapsed.addAndGet(duration); + // double dur = 1e-9*duration; + // if(dur > 0.05) + // System.err.println("duration=" + dur + "s. " + ops.size()); + } + }); + } + } + + s.acquire(acquireAmount); + + /* + * Here we are actively processing updates from client. + * Maintain necessary caching here. + */ + + clusters.streamLRU.acquireMutex(); + try { + swapChunks(); + } catch (AcornAccessVerificationException | IllegalAcornStateException e) { + e.printStackTrace(); + } finally { + clusters.streamLRU.releaseMutex(); + } + clusters.csLRU.acquireMutex(); + try { + swapCS(); + } catch (Throwable t) { + throw new IllegalAcornStateException(t); + } finally { + clusters.csLRU.releaseMutex(); + } + + TimeLogger.log("Performed updates"); + + } + + } catch (Throwable t) { + t.printStackTrace(); + } finally { + deathBarrier.release(); + } + } + + /* + * Mutex for streamLRU is assumed here + * + */ + private void swapChunks() throws AcornAccessVerificationException, IllegalAcornStateException { + + // Cache chunks during update operations + boolean written = clusters.streamLRU.swap(Integer.MAX_VALUE, CHUNK_CACHE_SIZE); + while(written) { + written = clusters.streamLRU.swap(Integer.MAX_VALUE, CHUNK_CACHE_SIZE); + } + } + + private void swapCS() throws AcornAccessVerificationException, IllegalAcornStateException { + + // Cache chunks during update operations + boolean written = clusters.csLRU.swap(Integer.MAX_VALUE, CHUNK_CACHE_SIZE); + while(written) { + written = clusters.csLRU.swap(Integer.MAX_VALUE, CHUNK_CACHE_SIZE); + } + } + + public synchronized void committed() { + + ClusterStreamChunk last = operations.isEmpty() ? null : operations.getLast(); + if (!alive) { + System.err.println("Trying to commit operation after MainProgram is closed! Operation is " + last); +// return; + } + if(last != null) last.commit(); + + } + + public synchronized void schedule(ClusterUpdateOperation operation) throws IllegalAcornStateException { + if (!alive) { + System.err.println("Trying to schedule operation after MainProgram is closed! Operation is " + operation); +// return; + } + clusters.streamLRU.acquireMutex(); + + try { + + ClusterStreamChunk last = operations.isEmpty() ? null : operations.getLast(); + if(last == null || last.isCommitted()) { + String id = "" + currentChangeSetId + "-" + nextChunkId++; + last = new ClusterStreamChunk(clusters, clusters.streamLRU, id); + operations.add(last); + } + + String chunkId = last.getKey(); + int chunkOffset = last.operations.size(); + operation.scheduled(chunkId + "." + chunkOffset); + + last.addOperation(operation); + + swapChunks(); + + notifyAll(); + } catch (IllegalAcornStateException e) { + throw e; + } catch (Throwable t) { + throw new IllegalAcornStateException(t); + } finally { + clusters.streamLRU.releaseMutex(); + } + } + + @Override + public void close() { + alive = false; + synchronized (this) { + notifyAll(); + } + try { + deathBarrier.acquire(); + } catch (InterruptedException e) { + } + + for (ExecutorService executor : clusterUpdateThreads) + executor.shutdown(); + + for (int i = 0; i < clusterUpdateThreads.length; i++) { + try { + ExecutorService executor = clusterUpdateThreads[i]; + executor.awaitTermination(500, TimeUnit.MILLISECONDS); + clusterUpdateThreads[i] = null; + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/MainState.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/MainState.java new file mode 100644 index 000000000..ec8451cca --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/MainState.java @@ -0,0 +1,180 @@ +package org.simantics.acorn; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.Serializable; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.simantics.acorn.exception.InvalidHeadStateException; +import org.simantics.databoard.Bindings; +import org.simantics.databoard.binding.mutable.MutableVariant; +import org.simantics.databoard.file.RuntimeIOException; +import org.simantics.databoard.serialization.Serializer; +import org.simantics.databoard.util.binary.BinaryMemory; +import org.simantics.utils.FileUtils; + +public class MainState implements Serializable { + + private static final long serialVersionUID = 6237383147637270225L; + + public static final String MAIN_STATE = "main.state"; + + public int headDir = 0; + + public MainState() { + } + + private MainState(int headDir) { + this.headDir = headDir; + } + + public static MainState load(Path directory, Consumer callback) throws IOException { + Files.createDirectories(directory); + Path mainState = directory.resolve(MAIN_STATE); + try { + byte[] bytes = Files.readAllBytes(mainState); + MainState state = null; + try (ByteArrayInputStream bais = new ByteArrayInputStream(bytes)) { + state = (MainState) org.simantics.databoard.Files.readFile(bais, Bindings.getBindingUnchecked(MainState.class)); + } + + while (true) { + Path latest = directory.resolve(Integer.toString(state.headDir - 1)); + try { + Path headState = latest.resolve(HeadState.HEAD_STATE); + HeadState.validateHeadStateIntegrity(headState); + break; + } catch (InvalidHeadStateException e) { + e.printStackTrace(); + state.headDir--; + callback.accept(e); + } finally { + cleanBaseDirectory(directory, latest, callback); + } + } + return state; + } catch(Exception i) { + callback.accept(i); + int largest = -1; + Path latest = findNewHeadStateDir(directory, callback); + if (latest != null) + largest = safeParseInt(-1, latest.getFileName().toString()); + // +1 because we want to return the next head version to use, + // not the latest existing version. + largest++; + MainState state = new MainState( largest ); + cleanBaseDirectory(directory, latest, callback); + return state; + } finally { + if (Files.exists(mainState)) { + Files.delete(mainState); + } + } + } + + public void save(Path directory) throws IOException { + Path f = directory.resolve(MAIN_STATE); + BinaryMemory rf = new BinaryMemory(4096); + try { + MutableVariant v = new MutableVariant(Bindings.getBindingUnchecked(MainState.class), this); + Serializer s = Bindings.getSerializerUnchecked( Bindings.VARIANT ); + s.serialize(rf, v); + } finally { + rf.close(); + } + byte[] bytes = rf.toByteBuffer().array(); + try (OutputStream out = Files.newOutputStream(f)) { + out.write(bytes); + } + FileIO.syncPath(f); + } + + private static boolean isInteger(Path p) { + try { + Integer.parseInt(p.getFileName().toString()); + return true; + } catch (NumberFormatException e) { + return false; + } + } + + /** + * + * @param directory + * @param callback + * @return + * @throws IOException + */ + private static Path findNewHeadStateDir(Path directory, Consumer callback) throws IOException { + try (Stream s = Files.walk(directory, 1)) { + List reverseSortedPaths = s + .filter(p -> !p.equals(directory) && isInteger(p) && Files.isDirectory(p)) + .sorted((p1, p2) -> { + int p1Name = Integer.parseInt(p1.getFileName().toString()); + int p2Name = Integer.parseInt(p2.getFileName().toString()); + return Integer.compare(p2Name, p1Name); + }).collect(Collectors.toList()); + + Path latest = null; + for (Path last : reverseSortedPaths) { + Path headState = last.resolve(HeadState.HEAD_STATE); + try { + HeadState.validateHeadStateIntegrity(headState); + latest = last; + break; + } catch (IOException | InvalidHeadStateException e) { + // Cleanup is done in {@link cleanBaseDirectory} method + callback.accept(e); + } + } + return latest; + } + } + + private static int safeParseInt(int defaultValue, String s) { + try { + return Integer.parseInt(s); + } catch (NumberFormatException e) { + return defaultValue; + } + } + + private static void cleanBaseDirectory(Path directory, Path latest, Consumer callback) throws IOException { + try (Stream s = Files.walk(directory, 1)) { + List reverseSortedPaths = s + .filter(p -> !p.equals(directory) && isInteger(p) && Files.isDirectory(p)) + .sorted((p1, p2) -> { + int p1Name = Integer.parseInt(p1.getFileName().toString()); + int p2Name = Integer.parseInt(p2.getFileName().toString()); + return Integer.compare(p2Name, p1Name); + }).collect(Collectors.toList()); + + for (Path p : reverseSortedPaths) { + if (!p.equals(latest)) { + // this indicates that there is a possibility that index and vg's are out of sync + // if we are able to find folders with higher number than the current head.state + callback.accept(null); + uncheckedDeleteAll(p); + } else { + break; + } + } + + } + } + + private static void uncheckedDeleteAll(Path path) { + try { + FileUtils.deleteAll(path.toFile()); + } catch (IOException e) { + throw new RuntimeIOException(e); + } + } + +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/Persistable.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/Persistable.java new file mode 100644 index 000000000..86dfdd435 --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/Persistable.java @@ -0,0 +1,15 @@ +package org.simantics.acorn; + +import java.io.IOException; +import java.nio.file.Path; + +import org.simantics.acorn.exception.AcornAccessVerificationException; +import org.simantics.acorn.exception.IllegalAcornStateException; +import org.simantics.db.exception.SDBException; + +public interface Persistable { + + void toFile(Path path) throws IOException ; + void fromFile(byte[] data) throws IllegalAcornStateException, AcornAccessVerificationException; + +} \ No newline at end of file diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/UndoClusterSupport.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/UndoClusterSupport.java new file mode 100644 index 000000000..1e7352c3e --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/UndoClusterSupport.java @@ -0,0 +1,170 @@ +package org.simantics.acorn; + +import java.io.InputStream; + +import org.simantics.db.Session; +import org.simantics.db.exception.DatabaseException; +import org.simantics.db.impl.ClusterBase; +import org.simantics.db.impl.ClusterSupport; +import org.simantics.db.impl.IClusterTable; +import org.simantics.db.service.ClusterUID; + +public class UndoClusterSupport implements ClusterSupport { + + final ClusterManager impl; + + public UndoClusterSupport(ClusterManager impl) { + this.impl = impl; + } + + @Override + public int createClusterKeyByClusterUID(ClusterUID clusterUID, + long clusterId) { + throw new UnsupportedOperationException(); + } + + @Override + public ClusterBase getClusterByClusterUIDOrMake(ClusterUID clusterUID) { + throw new UnsupportedOperationException(); + } + + @Override + public ClusterBase getClusterByClusterId(long clusterId) { + throw new UnsupportedOperationException(); + } + + @Override + public ClusterBase getClusterByClusterKey(int clusterKey) { + try { + return impl.getClusterByClusterKey(clusterKey); + } catch (DatabaseException e) { + e.printStackTrace(); + return null; + } + } + + @Override + public int getClusterKeyByClusterUIDOrMake(ClusterUID clusterUID) { + throw new UnsupportedOperationException(); + } + + @Override + public ClusterBase getClusterByResourceKey(int resourceKey) { + throw new UnsupportedOperationException(); + } + + @Override + public long getClusterIdOrCreate(ClusterUID clusterUID) { + throw new UnsupportedOperationException(); + } + + @Override + public void addStatement(Object cluster) { + } + + @Override + public void cancelStatement(Object cluster) { + throw new UnsupportedOperationException(); + } + + @Override + public void removeStatement(Object cluster) { + } + + @Override + public void cancelValue(Object cluster) { + throw new UnsupportedOperationException(); + } + + @Override + public void removeValue(Object cluster) { + throw new UnsupportedOperationException(); + } + + @Override + public void setValue(Object cluster, long clusterId, byte[] bytes, + int length) { + } + + @Override + public void modiValue(Object cluster, long clusterId, long voffset, + int length, byte[] bytes, int offset) { + throw new UnsupportedOperationException(); + } + + @Override + public void setImmutable(Object cluster, boolean immutable) { + throw new UnsupportedOperationException(); + } + + @Override + public void setDeleted(Object cluster, boolean deleted) { + throw new UnsupportedOperationException(); + } + + @Override + public void createResource(Object cluster, short resourceIndex, + long clusterId) { + throw new UnsupportedOperationException(); + } + + @Override + public void addStatementIndex(Object cluster, int resourceKey, + ClusterUID clusterUID, byte op) { + } + + @Override + public void setStreamOff(boolean setOff) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean getStreamOff() { + throw new UnsupportedOperationException(); + } + + @Override + public InputStream getValueStreamEx(int resourceIndex, long clusterId) + throws DatabaseException { + throw new UnsupportedOperationException(); + } + + @Override + public byte[] getValueEx(int resourceIndex, long clusterId) + throws DatabaseException { + throw new UnsupportedOperationException(); + } + + @Override + public byte[] getValueEx(int resourceIndex, long clusterId, long voffset, + int length) throws DatabaseException { + throw new UnsupportedOperationException(); + } + + @Override + public long getValueSizeEx(int resourceIndex, long clusterId) + throws DatabaseException { + throw new UnsupportedOperationException(); + } + + @Override + public int wait4RequestsLess(int limit) throws DatabaseException { + throw new UnsupportedOperationException(); + } + + @Override + public Session getSession() { + throw new UnsupportedOperationException(); + } + + @Override + public IClusterTable getClusterTable() { + throw new UnsupportedOperationException(); + } + + @Override + public int getClusterKeyByClusterUIDOrMake(long id1, long id2) { + throw new UnsupportedOperationException(); + } + +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/backup/AcornBackupProvider.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/backup/AcornBackupProvider.java new file mode 100644 index 000000000..3977ad73d --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/backup/AcornBackupProvider.java @@ -0,0 +1,321 @@ +package org.simantics.acorn.backup; + +import java.io.BufferedReader; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Arrays; +import java.util.concurrent.Future; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import org.simantics.acorn.AcornSessionManagerImpl; +import org.simantics.acorn.GraphClientImpl2; +import org.simantics.acorn.exception.IllegalAcornStateException; +import org.simantics.backup.BackupException; +import org.simantics.backup.IBackupProvider; +import org.simantics.db.server.ProCoreException; +import org.simantics.utils.FileUtils; + +/** + * @author Jani + * + * TODO: get rid of {@link GraphClientImpl2#getInstance()} invocations somehow in a cleaner way + */ +public class AcornBackupProvider implements IBackupProvider { + + private static final String IDENTIFIER = "AcornBackupProvider"; + private long trId = -1; + private final Semaphore lock = new Semaphore(1); + private final GraphClientImpl2 client; + + public AcornBackupProvider() { + this.client = AcornSessionManagerImpl.getInstance().getClient(); + } + + private static Path getAcornMetadataFile(Path dbFolder) { + return dbFolder.getParent().resolve(IDENTIFIER); + } + + @Override + public void lock() throws BackupException { + try { + if (trId != -1) + throw new IllegalStateException(this + " backup provider is already locked"); + trId = client.askWriteTransaction(-1).getTransactionId(); + } catch (ProCoreException e) { + e.printStackTrace(); + } + } + + @Override + public Future backup(Path targetPath, int revision) throws BackupException { + boolean releaseLock = true; + try { + lock.acquire(); + + client.makeSnapshot(true); + + Path dbDir = client.getDbFolder(); + int newestFolder = client.clusters.mainState.headDir - 1; + int latestFolder = -2; + Path AcornMetadataFile = getAcornMetadataFile(dbDir); + if (Files.exists(AcornMetadataFile)) { + try (BufferedReader br = Files.newBufferedReader(AcornMetadataFile)) { + latestFolder = Integer.parseInt( br.readLine() ); + } + } + + AcornBackupRunnable r = new AcornBackupRunnable( + lock, targetPath, revision, dbDir, latestFolder, newestFolder); + new Thread(r, "Acorn backup thread").start(); + + releaseLock = false; + return r; + } catch (InterruptedException e) { + releaseLock = false; + throw new BackupException("Failed to lock Acorn for backup.", e); + } catch (NumberFormatException e) { + throw new BackupException("Failed to read Acorn head state file.", e); + } catch (IllegalAcornStateException | IOException e) { + throw new BackupException("I/O problem during Acorn backup.", e); + } finally { + if (releaseLock) + lock.release(); + } + } + + @Override + public void unlock() throws BackupException { + try { + if (trId == -1) + throw new BackupException(this + " backup provider is not locked"); + client.endTransaction(trId); + trId = -1; + } catch (ProCoreException e) { + throw new BackupException(e); + } + } + + @Override + public void restore(Path fromPath, int revision) { + try { + // 1. Resolve initial backup restore target. + // This can be DB directory directly or a temporary directory that + // will replace the DB directory. + Path dbRoot = client.getDbFolder(); + Path restorePath = dbRoot; + if (!Files.exists(dbRoot, LinkOption.NOFOLLOW_LINKS)) { + Files.createDirectories(dbRoot); + } else { + Path dbRootParent = dbRoot.getParent(); + restorePath = dbRootParent == null ? Files.createTempDirectory("restore") + : Files.createTempDirectory(dbRootParent, "restore"); + } + + // 2. Restore the backup. + Files.walkFileTree(fromPath, new RestoreCopyVisitor(restorePath, revision)); + + // 3. Override existing DB root with restored temporary copy if necessary. + if (dbRoot != restorePath) { + FileUtils.deleteAll(dbRoot.toFile()); + Files.move(restorePath, dbRoot); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + private class RestoreCopyVisitor extends SimpleFileVisitor { + + private final Path toPath; + private final int revision; + private Path currentSubFolder; + + public RestoreCopyVisitor(Path toPath, int revision) { + this.toPath = toPath; + this.revision = revision; + } + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + Path dirName = dir.getFileName(); + if (dirName.toString().equals(IDENTIFIER)) { + currentSubFolder = dir; + return FileVisitResult.CONTINUE; + } else if (dir.getParent().getFileName().toString().equals(IDENTIFIER)) { + Path targetPath = toPath.resolve(dirName); + if (!Files.exists(targetPath)) { + Files.createDirectory(targetPath); + } + return FileVisitResult.CONTINUE; + } else if (dirName.toString().length() == 1 && Character.isDigit(dirName.toString().charAt(0))) { + int dirNameInt = Integer.parseInt(dirName.toString()); + if (dirNameInt <= revision) { + return FileVisitResult.CONTINUE; + } else { + return FileVisitResult.SKIP_SUBTREE; + } + } else { + return FileVisitResult.CONTINUE; + } + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + if (file.getFileName().toString().endsWith(".tar.gz")) + return FileVisitResult.CONTINUE; + System.out.println("Restore " + file + " to " + toPath.resolve(currentSubFolder.relativize(file))); + Files.copy(file, toPath.resolve(currentSubFolder.relativize(file)), StandardCopyOption.REPLACE_EXISTING); + return FileVisitResult.CONTINUE; + } + } + + private static class AcornBackupRunnable implements Runnable, Future { + + private final Semaphore lock; + private final Path targetPath; + private final int revision; + private final Path baseDir; + private final int latestFolder; + private final int newestFolder; + + private boolean done = false; + private final Semaphore completion = new Semaphore(0); + private BackupException exception = null; + + public AcornBackupRunnable(Semaphore lock, Path targetPath, int revision, + Path baseDir, int latestFolder, int newestFolder) { + this.lock = lock; + this.targetPath = targetPath; + this.revision = revision; + this.baseDir = baseDir; + this.latestFolder = latestFolder; + this.newestFolder = newestFolder; + } + + @Override + public void run() { + try { + doBackup(); + writeHeadstateFile(); + } catch (IOException e) { + exception = new BackupException("Acorn backup failed", e); + rollback(); + } finally { + done = true; + lock.release(); + completion.release(); + } + } + + private void doBackup() throws IOException { + Path target = targetPath.resolve(String.valueOf(revision)).resolve(IDENTIFIER); + if (!Files.exists(target)) + Files.createDirectories(target); + Files.walkFileTree(baseDir, + new BackupCopyVisitor(baseDir, target)); + } + + private void writeHeadstateFile() throws IOException { + Path AcornMetadataFile = getAcornMetadataFile(baseDir); + if (!Files.exists(AcornMetadataFile)) { + Files.createFile(AcornMetadataFile); + } + Files.write(AcornMetadataFile, + Arrays.asList(Integer.toString(newestFolder)), + StandardOpenOption.WRITE, + StandardOpenOption.TRUNCATE_EXISTING, + StandardOpenOption.CREATE); + } + + private void rollback() { + // TODO + } + + private class BackupCopyVisitor extends SimpleFileVisitor { + + private Path fromPath; + private Path toPath; + + public BackupCopyVisitor(Path fromPath, Path toPath) { + this.fromPath = fromPath; + this.toPath = toPath; + } + + @Override + public FileVisitResult preVisitDirectory(Path dir, + BasicFileAttributes attrs) throws IOException { + Path dirName = dir.getFileName(); + if (dirName.equals(fromPath)) { + Path targetPath = toPath.resolve(fromPath.relativize(dir)); + if (!Files.exists(targetPath)) { + Files.createDirectory(targetPath); + } + return FileVisitResult.CONTINUE; + } else { + int dirNameInt = Integer.parseInt(dirName.toString()); + if (latestFolder < dirNameInt && dirNameInt <= newestFolder) { + Path targetPath = toPath.resolve(fromPath + .relativize(dir)); + if (!Files.exists(targetPath)) { + Files.createDirectory(targetPath); + } + return FileVisitResult.CONTINUE; + } + return FileVisitResult.SKIP_SUBTREE; + } + } + + @Override + public FileVisitResult visitFile(Path file, + BasicFileAttributes attrs) throws IOException { + System.out.println("Backup " + file + " to " + + toPath.resolve(fromPath.relativize(file))); + Files.copy(file, toPath.resolve(fromPath.relativize(file)), + StandardCopyOption.REPLACE_EXISTING); + return FileVisitResult.CONTINUE; + } + } + + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + return false; + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public boolean isDone() { + return done; + } + + @Override + public BackupException get() throws InterruptedException { + completion.acquire(); + completion.release(); + return exception; + } + + @Override + public BackupException get(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException { + if (completion.tryAcquire(timeout, unit)) + completion.release(); + else + throw new TimeoutException("Acorn backup completion waiting timed out."); + return exception; + } + + } + +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/cluster/ClusterBig.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/cluster/ClusterBig.java new file mode 100644 index 000000000..51241728d --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/cluster/ClusterBig.java @@ -0,0 +1,1109 @@ +/******************************************************************************* + * Copyright (c) 2007, 2010 Association for Decentralized Information Management + * in Industry THTH ry. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * VTT Technical Research Centre of Finland - initial API and implementation + *******************************************************************************/ +package org.simantics.acorn.cluster; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; + +import org.simantics.acorn.exception.AcornAccessVerificationException; +import org.simantics.acorn.exception.IllegalAcornStateException; +import org.simantics.acorn.internal.ClusterChange; +import org.simantics.acorn.internal.ClusterStream; +import org.simantics.acorn.internal.ClusterSupport2; +import org.simantics.acorn.internal.DebugPolicy; +import org.simantics.db.Resource; +import org.simantics.db.exception.DatabaseException; +import org.simantics.db.exception.ExternalValueException; +import org.simantics.db.exception.ValidationException; +import org.simantics.db.impl.ClusterI; +import org.simantics.db.impl.ClusterI.PredicateProcedure; +import org.simantics.db.impl.ClusterSupport; +import org.simantics.db.impl.ClusterTraitsBase; +import org.simantics.db.impl.ForEachObjectContextProcedure; +import org.simantics.db.impl.ForEachObjectProcedure; +import org.simantics.db.impl.ForPossibleRelatedValueContextProcedure; +import org.simantics.db.impl.ForPossibleRelatedValueProcedure; +import org.simantics.db.impl.IClusterTable; +import org.simantics.db.impl.Table; +import org.simantics.db.impl.TableHeader; +import org.simantics.db.impl.graph.ReadGraphImpl; +import org.simantics.db.impl.query.QueryProcessor; +import org.simantics.db.procedure.AsyncContextMultiProcedure; +import org.simantics.db.procedure.AsyncMultiProcedure; +import org.simantics.db.procore.cluster.ClusterMap; +import org.simantics.db.procore.cluster.ClusterPrintDebugInfo; +import org.simantics.db.procore.cluster.ClusterTraits; +import org.simantics.db.procore.cluster.CompleteTable; +import org.simantics.db.procore.cluster.FlatTable; +import org.simantics.db.procore.cluster.ForeignTable; +import org.simantics.db.procore.cluster.ObjectTable; +import org.simantics.db.procore.cluster.PredicateTable; +import org.simantics.db.procore.cluster.ResourceTable; +import org.simantics.db.procore.cluster.ValueTable; +import org.simantics.db.service.ClusterUID; +import org.simantics.utils.datastructures.Callback; + +final public class ClusterBig extends ClusterImpl { + private static final int TABLE_HEADER_SIZE = TableHeader.HEADER_SIZE + TableHeader.EXTRA_SIZE; + private static final int RESOURCE_TABLE_OFFSET = 0; + private static final int PREDICATE_TABLE_OFFSET = RESOURCE_TABLE_OFFSET + TABLE_HEADER_SIZE; + private static final int OBJECT_TABLE_OFFSET = PREDICATE_TABLE_OFFSET + TABLE_HEADER_SIZE; + private static final int VALUE_TABLE_OFFSET = OBJECT_TABLE_OFFSET + TABLE_HEADER_SIZE; + private static final int FLAT_TABLE_OFFSET = VALUE_TABLE_OFFSET + TABLE_HEADER_SIZE; + private static final int COMPLETE_TABLE_OFFSET = FLAT_TABLE_OFFSET + TABLE_HEADER_SIZE; + private static final int FOREIGN_TABLE_OFFSET = COMPLETE_TABLE_OFFSET + TABLE_HEADER_SIZE; + private static final int INT_HEADER_SIZE = FOREIGN_TABLE_OFFSET + TABLE_HEADER_SIZE; + private final int clusterBits; + final private ResourceTable resourceTable; + //final private ResourceTable movedResourceTable; + final private PredicateTable predicateTable; + final private ObjectTable objectTable; + final private ValueTable valueTable; + final private FlatTable flatTable; + final private ForeignTable foreignTable; + final private CompleteTable completeTable; + final private ClusterMap clusterMap; + final private int[] headerTable; + final private ClusterSupport2 clusterSupport; + + public ClusterBig(IClusterTable clusterTable, ClusterUID clusterUID, int clusterKey, ClusterSupport2 support) { + super(clusterTable, clusterUID, clusterKey, support); + if(DebugPolicy.REPORT_CLUSTER_EVENTS) + new Exception(getClusterUID().toString()).printStackTrace(); + this.headerTable = new int[INT_HEADER_SIZE]; + this.resourceTable = new ResourceTable(this, headerTable, RESOURCE_TABLE_OFFSET); + this.foreignTable = new ForeignTable(this, headerTable, FOREIGN_TABLE_OFFSET); + this.predicateTable = new PredicateTable(this, headerTable, PREDICATE_TABLE_OFFSET); + this.objectTable = new ObjectTable(this, headerTable, OBJECT_TABLE_OFFSET); + this.valueTable = new ValueTable(this, headerTable, VALUE_TABLE_OFFSET); + this.completeTable = new CompleteTable(this, headerTable, COMPLETE_TABLE_OFFSET); + this.flatTable = null; + this.clusterMap = new ClusterMap(foreignTable, flatTable); + this.clusterSupport = support; + this.clusterBits = ClusterTraitsBase.getClusterBits(clusterKey); + this.importance = 0; +// clusterTable.setDirtySizeInBytes(true); + } + protected ClusterBig(IClusterTable clusterTable, long[] longs, int[] ints, byte[] bytes, ClusterSupport2 support, int clusterKey) + throws DatabaseException { + super(clusterTable, checkValidity(0, longs, ints, bytes), clusterKey, support); + if(DebugPolicy.REPORT_CLUSTER_EVENTS) + new Exception(getClusterUID().toString()).printStackTrace(); + if (ints.length < INT_HEADER_SIZE) + throw new IllegalArgumentException("Too small integer table for cluster."); + this.headerTable = ints; + this.resourceTable = new ResourceTable(this, ints, RESOURCE_TABLE_OFFSET, longs); + this.foreignTable = new ForeignTable(this, headerTable, FOREIGN_TABLE_OFFSET, longs); + this.predicateTable = new PredicateTable(this, ints, PREDICATE_TABLE_OFFSET, ints); + this.objectTable = new ObjectTable(this, ints, OBJECT_TABLE_OFFSET, ints); + this.valueTable = new ValueTable(this, ints, VALUE_TABLE_OFFSET, bytes); + this.flatTable = null; + this.completeTable = new CompleteTable(this, headerTable, COMPLETE_TABLE_OFFSET, ints); + this.clusterMap = new ClusterMap(foreignTable, flatTable); + this.clusterSupport = support; + this.clusterBits = ClusterTraitsBase.getClusterBits(clusterKey); + } + void analyse() { + System.out.println("Cluster " + clusterId); + System.out.println("-size:" + getUsedSpace()); + System.out.println(" -rt:" + (resourceTable.getTableCapacity() * 8 + 8)); + System.out.println(" -ft:" + foreignTable.getTableCapacity() * 8); + System.out.println(" -pt:" + predicateTable.getTableCapacity() * 4); + System.out.println(" -ot:" + objectTable.getTableCapacity() * 4); + System.out.println(" -ct:" + completeTable.getTableCapacity() * 4); + System.out.println(" -vt:" + valueTable.getTableCapacity()); + + System.out.println("-resourceTable:"); + System.out.println(" -resourceCount=" + resourceTable.getResourceCount()); + System.out.println(" -size=" + resourceTable.getTableSize()); + System.out.println(" -capacity=" + resourceTable.getTableCapacity()); + System.out.println(" -count=" + resourceTable.getTableCount()); + System.out.println(" -size=" + resourceTable.getTableSize()); + //resourceTable.analyse(); + } + public void checkDirectReference(int dr) + throws DatabaseException { + if (!ClusterTraits.statementIndexIsDirect(dr)) + throw new ValidationException("Reference is not direct. Reference=" + dr); + if (ClusterTraits.isFlat(dr)) + throw new ValidationException("Reference is flat. Reference=" + dr); + if (ClusterTraits.isLocal(dr)) { + if (dr < 1 || dr > resourceTable.getUsedSize()) + throw new ValidationException("Illegal local reference. Reference=" + dr); + } else { + int fi = ClusterTraits.getForeignIndexFromReference(dr); + int ri = ClusterTraits.getResourceIndexFromForeignReference(dr); + if (fi < 1 || fi > foreignTable.getUsedSize()) + throw new ValidationException("Illegal foreign reference. Reference=" + dr + " foreign index=" + fi); + if (ri < 1 || ri > ClusterTraits.getMaxNumberOfResources()) + throw new ValidationException("Illegal foreign reference. Reference=" + dr + " resource index=" + ri); + } + } + public void checkPredicateIndex(int pi) + throws DatabaseException { + predicateTable.checkPredicateSetIndex(this, pi); + } + public void checkObjectSetReference(int or) + throws DatabaseException { + if (ClusterTraits.statementIndexIsDirect(or)) + throw new ValidationException("Illegal object set reference. Reference=" + or); + int oi = ClusterTraits.statementIndexGet(or); + this.objectTable.checkObjectSetIndex(this, oi); + } + + public void checkValueInit() + throws DatabaseException { + valueTable.checkValueInit(); + } + public void checkValue(int capacity, int index) + throws DatabaseException { + valueTable.checkValue(capacity, index); + } + public void checkValueFini() + throws DatabaseException { + valueTable.checkValueFini(); + } + public void checkForeingIndex(int fi) + throws DatabaseException { + if (fi<1 || fi > foreignTable.getUsedSize()) + throw new ValidationException("Illegal foreign index=" + fi); + } + public void checkCompleteSetReference(int cr) + throws DatabaseException { + if (!ClusterTraits.completeReferenceIsMultiple(cr)) + throw new ValidationException("Illegal complete set reference. Reference=" + cr); + int ci = cr; + this.completeTable.checkCompleteSetIndex(this, ci); + } + public void check() + throws DatabaseException { + this.completeTable.check(this); + this.objectTable.check(this); + // Must be after object table check. + this.predicateTable.check(this); + this.resourceTable.check(this); + } + @Override + public CompleteTypeEnum getCompleteType(int resourceKey, ClusterSupport support) + throws DatabaseException { + final int resourceRef = getLocalReference(resourceKey); + int completeRef = resourceTable.getCompleteObjectRef(resourceRef); + CompleteTypeEnum ct = ClusterTraits.completeReferenceGetType(completeRef); + if (DEBUG) + System.out.println("Cluster.getCompleteType rk=" + resourceKey + " ct=" + ct); + int i = ct.getValue(); + switch (i) { + case 0: return CompleteTypeEnum.NotComplete; + case 1: return CompleteTypeEnum.InstanceOf; + case 2: return CompleteTypeEnum.Inherits; + case 3: return CompleteTypeEnum.SubrelationOf; + default: throw new DatabaseException("Illegal complete type enumeration."); + } + } + + @Override + public int getCompleteObjectKey(int resourceKey, ClusterSupport support) + throws DatabaseException { + final int resourceRef = getLocalReference(resourceKey); + int completeRef = resourceTable.getCompleteObjectRef(resourceRef); + int clusterIndex; + int resourceIndex = ClusterTraits.completeReferenceGetResourceIndex(completeRef); + + ClusterI.CompleteTypeEnum completeType = ClusterTraits.completeReferenceGetType(completeRef); + if (completeType == ClusterI.CompleteTypeEnum.NotComplete) + throw new DatabaseException("Resource has multiple complete objects. Resource key=" + resourceKey + "."); + + if (ClusterTraits.completeReferenceIsLocal(completeRef)) { + clusterIndex = clusterKey; + } else { + int foreignIndex = ClusterTraits.completeReferenceGetForeignIndex(completeRef); +// System.err.println("completeRef=" + completeRef + " foreignIndex=" + foreignIndex ); + ClusterUID clusterUID = foreignTable.getResourceUID(foreignIndex).asCID(); + ClusterI c = support.getClusterByClusterUIDOrMake(clusterUID); + clusterIndex = c.getClusterKey(); + } + int key = ClusterTraits.createResourceKey(clusterIndex, resourceIndex); + if (DEBUG) + System.out.println("Cluster.complete object rk=" + resourceKey + " ck=" + key); + return key; + } + + @Override + public boolean isComplete(int resourceKey, ClusterSupport support) + throws DatabaseException { + final int resourceRef = getLocalReference(resourceKey); + int completeRef = resourceTable.getCompleteObjectRef(resourceRef); + ClusterI.CompleteTypeEnum completeType = ClusterTraits.completeReferenceGetType(completeRef); + boolean complete = completeType != ClusterI.CompleteTypeEnum.NotComplete; + if (DEBUG) + System.out.println("Cluster.key=" + resourceKey + " isComplete=" + complete); + return complete; + } + + public int getSingleObject(int resourceKey, int predicateKey, int objectIndex, ClusterSupport support) throws DatabaseException { + if (DEBUG) + System.out.println("Cluster.getSingleObject: rk=" + resourceKey + " pk=" + predicateKey); + if (0 == objectIndex) { + final int resourceIndex = getLocalReference(resourceKey); + final int pRef = getInternalReferenceOrZero(predicateKey, support); + final ClusterI.CompleteTypeEnum pCompleteType = ClusterTraitsBase.getCompleteTypeFromResourceKey(predicateKey); + return resourceTable.getSingleObject(resourceIndex, support, pRef, pCompleteType, completeTable, this); + } + return objectTable.getSingleObject(objectIndex, support, this); + } + + public void forObjects(int resourceKey, int predicateKey, int objectIndex, QueryProcessor processor, ReadGraphImpl graph, AsyncMultiProcedure procedure, + ClusterSupport support) throws DatabaseException { + if (DEBUG) + System.out.println("Cluster.forObjects1: rk=" + resourceKey + " pk=" + predicateKey); + if (0 == objectIndex) { + final int resourceIndex = getLocalReference(resourceKey); + final int pRef = getInternalReferenceOrZero(predicateKey, support); + final ClusterI.CompleteTypeEnum pCompleteType = ClusterTraitsBase.getCompleteTypeFromResourceKey(predicateKey); + resourceTable.foreachObject(resourceIndex, graph, procedure, support, pRef, pCompleteType, completeTable, this); + return; + } + objectTable.foreachObject(graph, objectIndex, procedure, this); + } + public void forObjects(int resourceKey, int predicateKey, int objectIndex, QueryProcessor processor, ReadGraphImpl graph, C context, AsyncContextMultiProcedure procedure, + ClusterSupport support) throws DatabaseException { + if (DEBUG) + System.out.println("Cluster.forObjects1: rk=" + resourceKey + " pk=" + predicateKey); + if (0 == objectIndex) { + final int resourceIndex = getLocalReference(resourceKey); + final int pRef = getInternalReferenceOrZero(predicateKey, support); + final ClusterI.CompleteTypeEnum pCompleteType = ClusterTraitsBase.getCompleteTypeFromResourceKey(predicateKey); + resourceTable.foreachObject(resourceIndex, graph, context, procedure, support, pRef, pCompleteType, completeTable, this); + return; + } + objectTable.foreachObject(graph, objectIndex, context, procedure, this); + } + @Override + public boolean forObjects(int resourceKey, int predicateKey, int objectIndex, ObjectProcedure procedure, + Context context, ClusterSupport support) throws DatabaseException { + if (DEBUG) + System.out.println("Cluster.forObjects2: rk=" + resourceKey + " pk=" + predicateKey); + if (0 == objectIndex) { + final int resourceIndex = getLocalReference(resourceKey); + final int pRef = getInternalReferenceOrZero(predicateKey, support); + final ClusterI.CompleteTypeEnum pCompleteType = ClusterTraitsBase.getCompleteTypeFromResourceKey(predicateKey); + return resourceTable.foreachObject(resourceIndex, procedure, context, support, this, pRef, pCompleteType, completeTable); + } + return objectTable.foreachObject(objectIndex, procedure, context, support, this); + } + + @Override + public int getSingleObject(int resourceKey, int predicateKey, ClusterSupport support) throws DatabaseException { + if (DEBUG) + System.out.println("Cluster.getSingleObject2: rk=" + resourceKey + " pk=" + predicateKey); + final int resourceIndex = getLocalReference(resourceKey); + final int pRef = getInternalReferenceOrZero(predicateKey, support); + final ClusterI.CompleteTypeEnum pCompleteType = ClusterTraitsBase.getCompleteTypeFromResourceKey(predicateKey); + if (ClusterI.CompleteTypeEnum.NotComplete != pCompleteType) + return resourceTable.getSingleObject(resourceIndex, support, pRef, pCompleteType, completeTable, this); + final int predicateIndex = resourceTable.getPredicateIndex(resourceIndex); + if (0 == predicateIndex) + return resourceTable.getSingleObject(resourceIndex, support, pRef, pCompleteType, completeTable, this); + int objectIndex = predicateTable.getObjectIndex(predicateIndex, pRef); + return getSingleObject(resourceKey, predicateKey, objectIndex, support); + } + + @Override + public int getSingleObject(int resourceKey, ForPossibleRelatedValueProcedure procedure, ClusterSupport support) throws DatabaseException { + final int predicateKey = procedure.predicateKey; + if (DEBUG) + System.out.println("Cluster.getSingleObject2: rk=" + resourceKey + " pk=" + predicateKey); + final int resourceIndex = getLocalReference(resourceKey); + final int pRef = getInternalReferenceOrZero(predicateKey, support); + final ClusterI.CompleteTypeEnum pCompleteType = ClusterTraitsBase.getCompleteTypeFromResourceKey(predicateKey); + if (ClusterI.CompleteTypeEnum.NotComplete != pCompleteType) + return resourceTable.getSingleObject(resourceIndex, support, pRef, pCompleteType, completeTable, this); + final int predicateIndex = resourceTable.getPredicateIndex(resourceIndex); + if (0 == predicateIndex) + return resourceTable.getSingleObject(resourceIndex, support, pRef, pCompleteType, completeTable, this); + int objectIndex = predicateTable.getObjectIndex(predicateIndex, pRef); + return getSingleObject(resourceKey, predicateKey, objectIndex, support); + } + + @Override + public int getSingleObject(int resourceKey, ForPossibleRelatedValueContextProcedure procedure, ClusterSupport support) throws DatabaseException { + final int predicateKey = procedure.predicateKey; + if (DEBUG) + System.out.println("Cluster.getSingleObject2: rk=" + resourceKey + " pk=" + predicateKey); + final int resourceIndex = getLocalReference(resourceKey); + final int pRef = getInternalReferenceOrZero(predicateKey, support); + final ClusterI.CompleteTypeEnum pCompleteType = ClusterTraitsBase.getCompleteTypeFromResourceKey(predicateKey); + if (ClusterI.CompleteTypeEnum.NotComplete != pCompleteType) + return resourceTable.getSingleObject(resourceIndex, support, pRef, pCompleteType, completeTable, this); + final int predicateIndex = resourceTable.getPredicateIndex(resourceIndex); + if (0 == predicateIndex) + return resourceTable.getSingleObject(resourceIndex, support, pRef, pCompleteType, completeTable, this); + int objectIndex = predicateTable.getObjectIndex(predicateIndex, pRef); + return getSingleObject(resourceKey, predicateKey, objectIndex, support); + } + + @Override + public void forObjects(ReadGraphImpl graph, int resourceKey, + int predicateKey, AsyncMultiProcedure procedure) + throws DatabaseException { + + throw new UnsupportedOperationException(); + +// SessionImplSocket session = (SessionImplSocket)graph.getSession(); +// ClusterSupport support = session.clusterTranslator; +// +// if (DEBUG) +// System.out.println("Cluster.forObjects3: rk=" + resourceKey + " pk=" + predicateKey); +// final int resourceIndex = getLocalReference(resourceKey); +// final int pRef = getInternalReferenceOrZero(predicateKey, support); +// final ClusterI.CompleteTypeEnum pCompleteType = ClusterTraitsBase.getCompleteTypeFromResourceKey(predicateKey); +// if (ClusterI.CompleteTypeEnum.NotComplete != pCompleteType) { +// resourceTable.foreachObject(resourceIndex, graph, procedure, support, pRef, pCompleteType, completeTable, this); +// return; +// } +// final int predicateIndex = resourceTable.getPredicateIndex(resourceIndex); +// if (0 == predicateIndex) { +// resourceTable.foreachObject(resourceIndex, graph, procedure, support, pRef, pCompleteType, completeTable, this); +// return; +// } +// int objectIndex = predicateTable.getObjectIndex(predicateIndex, pRef); +// forObjects(resourceKey, predicateKey, objectIndex, graph.processor, graph, procedure, support); + + } + + @Override + public void forObjects(ReadGraphImpl graph, int resourceKey, ForEachObjectProcedure procedure) throws DatabaseException { + + throw new UnsupportedOperationException(); + +// SessionImplSocket session = (SessionImplSocket)graph.getSession(); +// ClusterSupport support = session.clusterTranslator; +// final int predicateKey = procedure.predicateKey; +// if (DEBUG) +// System.out.println("Cluster.forObjects3: rk=" + resourceKey + " pk=" + predicateKey); +// final int resourceIndex = getLocalReference(resourceKey); +// final int pRef = getInternalReferenceOrZero(predicateKey, support); +// final ClusterI.CompleteTypeEnum pCompleteType = ClusterTraitsBase.getCompleteTypeFromResourceKey(predicateKey); +// if (ClusterI.CompleteTypeEnum.NotComplete != pCompleteType) { +// resourceTable.foreachObject(resourceIndex, graph, procedure, support, pRef, pCompleteType, completeTable, this); +// return; +// } +// final int predicateIndex = resourceTable.getPredicateIndex(resourceIndex); +// if (0 == predicateIndex) { +// resourceTable.foreachObject(resourceIndex, graph, procedure, support, pRef, pCompleteType, completeTable, this); +// return; +// } +// int objectIndex = predicateTable.getObjectIndex(predicateIndex, pRef); +// forObjects(resourceKey, predicateKey, objectIndex, graph.processor, graph, procedure, support); + + } + @Override + public void forObjects(ReadGraphImpl graph, int resourceKey, C context, + ForEachObjectContextProcedure procedure) throws DatabaseException { + + throw new UnsupportedOperationException(); + +// SessionImplSocket session = (SessionImplSocket)graph.getSession(); +// ClusterSupport support = session.clusterTranslator; +// +// final int predicateKey = procedure.predicateKey; +// +// if (DEBUG) +// System.out.println("Cluster.forObjects3: rk=" + resourceKey + " pk=" + predicateKey); +// final int resourceIndex = getLocalReference(resourceKey); +// final int pRef = getInternalReferenceOrZero(predicateKey, support); +// final ClusterI.CompleteTypeEnum pCompleteType = ClusterTraitsBase.getCompleteTypeFromResourceKey(predicateKey); +// if (ClusterI.CompleteTypeEnum.NotComplete != pCompleteType) { +// resourceTable.foreachObject(resourceIndex, graph, context, procedure, support, pRef, pCompleteType, completeTable, this); +// return; +// } +// final int predicateIndex = resourceTable.getPredicateIndex(resourceIndex); +// if (0 == predicateIndex) { +// resourceTable.foreachObject(resourceIndex, graph, context, procedure, support, pRef, pCompleteType, completeTable, this); +// return; +// } +// int objectIndex = predicateTable.getObjectIndex(predicateIndex, pRef); +// forObjects(resourceKey, predicateKey, objectIndex, graph.processor, graph, context, procedure, support); + + } + + @Override + public boolean forObjects(int resourceKey, int predicateKey, + ObjectProcedure procedure, Context context, ClusterSupport support) + throws DatabaseException { + if (DEBUG) + System.out.println("Cluster.forObjects4: rk=" + resourceKey + " pk=" + predicateKey); + final int resourceIndex = getLocalReference(resourceKey); + final int pRef = getInternalReferenceOrZero(predicateKey, support); + final ClusterI.CompleteTypeEnum pCompleteType = ClusterTraitsBase.getCompleteTypeFromResourceKey(predicateKey); + if (ClusterI.CompleteTypeEnum.NotComplete != pCompleteType) + return resourceTable.foreachObject(resourceIndex, procedure, context, support, this, pRef, pCompleteType, completeTable); + final int predicateIndex = resourceTable.getPredicateIndex(resourceIndex); + if (0 == predicateIndex) + return resourceTable.foreachObject(resourceIndex, procedure, context, support, this, pRef, pCompleteType, completeTable); + int objectIndex = predicateTable.getObjectIndex(predicateIndex, pRef); + return forObjects(resourceKey, predicateKey, objectIndex, procedure, context, support); + } + @Override + public boolean forPredicates(int resourceKey, + PredicateProcedure procedure, Context context, ClusterSupport support) + throws DatabaseException { + if (DEBUG) + System.out.println("Cluster.forPredicates: rk=" + resourceKey); + final int resourceIndex = getLocalReference(resourceKey); + final int predicateIndex = resourceTable.getPredicateIndex(resourceIndex); + if (0 == predicateIndex) + return resourceTable.foreachPredicate(resourceIndex, + procedure, context, support, this, completeTable); + else { + boolean broken = resourceTable.foreachPredicate(resourceIndex, + procedure, context, support, this, completeTable); + if (broken) + return true; + } + return predicateTable.foreachPredicate(predicateIndex, procedure, context, support, this); + } + @Override + public ClusterI addRelation(int sResourceKey, ClusterUID puid, int pResourceKey, ClusterUID ouid, int oResourceKey, ClusterSupport support) + throws DatabaseException { + if (DEBUG) + System.out.println("add rk=" + sResourceKey + " pk=" + pResourceKey + " ok=" + oResourceKey); + int sri = getLocalReferenceAnd(sResourceKey, support, ClusterChange.ADD_OPERATION); + int pri = getReferenceOrCreateIfForeign(pResourceKey, puid, support, ClusterStream.NULL_OPERATION); + int ori = getReferenceOrCreateIfForeign(oResourceKey, ouid, support, ClusterStream.NULL_OPERATION); + ClusterI.CompleteTypeEnum completeType = ClusterTraitsBase.getCompleteTypeFromResourceKey(pResourceKey); + boolean ret = addRelationInternal(sri, pri, ori, completeType); +// check(); + if (ret) { + support.addStatement(this); + return this; + } else { + support.cancelStatement(this); + return null; + } + } + @Override + public ClusterI addRelation(int sResourceKey, int pResourceKey, int oResourceKey, ClusterSupport support) + throws DatabaseException { + if (DEBUG) + System.out.println("add rk=" + sResourceKey + " pk=" + pResourceKey + " ok=" + oResourceKey); + int sri = getLocalReferenceAnd(sResourceKey, support, ClusterChange.ADD_OPERATION); + int pri = getReferenceOrCreateIfForeign(pResourceKey, support, ClusterStream.NULL_OPERATION); + int ori = getReferenceOrCreateIfForeign(oResourceKey, support, ClusterStream.NULL_OPERATION); + ClusterI.CompleteTypeEnum completeType = ClusterTraitsBase.getCompleteTypeFromResourceKey(pResourceKey); + boolean ret = addRelationInternal(sri, pri, ori, completeType); +// check(); + if (ret) { + support.addStatement(this); + return this; + } else { + support.cancelStatement(this); + return null; + } + } + @Override + public boolean removeRelation(int sResourceKey, int pResourceKey, int oResourceKey, ClusterSupport support) + throws DatabaseException { +// check(); + int sri = getLocalReferenceAnd(sResourceKey, support, ClusterChange.REMOVE_OPERATION); + int pri = getInternalReferenceOrZeroAnd(pResourceKey, support, ClusterStream.NULL_OPERATION); + int ori = getInternalReferenceOrZeroAnd(oResourceKey, support, ClusterStream.NULL_OPERATION); + boolean ret = false; + if (0 != pri && 0 != ori) { + ClusterI.CompleteTypeEnum completeType = ClusterTraitsBase.getCompleteTypeFromResourceKey(pResourceKey); + ret = removeRelationInternal(sri, pri, ori, completeType, support); + } + if (ret) + support.removeStatement(this); + else + support.cancelStatement(this); +// check(); + return ret; + } + @Override + public void denyRelation(int sResourceKey, int pResourceKey, int oResourceKey, ClusterSupport support) + throws DatabaseException { + int sri = checkResourceKeyIsOursAndGetResourceIndexIf(sResourceKey, support); + ResourceIndexAndId p = checkResourceKeyAndGetResourceIndexIf(pResourceKey, support); + ResourceIndexAndId o = checkResourceKeyAndGetResourceIndexIf(oResourceKey, support); + if (0 == sri || 0 == p.index || 0 == o.index) + return; +// check(); + ClusterI.CompleteTypeEnum completeType = ClusterTraitsBase.getCompleteTypeFromResourceKey(pResourceKey); + boolean ret = removeRelationInternal(sri, p.reference, o.reference, completeType, support); + if (ret) { + support.addStatementIndex(this, sResourceKey, getClusterUID(), ClusterChange.REMOVE_OPERATION); + support.addStatementIndex(this, pResourceKey, p.clusterUID, ClusterStream.NULL_OPERATION); + support.addStatementIndex(this, oResourceKey, o.clusterUID, ClusterStream.NULL_OPERATION); + support.removeStatement(this); + } +// check(); + return; + } + @Override + public InputStream getValueStream(int rResourceId, ClusterSupport support) throws DatabaseException { + if (DEBUG) + System.out.println("ClusterBig.getValue " + rResourceId); + int resourceIndex = getLocalReference(rResourceId); + try { + byte[] buffer = resourceTable.getValue(valueTable, resourceIndex); + if(buffer == null) return null; + return new ByteArrayInputStream(buffer); + } catch (ExternalValueException e) { + return support.getValueStreamEx(resourceIndex, clusterUID.second); + } + } + @Override + public byte[] getValue(int rResourceId, ClusterSupport support) + throws DatabaseException { + if (DEBUG) + System.out.println("ClusterBig.getValue " + rResourceId); + int resourceIndex = getLocalReference(rResourceId); + try { + return resourceTable.getValue(valueTable, resourceIndex); + } catch (ExternalValueException e) { + try { + return clusterSupport.impl.getResourceFile(clusterUID.asBytes(), resourceIndex); + } catch (AcornAccessVerificationException | IllegalAcornStateException e1) { + throw new DatabaseException(e1); + } + } + } + @Override + public boolean hasValue(int rResourceId, ClusterSupport support) + throws DatabaseException { + int resourceIndex = getLocalReference(rResourceId); + return resourceTable.hasValue(resourceIndex); + } + @Override + public boolean removeValue(int rResourceId, ClusterSupport support) + throws DatabaseException { + int resourceIndex = getLocalReferenceAnd(rResourceId, support, ClusterChange.DELETE_OPERATION); + support.removeValue(this); + return resourceTable.removeValue(valueTable, resourceIndex); + } + + @Override + public ClusterI setValue(int rResourceId, byte[] value, int length, ClusterSupport support) + throws DatabaseException { + int resourceIndex = getLocalReferenceAnd(rResourceId, support, ClusterStream.SET_OPERATION); + support.setValue(this, getClusterId(), value, length); + resourceTable.setValue(valueTable, resourceIndex, value, length); + return this; + } + @Override + public ClusterI modiValueEx(int rResourceId, long voffset, int length, byte[] value, int offset, ClusterSupport support) + throws DatabaseException { + int resourceIndex = getLocalReferenceAnd(rResourceId, support, ClusterStream.MODI_OPERATION); + support.modiValue(this, getClusterId(), voffset, length, value, offset); + resourceTable.setValueEx(valueTable, resourceIndex); + return this; + } + @Override + public byte[] readValueEx(int rResourceId, long voffset, int length, ClusterSupport support) + throws DatabaseException { + int resourceIndex = getLocalReference(rResourceId); + boolean isExternal = resourceTable.isValueEx(valueTable, resourceIndex); + if (!isExternal) + throw new DatabaseException("ClusterI.readValue supported only for external value. Resource key=" + rResourceId); + return support.getValueEx(resourceIndex, getClusterId(), voffset, length); + } + @Override + public long getValueSizeEx(int resourceKey, ClusterSupport support) + throws DatabaseException, ExternalValueException { + int resourceIndex = getLocalReference(resourceKey); + boolean isExternal = resourceTable.isValueEx(valueTable, resourceIndex); + if (!isExternal) + throw new ExternalValueException("ClusterI.getSize supported only for external value. Resource key=" + resourceKey); + return support.getValueSizeEx(resourceIndex, getClusterId()); + } + public boolean isValueEx(int resourceKey) + throws DatabaseException { + int resourceIndex = getLocalReference(resourceKey); + return resourceTable.isValueEx(valueTable, resourceIndex); + } + @Override + public void setValueEx(int resourceKey) + throws DatabaseException { + int resourceIndex = getLocalReference(resourceKey); + resourceTable.setValueEx(valueTable, resourceIndex); + } + @Override + public int createResource(ClusterSupport support) + throws DatabaseException { + short resourceIndex = resourceTable.createResource(); + + if(DebugPolicy.REPORT_RESOURCE_ID_ALLOCATION) + System.out.println("[RID_ALLOCATION]: ClusterBig[" + clusterId + "] allocates " + resourceIndex); + + support.createResource(this, resourceIndex, clusterId); + return ClusterTraits.createResourceKey(clusterKey, resourceIndex); + } + @Override + public boolean hasResource(int resourceKey, ClusterSupport support) { + int clusterKey = ClusterTraitsBase.getClusterKeyFromResourceKeyNoThrow(resourceKey); + if (this.clusterKey != clusterKey) // foreign resource + return false; + int resourceIndex; + try { + resourceIndex = ClusterTraits.getResourceIndexFromResourceKey(resourceKey); + } catch (DatabaseException e) { + return false; + } + if (resourceIndex > 0 & resourceIndex <= resourceTable.getTableCount()) + return true; + else + return false; + } + @Override + public int getNumberOfResources(ClusterSupport support) { + return resourceTable.getUsedSize(); + } + @Override + public long getUsedSpace() { + long rt = resourceTable.getTableCapacity() * 8 + 8; // (8 = cluster id) + long ft = foreignTable.getTableCapacity() * 8; + long pt = predicateTable.getTableCapacity() * 4; + long ot = objectTable.getTableCapacity() * 4; + long ct = completeTable.getTableCapacity() * 4; + long vt = valueTable.getTableCapacity() * 1; + long cm = clusterMap.getUsedSpace(); + + return rt + ft + pt + ot + ct + vt + cm; +// System.out.println("resource table " + rt); +// System.out.println("foreign table (non flat cluster table) " + ft); +// System.out.println("predicate table " + pt); +// long pt2 = getRealSizeOfPredicateTable() * 4; +// System.out.println("predicate table real size " + pt2); +// System.out.println("object table " + ot); +// long ot2 = getRealSizeOfObjectTable() * 4; +// System.out.println("object table real size " + ot2); +// System.out.println("value table " + vt); + } + int getRealSizeOfPredicateTable() throws DatabaseException { + SizeOfPredicateTable proc = new SizeOfPredicateTable(resourceTable, predicateTable); + resourceTable.foreachResource(proc, 0, null, null); + return proc.getSize(); + } + int getRealSizeOfObjectTable() throws DatabaseException { + SizeOfObjectTable proc = new SizeOfObjectTable(resourceTable, predicateTable, objectTable); + resourceTable.foreachResource(proc, 0, null, null); + return proc.getSize(); + } + @Override + public boolean isEmpty() { + return resourceTable.getTableCount() == 0; + } + @Override + public void printDebugInfo(String message, ClusterSupport support) + throws DatabaseException { + predicateTable.printDebugInfo(); + objectTable.printDebugInfo(); + ClusterPrintDebugInfo proc = new ClusterPrintDebugInfo(this + , resourceTable, predicateTable, support, objectTable); + resourceTable.foreachResource(proc, 0, null, null); + } + private int getInternalReferenceOrZero(int resourceKey, ClusterSupport support) + throws DatabaseException { + int clusterKey = ClusterTraits.getClusterKeyFromResourceKey(resourceKey); + int resourceIndex = ClusterTraits.getResourceIndexFromResourceKey(resourceKey); + if (this.clusterKey != clusterKey) { // foreign resource + ClusterI foreignCluster = support.getClusterByClusterKey(clusterKey); + ClusterUID clusterUID = foreignCluster.getClusterUID(); + int foreignResourceIndex = clusterMap.getForeignReferenceOrZero(resourceIndex, clusterUID); + return foreignResourceIndex; + } + return resourceIndex; + } + private int getInternalReferenceOrZeroAnd(int resourceKey, ClusterSupport support, byte op) + throws DatabaseException { + int clusterKey = ClusterTraits.getClusterKeyFromResourceKey(resourceKey); + int resourceIndex = ClusterTraits.getResourceIndexFromResourceKey(resourceKey); + if (this.clusterKey != clusterKey) { // foreign resource + ClusterUID clusterUID = clusterSupport.getClusterUIDByResourceKey(resourceKey); + int foreignResourceIndex = clusterMap.getForeignReferenceOrZero(resourceIndex, clusterUID); + support.addStatementIndex(this, resourceKey, clusterUID, op); + return foreignResourceIndex; + } + support.addStatementIndex(this, resourceKey, getClusterUID(), op); + return resourceIndex; + } + private short getLocalReference(int resourceKey) throws DatabaseException { + return ClusterTraitsBase.getResourceIndexFromResourceKeyNoThrow(resourceKey); + } + private int getLocalReferenceAnd(int resourceKey, ClusterSupport support, byte op) + throws DatabaseException { + int resourceIndex = getLocalReference(resourceKey); + support.addStatementIndex(this, resourceKey, getClusterUID(), op); + return resourceIndex; + } + private int checkResourceKeyIsOursAndGetResourceIndexIf(int resourceKey, ClusterSupport support) + throws DatabaseException { + int clusterShortId = ClusterTraits.getClusterKeyFromResourceKey(resourceKey); + if (this.clusterKey != clusterShortId) + return 0; + int resourceIndex = ClusterTraits.getResourceIndexFromResourceKey(resourceKey); + return resourceIndex; + } + private int getReferenceOrCreateIfForeign(int resourceKey, ClusterUID clusterUID, ClusterSupport support, byte op) + throws DatabaseException { + int clusterKey = ClusterTraits.getClusterKeyFromResourceKey(resourceKey); + int resourceIndex = ClusterTraits.getResourceIndexFromResourceKey(resourceKey); + if (this.clusterKey != clusterKey) { + support.addStatementIndex(this, resourceKey, clusterUID, op); + return clusterMap.getForeignReferenceOrCreateByResourceKey(resourceKey, clusterUID); + } + support.addStatementIndex(this, resourceKey, getClusterUID(), op); + return resourceIndex; + } + private int getReferenceOrCreateIfForeign(int resourceKey, ClusterSupport support, byte op) + throws DatabaseException { + int clusterKey = ClusterTraits.getClusterKeyFromResourceKey(resourceKey); + int resourceIndex = ClusterTraits.getResourceIndexFromResourceKey(resourceKey); + if (this.clusterKey != clusterKey) { + ClusterUID clusterUID = clusterSupport.getClusterUIDByResourceKey(resourceKey); + support.addStatementIndex(this, resourceKey, clusterUID, op); + return clusterMap.getForeignReferenceOrCreateByResourceKey(resourceKey, clusterUID); + } + support.addStatementIndex(this, resourceKey, getClusterUID(), op); + return resourceIndex; + } + private class ResourceIndexAndId { + ResourceIndexAndId(int reference, int index, ClusterUID clusterUID) { + this.reference = reference; + this.index = index; + this.clusterUID = clusterUID; + } + public final int reference; + public final int index; + public final ClusterUID clusterUID; + } + private ResourceIndexAndId checkResourceKeyAndGetResourceIndexIf(int resourceKey, ClusterSupport support) + throws DatabaseException { + int clusterKey = ClusterTraits.getClusterKeyFromResourceKey(resourceKey); + int resourceIndex = ClusterTraits.getResourceIndexFromResourceKey(resourceKey); + if (this.clusterKey != clusterKey) { // foreign resource + ClusterI foreignCluster = support.getClusterByClusterKey(clusterKey); + ClusterUID clusterUID = foreignCluster.getClusterUID(); + int ref = clusterMap.getForeignReferenceOrCreateByResourceIndex(resourceIndex, clusterUID); + return new ResourceIndexAndId(ref, resourceIndex, clusterUID); + } + return new ResourceIndexAndId(resourceIndex, resourceIndex, getClusterUID()); + } + + @Override + final public int execute(int resourceIndex) throws DatabaseException { + int key; + if(resourceIndex > 0) { + key = clusterBits | resourceIndex; + } else { + ClusterUID clusterUID = clusterMap.getResourceUID(resourceIndex).asCID(); + ClusterI cluster = clusterSupport.getClusterByClusterUIDOrMake(clusterUID); + int foreingResourceIndex = clusterMap.getForeignResourceIndex(resourceIndex); + key = ClusterTraits.createResourceKey(cluster.getClusterKey(), foreingResourceIndex); + } + if (DEBUG) + System.out.println("Cluster.execute key=" + key); + return key; + } + + private boolean addRelationInternal(int sReference, int pReference, int oReference, ClusterI.CompleteTypeEnum completeType) + throws DatabaseException { + int predicateIndex = resourceTable.addStatement(sReference, pReference, + oReference, predicateTable, objectTable, completeType, completeTable); + if (0 == predicateIndex) + return true; // added to resourceTable + else if (0 > predicateIndex) + return false; // old complete statemenent + int newPredicateIndex = predicateTable.addPredicate(predicateIndex, + pReference, oReference, objectTable); + if (0 == newPredicateIndex) + return false; + if (predicateIndex != newPredicateIndex) + resourceTable.setPredicateIndex(sReference, newPredicateIndex); + return true; + } + private boolean removeRelationInternal(int sResourceIndex, int pResourceIndex, + int oResourceIndex, ClusterI.CompleteTypeEnum completeType, ClusterSupport support) + throws DatabaseException { + int predicateIndex = resourceTable.getPredicateIndex(sResourceIndex); + if (0 == predicateIndex || ClusterI.CompleteTypeEnum.NotComplete != completeType) + return resourceTable.removeStatementFromCache(sResourceIndex, + pResourceIndex, oResourceIndex, completeType, completeTable); + PredicateTable.Status ret = predicateTable.removePredicate(predicateIndex, pResourceIndex, oResourceIndex, objectTable); + switch (ret) { + case NothingRemoved: + return false; + case PredicateRemoved: { + if (0 == predicateTable.getPredicateSetSize(predicateIndex)) + resourceTable.setPredicateIndex(sResourceIndex, 0); + // intentionally dropping to next case + } default: + break; + } + resourceTable.removeStatement(sResourceIndex, + pResourceIndex, oResourceIndex, + completeType, completeTable, + predicateTable, objectTable, this, support); + return true; + } + @Override + public void load() { + throw new Error("Not supported."); + } + + @Override + public void load(Callback r) { + throw new Error("Not supported."); + } + + public int makeResourceKey(int resourceIndex) throws DatabaseException { + int key = 0; + if (resourceIndex > 0) // local resource + key = ClusterTraits.createResourceKey(clusterKey, resourceIndex); + else { + ClusterUID clusterUID = clusterMap.getResourceUID(resourceIndex).asCID(); + int clusterKey = clusterSupport.getClusterKeyByClusterUIDOrMake(clusterUID); + int foreingResourceIndex = clusterMap.getForeignResourceIndex(resourceIndex); + key = ClusterTraits.createResourceKey(clusterKey, foreingResourceIndex); + } + if (0 == key) + throw new DatabaseException("Failed to make resource key from " + resourceIndex); + return key; + } + @Override + public ClusterBig toBig(ClusterSupport support) throws DatabaseException { + throw new Error("Not implemented"); + } + @Override + public void load(ClusterSupport session, Runnable callback) { + throw new Error("Not implemented"); + } + @Override + public ClusterI getClusterByResourceKey(int resourceKey, + ClusterSupport support) { + throw new Error("Not implemented"); + } + @Override + public void increaseReferenceCount(int amount) { + throw new Error("Not implemented"); + } + @Override + + public void decreaseReferenceCount(int amount) { + throw new Error("Not implemented"); + } + @Override + public int getReferenceCount() { + throw new Error("Not implemented"); + } + @Override + public void releaseMemory() { + } + @Override + public void compact() { + clusterMap.compact(); + } + public boolean contains(int resourceKey) { + return ClusterTraitsBase.isCluster(clusterBits, resourceKey); + } + @Override + public ClusterTypeEnum getType() { + return ClusterTypeEnum.BIG; + } + @Override + public boolean getImmutable() { + int status = resourceTable.getClusterStatus(); + return (status & ClusterStatus.ImmutableMaskSet) == 1; + } + @Override + public void setImmutable(boolean immutable, ClusterSupport support) { + int status = resourceTable.getClusterStatus(); + if (immutable) + status |= ClusterStatus.ImmutableMaskSet; + else + status &= ClusterStatus.ImmutableMaskClear; + resourceTable.setClusterStatus(status); + support.setImmutable(this, immutable); + } + + @Override + public ClusterTables store() throws IOException { + + ClusterTables result = new ClusterTables(); + + int[] currentHeader = Arrays.copyOf(headerTable, INT_HEADER_SIZE); + + int byteSize = valueTable.getTableSize(); + byte[] byteBytes = new byte[byteSize]; + valueTable.store(byteBytes, 0); + + //FileUtils.writeFile(bytes, valueTable.table); + + result.bytes = byteBytes; + + int longSize = LONG_HEADER_SIZE + resourceTable.getTableSize() + foreignTable.getTableSize(); + long[] longBytes = new long[longSize]; + + longBytes[0] = 0; + longBytes[1] = LONG_HEADER_VERSION; + longBytes[2] = 0; + longBytes[3] = clusterUID.second; + +// Bytes.writeLE8(longBytes, 0, 0); +// Bytes.writeLE8(longBytes, 8, LONG_HEADER_VERSION); +// Bytes.writeLE8(longBytes, 16, 0); +// Bytes.writeLE8(longBytes, 24, clusterUID.second); + + int longPos = resourceTable.store(longBytes, LONG_HEADER_SIZE); + foreignTable.store(longBytes, longPos); + + result.longs = longBytes; + +// FileUtils.writeFile(longs, longBytes); + + int intSize = INT_HEADER_SIZE + predicateTable.getTableSize() + objectTable.getTableSize() + completeTable.getTableSize(); + int[] intBytes = new int[intSize]; + int intPos = INT_HEADER_SIZE; + intPos = predicateTable.store(intBytes, intPos); + intPos = objectTable.store(intBytes, intPos); + intPos = completeTable.store(intBytes, intPos); + // write header + for(int i=0;i getPredicateTable() { + return predicateTable; + } + @Override + public Table getForeignTable() { + return foreignTable; + } + @Override + public Table getCompleteTable() { + return completeTable; + } + @Override + public Table getValueTable() { + return valueTable; + } + @Override + public Table getObjectTable() { + return objectTable; + } +} + +class SizeOfPredicateTable implements ClusterI.ObjectProcedure { + private final ResourceTable mrResourceTable; + private final PredicateTable mrPredicateTable; + private int size = 0; + SizeOfPredicateTable(ResourceTable resourceTable + , PredicateTable predicateTable) { + mrResourceTable = resourceTable; + mrPredicateTable = predicateTable; + } + @Override + public boolean execute(Integer i, int resourceRef) { + int predicateIndex = mrResourceTable.getPredicateIndex(resourceRef); + if (0 == predicateIndex) + return false; // continue loop + size += mrPredicateTable.getPredicateSetSize(predicateIndex); + return false; // continue loop + } + + public int getSize() { + return size; + } + +} + +class SizeOfObjectTable implements ClusterI.ObjectProcedure { + private final ResourceTable mrResourceTable; + private final PredicateTable mrPredicateTable; + private final ObjectTable mrObjectTable; + private int size = 0; + SizeOfObjectTable(ResourceTable resourceTable + , PredicateTable predicateTable, ObjectTable objectTable) { + mrResourceTable = resourceTable; + mrPredicateTable = predicateTable; + mrObjectTable = objectTable; + } + + @Override + public boolean execute(Integer i, int resourceRef) { + int predicateIndex = mrResourceTable.getPredicateIndex(resourceRef); + if (0 == predicateIndex) + return false; // continue loop + ClusterI.PredicateProcedure procedure = new PredicateProcedure() { + @Override + public boolean execute(Object context, int pRef, int oIndex) { + if (ClusterTraits.statementIndexIsDirect(oIndex)) + return false; // no table space reserved, continue looping + int objectIndex; + try { + objectIndex = ClusterTraits.statementIndexGet(oIndex); + size += mrObjectTable.getObjectSetSize(objectIndex); + } catch (DatabaseException e) { + e.printStackTrace(); + } + return false; // continue looping + } + }; + try { + mrPredicateTable.foreachPredicate(predicateIndex, procedure, null, null, null); + } catch (DatabaseException e) { + e.printStackTrace(); + } + return false; // continue loop + } + + public int getSize() { + return size; + } + +} \ No newline at end of file diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/cluster/ClusterImpl.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/cluster/ClusterImpl.java new file mode 100644 index 000000000..353d9382a --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/cluster/ClusterImpl.java @@ -0,0 +1,226 @@ +/******************************************************************************* + * Copyright (c) 2007, 2010 Association for Decentralized Information Management + * in Industry THTH ry. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * VTT Technical Research Centre of Finland - initial API and implementation + *******************************************************************************/ +package org.simantics.acorn.cluster; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.UUID; + +import org.simantics.acorn.internal.Change; +import org.simantics.acorn.internal.ClusterChange; +import org.simantics.acorn.internal.ClusterSupport2; +import org.simantics.db.common.utils.Logger; +import org.simantics.db.exception.DatabaseException; +import org.simantics.db.exception.InvalidClusterException; +import org.simantics.db.impl.ClusterBase; +import org.simantics.db.impl.ClusterI; +import org.simantics.db.impl.ClusterSupport; +import org.simantics.db.impl.IClusterTable; +import org.simantics.db.impl.Modifier; +import org.simantics.db.service.ClusterCollectorPolicy.CollectorCluster; +import org.simantics.db.service.ClusterUID; +import org.simantics.db.service.ClusteringSupport.Id; +import org.simantics.utils.strings.AlphanumComparator; + +public abstract class ClusterImpl extends ClusterBase implements Modifier, CollectorCluster { + protected static final int LONG_HEADER_SIZE = 7; + protected static final long LONG_HEADER_VERSION = 1; + protected static ClusterUID checkValidity(long type, long[] longs, int[] ints, byte[] bytes) + throws InvalidClusterException { + if (longs.length < LONG_HEADER_SIZE) + throw new InvalidClusterException("Header size mismatch. Expected=" + ClusterImpl.LONG_HEADER_SIZE + ", got=" + longs.length); + if (longs[0] != type) + throw new InvalidClusterException("Type mismatch. Expected=" + type + ", got=" + longs[0] + " " + ClusterUID.make(longs[2], longs[3])); + if (longs[1] != ClusterImpl.LONG_HEADER_VERSION) + throw new InvalidClusterException("Header size mismatch. Expected=" + ClusterImpl.LONG_HEADER_VERSION + ", got=" + longs[1]); + return ClusterUID.make(longs[2], longs[3]); + } + protected static Id getUniqueId(long[] longs) { + return new IdImpl(new UUID(longs[3], longs[4])); + } + static final boolean DEBUG = false; + final public IClusterTable clusterTable; + // This can be null iff the cluster has been converted to big + public Change change = new Change(); + public ClusterChange cc; + public byte[] foreignLookup; + + private boolean dirtySizeInBytes = true; + private long sizeInBytes = 0; + + protected ClusterImpl() { + clusterTable = null; + } + + public ClusterImpl(IClusterTable clusterTable, ClusterUID clusterUID, int clusterKey, ClusterSupport support) { + super(support, clusterUID, clusterKey); +// SessionImplSocket session = (SessionImplSocket)support.getSession(); +// if(session != null) + this.clusterTable = clusterTable; +// else + } + + public static ClusterImpl dummy() { + return new ClusterSmall(); + } + + public static ClusterImpl make(IClusterTable clusterTable, ClusterUID clusterUID, int clusterKey, ClusterSupport2 support) { + return new ClusterSmall(clusterUID, clusterKey, support, clusterTable); + } + public static ClusterSmall proxy(IClusterTable clusterTable, ClusterUID clusterUID, int clusterKey, long clusterId, ClusterSupport2 support) { + if (DEBUG) + new Exception("Cluster proxy for " + clusterUID).printStackTrace(); + return new ClusterSmall(null, clusterUID, clusterKey, support); + } + public static ClusterImpl make(IClusterTable clusterTable, long[] longs, int[] ints, byte[] bytes, ClusterSupport2 support, int clusterKey) + throws DatabaseException { + if (longs[0] == 0) + return new ClusterBig(clusterTable, longs, ints, bytes, support, clusterKey); + else + return new ClusterSmall(clusterTable, longs, ints, bytes, support, clusterKey); + } + +// public boolean virtual = false; + + @Override + public boolean hasVirtual() { + return false; +// return clusterTable.hasVirtual(clusterKey); + } + + @Override + public void markVirtual() { +// clusterTable.markVirtual(clusterKey); +// virtual = true; + } + + @Override + public boolean isWriteOnly() { + return false; + } + @Override + public boolean isLoaded() { + return true; + } + + @Override + public void resized() { + dirtySizeInBytes = true; +// if(clusterTable != null) +// clusterTable.setDirtySizeInBytes(true); + } + + public long getCachedSize() { + if(dirtySizeInBytes) { + try { + sizeInBytes = getUsedSpace(); + //System.err.println("recomputed size of cluster " + getClusterId() + " => " + sizeInBytes); + } catch (DatabaseException e) { + Logger.defaultLogError(e); + } + dirtySizeInBytes = false; + } + return sizeInBytes; + } + + protected void calculateModifiedId() { +// setModifiedId(new IdImpl(UUID.randomUUID())); + } + + public static class ClusterTables { + public byte[] bytes; + public int[] ints; + public long[] longs; + } + + public byte[] storeBytes() throws IOException { + throw new UnsupportedOperationException(); + } + + public ClusterTables store() throws IOException { + throw new UnsupportedOperationException(); + } + + abstract protected int getResourceTableCount(); + + public String dump(final ClusterSupport support) { + + StringBuilder sb = new StringBuilder(); + for(int i=1;i stms = new ArrayList(); + try { + + byte[] value = getValue(i, support); + if(value != null) + sb.append(" bytes: " + Arrays.toString(value) + "\n"); + + forPredicates(i, new PredicateProcedure() { + + @Override + public boolean execute(Integer c, final int predicateKey, int objectIndex) { + + try { + + forObjects(resourceKey, predicateKey, objectIndex, new ObjectProcedure() { + + @Override + public boolean execute(Integer context, int objectKey) throws DatabaseException { + + ClusterUID puid = support.getClusterByResourceKey(predicateKey).getClusterUID(); + ClusterUID ouid = support.getClusterByResourceKey(objectKey).getClusterUID(); + + stms.add(" " + puid + " " + (predicateKey&0xFFF) + " " + ouid + " " + (objectKey&0xFFF)); + + return false; + + } + + }, 0, support); + } catch (DatabaseException e) { + e.printStackTrace(); + } + + return false; + + } + + },0,support); + + Collections.sort(stms, AlphanumComparator.COMPARATOR); + + for(String s : stms) { + sb.append(s); + sb.append("\n"); + } + + } catch (DatabaseException e) { + e.printStackTrace(); + } + } + + return sb.toString(); + + } + + abstract public boolean isValueEx(int resourceIndex) throws DatabaseException; + + abstract public ClusterI addRelation(int resourceKey, ClusterUID puid, int predicateKey, ClusterUID ouid, int objectKey, ClusterSupport support) throws DatabaseException; + + @Override + public IClusterTable getClusterTable() { + return clusterTable; + } +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/cluster/ClusterSmall.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/cluster/ClusterSmall.java new file mode 100644 index 000000000..b84d4d51f --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/cluster/ClusterSmall.java @@ -0,0 +1,1309 @@ +/******************************************************************************* + * Copyright (c) 2007, 2010 Association for Decentralized Information Management + * in Industry THTH ry. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * VTT Technical Research Centre of Finland - initial API and implementation + *******************************************************************************/ +package org.simantics.acorn.cluster; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; + +import org.simantics.acorn.exception.IllegalAcornStateException; +import org.simantics.acorn.internal.ClusterChange; +import org.simantics.acorn.internal.ClusterStream; +import org.simantics.acorn.internal.ClusterSupport2; +import org.simantics.acorn.internal.DebugPolicy; +import org.simantics.db.Resource; +import org.simantics.db.common.utils.Logger; +import org.simantics.db.exception.DatabaseException; +import org.simantics.db.exception.ExternalValueException; +import org.simantics.db.exception.ValidationException; +import org.simantics.db.impl.ClusterI; +import org.simantics.db.impl.ClusterSupport; +import org.simantics.db.impl.ClusterTraitsBase; +import org.simantics.db.impl.ForEachObjectContextProcedure; +import org.simantics.db.impl.ForEachObjectProcedure; +import org.simantics.db.impl.ForPossibleRelatedValueContextProcedure; +import org.simantics.db.impl.ForPossibleRelatedValueProcedure; +import org.simantics.db.impl.IClusterTable; +import org.simantics.db.impl.Table; +import org.simantics.db.impl.TableHeader; +import org.simantics.db.impl.graph.ReadGraphImpl; +import org.simantics.db.procedure.AsyncContextMultiProcedure; +import org.simantics.db.procedure.AsyncMultiProcedure; +import org.simantics.db.procore.cluster.ClusterMapSmall; +import org.simantics.db.procore.cluster.ClusterTraits; +import org.simantics.db.procore.cluster.ClusterTraitsSmall; +import org.simantics.db.procore.cluster.CompleteTableSmall; +import org.simantics.db.procore.cluster.ForeignTableSmall; +import org.simantics.db.procore.cluster.ObjectTable; +import org.simantics.db.procore.cluster.OutOfSpaceException; +import org.simantics.db.procore.cluster.PredicateTable; +import org.simantics.db.procore.cluster.ResourceTableSmall; +import org.simantics.db.procore.cluster.ValueTableSmall; +import org.simantics.db.service.Bytes; +import org.simantics.db.service.ClusterUID; +import org.simantics.db.service.ResourceUID; +import org.simantics.utils.datastructures.Callback; + +import gnu.trove.map.hash.TIntShortHashMap; +import gnu.trove.procedure.TIntProcedure; +import gnu.trove.set.hash.TIntHashSet; + +final public class ClusterSmall extends ClusterImpl { + private static final int TABLE_HEADER_SIZE = TableHeader.HEADER_SIZE + TableHeader.EXTRA_SIZE; + private static final int RESOURCE_TABLE_OFFSET = 0; + private static final int PREDICATE_TABLE_OFFSET = RESOURCE_TABLE_OFFSET + TABLE_HEADER_SIZE; + private static final int OBJECT_TABLE_OFFSET = PREDICATE_TABLE_OFFSET + TABLE_HEADER_SIZE; + private static final int VALUE_TABLE_OFFSET = OBJECT_TABLE_OFFSET + TABLE_HEADER_SIZE; + private static final int FLAT_TABLE_OFFSET = VALUE_TABLE_OFFSET + TABLE_HEADER_SIZE; + private static final int COMPLETE_TABLE_OFFSET = FLAT_TABLE_OFFSET + TABLE_HEADER_SIZE; + private static final int FOREIGN_TABLE_OFFSET = COMPLETE_TABLE_OFFSET + TABLE_HEADER_SIZE; + private static final int INT_HEADER_SIZE = FOREIGN_TABLE_OFFSET + TABLE_HEADER_SIZE; + private final int clusterBits; + private final ResourceTableSmall resourceTable; + private final PredicateTable predicateTable; + private final ObjectTable objectTable; + private final ValueTableSmall valueTable; + private final ForeignTableSmall foreignTable; + private final CompleteTableSmall completeTable; + private final ClusterMapSmall clusterMap; + private final int[] headerTable; + public final ClusterSupport2 clusterSupport; + private boolean proxy; + private boolean deleted = false; + + protected ClusterSmall() { + this.proxy = true; + this.headerTable = null; + this.resourceTable = null; + this.foreignTable = null; + this.predicateTable = null; + this.objectTable = null; + this.valueTable = null; + this.completeTable = null; + this.clusterMap = null; + this.clusterSupport = null; + this.clusterBits = 0; + this.importance = 0; + } + + public ClusterSmall(IClusterTable clusterTable, ClusterUID clusterUID, int clusterKey, ClusterSupport2 support) { + super(clusterTable, clusterUID, clusterKey, support); + if(DebugPolicy.REPORT_CLUSTER_EVENTS) + new Exception(clusterUID.toString()).printStackTrace(); + this.proxy = true; + this.headerTable = null; + this.resourceTable = null; + this.foreignTable = null; + this.predicateTable = null; + this.objectTable = null; + this.valueTable = null; + this.completeTable = null; + this.clusterMap = null; + this.clusterSupport = support; + this.clusterBits = 0; + this.importance = 0; +// new Exception("ClusterSmall " + clusterKey).printStackTrace(); + } + ClusterSmall(ClusterUID clusterUID, int clusterKey, ClusterSupport2 support, IClusterTable clusterTable) { + super(clusterTable, clusterUID, clusterKey, support); + if(DebugPolicy.REPORT_CLUSTER_EVENTS) + new Exception(clusterUID.toString()).printStackTrace(); + this.proxy = false; + this.clusterSupport = support; + this.headerTable = new int[INT_HEADER_SIZE]; + this.resourceTable = new ResourceTableSmall(this, headerTable, RESOURCE_TABLE_OFFSET); + this.foreignTable = new ForeignTableSmall(this, headerTable, FOREIGN_TABLE_OFFSET); + this.predicateTable = new PredicateTable(this, headerTable, PREDICATE_TABLE_OFFSET); + this.objectTable = new ObjectTable(this, headerTable, OBJECT_TABLE_OFFSET); + this.valueTable = new ValueTableSmall(this, headerTable, VALUE_TABLE_OFFSET); + this.completeTable = new CompleteTableSmall(this, headerTable, COMPLETE_TABLE_OFFSET); + this.clusterMap = new ClusterMapSmall(this, foreignTable); + this.clusterBits = ClusterTraitsBase.getClusterBits(clusterKey); +// if(clusterTable != null) +// this.importance = -clusterTable.timeCounter(); +// else + this.importance = 0; +// new Exception("ClusterSmall " + clusterKey).printStackTrace(); + } + protected ClusterSmall(IClusterTable clusterTable, long[] longs, int[] ints, byte[] bytes, ClusterSupport2 support, int clusterKey) + throws DatabaseException { + super(clusterTable, checkValidity(-1, longs, ints, bytes), clusterKey, support); + this.proxy = false; + this.clusterSupport = support; + if (ints.length < INT_HEADER_SIZE) + throw new IllegalArgumentException("Too small integer table for cluster."); + this.headerTable = ints; + if(DebugPolicy.REPORT_CLUSTER_EVENTS) new Exception(Long.toString(clusterId)).printStackTrace(); + this.resourceTable = new ResourceTableSmall(this, ints, RESOURCE_TABLE_OFFSET, longs); + this.foreignTable = new ForeignTableSmall(this, headerTable, FOREIGN_TABLE_OFFSET, longs); + this.predicateTable = new PredicateTable(this, ints, PREDICATE_TABLE_OFFSET, ints); + this.objectTable = new ObjectTable(this, ints, OBJECT_TABLE_OFFSET, ints); + this.valueTable = new ValueTableSmall(this, ints, VALUE_TABLE_OFFSET, bytes); + this.completeTable = new CompleteTableSmall(this, headerTable, COMPLETE_TABLE_OFFSET, ints); + this.clusterMap = new ClusterMapSmall(this, foreignTable); + this.clusterBits = ClusterTraitsBase.getClusterBits(clusterKey); +// if(clusterTable != null) { +// this.importance = clusterTable.timeCounter(); +// clusterTable.markImmutable(this, getImmutable()); +// } +// new Exception("ClusterSmall " + clusterKey).printStackTrace(); + } + void analyse() { + System.out.println("Cluster " + clusterId); + System.out.println("-size:" + getUsedSpace()); + System.out.println(" -rt:" + (resourceTable.getTableCapacity() * 8 + 8)); + System.out.println(" -ft:" + foreignTable.getTableCapacity() * 8); + System.out.println(" -pt:" + predicateTable.getTableCapacity() * 4); + System.out.println(" -ot:" + objectTable.getTableCapacity() * 4); + System.out.println(" -ct:" + completeTable.getTableCapacity() * 4); + System.out.println(" -vt:" + valueTable.getTableCapacity()); + + System.out.println("-resourceTable:"); + System.out.println(" -resourceCount=" + resourceTable.getResourceCount()); + System.out.println(" -size=" + resourceTable.getTableSize()); + System.out.println(" -capacity=" + resourceTable.getTableCapacity()); + System.out.println(" -count=" + resourceTable.getTableCount()); + System.out.println(" -size=" + resourceTable.getTableSize()); + //resourceTable.analyse(); + } + public void checkDirectReference(int dr) + throws DatabaseException { + if (!ClusterTraits.statementIndexIsDirect(dr)) + throw new ValidationException("Reference is not direct. Reference=" + dr); + if (ClusterTraits.isFlat(dr)) + throw new ValidationException("Reference is flat. Reference=" + dr); + if (ClusterTraits.isLocal(dr)) { + if (dr < 1 || dr > resourceTable.getUsedSize()) + throw new ValidationException("Illegal local reference. Reference=" + dr); + } else { + int fi = ClusterTraits.getForeignIndexFromReference(dr); + int ri = ClusterTraits.getResourceIndexFromForeignReference(dr); + if (fi < 1 || fi > foreignTable.getUsedSize()) + throw new ValidationException("Illegal foreign reference. Reference=" + dr + " foreign index=" + fi); + if (ri < 1 || ri > ClusterTraits.getMaxNumberOfResources()) + throw new ValidationException("Illegal foreign reference. Reference=" + dr + " resource index=" + ri); + } + } + public void checkPredicateIndex(int pi) + throws DatabaseException { + // predicateTable.checkPredicateSetIndex(this, pi); + } + public void checkObjectSetReference(int or) + throws DatabaseException { + if (ClusterTraits.statementIndexIsDirect(or)) + throw new ValidationException("Illegal object set reference. Reference=" + or); + int oi = ClusterTraits.statementIndexGet(or); + this.objectTable.checkObjectSetIndex(this, oi); + } + + public void checkValueInit() + throws DatabaseException { + valueTable.checkValueInit(); + } + public void checkValue(int capacity, int index) + throws DatabaseException { + valueTable.checkValue(capacity, index); + } + public void checkValueFini() + throws DatabaseException { + valueTable.checkValueFini(); + } + public void checkForeingIndex(int fi) + throws DatabaseException { + if (fi<1 || fi > foreignTable.getUsedSize()) + throw new ValidationException("Illegal foreign index=" + fi); + } + public void checkCompleteSetReference(int cr) + throws DatabaseException { + if (!ClusterTraits.completeReferenceIsMultiple(cr)) + throw new ValidationException("Illegal complete set reference. Reference=" + cr); + int ci = cr; + this.completeTable.checkCompleteSetIndex(this, ci); + } + public void check() + throws DatabaseException { +// this.completeTable.check(this); +// this.objectTable.check(this); +// // Must be after object table check. +// this.predicateTable.check(this); +// this.resourceTable.check(this); + } + @Override + public CompleteTypeEnum getCompleteType(int resourceKey, ClusterSupport support) + throws DatabaseException { + final int resourceRef = getLocalReference(resourceKey); + CompleteTypeEnum ct = resourceTable.getCompleteType(resourceRef); + if (DEBUG) + System.out.println("ClusterSmall.getCompleteType rk=" + resourceKey + " ct=" + ct); + return ct; + } + + @Override + public int getCompleteObjectKey(int resourceKey, ClusterSupport support) + throws DatabaseException { + final int resourceIndexOld = getLocalReference(resourceKey); + short completeRef = resourceTable.getCompleteObjectRef(resourceIndexOld); + int clusterIndex; + int resourceIndex; + if (0 == completeRef) + throw new DatabaseException("Resource's complete object refernce is null. Resource key=" + resourceKey + "."); + ClusterI.CompleteTypeEnum completeType = resourceTable.getCompleteType(resourceIndexOld); + if (completeType == ClusterI.CompleteTypeEnum.NotComplete) + throw new DatabaseException("Resource has multiple complete objects. Resource key=" + resourceKey + "."); + if (ClusterTraitsSmall.resourceRefIsLocal(completeRef)) { + clusterIndex = clusterKey; + resourceIndex = completeRef; + } else { // Resource has one complete statement. + ResourceUID resourceUID = clusterMap.getForeignResourceUID(completeRef); + ClusterUID uid = resourceUID.asCID(); + clusterIndex = clusterSupport.getClusterKeyByUID(0, uid.second); + //ClusterI c = clusterTable.getClusterByClusterUIDOrMakeProxy(uid); + //clusterIndex = c.getClusterKey(); + //assert(clusterIndex == clusterTable.getClusterByClusterUIDOrMakeProxy(uid).getClusterKey()); + resourceIndex = resourceUID.getIndex(); + } + int key = ClusterTraits.createResourceKey(clusterIndex, resourceIndex); + if (DEBUG) + System.out.println("ClusterSmall.complete object rk=" + resourceKey + " ck=" + key); + return key; + } + + @Override + public boolean isComplete(int resourceKey, ClusterSupport support) + throws DatabaseException { + final int resourceRef = getLocalReference(resourceKey); + final ClusterI.CompleteTypeEnum completeType = resourceTable.getCompleteType(resourceRef); + boolean complete = completeType != ClusterI.CompleteTypeEnum.NotComplete; + if (DEBUG) + System.out.println("ClusterSmall.key=" + resourceKey + " isComplete=" + complete); + return complete; + } + public int getSingleObject(int resourceKey, int predicateKey, int objectIndex, ClusterSupport support) throws DatabaseException { + if (DEBUG) + System.out.println("ClusterSmall.getSingleObject: rk=" + resourceKey + " pk=" + predicateKey); + if (0 == objectIndex) { + final int resourceIndex = ClusterTraitsBase.getResourceIndexFromResourceKeyNoThrow(resourceKey); + final short pRef = getInternalReferenceOrZero2(predicateKey, support); + final ClusterI.CompleteTypeEnum pCompleteType = ClusterTraitsBase.getCompleteTypeFromResourceKey(predicateKey); + return resourceTable.getSingleObject(resourceIndex, support, pRef, pCompleteType, completeTable, this); + } + return objectTable.getSingleObject(objectIndex, support, this); + } + + public void forObjects(ReadGraphImpl graph, int resourceKey, int predicateKey, int objectIndex, AsyncMultiProcedure procedure, + ClusterSupport support) throws DatabaseException { + if (DEBUG) + System.out.println("ClusterSmall.forObjects1: rk=" + resourceKey + " pk=" + predicateKey); + if (0 == objectIndex) { + final int resourceIndex = ClusterTraitsBase.getResourceIndexFromResourceKey(resourceKey); + final int pRef = getInternalReferenceOrZero2(predicateKey, support); + final ClusterI.CompleteTypeEnum pCompleteType = ClusterTraitsBase.getCompleteTypeFromResourceKey(predicateKey); + resourceTable.foreachObject(resourceIndex, graph, procedure, support, pRef, pCompleteType, completeTable, this); + return; + } + objectTable.foreachObject(graph, objectIndex, procedure, this); + } + + public void forObjects(ReadGraphImpl graph, int resourceKey, int predicateKey, int objectIndex, C context, AsyncContextMultiProcedure procedure, + ClusterSupport support) throws DatabaseException { + if (DEBUG) + System.out.println("ClusterSmall.forObjects1: rk=" + resourceKey + " pk=" + predicateKey); + if (0 == objectIndex) { + final int resourceIndex = ClusterTraitsBase.getResourceIndexFromResourceKey(resourceKey); + final int pRef = getInternalReferenceOrZero2(predicateKey, support); + final ClusterI.CompleteTypeEnum pCompleteType = ClusterTraitsBase.getCompleteTypeFromResourceKey(predicateKey); + resourceTable.foreachObject(resourceIndex, graph, context, procedure, support, pRef, pCompleteType, completeTable, this); + return; + } + objectTable.foreachObject(graph, objectIndex, context, procedure, this); + } + + @Override + public boolean forObjects(int resourceKey, int predicateKey, int objectIndex, ObjectProcedure procedure, + Context context, ClusterSupport support) throws DatabaseException { + if (DEBUG) + System.out.println("ClusterSmall.forObjects2: rk=" + resourceKey + " pk=" + predicateKey); + if (0 == objectIndex) { + final int resourceIndex = ClusterTraitsBase.getResourceIndexFromResourceKey(resourceKey); + final short pRef = getInternalReferenceOrZero2(predicateKey, support); + final ClusterI.CompleteTypeEnum pCompleteType = ClusterTraitsBase.getCompleteTypeFromResourceKey(predicateKey); + return resourceTable.foreachObject(resourceIndex, procedure, context, support, this, pRef, pCompleteType, completeTable); + } + return objectTable.foreachObject(objectIndex, procedure, context, support, this); + } + + @Override + public int getSingleObject(int resourceKey, int predicateKey, ClusterSupport support) throws DatabaseException { + if (DEBUG) + System.out.println("ClusterSmall.getSingleObject2: rk=" + resourceKey + " pk=" + predicateKey); + final int resourceIndex = ClusterTraitsBase.getResourceIndexFromResourceKey(resourceKey); + final short pRef = getInternalReferenceOrZero2(predicateKey, support); + final int completeType = ClusterTraitsBase.getCompleteTypeIntFromResourceKey(predicateKey); + final ClusterI.CompleteTypeEnum pCompleteType = CompleteTypeEnum.make(completeType); + if (completeType > 0) + return resourceTable.getSingleObject(resourceIndex, support, pRef, pCompleteType, completeTable, this); + final int predicateIndex = (int)resourceTable.table[(resourceIndex<<1) - 1 + resourceTable.offset] & 0xFFFFFF; + if (0 == predicateIndex) // All relevant data is in resource table. + return resourceTable.getSingleObject(resourceIndex, support, pRef, pCompleteType, completeTable, this); + int objectIndex = predicateTable.getObjectIndex(predicateIndex, pRef & 0xFFFF); + return getSingleObject(resourceKey, predicateKey, objectIndex, support); + } + + @Override + public int getSingleObject(int resourceKey, ForPossibleRelatedValueProcedure procedure, ClusterSupport support) throws DatabaseException { + final short resourceIndex = (short)ClusterTraitsBase.getResourceIndexFromResourceKey(resourceKey); + final int predicateKey = procedure.predicateKey; + int clusterKey = ClusterTraitsBase.getClusterMaskFromResourceKey(resourceKey); + short pRef = 0; + if(procedure.clusterKey[0] == clusterKey) { + pRef = (short)procedure.predicateReference[0]; + } else { + pRef = getInternalReferenceOrZero2(predicateKey, support); + procedure.clusterKey[0] = clusterKey; + procedure.predicateReference[0] = pRef; + } + + final ClusterI.CompleteTypeEnum pCompleteType = procedure.completeType; + if (CompleteTypeEnum.NotComplete != pCompleteType) + return resourceTable.getSingleObject(resourceIndex, support, pRef, pCompleteType, completeTable, this); + final int predicateIndex = (int)resourceTable.table[(resourceIndex<<1) - 1 + resourceTable.offset] & 0xFFFFFF; + if (0 == predicateIndex) // All relevant data is in resource table. + return resourceTable.getSingleObject(resourceIndex, support, pRef, pCompleteType, completeTable, this); + int objectIndex = predicateTable.getObjectIndex(predicateIndex, pRef & 0xFFFF); + return getSingleObject(resourceKey, predicateKey, objectIndex, support); + } + + @Override + public int getSingleObject(int resourceKey, ForPossibleRelatedValueContextProcedure procedure, ClusterSupport support) throws DatabaseException { + final short resourceIndex = (short)ClusterTraitsBase.getResourceIndexFromResourceKey(resourceKey); + final int predicateKey = procedure.predicateKey; + int clusterKey = ClusterTraitsBase.getClusterMaskFromResourceKey(resourceKey); + short pRef = 0; + if(procedure.clusterKey[0] == clusterKey) { + pRef = (short)procedure.predicateReference[0]; + } else { + pRef = getInternalReferenceOrZero2(predicateKey, support); + procedure.clusterKey[0] = clusterKey; + procedure.predicateReference[0] = pRef; + } + final ClusterI.CompleteTypeEnum pCompleteType = procedure.completeType; + if (CompleteTypeEnum.NotComplete != pCompleteType) + return resourceTable.getSingleObject(resourceIndex, support, pRef, pCompleteType, completeTable, this); + final int predicateIndex = (int)resourceTable.table[(resourceIndex<<1) - 1 + resourceTable.offset] & 0xFFFFFF; + if (0 == predicateIndex) // All relevant data is in resource table. + return resourceTable.getSingleObject(resourceIndex, support, pRef, pCompleteType, completeTable, this); + int objectIndex = predicateTable.getObjectIndex(predicateIndex, pRef & 0xFFFF); + return getSingleObject(resourceKey, predicateKey, objectIndex, support); + } + + @Override + public void forObjects(ReadGraphImpl graph, int resourceKey, + int predicateKey, AsyncMultiProcedure procedure) throws DatabaseException { + + throw new UnsupportedOperationException(); + +// SessionImplSocket session = (SessionImplSocket)graph.getSession(); +// ClusterSupport support = session.clusterTranslator; +// if (DEBUG) +// System.out.println("ClusterSmall.forObjects3: rk=" + resourceKey + " pk=" + predicateKey); +// final int resourceIndex = ClusterTraitsBase.getResourceIndexFromResourceKey(resourceKey); +// final int pRef = getInternalReferenceOrZero2(predicateKey, support); +// final int completeType = ClusterTraitsBase.getCompleteTypeIntFromResourceKey(predicateKey); +// final ClusterI.CompleteTypeEnum pCompleteType = CompleteTypeEnum.make(completeType); +// if (completeType > 0) { +// resourceTable.foreachObject(resourceIndex, graph, procedure, support, pRef, pCompleteType, completeTable, this); +// return; +// } +// final int predicateIndex = (int)resourceTable.table[(resourceIndex<<1) - 1 + resourceTable.offset] & 0xFFFFFF; +// if (0 == predicateIndex) { +// resourceTable.foreachObject(resourceIndex, graph, procedure, support, pRef, pCompleteType, completeTable, this); +// return; +// } +// int objectIndex = predicateTable.getObjectIndex(predicateIndex, pRef & 0xFFFF); +// forObjects(graph, resourceKey, predicateKey, objectIndex, procedure, support); + } + + public void forObjects(ReadGraphImpl graph, int resourceKey, ForEachObjectProcedure procedure) throws DatabaseException { + + throw new UnsupportedOperationException(); + +// final int resourceIndex = ClusterTraitsBase.getResourceIndexFromResourceKey(resourceKey); +// final int predicateKey = procedure.predicateKey; +// int clusterKey = ClusterTraitsBase.getClusterMaskFromResourceKey(resourceKey); +// int pRef = 0; +// if(procedure.clusterKey[0] == clusterKey) { +// pRef = procedure.predicateReference[0]; +// } else { +// SessionImplSocket session = (SessionImplSocket)graph.getSession(); +// ClusterSupport support = session.clusterTranslator; +// pRef = getInternalReferenceOrZero2(predicateKey, support); +// procedure.clusterKey[0] = clusterKey; +// procedure.predicateReference[0] = pRef; +// } +// final ClusterI.CompleteTypeEnum pCompleteType = procedure.completeType; +// if (ClusterI.CompleteTypeEnum.NotComplete != pCompleteType) { +// SessionImplSocket session = (SessionImplSocket)graph.getSession(); +// ClusterSupport support = session.clusterTranslator; +// resourceTable.foreachObject(resourceIndex, graph, procedure, support, pRef, pCompleteType, completeTable, this); +// return; +// } +// final int predicateIndex = (int)resourceTable.table[(resourceIndex<<1) - 1 + resourceTable.offset] & 0xFFFFFF; +// if (0 == predicateIndex) { +// SessionImplSocket session = (SessionImplSocket)graph.getSession(); +// ClusterSupport support = session.clusterTranslator; +// resourceTable.foreachObject(resourceIndex, graph, procedure, support, pRef, pCompleteType, completeTable, this); +// return; +// } +// int hashBase = predicateIndex + predicateTable.offset; +// if (predicateTable.table[hashBase-1] < 0) { +// int objectIndex = TableIntArraySet2.get(predicateTable.table, hashBase, pRef & 0xFFFF); +// //int objectIndex = predicateTable.getObjectIndex(predicateIndex, pRef & 0xFFFF); +// SessionImplSocket session = (SessionImplSocket)graph.getSession(); +// ClusterSupport support = session.clusterTranslator; +// forObjects(graph, resourceKey, predicateKey, objectIndex, procedure, support); +// } else { +// procedure.finished(graph); +//// graph.dec(); +// } + } + + public void forObjects(ReadGraphImpl graph, int resourceKey, C context, ForEachObjectContextProcedure procedure) throws DatabaseException { + + throw new UnsupportedOperationException(); + +// final int resourceIndex = ClusterTraitsBase.getResourceIndexFromResourceKey(resourceKey); +// final int predicateKey = procedure.predicateKey; +// int clusterKey = ClusterTraitsBase.getClusterMaskFromResourceKey(resourceKey); +// int pRef = 0; +// if(procedure.clusterKey[0] == clusterKey) { +// pRef = procedure.predicateReference[0]; +// } else { +// SessionImplSocket session = (SessionImplSocket)graph.getSession(); +// ClusterSupport support = session.clusterTranslator; +// pRef = getInternalReferenceOrZero2(predicateKey, support); +// procedure.clusterKey[0] = clusterKey; +// procedure.predicateReference[0] = pRef; +// } +// +// final ClusterI.CompleteTypeEnum pCompleteType = procedure.completeType; +// if (ClusterI.CompleteTypeEnum.NotComplete != pCompleteType) { +// SessionImplSocket session = (SessionImplSocket)graph.getSession(); +// ClusterSupport support = session.clusterTranslator; +// resourceTable.foreachObject(resourceIndex, graph, context, procedure, support, pRef, pCompleteType, completeTable, this); +// return; +// } +// final int predicateIndex = (int)resourceTable.table[(resourceIndex<<1) - 1 + resourceTable.offset] & 0xFFFFFF; +// if (0 == predicateIndex) { +// SessionImplSocket session = (SessionImplSocket)graph.getSession(); +// ClusterSupport support = session.clusterTranslator; +// resourceTable.foreachObject(resourceIndex, graph, context, procedure, support, pRef, pCompleteType, completeTable, this); +// return; +// } +// int hashBase = predicateIndex + predicateTable.offset; +// if(predicateTable.table[hashBase-1] < 0) { +// int objectIndex = TableIntArraySet2.get(predicateTable.table, hashBase, pRef & 0xFFFF); +// SessionImplSocket session = (SessionImplSocket)graph.getSession(); +// ClusterSupport support = session.clusterTranslator; +// forObjects(graph, resourceKey, predicateKey, objectIndex, context, procedure, support); +// } else { +// int objectIndex = TableIntSet2.get(predicateTable.table, hashBase, pRef & 0xFFFF); +// SessionImplSocket session = (SessionImplSocket)graph.getSession(); +// ClusterSupport support = session.clusterTranslator; +// forObjects(graph, resourceKey, predicateKey, objectIndex, context, procedure, support); +// } + } + @Override + public boolean forObjects(int resourceKey, int predicateKey, + ObjectProcedure procedure, Context context, ClusterSupport support) + throws DatabaseException { + if (DEBUG) + System.out.println("ClusterSmall.forObjects4: rk=" + resourceKey + " pk=" + predicateKey); + final short resourceIndex = (short)ClusterTraitsBase.getResourceIndexFromResourceKey(resourceKey); + final short pRef = getInternalReferenceOrZero2(predicateKey, support); + final ClusterI.CompleteTypeEnum pCompleteType = ClusterTraitsBase.getCompleteTypeFromResourceKey(predicateKey); + // PredicateType is complete i.e. all relevant data is in resource table. + if (ClusterI.CompleteTypeEnum.NotComplete != pCompleteType) { + if (DEBUG) + System.out.println("ClusterSmall.forObjects: complete type was " + pCompleteType + " cluster=" + getClusterUID()); + return resourceTable.foreachObject(resourceIndex, procedure, context, support, this, pRef, pCompleteType, completeTable); + } + final int predicateIndex = resourceTable.getPredicateIndex(resourceIndex); + if (0 == predicateIndex) { // All relevant data is in resource table. + if (DEBUG) + System.out.println("ClusterSmall.forObjects: no predicate table " + pCompleteType); + return resourceTable.foreachObject(resourceIndex, procedure, context, support, this, pRef, pCompleteType, completeTable); + } + int objectIndex = predicateTable.getObjectIndex(predicateIndex, pRef & 0xFFFF); + return forObjects(resourceKey, predicateKey, objectIndex, procedure, context, support); + } + @Override + public boolean forPredicates(int resourceKey, + PredicateProcedure procedure, Context context, ClusterSupport support) + throws DatabaseException { + if (DEBUG) + System.out.println("ClusterSmall.forPredicates: rk=" + resourceKey ); + final int resourceIndex = getLocalReference(resourceKey); + final int predicateIndex = resourceTable.getPredicateIndex(resourceIndex); + if (0 == predicateIndex) + return resourceTable.foreachPredicate(resourceIndex, + procedure, context, support, this, completeTable); + else { + boolean broken = resourceTable.foreachPredicate(resourceIndex, + procedure, context, support, this, completeTable); + if (broken) + return true; + } + return predicateTable.foreachPredicate(predicateIndex, + procedure, context, support, this); + } + + @Override + public ClusterI addRelation(int sResourceKey, ClusterUID puid, int pResourceKey, ClusterUID ouid, int oResourceKey, ClusterSupport support) throws DatabaseException { + + if(proxy) { + throw new UnsupportedOperationException(); +// ClusterImpl cluster = clusterTable.load2(clusterId, clusterKey); +// return cluster.addRelation(sResourceKey, pResourceKey, oResourceKey, support); + } + + // check(); + boolean ret; + try { + short sri = getLocalReferenceAnd(sResourceKey, support, ClusterChange.ADD_OPERATION); + short pri = getReferenceOrCreateIfForeign(pResourceKey, puid, support, ClusterStream.NULL_OPERATION); + short ori = getReferenceOrCreateIfForeign(oResourceKey, ouid, support, ClusterStream.NULL_OPERATION); + ClusterI.CompleteTypeEnum completeType = ClusterTraitsBase.getCompleteTypeFromResourceKey(pResourceKey); + ret = addRelationInternal(sri, pri, ori, completeType); + calculateModifiedId(); + } catch (OutOfSpaceException e) { + boolean streamOff = support.getStreamOff(); + if (!streamOff) { + support.cancelStatement(this); + support.setStreamOff(true); + } + ClusterI cluster = toBig(clusterSupport); + if (!streamOff) + support.setStreamOff(false); + ClusterI cluster2 = cluster.addRelation(sResourceKey, pResourceKey, oResourceKey, support); + if (cluster != cluster2) + throw new DatabaseException("Internal error. Contact application support."); + return cluster; + } +// check(); + if (ret) { + support.addStatement(this); + return this; + } else { + support.cancelStatement(this); + return null; + } + + } + + @Override + public ClusterI addRelation(int sResourceKey, int pResourceKey, int oResourceKey, ClusterSupport support) throws DatabaseException { + + if (DEBUG) + System.out.println("add rk=" + sResourceKey + " pk=" + pResourceKey + " ok=" + oResourceKey); + + if(proxy) { + throw new UnsupportedOperationException(); +// ClusterImpl cluster = clusterTable.load2(clusterId, clusterKey); +// return cluster.addRelation(sResourceKey, pResourceKey, oResourceKey, support); + } + + // check(); + boolean ret; + try { + short sri = getLocalReferenceAnd(sResourceKey, support, ClusterChange.ADD_OPERATION); + short pri = getReferenceOrCreateIfForeign(pResourceKey, support, ClusterStream.NULL_OPERATION); + short ori = getReferenceOrCreateIfForeign(oResourceKey, support, ClusterStream.NULL_OPERATION); + ClusterI.CompleteTypeEnum completeType = ClusterTraitsBase.getCompleteTypeFromResourceKey(pResourceKey); + ret = addRelationInternal(sri, pri, ori, completeType); + calculateModifiedId(); + } catch (OutOfSpaceException e) { + boolean streamOff = support.getStreamOff(); + if (!streamOff) { + support.cancelStatement(this); + support.setStreamOff(true); + } + ClusterI cluster = toBig(clusterSupport); + if (!streamOff) + support.setStreamOff(false); + ClusterI cluster2 = cluster.addRelation(sResourceKey, pResourceKey, oResourceKey, support); + if (cluster != cluster2) + throw new DatabaseException("Internal error. Contact application support."); + return cluster; + } +// check(); + if (ret) { + support.addStatement(this); + return this; + } else { + support.cancelStatement(this); + return null; + } + } + @Override + public boolean removeRelation(int sResourceKey, int pResourceKey, int oResourceKey, ClusterSupport support) + throws DatabaseException { + // check(); + short sri = getLocalReferenceAnd(sResourceKey, support, ClusterChange.REMOVE_OPERATION); + short pri = getInternalReferenceOrZeroAnd(pResourceKey, support, ClusterStream.NULL_OPERATION); + short ori = getInternalReferenceOrZeroAnd(oResourceKey, support, ClusterStream.NULL_OPERATION); + boolean ret = false; + if (0 != pri && 0 != ori) { + ClusterI.CompleteTypeEnum completeType = ClusterTraitsBase.getCompleteTypeFromResourceKey(pResourceKey); + ret = removeRelationInternal(sri, pri, ori, completeType, support); + calculateModifiedId(); + } + if (ret) + support.removeStatement(this); + else + support.cancelStatement(this); + // check(); + return ret; + } + @Override + public void denyRelation(int sResourceKey, int pResourceKey, int oResourceKey, ClusterSupport support) + throws DatabaseException { + short s = checkResourceKeyIsOursAndGetResourceIndexIf(sResourceKey, support); + ResourceReferenceAndCluster p = checkResourceKeyAndGetResourceIndexIf(pResourceKey, support); + ResourceReferenceAndCluster o = checkResourceKeyAndGetResourceIndexIf(oResourceKey, support); + if (0 == s || 0 == p.reference || 0 == o.reference) + return; + // check(); + ClusterI.CompleteTypeEnum completeType = ClusterTraitsBase.getCompleteTypeFromResourceKey(pResourceKey); + boolean ret = removeRelationInternal(s, p.reference, o.reference, completeType, support); + if (ret) { + support.addStatementIndex(this, sResourceKey, getClusterUID(), ClusterChange.REMOVE_OPERATION); + support.addStatementIndex(this, pResourceKey, p.clusterUID, ClusterStream.NULL_OPERATION); + support.addStatementIndex(this, oResourceKey, o.clusterUID, ClusterStream.NULL_OPERATION); + support.removeStatement(this); + } + calculateModifiedId(); + // check(); + return; + } + @Override + public InputStream getValueStream(int resourceKey, ClusterSupport support) throws DatabaseException { + if (DEBUG) + System.out.println("ClusterSmall.getValue " + resourceKey); + int resourceIndex = ClusterTraitsBase.getResourceIndexFromResourceKeyNoThrow(resourceKey); + try { + byte[] buffer = resourceTable.getValue(valueTable, resourceIndex); + if(buffer == null) return null; + return new ByteArrayInputStream(buffer); + } catch (ExternalValueException e) { + return support.getValueStreamEx(resourceIndex, clusterUID.second); + } + } + @Override + public byte[] getValue(int resourceKey, ClusterSupport support) throws DatabaseException { + if (DEBUG) + System.out.println("ClusterSmall.getValue " + resourceKey); + int resourceIndex = ClusterTraitsBase.getResourceIndexFromResourceKeyNoThrow(resourceKey); + try { + return resourceTable.getValue(valueTable, resourceIndex); + } catch (ExternalValueException e) { + return clusterSupport.impl.getResourceFile(clusterUID.asBytes(), resourceIndex); + //return support.getValueEx(resourceIndex, clusterUID.second); + } + } + @Override + public boolean hasValue(int resourceKey, ClusterSupport support) + throws DatabaseException { + int resourceIndex = getLocalReference(resourceKey); + return resourceTable.hasValue(resourceIndex); + } + @Override + public boolean removeValue(int resourceKey, ClusterSupport support) + throws DatabaseException { + int resourceIndex = getLocalReferenceAnd(resourceKey, support, ClusterChange.DELETE_OPERATION); + support.removeValue(this); + calculateModifiedId(); + return resourceTable.removeValue(valueTable, resourceIndex); + } + @Override + public ClusterI setValue(int rResourceId, byte[] value, int length, ClusterSupport support) + throws DatabaseException { + int resourceIndex = getLocalReferenceAnd(rResourceId, support, ClusterStream.SET_OPERATION); + support.setValue(this, getClusterId(), value, length); + try { + resourceTable.setValue(valueTable, resourceIndex, value, length); + calculateModifiedId(); + return this; + } catch (OutOfSpaceException e) { + boolean streamOff = support.getStreamOff(); + if (!streamOff) + support.setStreamOff(true); + ClusterI cluster = toBig(support); + cluster.setValue(rResourceId, value, length, support); + if (!streamOff) + support.setStreamOff(false); + return cluster; + } + } + @Override + public ClusterI modiValueEx(int rResourceId, long voffset, int length, byte[] value, int offset, ClusterSupport support) + throws DatabaseException { + int resourceIndex = getLocalReferenceAnd(rResourceId, support, ClusterStream.MODI_OPERATION); + support.modiValue(this, getClusterId(), voffset, length, value, offset); + resourceTable.setValueEx(valueTable, resourceIndex); + calculateModifiedId(); + return this; + } + @Override + public byte[] readValueEx(int rResourceId, long voffset, int length, ClusterSupport support) + throws DatabaseException { + int resourceIndex = getLocalReference(rResourceId); + boolean isExternal = resourceTable.isValueEx(valueTable, resourceIndex); + if (!isExternal) + throw new DatabaseException("ClusterI.readValue supported only for external value. Resource key=" + rResourceId); + return support.getValueEx(resourceIndex, getClusterId(), voffset, length); + } + @Override + public boolean isValueEx(int resourceKey) throws DatabaseException { + int resourceIndex = getLocalReference(resourceKey); + return resourceTable.isValueEx(valueTable, resourceIndex); + } + @Override + public long getValueSizeEx(int rResourceId, ClusterSupport support) + throws DatabaseException, ExternalValueException { + int resourceIndex = getLocalReference(rResourceId); + boolean isExternal = resourceTable.isValueEx(valueTable, resourceIndex); + if (!isExternal) + throw new ExternalValueException("ClusterI.getValueSizeEx supported only for external value. Resource key=" + rResourceId); + return support.getValueSizeEx(resourceIndex, getClusterId()); + } + @Override + public void setValueEx(int rResourceId) + throws DatabaseException { + int resourceIndex = getLocalReference(rResourceId); + resourceTable.setValueEx(valueTable, resourceIndex); + } + @Override + public int createResource(ClusterSupport support) + throws DatabaseException { + + if(proxy) { + throw new UnsupportedOperationException(); +// ClusterImpl cluster = clusterTable.load2(clusterId, clusterKey); +// return cluster.createResource(support); + } + + short resourceIndex = resourceTable.createResource(); + calculateModifiedId(); + if(DebugPolicy.REPORT_RESOURCE_ID_ALLOCATION) + System.out.println("[RID_ALLOCATION]: ClusterSmall[" + clusterId + "] allocates " + resourceIndex); + support.createResource(this, resourceIndex, getClusterId()); + return ClusterTraits.createResourceKey(clusterKey, resourceIndex); + } + @Override + public boolean hasResource(int resourceKey, ClusterSupport support) { + int clusterKey = ClusterTraitsBase.getClusterKeyFromResourceKeyNoThrow(resourceKey); + if (this.clusterKey != clusterKey) // foreign resource + return false; + int resourceIndex; + try { + resourceIndex = ClusterTraits.getResourceIndexFromResourceKey(resourceKey); + } catch (DatabaseException e) { + return false; + } + if (resourceIndex > 0 & resourceIndex <= resourceTable.getTableCount()) + return true; + else + return false; + } + @Override + public int getNumberOfResources(ClusterSupport support) + throws DatabaseException { + + if(proxy) { + throw new UnsupportedOperationException(); +// ClusterImpl cluster = clusterTable.load2(clusterId, clusterKey); +// return cluster.getNumberOfResources(support); + } + + return resourceTable.getUsedSize(); + } + + public int getNumberOfResources() throws IllegalAcornStateException { + if(proxy) + throw new IllegalAcornStateException("proxy == true for " + clusterId); + + return resourceTable.getUsedSize(); + } + + @Override + public long getUsedSpace() { + if(isEmpty()) return 0; + long rt = resourceTable.getTableCapacity() * 8 + 8; // (8 = cluster id) + long ft = foreignTable.getTableCapacity() * 8; + long pt = predicateTable.getTableCapacity() * 4; + long ot = objectTable.getTableCapacity() * 4; + long ct = completeTable.getTableCapacity() * 4; + long vt = valueTable.getTableCapacity() * 1; + long cm = clusterMap.getUsedSpace(); + return rt + ft + pt + ot + ct + vt + cm; + } + @Override + public boolean isEmpty() { + if(resourceTable == null) return true; + return resourceTable.getTableCount() == 0; + } + @Override + public void printDebugInfo(String message, ClusterSupport support) + throws DatabaseException { + throw new DatabaseException("Not implemented!"); + } + private short getInternalReferenceOrZero2(int resourceKey, ClusterSupport support) throws DatabaseException { + int resourceIndex = ClusterTraitsBase.getResourceIndexFromResourceKeyNoThrow(resourceKey); + if (!ClusterTraitsBase.isCluster(clusterBits, resourceKey)) { + return clusterMap.getForeignReferenceOrZero(resourceKey); + } else { + return (short)resourceIndex; + } + } + private short getInternalReferenceOrZeroAnd(int resourceKey, ClusterSupport support, byte op) + throws DatabaseException { + int clusterKey = ClusterTraits.getClusterKeyFromResourceKey(resourceKey); + int resourceIndex = ClusterTraits.getResourceIndexFromResourceKey(resourceKey); + if (this.clusterKey != clusterKey) { // foreign resource + ClusterUID clusterUID = clusterSupport.getClusterUIDByResourceKey(resourceKey); + short foreignRef = clusterMap.getForeignReferenceOrZero(resourceKey); + support.addStatementIndex(this, resourceKey, clusterUID, op); + return foreignRef; + } + support.addStatementIndex(this, resourceKey, getClusterUID(), op); + return (short)resourceIndex; + } + private final short getLocalReference(int resourceKey) throws DatabaseException { + return ClusterTraits.getResourceIndexFromResourceKeyNoThrow(resourceKey); + } + private final short getLocalReferenceAnd(int resourceKey, ClusterSupport support, byte op) + throws DatabaseException { + short resourceIndex = getLocalReference(resourceKey); + support.addStatementIndex(this, resourceKey, getClusterUID(), op); + return resourceIndex; + } + private short checkResourceKeyIsOursAndGetResourceIndexIf(int resourceKey, ClusterSupport support) + throws DatabaseException { + int clusterShortId = ClusterTraits.getClusterKeyFromResourceKey(resourceKey); + if (this.clusterKey != clusterShortId) + return 0; + int resourceIndex = ClusterTraits.getResourceIndexFromResourceKey(resourceKey); + return (short)resourceIndex; + } + private short getReferenceOrCreateIfForeign(int resourceKey, ClusterUID clusterUID, ClusterSupport support, byte op) + throws DatabaseException { + int clusterKey = ClusterTraits.getClusterKeyFromResourceKey(resourceKey); + short resourceIndex = (short)ClusterTraits.getResourceIndexFromResourceKey(resourceKey); + if (this.clusterKey != clusterKey) { + support.addStatementIndex(this, resourceKey, clusterUID, op); + short ref = clusterMap.getForeignReferenceOrCreateByResourceKey(resourceKey, clusterUID); + return ref; + } + support.addStatementIndex(this, resourceKey, getClusterUID(), op); + return resourceIndex; + } + private short getReferenceOrCreateIfForeign(int resourceKey, ClusterSupport support, byte op) + throws DatabaseException { + int clusterKey = ClusterTraits.getClusterKeyFromResourceKey(resourceKey); + short resourceIndex = (short)ClusterTraits.getResourceIndexFromResourceKey(resourceKey); + if (this.clusterKey != clusterKey) { + ClusterUID clusterUID = clusterSupport.getClusterUIDByResourceKey(resourceKey); + support.addStatementIndex(this, resourceKey, clusterUID, op); + short ref = clusterMap.getForeignReferenceOrCreateByResourceKey(resourceKey, clusterUID); + return ref; + } + support.addStatementIndex(this, resourceKey, getClusterUID(), op); + return resourceIndex; + } + private class ResourceReferenceAndCluster { + ResourceReferenceAndCluster(short reference, ClusterUID clusterUID) { + this.reference = reference; + this.clusterUID = clusterUID; + } + public final short reference; + public final ClusterUID clusterUID; + } + private ResourceReferenceAndCluster checkResourceKeyAndGetResourceIndexIf(int resourceKey, ClusterSupport support) + throws DatabaseException { + int clusterKey = ClusterTraits.getClusterKeyFromResourceKey(resourceKey); + short resourceIndex = (short)ClusterTraits.getResourceIndexFromResourceKey(resourceKey); + if (this.clusterKey != clusterKey) { // foreign resource + ClusterI foreignCluster = support.getClusterByClusterKey(clusterKey); + ClusterUID clusterUID = foreignCluster.getClusterUID(); + short ref = clusterMap.getForeignReferenceOrZero(resourceKey); + return new ResourceReferenceAndCluster(ref, clusterUID); + } + return new ResourceReferenceAndCluster(resourceIndex, getClusterUID()); + } + + static long fTime = 0; + + @Override + final public int execute(int resourceReference) throws DatabaseException { + short resourceRef = (short)resourceReference; + int key; + if (ClusterTraitsSmall.resourceRefIsLocal(resourceRef)) { + key = clusterBits | resourceRef; + } else { + short foreignIndex = ClusterTraitsSmall.resourceRefGetForeignIndex((short)resourceRef); + //long start = System.nanoTime(); + ResourceUID resourceUID = foreignTable.getResourceUID(foreignIndex); + int clusterKey = clusterSupport.getClusterKeyByClusterUIDOrMake(resourceUID.asCID()); +// ClusterBase cluster = clusterSupport.getClusterByClusterUIDOrMake(resourceUID.asCID()); + key = ClusterTraitsBase.createResourceKey(clusterKey, resourceUID.getIndex()); + //fTime += System.nanoTime() - start; + //System.err.println("fTime: " + 1e-9*fTime); + } + if (DEBUG) + System.out.println("ClusterSmall.execute key=" + key); + return key; + } + + private boolean addRelationInternal(short sReference, short pReference, short oReference, ClusterI.CompleteTypeEnum completeType) + throws DatabaseException { + int predicateIndex = resourceTable.addStatement(sReference, pReference, oReference, predicateTable, objectTable, completeType, completeTable); + if (0 == predicateIndex) + return true; // added to resourceTable + else if (0 > predicateIndex) + return false; // old complete statemenent + int newPredicateIndex = predicateTable.addPredicate(predicateIndex, 0xFFFF & pReference, 0xFFFF & oReference, objectTable); + if (0 == newPredicateIndex) + return false; + if (predicateIndex != newPredicateIndex) + resourceTable.setPredicateIndex(sReference, newPredicateIndex); + return true; + } + private boolean removeRelationInternal(int sResourceIndex, short pResourceIndex, + short oResourceIndex, ClusterI.CompleteTypeEnum completeType, ClusterSupport support) + throws DatabaseException { + int predicateIndex = resourceTable.getPredicateIndex(sResourceIndex); + if (0 == predicateIndex || ClusterI.CompleteTypeEnum.NotComplete != completeType) + return resourceTable.removeStatementFromCache(sResourceIndex, + pResourceIndex, oResourceIndex, completeType, completeTable); + PredicateTable.Status ret = predicateTable.removePredicate(predicateIndex, 0xFFFF & pResourceIndex, 0xFFFF & oResourceIndex, objectTable); + switch (ret) { + case NothingRemoved: + return false; + case PredicateRemoved: { + if (0 == predicateTable.getPredicateSetSize(predicateIndex)) + resourceTable.setPredicateIndex(sResourceIndex, 0); + // intentionally dropping to next case + } default: + break; + } + resourceTable.removeStatement(sResourceIndex, + pResourceIndex, oResourceIndex, + completeType, completeTable, + predicateTable, objectTable, support); + return true; + } + @Override + public void load() { + throw new Error("Not supported."); + } + + @Override + public void load(Callback r) { + throw new Error("Not supported."); + } + + public boolean contains(int resourceKey) { + return ClusterTraitsBase.isCluster(clusterBits, resourceKey); + } + @Override + public void load(final ClusterSupport support, final Runnable callback) { + + throw new UnsupportedOperationException(); + +// try { +// clusterTable.load2(clusterId, clusterKey); +// callback.run(); +// } catch (DatabaseException e) { +// e.printStackTrace(); +// } + + } + @Override + public ClusterI getClusterByResourceKey(int resourceKey, + ClusterSupport support) { + throw new Error(); + } + @Override + public void increaseReferenceCount(int amount) { + throw new Error(); + } + @Override + public void decreaseReferenceCount(int amount) { + throw new Error(); + } + @Override + public int getReferenceCount() { + throw new Error(); + } + @Override + public void releaseMemory() { + } + @Override + public void compact() { + clusterMap.compact(); + } + @Override + public boolean isLoaded() { + return !proxy; + } + +// public ClusterImpl tryLoad(SessionImplSocket sessionImpl) { +// +// throw new UnsupportedOperationException(); +// assert(Constants.ReservedClusterId != clusterId); +// +// return clusterTable.tryLoad(clusterId, clusterKey); +// +// } + + + @Override + public ClusterBig toBig(ClusterSupport support) + throws DatabaseException { + if (DEBUG) { + System.out.println("DEBUG: toBig cluster=" + clusterId); + new Exception().printStackTrace(); + } + ClusterBig big = new ClusterBig(clusterSupport, getClusterUID(), clusterKey, (ClusterSupport2)support); + big.cc = this.cc; +// if(big.cc != null) +// big.cc.clusterImpl = this; + resourceTable.toBig(big, support, this); + big.foreignLookup = this.foreignLookup; + big.change = this.change; + this.cc = null; + this.foreignLookup = null; + this.change = null; + return big; + } + + @Override + public ClusterTypeEnum getType() { + return ClusterTypeEnum.SMALL; + } + @Override + public boolean getImmutable() { + int status = resourceTable.getClusterStatus(); + return (status & ClusterStatus.ImmutableMaskSet) == 1; + } + @Override + public void setImmutable(boolean immutable, ClusterSupport support) { + if(resourceTable != null) { + int status = resourceTable.getClusterStatus(); + if (immutable) + status |= ClusterStatus.ImmutableMaskSet; + else + status &= ClusterStatus.ImmutableMaskClear; + resourceTable.setClusterStatus(status); + } + support.setImmutable(this, immutable); + } + + @Override + public String toString() { + try { + final TIntHashSet set = new TIntHashSet(); + TIntShortHashMap map = foreignTable.getResourceHashMap(); + map.forEachKey(new TIntProcedure() { + @Override + public boolean execute(int value) { + set.add(value & 0xfffff000); + return true; + } + }); + return "ClusterSmall[" + getClusterUID() + " - " + getClusterId() + " - " + getNumberOfResources() + " - " + foreignTable.getResourceHashMap().size() + " - " + set.size() + "]"; + } catch (DatabaseException e) { + try { + return "ClusterSmall[" + getNumberOfResources() + "]"; + } catch (IllegalAcornStateException e1) { + Logger.defaultLogError(e1); + e1.printStackTrace(); + return "An exception occured!!"; + } + } + } + + // Memory map + // bytes (b) | headers(i) | predicateTable (i) | objectTable (i) | completeTable (i) | resourceTable (l) | foreignTable (l) + + @Override + public byte[] storeBytes() throws IOException { + + int byteSize = valueTable.getTableSize(); + int longSize = LONG_HEADER_SIZE + resourceTable.getTableSize() + foreignTable.getTableSize(); + int intSize = INT_HEADER_SIZE + predicateTable.getTableSize() + objectTable.getTableSize() + completeTable.getTableSize(); + + byte[] raw = new byte[12 + byteSize + 8*longSize + 4*intSize]; + + int[] currentHeader = Arrays.copyOf(headerTable, INT_HEADER_SIZE); + + Bytes.writeLE(raw, 0, byteSize); + Bytes.writeLE(raw, 4, intSize); + Bytes.writeLE(raw, 8, longSize); + + int rawPos = valueTable.storeBytes(raw, 0, 12); + + int intBase = rawPos; + + rawPos += 4*INT_HEADER_SIZE; + rawPos = predicateTable.storeBytes(raw, (rawPos-intBase)>>2, rawPos); + rawPos = objectTable.storeBytes(raw, (rawPos-intBase)>>2, rawPos); + rawPos = completeTable.storeBytes(raw, (rawPos-intBase)>>2, rawPos); + + int longBase = rawPos; + + rawPos += 8*LONG_HEADER_SIZE; + rawPos = resourceTable.storeBytes(raw, (rawPos-longBase)>>3, rawPos); + rawPos = foreignTable.storeBytes(raw, (rawPos-longBase)>>3, rawPos); + + Bytes.writeLE8(raw, longBase, -1); + Bytes.writeLE8(raw, longBase+8, LONG_HEADER_VERSION); + Bytes.writeLE8(raw, longBase+16, 0); + Bytes.writeLE8(raw, longBase+24, clusterUID.second); + + // write header + for(int i=0;i getPredicateTable() { + return predicateTable; + } + + @Override + public Table getForeignTable() { + return foreignTable; + } + + @Override + public int makeResourceKey(int pRef) throws DatabaseException { + throw new UnsupportedOperationException(); + } + + @Override + public Table getCompleteTable() { + return completeTable; + } + + @Override + public Table getValueTable() { + return valueTable; + } + + @Override + public Table getObjectTable() { + return objectTable; + } + +} + +class ClusterStatus { + public static final int ImmutableMaskClear = 0xFFFFFFFE; + public static final int ImmutableMaskSet = 0x00000001; + public static final int DeletedMaskClear = 0xFFFFFFFD; + public static final int DeletedMaskSet = 0x00000002; +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/exception/AcornAccessVerificationException.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/exception/AcornAccessVerificationException.java new file mode 100644 index 000000000..6269a60b3 --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/exception/AcornAccessVerificationException.java @@ -0,0 +1,20 @@ +package org.simantics.acorn.exception; + +import org.simantics.db.exception.SDBException; + +public class AcornAccessVerificationException extends SDBException { + + private static final long serialVersionUID = 6601855907356895356L; + + public AcornAccessVerificationException(String message, Throwable cause) { + super(message, cause); + } + + public AcornAccessVerificationException(String message) { + super(message); + } + + public AcornAccessVerificationException(Throwable cause) { + super(cause); + } +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/exception/IllegalAcornStateException.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/exception/IllegalAcornStateException.java new file mode 100644 index 000000000..8228d59c1 --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/exception/IllegalAcornStateException.java @@ -0,0 +1,21 @@ +package org.simantics.acorn.exception; + +import org.simantics.db.exception.SDBException; + +public class IllegalAcornStateException extends SDBException { + + private static final long serialVersionUID = -8255505454138490120L; + + public IllegalAcornStateException(String message, Throwable cause) { + super(message, cause); + } + + public IllegalAcornStateException(String message) { + super(message); + } + + public IllegalAcornStateException(Throwable cause) { + super(cause); + } + +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/exception/InvalidHeadStateException.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/exception/InvalidHeadStateException.java new file mode 100644 index 000000000..7c8510399 --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/exception/InvalidHeadStateException.java @@ -0,0 +1,27 @@ +package org.simantics.acorn.exception; + +public class InvalidHeadStateException extends Exception { + + private static final long serialVersionUID = -7291859180968235955L; + + public InvalidHeadStateException() { + super(); + } + + public InvalidHeadStateException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } + + public InvalidHeadStateException(String message, Throwable cause) { + super(message, cause); + } + + public InvalidHeadStateException(String message) { + super(message); + } + + public InvalidHeadStateException(Throwable cause) { + super(cause); + } +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/AcornDatabase.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/AcornDatabase.java new file mode 100644 index 000000000..be505c603 --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/AcornDatabase.java @@ -0,0 +1,263 @@ +package org.simantics.acorn.internal; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.channels.FileLock; +import java.nio.file.DirectoryStream; +import java.nio.file.FileVisitOption; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.EnumSet; +import java.util.Properties; + +import org.simantics.acorn.GraphClientImpl2; +import org.simantics.db.Database; +import org.simantics.db.DatabaseUserAgent; +import org.simantics.db.ServiceLocator; +import org.simantics.db.common.utils.Logger; +import org.simantics.db.server.ProCoreException; + +/** + * @author Tuukka Lehtonen + */ +public class AcornDatabase implements Database { + + private final Path folder; + + private DatabaseUserAgent userAgent; + + private RandomAccessFile raLockFile; + + private FileLock lock; + + private boolean isRunning; + + public AcornDatabase(Path folder) { + this.folder = folder; + } + + @Override + public DatabaseUserAgent getUserAgent() { + return userAgent; + } + + @Override + public void setUserAgent(DatabaseUserAgent dbUserAgent) { + userAgent = dbUserAgent; + } + + @Override + public Status getStatus() { + return Status.Local; + } + + @Override + public File getFolder() { + return folder.toFile(); + } + + @Override + public boolean isFolderOk() { + return isFolderOk(folder.toFile()); + } + + @Override + public boolean isFolderOk(File aFolder) { + if (!aFolder.isDirectory()) + return false; + return true; + } + + @Override + public boolean isFolderEmpty() { + return isFolderEmpty(folder.toFile()); + } + + @Override + public boolean isFolderEmpty(File aFolder) { + Path path = aFolder.toPath(); + if (!Files.isDirectory(path)) + return false; + try (DirectoryStream folderStream = Files.newDirectoryStream(path)) { + return !folderStream.iterator().hasNext(); + } catch (IOException e) { + Logger.defaultLogError("Failed to open folder stream. folder=" + path, e); + return false; + } + } + + @Override + public void initFolder(Properties properties) throws ProCoreException { + try { + Files.createDirectories(folder); + } catch (IOException e) { + throw new ProCoreException(e); + } + } + + @Override + public void deleteFiles() throws ProCoreException { + deleteTree(folder); + } + + @Override + public void start() throws ProCoreException { + Path lockFile = folder.resolve("lock"); + try { + if (!Files.exists(lockFile)) + Files.createFile(lockFile); + + raLockFile = new RandomAccessFile(lockFile.toFile(), "rw"); + lock = raLockFile.getChannel().tryLock(); + if (lock == null) { + throw new ProCoreException("The database in folder " + folder.toAbsolutePath() + " is already in use!"); + } + + isRunning = true; + + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Override + public boolean isRunning() throws ProCoreException { + return isRunning; + } + + @Override + public boolean tryToStop() throws ProCoreException { + try { + lock.release(); + raLockFile.close(); + + Files.deleteIfExists(folder.resolve("lock")); + + isRunning = false; + + } catch (IOException e) { + e.printStackTrace(); + } + + return true; + } + + @Override + public void connect() throws ProCoreException { + } + + @Override + public boolean isConnected() throws ProCoreException { + return isRunning; + } + + @Override + public String execute(String command) throws ProCoreException { + throw new UnsupportedOperationException("execute(" + command + ")"); + } + + @Override + public void disconnect() throws ProCoreException { + } + + @Override + public void clone(File to, int revision, boolean saveHistory) throws ProCoreException { + // TODO: implement + throw new UnsupportedOperationException(); + } + + @Override + public Path createFromChangeSets(int revision) throws ProCoreException { + // TODO: implement + throw new UnsupportedOperationException(); + } + + @Override + public void deleteGuard() throws ProCoreException { + // TODO: implement + throw new UnsupportedOperationException(); + } + + @Override + public Path dumpChangeSets() throws ProCoreException { + // TODO: implement + throw new UnsupportedOperationException(); + } + + @Override + public void purgeDatabase() throws ProCoreException { + // TODO: implement + throw new UnsupportedOperationException(); + } + + @Override + public long serverGetTailChangeSetId() throws ProCoreException { + // "We have it all" + // But after purging we don't so beware. + // TODO: beware for purge + return 1; + } + + @Override + public Session newSession(ServiceLocator locator) throws ProCoreException { + try { + return new GraphClientImpl2(this, folder, locator); + } catch (IOException e) { + throw new ProCoreException(e); + } + } + + @Override + public Journal getJournal() throws ProCoreException { + // TODO: implement + throw new UnsupportedOperationException(); + } + + private static void deleteTree(Path path) throws ProCoreException { + if (!Files.exists(path)) + return; + + class Visitor extends SimpleFileVisitor { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + try { + Files.delete(file); + } catch (IOException ioe) { + ioe.printStackTrace(); + throw ioe; + } + return FileVisitResult.CONTINUE; + } + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException e) throws IOException { + if (e == null) { + try { + Files.delete(dir); + } catch (IOException ioe) { + ioe.printStackTrace(); + throw ioe; + } + return FileVisitResult.CONTINUE; + } + throw e; + } + } + try { + Visitor v = new Visitor(); + EnumSet opts = EnumSet.noneOf(FileVisitOption.class); + Files.walkFileTree(path, opts, Integer.MAX_VALUE, v); + } catch (IOException e) { + throw new ProCoreException("Could not delete " + path, e); + } + } + + @Override + public String getCompression() { + return "LZ4"; + } + +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/Activator.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/Activator.java new file mode 100644 index 000000000..b6cb59b40 --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/Activator.java @@ -0,0 +1,62 @@ +/******************************************************************************* + * Copyright (c) 2007, 2010 Association for Decentralized Information Management + * in Industry THTH ry. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * VTT Technical Research Centre of Finland - initial API and implementation + *******************************************************************************/ +package org.simantics.acorn.internal; + +import org.eclipse.core.runtime.Plugin; +import org.osgi.framework.BundleContext; + +/** + * @author Antti Villberg + */ +public class Activator extends Plugin { + + // The plug-in ID + public static final String BUNDLE_ID = "org.simantics.acorn"; //$NON-NLS-1$ + // The shared instance + private static Activator plugin; + + /** + * The constructor + */ + public Activator() { + } + + /* + * (non-Javadoc) + * @see org.eclipse.ui.plugin.AbstractUIPlugin#start(org.osgi.framework.BundleContext) + */ + @Override + public void start(BundleContext context) throws Exception { + super.start(context); + plugin = this; + } + + /* + * (non-Javadoc) + * @see org.eclipse.ui.plugin.AbstractUIPlugin#stop(org.osgi.framework.BundleContext) + */ + @Override + public void stop(BundleContext context) throws Exception { + plugin = null; + super.stop(context); + } + + /** + * Returns the shared instance + * + * @return the shared instance + */ + public static Activator getDefault() { + return plugin; + } + +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/BijectionMap.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/BijectionMap.java new file mode 100644 index 000000000..3de77d2aa --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/BijectionMap.java @@ -0,0 +1,119 @@ +/******************************************************************************* + * Copyright (c) 2007, 2010 Association for Decentralized Information Management + * in Industry THTH ry. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * VTT Technical Research Centre of Finland - initial API and implementation + *******************************************************************************/ +/* + * Created on Jan 21, 2005 + * + * Copyright Toni Kalajainen + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.simantics.acorn.internal; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * Bijection map is a Map that has no values or keys, only 1:1 mappings + * of values. These value/keys will be called with left and right side + * values. + * + * Each value can exist only once on a side + * + * @author Toni Kalajainen + */ +public class BijectionMap { + + /** The keys of tableLeft are left-side-values and + * values are right-side-values */ + private final Map tableLeft = new HashMap(); + /** The keys of tableRight are right-side-values and + * values on it are left-side-values */ + private final Map tableRight = new HashMap(); + + public boolean containsLeft(L leftValue) + { + return tableLeft.containsKey(leftValue); + } + + public boolean containsRight(R rightValue) + { + return tableRight.containsKey(rightValue); + } + + public void map(L leftValue, R rightValue) + { + // Remove possible old mapping + R oldRight = tableLeft.remove(leftValue); + if (oldRight != null) { + tableRight.remove(oldRight); + } else { + L oldLeft = tableRight.remove(rightValue); + if (oldLeft != null) { + tableLeft.remove(oldLeft); + } + } + + tableLeft.put(leftValue, rightValue); + tableRight.put(rightValue, leftValue); + } + + public int size() + { + return tableLeft.size(); + } + + public L getLeft(R rightValue) { + return tableRight.get(rightValue); + } + + public R getRight(L leftValue) { + return tableLeft.get(leftValue); + } + + public R removeWithLeft(L leftValue) { + R rightValue = tableLeft.remove(leftValue); + if (rightValue!=null) + tableRight.remove(rightValue); + return rightValue; + } + + public L removeWithRight(R rightValue) { + L leftValue = tableRight.remove(rightValue); + if (leftValue!=null) + tableLeft.remove(leftValue); + return leftValue; + } + + public Set getLeftSet() { + return tableLeft.keySet(); + } + + public Set getRightSet() { + return tableRight.keySet(); + } + + public void clear() { + tableLeft.clear(); + tableRight.clear(); + } +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/Change.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/Change.java new file mode 100644 index 000000000..305e31537 --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/Change.java @@ -0,0 +1,70 @@ +package org.simantics.acorn.internal; + +import org.simantics.db.service.ClusterUID; + +final public class Change { + + byte op0; + int key0; + int key1; + int key2; + ClusterUID clusterUID1; + ClusterUID clusterUID2; + byte[] lookup1; + byte[] lookup2; + byte lookIndex1; + byte lookIndex2; + int lastArg = 0; + + @Override + public String toString() { + return "Change " + (key0&0xffff) + " " + (key1&0xffff) + " " + (key2&0xffff) + " " + clusterUID2 + " " + clusterUID2; + } + + public final void init() { + lastArg = 0; + } + + public final void initValue() { + lastArg = 0; + } + + final void addStatementIndex0(int key, byte op) { + assert (op != 0); + key0 = key; + op0 = op; + } + + final void addStatementIndex1(int key, ClusterUID clusterUID, byte lookIndex, byte[] lookup) { + key1 = key; + clusterUID1 = clusterUID; + lookIndex1 = lookIndex; + lookup1 = lookup; +// if(lookIndex > 0) +// System.err.println("statementIndex1 " + pos + " " + lookIndex); + } + + final void addStatementIndex2(int key, ClusterUID clusterUID, byte lookIndex, byte[] lookup) { + key2 = key; + clusterUID2 = clusterUID; + lookIndex2 = lookIndex; + lookup2 = lookup; + } + + final public void addStatementIndex(int key, ClusterUID clusterUID, byte op) { + + // new Exception("lastArg=" + lastArg).printStackTrace(); + + assert (lastArg < 3); + + if (0 == lastArg) + addStatementIndex0(key, op); + else if (1 == lastArg) + addStatementIndex1(key, clusterUID, (byte)0, null); + else if (2 == lastArg) + addStatementIndex2(key, clusterUID, (byte)0, null); + + lastArg++; + + } +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/ClusterChange.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/ClusterChange.java new file mode 100644 index 000000000..b1fbb5d9c --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/ClusterChange.java @@ -0,0 +1,735 @@ +package org.simantics.acorn.internal; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.simantics.acorn.internal.ClusterStream.ClusterEnum; +import org.simantics.acorn.internal.ClusterStream.Data; +import org.simantics.acorn.internal.ClusterStream.DebugInfo; +import org.simantics.acorn.internal.ClusterStream.OpEnum; +import org.simantics.acorn.internal.ClusterStream.StmEnum; +import org.simantics.compressions.Compressions; +import org.simantics.db.exception.RuntimeDatabaseException; +import org.simantics.db.impl.ClusterTraitsBase; +import org.simantics.db.procore.cluster.ClusterTraits; +import org.simantics.db.procore.cluster.ClusterTraitsSmall; +import org.simantics.db.service.Bytes; +import org.simantics.db.service.ClusterUID; +import org.simantics.utils.datastructures.Pair; + +import gnu.trove.map.hash.TIntByteHashMap; +import gnu.trove.map.hash.TLongIntHashMap; + + +public final class ClusterChange { + + public static final int VERSION = 1; + public static final byte ADD_OPERATION = 2; + public static final byte REMOVE_OPERATION = 3; + public static final byte DELETE_OPERATION = 5; + + public static final boolean DEBUG = false; + public static final boolean DEBUG_STAT = false; + public static final boolean DEBUG_CCS = false; + + private static DebugInfo sum = new DebugInfo(); + + public final TIntByteHashMap foreignTable = new TIntByteHashMap(); + private final DebugInfo info; +// private final GraphSession graphSession; + public final ClusterUID clusterUID; + private final int SIZE_OFFSET; +// private final int HEADER_SIZE; + // How much buffer is used before stream is flushed to server. The bigger the better. + public static final int MAX_FIXED_BYTES = (1<<15) + (1<<14); + private static final int MAX_FIXED_OPERATION_SIZE = 17 + 16; + private static final int MAX_FIXED_OPERATION_SIZE_AND_ROOM_FOR_ERROR = MAX_FIXED_OPERATION_SIZE + 36; + private int nextSize = MAX_FIXED_BYTES; + int byteIndex = 0; + private byte[] bytes = null; // Operation data. +// private final byte[] header; + private boolean flushed = false; + private ArrayList> stream; + +// public ClusterImpl clusterImpl; + + public ClusterChange( ArrayList> stream, ClusterUID clusterUID) { + this.clusterUID = clusterUID; + long[] longs = new long[ClusterUID.getLongLength()]; + clusterUID.toLong(longs, 0); + this.stream = stream; +// this.graphSession = clusterStream.graphSession; + info = new DebugInfo(); +// HEADER_SIZE = 8 + longs.length * 8; +// header = new byte[HEADER_SIZE]; + SIZE_OFFSET = 0; +// Bytes.writeLE(header, SIZE_OFFSET + 0, 0); // Correct byte vector size is set with setHeaderVectorSize() later. +// Bytes.writeLE(header, SIZE_OFFSET + 4, VERSION); +// for (int i=0, offset=8; i>>8); + } + void flushCollect(Change c) { + throw new UnsupportedOperationException(); +// flushInternal(graphSession, clusterUID); +// if (DEBUG) +// printlnd("Cluster change data was flushed."); +// if (null != c) { +// if (DEBUG) +// printlnd("Clearing lookup for " + c.toString()); +// c.lookup1 = null; +// c.lookup2 = null; +// } +// if (null != clusterImpl) { +// clusterImpl.foreignLookup = null; +// } + } + + private final boolean checkBufferSpace(Change c) { +// clusterStream.changes.checkFlush(); + if(bytes == null) initBuffer(); + if (MAX_FIXED_BYTES - byteIndex > MAX_FIXED_OPERATION_SIZE_AND_ROOM_FOR_ERROR) { + return false; + } + flush(); +// initBuffer(); + return true; + } + + private final void checkBufferSpace(int size) { + if(bytes == null) initBuffer(); + if (bytes.length - byteIndex >= size) + return; + nextSize = Math.max(MAX_FIXED_BYTES, size); + flush(); + initBuffer(); + } + + public final void addChange(Change c) { + checkInitialization(); + checkBufferSpace(c); + byte operation = c.op0; + if(operation == ADD_OPERATION) + addStm(c, StmEnum.Add); + else if (operation == REMOVE_OPERATION) + addStm(c, StmEnum.Remove); + else if (operation == DELETE_OPERATION) { + if (DEBUG) + printlnd("Delete value offset=" + byteIndex + " " + c); + addByte(OpEnum.Delete.getOrMask()); + addShort(ClusterTraitsBase.getResourceIndexFromResourceKeyNoThrow(c.key0)); + } + c.lastArg = 0; + } + + private final void addForeignLong(short index, ClusterUID clusterUID) { + byteIndex = clusterUID.toByte(bytes, byteIndex); + bytes[byteIndex++] = (byte)(index & 0xFF); + bytes[byteIndex++] = (byte)(index >>> 8); + } + + private final ClusterEnum addIndexAndCluster(int key, ClusterUID clusterUID, byte lookIndex, byte[] lookup) { + assert(!clusterUID.equals(ClusterUID.Null)); + short resourceIndex = ClusterTraitsBase.getResourceIndexFromResourceKeyNoThrow(key); + if (clusterUID.equals(this.clusterUID)) { + bytes[byteIndex++] = (byte)(resourceIndex & 0xFF); + bytes[byteIndex++] = (byte)(resourceIndex >>> 8); + return ClusterEnum.Local; + } + + byte foreign = 0; + if(lookIndex > 0) { + if(lookup != null) + foreign = lookup[lookIndex]; + } else { + foreign = foreignTable.get(key); + } + if (0 != foreign) { + if (foreign > 256) + throw new RuntimeDatabaseException("Internal error, contact application support." + + "Too big foreing index=" + foreign + " max=256"); + --foreign; + bytes[byteIndex++] = foreign; + return ClusterEnum.ForeignShort; + } else { + byte position = (byte) (foreignTable.size() + 1); + if(lookup != null) + lookup[lookIndex] = position; + foreignTable.put(key, position); + if (DEBUG_STAT) + info.sForeign = foreignTable.size(); + if (clusterUID.equals(ClusterUID.Null)) + throw new RuntimeDatabaseException("Internal error, contact application support." + + "Cluster unique id not defined for foreing cluster."); + addForeignLong(resourceIndex, clusterUID); + return ClusterEnum.ForeignLong; + } + } + + private final void addByte(byte b) { + bytes[byteIndex++] = b; + } + + private final void addShort(short s) { + bytes[byteIndex++] = (byte)(s & 0xFF); + bytes[byteIndex++] = (byte)(s >>> 8); + } + +// private final void addShort(int s) { +// bytes[byteIndex++] = (byte) (s & 0xFF); +// bytes[byteIndex++] = (byte) ((s >>> 8) & 0xFF); +// } + + private final void addInt(int i) { +// System.err.println("addInt " + i + " " + i); + bytes[byteIndex++] = (byte) (i & 0xFF); + bytes[byteIndex++] = (byte) ((i >>> 8) & 0xFF); + bytes[byteIndex++] = (byte) ((i >>> 16) & 0xFF); + bytes[byteIndex++] = (byte) ((i >>> 24) & 0xFF); + // buffer.asIntBuffer().put(i); + // buffer.position(buffer.position()+4); + } + +// private void addLong6(long l) { +//// System.err.println("addLong " + l); +// bytes[byteIndex++] = (byte) (l & 0xFF); +// bytes[byteIndex++] = (byte) ((l >>> 8) & 0xFF); +// bytes[byteIndex++] = (byte) ((l >>> 16) & 0xFF); +// bytes[byteIndex++] = (byte) ((l >>> 24) & 0xFF); +// bytes[byteIndex++] = (byte) ((l >>> 32) & 0xFF); +// bytes[byteIndex++] = (byte) ((l >>> 40) & 0xFF); +// // buffer.asLongBuffer().put(l); +// // buffer.position(buffer.position() + 6); +// } + + private void addLong7(long l) { + bytes[byteIndex++] = (byte) (l & 0xFF); + bytes[byteIndex++] = (byte) ((l >>> 8) & 0xFF); + bytes[byteIndex++] = (byte) ((l >>> 16) & 0xFF); + bytes[byteIndex++] = (byte) ((l >>> 24) & 0xFF); + bytes[byteIndex++] = (byte) ((l >>> 32) & 0xFF); + bytes[byteIndex++] = (byte) ((l >>> 40) & 0xFF); + bytes[byteIndex++] = (byte) ((l >>> 48) & 0xFF); + // buffer.asLongBuffer().put(l); + // buffer.position(buffer.position() + 7); + } + +// private void addLong(long l) { +// bytes[byteIndex++] = (byte) (l & 0xFF); +// bytes[byteIndex++] = (byte) ((l >>> 8) & 0xFF); +// bytes[byteIndex++] = (byte) ((l >>> 16) & 0xFF); +// bytes[byteIndex++] = (byte) ((l >>> 24) & 0xFF); +// bytes[byteIndex++] = (byte) ((l >>> 32) & 0xFF); +// bytes[byteIndex++] = (byte) ((l >>> 40) & 0xFF); +// bytes[byteIndex++] = (byte) ((l >>> 48) & 0xFF); +// bytes[byteIndex++] = (byte) ((l >>> 56) & 0xFF); +// } + private final byte bufferPop() { + return bytes[--byteIndex]; + } + + final class DebugStm { + StmEnum e; + int r; + int p; + int o; + ClusterUID pc; + ClusterUID oc; + + DebugStm(StmEnum e, int r, int p, ClusterUID pc, int o, ClusterUID oc) { + this.e = e; + this.r = r; + this.p = p; + this.o = o; + this.pc = pc; + this.oc = oc; + } + + @Override + public String toString() { + short ri = ClusterTraitsBase.getResourceIndexFromResourceKeyNoThrow(r); + short pi = ClusterTraitsBase.getResourceIndexFromResourceKeyNoThrow(p); + short oi = ClusterTraitsBase.getResourceIndexFromResourceKeyNoThrow(o); + return "" + e + " rk=" + r + " ri=" + ri + " rc=" + clusterUID + + " pk=" + p + " pi=" + pi + " pc=" + pc + + " ok=" + o + " oi=" + oi + " oc=" + oc; + } + + public String toString2() { + return "" + e + " r=" + r + " rc=" + clusterUID + " p=" + p + + " pc=" + pc + " o=" + o + " oc=" + oc; + } + + public String toString3() { + short ri = ClusterTraitsBase.getResourceIndexFromResourceKeyNoThrow(r); + short pi = ClusterTraitsBase.getResourceIndexFromResourceKeyNoThrow(p); + short oi = ClusterTraitsBase.getResourceIndexFromResourceKeyNoThrow(o); + return "" + e + " ri=" + ri + + " pi=" + pi + " pc=" + pc + + " oi=" + oi + " oc=" + oc; + } + } + + private List debugStms = new ArrayList(); + + @SuppressWarnings("unused") + private final void addStm(Change c, StmEnum stmEnum) { + + if (DEBUG_STAT) + ++info.nStms; + if (DEBUG || DEBUG_CCS) { + DebugStm d = new DebugStm(stmEnum, c.key0, c.key1, c.clusterUID1, c.key2, c.clusterUID2); + if (DEBUG_CCS) + debugStms.add(d); + if (DEBUG) { + printlnd(d.toString3() + " offset=" + byteIndex); + } + } + // int opPos = buffer.position(); + int opPos = byteIndex++; + // buffer.put((byte)0); // operation code + // addByte((byte)0); + + boolean done = true; + + ClusterEnum a = addIndexAndCluster(c.key1, c.clusterUID1, c.lookIndex1, c.lookup1); + byte ab = 0; + + // ForeignShort = byte + // Local = short + // ForeignLong = 8 byte + if (a != ClusterEnum.ForeignShort) { + ab = bufferPop(); + done = false; + } + + ClusterEnum b = addIndexAndCluster(c.key2, c.clusterUID2, c.lookIndex2, c.lookup2); + byte bb = 0; + if (b != ClusterEnum.ForeignShort) { + bb = bufferPop(); + done = false; + } + + int ri = ClusterTraitsBase.getResourceIndexFromResourceKeyNoThrow(c.key0); + if (ClusterTraitsSmall.isIllegalResourceIndex(ri)) + throw new RuntimeDatabaseException("Assertion error. Illegal resource index=" + ri); + bytes[byteIndex++] = (byte)ri; // index low byte + if(!done) { + Data data = ClusterEnum.getData(stmEnum, a, b); + int left = 6 - data.bits; + int op = ri >>> (8 + left); + ri >>>= 8; + ri &= (1 << left) - 1; + if (a != ClusterEnum.ForeignShort) { + ri |= ab << left; + left += 6; + } + if (b != ClusterEnum.ForeignShort) { + ri |= bb << left; + left += 6; + } + switch (data.bytes) { + default: + throw new RuntimeDatabaseException("Assertion error. Illegal number of bytes=" + data.bytes); + case 2: + bytes[byteIndex++] = (byte)(ri & 0xFF); + bytes[byteIndex++] = (byte)((ri >>> 8) & 0xFF); + break; + case 1: + bytes[byteIndex++] = (byte)(ri & 0xFF); + break; + case 0: + break; + } + op |= data.mask; + this.bytes[opPos] = (byte)op; + } else { + if (stmEnum == StmEnum.Add) + bytes[opPos] = (byte)((ri >>> 8) + 64); + else + bytes[opPos] = (byte)((ri >>> 8) + 128); + } + if (DEBUG_STAT) { + if (a == ClusterEnum.Local && b == ClusterEnum.Local) { + ++info.nLocal; + } else if (a == ClusterEnum.Local || b == ClusterEnum.Local) { + ++info.nPartly; + } else { + ++info.nForeign; + } + } + if (foreignTable.size() > 252) + flush(); +// throw new UnsupportedOperationException(); + //flushInternal(graphSession, clusterUID); + } + + private final int modiValue(int ri, long value_offset, byte[] bytes, int offset, int size) { + if (DEBUG) + printlnd("Modify value ri=" + ri + " vo=" + value_offset + " size=" + size + " total=" + bytes.length); + if (ClusterTraitsBase.isIllegalResourceIndex(ri)) + throw new RuntimeDatabaseException("Assertion error. Illegal resource index=" + ri); + if (value_offset > (1L << 58 - 1)) + throw new RuntimeDatabaseException("Illegal value offset=" + + value_offset); + if (size < 0 || size > MAX_FIXED_BYTES - 1) + throw new RuntimeDatabaseException("Illegal value size=" + size); + if (offset + size > bytes.length) + throw new RuntimeDatabaseException("Illegal value size=" + size); + checkBufferSpace(12 + size); + addByte(OpEnum.Modify.getOrMask()); + ri |= (value_offset >>> 56) << 14; // add top two bits + addShort((short) ri); + value_offset &= (1L << 56) - 1; + addLong7(value_offset); + addShort((short) size); + if (DEBUG) + System.out.println("Modify value fixed part end offset=" + byteIndex); + int copied = Math.min(size, this.bytes.length - byteIndex); + System.arraycopy(bytes, offset, this.bytes, byteIndex, copied); + byteIndex += size; + return copied; + } + +// private final void modiValueBig(int ri, long voffset, int left, byte[] bytes, int offset) { +// checkBufferSpace(0); +// int current = Math.min(this.bytes.length - byteIndex - 12, left); +// if(current >= 0) { +// int written = modiValue(ri, voffset, bytes, offset, current); +// voffset += written; +// offset += written; +// left -= written; +// } +//// flushInternal(graphSession, clusterUID); +// while (left > 0) { +// int length = Math.min(left, (1 << 16) - 1); +// if (DEBUG) +// printlnd("Modify big value ri=" + ri + " vo=" + voffset + " len=" + length); +// int psize = length + 12; +//// setHeaderVectorSize(psize); +// byte[] message = new byte[psize/*+HEADER_SIZE*/]; +//// System.arraycopy(header, 0, message, 0, HEADER_SIZE); +// int to = 0; +// Bytes.write(message, to++, OpEnum.Modify.getOrMask()); +// short index = (short)(ri | (voffset >>> 56)<<14); // add top two bits +// Bytes.writeLE(message, to, index); to += 2; +// Bytes.writeLE7(message, to, voffset & ((1L << 56) - 1)); to += 7; +// Bytes.writeLE(message, to, (short)length); to += 2; +// System.arraycopy(bytes, offset, message, to, length); +//// graphSession.updateCluster(new UpdateClusterFunction(message)); +// voffset += length; +// offset += length; +// left -= length; +// } +// } + + private final int setValueBig(int ri, byte[] bytes, int length_) { + checkBufferSpace(12); + int sum = 0; + int voffset = 0; + int offset = 0; + int left = length_; + while (left > 0) { + int length = Math.min(left, MAX_FIXED_BYTES - 12 - byteIndex); + if (DEBUG) + printlnd("Set big value ri=" + ri + " vo=" + voffset + " len=" + length); + int written = modiValue(ri, voffset, bytes, offset, length); + sum += written; + voffset += written; + offset += written; + left -= written; + checkBufferSpace(12); + } + return sum; + } + + private final int setValueSmall(int ri, byte[] bytes, int length) { + checkBufferSpace(5 + length); + int pos = byteIndex; + int i = length << 14 | ri; + if (length < 32) { + byte op = (byte) (OpEnum.SetShort.getOrMask() | length >>> 2); + addByte(op); + short s = (short) i; + addShort(s); + } else { + addByte(OpEnum.Set.getOrMask()); + addInt(i); + } + System.arraycopy(bytes, 0, this.bytes, byteIndex, length); + byteIndex += length; + int len = byteIndex - pos; + return len; + } + + final void setValue(short index, byte[] bytes) { + setValue(index, bytes, bytes.length); + } + + final public void setValue(short index, byte[] bytes, int length) { + checkInitialization(); + if (ClusterTraitsBase.isIllegalResourceIndex(index)) + throw new RuntimeDatabaseException("Assertion error. Illegal resource index=" + index); + if (DEBUG) + printlnd("Set value ri=" + index + + " len=" + length + + " bytes=" + Arrays.toString(Arrays.copyOfRange(bytes, 0, Math.min(10, length)))); + int len; + /* + * The limit for the cluster stream is (1<18)-1 but this avoids the + * conversion to big cluster. + */ + if (length > ClusterTraitsSmall.VALUE_SIZE_MAX) + len = setValueBig(index, bytes, length); + else + len = setValueSmall(index, bytes, length); + if (DEBUG_STAT) { + ++info.nValues; + info.sValues += len + length; + } + } + +// final void setValue(Change c, byte[] bytes, int length) { +// short ri = ClusterTraitsBase.getResourceIndexFromResourceKeyNoThrow(c.key0); +// setValue(ri, bytes, length); +// c.initValue(); +// } + +// final void modiValue(Change c, long voffset, int length, byte[] bytes, int offset) { +// checkInitialization(); +// int ri = ClusterTraitsBase.getResourceIndexFromResourceKeyNoThrow(c.key0); +// if (DEBUG) +// printlnd("Modify value ri=" + ri +// + " voff=" + voffset +// + " vlen=" + length +// + " blen=" + bytes.length +// + " boff=" + offset +// + " bytes=" + Arrays.toString(Arrays.copyOfRange(bytes, 0, Math.min(10, bytes.length)))); +// modiValueBig(ri, voffset, length, bytes, offset); +// c.init(); +// if (DEBUG_STAT) { +// ++info.nValues; +// info.sValues += length; +// } +// } + final void setImmutable(boolean immutable) { + checkInitialization(); +// clusterChange2.setImmutable(immutable); + } + final void undoValueEx(int resourceIndex) { + checkInitialization(); +// clusterChange2.undoValueEx(resourceIndex); + } + final void setDeleted(boolean deleted) { + checkInitialization(); +// clusterChange2.setDeleted(deleted); + } + final void corrupt() { + checkInitialization(); + addByte((byte)0); + } + + public byte[] getBytes() { + byte[] copy = new byte[byteIndex]; + System.arraycopy(bytes, 0, copy, 0, byteIndex); + return copy; + } + + /** + * @param graphSession + * @param clusterId + * @return true if actually flushed something + */ + final boolean flush(/*GraphSession graphSession,*/ ClusterUID clusterUID) { + throw new UnsupportedOperationException(); +// if (byteIndex > 0) { +// if(DebugPolicy.REPORT_CLUSTER_STREAM) +// System.err.println("Flush cluster change set stream " + this); +// setHeaderVectorSize(byteIndex); +// byte[] copy = new byte[byteIndex + HEADER_SIZE]; +// System.arraycopy(header, 0, copy, 0, HEADER_SIZE); +// System.arraycopy(bytes, 0, copy, HEADER_SIZE, byteIndex); +// UpdateClusterFunction updateClusterFunction = new UpdateClusterFunction(copy); +// if (DEBUG_CCS) { +// for (DebugStm stm : debugStms) +// printlnd(stm.toString2()); +// debugStms.clear(); +// } +// if (DEBUG_STAT) { +// info.tot = updateClusterFunction.operation.length; +// printlnd("ReallyFlush: " + info.toString()); +// sum.add(info); +// printlnd("ReallyFlush sum: " + sum.toString()); +// } +// // long start = System.nanoTime(); +// graphSession.updateCluster(updateClusterFunction); +// // long duration = System.nanoTime() - start; +// // duration2 += duration; +// // System.err.println("updateCluster " + 1e-9*duration); +// // System.err.println("updateCluster total " + 1e-9*duration2); +// clear(); +// clusterChange2.flush(graphSession); +// return true; +// } else if (clusterChange2.isDirty()) { +// clusterChange2.flush(graphSession); +// clear(); +// return true; +// } else if (flushed) { +// flushed = false; +// return true; +// } else { +// return true; +// } + } + + final void flushInternal(ClusterUID clusterUID) { + throw new UnsupportedOperationException(); +// flush(graphSession, clusterUID); +// flushed = true; + } + + final class ForeignTable { + private final TLongIntHashMap table = new TLongIntHashMap(); + + private long createKey(short index, long cluster) { + assert (cluster <= (1L << 48) - 1); + return (cluster << 14) | index; + } + + public int get(short index, long cluster) { + int value = table.get(createKey(index, cluster)); + if (DEBUG) + printlnd("ForeignTable get c=" + clusterUID + " i=" + + (value - 1) + " r=" + index + " rc=" + cluster); + return value; + } + + public int put(short index, long cluster, int value) { + if (DEBUG) + printlnd("ForeignTable put c=" + clusterUID + " i=" + + (value - 1) + " r=" + index + " rc=" + cluster); + return table.put(createKey(index, cluster), value); + } + + public int size() { + return table.size(); + } + + public void clear() { + table.clear(); + } + } + + @Override + public int hashCode() { + return 31*clusterUID.hashCode(); + } + + @Override + public boolean equals(Object object) { + if (this == object) + return true; + else if (object == null) + return false; + else if (!(object instanceof ClusterChange)) + return false; + ClusterChange r = (ClusterChange)object; + return r.clusterUID.equals(clusterUID); + } + + public void flush() { + + if(byteIndex > 0) { + + final ClusterUID cuid = clusterUID; + + byte[] block = getBytes(); + byte[] raw = new byte[block.length + 28]; + Bytes.writeLE(raw, 0, 1); + System.arraycopy(cuid.asBytes(), 0, raw, 4, 16); + Bytes.writeLE(raw, 20, block.length); + System.arraycopy(block, 0, raw, 24, block.length); + Bytes.writeLE(raw, 24+block.length, 0); + + ByteBuffer rawBB = ByteBuffer.wrap(raw); + ByteBuffer outputBB = ByteBuffer.allocate(raw.length + raw.length/8); + //outputBB.order(ByteOrder.LITTLE_ENDIAN); + int compressedSize = Compressions.get(Compressions.LZ4).compressBuffer(rawBB, 0, raw.length, outputBB, 0); + + byte[] data_ = null; + if(compressedSize < raw.length) { + data_ = new byte[compressedSize]; + outputBB.get(data_,0,compressedSize); + } else { + data_ = raw; + } + + byte[] data = new byte[data_.length+24]; + Bytes.writeLE(data, 0, 0); + Bytes.writeLE(data, 4, 0); + Bytes.writeLE(data, 8, raw.length); + Bytes.writeLE(data, 12, raw.length); + Bytes.writeLE(data, 16, data_.length); + System.arraycopy(data_, 0, data, 20, data_.length); + Bytes.writeLE(data, 20+data_.length, 0); + + stream.add(Pair.make(clusterUID, data)); + clear(); + initBuffer(); + + } + } + +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/ClusterChange2.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/ClusterChange2.java new file mode 100644 index 000000000..472b4d7b7 --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/ClusterChange2.java @@ -0,0 +1,70 @@ +package org.simantics.acorn.internal; + + +public class ClusterChange2 { + public static final int VERSION = 2; + public static final byte SET_IMMUTABLE_OPERATION = 1; // + public static final byte UNDO_VALUE_OPERATION = 2; // + private static final int INCREMENT = 1<<10; +// private boolean dirty = false; +// private byte[] bytes; +// private int byteIndex; +// private ClusterUID clusterUID; +// private ClusterImpl cluster; +// ClusterChange2(ClusterUID clusterUID, ClusterImpl cluster) { +// this.clusterUID = clusterUID; +// this.cluster = cluster; +// init(); +// } +// void init() { +//// System.err.println("clusterChange2 dirty " + cluster.clusterId); +// dirty = false; +// bytes = new byte[INCREMENT]; +// byteIndex = 0; +// addInt(0); // Size of byte vector. Set by flush. +// addInt(VERSION); +// byteIndex = clusterUID.toByte(bytes, 8); +// } +// boolean isDirty() { +// return dirty; +// } +// void flush(GraphSession graphSession) { +//// System.err.println("flush2 clusterChange2 " + dirty + this); +// if (!dirty) +// return; +// Bytes.writeLE(bytes, 0, byteIndex - 4); +// byte[] ops = Arrays.copyOf(bytes, byteIndex); +//// System.err.println("flush2 clusterChange2 " + cluster.clusterId + " " + ops.length + " bytes."); +// graphSession.updateCluster(new UpdateClusterFunction(ops)); +// init(); +// } +// void setImmutable(boolean immutable) { +// dirty = true; +// addByte(SET_IMMUTABLE_OPERATION); +// addByte((byte)(immutable ? -1 : 0)); +// } +// void undoValueEx(int resourceIndex) { +// dirty = true; +// addByte(UNDO_VALUE_OPERATION); +// addInt(resourceIndex); +// } +// private final void checkSpace(int len) { +// if (bytes.length - byteIndex > len) +// return; +// bytes = Arrays.copyOf(bytes, bytes.length + len + INCREMENT); +// } +// private final void addByte(byte value) { +// checkSpace(1); +// bytes[byteIndex++] = value; +// } +// private final void addInt(int value) { +// checkSpace(4); +// Bytes.writeLE(bytes, byteIndex, value); +// byteIndex += 4; +// } +//// private void addLong(long value) { +//// checkSpace(8); +//// Bytes.writeLE(bytes, byteIndex, value); +//// byteIndex += 8; +//// } +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/ClusterStream.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/ClusterStream.java new file mode 100644 index 000000000..2b1ae1979 --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/ClusterStream.java @@ -0,0 +1,437 @@ +/******************************************************************************* + * Copyright (c) 2007, 2010 Association for Decentralized Information Management + * in Industry THTH ry. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * VTT Technical Research Centre of Finland - initial API and implementation + *******************************************************************************/ +package org.simantics.acorn.internal; + +import java.util.ArrayList; + +import org.simantics.db.exception.DatabaseException; +import org.simantics.db.service.ClusterUID; + +final public class ClusterStream { + +// // public static long duration2 = 0; +// + public static final boolean DEBUG = false; + public static final byte NULL_OPERATION = 0; + public static final byte CREATE_OPERATION = 1; + public static final byte SET_OPERATION = 4; + public static final byte MODI_OPERATION = 6; + public static final byte KILL_OPERATION = 7; +// boolean off = false; +// public GraphSession graphSession; +// final SessionImplSocket session; +//// private int flushCount = 0; +// final private boolean alwaysOff; +// private int stamp; +// private int acceptedStamp; +// private boolean dirty = false; +//// final private ArrayList clusterChanges = new ArrayList(); +// +// final ClusterChangeManager changes = new ClusterChangeManager(); +// +//// final TLongObjectHashMap clusterChanges = new TLongObjectHashMap(); +// +// // private final Change lastChange = new Change(); +// ClusterStream(SessionImplSocket session, GraphSession graphSession, +// boolean alwaysOff) { +// this.session = session; +// this.graphSession = graphSession; +// this.alwaysOff = alwaysOff; +// } +// +// +// boolean isDirty() { +// return dirty; +// } +// +// void markDirty() { +// dirty = true; +// } +// +// void setOff(boolean value) { +// if (alwaysOff) { +// off = true; +// } else { +// off = value; +// } +// } +// +// boolean getOff() { +// return off; +// } +// +// void createResource(ClusterChange cc, short operationIndex, ClusterUID clusterUID) { +// if (off) +// return; +// assert (null != cc); +// assert (0 != operationIndex); +// assert (!ClusterUID.Null.equals(clusterUID)); +// if (DEBUG) +// System.out.println("DEBUG: Created resource index=" + operationIndex + " cluster=" + clusterUID); +// cc.createResource(operationIndex); +// } +// +// final void addStatementIndex(Change change, int key, ClusterUID clusterUID, byte op) { +// if (off) +// return; +// assert (key > 0); +// assert (null != change); +// assert (!ClusterUID.Null.equals(clusterUID)); +// change.addStatementIndex(key, clusterUID, op); +// } +// +// void addStatement(ClusterChange cc, Change change) { +// if (off) +// return; +// assert (null != cc); +// assert (null != change); +// cc.addChange(change); +// } +// +// void cancelStatement(Change change) { +// if (off) +// return; +// assert (null != change); +// change.init(); +// } +// +// void removeStatement(ClusterChange cc, Change change, long clusterId) { +// if (off) +// return; +// assert (null != cc); +// assert (null != change); +// cc.addChange(change); +// } +// +// void cancelValue(Change change) { +// if (off) +// return; +// assert (null != change); +// change.init(); +// } +// +// void removeValue(ClusterChange cc, Change change, long clusterId) { +// if (off) +// return; +// // ClusterChange cc = getClusterChange(clusterId); +// assert (null != cc); +// assert (null != change); +// cc.addChange(change); +// } +// +// void setValue(ClusterChange cc, Change change, long clusterId, byte[] bytes, int length) { +// if (off) +// return; +// assert (null != cc); +// assert (null != change); +// // ClusterChange cc = getClusterChange(clusterId); +// cc.setValue(change, bytes, length); +// } +// +// void modiValue(ClusterChange cc, Change change, long clusterId, +// long voffset, int length, byte[] bytes, int offset) { +// assert (null != cc); +// assert (null != change); +// cc.modiValue(change, voffset, length, bytes, offset); +// } +// +// void undoValueEx(ClusterChange cc, Change change, int resourceIndex) { +// cc.undoValueEx(resourceIndex); +// } +// void setImmutable(ClusterChange cc, Change change, long clusterId, boolean immutable) { +// if (off) +// return; +// cc.setImmutable(immutable); +// } +// public void corruptCluster(ClusterChange cc, long clusterId) +// throws DatabaseException { +// if (off) +// return; +// if (DEBUG) +// System.out.println("ClusterStream.corrupt cid=" + clusterId + "."); +// assert (null != cc); +// cc.corrupt(); +// } +// +// int getStamp() { +// return stamp; +// } +// +// void flush() { +// if (off) +// return; +//// flushCount++; +// return; +// } +// +// void flush(long clusterId) { +// if (off) +// return; +// ClusterUID clusterUID = session.clusterTable.clusterIds.getClusterUID(clusterId); +// ArrayList ccs = new ArrayList(); +// for(ClusterChange cc : changes.get()) { +// if(cc.clusterUID.equals(clusterUID)) { +// if (cc.flush(graphSession, cc.clusterUID)) { +// ccs.add(cc); +// if (stamp == acceptedStamp) +// ++stamp; +// } else { +//// System.err.println("kasdi"); +// } +// } +// } +// changes.remove(ccs); +// } +// +// /** +// * @return true if the stream has accepted all changes +// */ +// public boolean reallyFlush() { +// // Last possibility to mark clusters immutable before write only clusters are gone +// session.handleCreatedClusters(); +// // These shall be requested from server +// session.clusterTable.removeWriteOnlyClusters(); +// if (!off && changes.size() > 0) { +// for(ClusterChange cc : changes.get()) { +// if (cc.flush(graphSession, cc.clusterUID)) +// if (stamp == acceptedStamp) +// ++stamp; +// } +// changes.clear(); +// } +// dirty = false; +// return hasAcceptedAllChanges(); +// } +// +// /** +// * Clear all changes and set stream status to empty. +// */ +// public void clear() { +// changes.clear(); +// acceptedStamp = stamp; +// dirty = false; +// } +// +// private boolean hasAcceptedAllChanges() { +// return stamp == acceptedStamp; +// } +// +// void accept() { +// acceptedStamp = stamp; +// } +// +// + + static class DebugInfo { + long nStms; + long nLocal; + long nPartly; + long nForeign; + long nValues; + long sValues; + long sForeign; + long tot; + + void clear() { + nStms = 0; + nLocal = 0; + nPartly = 0; + nForeign = 0; + sForeign = 0; + nValues = 0; + sValues = 0; + tot = 0; + } + + void add(DebugInfo di) { + nStms += di.nStms; + nLocal += di.nLocal; + nPartly += di.nPartly; + nForeign += di.nForeign; + sForeign += di.sForeign; + nValues += di.nValues; + sValues += di.sValues; + tot += di.tot; + } + + @Override + public String toString() { + return "val=" + nValues + " stm=" + nStms + " loc=" + nLocal + + " par=" + nPartly + " ful=" + nForeign + " for=" + + sForeign + " vat=" + sValues + " tot=" + tot; + } + } + + enum StmEnum { + Add(0, (byte) 0), Remove(1, (byte) 0x20); + StmEnum(int ordinal, byte mask) { + this.ordinal = ordinal; + this.mask = mask; + } + + public int ordinal; + private byte mask; + + byte getOrMask() { + return mask; + } + } + + final static class Data { + + final byte mask; // or mask for operation code (don't care bits are zero) + final short bits; // how many bits are reserved for resource index (0,2,4,6) + final int bytes; + + Data(int mask, int bits, ClusterEnum a, ClusterEnum b) { + this.mask = (byte) (mask << bits); + this.bits = (short) bits; + this.bytes = bytes(bits, a, b); + } + + private static int bytes(int bits, ClusterEnum a, ClusterEnum b) { + int left = 6 - bits; + if (a != ClusterEnum.ForeignShort) { + left += 6; + } + if (b != ClusterEnum.ForeignShort) { + left += 6; + } + int bytes = left >>> 3; + if ((left & 7) != 0) + bytes++; + return bytes; + } + + } + + enum ClusterEnum { + Local(0), ForeignShort(1), ForeignLong(2); + public int ordinal; + + ClusterEnum(int ordinal) { + this.ordinal = ordinal; + } + + static Data[][][] maps = new Data[2][3][3]; + static { + // mask: 00000000 + // op: 000000|r12-13 + // p1 + // o1 + // r0-7 + // o2 | p2 | r8-11 + maps[StmEnum.Add.ordinal][Local.ordinal][Local.ordinal] = new Data( + 0, 2, Local, Local); + // mask: 11000000 + // op: 1100 | r10-13 + // p1 + // o for index + // r0-7 + // p2 | ri 8-9 + maps[StmEnum.Add.ordinal][Local.ordinal][ForeignShort.ordinal] = new Data( + 12, 4, Local, ForeignShort); + // mask: 00001000 + // op: 000010 | r12-13 + maps[StmEnum.Add.ordinal][Local.ordinal][ForeignLong.ordinal] = new Data( + 2, 2, Local, ForeignLong); + // mask: 11010000 + // op: 1101 | r10-13 + maps[StmEnum.Add.ordinal][ForeignShort.ordinal][Local.ordinal] = new Data( + 13, 4, ForeignShort, Local); + + // mask: 01000000 + // op: 01 | r8-13 + // p for index + // o for index + // r0-7 + maps[StmEnum.Add.ordinal][ForeignShort.ordinal][ForeignShort.ordinal] = new Data( + 1, 6, ForeignShort, ForeignShort); + // mask: 11100000 + // op: 1110 | r10-13 + maps[StmEnum.Add.ordinal][ForeignShort.ordinal][ForeignLong.ordinal] = new Data( + 14, 4, ForeignShort, ForeignLong); + // mask: 00010000 + // op: 000100 | r12-13 + maps[StmEnum.Add.ordinal][ForeignLong.ordinal][Local.ordinal] = new Data( + 4, 2, ForeignLong, Local); + // mask: 11110000 + // op: 1111 | r10-13 + maps[StmEnum.Add.ordinal][ForeignLong.ordinal][ForeignShort.ordinal] = new Data( + 15, 4, ForeignLong, ForeignShort); + // mask: 00011000 + // op: 000110 | r12-13 + maps[StmEnum.Add.ordinal][ForeignLong.ordinal][ForeignLong.ordinal] = new Data( + 6, 2, ForeignLong, ForeignLong); + + // mask: 00000100 + // op: 000001 | r12-13 + maps[StmEnum.Remove.ordinal][Local.ordinal][Local.ordinal] = new Data( + 1, 2, Local, Local); + // mask: 01100001 + // op: 01100001 + // p1 + // o for index + // r0-7 + // p2 | ri 8-13 + maps[StmEnum.Remove.ordinal][Local.ordinal][ForeignShort.ordinal] = new Data( + 49, 0, Local, ForeignShort); + // mask: 00001100 + // op: 000011 | r12-13 + maps[StmEnum.Remove.ordinal][Local.ordinal][ForeignLong.ordinal] = new Data( + 3, 2, Local, ForeignLong); + // mask: 00100000 + // op: 0010 | r10-13 + maps[StmEnum.Remove.ordinal][ForeignShort.ordinal][Local.ordinal] = new Data( + 2, 4, ForeignShort, Local); + // mask: 10000000 + // op: 10 | r8-13 + maps[StmEnum.Remove.ordinal][ForeignShort.ordinal][ForeignShort.ordinal] = new Data( + 2, 6, ForeignShort, ForeignShort); + // mask: 00110010 + // op: 00110010 + maps[StmEnum.Remove.ordinal][ForeignShort.ordinal][ForeignLong.ordinal] = new Data( + 50, 0, ForeignShort, ForeignLong); + // mask: 00010100 + // op: 000101 | r12-13 + maps[StmEnum.Remove.ordinal][ForeignLong.ordinal][Local.ordinal] = new Data( + 5, 2, ForeignLong, Local); + // mask: 00110011 + // op: 00110011 + maps[StmEnum.Remove.ordinal][ForeignLong.ordinal][ForeignShort.ordinal] = new Data( + 51, 0, ForeignLong, ForeignShort); + // mask: 00011100 + // op: 000111 | r12-13 + maps[StmEnum.Remove.ordinal][ForeignLong.ordinal][ForeignLong.ordinal] = new Data( + 7, 2, ForeignLong, ForeignLong); + } + + static Data getData(StmEnum s, ClusterEnum a, ClusterEnum b) { + return maps[s.ordinal][a.ordinal][b.ordinal]; + // return maps.get(s).get(a).get(b); + } + } + + enum OpEnum { + Create((byte) 52), Set((byte) 53), SetShort((byte) 56), Delete( + (byte) 54), Modify((byte) 55); + OpEnum(byte mask) { + this.mask = mask; + } + + public byte getOrMask() { + return mask; + } + + private byte mask; + } +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/ClusterSupport2.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/ClusterSupport2.java new file mode 100644 index 000000000..b1b1e8365 --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/ClusterSupport2.java @@ -0,0 +1,352 @@ +package org.simantics.acorn.internal; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; +import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; + +import org.simantics.acorn.ClusterManager; +import org.simantics.acorn.exception.AcornAccessVerificationException; +import org.simantics.acorn.exception.IllegalAcornStateException; +import org.simantics.db.Session; +import org.simantics.db.exception.DatabaseException; +import org.simantics.db.impl.ClusterBase; +import org.simantics.db.impl.ClusterI; +import org.simantics.db.impl.ClusterSupport; +import org.simantics.db.impl.IClusterTable; +import org.simantics.db.service.ClusterUID; + +import gnu.trove.map.hash.TIntObjectHashMap; + +public class ClusterSupport2 implements ClusterSupport, IClusterTable { + + final private static boolean DEBUG = false; + + public ClusterManager impl; + + public TIntObjectHashMap uidCache = new TIntObjectHashMap(); + + public ClusterSupport2(ClusterManager impl) { + this.impl = impl; + } + + @Override + public int createClusterKeyByClusterUID(ClusterUID clusterUID, long clusterId) { + throw new UnsupportedOperationException(); + } + + @Override + public ClusterBase getClusterByClusterUIDOrMake(ClusterUID clusterUID) { + try { + return impl.getClusterByClusterUIDOrMake(clusterUID); + } catch (DatabaseException e) { + e.printStackTrace(); + return null; + } + } + + @Override + public ClusterBase getClusterByClusterId(long clusterId) { + throw new UnsupportedOperationException(); + } + + @Override + public ClusterBase getClusterByClusterKey(int clusterKey) { + throw new UnsupportedOperationException(); + } + + ReentrantReadWriteLock uidLock = new ReentrantReadWriteLock(); + ReadLock uidRead = uidLock.readLock(); + WriteLock uidWrite = uidLock.writeLock(); + + @Override + public ClusterUID getClusterUIDByResourceKey(int resourceKey) throws DatabaseException { + + ClusterUID cuid; + + uidRead.lock(); + cuid = uidCache.get(resourceKey >> 12); + uidRead.unlock(); + if(cuid != null) return cuid; + uidWrite.lock(); + cuid = uidCache.get(resourceKey >> 12); + if(cuid == null) { + cuid = impl.getClusterUIDByResourceKeyWithoutMutex(resourceKey); + uidCache.put(resourceKey >> 12, cuid); + } + uidWrite.unlock(); + + return cuid; + + } + + @Override + public int getClusterKeyByClusterUIDOrMake(ClusterUID clusterUID) { + try { + return impl.getClusterKeyByClusterUIDOrMakeWithoutMutex(clusterUID); + } catch (IllegalAcornStateException | AcornAccessVerificationException e) { + throw new RuntimeException(e); + } + } + + @Override + public int getClusterKeyByClusterUIDOrMake(long id1, long id2) { + throw new UnsupportedOperationException(); + } + + @Override + public ClusterBase getClusterByResourceKey(int resourceKey) { + throw new UnsupportedOperationException(); +// return impl.getClusterByResourceKey(resourceKey); + } + + @Override + public long getClusterIdOrCreate(ClusterUID clusterUID) { + return impl.getClusterIdOrCreate(clusterUID); + } + + @Override + public void addStatement(Object cluster) { + // nop + } + + @Override + public void cancelStatement(Object cluster) { + // nop + } + + @Override + public void removeStatement(Object cluster) { + // nop + } + + @Override + public void removeValue(Object cluster) { + // nop + } + + @Override + public void setImmutable(Object cluster, boolean immutable) { + // nop + } + + @Override + public void setDeleted(Object cluster, boolean deleted) { + // TODO Auto-generated method stub + + } + + + + @Override + public void cancelValue(Object cluster) { + throw new UnsupportedOperationException(); + } + + @Override + public void setValue(Object cluster, long clusterId, byte[] bytes, + int length) { + // nop + } + + @Override + public void modiValue(Object _cluster, long clusterId, long voffset, + int length, byte[] bytes, int offset) { + // nop + } + + @Override + public void createResource(Object cluster, short resourceIndex, + long clusterId) { + // No op + } + + @Override + public void addStatementIndex(Object cluster, int resourceKey, + ClusterUID clusterUID, byte op) { + // No op + } + + @Override + public void setStreamOff(boolean setOff) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean getStreamOff() { + return true; + } + + + private static class ResourceSegment { + public long valueSize; + + public byte[] bytes; + + ResourceSegment(long valueSize, byte[] bytes) { + this.valueSize = valueSize; + this.bytes = bytes; + } + } + + public ResourceSegment getResourceSegment(int resourceIndex, ClusterUID clusterUID, long offset, short size) throws DatabaseException { + if (DEBUG) + System.out.println("DEBUG: getResourceSegment ri=" + resourceIndex + " cid=" + clusterUID + " offset=" + offset + " size=" + size); + + try { + org.simantics.db.Database.Session.ResourceSegment t = impl.getResourceSegment(clusterUID.asBytes(), resourceIndex, offset, size); + return new ResourceSegment(t.getValueSize(), t.getSegment()); + } catch (AcornAccessVerificationException | IllegalAcornStateException e) { + throw new DatabaseException(e); + } + } + + protected byte[] getValueBig(ClusterBase cluster, int resourceIndex, int offset, int length) throws DatabaseException { + + assert(offset == 0); + assert(length == 0); + + ClusterUID clusterUID = cluster.clusterUID; + + try { + return impl.getResourceFile(clusterUID.asBytes(), resourceIndex); + } catch (AcornAccessVerificationException | IllegalAcornStateException e) { + throw new DatabaseException(e); + } + } + + protected InputStream getValueStreamBig(ClusterBase cluster, final int resourceIndex, int offset, int length) throws DatabaseException { + + final ClusterUID clusterUID = cluster.clusterUID; + + if (DEBUG) + System.out.println("DEBUG: getResourceFile ri=" + resourceIndex + " cid=" + clusterUID + " off=" + offset + " len=" + length); + final int IMAX = 0xFFFF; + short slen = (short)Math.min(length != 0 ? length : IMAX, IMAX); + final ResourceSegment s = getResourceSegment(resourceIndex, clusterUID, offset, slen); + if (s.valueSize < 0) + throw new DatabaseException("Failed to get value for resource index=" + resourceIndex + + " cluster=" + clusterUID + " off=" + offset + " len=" + length + " (1)."); + int ilen = (int)slen & 0xFFFF; + assert(s.bytes.length <= ilen); + if (0 == length) { + if (s.valueSize > Integer.MAX_VALUE) + throw new DatabaseException("Failed to get value for resource index=" + resourceIndex + + " cluster=" + clusterUID + " off=" + offset + " len=" + length + + ". Value size=" + s.valueSize + " (2)."); + length = (int)s.valueSize; + } + long rSize = s.valueSize - offset; + if (rSize < length) + throw new DatabaseException("Failed to get value for resource index=" + resourceIndex + + " cluster=" + clusterUID + " off=" + offset + " len=" + length + + ". Value size=" + s.valueSize + " (3)."); + else if (length <= IMAX) + return new ByteArrayInputStream(s.bytes); + + final int finalLength = length; + + return new InputStream() { + + int left = finalLength; + long valueOffset = 0; + int offset = 0; + ResourceSegment _s = s; + + @Override + public int read() throws IOException { + + if(left <= 0) + throw new IOException("left <= 0 for " + _s); + + if(offset == _s.bytes.length) { + short slen = (short)Math.min(left, IMAX); + valueOffset += _s.bytes.length; + try { + _s = getResourceSegment(resourceIndex, clusterUID, valueOffset, slen); + } catch (DatabaseException e) { + throw new IOException(e); + } + offset = 0; + } + + left--; + int result = _s.bytes[offset++]; + if(result < 0) result += 256; + return result; + + } + + }; + + } + + @Override + public InputStream getValueStreamEx(int resourceIndex, long clusterId) + throws DatabaseException { + ClusterBase cluster = impl.getClusterByClusterUIDOrMakeProxy(ClusterUID.make(0, clusterId)); + return getValueStreamBig(cluster, resourceIndex, 0, 0); + } + + @Override + public byte[] getValueEx(int resourceIndex, long clusterId) + throws DatabaseException { + ClusterBase cluster = impl.getClusterByClusterUIDOrMakeProxy(ClusterUID.make(0, clusterId)); + return getValueBig(cluster, resourceIndex, 0, 0); + } + + @Override + public byte[] getValueEx(int resourceIndex, long clusterId, long voffset, + int length) throws DatabaseException { + throw new UnsupportedOperationException(); + } + + @Override + public long getValueSizeEx(int resourceIndex, long clusterId) + throws DatabaseException { + throw new UnsupportedOperationException(); + } + + @Override + public int wait4RequestsLess(int limit) throws DatabaseException { + throw new UnsupportedOperationException(); + } + + @Override + public Session getSession() { + return null; + } + + @Override + public IClusterTable getClusterTable() { + return this; + } + + @Override + public T getClusterByClusterUIDOrMakeProxy(ClusterUID clusterUID) { + try { + return (T)impl.getClusterByClusterUIDOrMakeProxy(clusterUID); + } catch (DatabaseException e) { + e.printStackTrace(); + return null; + } + } + + @Override + public T getClusterProxyByResourceKey(int resourceKey) { + try { + return impl.getClusterProxyByResourceKey(resourceKey); + } catch (DatabaseException e) { + e.printStackTrace(); + return null; + } + } + + @Override + public int getClusterKeyByUID(long id1, long id2) throws DatabaseException { + return impl.getClusterKeyByUID(id1, id2); + } + +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/ClusterUpdateProcessor.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/ClusterUpdateProcessor.java new file mode 100644 index 000000000..d4381aeca --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/ClusterUpdateProcessor.java @@ -0,0 +1,86 @@ +package org.simantics.acorn.internal; + +import org.simantics.acorn.ClusterManager; +import org.simantics.acorn.cluster.ClusterImpl; +import org.simantics.acorn.exception.IllegalAcornStateException; +import org.simantics.acorn.lru.ClusterUpdateOperation; +import org.simantics.db.exception.DatabaseException; +import org.simantics.db.impl.ClusterSupport; +import org.simantics.db.service.ClusterUID; + +public class ClusterUpdateProcessor extends ClusterUpdateProcessorBase { + + final ClusterSupport support; + final ClusterUpdateOperation info; + private ClusterImpl cluster; + + public ClusterUpdateProcessor(ClusterManager client, ClusterSupport support, byte[] operations, ClusterUpdateOperation info) throws DatabaseException { + super(client, operations); + this.support = support; + this.info = info; + } + + @Override + void create() throws DatabaseException { + cluster.createResource(support); + } + + @Override + void delete(int ri) throws DatabaseException { + + boolean oldValueEx = cluster.isValueEx(ri); + byte[] old = cluster.getValue(ri, support); + if(old != null) cluster.removeValue(ri, support); + info.ccs.oldValueEx.add(oldValueEx ? (byte)1 : 0); + info.ccs.oldValues.add(old); + + } + + @Override + void modify(int resourceKey, long offset, int size, byte[] bytes, int pos) + throws DatabaseException { + + cluster = (ClusterImpl)cluster.modiValueEx(resourceKey, offset, size, bytes, pos, support); + manager.modiFileEx(cluster.getClusterUID(), resourceKey, offset, size, bytes, pos, support); + + } + + @Override + void set(int resourceKey, byte[] bytes, int length) + throws DatabaseException { + + byte[] old = cluster.getValue(resourceKey, support); + boolean oldValueEx = cluster.isValueEx(resourceKey); + cluster = (ClusterImpl)cluster.setValue(resourceKey, valueBuffer, length, support); + info.ccs.oldValueEx.add(oldValueEx ? (byte)1 : 0); + info.ccs.oldValues.add(old); + + } + + @Override + void claim(int resourceKey, int predicateKey, int objectKey, ClusterUID puid, ClusterUID ouid) + throws DatabaseException { + + ClusterImpl c = (ClusterImpl)cluster.addRelation(resourceKey, puid, predicateKey, ouid, objectKey, support); + if(c != null) cluster = c; + info.ccs.statementMask.add(c != null ? (byte)1 : 0); + + } + + @Override + void deny(int resourceKey, int predicateKey, int objectKey, ClusterUID puid, ClusterUID ouid) + throws DatabaseException { + + boolean modified = cluster.removeRelation(resourceKey, predicateKey, objectKey, support); + info.ccs.statementMask.add(modified ? (byte)1 : 0); + + } + + public ClusterImpl process(ClusterImpl cluster) throws IllegalAcornStateException { + this.cluster = cluster; + process(); + info.finish(); + return this.cluster; + } + +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/ClusterUpdateProcessor2.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/ClusterUpdateProcessor2.java new file mode 100644 index 000000000..61a8a8a9d --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/ClusterUpdateProcessor2.java @@ -0,0 +1,31 @@ +package org.simantics.acorn.internal; + +import org.simantics.acorn.cluster.ClusterImpl; +import org.simantics.acorn.exception.IllegalAcornStateException; +import org.simantics.acorn.lru.ClusterUpdateOperation; +import org.simantics.db.impl.ClusterSupport; + +public class ClusterUpdateProcessor2 extends ClusterUpdateProcessorBase2 { + + final ClusterSupport support; + final ClusterUpdateOperation info; + private ClusterImpl cluster; + + public ClusterUpdateProcessor2(ClusterSupport support, byte[] operations, ClusterUpdateOperation info) { + super(operations); + this.support = support; + this.info = info; + } + + public void process(ClusterImpl cluster) throws IllegalAcornStateException { + this.cluster = cluster; + process(); + info.finish(); + } + + @Override + void setImmutable(boolean value) { + cluster.setImmutable(value, support); + } + +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/ClusterUpdateProcessorBase.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/ClusterUpdateProcessorBase.java new file mode 100644 index 000000000..cd8130d9c --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/ClusterUpdateProcessorBase.java @@ -0,0 +1,476 @@ +package org.simantics.acorn.internal; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import org.simantics.acorn.ClusterManager; +import org.simantics.acorn.exception.IllegalAcornStateException; +import org.simantics.acorn.internal.ClusterStream.ClusterEnum; +import org.simantics.acorn.internal.ClusterStream.Data; +import org.simantics.acorn.internal.ClusterStream.StmEnum; +import org.simantics.db.exception.DatabaseException; +import org.simantics.db.service.Bytes; +import org.simantics.db.service.ClusterUID; + +abstract public class ClusterUpdateProcessorBase { + + public final static boolean DEBUG = false; + + final protected ClusterManager manager; + final public byte[] bytes; + private int pos = 0; + final private int len; + final private ClusterUID uid; + final private int clusterKey; + final public int version; + + final Map clusterKeyCache = new HashMap(); + + public int getResourceKey(ClusterUID uid, int index) throws IllegalAcornStateException { + Integer match = clusterKeyCache.get(uid); + if(match != null) return match+index; + int key = manager.getResourceKeyWitoutMutex(uid, 0); + clusterKeyCache.put(uid, key); + return key+index; + } + + + public ClusterUpdateProcessorBase(ClusterManager client, byte[] operations) throws DatabaseException { + this.manager = client; + this.bytes = operations; + this.len = Bytes.readLE4(bytes, 0)+4; // whatta? + version = Bytes.readLE4(bytes, 4); + long cuid1 = Bytes.readLE8(bytes, 8); + long cuid2 = Bytes.readLE8(bytes, 16); + uid = ClusterUID.make(cuid1, cuid2); + pos = 24; + client.clusterLRU.acquireMutex(); + try { + clusterKey = client.clusterLRU.getClusterKeyByUID(cuid1, cuid2) << 12; + } catch (Throwable t) { + throw new IllegalStateException(t); + } finally { + client.clusterLRU.releaseMutex(); + } + } + + public ClusterUID getClusterUID() { + return uid; + } + + private void processCreate() { + int r = Bytes.readLE2(bytes, pos); + pos+=2; + if(DEBUG) System.err.println("DEBUG: New ri=" + r + " offset=" + (pos-3-24)); + try { + create(); + } catch (DatabaseException e) { + e.printStackTrace(); + } + + } + + private void processDelete() { + + int ri = Bytes.readLE2(bytes, pos); + pos += 2; + + if(DEBUG) System.err.println("DEBUG: Delete " + ri); + + try { + delete(ri); + } catch (DatabaseException e) { + e.printStackTrace(); + } + + } + + private void processModify(int op) { + + int ri = Bytes.readLE2(bytes, pos); + pos+=2; + long offset = Bytes.readLE7(bytes, pos); + pos+=7; + int size = Bytes.readLE2(bytes, pos); + pos+=2; + + offset += (ri>>14) << 56; + ri = ri & 0x3FFF; + + if(size < 0) + throw new IllegalStateException(); + if(ri < 1) + throw new IllegalStateException(); + if(ri > 4095) + throw new IllegalStateException(); + + if(DEBUG) System.err.println("DEBUG: Modify " + ri + " " + offset + " " + size + " offset=" + (pos-1-24) + " " + Arrays.toString(Arrays.copyOf(valueBuffer,size))); + + try { + modify(clusterKey + ri, offset, size, bytes, pos); + } catch (DatabaseException e) { + e.printStackTrace(); + } + + pos += size; + + } + + private void processSet(int op) { + + int s = Bytes.readLE4(bytes, pos); + int length = (s >> 14); + if(length < 1) + throw new IllegalStateException(); + int r = s & 0x3FFF; + + pos += 4; + System.arraycopy(bytes, pos, valueBuffer, 0, length); + pos += length; + + if(DEBUG) System.err.println("DEBUG: Set " + r + " " + length + " offset=" + (pos-1-24) + " " + Arrays.toString(Arrays.copyOf(valueBuffer,length))); + + try { + set(clusterKey+r, valueBuffer, length); + } catch (DatabaseException e) { + e.printStackTrace(); + } + + } + + byte[] valueBuffer = new byte[65536]; + + private void processSetShort(int op) { + + int s = Bytes.readLE2(bytes, pos); + int length = ((op&7)<<2) + (s >> 14); + if(length < 1) + throw new IllegalStateException(); + if(length > 31) + throw new IllegalStateException(); + int r = s & 0x3FFF; + + if(DEBUG) System.err.println("DEBUG: SetShort " + r + " " + length + " offset=" + (pos-1-24) + " " + Arrays.toString(Arrays.copyOf(valueBuffer,length))); + pos += 2; + + System.arraycopy(bytes, pos, valueBuffer, 0, length); + pos += length; + + try { + set(clusterKey+r, valueBuffer, length); + } catch (DatabaseException e) { + e.printStackTrace(); + } + + } + + private void processStatementResource(ClusterEnum enu, int pOrO) { + if(ClusterEnum.ForeignShort == enu) { + int fs = bytes[pos++]&0xff; + foreignRefs[pOrO] = fs; + } else if(ClusterEnum.Local == enu) { + int lo = bytes[pos++]&0xff; + lows[pOrO] = lo; + } else { + long l1 = Bytes.readLE8(bytes, pos); + pos += 8; + long l2 = Bytes.readLE8(bytes, pos); + pos += 8; + ClusterUID cuid = ClusterUID.make(l1, l2); + foreignClusters[foreignPos] = cuid; + int lo = bytes[pos++]&0xff; + foreignIndices[foreignPos] = lo; + foreignRefs[pOrO] = foreignPos; + foreignPos++; + lows[pOrO] = lo; + } + } + + ClusterUID[] foreignClusters = new ClusterUID[256]; + int[] foreignIndices = new int[256]; + int foreignPos = 0; + int lows[] = new int[2]; + int foreignRefs[] = new int[2]; + + private void processStatement(int op, StmEnum stmEnum, ClusterEnum p, ClusterEnum o) throws IllegalAcornStateException { + + int curPos = pos-1-24; + + processStatementResource(p, 0); + processStatementResource(o, 1); + + int ri = bytes[pos++]&0xff; + int pi = 0; + int oi = 0; + + ClusterUID puid = uid; + ClusterUID ouid = puid; + + if(ClusterEnum.ForeignShort == p && ClusterEnum.ForeignShort == o) { + ri |= (op&0x3F) << 8; + } else { + Data data = ClusterEnum.getData(stmEnum, p, o); + // data.left is the amount of bytes in last two bytes + if(data.bytes == 0) { + ri = ri | ((op&0x3F)<<8); + } else { + int extra = 0; + int opBits = data.bits; + int extraBits = 6-opBits; + if(data.bytes == 1) { + extra = bytes[pos++]&0xff; + int high = extra >> extraBits; + if(ClusterEnum.ForeignShort == p) { + oi = lows[1] + (high<<8); + } else { + pi = lows[0] + (high<<8); + } + } else { + extra = Bytes.readLE2(bytes, pos); + pos += 2; + int high1 = (extra >> extraBits)&((1<<6)-1); + int high2 = (extra >> (extraBits+6))&((1<<6)-1); + if(ClusterEnum.ForeignShort == p) { + oi = lows[1] + (high1<<8); + } else { + pi = lows[0] + (high1<<8); + oi = lows[1] + (high2<<8); + } + } + ri = ri | ((extra&((1< 4095) + throw new IllegalStateException(); + if(pi > 4095) + throw new IllegalStateException(); + if(oi > 4095) + throw new IllegalStateException(); + + if(StmEnum.Add == stmEnum) { + + if(DEBUG) + System.err.println("DEBUG: ClusterChange " + uid + ": Add ri=" + ri + " pi=" + pi + " oi=" + oi + " pc=" + puid + " oc=" + ouid + " offset=" + curPos + " " + p.ordinal + " " + o.ordinal); + + int predicateKey = getResourceKey(puid, pi); + int objectKey = getResourceKey(ouid, oi); + try { + claim(clusterKey+ri, predicateKey, objectKey, puid, ouid); + } catch (DatabaseException e) { + e.printStackTrace(); + } + + } else { + + if(DEBUG) + System.err.println("DEBUG: ClusterChange " + uid + ": Rem ri=" + ri + " pi=" + pi + " oi=" + oi + " pc=" + puid + " oc=" + ouid + " offset=" + curPos + " " + p.ordinal + " " + o.ordinal); + + int predicateKey = getResourceKey(puid, pi); + int objectKey = getResourceKey(ouid, oi); + try { + deny(clusterKey+ri, predicateKey, objectKey, puid, ouid); + } catch (DatabaseException e) { + e.printStackTrace(); + } + + } + + } + + public void process() throws IllegalAcornStateException { + + foreignPos = 0; + + if(DEBUG) System.err.println("DEBUG: process " + uid + " " + len); + + // op resolution for statement operation: + + // 2 first bits + // op: 01 | r8-13 + // op: 10 | r8-13 + + // 3 first bits (000) + // op: 000000 | r12-13 + // op: 000001 | r12-13 + // op: 000010 | r12-13 + // op: 000011 | r12-13 + // op: 000100 | r12-13 + // op: 000101 | r12-13 + // op: 000110 | r12-13 + // op: 000111 | r12-13 + + // 4 first bits + // op: 1100 | r10-13 + // op: 1101 | r10-13 + // op: 1110 | r10-13 + // op: 1111 | r10-13 + // op: 0010 | r10-13 + + // 6 bits + // op: 00110001 = 49 + // op: 00110010 = 50 + // op: 00110011 = 51 + // other: 0011xxxx + + while(pos < len) { + + int op = bytes[pos++]&0xff; + + // common prefix: 0011 + switch(op) { + + case 49: + processStatement(op, StmEnum.Remove, ClusterEnum.Local, ClusterEnum.ForeignShort); + break; + case 50: + processStatement(op, StmEnum.Remove, ClusterEnum.ForeignShort, ClusterEnum.ForeignLong); + break; + case 51: + processStatement(op, StmEnum.Remove, ClusterEnum.ForeignLong, ClusterEnum.ForeignShort); + break; + // 52 = 32+16+4 = 00110100 + case 52: + processCreate(); + break; + // 53 = 32+16+4+1 = 00110101 + case 53: + processSet(op); + break; + // 54 = 32+16+4+2 = 00110110 + case 54: + processDelete(); + break; + // 55 = 32+16+4+2+1 = 00110111 + case 55: + processModify(op); + break; + default: + + int bits6 = ((int)op)&0xC0; + switch(bits6) { + + case 0x40: + processStatement(op, StmEnum.Add, ClusterEnum.ForeignShort, ClusterEnum.ForeignShort); + break; + case 0x80: + processStatement(op, StmEnum.Remove, ClusterEnum.ForeignShort, ClusterEnum.ForeignShort); + break; + default: + + int bits5 = ((int)op)&0xE0; + if(bits5 == 0) { + + int bits2 = (((int)op)&0xFC) >> 2; + + // 3 top bits are 0 + // 6 bits of op + + switch(bits2) { + + case 0: + processStatement(op, StmEnum.Add, ClusterEnum.Local, ClusterEnum.Local); + break; + case 1: + processStatement(op, StmEnum.Remove, ClusterEnum.Local, ClusterEnum.Local); + break; + case 2: + processStatement(op, StmEnum.Add, ClusterEnum.Local, ClusterEnum.ForeignLong); + break; + case 3: + processStatement(op, StmEnum.Remove, ClusterEnum.Local, ClusterEnum.ForeignLong); + break; + case 4: + processStatement(op, StmEnum.Add, ClusterEnum.ForeignLong, ClusterEnum.Local); + break; + case 5: + processStatement(op, StmEnum.Remove, ClusterEnum.ForeignLong, ClusterEnum.Local); + break; + case 6: + processStatement(op, StmEnum.Add, ClusterEnum.ForeignLong, ClusterEnum.ForeignLong); + break; + case 7: + processStatement(op, StmEnum.Remove, ClusterEnum.ForeignLong, ClusterEnum.ForeignLong); + break; + + } + + } else { + + // 4 top bits of op + // 4 low bits of payload + + int bits4 = (((int)op)&0xF0)>>4; + switch(bits4) { + case 0b1100: + processStatement(op, StmEnum.Add, ClusterEnum.Local, ClusterEnum.ForeignShort); + break; + case 0b1101: + processStatement(op, StmEnum.Add, ClusterEnum.ForeignShort, ClusterEnum.Local); + break; + case 0b1110: + processStatement(op, StmEnum.Add, ClusterEnum.ForeignShort, ClusterEnum.ForeignLong); + break; + case 0b1111: + processStatement(op, StmEnum.Add, ClusterEnum.ForeignLong, ClusterEnum.ForeignShort); + break; + case 0b0010: + processStatement(op, StmEnum.Remove, ClusterEnum.ForeignShort, ClusterEnum.Local); + break; + case 0b0011: + int bits3 = (((int)op)&0xF8)>>3; + if(bits3 == 7) + processSetShort(op); + break; + } + + } + + } + + } + + } + + } + + + abstract void create() throws DatabaseException; + abstract void delete(int resourceIndex) throws DatabaseException; + abstract void modify(int resourceKey, long offset, int size, byte[] bytes, int pos) throws DatabaseException; + abstract void set(int resourceKey, byte[] bytes, int length) throws DatabaseException; + + abstract void claim(int resourceKey, int predicateKey, int objectKey, ClusterUID puid, ClusterUID ouid) throws DatabaseException; + abstract void deny(int resourceKey, int predicateKey, int objectKey, ClusterUID puid, ClusterUID ouid) throws DatabaseException; + +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/ClusterUpdateProcessorBase2.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/ClusterUpdateProcessorBase2.java new file mode 100644 index 000000000..502729c0b --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/ClusterUpdateProcessorBase2.java @@ -0,0 +1,62 @@ +package org.simantics.acorn.internal; + +import org.simantics.acorn.exception.IllegalAcornStateException; +import org.simantics.db.service.Bytes; +import org.simantics.db.service.ClusterUID; + +public abstract class ClusterUpdateProcessorBase2 { + + final private byte[] bytes; + private int pos = 0; + final private int len; + final private ClusterUID uid; + + public ClusterUpdateProcessorBase2(byte[] operations) { + this.bytes = operations; + this.len = Bytes.readLE4(bytes, 0) + 4; // whatta? + int version = Bytes.readLE4(bytes, 4); + assert(version == ClusterChange2.VERSION); + long cuid1 = Bytes.readLE8(bytes, 8); + long cuid2 = Bytes.readLE8(bytes, 16); + pos = 24; + uid = ClusterUID.make(cuid1, cuid2); + } + + public ClusterUID getClusterUID() { + return uid; + } + + private void processSetImmutable(int op) { + int value = bytes[pos++]&0xff; + setImmutable(value > 0); + } + + private void processUndoValue(int op) { + Bytes.readLE4(bytes, pos); + pos+=4; + } + + public void process() throws IllegalAcornStateException { + + while(pos < len) { + + int op = bytes[pos++]&0xff; + + switch(op) { + + case ClusterChange2.SET_IMMUTABLE_OPERATION: + processSetImmutable(op); + break; + case ClusterChange2.UNDO_VALUE_OPERATION: + processUndoValue(op); + break; + default: + throw new IllegalAcornStateException("Can not process cluster " + uid); + + } + } + } + + abstract void setImmutable(boolean value); + +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/DebugPolicy.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/DebugPolicy.java new file mode 100644 index 000000000..d694abe83 --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/DebugPolicy.java @@ -0,0 +1,19 @@ +package org.simantics.acorn.internal; + + +/** + * @author Antti Villberg + */ +public final class DebugPolicy { + + public static final boolean REPORT_RESOURCE_ID_ALLOCATION = false; + public static final boolean REPORT_CLUSTER_ID_ALLOCATION = false; + public static final boolean REPORT_CLUSTER_EVENTS = false; + public static final boolean REPORT_CLUSTER_LOADING = false; + public static final boolean REPORT_CLUSTER_LOADING_STACKS = false; + public static final boolean REPORT_CLUSTER_STREAM = false; + public static final boolean CLUSTER_COLLECTION = false; + public static final boolean LOG_SERVER_EVENTS = false; + public static final boolean SHOW_SERVER_EVENTS = false; // Requires LOG_SERVER_EVENTS to be true. + +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/UndoClusterUpdateProcessor.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/UndoClusterUpdateProcessor.java new file mode 100644 index 000000000..8b3e4f066 --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/internal/UndoClusterUpdateProcessor.java @@ -0,0 +1,114 @@ +package org.simantics.acorn.internal; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.simantics.acorn.ClusterManager; +import org.simantics.acorn.exception.AcornAccessVerificationException; +import org.simantics.acorn.exception.IllegalAcornStateException; +import org.simantics.acorn.lru.ClusterChangeSet; +import org.simantics.acorn.lru.ClusterStreamChunk; +import org.simantics.acorn.lru.ClusterChangeSet.Entry; +import org.simantics.acorn.lru.ClusterChangeSet.Type; +import org.simantics.db.exception.DatabaseException; +import org.simantics.db.service.ClusterUID; + +public class UndoClusterUpdateProcessor extends ClusterUpdateProcessorBase { + + public final static boolean DEBUG = false; + + final private ClusterChangeSet ccs; + + private int oldValuesIndex = 0; + private int statementMaskIndex = 0; + + final public List entries = new ArrayList(); + + public UndoClusterUpdateProcessor(ClusterManager client, ClusterStreamChunk chunk, ClusterChangeSet ccs) throws DatabaseException { + super(client, readOperation(client, chunk, ccs)); + this.ccs = ccs; + } + + private static byte[] readOperation(ClusterManager manager, ClusterStreamChunk chunk, ClusterChangeSet ccs) throws AcornAccessVerificationException, IllegalAcornStateException { + +// ClusterStreamChunk chunk; +// manager.streamLRU.acquireMutex(); +// try { +// chunk = ccs.getChunk(manager); +// } catch (Throwable t) { +// throw new IllegalStateException(t); +// } finally { +// manager.streamLRU.releaseMutex(); +// } +// +// chunk.acquireMutex(); +// try { +// chunk.ve + chunk.makeResident(); + return chunk.getOperation(ccs.chunkOffset); +// } catch (Throwable t) { +// throw new IllegalStateException(t); +// } finally { +// chunk.releaseMutex(); +// } + } + + @Override + void create() throws DatabaseException { + } + + @Override + void delete(int ri) throws DatabaseException { + + byte[] old = ccs.oldValues.get(oldValuesIndex); + boolean oldValueEx = ccs.oldValueEx.get(oldValuesIndex) > 0; + oldValuesIndex++; + + if(old != null) { + entries.add(new Entry(ri, oldValueEx, old, null)); + } + + } + + @Override + void modify(int resourceKey, long offset, int size, byte[] bytes, int pos) + throws DatabaseException { + + } + + @Override + void set(int resourceKey, byte[] bytes, int length) + throws DatabaseException { + + byte[] old = ccs.oldValues.get(oldValuesIndex); + boolean oldValueEx = ccs.oldValueEx.get(oldValuesIndex) > 0; + oldValuesIndex++; + + entries.add(new Entry(resourceKey, oldValueEx, old, Arrays.copyOf(valueBuffer, length))); + + } + + @Override + void claim(int resourceKey, int predicateKey, int objectKey, ClusterUID puid, ClusterUID ouid) + throws DatabaseException { + + boolean add = ccs.statementMask.get(statementMaskIndex++) > 0; + if(add) { + entries.add(new Entry(Type.ADD, resourceKey, puid, predicateKey & 0xFFF, ouid, objectKey & 0xFFF)); + } + + } + + @Override + void deny(int resourceKey, int predicateKey, int objectKey, ClusterUID puid, ClusterUID ouid) + throws DatabaseException { + + boolean remove = ccs.statementMask.get(statementMaskIndex++) > 0; + if(remove) { + entries.add(new Entry(Type.REMOVE, resourceKey, puid, predicateKey & 0xFFF, ouid, objectKey & 0xFFF)); + } + + } + +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/AccessTime.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/AccessTime.java new file mode 100644 index 000000000..8a32ef230 --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/AccessTime.java @@ -0,0 +1,23 @@ +package org.simantics.acorn.lru; + +public class AccessTime { + + private long last = 0; + + private static AccessTime INSTANCE = new AccessTime(); + + private AccessTime() { + + } + + public static AccessTime getInstance() { + return INSTANCE; + } + + public synchronized long getAccessTime() { + long result = System.nanoTime(); + last = Math.max(result, last+1); + return last; + } + +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/CachingClusterSupport.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/CachingClusterSupport.java new file mode 100644 index 000000000..a2c489901 --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/CachingClusterSupport.java @@ -0,0 +1,160 @@ +package org.simantics.acorn.lru; + +import java.io.InputStream; + +import org.simantics.db.Session; +import org.simantics.db.exception.DatabaseException; +import org.simantics.db.impl.ClusterBase; +import org.simantics.db.impl.ClusterSupport; +import org.simantics.db.impl.IClusterTable; +import org.simantics.db.service.ClusterUID; + +public class CachingClusterSupport implements ClusterSupport { + + private ClusterSupport backend; + + public CachingClusterSupport(ClusterSupport backend) { + this.backend = backend; + } + + @Override + public int createClusterKeyByClusterUID(ClusterUID clusterUID, long clusterId) { + throw new UnsupportedOperationException(); + } + + @Override + public ClusterBase getClusterByClusterUIDOrMake(ClusterUID clusterUID) { + throw new UnsupportedOperationException(); + } + + @Override + public ClusterBase getClusterByClusterId(long clusterId) { + throw new UnsupportedOperationException(); + } + + @Override + public ClusterBase getClusterByClusterKey(int clusterKey) { + throw new UnsupportedOperationException(); + } + + @Override + public int getClusterKeyByClusterUIDOrMake(ClusterUID clusterUID) { + throw new UnsupportedOperationException(); + } + + @Override + public int getClusterKeyByClusterUIDOrMake(long id1, long id2) { + throw new UnsupportedOperationException(); + } + + @Override + public ClusterBase getClusterByResourceKey(int resourceKey) { + throw new UnsupportedOperationException(); + } + + @Override + public long getClusterIdOrCreate(ClusterUID clusterUID) { + throw new UnsupportedOperationException(); + } + + @Override + public void addStatement(Object cluster) { + throw new UnsupportedOperationException(); + } + + @Override + public void cancelStatement(Object cluster) { + throw new UnsupportedOperationException(); + } + + @Override + public void removeStatement(Object cluster) { + throw new UnsupportedOperationException(); + } + + @Override + public void cancelValue(Object cluster) { + throw new UnsupportedOperationException(); + } + + @Override + public void removeValue(Object cluster) { + throw new UnsupportedOperationException(); + } + + @Override + public void setValue(Object cluster, long clusterId, byte[] bytes, int length) { + throw new UnsupportedOperationException(); + } + + @Override + public void modiValue(Object cluster, long clusterId, long voffset, int length, byte[] bytes, int offset) { + throw new UnsupportedOperationException(); + } + + @Override + public void setImmutable(Object cluster, boolean immutable) { + throw new UnsupportedOperationException(); + } + + @Override + public void setDeleted(Object cluster, boolean deleted) { + throw new UnsupportedOperationException(); + } + + @Override + public void createResource(Object cluster, short resourceIndex, long clusterId) { + backend.createResource(cluster, resourceIndex, clusterId); + } + + @Override + public void addStatementIndex(Object cluster, int resourceKey, ClusterUID clusterUID, byte op) { + throw new UnsupportedOperationException(); + } + + @Override + public void setStreamOff(boolean setOff) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean getStreamOff() { + throw new UnsupportedOperationException(); + } + + @Override + public InputStream getValueStreamEx(int resourceIndex, long clusterId) throws DatabaseException { + throw new UnsupportedOperationException(); + } + + @Override + public byte[] getValueEx(int resourceIndex, long clusterId) throws DatabaseException { + throw new UnsupportedOperationException(); + } + + @Override + public byte[] getValueEx(int resourceIndex, long clusterId, long voffset, int length) throws DatabaseException { + throw new UnsupportedOperationException(); + } + + @Override + public long getValueSizeEx(int resourceIndex, long clusterId) throws DatabaseException { + throw new UnsupportedOperationException(); + } + + @Override + public int wait4RequestsLess(int limit) throws DatabaseException { + throw new UnsupportedOperationException(); + } + + @Override + public Session getSession() { + throw new UnsupportedOperationException(); + } + + @Override + public IClusterTable getClusterTable() { + throw new UnsupportedOperationException(); + } + +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/ChangeSetInfo.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/ChangeSetInfo.java new file mode 100644 index 000000000..a730e136d --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/ChangeSetInfo.java @@ -0,0 +1,113 @@ +package org.simantics.acorn.lru; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; + +import org.simantics.acorn.exception.AcornAccessVerificationException; +import org.simantics.acorn.exception.IllegalAcornStateException; +import org.simantics.db.service.Bytes; +import org.simantics.utils.datastructures.Pair; + +import gnu.trove.list.array.TByteArrayList; + +public class ChangeSetInfo extends LRUObject { + + private byte[] metadataBytes; + private ArrayList clusterChangeSetIds; + + // Stub + public ChangeSetInfo(LRU LRU, Path readDir, Long revision, int offset, int length) throws AcornAccessVerificationException { + super(LRU, revision, readDir, "clusterStream", offset, length, false, false); + LRU.map(this); + } + + // New + public ChangeSetInfo(LRU LRU, Long revision, byte[] bytes, ArrayList clusterChangeSetIds) throws AcornAccessVerificationException { + super(LRU, revision, LRU.getDirectory(), "clusterStream", true, true); + this.metadataBytes = bytes; + this.metadataBytes = bytes; + this.clusterChangeSetIds = clusterChangeSetIds; + LRU.insert(this, accessTime); + } + + public ArrayList getCSSIds() throws AcornAccessVerificationException { + if(VERIFY) verifyAccess(); + return clusterChangeSetIds; + } + + public byte[] getMetadataBytes() throws AcornAccessVerificationException, IllegalAcornStateException { + if(VERIFY) + verifyAccess(); + + makeResident(); + return metadataBytes; + } + + private static void writeLE(TByteArrayList bytes, int value) { + + bytes.add( (byte) (value & 0xFF)); + bytes.add((byte) ((value >>> 8) & 0xFF)); + bytes.add((byte) ((value >>> 16) & 0xFF)); + bytes.add((byte) ((value >>> 24) & 0xFF)); + + } + + @Override + protected Pair toBytes() { + + TByteArrayList result = new TByteArrayList(); + writeLE(result, metadataBytes.length); + result.add(metadataBytes); + writeLE(result, clusterChangeSetIds.size()); + for(String id : clusterChangeSetIds) { + byte[] bb = id.getBytes(); + writeLE(result, bb.length); + result.add(bb); + } + + release(); + + byte[] ret = result.toArray(); + + return Pair.make(ret, ret.length); + + } + + @Override + void release() { + clusterChangeSetIds = null; + metadataBytes = null; + } + + @Override + public void fromFile(byte[] data) { + + clusterChangeSetIds = new ArrayList(); + + int metadataLength = Bytes.readLE4(data, 0); + metadataBytes = Arrays.copyOfRange(data, 4, 4+metadataLength); + int offset = 4+metadataLength; + int numberOfChangeSets = Bytes.readLE4(data, offset); + offset += 4; + for(int i=0;i oldValues = new ArrayList(); + + public ClusterChangeSet(String id ,ClusterUID cuid) { + this.id = id; + this.cuid = cuid; + String[] ss = id.split("\\."); + chunkKey = ss[0]; + chunkOffset = Integer.parseInt(ss[1]); + } + + public ClusterStreamChunk getChunk(ClusterManager manager) throws AcornAccessVerificationException { + return manager.streamLRU.get(chunkKey); + } + +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/ClusterInfo.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/ClusterInfo.java new file mode 100644 index 000000000..57dfe9f41 --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/ClusterInfo.java @@ -0,0 +1,342 @@ +package org.simantics.acorn.lru; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Path; + +import org.simantics.acorn.ClusterManager; +import org.simantics.acorn.Persistable; +import org.simantics.acorn.cluster.ClusterImpl; +import org.simantics.acorn.cluster.ClusterSmall; +import org.simantics.acorn.exception.AcornAccessVerificationException; +import org.simantics.acorn.exception.IllegalAcornStateException; +import org.simantics.acorn.cluster.ClusterImpl.ClusterTables; +import org.simantics.acorn.internal.ClusterSupport2; +import org.simantics.compressions.CompressionCodec; +import org.simantics.compressions.Compressions; +import org.simantics.db.ClusterCreator; +import org.simantics.db.exception.DatabaseException; +import org.simantics.db.exception.SDBException; +import org.simantics.db.service.Bytes; +import org.simantics.db.service.ClusterUID; +import org.simantics.utils.datastructures.Pair; + +public class ClusterInfo extends LRUObject implements Persistable { + + final private ClusterManager manager; + private ClusterImpl cluster; + public int changeSetId; + private ClusterUpdateState updateState; + + public static final String COMPRESSION = "LZ4"; + + // Stub + public ClusterInfo(ClusterManager manager, LRU LRU, Path readDirectory, ClusterUID uid, int offset, int length) throws AcornAccessVerificationException { + super(LRU, uid, readDirectory, uid.toString() + ".cluster", offset, length, false, false); + this.manager = manager; + this.cluster = null; + LRU.map(this); + } + + // New + public ClusterInfo(ClusterManager manager, LRU LRU, ClusterImpl cluster) throws AcornAccessVerificationException, IllegalAcornStateException { + super(LRU, cluster.getClusterUID(), LRU.getDirectory(), cluster.getClusterUID().toString() + ".cluster", true, true); + this.manager = manager; + this.cluster = cluster; + LRU.insert(this, accessTime); + LRU.swap(getKey()); + } + + public T clone(ClusterUID uid, ClusterCreator creator) throws IOException, AcornAccessVerificationException, IllegalAcornStateException { + + // Updates have been ensured at this point + + acquireMutex(); + + try { + if(isResident()) { + ClusterTables tables = cluster.store(); + return creator.create(uid, tables.bytes, tables.ints, tables.longs); + } + } catch (IOException e) { + throw e; + } catch (Throwable t) { + throw new IllegalAcornStateException(t); + } finally { + releaseMutex(); + } + + // Ensure pending updates here - this may take some time + LRU.waitPending(this, false); + + acquireMutex(); + try { + + if(isResident()) { + ClusterTables tables = cluster.store(); + return creator.create(uid, tables.bytes, tables.ints, tables.longs); + } else { + byte[] data = readFile(); + ClusterTables tables = new ClusterTables(); + loadCluster(getKey(), manager.support, data, tables); + return creator.create(uid, tables.bytes, tables.ints, tables.longs); + } + + } catch (IOException e) { + throw e; + } catch (Throwable t) { + throw new IllegalAcornStateException(t); + } finally { + releaseMutex(); + } + + } + + static class ClusterDecompressor { + + byte[] decompressBuffer = new byte[1024*1024]; + + public synchronized ClusterTables readCluster(ClusterUID uid, byte[] compressed) throws IOException { + + int deflatedSize = Bytes.readLE4(compressed, compressed.length-4); + + if(decompressBuffer.length < deflatedSize) + decompressBuffer = new byte[Math.max(3*decompressBuffer.length / 2, deflatedSize)]; + + CompressionCodec codec = Compressions.get(Compressions.LZ4); + + ByteBuffer input = ByteBuffer.wrap(compressed); + ByteBuffer output = ByteBuffer.wrap(decompressBuffer); + + int decompressedSize = codec.decompressBuffer(input, 0, compressed.length-4, output, 0, decompressBuffer.length); + assert(decompressedSize <= decompressBuffer.length); + + int byteLength = Bytes.readLE4(decompressBuffer, 0); + int intLength = Bytes.readLE4(decompressBuffer, 4); + int longLength = Bytes.readLE4(decompressBuffer, 8); + + byte[] bytes = new byte[byteLength]; + int[] ints = new int[intLength]; + long[] longs = new long[longLength]; + + System.arraycopy(decompressBuffer, 12, bytes, 0, byteLength); + + int offset = 12+byteLength; + for(int i=0;i toBytes() throws IllegalAcornStateException { + try { + byte[] raw = null; + + if(cluster instanceof ClusterSmall) { + raw = cluster.storeBytes(); + } else { + + ClusterTables tables = cluster.store(); + + raw = new byte[12 + tables.bytes.length + (tables.ints.length<<2) + (tables.longs.length<<3)]; + + Bytes.writeLE(raw, 0, tables.bytes.length); + Bytes.writeLE(raw, 4, tables.ints.length); + Bytes.writeLE(raw, 8, tables.longs.length); + + System.arraycopy(tables.bytes, 0, raw, 12, tables.bytes.length); + int offset = 12+tables.bytes.length; + for(int i=0;i { + + final private BijectionMap clusterMapping = new BijectionMap(); + + public ClusterLRU(ClusterManager manager, String identifier, Path writeDir) { + super(manager, identifier, writeDir); + + clusterMapping.map(ClusterUID.make(0,2), clusterMapping.size() + 1); + } + + public ClusterInfo getOrCreate(ClusterUID uid, boolean makeIfNull) throws IllegalAcornStateException, AcornAccessVerificationException { + + try { + + acquireMutex(); + + ClusterInfo info = get(uid); + + if (info == null) { + + if(!makeIfNull) throw new IllegalAcornStateException("Asked for an existing cluster " + uid + " that was not found."); + + Integer clusterKey = clusterMapping.getRight(uid); + if (clusterKey == null) { + clusterKey = clusterMapping.size() + 1; + clusterMapping.map(uid, clusterKey); + } + + info = new ClusterInfo(manager, this, ClusterImpl.make(manager.support, + uid, clusterKey, manager.support)); + + } + + return info; + } catch (IllegalAcornStateException | AcornAccessVerificationException e) { + throw e; + } catch (Throwable t) { + throw new IllegalAcornStateException(t); + } finally { + + releaseMutex(); + + } + + } + + /* + * This method waits - we have no locks here + */ + public void ensureUpdates(ClusterUID uid) throws ClusterDoesNotExistException, AcornAccessVerificationException, IllegalAcornStateException { + + ClusterInfo info = getWithoutMutex(uid); + if(info == null) + throw new ClusterDoesNotExistException("Asked a cluster which does not exist: " + uid); + info.waitForUpdates(); + } + + public ClusterInfo get(ClusterUID uid, boolean makeIfNull, boolean ensureUpdates) throws AcornAccessVerificationException, IllegalAcornStateException { + + if (ensureUpdates) { + try { + ensureUpdates(uid); + } catch (ClusterDoesNotExistException e) { + if (makeIfNull) { + Logger.defaultLogError("For debug purposes, creating cluster which does not exist", e); + } else { + throw new IllegalAcornStateException(e); + } + } + } + return getOrCreate(uid, makeIfNull); + } + + public ClusterInfo get(ClusterUID uid, boolean makeIfNull) throws AcornAccessVerificationException, IllegalAcornStateException { + return get(uid, makeIfNull, true); + } + + public int getResourceKey(ClusterUID uid, int index) throws AcornAccessVerificationException { + + if(VERIFY) verifyAccess(); + + Integer i = clusterMapping.getRight(uid); + if (i == null) { + i = clusterMapping.size() + 1; + clusterMapping.map(uid, i); + } + return (i << 12) + index; + + } + + public int getResourceKeyWithoutMutex(ClusterUID uid, int index) throws IllegalAcornStateException { + + acquireMutex(); + try { + return getResourceKey(uid, index); + } catch (Throwable t) { + throw new IllegalAcornStateException(t); + } finally { + releaseMutex(); + } + } + + public int createClusterKeyByClusterUID(ClusterUID uid) throws AcornAccessVerificationException { + + if(VERIFY) verifyAccess(); + + Integer i = clusterMapping.getRight(uid); + if (i == null) { + i = clusterMapping.size() + 1; + clusterMapping.map(uid, i); + } + return i; + + } + + public ClusterBase getClusterByClusterUIDOrMake(ClusterUID uid) throws AcornAccessVerificationException, IllegalAcornStateException { + + if(VERIFY) verifyAccess(); + + int key = createClusterKeyByClusterUID(uid); + return getClusterByClusterKey(key); + + } + + public int getClusterKeyByClusterUIDOrMake(ClusterUID clusterUID) throws AcornAccessVerificationException { + + if(VERIFY) verifyAccess(); + + return createClusterKeyByClusterUID(clusterUID); + + } + + public int getClusterKeyByClusterUIDOrMakeWithoutMutex(ClusterUID clusterUID) throws IllegalAcornStateException, AcornAccessVerificationException { + acquireMutex(); + try { + return getClusterKeyByClusterUIDOrMake(clusterUID); + } catch (AcornAccessVerificationException e) { + throw e; + } catch (Throwable t) { + throw new IllegalAcornStateException(t); + } finally { + releaseMutex(); + } + } + + public ClusterBase getClusterByClusterKey(int clusterKey) throws AcornAccessVerificationException, IllegalAcornStateException { + + if(VERIFY) verifyAccess(); + + ClusterUID uid = clusterMapping.getLeft(clusterKey); + ClusterInfo info = get(uid, true); + info.acquireMutex(); + try { + return info.getCluster(); + } catch (IllegalAcornStateException | AcornAccessVerificationException e) { + throw e; + } catch (Throwable t) { + throw new IllegalAcornStateException(t); + } finally { + info.releaseMutex(); + } + } + + public ClusterUID getClusterUIDByResourceKey(int resourceKey) throws AcornAccessVerificationException { + + if(VERIFY) verifyAccess(); + + int clusterKey = resourceKey >> 12; + return clusterMapping.getLeft(clusterKey); + + } + + public ClusterUID getClusterUIDByResourceKeyWithoutMutex(int resourceKey) throws IllegalAcornStateException, AcornAccessVerificationException { + acquireMutex(); + try { + return getClusterUIDByResourceKey(resourceKey); + } finally { + releaseMutex(); + } + } + + @SuppressWarnings("unchecked") + public T getClusterByClusterUIDOrMakeProxy(ClusterUID uid) throws DatabaseException, AcornAccessVerificationException, IllegalAcornStateException { + return (T) getClusterByClusterUIDOrMake(uid); + } + + @SuppressWarnings("unchecked") + public T getClusterProxyByResourceKey(int resourceKey) throws DatabaseException, AcornAccessVerificationException, IllegalAcornStateException { + + if(VERIFY) verifyAccess(); + + return (T) getClusterByClusterKey(resourceKey >> 12); + + } + + public int getClusterKeyByUID(long id1, long id2) throws DatabaseException, AcornAccessVerificationException { + + if(VERIFY) verifyAccess(); + + return getClusterKeyByClusterUIDOrMake(ClusterUID.make(id1, id2)); + + } + + public int getClusterKeyByUIDWithoutMutex(long id1, long id2) throws DatabaseException, IllegalAcornStateException { + acquireMutex(); + try { + return getClusterKeyByClusterUIDOrMake(ClusterUID.make(id1, id2)); + } catch (Throwable t) { + throw new IllegalAcornStateException(t); + } finally { + releaseMutex(); + } + } + + + public static void main(String[] args) throws Exception { + + long start = System.nanoTime(); + + final TIntIntHashMap map = new TIntIntHashMap(0, 0.9f); + + AtomicInteger counter = new AtomicInteger(0); + AtomicBoolean written = new AtomicBoolean(false); + + //final Semaphore ws = new Semaphore(1); + + Thread write = new Thread() { + + @Override + public void run() { + try { + for(int i=0;i<100000000;i++) { + synchronized(map) { +// ws.acquire(); + map.put(i, i); +// ws.release(); + } + //if((i & 0xfffff) == 0) System.err.println("Write " + i); + counter.incrementAndGet(); + } + written.set(true); + } catch (Throwable e) { + e.printStackTrace(); + } + } + + }; + write.start(); + + Thread read = new Thread() { + + @Override + public void run() { + try { + while(!written.get()) { + double r = Math.random(); + double max = counter.get(); + int key = (int)(max*r); + int value = map.get(key); + if(key != value) { + //System.err.println("Read failed " + key + " vs. " + value); + //ws.acquire(); + synchronized(map) { + value = map.get(key); + if(key != value) { + System.err.println("Read failed for real " + key + " vs. " + value); + } + //ws.release(); + } + } + //if((key & 0xfffff) == 0) System.err.println("Read " + key); + } + } catch (Throwable e) { + e.printStackTrace(); + } + } + + }; + read.start(); + + write.join(); + read.join(); + + long duration = System.nanoTime() - start; + System.err.println("took " + 1e-9*duration + "s."); + + } + +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/ClusterStreamChunk.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/ClusterStreamChunk.java new file mode 100644 index 000000000..57d6f6a04 --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/ClusterStreamChunk.java @@ -0,0 +1,300 @@ +package org.simantics.acorn.lru; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.util.ArrayList; + +import org.simantics.acorn.ClusterManager; +import org.simantics.acorn.Persistable; +import org.simantics.acorn.exception.AcornAccessVerificationException; +import org.simantics.acorn.exception.IllegalAcornStateException; +import org.simantics.acorn.internal.ClusterChange; +import org.simantics.acorn.internal.UndoClusterUpdateProcessor; +import org.simantics.compressions.CompressionCodec; +import org.simantics.compressions.Compressions; +import org.simantics.db.exception.DatabaseException; +import org.simantics.db.service.Bytes; +import org.simantics.utils.datastructures.Pair; + +import gnu.trove.list.array.TByteArrayList; + +public class ClusterStreamChunk extends LRUObject implements Persistable { + + // 500KB is a fine chunk + private static int MAX_CHUNK_SIZE = 500*1024; + + int size = 0; + final private ClusterManager manager; + private boolean committed = false; + + public int nextToProcess = 0; + + public ArrayList operations = new ArrayList(); + + // Stub + public ClusterStreamChunk(ClusterManager manager, LRU LRU, Path readDir, String id, int offset, int length) throws AcornAccessVerificationException { + super(LRU, id, readDir, "clusterStream", offset, length, false, false); + this.manager = manager; + LRU.map(this); + } + + // Creation + public ClusterStreamChunk(ClusterManager manager, LRU LRU, String id) throws AcornAccessVerificationException { + super(LRU, id, LRU.getDirectory(), "clusterStream", true, true); + this.manager = manager; + LRU.insert(this, accessTime); + } + + public UndoClusterUpdateProcessor getUndoProcessor(ClusterManager clusters, int chunkOffset, String ccsId) throws DatabaseException { + + if(VERIFY) verifyAccess(); + + makeResident(true); + + ClusterUpdateOperation op = operations.get(chunkOffset); + if(op == null) throw new IllegalAcornStateException("Cluster Update Operation " + ccsId + " was not found."); + if(op.ccs == null) throw new IllegalAcornStateException("Cluster ChangeSet " + ccsId + " was not found."); + + UndoClusterUpdateProcessor proc = new UndoClusterUpdateProcessor(clusters, this, op.ccs); + if(proc.version != ClusterChange.VERSION) + return null; + + // This cluster and CCS can still be under preparation => wait + clusters.clusterLRU.ensureUpdates(proc.getClusterUID()); + + proc.process(); + + cancelForceResident(); + + return proc; + + } + + public void addOperation(ClusterUpdateOperation op) throws IllegalAcornStateException { + if(committed) + throw new IllegalAcornStateException("Cannot add operation " + op + " to " + this + " if commited == true"); + operations.add(op); + size += op.data.length; +// if(isCommitted()) { +// LRU.refresh(this); +// } + } + + public byte[] getOperation(int index) { + return operations.get(index).data; + } + + public void commit() { + committed = true; + } + + public boolean isCommitted() { + if(size > MAX_CHUNK_SIZE) committed = true; + return committed; + } + + @Override + public boolean canBePersisted() throws AcornAccessVerificationException { + if(!super.canBePersisted()) return false; + if(!isCommitted()) return false; + for(ClusterUpdateOperation op : operations) { + if(!op.finished) return false; + } + return true; + } + + private static void writeLE(TByteArrayList bytes, int value) { + + bytes.add( (byte) (value & 0xFF)); + bytes.add((byte) ((value >>> 8) & 0xFF)); + bytes.add((byte) ((value >>> 16) & 0xFF)); + bytes.add((byte) ((value >>> 24) & 0xFF)); + + } + + final public static void writeLE8(TByteArrayList bytes, long value) { + + bytes.add( (byte) (value & 0xFF)); + bytes.add((byte) ((value >>> 8) & 0xFF)); + bytes.add((byte) ((value >>> 16) & 0xFF)); + bytes.add((byte) ((value >>> 24) & 0xFF)); + bytes.add((byte) ((value >>> 32) & 0xFF)); + bytes.add((byte) ((value >>> 40) & 0xFF)); + bytes.add((byte) ((value >>> 48) & 0xFF)); + bytes.add((byte) ((value >>> 56) & 0xFF)); + + } + + @Override + protected Pair toBytes() { + + assert(isCommitted()); + + TByteArrayList raw = new TByteArrayList(); + + writeLE(raw, operations.size()); + + for(ClusterUpdateOperation op : operations) { + + writeLE(raw, op.data.length); + raw.add(op.data); + op.data = null; + + writeLE(raw, op.ccs.statementMask.size()); + raw.add(op.ccs.statementMask.toArray()); + writeLE(raw, op.ccs.oldValueEx.size()); + raw.add(op.ccs.oldValueEx.toArray()); + writeLE(raw, op.ccs.oldValues.size()); + + for(byte[] oldValue : op.ccs.oldValues) { + int len = (oldValue != null ? oldValue.length : -1); + writeLE(raw, len); + if(oldValue != null) { + raw.add(oldValue); + } + } + + } + + byte[] raw_ = raw.toArray(); + CompressionCodec codec = Compressions.get(Compressions.LZ4); + ByteBuffer input = ByteBuffer.wrap(raw_); + ByteBuffer output = ByteBuffer.allocate(raw_.length + raw_.length/8); + int compressedSize = codec.compressBuffer(input, 0, raw_.length, output, 0); + + // We append inflated size - cannot prepend since decompression cannot handle offsets in input + final byte[] rawOutput = new byte[compressedSize+4]; + output.get(rawOutput,0,compressedSize); + Bytes.writeLE(rawOutput, compressedSize, raw_.length); + + release(); + + return Pair.make(rawOutput, rawOutput.length); + + } + + @Override + void release() { + + for(ClusterUpdateOperation op : operations) { + op.data = null; + op.ccs = null; + } + + } + + static class StreamDecompressor { + +// byte[] decompressBuffer = new byte[1024*1024]; + + public synchronized byte[] decompressBuffer(byte[] compressed) throws IOException { + + int deflatedSize = Bytes.readLE4(compressed, compressed.length-4); + + byte[] result = new byte[deflatedSize]; + +// if(decompressBuffer.length < deflatedSize) +// decompressBuffer = new byte[Math.max(3*decompressBuffer.length / 2, deflatedSize)]; + + CompressionCodec codec = Compressions.get(Compressions.LZ4); + + ByteBuffer input = ByteBuffer.wrap(compressed); + ByteBuffer output = ByteBuffer.wrap(result); + + int decompressedSize = codec.decompressBuffer(input, 0, compressed.length-4, output, 0, result.length); + assert(decompressedSize == deflatedSize); + + return result; + + } + + + } + + private static StreamDecompressor decompressor = new StreamDecompressor(); + + @Override + public void fromFile(byte[] data_) throws IllegalAcornStateException, AcornAccessVerificationException { + + try { + + byte[] data = decompressor.decompressBuffer(data_); + + operations = new ArrayList(); + + int offset = 0; + int opLen = Bytes.readLE4(data, offset); + offset += 4; + + for(int i=0;i(oldValuesSize); + for(int j=0;j LRU, String id, int size) throws AcornAccessVerificationException { + super(LRU, id, LRU.getDirectory(), id.toString() + ".extFile", true, true); + this.bytes = new TByteArrayList(size); + LRU.insert(this, accessTime); + } + + public byte[] getResourceFile() throws AcornAccessVerificationException, IllegalAcornStateException { + + if(VERIFY) verifyAccess(); + + makeResident(); + return bytes.toArray(); + } + + + public ResourceSegment getResourceSegment(final byte[] clusterUID, final int resourceIndex, final long segmentOffset, short segmentSize) throws AcornAccessVerificationException, IllegalAcornStateException { + + if(VERIFY) verifyAccess(); + + makeResident(); + try { + int segSize = segmentSize; + if (segSize < 0) + segSize += 65536; + if (segmentSize == -1) + segSize = Math.min(65535, bytes.size()); + + final long valueSize = bytes.size(); + final byte[] segment = bytes.toArray((int) segmentOffset, segSize); + + return new ResourceSegment() { + + @Override + public long getValueSize() { + return valueSize; + } + + @Override + public byte[] getSegment() { + return segment; + } + + @Override + public int getResourceIndex() { + return resourceIndex; + } + + @Override + public long getOffset() { + return segmentOffset; + } + + @Override + public byte[] getClusterId() { + return clusterUID; + } + }; + } catch (Throwable t) { + throw new IllegalAcornStateException(t); + } + } + + public void updateData(byte[] newBytes, long offset, long pos, long size) throws AcornAccessVerificationException, IllegalAcornStateException { + + if(VERIFY) verifyAccess(); + makeResident(); + + if(size == 0) { + bytes.remove((int)offset, (int)(bytes.size()-offset)); + } else { + bytes.fill((int) (offset + size), (int) (offset + size), (byte) 0); + bytes.set((int) offset, newBytes, (int) pos, (int) size); + } + + setDirty(); + + } + + @Override + public Pair toBytes() { + byte[] result = bytes.toArray(); + release(); + return Pair.make(result, result.length); + } + + @Override + protected void release() { + bytes = null; + } + + @Override + public void fromFile(byte[] data) { + bytes = new TByteArrayList(data); + } + + @Override + protected String getExtension() { + return "extFile"; + } + + @Override + protected boolean overwrite() { + return true; + } + +} \ No newline at end of file diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/LRU.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/LRU.java new file mode 100644 index 000000000..323d66d3d --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/LRU.java @@ -0,0 +1,590 @@ +package org.simantics.acorn.lru; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.Semaphore; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; + +import org.simantics.acorn.ClusterManager; +import org.simantics.acorn.GraphClientImpl2; +import org.simantics.acorn.exception.AcornAccessVerificationException; +import org.simantics.acorn.exception.IllegalAcornStateException; +import org.simantics.db.common.utils.Logger; + +/* + * The order rule of synchronization for LRU and LRUObject is: + * § Always lock LRUObject first! + * + */ + +public class LRU> { + + public static boolean VERIFY = true; + + final private long swapTime = 5L*1000000000L; + final private int swapSize = 200; + + final private HashMap map = new HashMap(); + final private TreeMap priorityQueue = new TreeMap(); + + final private Semaphore mutex = new Semaphore(1); + final private String identifier; + + private Path writeDir; + + private Thread mutexOwner; + + public Map pending = new HashMap(); + + protected final ClusterManager manager; + + public LRU(ClusterManager manager, String identifier, Path writeDir) { + this.manager = manager; + this.identifier = identifier; + this.writeDir = writeDir; + resume(); + } + + /* + * Public interface + */ + + public void acquireMutex() throws IllegalAcornStateException { + try { + while(!mutex.tryAcquire(3, TimeUnit.SECONDS)) { + System.err.println("Mutex is taking a long time to acquire - owner is " + mutexOwner); + } + if(VERIFY) + mutexOwner = Thread.currentThread(); + } catch (InterruptedException e) { + throw new IllegalAcornStateException(e); + } + } + + public void releaseMutex() { + mutex.release(); + mutexOwner = null; + } + + public void shutdown() { + if (GraphClientImpl2.DEBUG) + System.err.println("Shutting down LRU writers " + writers); + writers.shutdown(); + try { + writers.awaitTermination(60, TimeUnit.SECONDS); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + } + + public void resume() { + writers = new ScheduledThreadPoolExecutor(2, new ThreadFactory() { + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, identifier + " File Writer"); + } + }); + if (GraphClientImpl2.DEBUG) + System.err.println("Resuming LRU writers " + writers); + } + + /* + * This method violates the synchronization order rule between LRU and MapVAlue + * External synchronization is used to ensure correct operation + */ + public void persist(ArrayList state) throws IllegalAcornStateException { + + acquireMutex(); + try { + for (MapValue value : values()) { + value.acquireMutex(); + // for debugging purposes + boolean persisted = false; + try { + // Persist the value if needed + persisted = value.persist(); + } finally { + // WriteRunnable may want to + value.releaseMutex(); + } + // Wait pending if value was actually persisted + waitPending(value, false); + // Take lock again + value.acquireMutex(); + try { + // Record the value + state.add(value.getStateKey()); + } finally { + value.releaseMutex(); + } + } + } catch (IllegalAcornStateException e) { + throw e; + } catch (IOException e) { + throw new IllegalAcornStateException("Unable to waitPending for " + this.identifier, e); + } catch (Throwable t) { + throw new IllegalAcornStateException("Fatal error occured for " + this.identifier, t); + } finally { + releaseMutex(); + } + } + + public MapValue getWithoutMutex(MapKey key) throws AcornAccessVerificationException, IllegalAcornStateException { + + acquireMutex(); + try { + return get(key); + } finally { + releaseMutex(); + } + } + + public MapValue get(MapKey key) throws AcornAccessVerificationException { + + if(VERIFY) verifyAccess(); + + return map.get(key); + } + + public void map(MapValue info) throws AcornAccessVerificationException { + + if(VERIFY) verifyAccess(); + + map.put(info.getKey(), info); + } + + public Collection values() throws AcornAccessVerificationException { + + if(VERIFY) verifyAccess(); + + return map.values(); + } + + public boolean swapForced() throws IllegalAcornStateException, AcornAccessVerificationException { + + acquireMutex(); + + try { + return swap(0, 0, null); + } finally { + releaseMutex(); + } + + } + + public boolean swap(long lifeTime, int targetSize) throws AcornAccessVerificationException, IllegalAcornStateException { + + if(VERIFY) verifyAccess(); + + return swap(lifeTime, targetSize, null); + } + + /* + * This is called under global lock + */ + public void setWriteDir(Path dir) { + + this.writeDir = dir; + } + + + /* + * Package access + */ + + void insert(MapValue info, long accessTime) throws AcornAccessVerificationException { + + if(VERIFY) verifyAccess(); + + map.put(info.getKey(), info); + priorityQueue.put(accessTime, info.getKey()); + } + + /* + * We have access to ClusterLRU - try to refresh value if available + */ + boolean tryRefresh(MapValue info) throws AcornAccessVerificationException, IllegalAcornStateException { + + if(VERIFY) verifyAccess(); + + if(!info.tryAcquireMutex()) + return false; + + try { + priorityQueue.remove(info.getLastAccessTime()); + info.accessed(); + map.put(info.getKey(), info); + priorityQueue.put(info.getLastAccessTime(), info.getKey()); + return true; + } finally { + info.releaseMutex(); + } + } + + /* + * We have access to MapValue and no access to clusterLRU + */ + void refresh(MapValue info, boolean needMutex) throws AcornAccessVerificationException, IllegalAcornStateException { + + if(VERIFY) { + if(!needMutex) verifyAccess(); + info.verifyAccess(); + } + + if(needMutex) + acquireMutex(); + + try { + + priorityQueue.remove(info.getLastAccessTime()); + info.accessed(); + map.put(info.getKey(), info); + priorityQueue.put(info.getLastAccessTime(), info.getKey()); + + } catch (AcornAccessVerificationException e) { + throw e; + } catch (Throwable t) { + throw new IllegalAcornStateException(t); + } finally { + if(needMutex) + releaseMutex(); + } + } + + /* + * Private implementation + */ + + int size() throws AcornAccessVerificationException { + if(VERIFY) verifyAccess(); + return priorityQueue.size(); + } + + boolean swap(MapKey excluded) throws AcornAccessVerificationException, IllegalAcornStateException { + if(VERIFY) verifyAccess(); + return swap(swapTime, swapSize, excluded); + } + + boolean swap(long lifeTime, int targetSize, MapKey excluded) throws AcornAccessVerificationException, IllegalAcornStateException { + + if(VERIFY) verifyAccess(); + + MapValue valueToSwap = getValueToSwap(lifeTime, targetSize, excluded); + if(valueToSwap != null) { + + if(valueToSwap.tryAcquireMutex()) { + try { + if(valueToSwap.canBePersisted()) { + valueToSwap.persist(); + return true; + } + } catch (Throwable t) { + throw new IllegalAcornStateException(t); + } finally { + valueToSwap.releaseMutex(); + } + } + } + return false; + } + + private MapValue getValueToSwap1(long lifeTime, int targetSize, MapKey excluded) throws AcornAccessVerificationException, IllegalAcornStateException { + + if(VERIFY) verifyAccess(); + + for(int i=0;i<10;i++) { + + long candidate = getSwapCandidate(lifeTime, targetSize); + if(candidate == 0) return null; + + MapKey key = priorityQueue.remove(candidate); + if(key.equals(excluded)) { + tryRefresh(map.get(key)); + continue; + } + + return map.get(key); + } + return null; + } + + + private MapValue getValueToSwap(long lifeTime, int targetSize, MapKey excluded) throws AcornAccessVerificationException, IllegalAcornStateException { + + if(VERIFY) verifyAccess(); + + for(int i=0;i<10;i++) { + + // Lock LRU and get a candidate + MapValue value = getValueToSwap1(lifeTime, targetSize, excluded); + if(value == null) return null; + + if(value.tryAcquireMutex()) { + + try { + // This may lock the object + if(value.canBePersisted()) + return value; + // Insert back the value + refresh(value, false); + } finally { + value.releaseMutex(); + } + } + } + return null; + } + + private long getSwapCandidate(long lifeTime, int targetSize) throws AcornAccessVerificationException { + + if(VERIFY) verifyAccess(); + + if(priorityQueue.isEmpty()) return 0; + + long currentTime = System.nanoTime(); + Long lowest = priorityQueue.firstKey(); + + if(currentTime - lowest > lifeTime || priorityQueue.size() > targetSize) { + return lowest; + } + + return 0; + + } + + /* + * Tries to persist this object. Can fail if the object cannot be persisted at this time. + * + */ + boolean persist(Object object_) throws AcornAccessVerificationException { + + MapValue object = (MapValue)object_; + + if(VERIFY) object.verifyAccess(); + + if(object.isDirty()) { + // It is possible that this just became unpersistable. Fail here in this case. + if(!object.canBePersisted()) { + return false; + } + + assert(object.isResident()); + + Path f = writeDir.resolve(object.getFileName()); + + WriteRunnable runnable = new WriteRunnable(f, object); + + synchronized(pending) { + WriteRunnable existing = pending.put(object.getKey().toString(), runnable); + assert(existing == null); + } + + writers.execute(runnable); + + object.setResident(false); + object.setDirty(false); + + return true; + + } else if(object.isResident()) { + + object.release(); + object.setResident(false); + return false; + } + return false; + } + + int makeResident(Object object_, boolean keepResident) throws AcornAccessVerificationException, IllegalAcornStateException { + + MapValue object = (MapValue)object_; + + if(VERIFY) object.verifyAccess(); + + try { + object.setForceResident(keepResident); + + if(object.isResident()) { + refresh(object, true); + return 0; + } + + waitPending(object, true); + + byte[] data = object.readFile(); + + object.fromFile(data); + object.setResident(true); + + acquireMutex(); + try { + refresh(object, false); + swap(swapTime, swapSize, object.getKey()); + } finally { + releaseMutex(); + } + return data.length; + } catch (IOException e) { + throw new IllegalAcornStateException("Unable to makeResident " + identifier, e); + } + } + + static int readCounter = 0; + static int writeCounter = 0; + + ScheduledThreadPoolExecutor writers; + + void waitPending(MapValue value, boolean hasMutex) throws IOException, AcornAccessVerificationException, IllegalAcornStateException { + + WriteRunnable runnable = null; + boolean inProgress = false; + synchronized(pending) { + runnable = pending.get(value.getKey().toString()); + if(runnable != null) { + synchronized(runnable) { + if(runnable.committed) { + // just being written - just need to wait + inProgress = true; + } else { + runnable.committed = true; + // we do the writing + } + } + } + } + if(runnable != null) { + if(inProgress) { +// System.err.println("reader waits for WriteRunnable to finish"); + try { + if(hasMutex) { + runnable.borrowMutex = true; + } + runnable.s.acquire(); + } catch (InterruptedException e) { + throw new IllegalAcornStateException(e); + } + } else { +// System.err.println("reader took WriteRunnable"); + runnable.runReally(hasMutex); + } + } + } + + public class WriteRunnable implements Runnable { + + private Path bytes; + private MapValue impl; + private boolean committed = false; + private boolean borrowMutex = false; + private Semaphore s = new Semaphore(0); + + WriteRunnable(Path bytes, MapValue impl) { + this.bytes = bytes; + this.impl = impl; + } + + @Override + public void run() { + try { + synchronized(impl) { + + synchronized(this) { + + if(committed) + return; + + committed = true; + } + runReally(false); + } + } catch (Throwable t) { + if (t instanceof IllegalAcornStateException) { + manager.notSafeToMakeSnapshot((IllegalAcornStateException)t); + } else { + manager.notSafeToMakeSnapshot(new IllegalAcornStateException(t)); + } + t.printStackTrace(); + Logger.defaultLogError(t); + } + } + + public void runWithMutex() throws IOException, IllegalAcornStateException, AcornAccessVerificationException { + + try { + // These have been set in method persist + assert (!impl.isResident()); + assert (!impl.isDirty()); + + impl.toFile(bytes); + } finally { + synchronized (pending) { + pending.remove(impl.getKey().toString()); + s.release(Integer.MAX_VALUE); + } + } + + } + + // Fix WriteRunnable.runReally() to use LRU.MapValue mutex instead of + // borrowMutex + public void runReally(boolean hasMutex) throws IOException, IllegalAcornStateException, AcornAccessVerificationException { + + if (hasMutex) { + + runWithMutex(); + + } else { + + boolean gotMutex = impl.tryAcquireMutex(); + + boolean done = false; + while (!done) { + + if (gotMutex || borrowMutex) { + runWithMutex(); + done = true; + } else { + System.err.println("Retry mutex acquire"); + gotMutex = impl.tryAcquireMutex(); + } + + } + + if (gotMutex) + impl.releaseMutex(); + + } + + } + } + + public Path getDirectory() { + return writeDir; + } + + /* + * Protected implementation + * + */ + + protected void verifyAccess() throws AcornAccessVerificationException { + if (mutex.availablePermits() != 0) + throw new AcornAccessVerificationException("identifier=" + identifier + " mutex has " + mutex.availablePermits() + " available permits, should be 0! Current mutexOwner is " + mutexOwner); + } + + /* + * Private implementation + * + */ + + +} diff --git a/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/LRUObject.java b/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/LRUObject.java new file mode 100644 index 000000000..3194d591e --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/acorn/lru/LRUObject.java @@ -0,0 +1,247 @@ +package org.simantics.acorn.lru; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; + +import org.simantics.acorn.FileIO; +import org.simantics.acorn.Persistable; +import org.simantics.acorn.exception.AcornAccessVerificationException; +import org.simantics.acorn.exception.IllegalAcornStateException; +import org.simantics.utils.datastructures.Pair; + +public abstract class LRUObject> implements Persistable { + + public static boolean VERIFY = true; + + // Final stuff + final protected LRU LRU; + final private Semaphore mutex = new Semaphore(1); + final private MapKey key; + final private String fileName; + + // Mutable stuff + protected long accessTime = AccessTime.getInstance().getAccessTime(); + private int offset; + private int length; + private boolean resident = true; + private boolean dirty = true; + private boolean forceResident = false; + + // DEBUG +// private boolean isForceResidentSetAfterLastGet = false; + + private Path readDirectory; + + private Thread mutexOwner; + + // for loading + public LRUObject(LRU LRU, MapKey key, Path readDirectory, String fileName, int offset, int length, boolean dirty, boolean resident) { + this.LRU = LRU; + this.key = key; + this.fileName = fileName; + this.offset = offset; + this.length = length; + this.readDirectory = readDirectory; + this.dirty = dirty; + this.resident = resident; + } + + // for creating + public LRUObject(LRU LRU, MapKey key, Path readDirectory, String fileName, boolean dirty, boolean resident) { + this(LRU, key, readDirectory, fileName, -1, -1, dirty, resident); + } + + /* + * Public interface + */ + public MapKey getKey() { + // This can be called without mutex + return key; + } + + public void acquireMutex() throws IllegalAcornStateException { + try { + while(!mutex.tryAcquire(3, TimeUnit.SECONDS)) { + System.err.println("Mutex is taking a long time to acquire - owner is " + mutexOwner); + } + + if(VERIFY) + mutexOwner = Thread.currentThread(); + + } catch (InterruptedException e) { + throw new IllegalAcornStateException(e); + } + } + + public boolean tryAcquireMutex() { + return mutex.tryAcquire(); + } + + public void releaseMutex() { + mutex.release(); + } + + @Override + public void toFile(Path bytes) throws IOException { + if(VERIFY) { + try { + verifyAccess(); + } catch (AcornAccessVerificationException e) { + throw new IOException("Exception occured during toFile for file " + fileName, e); + } + } + try { + Pair pair = toBytes(); + byte[] data = pair.first; + int length = pair.second; + FileIO fio = FileIO.get(bytes); + int offset = fio.saveBytes(data, length, overwrite()); + setPosition(offset, length); + } catch (AcornAccessVerificationException | IllegalAcornStateException e) { + throw new IOException("Exception occured during toFile for file " + fileName, e); + } + } + + public int makeResident() throws AcornAccessVerificationException, IllegalAcornStateException { + if(VERIFY) verifyAccess(); + return LRU.makeResident(this, false); + } + + public int makeResident(boolean keepResident) throws AcornAccessVerificationException, IllegalAcornStateException { + if(VERIFY) verifyAccess(); + return LRU.makeResident(this, true); + } + + /* + * Package implementation details + */ + + abstract void release(); + abstract String getExtension(); + + String getStateKey() throws IllegalAcornStateException, AcornAccessVerificationException { + String result = getKey().toString() + "#" + getDirectory().getFileName() + "#" + getOffset() + "#" + getLength(); + if(offset == -1) + throw new IllegalAcornStateException(result); + return result; + } + + long getLastAccessTime() throws AcornAccessVerificationException { + if(VERIFY) verifyAccess(); + return accessTime; + } + + void accessed() throws AcornAccessVerificationException { + if(VERIFY) verifyAccess(); + accessTime = AccessTime.getInstance().getAccessTime(); + } + + boolean persist() throws AcornAccessVerificationException { + if(VERIFY) verifyAccess(); + if(LRU.persist(this)) { + readDirectory = LRU.getDirectory(); + return true; + } else { + return false; + } + } + + void setForceResident(boolean value) throws AcornAccessVerificationException { + if(VERIFY) verifyAccess(); + forceResident = value; +// isForceResidentSetAfterLastGet = true; + } + + boolean canBePersisted() throws AcornAccessVerificationException { + if(VERIFY) verifyAccess(); +// isForceResidentSetAfterLastGet = false; + return !forceResident; + } + + boolean isDirty() throws AcornAccessVerificationException { + if(VERIFY) verifyAccess(); + return dirty; + } + + boolean isResident() throws AcornAccessVerificationException { + if(VERIFY) verifyAccess(); + return resident; + } + + String getFileName() throws AcornAccessVerificationException { + if(VERIFY) verifyAccess(); + return fileName; + } + + void setResident(boolean value) throws AcornAccessVerificationException { + if(VERIFY) verifyAccess(); + resident = value; + } + + void setDirty(boolean value) throws AcornAccessVerificationException { + if(VERIFY) verifyAccess(); + dirty = value; + } + + byte[] readFile() throws IOException, AcornAccessVerificationException { + if(VERIFY) verifyAccess(); + Path dir = getDirectory(); + Path f = dir.resolve(getFileName()); + FileIO fio = FileIO.get(f); + return fio.readBytes(getOffset(), getLength()); + } + + /* + * Protected implementation details + */ + + abstract protected boolean overwrite(); + + abstract protected Pair toBytes() throws IllegalAcornStateException; + + protected void setDirty() throws AcornAccessVerificationException { + if(VERIFY) verifyAccess(); + dirty = true; + } + + protected void verifyAccess() throws AcornAccessVerificationException { + if (mutex.availablePermits() != 0) + throw new AcornAccessVerificationException("fileName=" + fileName + " mutex has " + mutex.availablePermits() + " available permits, should be 0! Current mutexOwner is " + mutexOwner); + } + + protected synchronized void cancelForceResident() throws AcornAccessVerificationException { + setForceResident(false); + } + + /* + * Private implementation details + */ + + private int getOffset() throws AcornAccessVerificationException { + if(VERIFY) verifyAccess(); + return offset; + } + + private int getLength() throws AcornAccessVerificationException { + if(VERIFY) verifyAccess(); + return length; + } + + private void setPosition(int offset, int length) throws AcornAccessVerificationException, IllegalAcornStateException { + if(VERIFY) verifyAccess(); + if(offset == -1) + throw new IllegalAcornStateException("offset == -1 for " + fileName + " in " + readDirectory.toAbsolutePath() + ", dirty=" + dirty + ", resident=" + resident + ", forceResident=" + forceResident); + this.offset = offset; + this.length = length; + if(overwrite() && offset > 0) + throw new IllegalAcornStateException("overwrite() == true && offset > 0 for " + fileName + " in " + readDirectory.toAbsolutePath() + ", dirty=" + dirty + ", resident=" + resident + ", forceResident=" + forceResident); + } + + private Path getDirectory() throws AcornAccessVerificationException { + if(VERIFY) verifyAccess(); + return readDirectory; + } + +} \ No newline at end of file diff --git a/bundles/org.simantics.acorn/src/org/simantics/db/javacore/HeadState.java b/bundles/org.simantics.acorn/src/org/simantics/db/javacore/HeadState.java new file mode 100644 index 000000000..0fb29333b --- /dev/null +++ b/bundles/org.simantics.acorn/src/org/simantics/db/javacore/HeadState.java @@ -0,0 +1,73 @@ +package org.simantics.db.javacore; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.Serializable; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; + +import org.simantics.acorn.exception.InvalidHeadStateException; + +public class HeadState implements Serializable { + + private static final long serialVersionUID = -4135031566499790077L; + + public int headChangeSetId = 0; + public long transactionId = 1; + public long reservedIds = 3; + + public ArrayList clusters = new ArrayList<>(); + public ArrayList files = new ArrayList<>(); + public ArrayList stream = new ArrayList<>(); + public ArrayList cs = new ArrayList<>(); +// public ArrayList ccs = new ArrayList(); + + public static HeadState load(Path directory) throws InvalidHeadStateException { + Path f = directory.resolve("head.state"); + try { + byte[] bytes = Files.readAllBytes(f); + MessageDigest sha1 = MessageDigest.getInstance("SHA-1"); + int digestLength = sha1.getDigestLength(); + sha1.update(bytes, digestLength, bytes.length - digestLength); + byte[] newChecksum = sha1.digest(); + if (!Arrays.equals(newChecksum, Arrays.copyOfRange(bytes, 0, digestLength))) { + throw new InvalidHeadStateException( + "Checksum " + Arrays.toString(newChecksum) + " does not match excpected " + + Arrays.toString(Arrays.copyOfRange(bytes, 0, digestLength)) + " for " + f.toAbsolutePath()); + } + try (ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bytes, digestLength, bytes.length - digestLength))) { + HeadState state = (HeadState) ois.readObject(); + return state; + } + } catch (IOException i) { + return new HeadState(); + } catch (ClassNotFoundException c) { +// throw new Error("HeadState class not found", c); + return new HeadState(); + } catch (NoSuchAlgorithmException e) { + throw new Error("SHA-1 Algorithm not found", e); + } + } + + public static void validateHeadStateIntegrity(Path headState) throws InvalidHeadStateException, IOException { + try { + byte[] bytes = Files.readAllBytes(headState); + MessageDigest sha1 = MessageDigest.getInstance("SHA-1"); + int digestLength = sha1.getDigestLength(); + sha1.update(bytes, digestLength, bytes.length - digestLength); + byte[] newChecksum = sha1.digest(); + if (!Arrays.equals(newChecksum, Arrays.copyOfRange(bytes, 0, digestLength))) { + throw new InvalidHeadStateException( + "Checksum " + Arrays.toString(newChecksum) + " does not match excpected " + + Arrays.toString(Arrays.copyOfRange(bytes, 0, digestLength)) + " for " + headState.toAbsolutePath()); + } + } catch (NoSuchAlgorithmException e) { + throw new Error("SHA-1 digest not found, should not happen", e); + } + } +} diff --git a/bundles/org.simantics.annotation.ui/META-INF/MANIFEST.MF b/bundles/org.simantics.annotation.ui/META-INF/MANIFEST.MF index e962c736e..9435da36d 100644 --- a/bundles/org.simantics.annotation.ui/META-INF/MANIFEST.MF +++ b/bundles/org.simantics.annotation.ui/META-INF/MANIFEST.MF @@ -16,7 +16,8 @@ Require-Bundle: org.eclipse.ui, org.simantics.browsing.ui.model;bundle-version="1.0.0", org.simantics.modeling.ui;bundle-version="1.1.1", org.simantics.graph.db;bundle-version="1.1.9", - org.simantics.views.swt.client;bundle-version="1.0.0" + org.simantics.views.swt.client;bundle-version="1.0.0", + org.slf4j.api;bundle-version="1.7.20" Bundle-ActivationPolicy: lazy Bundle-RequiredExecutionEnvironment: JavaSE-1.8 Bundle-Vendor: VTT Technical Research Centre of Finland diff --git a/bundles/org.simantics.annotation.ui/src/org/simantics/annotation/ui/SCL.java b/bundles/org.simantics.annotation.ui/src/org/simantics/annotation/ui/SCL.java index 7a7977f33..f4f99997c 100644 --- a/bundles/org.simantics.annotation.ui/src/org/simantics/annotation/ui/SCL.java +++ b/bundles/org.simantics.annotation.ui/src/org/simantics/annotation/ui/SCL.java @@ -34,7 +34,6 @@ import org.simantics.db.VirtualGraph; import org.simantics.db.WriteGraph; import org.simantics.db.common.request.UnaryRead; import org.simantics.db.common.request.WriteRequest; -import org.simantics.db.common.utils.Logger; import org.simantics.db.exception.DatabaseException; import org.simantics.db.layer0.util.Layer0Utils; import org.simantics.db.layer0.util.RemoverUtil; @@ -60,6 +59,8 @@ import org.simantics.utils.strings.AlphanumComparator; import org.simantics.utils.ui.ISelectionUtils; import org.simantics.views.swt.client.base.ISWTViewNode; import org.simantics.views.swt.client.impl.SWTExplorer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import gnu.trove.map.hash.THashMap; import gnu.trove.set.hash.THashSet; @@ -69,6 +70,8 @@ import gnu.trove.set.hash.THashSet; * @author Tuukka Lehtonen */ public class SCL { + + private static final Logger LOGGER = LoggerFactory.getLogger(SCL.class); final public static String EMPTY = ""; final public static String MAPPED = "Mapped"; @@ -493,7 +496,7 @@ public class SCL { try { doAdd((Variable)properties.input); } catch (DatabaseException e) { - Logger.defaultLogError(e); + LOGGER.error("newAnnotationModifier failed", e); } return null; @@ -670,7 +673,7 @@ public class SCL { String name = graph.getPossibleRelatedValue(r, L0.HasName, Bindings.STRING); if(name != null) { if (result.put(name, r) != null) - System.err.println(this + ": The database contains siblings with the same name " + name + " (resource=$" + resource.getResourceId() +")."); + LOGGER.error("The database contains siblings with the same name " + name + " (resource=$" + resource.getResourceId() +")."); } } return result; diff --git a/bundles/org.simantics.backup.ontology/build.properties b/bundles/org.simantics.backup.ontology/build.properties index 022de1735..ecdc7c302 100644 --- a/bundles/org.simantics.backup.ontology/build.properties +++ b/bundles/org.simantics.backup.ontology/build.properties @@ -1,6 +1,5 @@ source.. = src/ output.. = bin/ -bin.includes = plugin.xml,\ - META-INF/,\ +bin.includes = META-INF/,\ .,\ graph.tg diff --git a/bundles/org.simantics.charts/src/org/simantics/charts/Charts.java b/bundles/org.simantics.charts/src/org/simantics/charts/Charts.java index 1e503159e..1358c4023 100644 --- a/bundles/org.simantics.charts/src/org/simantics/charts/Charts.java +++ b/bundles/org.simantics.charts/src/org/simantics/charts/Charts.java @@ -1,75 +1,75 @@ -package org.simantics.charts; - -import java.util.Collections; -import java.util.List; - -import org.simantics.charts.editor.ChartData; -import org.simantics.charts.editor.ChartKeys; -import org.simantics.databoard.binding.error.BindingException; -import org.simantics.databoard.util.Bean; -import org.simantics.db.ReadGraph; -import org.simantics.db.Resource; -import org.simantics.db.common.request.PossibleIndexRoot; -import org.simantics.db.exception.DatabaseException; -import org.simantics.db.layer0.variable.Variable; -import org.simantics.history.HistoryException; -import org.simantics.history.HistorySamplerItem; -import org.simantics.history.ItemManager; -import org.simantics.history.util.subscription.SamplingFormat; -import org.simantics.modeling.subscription.SubscriptionItem; -import org.simantics.modeling.subscription.SubscriptionItemQuery; -import org.simantics.project.IProject; -import org.simantics.simulation.experiment.IExperiment; - -/** - * Main facade for externally dealing with the trending system. - * - * @author Tuukka Lehtonen - * @author Antti Villberg - * - */ -public final class Charts { - - public static void resetChartEditorData(IProject project, Resource model, ChartData editorData) { - if (editorData != null) { - project.setHint(ChartKeys.chartSourceKey(model), editorData); - } else { - project.removeHint(ChartKeys.chartSourceKey(model)); - } - } - - public static HistorySamplerItem createHistorySamplerItem(ReadGraph graph, Variable run, Resource subscriptionItem) throws DatabaseException { - IExperiment exp = (IExperiment) run.getPropertyValue(graph, "iExperiment"); - ITrendSupport support = exp.getService(ITrendSupport.class); - ChartData data = support.getChartData(); - return createHistorySamplerItem(graph, subscriptionItem, data); - } - - public static HistorySamplerItem createHistorySamplerItem(ReadGraph graph, Resource subscriptionItem, ChartData data) throws DatabaseException { - - try { - Resource model = graph.syncRequest(new PossibleIndexRoot(subscriptionItem)); - if (model == null) { - throw new DatabaseException("There is no model for " + subscriptionItem); - } - - ItemManager im = new ItemManager(data.history.getItems()); - - SubscriptionItem i = graph.syncRequest(new SubscriptionItemQuery(subscriptionItem)); - - List items = im.search("variableId", i.variableId); - Collections.sort(items, SamplingFormat.INTERVAL_COMPARATOR); - if (items.isEmpty()) - new DatabaseException("There is history item for " + i.variableId); - Bean config = items.get(0); - String historyId = (String) config.getFieldUnchecked("id"); - - return new HistorySamplerItem(data.collector, data.history, historyId, System.identityHashCode(data)); - } catch (HistoryException e) { - throw new DatabaseException(e); - } catch (BindingException e) { - throw new DatabaseException(e); - } - } - -} +package org.simantics.charts; + +import java.util.Collections; +import java.util.List; + +import org.simantics.charts.editor.ChartData; +import org.simantics.charts.editor.ChartKeys; +import org.simantics.databoard.binding.error.BindingException; +import org.simantics.databoard.util.Bean; +import org.simantics.db.ReadGraph; +import org.simantics.db.Resource; +import org.simantics.db.common.request.PossibleIndexRoot; +import org.simantics.db.exception.DatabaseException; +import org.simantics.db.layer0.variable.Variable; +import org.simantics.history.HistoryException; +import org.simantics.history.HistorySamplerItem; +import org.simantics.history.ItemManager; +import org.simantics.history.util.subscription.SamplingFormat; +import org.simantics.modeling.subscription.SubscriptionItem; +import org.simantics.modeling.subscription.SubscriptionItemQuery; +import org.simantics.project.IProject; +import org.simantics.simulation.experiment.IExperiment; + +/** + * Main facade for externally dealing with the trending system. + * + * @author Tuukka Lehtonen + * @author Antti Villberg + * + */ +public final class Charts { + + public static void resetChartEditorData(IProject project, Resource model, ChartData editorData) { + if (editorData != null) { + project.setHint(ChartKeys.chartSourceKey(model), editorData); + } else { + project.removeHint(ChartKeys.chartSourceKey(model)); + } + } + + public static HistorySamplerItem createHistorySamplerItem(ReadGraph graph, Variable run, Resource subscriptionItem) throws DatabaseException { + IExperiment exp = (IExperiment) run.getPropertyValue(graph, "iExperiment"); + ITrendSupport support = exp.getService(ITrendSupport.class); + ChartData data = support.getChartData(); + return createHistorySamplerItem(graph, subscriptionItem, data); + } + + public static HistorySamplerItem createHistorySamplerItem(ReadGraph graph, Resource subscriptionItem, ChartData data) throws DatabaseException { + + try { + Resource model = graph.syncRequest(new PossibleIndexRoot(subscriptionItem)); + if (model == null) { + throw new DatabaseException("There is no model for " + subscriptionItem); + } + + ItemManager im = new ItemManager(data.history.getItems()); + + SubscriptionItem i = graph.syncRequest(new SubscriptionItemQuery(subscriptionItem)); + + List items = im.search("variableId", i.variableId); + Collections.sort(items, SamplingFormat.INTERVAL_COMPARATOR); + if (items.isEmpty()) + new DatabaseException("There is history item for " + i.variableId); + Bean config = items.get(0); + String historyId = (String) config.getFieldUnchecked("id"); + + return new HistorySamplerItem(data.collector, data.history, historyId, System.identityHashCode(data)); + } catch (HistoryException e) { + throw new DatabaseException(e); + } catch (BindingException e) { + throw new DatabaseException(e); + } + } + +} diff --git a/bundles/org.simantics.compressions/src/org/simantics/compressions/impl/DecompressingInputStream.java b/bundles/org.simantics.compressions/src/org/simantics/compressions/impl/DecompressingInputStream.java index 8c7987177..9329d83c4 100644 --- a/bundles/org.simantics.compressions/src/org/simantics/compressions/impl/DecompressingInputStream.java +++ b/bundles/org.simantics.compressions/src/org/simantics/compressions/impl/DecompressingInputStream.java @@ -134,16 +134,20 @@ public abstract class DecompressingInputStream extends InputStream { return true; } - private static ByteBuffer ensureBufferSize(ByteBuffer buffer, int minCapacity) { + private ByteBuffer ensureBufferSize(ByteBuffer buffer, int minCapacity) { int oldCapacity = buffer != null ? buffer.capacity() : 0; if (buffer == null || oldCapacity < minCapacity) { int newCapacity = grow(oldCapacity, minCapacity); //System.out.println("ensureBufferSize(" + oldCapacity + ", " + minCapacity + "), new capacity " + newCapacity); - buffer = ByteBuffer.allocateDirect(newCapacity); + buffer = allocateBuffer(newCapacity); } return buffer; } + protected ByteBuffer allocateBuffer(int capacity) { + return ByteBuffer.allocateDirect(capacity); + } + /** * @param oldCapacity current capacity of a buffer * @param minCapacity diff --git a/bundles/org.simantics.databoard/src/org/simantics/databoard/util/URIStringUtils.java b/bundles/org.simantics.databoard/src/org/simantics/databoard/util/URIStringUtils.java index dde498a2c..a11579f07 100644 --- a/bundles/org.simantics.databoard/src/org/simantics/databoard/util/URIStringUtils.java +++ b/bundles/org.simantics.databoard/src/org/simantics/databoard/util/URIStringUtils.java @@ -45,8 +45,7 @@ package org.simantics.databoard.util; -import java.nio.charset.Charset; -import java.util.ArrayList; +import java.util.Arrays; import java.util.List; @@ -194,13 +193,14 @@ public final class URIStringUtils { return name; } - final private static int HTTP_POSITION = "http://".length(); + final private static String HTTP_PREFIX = "http://"; + final private static int HTTP_POSITION = HTTP_PREFIX.length(); public static String[] splitURI(String uri) { int nextPathSeparator = uri.lastIndexOf(URIStringUtils.NAMESPACE_PATH_SEPARATOR); if (nextPathSeparator == -1) return null; if (nextPathSeparator == HTTP_POSITION - 1) { - if(uri.startsWith("http://")) return new String[] { "http://", uri.substring(HTTP_POSITION, uri.length()) }; + if(uri.startsWith(HTTP_PREFIX)) return new String[] { HTTP_PREFIX, uri.substring(HTTP_POSITION, uri.length()) }; else return null; } return new String[] { @@ -208,12 +208,10 @@ public final class URIStringUtils { uri.substring(nextPathSeparator + 1, uri.length()) }; } - + public static List splitURISCL(String uri) { String[] result = splitURI(uri); - ArrayList list = new ArrayList(result.length); - for(String s : result) list.add(s); - return list; + return Arrays.asList(result); } /** @@ -263,8 +261,7 @@ public final class URIStringUtils { public static String escapeURI(String localName) { if (localName == null) throw new NullPointerException("null local name"); - String result = encode(localName); - return result; + return encode(localName); } /** @@ -276,8 +273,7 @@ public final class URIStringUtils { * @return the joined namespace */ public static String appendURINamespace(String namespace, String suffix) { - //return namespace + NAMESPACE_PATH_SEPARATOR + suffix; - return new StringBuffer(namespace.length() + 1 + suffix.length()) + return new StringBuilder(namespace.length() + 1 + suffix.length()) .append(namespace) .append(NAMESPACE_PATH_SEPARATOR) .append(suffix) @@ -293,9 +289,8 @@ public final class URIStringUtils { * @return the joined URI */ public static String makeURI(String namespace, String localName) { - //return namespace + NAMESPACE_LOCAL_SEPARATOR + escapeURI(localName); String escapedLocalName = escapeURI(localName); - return new StringBuffer(namespace.length() + 1 + escapedLocalName.length()) + return new StringBuilder(namespace.length() + 1 + escapedLocalName.length()) .append(namespace) .append(NAMESPACE_LOCAL_SEPARATOR) .append(escapedLocalName) @@ -332,94 +327,59 @@ public final class URIStringUtils { } - final private static Charset UTF8 = Charset.forName("UTF-8"); - final private static Charset ASCII = Charset.forName("US-ASCII"); - - /* Copied and modified from Jena 2.4 com.hp.hpl.jena.util.URIref */ - private static String encode(String unicode) { - boolean needsEscapes = needsEscaping(unicode); - if (!needsEscapes) - return unicode; - - byte utf8[] = unicode.getBytes(UTF8); - byte rsltAscii[] = new byte[utf8.length * 6]; - int in = 0; - int out = 0; - while (in < utf8.length) { - switch (utf8[in]) { - case (byte)'a': case (byte)'b': case (byte)'c': case (byte)'d': case (byte)'e': case (byte)'f': case (byte)'g': case (byte)'h': case (byte)'i': case (byte)'j': case (byte)'k': case (byte)'l': case (byte)'m': case (byte)'n': case (byte)'o': case (byte)'p': case (byte)'q': case (byte)'r': case (byte)'s': case (byte)'t': case (byte)'u': case (byte)'v': case (byte)'w': case (byte)'x': case (byte)'y': case (byte)'z': - case (byte)'A': case (byte)'B': case (byte)'C': case (byte)'D': case (byte)'E': case (byte)'F': case (byte)'G': case (byte)'H': case (byte)'I': case (byte)'J': case (byte)'K': case (byte)'L': case (byte)'M': case (byte)'N': case (byte)'O': case (byte)'P': case (byte)'Q': case (byte)'R': case (byte)'S': case (byte)'T': case (byte)'U': case (byte)'V': case (byte)'W': case (byte)'X': case (byte)'Y': case (byte)'Z': - case (byte)'0': case (byte)'1': case (byte)'2': case (byte)'3': case (byte)'4': case (byte)'5': case (byte)'6': case (byte)'7': case (byte)'8': case (byte)'9': - case (byte)';': case (byte)'?': case (byte)':': case (byte)'@': case (byte)'=': case (byte)'+': case (byte)'$': case (byte)',': - case (byte)'-': case (byte)'_': case (byte)'.': case (byte)'!': case (byte)'~': case (byte)'*': case (byte)'\'': case (byte)'(': case (byte)')': - case (byte)'[': case (byte)']': - rsltAscii[out] = utf8[in]; - out++; - in++; - break; - case (byte)' ': - rsltAscii[out++] = (byte) '%'; - rsltAscii[out++] = '2'; - rsltAscii[out++] = '0'; - in++; - break; - case (byte) '%': - // [lehtonen] NOTE: all input needs to be escaped, i.e. "%01" should result in "%2501", not "%01". - // escape+unescape is a bijection, not an idempotent operation. - // Fall through to to escape '%' as '%25' - case (byte) '#': - case (byte) '/': - // Fall through to escape '/' - case (byte)'&': - // Fall through to escape '&' characters to avoid them - // being interpreted as SGML entities. - default: - rsltAscii[out++] = (byte) '%'; - // Get rid of sign ... - int c = (utf8[in]) & 255; - rsltAscii[out++] = hexEncode(c / 16); - rsltAscii[out++] = hexEncode(c % 16); - in++; - break; - } - } - return new String(rsltAscii, 0, out, ASCII); - } - /* * RFC 3986 section 2.2 Reserved Characters (January 2005) * !*'();:@&=+$,/?#[] */ - private static boolean needsEscaping(String unicode) { + private static boolean[] ESCAPED_US_ASCII_CHARS = new boolean[128]; + + static { + ESCAPED_US_ASCII_CHARS[' '] = true; + // IMPORTANT NOTE: every time escape is invoked, all input needs to be escaped, + // i.e. escape("%01") should result in "%2501", not "%01". + // escape and unescape form a bijection, where neither + // of them is an idempotent operation. + ESCAPED_US_ASCII_CHARS['%'] = true; + // '#' and '/' are URL segment/fragment delimiters, need to be escaped in names. + ESCAPED_US_ASCII_CHARS['#'] = true; + ESCAPED_US_ASCII_CHARS['/'] = true; + // Escape '&' characters to avoid them being interpreted as SGML entities. + ESCAPED_US_ASCII_CHARS['&'] = true; + } + + private static int needsEscaping(String unicode) { int len = unicode.length(); + int escapeCount = 0; for (int i = 0; i < len; ++i) { - switch (unicode.charAt(i)) { - case (byte)'!': - case (byte)'*': - case (byte)'\'': - case (byte)'(': - case (byte)')': - case (byte)';': - case (byte)':': - case (byte)'@': - case (byte)'=': - case (byte)'+': - case (byte)'$': - case (byte)',': - case (byte)'?': - case (byte)'~': - case (byte)'[': - case (byte)']': - break; - case (byte)' ': - case (byte) '#': - case (byte) '%': - case (byte) '/': - case (byte)'&': - return true; + char ch = unicode.charAt(i); + if (ch < 128 && ESCAPED_US_ASCII_CHARS[ch]) + ++escapeCount; + } + return escapeCount; + } + + private static String encode(String unicode) { + int needsEscapes = needsEscaping(unicode); + if (needsEscapes == 0) + return unicode; + + int len = unicode.length(); + char result[] = new char[(len - needsEscapes) + needsEscapes * 3]; + int in = 0; + int out = 0; + while (in < len) { + char inCh = unicode.charAt(in++); + if (inCh >= 128 || !ESCAPED_US_ASCII_CHARS[inCh]) { + result[out++] = inCh; + } else { + // Only selected 7-bit US-ASCII characters are escaped + int c = inCh & 255; + result[out++] = '%'; + result[out++] = (char) hexEncode(c / 16); + result[out++] = (char) hexEncode(c % 16); } } - return false; + return new String(result, 0, out); } private static boolean needsUnescaping(String unicode) { @@ -427,13 +387,12 @@ public final class URIStringUtils { } /** - * Convert a URI, in US-ASCII, with escaped characters taken from UTF-8, to - * the corresponding Unicode string. On ill-formed input the results are - * undefined, specifically if the unescaped version is not a UTF-8 String, - * some String will be returned. + * Convert a URI, in UTF-16 with escaped characters taken from US-ASCII, to + * the corresponding unescaped Unicode string. On ill-formed input the results are + * undefined. * * @param uri the uri, in characters specified by RFC 2396 + '#'. - * @return the corresponding Unicode String. + * @return the corresponding unescaped Unicode String. * @exception IllegalArgumentException if a % hex sequence is ill-formed. */ public static String unescape(String uri) { @@ -441,26 +400,29 @@ public final class URIStringUtils { if (!needsUnescaping(uri)) return uri; - byte ascii[] = uri.getBytes("US-ASCII"); - byte utf8[] = new byte[ascii.length]; + int len = uri.length(); + String unicode = uri; + char result[] = new char[len]; int in = 0; int out = 0; - while ( in < ascii.length ) { - if (ascii[in] == (byte) '%') { - in++; - utf8[out++] = (byte) (hexDecode(ascii[in]) * 16 | hexDecode(ascii[in + 1])); + while (in < len) { + char inCh = unicode.charAt(in++); + if (inCh == '%') { + char d1 = unicode.charAt(in); + char d2 = unicode.charAt(in+1); + if (d1 > 127 || d2 > 127) + throw new IllegalArgumentException("Invalid hex digit escape sequence in " + uri + " at " + in); + result[out++] = (char) (hexDecode((byte) d1) * 16 | hexDecode((byte) d2)); in += 2; } else { - utf8[out++] = ascii[in++]; + result[out++] = inCh; } } - return new String(utf8, 0, out, "UTF-8"); + return new String(result, 0, out); } catch (IllegalArgumentException e) { throw new IllegalArgumentException("Problem while unescaping string: " + uri, e); - } catch (java.io.UnsupportedEncodingException e) { - throw new Error("The JVM is required to support UTF-8 and US-ASCII encodings."); - } catch (ArrayIndexOutOfBoundsException ee) { - throw new IllegalArgumentException("Incomplete Hex escape sequence in " + uri); + } catch (IndexOutOfBoundsException ee) { + throw new IllegalArgumentException("Incomplete hex digit escape sequence in " + uri); } } @@ -491,38 +453,36 @@ public final class URIStringUtils { * @param args */ public static void main(String[] args) { - String s; - s = "http://www.vtt.fi%2FSome- %25 Namespace/Jotain"; - System.out.println(String.format("escape+unescape: %s -> %s -> %s", s, escape(s), unescape(escape(s)))); - s = "http://www.vtt.fi%2FPSK"; - System.out.println(String.format("unescape: %s -> %s", s, unescape(s))); - s = "http://www.vtt.fi%2FSome-Namespace/Jotain / Muuta"; - System.out.println(String.format("escape: %s -> %s", s, escape(s))); - s = "Jotain / Muuta"; - System.out.println(String.format("escape: %s -> %s", s, escape(s))); - - System.out.println("escapeURI: " + escapeURI("foo/bar/org%2Fnet")); - System.out.println("escapeURI('...#...'): " + escapeURI("foo/bar#org%2Fnet")); - s = makeURI("http://foo.bar.com/foo/bar", "baz/guuk/org%2Fnet"); + String s = makeURI("http://foo.bar.com/foo/bar", "baz/guuk/org%2Fnet"); System.out.println("escapeURI: " + s); System.out.println("getNamespace: " + getNamespace(s)); System.out.println("getLocalName: " + getLocalName(s)); + System.out.println("escapeURI: " + escapeURI("foo/bar/org%2Fnet")); + System.out.println("escapeURI('...#...'): " + escapeURI("foo/bar#org%2Fnet")); + testEscape("/", "%2F"); testEscape("#", "%23"); testEscape("%", "%25"); testEscape("%01", "%2501"); testEscape("%GG", "%25GG"); + testEscape("säätö venttiili", "säätö%20venttiili"); + testEscape("säätö", "säätö"); + testEscape("Something / Else", "Something%20%2F%20Else"); + testEscape("http://www.vtt.fi%2FSome- %25 Namespace/Something", "http:%2F%2Fwww.vtt.fi%252FSome-%20%2525%20Namespace%2FSomething"); + testEscape("http://www.vtt.fi/PSK", "http:%2F%2Fwww.vtt.fi%2FPSK"); + testEscape("http://www.vtt.fi%2FSome-Namespace/Something / Else", "http:%2F%2Fwww.vtt.fi%252FSome-Namespace%2FSomething%20%2F%20Else"); } private static void testEscape(String unescaped, String expectedEscaped) { String esc = escape(unescaped); String unesc = unescape(esc); - System.out.format("escape('%s')='%s', unescape('%s')='%s'\n", unescaped, esc, esc, unesc); + System.out.format("escape('%s') -> '%s', unescape('%s') -> '%s'", unescaped, esc, esc, unesc); if (!esc.equals(expectedEscaped)) throw new AssertionError("escape('" + unescaped + "') was expected to return '" + expectedEscaped + "' but returned '" + esc + "'"); if (!unesc.equals(unescaped)) throw new AssertionError("unescape(escape('" + unescaped + "'))=unescape(" + esc + ") was expected to return '" + unescaped + "' but returned '" + unesc + "'"); + System.out.println(" OK"); } } diff --git a/bundles/org.simantics.db.common/META-INF/MANIFEST.MF b/bundles/org.simantics.db.common/META-INF/MANIFEST.MF index 89395e6ea..5ba6379d6 100644 --- a/bundles/org.simantics.db.common/META-INF/MANIFEST.MF +++ b/bundles/org.simantics.db.common/META-INF/MANIFEST.MF @@ -14,7 +14,8 @@ Require-Bundle: org.simantics.db;bundle-version="1.1.0";visibility:=reexport, org.simantics.scl.runtime;bundle-version="0.1.4", org.simantics.user.ontology;bundle-version="1.0.0", org.simantics.layer0x.ontology;bundle-version="1.0.0", - org.simantics.issues.ontology;bundle-version="1.2.0" + org.simantics.issues.ontology;bundle-version="1.2.0", + org.slf4j.api Export-Package: org.simantics.db.common, org.simantics.db.common.adaption, org.simantics.db.common.auth, diff --git a/bundles/org.simantics.db.common/src/org/simantics/db/common/request/PropertyMapOfResource.java b/bundles/org.simantics.db.common/src/org/simantics/db/common/request/PropertyMapOfResource.java index 5a92ef62c..1d06cc33d 100644 --- a/bundles/org.simantics.db.common/src/org/simantics/db/common/request/PropertyMapOfResource.java +++ b/bundles/org.simantics.db.common/src/org/simantics/db/common/request/PropertyMapOfResource.java @@ -22,9 +22,13 @@ import org.simantics.db.ReadGraph; import org.simantics.db.Resource; import org.simantics.db.exception.DatabaseException; import org.simantics.layer0.Layer0; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class PropertyMapOfResource extends ResourceRead> { + private static final Logger LOGGER = LoggerFactory.getLogger(PropertyMapOfResource.class); + public PropertyMapOfResource(Resource resource) { super(resource); } @@ -41,7 +45,7 @@ public class PropertyMapOfResource extends ResourceRead> { if(name != null) { String escapedName = URIStringUtils.escape(name); if (result.put(escapedName, predicate) != null) - System.err.println(this + ": The database contains siblings with the same name " + name + " (resource=$" + resource.getResourceId() +")."); + LOGGER.error("The database contains siblings with the same name " + name + " (resource=$" + resource.getResourceId() +")."); } } } diff --git a/bundles/org.simantics.db.common/src/org/simantics/db/common/uri/EscapedChildMapOfResource.java b/bundles/org.simantics.db.common/src/org/simantics/db/common/uri/EscapedChildMapOfResource.java index 99b20ff73..1d92da9c8 100644 --- a/bundles/org.simantics.db.common/src/org/simantics/db/common/uri/EscapedChildMapOfResource.java +++ b/bundles/org.simantics.db.common/src/org/simantics/db/common/uri/EscapedChildMapOfResource.java @@ -20,12 +20,17 @@ import org.simantics.db.Resource; import org.simantics.db.common.ProcedureBarrier; import org.simantics.db.common.WriteBindings; import org.simantics.db.common.procedure.adapter.AsyncMultiProcedureAdapter; +import org.simantics.db.common.request.PropertyMapOfResource; import org.simantics.db.common.request.ResourceAsyncRead; import org.simantics.db.procedure.AsyncProcedure; import org.simantics.layer0.Layer0; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class EscapedChildMapOfResource extends ResourceAsyncRead> { + private static final Logger LOGGER = LoggerFactory.getLogger(EscapedChildMapOfResource.class); + public EscapedChildMapOfResource(Resource resource) { super(resource); } @@ -50,7 +55,7 @@ public class EscapedChildMapOfResource extends ResourceAsyncRead> { + private static final Logger LOGGER = LoggerFactory.getLogger(UnescapedChildMapOfResource.class); + public UnescapedChildMapOfResource(Resource resource) { super(resource); } @@ -40,10 +44,10 @@ public class UnescapedChildMapOfResource extends ResourceRead { - Object getValue(Node node); - void setValue(Node node, Object value); + Object getValue(Node node) throws NodeManagerException; + void setValue(Node node, Object value) throws NodeManagerException; String getName(Node node); Map getChildren(Node node); Map getProperties(Node node); diff --git a/bundles/org.simantics.db.procore.ui/src/org/simantics/db/procore/ui/ProCoreUserAgent.java b/bundles/org.simantics.db.procore.ui/src/org/simantics/db/procore/ui/ProCoreUserAgent.java index 54b892070..9b0438042 100644 --- a/bundles/org.simantics.db.procore.ui/src/org/simantics/db/procore/ui/ProCoreUserAgent.java +++ b/bundles/org.simantics.db.procore.ui/src/org/simantics/db/procore/ui/ProCoreUserAgent.java @@ -8,38 +8,38 @@ import org.simantics.db.procore.ProCoreDriver; public final class ProCoreUserAgent implements DatabaseUserAgent { private static Shell getShell() { - Shell shell = null; - Display d = getDisplay(); - if (d == null) - return null; - shell = d.getActiveShell(); - if (null == shell) { - Shell[] shells = d.getShells(); - if (null != shells && shells.length > 0) - shell = shells[0]; - } - return shell; + Shell shell = null; + Display d = getDisplay(); + if (d == null) + return null; + shell = d.getActiveShell(); + if (null == shell) { + Shell[] shells = d.getShells(); + if (null != shells && shells.length > 0) + shell = shells[0]; + } + return shell; } private static Display getDisplay() { - Display d = Display.getCurrent(); - if (d == null) - d = Display.getDefault(); - return d; + Display d = Display.getCurrent(); + if (d == null) + d = Display.getDefault(); + return d; } - @Override - public boolean handleStart(InternalException exception) { - Shell shell = getShell(); - if (null == shell) - return false; // no can do - try { - return Auxiliary.handleStart(shell, exception); - } catch (InternalException e) { - return false; // no could do - } - } - - @Override - public String getId() { - return ProCoreDriver.ProCoreDriverName; - } -} \ No newline at end of file + @Override + public boolean handleStart(InternalException exception) { + Shell shell = getShell(); + if (null == shell) + return false; // no can do + try { + return Auxiliary.handleStart(shell, exception); + } catch (InternalException e) { + return false; // no could do + } + } + + @Override + public String getId() { + return ProCoreDriver.ProCoreDriverName; + } +} diff --git a/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/ClusterSetsSupportImpl.java b/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/ClusterSetsSupportImpl.java index 0f03141ea..c4ed5739c 100644 --- a/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/ClusterSetsSupportImpl.java +++ b/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/ClusterSetsSupportImpl.java @@ -70,14 +70,14 @@ public class ClusterSetsSupportImpl implements ClusterSetsSupport, Disposable { clusterSets.clear(); } - @Override - public void setReadDirectory(Path read) { - // TODO Auto-generated method stub - - } @Override public void updateWriteDirectory(Path write) { // Nothing to do here } + @Override + public void setReadDirectory(Path read) { + + } + } diff --git a/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/ClusterSetsSupportImpl2.java b/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/ClusterSetsSupportImpl2.java index db28af306..5da4e3761 100644 --- a/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/ClusterSetsSupportImpl2.java +++ b/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/ClusterSetsSupportImpl2.java @@ -77,7 +77,7 @@ public class ClusterSetsSupportImpl2 implements ClusterSetsSupport, Disposable { clusterSets.touch(); } } - + @Override public void setReadDirectory(Path read) { this.readDirectory = read; diff --git a/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/GraphSessionVirtual.java b/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/GraphSessionVirtual.java index 1dc1b015a..f6eba7eb8 100644 --- a/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/GraphSessionVirtual.java +++ b/bundles/org.simantics.db.procore/src/fi/vtt/simantics/procore/internal/GraphSessionVirtual.java @@ -214,4 +214,4 @@ class GraphSessionVirtual extends GraphSession { // Äsh! This extends relation id = virtualGraphServerSupport.createVirtual(); return id; } -} \ No newline at end of file +} diff --git a/bundles/org.simantics.db/src/org/simantics/db/service/ClusterSetsSupport.java b/bundles/org.simantics.db/src/org/simantics/db/service/ClusterSetsSupport.java index e79433870..d935a2b54 100644 --- a/bundles/org.simantics.db/src/org/simantics/db/service/ClusterSetsSupport.java +++ b/bundles/org.simantics.db/src/org/simantics/db/service/ClusterSetsSupport.java @@ -12,7 +12,7 @@ public interface ClusterSetsSupport { void put(long resourceId, long clusterId); void save() throws IOException; void clear(); - void updateWriteDirectory(Path write); void setReadDirectory(Path read); + void updateWriteDirectory(Path write); } diff --git a/bundles/org.simantics.document.server/META-INF/MANIFEST.MF b/bundles/org.simantics.document.server/META-INF/MANIFEST.MF index a345498be..184361779 100644 --- a/bundles/org.simantics.document.server/META-INF/MANIFEST.MF +++ b/bundles/org.simantics.document.server/META-INF/MANIFEST.MF @@ -22,7 +22,8 @@ Require-Bundle: org.simantics.document.base.ontology;bundle-version="1.1.0", org.simantics.modeling;bundle-version="1.1.1", org.simantics.document.server.io;visibility:=reexport, - org.simantics.scl.db;bundle-version="0.1.3" + org.simantics.scl.db;bundle-version="0.1.3", + org.slf4j.api Bundle-ActivationPolicy: lazy Bundle-Activator: org.simantics.document.server.Activator Export-Package: org.simantics.document.server, diff --git a/bundles/org.simantics.document.server/src/org/simantics/document/server/DocumentHistoryListener.java b/bundles/org.simantics.document.server/src/org/simantics/document/server/DocumentHistoryListener.java index ca4adef25..ef764bddc 100644 --- a/bundles/org.simantics.document.server/src/org/simantics/document/server/DocumentHistoryListener.java +++ b/bundles/org.simantics.document.server/src/org/simantics/document/server/DocumentHistoryListener.java @@ -2,10 +2,12 @@ package org.simantics.document.server; import java.util.List; -import org.simantics.Logger; import org.simantics.db.procedure.Listener; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class DocumentHistoryListener implements Listener> { + private static final Logger LOGGER = LoggerFactory.getLogger(DocumentHistoryListener.class); final private DocumentHistory history; @@ -32,7 +34,7 @@ public class DocumentHistoryListener implements Listener> { @Override public void exception(Throwable t) { - Logger.defaultLogError(t); + LOGGER.warn("DocumentHistoryListener received an exception.", t); } @Override diff --git a/bundles/org.simantics.document.server/src/org/simantics/document/server/request/NodeRequestUtils.java b/bundles/org.simantics.document.server/src/org/simantics/document/server/request/NodeRequestUtils.java index 26f88608f..2e67010eb 100644 --- a/bundles/org.simantics.document.server/src/org/simantics/document/server/request/NodeRequestUtils.java +++ b/bundles/org.simantics.document.server/src/org/simantics/document/server/request/NodeRequestUtils.java @@ -4,15 +4,17 @@ import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.io.UnsupportedEncodingException; -import org.simantics.Logger; import org.simantics.db.layer0.exception.MissingVariableException; import org.simantics.db.layer0.scl.SCLDatabaseException; import org.simantics.document.server.DocumentException; import org.simantics.scl.compiler.module.repository.ImportFailure; import org.simantics.scl.compiler.module.repository.ImportFailureException; import org.simantics.scl.compiler.top.NotFoundException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class NodeRequestUtils { + private static final Logger LOGGER = LoggerFactory.getLogger(NodeRequestUtils.class); public static String formatErrorMessage(String name, Throwable t) { @@ -36,8 +38,8 @@ public class NodeRequestUtils { sb.append(" " + f.moduleName + "\n"); return sb.toString(); } else { - Logger.defaultLogError(t); - + LOGGER.error("Node request error:", t); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintStream ps = new PrintStream(baos); t.printStackTrace(ps); diff --git a/bundles/org.simantics.export.core/META-INF/MANIFEST.MF b/bundles/org.simantics.export.core/META-INF/MANIFEST.MF index eadcf7e08..88dfc79c7 100644 --- a/bundles/org.simantics.export.core/META-INF/MANIFEST.MF +++ b/bundles/org.simantics.export.core/META-INF/MANIFEST.MF @@ -15,7 +15,8 @@ Require-Bundle: org.eclipse.core.runtime, org.simantics.utils.ui;bundle-version="1.1.0", org.simantics.graph.db;bundle-version="1.1.9", org.simantics;bundle-version="1.0.0", - com.lowagie.text;bundle-version="2.1.7";resolution:=optional + com.lowagie.text;bundle-version="2.1.7";resolution:=optional, + org.slf4j.api Bundle-RequiredExecutionEnvironment: JavaSE-1.8 Export-Package: org.simantics.export.core, org.simantics.export.core.error, diff --git a/bundles/org.simantics.export.core/src/org/simantics/export/core/pdf/ServiceBasedPdfExportPageEvent.java b/bundles/org.simantics.export.core/src/org/simantics/export/core/pdf/ServiceBasedPdfExportPageEvent.java index c513654df..65cb53d13 100644 --- a/bundles/org.simantics.export.core/src/org/simantics/export/core/pdf/ServiceBasedPdfExportPageEvent.java +++ b/bundles/org.simantics.export.core/src/org/simantics/export/core/pdf/ServiceBasedPdfExportPageEvent.java @@ -20,8 +20,9 @@ import java.util.function.Consumer; import org.osgi.framework.BundleContext; import org.osgi.framework.InvalidSyntaxException; import org.osgi.framework.ServiceReference; -import org.simantics.Logger; import org.simantics.export.core.internal.Activator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.lowagie.text.Document; import com.lowagie.text.Paragraph; @@ -34,6 +35,7 @@ import com.lowagie.text.pdf.PdfWriter; * @since 1.22.2 */ public class ServiceBasedPdfExportPageEvent extends PdfPageEventHelper { + private static final Logger LOGGER = LoggerFactory.getLogger(ServiceBasedPdfExportPageEvent.class); Map events; @@ -64,7 +66,7 @@ public class ServiceBasedPdfExportPageEvent extends PdfPageEventHelper { try { r.accept(event); } catch (Exception e) { - Logger.defaultLogError("Failed to invoke PdfExportPageEvent::" + eventName + " for " + event.toString(), e); + LOGGER.error("Failed to invoke PdfExportPageEvent::" + eventName + " for " + event.toString(), e); } } diff --git a/bundles/org.simantics.fastlz/LICENSE b/bundles/org.simantics.fastlz/LICENSE index 299e8e19c..c7d105725 100644 --- a/bundles/org.simantics.fastlz/LICENSE +++ b/bundles/org.simantics.fastlz/LICENSE @@ -24,38 +24,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - - -LZ4 native/lz4{,hc}.{c,h} ---------------------------------------------------------------------- - LZ4 - Fast LZ compression algorithm - Copyright (C) 2011-2012, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html - - LZ4 source repository : http://code.google.com/p/lz4/ +THE SOFTWARE. \ No newline at end of file diff --git a/bundles/org.simantics.fastlz/README.txt b/bundles/org.simantics.fastlz/README.txt index 134b4e111..8f9e06cd3 100644 --- a/bundles/org.simantics.fastlz/README.txt +++ b/bundles/org.simantics.fastlz/README.txt @@ -1,12 +1,12 @@ The native/ -directory contains the sources for the -native parts of the FastLZ and LZ4 compressions algorithms. +native parts of the FastLZ compression algorithm. To compile in the native libraries follow these instructions: == Windows == -* Install MS Visual Studio 2010 with C++ support and Microsoft SDK 7.1 to get 64-bit compiler -* Open native/vs2010/fastlz.sln in visual studio +* Install MS Visual Studio 2012 with C++ support and Microsoft Platform SDK for Windows Server 2003 R2 to get 64-bit compiler +* Open native/vs2012/fastlz.sln in Visual Studio 2012 * Select ''Batch Build'' from the solution context menu, select Win32 + x64 Release and press ''Rebuild'' * The build will copy the resulting fastlz-windows-{x86,x86_64}.dll into src/ diff --git a/bundles/org.simantics.fastlz/native/Makefile b/bundles/org.simantics.fastlz/native/Makefile deleted file mode 100644 index 8a2696aef..000000000 --- a/bundles/org.simantics.fastlz/native/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -OS := $(shell uname) - -ifeq ($(OS),Linux) - OUTPUT32 = lz4demo32 - OUTPUT64 = lz4demo64 -else - OUTPUT32 = LZ4Demo32.exe - OUTPUT64 = LZ4Demo64.exe -endif - -all: lz4demo64 lz4demo32 - -lz4demo64: lz4.c lz4.h lz4hc.c lz4hc.h bench.c lz4demo.c - gcc -O3 -I. -std=c99 -Wall -W -Wundef -Wno-implicit-function-declaration lz4hc.c lz4.c bench.c lz4demo.c -o $(OUTPUT64) - -lz4demo32: lz4.c lz4.h lz4hc.c lz4hc.h bench.c lz4demo.c - gcc -m32 -Os -march=native -I. -std=c99 -Wall -W -Wundef -Wno-implicit-function-declaration lz4hc.c lz4.c bench.c lz4demo.c -o $(OUTPUT32) - -clean: - rm -f core *.o $(OUTPUT32) $(OUTPUT64) diff --git a/bundles/org.simantics.fastlz/native/compile-x64.bat b/bundles/org.simantics.fastlz/native/compile-x64.bat index c8628b88a..2eb226eb0 100644 --- a/bundles/org.simantics.fastlz/native/compile-x64.bat +++ b/bundles/org.simantics.fastlz/native/compile-x64.bat @@ -11,6 +11,6 @@ @rem *************************************************************************** @echo off -cl /O2 /Oi /GL /I "%JAVA_HOME%/include/win32" /I "%JAVA_HOME%/include" /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /D "_USRDLL" /D "VC64_EXPORTS" /D "_UNICODE" /D "UNICODE" /D "_WINDLL" /EHsc /LD /Gy /GS- /W3 /nologo /c /Zi /TC /errorReport:prompt fastlz.c lz4.c lz4hc.c jniWrapper.c +cl /O2 /Oi /GL /I "%JAVA_HOME%/include/win32" /I "%JAVA_HOME%/include" /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /D "_USRDLL" /D "VC64_EXPORTS" /D "_UNICODE" /D "UNICODE" /D "_WINDLL" /EHsc /LD /Gy /GS- /W3 /nologo /c /Zi /TC /errorReport:prompt fastlz.c jniWrapper.c -link /OUT:"..\src\fastlz-windows-x86_64.dll" /INCREMENTAL:NO /NOLOGO /DLL /MANIFEST /MANIFESTFILE:"..\fastlz-windows-x86_64.dll.intermediate.manifest" /MANIFESTUAC:"level='asInvoker' uiAccess='false'" /PDB:"fastlz-windows-x86_64.pdb" /SUBSYSTEM:WINDOWS /OPT:REF /OPT:ICF /LTCG /DYNAMICBASE /NXCOMPAT /MACHINE:X64 /ERRORREPORT:PROMPT kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib fastlz.obj lz4.obj lz4hc.obj jniWrapper.obj +link /OUT:"..\src\fastlz-windows-x86_64.dll" /INCREMENTAL:NO /NOLOGO /DLL /MANIFEST /MANIFESTFILE:"..\fastlz-windows-x86_64.dll.intermediate.manifest" /MANIFESTUAC:"level='asInvoker' uiAccess='false'" /PDB:"fastlz-windows-x86_64.pdb" /SUBSYSTEM:WINDOWS /OPT:REF /OPT:ICF /LTCG /DYNAMICBASE /NXCOMPAT /MACHINE:X64 /ERRORREPORT:PROMPT kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib fastlz.obj jniWrapper.obj diff --git a/bundles/org.simantics.fastlz/native/compile-x86.bat b/bundles/org.simantics.fastlz/native/compile-x86.bat index 0e9570786..0141073b7 100644 --- a/bundles/org.simantics.fastlz/native/compile-x86.bat +++ b/bundles/org.simantics.fastlz/native/compile-x86.bat @@ -11,6 +11,6 @@ @rem *************************************************************************** @echo off -cl /O2 /Oi /GL /I "%JAVA_HOME%/include/win32" /I "%JAVA_HOME%/include" /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /D "_USRDLL" /D "_UNICODE" /D "UNICODE" /D "_WINDLL" /EHsc /LD /Gy /GS- /W3 /nologo /c /Zi /TC /errorReport:prompt fastlz.c lz4.c lz4hc.c jniWrapper.c +cl /O2 /Oi /GL /I "%JAVA_HOME%/include/win32" /I "%JAVA_HOME%/include" /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /D "_USRDLL" /D "_UNICODE" /D "UNICODE" /D "_WINDLL" /EHsc /LD /Gy /GS- /W3 /nologo /c /Zi /TC /errorReport:prompt fastlz.c jniWrapper.c -link /OUT:"..\src\fastlz-windows-x86.dll" /INCREMENTAL:NO /NOLOGO /DLL /MANIFEST /MANIFESTFILE:"..\fastlz-windows-x86.dll.intermediate.manifest" /MANIFESTUAC:"level='asInvoker' uiAccess='false'" /PDB:"fastlz-windows-x86.pdb" /SUBSYSTEM:WINDOWS /OPT:REF /OPT:ICF /LTCG /DYNAMICBASE /NXCOMPAT /MACHINE:X86 /ERRORREPORT:PROMPT kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib fastlz.obj lz4.obj lz4hc.obj jniWrapper.obj +link /OUT:"..\src\fastlz-windows-x86.dll" /INCREMENTAL:NO /NOLOGO /DLL /MANIFEST /MANIFESTFILE:"..\fastlz-windows-x86.dll.intermediate.manifest" /MANIFESTUAC:"level='asInvoker' uiAccess='false'" /PDB:"fastlz-windows-x86.pdb" /SUBSYSTEM:WINDOWS /OPT:REF /OPT:ICF /LTCG /DYNAMICBASE /NXCOMPAT /MACHINE:X86 /ERRORREPORT:PROMPT kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib fastlz.obj jniWrapper.obj diff --git a/bundles/org.simantics.fastlz/native/compile.bat b/bundles/org.simantics.fastlz/native/compile.bat index 22968b87c..7904b5198 100644 --- a/bundles/org.simantics.fastlz/native/compile.bat +++ b/bundles/org.simantics.fastlz/native/compile.bat @@ -9,6 +9,6 @@ @rem Contributors: @rem VTT Technical Research Centre of Finland - initial API and implementation @rem *************************************************************************** -rem gcc -mno-cygwin "-I$jdk/include" "-I$jdk$jdk/include/win32" -Wl,--add-stdcall-alias -shared -o ../src/fastlz-windows-x86.dll fastlz.c lz4.c lz4hc.c jniWrapper.c -gcc -mno-cygwin "-I%JAVA_HOME%/include" "-I%JAVA_HOME%/include/win32" -Wl,--add-stdcall-alias -shared -o ../src/fastlz-windows-x86.dll fastlz.c lz4.c lz4hc.c jniWrapper.c +rem gcc -mno-cygwin "-I$jdk/include" "-I$jdk$jdk/include/win32" -Wl,--add-stdcall-alias -shared -o ../src/fastlz-windows-x86.dll fastlz.c jniWrapper.c +gcc -mno-cygwin "-I%JAVA_HOME%/include" "-I%JAVA_HOME%/include/win32" -Wl,--add-stdcall-alias -shared -o ../src/fastlz-windows-x86.dll fastlz.c jniWrapper.c gcc -o fastlz_test.exe fastlz.c fastlz_read.c fastlz_write.c fastlz_test.c diff --git a/bundles/org.simantics.fastlz/native/compile.sh b/bundles/org.simantics.fastlz/native/compile.sh index 05aac11f9..21899d5dd 100755 --- a/bundles/org.simantics.fastlz/native/compile.sh +++ b/bundles/org.simantics.fastlz/native/compile.sh @@ -54,7 +54,7 @@ output="../src/libfastlz-${kernel}-${arch}" case $kernel in darwin*) - output="${output}.jnilib" + output="${output}.dylib" ;; *) output="${output}.so" @@ -65,7 +65,7 @@ echo "Architecture: $arch" echo "Output library: $output" echo "Compiler options: $options" -gcc ${options} -o ${output} fastlz.c lz4.c lz4hc.c jniWrapper.c +gcc ${options} -o ${output} fastlz.c jniWrapper.c size=`ls -l $output | cut -d " " -f 5` echo "library size before stripping: $size" diff --git a/bundles/org.simantics.fastlz/native/jniWrapper.c b/bundles/org.simantics.fastlz/native/jniWrapper.c index 0c696e1f5..6f497627a 100644 --- a/bundles/org.simantics.fastlz/native/jniWrapper.c +++ b/bundles/org.simantics.fastlz/native/jniWrapper.c @@ -87,71 +87,3 @@ JNIEXPORT jint JNICALL Java_org_simantics_fastlz_FastLZ_decompressCluster(JNIEnv } -JNIEXPORT jint JNICALL Java_org_simantics_fastlz_LZ4_compress(JNIEnv* env, jclass clazz, - jobject input, jint inputOffset, jint length, - jobject output, jint outputOffset) { - void* inputAddress = (char*)(*env)->GetDirectBufferAddress(env, input) + inputOffset; - void* outputAddress = (char*)(*env)->GetDirectBufferAddress(env, output) + outputOffset; - return LZ4_compress(inputAddress, outputAddress, length); -} - -JNIEXPORT jint JNICALL Java_org_simantics_fastlz_LZ4_decompress(JNIEnv* env, jclass clazz, - jobject input, jint inputOffset, jint length, - jobject output, jint outputOffset, jint maxout) { - void* inputAddress = (char*)(*env)->GetDirectBufferAddress(env, input) + inputOffset; - void* outputAddress = (char*)(*env)->GetDirectBufferAddress(env, output) + outputOffset; - return LZ4_uncompress_unknownOutputSize(inputAddress, outputAddress, length, maxout); -} - -JNIEXPORT jint JNICALL Java_org_simantics_fastlz_LZ4_decompressCluster(JNIEnv* env, jclass clazz, jobject deflated, jint deflatedSize, jint inflatedSize, jobjectArray arrays) { - - static char *inflateBuffer = 0; - static int inflateBufferSize = 0; - - int ll, il, bl; - - jlongArray longs; - jintArray ints; - jbyteArray bytes; - - char *input = (char*)(*env)->GetDirectBufferAddress(env, deflated); - char *address; - - if(inflateBufferSize < inflatedSize) { - if(!inflateBuffer) { - if(inflatedSize < INITIAL_SIZE) inflatedSize = INITIAL_SIZE; - inflateBuffer = malloc(inflatedSize); - inflateBufferSize = inflatedSize; - } else { - if(inflateBuffer) free(inflateBuffer); - inflateBuffer = malloc(inflatedSize); - inflateBufferSize = inflatedSize; - } - } - - address = inflateBuffer; - - LZ4_uncompress_unknownOutputSize(input, inflateBuffer, deflatedSize, inflateBufferSize); - - ll = *(int *)address; - longs = (*env)->NewLongArray(env, ll); - (*env)->SetLongArrayRegion(env, longs, 0, ll, (const jlong *)(address + 4)); - (*env)->SetObjectArrayElement(env, arrays, 0, longs); - - address += 4 + 8 * ll; - - il = *(int *)address; - ints = (*env)->NewIntArray(env, il); - (*env)->SetIntArrayRegion(env, ints, 0, il, (const jint *)(address + 4)); - (*env)->SetObjectArrayElement(env, arrays, 1, ints); - - address += 4 * il + 4; - - bl = *(int *)address; - bytes = (*env)->NewByteArray(env, bl); - (*env)->SetByteArrayRegion(env, bytes, 0, bl, (const jbyte *)(address + 4)); - (*env)->SetObjectArrayElement(env, arrays, 2, bytes); - - return 0; - -} diff --git a/bundles/org.simantics.fastlz/native/lz4.c b/bundles/org.simantics.fastlz/native/lz4.c deleted file mode 100644 index 06e282970..000000000 --- a/bundles/org.simantics.fastlz/native/lz4.c +++ /dev/null @@ -1,819 +0,0 @@ -/* - LZ4 - Fast LZ compression algorithm - Copyright (C) 2011-2012, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html - - LZ4 source repository : http://code.google.com/p/lz4/ -*/ - -//************************************** -// Tuning parameters -//************************************** -// COMPRESSIONLEVEL : -// Increasing this value improves compression ratio -// Lowering this value reduces memory usage -// Reduced memory usage typically improves speed, due to cache effect (ex : L1 32KB for Intel, L1 64KB for AMD) -// Memory usage formula : N->2^(N+2) Bytes (examples : 12 -> 16KB ; 17 -> 512KB) -#define COMPRESSIONLEVEL 12 - -// NOTCOMPRESSIBLE_CONFIRMATION : -// Decreasing this value will make the algorithm skip faster data segments considered "incompressible" -// This may decrease compression ratio dramatically, but will be faster on incompressible data -// Increasing this value will make the algorithm search more before declaring a segment "incompressible" -// This could improve compression a bit, but will be slower on incompressible data -// The default value (6) is recommended -#define NOTCOMPRESSIBLE_CONFIRMATION 6 - -// LZ4_COMPRESSMIN : -// Compression function will *fail* if it is not successful at compressing input by at least LZ4_COMPRESSMIN bytes -// Since the compression function stops working prematurely, it results in a speed gain -// The output however is unusable. Compression function result will be zero. -// Default : 0 = disabled -#define LZ4_COMPRESSMIN 0 - -// BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE : -// This will provide a boost to performance for big endian cpu, but the resulting compressed stream will be incompatible with little-endian CPU. -// You can set this option to 1 in situations where data will stay within closed environment -// This option is useless on Little_Endian CPU (such as x86) -//#define BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE 1 - - - -//************************************** -// CPU Feature Detection -//************************************** -// 32 or 64 bits ? -#if (defined(__x86_64__) || defined(__x86_64) || defined(__amd64__) || defined(__amd64) || defined(__ppc64__) || defined(_WIN64) || defined(__LP64__) || defined(_LP64) ) // Detects 64 bits mode -# define LZ4_ARCH64 1 -#else -# define LZ4_ARCH64 0 -#endif - -// Little Endian or Big Endian ? -// Note : overwrite the below #define if you know your architecture endianess -#if (defined(__BIG_ENDIAN__) || defined(__BIG_ENDIAN) || defined(_BIG_ENDIAN) || defined(_ARCH_PPC) || defined(__PPC__) || defined(__PPC) || defined(PPC) || defined(__powerpc__) || defined(__powerpc) || defined(powerpc) || ((defined(__BYTE_ORDER__)&&(__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))) ) -# define LZ4_BIG_ENDIAN 1 -#else -// Little Endian assumed. PDP Endian and other very rare endian format are unsupported. -#endif - -// Unaligned memory access is automatically enabled for "common" CPU, such as x86. -// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected -// If you know your target CPU supports unaligned memory access, you may want to force this option manually to improve performance -#if defined(__ARM_FEATURE_UNALIGNED) -# define LZ4_FORCE_UNALIGNED_ACCESS 1 -#endif - -// Define this parameter if your target system or compiler does not support hardware bit count -#if defined(_MSC_VER) && defined(_WIN32_WCE) // Visual Studio for Windows CE does not support Hardware bit count -# define LZ4_FORCE_SW_BITCOUNT -#endif - - -//************************************** -// Compiler Options -//************************************** -#if __STDC_VERSION__ >= 199901L // C99 -/* "restrict" is a known keyword */ -#else -# define restrict // Disable restrict -#endif - -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) - -#ifdef _MSC_VER // Visual Studio -# define inline __forceinline // Visual is not C99, but supports some kind of inline -# if LZ4_ARCH64 // 64-bit -# pragma intrinsic(_BitScanForward64) // For Visual 2005 -# pragma intrinsic(_BitScanReverse64) // For Visual 2005 -# else -# pragma intrinsic(_BitScanForward) // For Visual 2005 -# pragma intrinsic(_BitScanReverse) // For Visual 2005 -# endif -#endif - -#ifdef _MSC_VER -# define lz4_bswap16(x) _byteswap_ushort(x) -#else -# define lz4_bswap16(x) ((unsigned short int) ((((x) >> 8) & 0xffu) | (((x) & 0xffu) << 8))) -#endif - -#if (GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__) -# define expect(expr,value) (__builtin_expect ((expr),(value)) ) -#else -# define expect(expr,value) (expr) -#endif - -#define likely(expr) expect((expr) != 0, 1) -#define unlikely(expr) expect((expr) != 0, 0) - - -//************************************** -// Includes -//************************************** -#include // for malloc -#include // for memset -#include "lz4.h" - - -//************************************** -// Basic Types -//************************************** -#if defined(_MSC_VER) // Visual Studio does not support 'stdint' natively -# define BYTE unsigned __int8 -# define U16 unsigned __int16 -# define U32 unsigned __int32 -# define S32 __int32 -# define U64 unsigned __int64 -#else -# include -# define BYTE uint8_t -# define U16 uint16_t -# define U32 uint32_t -# define S32 int32_t -# define U64 uint64_t -#endif - -#ifndef LZ4_FORCE_UNALIGNED_ACCESS -# pragma pack(push, 1) -#endif - -typedef struct _U16_S { U16 v; } U16_S; -typedef struct _U32_S { U32 v; } U32_S; -typedef struct _U64_S { U64 v; } U64_S; - -#ifndef LZ4_FORCE_UNALIGNED_ACCESS -# pragma pack(pop) -#endif - -#define A64(x) (((U64_S *)(x))->v) -#define A32(x) (((U32_S *)(x))->v) -#define A16(x) (((U16_S *)(x))->v) - - -//************************************** -// Constants -//************************************** -#define MINMATCH 4 - -#define HASH_LOG COMPRESSIONLEVEL -#define HASHTABLESIZE (1 << HASH_LOG) -#define HASH_MASK (HASHTABLESIZE - 1) - -#define SKIPSTRENGTH (NOTCOMPRESSIBLE_CONFIRMATION>2?NOTCOMPRESSIBLE_CONFIRMATION:2) -#define STACKLIMIT 13 -#define HEAPMODE (HASH_LOG>STACKLIMIT) // Defines if memory is allocated into the stack (local variable), or into the heap (malloc()). -#define COPYLENGTH 8 -#define LASTLITERALS 5 -#define MFLIMIT (COPYLENGTH+MINMATCH) -#define MINLENGTH (MFLIMIT+1) - -#define MAXD_LOG 16 -#define MAX_DISTANCE ((1 << MAXD_LOG) - 1) - -#define ML_BITS 4 -#define ML_MASK ((1U<> ((MINMATCH*8)-HASH_LOG)) -#define LZ4_HASH_VALUE(p) LZ4_HASH_FUNCTION(A32(p)) -#define LZ4_WILDCOPY(s,d,e) do { LZ4_COPYPACKET(s,d) } while (d>3); - #elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clzll(val) >> 3); - #else - int r; - if (!(val>>32)) { r=4; } else { r=0; val>>=32; } - if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } - r += (!val); - return r; - #endif -#else - #if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanForward64( &r, val ); - return (int)(r>>3); - #elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctzll(val) >> 3); - #else - static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; - return DeBruijnBytePos[((U64)((val & -val) * 0x0218A392CDABBD3F)) >> 58]; - #endif -#endif -} - -#else - -inline static int LZ4_NbCommonBytes (register U32 val) -{ -#if defined(LZ4_BIG_ENDIAN) - #if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse( &r, val ); - return (int)(r>>3); - #elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clz(val) >> 3); - #else - int r; - if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } - r += (!val); - return r; - #endif -#else - #if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanForward( &r, val ); - return (int)(r>>3); - #elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctz(val) >> 3); - #else - static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; - return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; - #endif -#endif -} - -#endif - - -//**************************** -// Public functions -//**************************** - -int LZ4_compressBound(int isize) -{ - return (isize + (isize/255) + 16); -} - - - -//****************************** -// Compression functions -//****************************** - -int LZ4_compressCtx(void** ctx, - const char* source, - char* dest, - int isize) -{ -#if HEAPMODE - struct refTables *srt = (struct refTables *) (*ctx); - HTYPE* HashTable; -#else - HTYPE HashTable[HASHTABLESIZE] = {0}; -#endif - - const BYTE* ip = (BYTE*) source; - INITBASE(base); - const BYTE* anchor = ip; - const BYTE* const iend = ip + isize; - const BYTE* const mflimit = iend - MFLIMIT; -#define matchlimit (iend - LASTLITERALS) - - BYTE* op = (BYTE*) dest; - - int len, length; - const int skipStrength = SKIPSTRENGTH; - U32 forwardH; - - - // Init - if (isizehashTable); - memset((void*)HashTable, 0, sizeof(srt->hashTable)); -#else - (void) ctx; -#endif - - - // First Byte - HashTable[LZ4_HASH_VALUE(ip)] = ip - base; - ip++; forwardH = LZ4_HASH_VALUE(ip); - - // Main Loop - for ( ; ; ) - { - int findMatchAttempts = (1U << skipStrength) + 3; - const BYTE* forwardIp = ip; - const BYTE* ref; - BYTE* token; - - // Find a match - do { - U32 h = forwardH; - int step = findMatchAttempts++ >> skipStrength; - ip = forwardIp; - forwardIp = ip + step; - - if unlikely(forwardIp > mflimit) { goto _last_literals; } - - forwardH = LZ4_HASH_VALUE(forwardIp); - ref = base + HashTable[h]; - HashTable[h] = ip - base; - - } while ((ref < ip - MAX_DISTANCE) || (A32(ref) != A32(ip))); - - // Catch up - while ((ip>anchor) && (ref>(BYTE*)source) && unlikely(ip[-1]==ref[-1])) { ip--; ref--; } - - // Encode Literal length - length = ip - anchor; - token = op++; - if (length>=(int)RUN_MASK) { *token=(RUN_MASK< 254 ; len-=255) *op++ = 255; *op++ = (BYTE)len; } - else *token = (length<=(int)ML_MASK) { *token+=ML_MASK; len-=ML_MASK; for(; len > 509 ; len-=510) { *op++ = 255; *op++ = 255; } if (len > 254) { len-=255; *op++ = 255; } *op++ = (BYTE)len; } - else *token += len; - - // Test end of chunk - if (ip > mflimit) { anchor = ip; break; } - - // Fill table - HashTable[LZ4_HASH_VALUE(ip-2)] = ip - 2 - base; - - // Test next position - ref = base + HashTable[LZ4_HASH_VALUE(ip)]; - HashTable[LZ4_HASH_VALUE(ip)] = ip - base; - if ((ref > ip - (MAX_DISTANCE + 1)) && (A32(ref) == A32(ip))) { token = op++; *token=0; goto _next_match; } - - // Prepare next loop - anchor = ip++; - forwardH = LZ4_HASH_VALUE(ip); - } - -_last_literals: - // Encode Last Literals - { - int lastRun = iend - anchor; - if ((LZ4_COMPRESSMIN>0) && (((op - (BYTE*)dest) + lastRun + 1 + ((lastRun-15)/255)) > isize - LZ4_COMPRESSMIN)) return 0; - if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } - else *op++ = (lastRun<> ((MINMATCH*8)-HASHLOG64K)) -#define LZ4_HASH64K_VALUE(p) LZ4_HASH64K_FUNCTION(A32(p)) -int LZ4_compress64kCtx(void** ctx, - const char* source, - char* dest, - int isize) -{ -#if HEAPMODE - struct refTables *srt = (struct refTables *) (*ctx); - U16* HashTable; -#else - U16 HashTable[HASH64KTABLESIZE] = {0}; -#endif - - const BYTE* ip = (BYTE*) source; - const BYTE* anchor = ip; - const BYTE* const base = ip; - const BYTE* const iend = ip + isize; - const BYTE* const mflimit = iend - MFLIMIT; -#define matchlimit (iend - LASTLITERALS) - - BYTE* op = (BYTE*) dest; - - int len, length; - const int skipStrength = SKIPSTRENGTH; - U32 forwardH; - - - // Init - if (isizehashTable); - memset((void*)HashTable, 0, sizeof(srt->hashTable)); -#else - (void) ctx; -#endif - - - // First Byte - ip++; forwardH = LZ4_HASH64K_VALUE(ip); - - // Main Loop - for ( ; ; ) - { - int findMatchAttempts = (1U << skipStrength) + 3; - const BYTE* forwardIp = ip; - const BYTE* ref; - BYTE* token; - - // Find a match - do { - U32 h = forwardH; - int step = findMatchAttempts++ >> skipStrength; - ip = forwardIp; - forwardIp = ip + step; - - if (forwardIp > mflimit) { goto _last_literals; } - - forwardH = LZ4_HASH64K_VALUE(forwardIp); - ref = base + HashTable[h]; - HashTable[h] = ip - base; - - } while (A32(ref) != A32(ip)); - - // Catch up - while ((ip>anchor) && (ref>(BYTE*)source) && (ip[-1]==ref[-1])) { ip--; ref--; } - - // Encode Literal length - length = ip - anchor; - token = op++; - if (length>=(int)RUN_MASK) { *token=(RUN_MASK< 254 ; len-=255) *op++ = 255; *op++ = (BYTE)len; } - else *token = (length<=(int)ML_MASK) { *token+=ML_MASK; len-=ML_MASK; for(; len > 509 ; len-=510) { *op++ = 255; *op++ = 255; } if (len > 254) { len-=255; *op++ = 255; } *op++ = (BYTE)len; } - else *token += len; - - // Test end of chunk - if (ip > mflimit) { anchor = ip; break; } - - // Fill table - HashTable[LZ4_HASH64K_VALUE(ip-2)] = ip - 2 - base; - - // Test next position - ref = base + HashTable[LZ4_HASH64K_VALUE(ip)]; - HashTable[LZ4_HASH64K_VALUE(ip)] = ip - base; - if (A32(ref) == A32(ip)) { token = op++; *token=0; goto _next_match; } - - // Prepare next loop - anchor = ip++; - forwardH = LZ4_HASH64K_VALUE(ip); - } - -_last_literals: - // Encode Last Literals - { - int lastRun = iend - anchor; - if ((LZ4_COMPRESSMIN>0) && (((op - (BYTE*)dest) + lastRun + 1 + ((lastRun-15)/255)) > isize - LZ4_COMPRESSMIN)) return 0; - if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } - else *op++ = (lastRun<>ML_BITS)) == RUN_MASK) { for (;(len=*ip++)==255;length+=255){} length += len; } - - // copy literals - cpy = op+length; - if unlikely(cpy>oend-COPYLENGTH) - { - if (cpy > oend) goto _output_error; // Error : request to write beyond destination buffer - memcpy(op, ip, length); - ip += length; - break; // Necessarily EOF - } - LZ4_WILDCOPY(ip, op, cpy); ip -= (op-cpy); op = cpy; - - // get offset - LZ4_READ_LITTLEENDIAN_16(ref,cpy,ip); ip+=2; - if (ref < (BYTE* const)dest) goto _output_error; // Error : offset create reference outside destination buffer - - // get matchlength - if ((length=(token&ML_MASK)) == ML_MASK) { for (;*ip==255;length+=255) {ip++;} length += *ip++; } - - // copy repeated sequence - if unlikely(op-refoend-COPYLENGTH) - { - if (cpy > oend) goto _output_error; // Error : request to write beyond destination buffer - LZ4_SECURECOPY(ref, op, (oend-COPYLENGTH)); - while(op>ML_BITS)) == RUN_MASK) { int s=255; while ((ipoend-COPYLENGTH) || (ip+length>iend-COPYLENGTH)) - { - if (cpy > oend) goto _output_error; // Error : request to write beyond destination buffer - if (ip+length > iend) goto _output_error; // Error : request to read beyond source buffer - memcpy(op, ip, length); - op += length; - ip += length; - if (ipoend-COPYLENGTH) - { - if (cpy > oend) goto _output_error; // Error : request to write outside of destination buffer - LZ4_SECURECOPY(ref, op, (oend-COPYLENGTH)); - while(op= 255 -- 10 : (=280 - 15 - 255) ) remaining length to reach 280 - -Example 3 : A length of 15 will be represented as : -- 15 : value for the 4-bits High field -- 0 : (=15-15) yes, the zero must be output - -Following the token and optional length bytes, are the literals themselves. -They are exactly as numerous as previously decoded (length of literals). -It's possible that there are zero literal. - - -Following the literals is the match copy operation. - -It starts by the offset. -This is a 2 bytes value, in little endian format : -the lower byte is the first one in the stream. - -The offset represents the position of the match to be copied from. -1 means "current position - 1 byte". -The maximum offset value is 65535, 65536 cannot be coded. -Note that 0 is an invalid value, not used. - -Then we need to extract the match length. -For this, we use the second token field, the low 4-bits. -Value, obviously, ranges from 0 to 15. -However here, 0 means that the copy operation will be minimal. -The minimum length of a match, called minmatch, is 4. -As a consequence, a 0 value means 4 bytes, and a value of 15 means 19+ bytes. -Similar to literal length, on reaching the highest possible value (15), -we output additional bytes, one at a time, with values ranging from 0 to 255. -They are added to total to provide the final match length. -A 255 value means there is another byte to read and add. -There is no limit to the number of optional bytes that can be output this way. -(This points towards a maximum achievable compression ratio of ~250). - -With the offset and the matchlength, -the decoder can now proceed to copy the data from the already decoded buffer. -On decoding the matchlength, we reach the end of the compressed sequence, -and therefore start another one. - - --- Parsing restrictions -- - -There are specific parsing rules to respect in order to remain compatible -with assumptions made by the decoder : -1) The last 5 bytes are always literals -2) The last match must start at least 12 bytes before end of stream -Consequently, a file with less than 13 bytes cannot be compressed. -These rules are in place to ensure that the decoder -will never read beyond the input buffer, nor write beyond the output buffer. - -Note that the last sequence is also incomplete, -and stops right after literals. - - --- Additional notes -- - -There is no assumption nor limits to the way the compressor -searches and selects matches within the source stream. -It could be a fast scan, a multi-probe, a full search using BST, -standard hash chains or MMC, well whatever. - -Advanced parsing strategies can also be implemented, such as lazy match, -or full optimal parsing. - -All these trade-off offer distinctive speed/memory/compression advantages. -Whatever the method used by the compressor, its result will be decodable -by any LZ4 decoder if it follows the format specification described above. - diff --git a/bundles/org.simantics.fastlz/native/lz4hc.c b/bundles/org.simantics.fastlz/native/lz4hc.c deleted file mode 100644 index cca755c26..000000000 --- a/bundles/org.simantics.fastlz/native/lz4hc.c +++ /dev/null @@ -1,663 +0,0 @@ -/* - LZ4 HC - High Compression Mode of LZ4 - Copyright (C) 2011-2012, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html - - LZ4 source repository : http://code.google.com/p/lz4/ -*/ - - -//************************************** -// CPU Feature Detection -//************************************** -// 32 or 64 bits ? -#if (defined(__x86_64__) || defined(__x86_64) || defined(__amd64__) || defined(__amd64) || defined(__ppc64__) || defined(_WIN64) || defined(__LP64__) || defined(_LP64) ) // Detects 64 bits mode -#define LZ4_ARCH64 1 -#else -#define LZ4_ARCH64 0 -#endif - -// Little Endian or Big Endian ? -#if (defined(__BIG_ENDIAN__) || defined(__BIG_ENDIAN) || defined(_BIG_ENDIAN) || defined(_ARCH_PPC) || defined(__PPC__) || defined(__PPC) || defined(PPC) || defined(__powerpc__) || defined(__powerpc) || defined(powerpc) || ((defined(__BYTE_ORDER__)&&(__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))) ) -#define LZ4_BIG_ENDIAN 1 -#else -// Little Endian assumed. PDP Endian and other very rare endian format are unsupported. -#endif - -// Unaligned memory access is automatically enabled for "common" CPU, such as x86. -// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected -// If you know your target CPU supports unaligned memory access, you may want to force this option manually to improve performance -#if defined(__ARM_FEATURE_UNALIGNED) -#define LZ4_FORCE_UNALIGNED_ACCESS 1 -#endif - - -//************************************** -// Compiler Options -//************************************** -#if __STDC_VERSION__ >= 199901L // C99 - /* "restrict" is a known keyword */ -#else -#define restrict // Disable restrict -#endif - -#ifdef _MSC_VER -#define inline __forceinline // Visual is not C99, but supports some kind of inline -#endif - -#ifdef _MSC_VER // Visual Studio -#define bswap16(x) _byteswap_ushort(x) -#else -#define bswap16(x) ((unsigned short int) ((((x) >> 8) & 0xffu) | (((x) & 0xffu) << 8))) -#endif - - -//************************************** -// Includes -//************************************** -#include // calloc, free -#include // memset, memcpy -#include "lz4hc.h" - -#define ALLOCATOR(s) calloc(1,s) -#define FREEMEM free -#define MEM_INIT memset - - -//************************************** -// Basic Types -//************************************** -#if defined(_MSC_VER) // Visual Studio does not support 'stdint' natively -#define BYTE unsigned __int8 -#define U16 unsigned __int16 -#define U32 unsigned __int32 -#define S32 __int32 -#define U64 unsigned __int64 -#else -#include -#define BYTE uint8_t -#define U16 uint16_t -#define U32 uint32_t -#define S32 int32_t -#define U64 uint64_t -#endif - -#ifndef LZ4_FORCE_UNALIGNED_ACCESS -#pragma pack(push, 1) -#endif - -typedef struct _U16_S { U16 v; } U16_S; -typedef struct _U32_S { U32 v; } U32_S; -typedef struct _U64_S { U64 v; } U64_S; - -#ifndef LZ4_FORCE_UNALIGNED_ACCESS -#pragma pack(pop) -#endif - -#define A64(x) (((U64_S *)(x))->v) -#define A32(x) (((U32_S *)(x))->v) -#define A16(x) (((U16_S *)(x))->v) - - -//************************************** -// Constants -//************************************** -#define MINMATCH 4 - -#define DICTIONARY_LOGSIZE 16 -#define MAXD (1<> ((MINMATCH*8)-HASH_LOG)) -#define HASH_VALUE(p) HASH_FUNCTION(*(U32*)(p)) -#define HASH_POINTER(p) (HashTable[HASH_VALUE(p)] + base) -#define DELTANEXT(p) chainTable[(size_t)(p) & MAXD_MASK] -#define GETNEXT(p) ((p) - (size_t)DELTANEXT(p)) -#define ADD_HASH(p) { size_t delta = (p) - HASH_POINTER(p); if (delta>MAX_DISTANCE) delta = MAX_DISTANCE; DELTANEXT(p) = (U16)delta; HashTable[HASH_VALUE(p)] = (p) - base; } - - -//************************************** -// Private functions -//************************************** -#if LZ4_ARCH64 - -inline static int LZ4_NbCommonBytes (register U64 val) -{ -#if defined(LZ4_BIG_ENDIAN) - #if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse64( &r, val ); - return (int)(r>>3); - #elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clzll(val) >> 3); - #else - int r; - if (!(val>>32)) { r=4; } else { r=0; val>>=32; } - if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } - r += (!val); - return r; - #endif -#else - #if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanForward64( &r, val ); - return (int)(r>>3); - #elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctzll(val) >> 3); - #else - static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; - return DeBruijnBytePos[((U64)((val & -val) * 0x0218A392CDABBD3F)) >> 58]; - #endif -#endif -} - -#else - -inline static int LZ4_NbCommonBytes (register U32 val) -{ -#if defined(LZ4_BIG_ENDIAN) - #if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanReverse( &r, val ); - return (int)(r>>3); - #elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_clz(val) >> 3); - #else - int r; - if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } - r += (!val); - return r; - #endif -#else - #if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) - unsigned long r = 0; - _BitScanForward( &r, val ); - return (int)(r>>3); - #elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) - return (__builtin_ctz(val) >> 3); - #else - static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; - return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; - #endif -#endif -} - -#endif - - -inline static int LZ4HC_Init (LZ4HC_Data_Structure* hc4, const BYTE* base) -{ - MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); - MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); - hc4->nextToUpdate = base + LZ4_ARCH64; - hc4->base = base; - return 1; -} - - -inline static void* LZ4HC_Create (const BYTE* base) -{ - void* hc4 = ALLOCATOR(sizeof(LZ4HC_Data_Structure)); - - LZ4HC_Init (hc4, base); - return hc4; -} - - -inline static int LZ4HC_Free (void** LZ4HC_Data) -{ - FREEMEM(*LZ4HC_Data); - *LZ4HC_Data = NULL; - return (1); -} - - -inline static void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip) -{ - U16* chainTable = hc4->chainTable; - HTYPE* HashTable = hc4->hashTable; - INITBASE(base,hc4->base); - - while(hc4->nextToUpdate < ip) - { - ADD_HASH(hc4->nextToUpdate); - hc4->nextToUpdate++; - } -} - - -inline static int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* const matchlimit, const BYTE** matchpos) -{ - U16* const chainTable = hc4->chainTable; - HTYPE* const HashTable = hc4->hashTable; - const BYTE* ref; - INITBASE(base,hc4->base); - int nbAttempts=MAX_NB_ATTEMPTS; - int ml=0; - - // HC4 match finder - LZ4HC_Insert(hc4, ip); - ref = HASH_POINTER(ip); - while ((ref > (ip-MAX_DISTANCE)) && (nbAttempts)) - { - nbAttempts--; - if (*(ref+ml) == *(ip+ml)) - if (*(U32*)ref == *(U32*)ip) - { - const BYTE* reft = ref+MINMATCH; - const BYTE* ipt = ip+MINMATCH; - - while (ipt ml) { ml = ipt-ip; *matchpos = ref; } - } - ref = GETNEXT(ref); - } - - return ml; -} - - -inline static int LZ4HC_InsertAndGetWiderMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* startLimit, const BYTE* matchlimit, int longest, const BYTE** matchpos, const BYTE** startpos) -{ - U16* const chainTable = hc4->chainTable; - HTYPE* const HashTable = hc4->hashTable; - INITBASE(base,hc4->base); - const BYTE* ref; - int nbAttempts = MAX_NB_ATTEMPTS; - int delta = ip-startLimit; - - // First Match - LZ4HC_Insert(hc4, ip); - ref = HASH_POINTER(ip); - - while ((ref > ip-MAX_DISTANCE) && (ref >= hc4->base) && (nbAttempts)) - { - nbAttempts--; - if (*(startLimit + longest) == *(ref - delta + longest)) - if (*(U32*)ref == *(U32*)ip) - { - const BYTE* reft = ref+MINMATCH; - const BYTE* ipt = ip+MINMATCH; - const BYTE* startt = ip; - - while (iptstartLimit) && (reft > hc4->base) && (startt[-1] == reft[-1])) {startt--; reft--;} - - if ((ipt-startt) > longest) - { - longest = ipt-startt; - *matchpos = reft; - *startpos = startt; - } - } - ref = GETNEXT(ref); - } - - return longest; -} - - -inline static int LZ4_encodeSequence(const BYTE** ip, BYTE** op, const BYTE** anchor, int ml, const BYTE* ref) -{ - int length, len; - BYTE* token; - - // Encode Literal length - length = *ip - *anchor; - token = (*op)++; - if (length>=(int)RUN_MASK) { *token=(RUN_MASK< 254 ; len-=255) *(*op)++ = 255; *(*op)++ = (BYTE)len; } - else *token = (length<=(int)ML_MASK) { *token+=ML_MASK; len-=ML_MASK; for(; len > 509 ; len-=510) { *(*op)++ = 255; *(*op)++ = 255; } if (len > 254) { len-=255; *(*op)++ = 255; } *(*op)++ = (BYTE)len; } - else *token += len; - - // Prepare next loop - *ip += ml; - *anchor = *ip; - - return 0; -} - - -//**************************** -// Compression CODE -//**************************** - -int LZ4_compressHCCtx(LZ4HC_Data_Structure* ctx, - const char* source, - char* dest, - int isize) -{ - const BYTE* ip = (const BYTE*) source; - const BYTE* anchor = ip; - const BYTE* const iend = ip + isize; - const BYTE* const mflimit = iend - MFLIMIT; - const BYTE* const matchlimit = (iend - LASTLITERALS); - - BYTE* op = (BYTE*) dest; - - int ml, ml2, ml3, ml0; - const BYTE* ref=NULL; - const BYTE* start2=NULL; - const BYTE* ref2=NULL; - const BYTE* start3=NULL; - const BYTE* ref3=NULL; - const BYTE* start0; - const BYTE* ref0; - - ip++; - - // Main Loop - while (ip < mflimit) - { - ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref)); - if (!ml) { ip++; continue; } - - // saved, in case we would skip too much - start0 = ip; - ref0 = ref; - ml0 = ml; - -_Search2: - if (ip+ml < mflimit) - ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2); - else ml2=ml; - - if (ml2 == ml) // No better match - { - LZ4_encodeSequence(&ip, &op, &anchor, ml, ref); - continue; - } - - if (start0 < ip) - { - if (start2 < ip + ml0) // empirical - { - ip = start0; - ref = ref0; - ml = ml0; - } - } - - // Here, start0==ip - if ((start2 - ip) < 3) // First Match too small : removed - { - ml = ml2; - ip = start2; - ref =ref2; - goto _Search2; - } - -_Search3: - // Currently we have : - // ml2 > ml1, and - // ip1+3 <= ip2 (usually < ip1+ml1) - if ((start2 - ip) < OPTIMAL_ML) - { - int correction; - int new_ml = ml; - if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML; - if (ip+new_ml > start2 + ml2 - MINMATCH) new_ml = start2 - ip + ml2 - MINMATCH; - correction = new_ml - (start2 - ip); - if (correction > 0) - { - start2 += correction; - ref2 += correction; - ml2 -= correction; - } - } - // Now, we have start2 = ip+new_ml, with new_ml=min(ml, OPTIMAL_ML=18) - - if (start2 + ml2 < mflimit) - ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3); - else ml3=ml2; - - if (ml3 == ml2) // No better match : 2 sequences to encode - { - // ip & ref are known; Now for ml - if (start2 < ip+ml) - { - if ((start2 - ip) < OPTIMAL_ML) - { - int correction; - if (ml > OPTIMAL_ML) ml = OPTIMAL_ML; - if (ip+ml > start2 + ml2 - MINMATCH) ml = start2 - ip + ml2 - MINMATCH; - correction = ml - (start2 - ip); - if (correction > 0) - { - start2 += correction; - ref2 += correction; - ml2 -= correction; - } - } - else - { - ml = start2 - ip; - } - } - // Now, encode 2 sequences - LZ4_encodeSequence(&ip, &op, &anchor, ml, ref); - ip = start2; - LZ4_encodeSequence(&ip, &op, &anchor, ml2, ref2); - continue; - } - - if (start3 < ip+ml+3) // Not enough space for match 2 : remove it - { - if (start3 >= (ip+ml)) // can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 - { - if (start2 < ip+ml) - { - int correction = (ip+ml) - start2; - start2 += correction; - ref2 += correction; - ml2 -= correction; - if (ml2 < MINMATCH) - { - start2 = start3; - ref2 = ref3; - ml2 = ml3; - } - } - - LZ4_encodeSequence(&ip, &op, &anchor, ml, ref); - ip = start3; - ref = ref3; - ml = ml3; - - start0 = start2; - ref0 = ref2; - ml0 = ml2; - goto _Search2; - } - - start2 = start3; - ref2 = ref3; - ml2 = ml3; - goto _Search3; - } - - // OK, now we have 3 ascending matches; let's write at least the first one - // ip & ref are known; Now for ml - if (start2 < ip+ml) - { - if ((start2 - ip) < (int)ML_MASK) - { - int correction; - if (ml > OPTIMAL_ML) ml = OPTIMAL_ML; - if (ip + ml > start2 + ml2 - MINMATCH) ml = start2 - ip + ml2 - MINMATCH; - correction = ml - (start2 - ip); - if (correction > 0) - { - start2 += correction; - ref2 += correction; - ml2 -= correction; - } - } - else - { - ml = start2 - ip; - } - } - LZ4_encodeSequence(&ip, &op, &anchor, ml, ref); - - ip = start2; - ref = ref2; - ml = ml2; - - start2 = start3; - ref2 = ref3; - ml2 = ml3; - - goto _Search3; - - } - - // Encode Last Literals - { - int lastRun = iend - anchor; - if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } - else *op++ = (lastRun< + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {2C249AD2-A0AE-4A88-8DCD-71F96133690E} + fastlz + Win32Proj + + + + DynamicLibrary + Unicode + true + v110 + + + DynamicLibrary + Unicode + v110 + + + DynamicLibrary + Unicode + true + v110 + + + DynamicLibrary + Unicode + v110 + + + + + + + + + + + + + + + + + + + <_ProjectFileVersion>10.0.30319.1 + $(SolutionDir)$(Configuration)\ + $(Configuration)\ + true + $(SolutionDir)$(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ + true + $(SolutionDir)$(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ + false + $(SolutionDir)$(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ + false + + + + Disabled + $(JAVA_HOME)\include;$(JAVA_HOME)\include\win32;%(AdditionalIncludeDirectories) + WIN32;_DEBUG;_WINDOWS;_USRDLL;FASTLZ_EXPORTS;%(PreprocessorDefinitions) + true + EnableFastChecks + MultiThreadedDebug + + + Level3 + EditAndContinue + + + true + Windows + MachineX86 + + + copy "$(TargetPath)" "$(ProjectDir)..\..\src\fastlz-windows-x86.dll" + + + + + X64 + + + Disabled + $(JAVA_HOME)\include;$(JAVA_HOME)\include\win32;%(AdditionalIncludeDirectories) + WIN32;_DEBUG;_WINDOWS;_USRDLL;FASTLZ_EXPORTS;%(PreprocessorDefinitions) + true + EnableFastChecks + MultiThreadedDebug + + + Level3 + ProgramDatabase + + + true + Windows + MachineX64 + + + copy "$(TargetPath)" "$(ProjectDir)..\..\src\fastlz-windows-x86_64.dll" + + + + + MaxSpeed + true + $(JAVA_HOME)\include;$(JAVA_HOME)\include\win32;%(AdditionalIncludeDirectories) + WIN32;NDEBUG;_WINDOWS;_USRDLL;FASTLZ_EXPORTS;%(PreprocessorDefinitions) + MultiThreaded + true + + + Level3 + ProgramDatabase + + + true + Windows + true + true + MachineX86 + + + copy "$(TargetPath)" "$(ProjectDir)..\..\src\fastlz-windows-x86.dll" + + + + + X64 + + + MaxSpeed + true + $(JAVA_HOME)\include;$(JAVA_HOME)\include\win32;%(AdditionalIncludeDirectories) + WIN32;NDEBUG;_WINDOWS;_USRDLL;FASTLZ_EXPORTS;%(PreprocessorDefinitions) + MultiThreaded + true + + + Level3 + ProgramDatabase + + + true + Windows + true + true + MachineX64 + + + copy "$(TargetPath)" "$(ProjectDir)..\..\src\fastlz-windows-x86_64.dll" + + + + + + + + + + + + + \ No newline at end of file diff --git a/bundles/org.simantics.fastlz/native/vs2012/fastlz.vcxproj.filters b/bundles/org.simantics.fastlz/native/vs2012/fastlz.vcxproj.filters new file mode 100644 index 000000000..114e17494 --- /dev/null +++ b/bundles/org.simantics.fastlz/native/vs2012/fastlz.vcxproj.filters @@ -0,0 +1,30 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav + + + + + Source Files + + + Source Files + + + + + Resource Files + + + \ No newline at end of file diff --git a/bundles/org.simantics.fastlz/src/libfastlz-darwin-x86_64.jnilib b/bundles/org.simantics.fastlz/src/libfastlz-darwin-x86_64.dylib similarity index 100% rename from bundles/org.simantics.fastlz/src/libfastlz-darwin-x86_64.jnilib rename to bundles/org.simantics.fastlz/src/libfastlz-darwin-x86_64.dylib diff --git a/bundles/org.simantics.fastlz/src/org/simantics/fastlz/java/FastLZJavaInputStream.java b/bundles/org.simantics.fastlz/src/org/simantics/fastlz/java/FastLZJavaInputStream.java index d6ac30896..290990b8d 100644 --- a/bundles/org.simantics.fastlz/src/org/simantics/fastlz/java/FastLZJavaInputStream.java +++ b/bundles/org.simantics.fastlz/src/org/simantics/fastlz/java/FastLZJavaInputStream.java @@ -43,6 +43,11 @@ public class FastLZJavaInputStream extends DecompressingInputStream { super(stream, channel); } + @Override + protected ByteBuffer allocateBuffer(int capacity) { + return ByteBuffer.allocate(capacity); + } + @Override public void decompress(ByteBuffer compressed, int compressedOffset, int compressedSize, ByteBuffer uncompressed, int uncompressedOffset, int uncompressedSize) throws IOException { diff --git a/bundles/org.simantics.fastlz/testcases/org/simantics/fastlz/FastLZBasicTests.java b/bundles/org.simantics.fastlz/testcases/org/simantics/fastlz/FastLZBasicTests.java index 177c8d1db..572ee1189 100644 --- a/bundles/org.simantics.fastlz/testcases/org/simantics/fastlz/FastLZBasicTests.java +++ b/bundles/org.simantics.fastlz/testcases/org/simantics/fastlz/FastLZBasicTests.java @@ -60,7 +60,6 @@ public class FastLZBasicTests { @Test public void validateCompress() throws IOException { validateCompress(testData1); - validateCompress(new File("grades.snp")); } private void validateCompress(File testData) throws IOException { diff --git a/bundles/org.simantics.graph.compiler/META-INF/MANIFEST.MF b/bundles/org.simantics.graph.compiler/META-INF/MANIFEST.MF index 6d785149f..d64d3d564 100644 --- a/bundles/org.simantics.graph.compiler/META-INF/MANIFEST.MF +++ b/bundles/org.simantics.graph.compiler/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: Compiler Bundle-SymbolicName: org.simantics.graph.compiler;singleton:=true -Bundle-Version: 1.1.11.qualifier +Bundle-Version: 1.1.15.qualifier Bundle-RequiredExecutionEnvironment: JavaSE-1.8 Require-Bundle: org.simantics.graph;bundle-version="1.0.0";visibility:=reexport, org.simantics.ltk.antlr;bundle-version="1.0.0", diff --git a/bundles/org.simantics.graph.compiler/src/org/simantics/graph/compiler/GraphCompiler.java b/bundles/org.simantics.graph.compiler/src/org/simantics/graph/compiler/GraphCompiler.java index 21761a3b1..5f30082a8 100644 --- a/bundles/org.simantics.graph.compiler/src/org/simantics/graph/compiler/GraphCompiler.java +++ b/bundles/org.simantics.graph.compiler/src/org/simantics/graph/compiler/GraphCompiler.java @@ -124,7 +124,7 @@ public class GraphCompiler { run(new CreateInverseRelations(graph, store)); run(new AddConsistsOf(paths, store)); run(new ConvertPreValues(graph, store, errors)); - run(new ReportCollisions(errors, store)); + run(new ReportCollisions(preferences, errors, store)); if(preferences.validate) run(new ValidateGraph(graph, errors, store, preferences)); diff --git a/bundles/org.simantics.graph.compiler/src/org/simantics/graph/compiler/internal/parsing/Graph.g b/bundles/org.simantics.graph.compiler/src/org/simantics/graph/compiler/internal/parsing/Graph.g index 67b11fbaa..995960f94 100644 --- a/bundles/org.simantics.graph.compiler/src/org/simantics/graph/compiler/internal/parsing/Graph.g +++ b/bundles/org.simantics.graph.compiler/src/org/simantics/graph/compiler/internal/parsing/Graph.g @@ -85,7 +85,7 @@ public Token nextToken() { emit balancing number of DEDENT tokens. */ if(iStack.size() <= 1) - return Token.EOF_TOKEN; + return getEOFToken(); else { while(iStack.size() > 1) { iStack.removeAt(iStack.size()-1); @@ -450,4 +450,4 @@ mapAssignment : value '=' value -> ^(ASSIGNMENT value*) ; - \ No newline at end of file + diff --git a/bundles/org.simantics.graph.compiler/src/org/simantics/graph/compiler/internal/parsing/GraphLexer.java b/bundles/org.simantics.graph.compiler/src/org/simantics/graph/compiler/internal/parsing/GraphLexer.java index 98a10ec97..d75d8cc93 100644 --- a/bundles/org.simantics.graph.compiler/src/org/simantics/graph/compiler/internal/parsing/GraphLexer.java +++ b/bundles/org.simantics.graph.compiler/src/org/simantics/graph/compiler/internal/parsing/GraphLexer.java @@ -108,7 +108,7 @@ public class GraphLexer extends Lexer { emit balancing number of DEDENT tokens. */ if(iStack.size() <= 1) - return Token.EOF_TOKEN; + return getEOFToken(); else { while(iStack.size() > 1) { iStack.removeAt(iStack.size()-1); @@ -2471,4 +2471,4 @@ public class GraphLexer extends Lexer { } -} \ No newline at end of file +} diff --git a/bundles/org.simantics.graph.compiler/src/org/simantics/graph/compiler/internal/validation/ReportCollisions.java b/bundles/org.simantics.graph.compiler/src/org/simantics/graph/compiler/internal/validation/ReportCollisions.java index e7346e945..277ddc7b1 100644 --- a/bundles/org.simantics.graph.compiler/src/org/simantics/graph/compiler/internal/validation/ReportCollisions.java +++ b/bundles/org.simantics.graph.compiler/src/org/simantics/graph/compiler/internal/validation/ReportCollisions.java @@ -2,22 +2,34 @@ package org.simantics.graph.compiler.internal.validation; import java.util.Collection; +import org.simantics.graph.compiler.GraphCompilerPreferences; import org.simantics.graph.compiler.internal.store.LocationStore; import org.simantics.graph.compiler.internal.templates.TemplateDefinitionStore; +import org.simantics.graph.query.Res; import org.simantics.graph.store.GraphStore; +import org.simantics.graph.store.StatementCollision; import org.simantics.ltk.Problem; public class ReportCollisions implements Runnable { + GraphCompilerPreferences preferences; Collection problems; GraphStore store; public ReportCollisions( - Collection problems, + GraphCompilerPreferences preferences, Collection problems, GraphStore store) { + this.preferences = preferences; this.problems = problems; this.store = store; } + private static String abbreviateURI(Res res) { + if(res == null) + return "null"; + String uri = res.toString(); + return uri.replace("http://www.simantics.org/", ""); + } + @Override public void run() { LocationStore locations = store.getStore(LocationStore.class); @@ -32,7 +44,16 @@ public class ReportCollisions implements Runnable { for(int c : store.getStore(TemplateDefinitionStore.class).getCollisions().toArray()) problems.add(new Problem( locations.getLocation(c), - "Two tempalate definitions are given for the same resource.")); + "Two template definitions are given for the same resource.")); + if(preferences.validate) + for(StatementCollision collision : store.statements.getCollisions()) { + problems.add(new Problem( + locations.getLocation(collision.subject), + "The same statement is defined " + collision.count + " times: " + + abbreviateURI(store.idToRes(collision.subject)) + ", " + + abbreviateURI(store.idToRes(collision.predicate)) + ", " + + abbreviateURI(store.idToRes(collision.object)))); + } } } diff --git a/bundles/org.simantics.graph/META-INF/MANIFEST.MF b/bundles/org.simantics.graph/META-INF/MANIFEST.MF index 448d8556b..5f4404acc 100644 --- a/bundles/org.simantics.graph/META-INF/MANIFEST.MF +++ b/bundles/org.simantics.graph/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: Transferable Graph Runtime Bundle-SymbolicName: org.simantics.graph -Bundle-Version: 1.1.11.qualifier +Bundle-Version: 1.1.15.qualifier Bundle-RequiredExecutionEnvironment: JavaSE-1.8 Require-Bundle: org.simantics.databoard;bundle-version="0.5.1", gnu.trove3;bundle-version="3.0.0";visibility:=reexport, diff --git a/bundles/org.simantics.graph/src/org/simantics/graph/store/StatementCollision.java b/bundles/org.simantics.graph/src/org/simantics/graph/store/StatementCollision.java new file mode 100644 index 000000000..3bcd4f0bf --- /dev/null +++ b/bundles/org.simantics.graph/src/org/simantics/graph/store/StatementCollision.java @@ -0,0 +1,16 @@ +package org.simantics.graph.store; + +public class StatementCollision { + public final int subject; + public final int predicate; + public final int object; + public final int count; + + public StatementCollision(int subject, int predicate, int object, int count) { + this.subject = subject; + this.predicate = predicate; + this.object = object; + this.count = count; + } +} + diff --git a/bundles/org.simantics.graph/src/org/simantics/graph/store/StatementStore.java b/bundles/org.simantics.graph/src/org/simantics/graph/store/StatementStore.java index 96b6b1c72..02dfddde8 100644 --- a/bundles/org.simantics.graph/src/org/simantics/graph/store/StatementStore.java +++ b/bundles/org.simantics.graph/src/org/simantics/graph/store/StatementStore.java @@ -1,5 +1,7 @@ package org.simantics.graph.store; +import java.util.ArrayList; + import gnu.trove.list.array.TIntArrayList; import gnu.trove.map.hash.TIntIntHashMap; import gnu.trove.map.hash.TIntObjectHashMap; @@ -10,7 +12,7 @@ import gnu.trove.set.hash.TIntHashSet; /** * Statement store indexes a set of statements. - * @author Hannu Niemistö + * @author Hannu Niemist� */ public class StatementStore implements IStore { @@ -287,5 +289,64 @@ public class StatementStore implements IStore { } }); return statements.toArray(); + } + + private static class CollisionSubjectProcedure implements TIntObjectProcedure> { + CollisionPredicateProcedure predicateProcedure; + + public CollisionSubjectProcedure(CollisionPredicateProcedure predicateProcedure) { + this.predicateProcedure = predicateProcedure; + } + + @Override + public boolean execute(int subject, TIntObjectHashMap predicateObjectMap) { + predicateProcedure.subject = subject; + predicateObjectMap.forEachEntry(predicateProcedure); + return true; + } + + } + + private static class CollisionPredicateProcedure implements TIntObjectProcedure { + ArrayList collisions; + int subject; + + public CollisionPredicateProcedure(ArrayList collisions) { + this.collisions = collisions; + } + + @Override + public boolean execute(int predicate, TIntArrayList objects) { + if(objects.size() > 1) { + objects.sort(); + int oldObject = objects.get(0); + int collisionCount = 1; + for(int i=1;i 1) { + collisions.add(new StatementCollision(subject, predicate, oldObject, collisionCount)); + collisionCount = 1; + } + oldObject = curObject; + } + } + if(collisionCount > 1) + collisions.add(new StatementCollision(subject, predicate, oldObject, collisionCount)); + } + return true; + } + + } + + public ArrayList getCollisions() { + ArrayList collisions = new ArrayList(); + CollisionPredicateProcedure predicateProcedure = new CollisionPredicateProcedure(collisions); + CollisionSubjectProcedure subjectProcedure = new CollisionSubjectProcedure(predicateProcedure); + statements.forEachEntry(subjectProcedure); + return collisions; } } diff --git a/bundles/org.simantics.help.base/.classpath b/bundles/org.simantics.help.base/.classpath index 6cf33fb32..b1dabee38 100644 --- a/bundles/org.simantics.help.base/.classpath +++ b/bundles/org.simantics.help.base/.classpath @@ -1,9 +1,5 @@ - - - - diff --git a/bundles/org.simantics.help.base/META-INF/MANIFEST.MF b/bundles/org.simantics.help.base/META-INF/MANIFEST.MF index d9c6d4585..a20f8e475 100644 --- a/bundles/org.simantics.help.base/META-INF/MANIFEST.MF +++ b/bundles/org.simantics.help.base/META-INF/MANIFEST.MF @@ -8,11 +8,8 @@ Bundle-Vendor: Semantum Oy Require-Bundle: org.eclipse.core.runtime, org.eclipse.help.base, org.apache.commons.logging;bundle-version="1.0.4", - org.bouncycastle;bundle-version="1.47.0" + org.apache.pdfbox;bundle-version="2.0.2", + org.apache.pdfbox.fontbox;bundle-version="2.0.2" Bundle-RequiredExecutionEnvironment: JavaSE-1.7 Bundle-ActivationPolicy: lazy -Bundle-ClassPath: ., - xmpbox-1.8.10.jar, - jempbox-1.8.10.jar, - fontbox-1.8.10.jar, - pdfbox-1.8.10.jar +Bundle-ClassPath: . diff --git a/bundles/org.simantics.help.base/build.properties b/bundles/org.simantics.help.base/build.properties index 7904aba26..c125ab4d7 100644 --- a/bundles/org.simantics.help.base/build.properties +++ b/bundles/org.simantics.help.base/build.properties @@ -2,9 +2,5 @@ source.. = src/ output.. = bin/ bin.includes = META-INF/,\ .,\ - plugin.xml,\ - pdfbox-1.8.10.jar,\ - fontbox-1.8.10.jar,\ - jempbox-1.8.10.jar,\ - xmpbox-1.8.10.jar + plugin.xml source.. = src/ diff --git a/bundles/org.simantics.help.base/fontbox-1.8.10.jar b/bundles/org.simantics.help.base/fontbox-1.8.10.jar deleted file mode 100644 index 3284950c8..000000000 Binary files a/bundles/org.simantics.help.base/fontbox-1.8.10.jar and /dev/null differ diff --git a/bundles/org.simantics.help.base/jempbox-1.8.10.jar b/bundles/org.simantics.help.base/jempbox-1.8.10.jar deleted file mode 100644 index 48cc63375..000000000 Binary files a/bundles/org.simantics.help.base/jempbox-1.8.10.jar and /dev/null differ diff --git a/bundles/org.simantics.help.base/pdfbox-1.8.10-src.zip b/bundles/org.simantics.help.base/pdfbox-1.8.10-src.zip deleted file mode 100644 index e05aa634b..000000000 Binary files a/bundles/org.simantics.help.base/pdfbox-1.8.10-src.zip and /dev/null differ diff --git a/bundles/org.simantics.help.base/pdfbox-1.8.10.jar b/bundles/org.simantics.help.base/pdfbox-1.8.10.jar deleted file mode 100644 index 87bb9a704..000000000 Binary files a/bundles/org.simantics.help.base/pdfbox-1.8.10.jar and /dev/null differ diff --git a/bundles/org.simantics.help.base/src/org/simantics/help/base/internal/PDFUtil.java b/bundles/org.simantics.help.base/src/org/simantics/help/base/internal/PDFUtil.java index c12e56b4d..43fac6c70 100644 --- a/bundles/org.simantics.help.base/src/org/simantics/help/base/internal/PDFUtil.java +++ b/bundles/org.simantics.help.base/src/org/simantics/help/base/internal/PDFUtil.java @@ -1,14 +1,14 @@ package org.simantics.help.base.internal; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import org.apache.pdfbox.cos.COSDocument; +import org.apache.pdfbox.io.RandomAccessFile; import org.apache.pdfbox.pdfparser.PDFParser; import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.pdmodel.PDDocumentInformation; -import org.apache.pdfbox.util.PDFTextStripper; +import org.apache.pdfbox.text.PDFTextStripper; import org.eclipse.help.search.ISearchDocument; /** @@ -17,7 +17,7 @@ import org.eclipse.help.search.ISearchDocument; public class PDFUtil { public static void stripText(File fromPdf, ISearchDocument doc) throws IOException { - PDFParser parser = new PDFParser(new FileInputStream(fromPdf)); + PDFParser parser = new PDFParser(new RandomAccessFile(fromPdf, "r")); parser.parse(); try (COSDocument cosDoc = parser.getDocument()) { diff --git a/bundles/org.simantics.help.base/xmpbox-1.8.10.jar b/bundles/org.simantics.help.base/xmpbox-1.8.10.jar deleted file mode 100644 index 7a8a46594..000000000 Binary files a/bundles/org.simantics.help.base/xmpbox-1.8.10.jar and /dev/null differ diff --git a/bundles/org.simantics.help.ui/META-INF/MANIFEST.MF b/bundles/org.simantics.help.ui/META-INF/MANIFEST.MF index e9f8ba1b9..0bc7d909d 100644 --- a/bundles/org.simantics.help.ui/META-INF/MANIFEST.MF +++ b/bundles/org.simantics.help.ui/META-INF/MANIFEST.MF @@ -12,7 +12,8 @@ Require-Bundle: org.eclipse.ui, org.eclipse.ui.editors;bundle-version="3.9.0", org.eclipse.jface.text;bundle-version="3.10.0", org.eclipse.core.resources;bundle-version="3.10.1", - org.simantics.help.core + org.simantics.help.core, + org.slf4j.api Bundle-RequiredExecutionEnvironment: JavaSE-1.8 Bundle-ActivationPolicy: lazy Bundle-ClassPath: . diff --git a/bundles/org.simantics.help.ui/src/org/simantics/help/ui/OpenHelpFileAdapter.java b/bundles/org.simantics.help.ui/src/org/simantics/help/ui/OpenHelpFileAdapter.java index c5847ce0d..6a5e2d968 100644 --- a/bundles/org.simantics.help.ui/src/org/simantics/help/ui/OpenHelpFileAdapter.java +++ b/bundles/org.simantics.help.ui/src/org/simantics/help/ui/OpenHelpFileAdapter.java @@ -2,7 +2,6 @@ package org.simantics.help.ui; import org.eclipse.ui.PartInitException; import org.eclipse.ui.PlatformUI; -import org.simantics.Logger; import org.simantics.Simantics; import org.simantics.db.ReadGraph; import org.simantics.db.Resource; @@ -16,8 +15,11 @@ import org.simantics.help.HelpResources; import org.simantics.ui.workbench.ResourceEditorInput2; import org.simantics.ui.workbench.editor.AbstractResourceEditorAdapter; import org.simantics.utils.ui.workbench.WorkbenchUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class OpenHelpFileAdapter extends AbstractResourceEditorAdapter { + private static final Logger LOGGER = LoggerFactory.getLogger(OpenHelpFileAdapter.class); public OpenHelpFileAdapter() { super("Help File Editor"); @@ -49,7 +51,7 @@ public class OpenHelpFileAdapter extends AbstractResourceEditorAdapter { String editorId = getEditorId(); WorkbenchUtils.openEditor(editorId, new ResourceEditorInput2(editorId, input, model, rvi)); } catch (PartInitException e) { - Logger.defaultLogError(e); + LOGGER.error("Failed to open an editor for help file.", e); } } }); diff --git a/bundles/org.simantics.image.ui/scl/Simantics/Image.scl b/bundles/org.simantics.image.ui/scl/Simantics/Image.scl index 74d197f2f..df9ba8360 100644 --- a/bundles/org.simantics.image.ui/scl/Simantics/Image.scl +++ b/bundles/org.simantics.image.ui/scl/Simantics/Image.scl @@ -8,6 +8,7 @@ type Image = Resource importJava "org.simantics.image.ui.SCLImage" where @JavaName importImageFromFile importImage :: File -> Library -> Image - - @JavaName linkImage - linkImage :: SVGImage -> Image -> () \ No newline at end of file + +// HN: does not exist +// @JavaName linkImage +// linkImage :: SVGImage -> Image -> () \ No newline at end of file diff --git a/bundles/org.simantics.issues.common/META-INF/MANIFEST.MF b/bundles/org.simantics.issues.common/META-INF/MANIFEST.MF index 421606517..23b688bcd 100644 --- a/bundles/org.simantics.issues.common/META-INF/MANIFEST.MF +++ b/bundles/org.simantics.issues.common/META-INF/MANIFEST.MF @@ -11,7 +11,8 @@ Require-Bundle: org.simantics.issues;bundle-version="1.1.0", org.simantics.layer0.utils;bundle-version="0.6.2", org.simantics.db.services;bundle-version="0.6.2", org.simantics;bundle-version="1.0.0", - org.simantics.issues.ui.ontology;bundle-version="1.0.0" + org.simantics.issues.ui.ontology;bundle-version="1.0.0", + org.slf4j.api Export-Package: org.simantics.issues.common, org.simantics.issues.common.preferences Bundle-Vendor: VTT Technical Research Centre of Finland diff --git a/bundles/org.simantics.issues.common/src/org/simantics/issues/common/DependencyIssueValidator2.java b/bundles/org.simantics.issues.common/src/org/simantics/issues/common/DependencyIssueValidator2.java index 7c6cdeccf..2f8c42b90 100644 --- a/bundles/org.simantics.issues.common/src/org/simantics/issues/common/DependencyIssueValidator2.java +++ b/bundles/org.simantics.issues.common/src/org/simantics/issues/common/DependencyIssueValidator2.java @@ -5,7 +5,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import org.simantics.Logger; import org.simantics.db.Issue; import org.simantics.db.ReadGraph; import org.simantics.db.Resource; @@ -17,8 +16,11 @@ import org.simantics.db.common.utils.NameUtils; import org.simantics.db.exception.DatabaseException; import org.simantics.issues.ontology.IssueResource; import org.simantics.layer0.Layer0; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class DependencyIssueValidator2 extends ResourceRead3 { + private static final Logger LOGGER = LoggerFactory.getLogger(DependencyIssueValidator2.class); public static final boolean DEBUG = false; @@ -65,7 +67,7 @@ public class DependencyIssueValidator2 extends ResourceRead3 { if(DEBUG) System.err.println("Validator found: " + contexts.size() + " issues (" + contexts + ")"); return contexts; } catch (DatabaseException e) { - Logger.defaultLogError(e); + LOGGER.error("Reading a constraint validator failed", e); return Collections.emptySet(); } diff --git a/bundles/org.simantics.issues.common/src/org/simantics/issues/common/IssueUtils.java b/bundles/org.simantics.issues.common/src/org/simantics/issues/common/IssueUtils.java index 26f0167ad..f1dcadc89 100644 --- a/bundles/org.simantics.issues.common/src/org/simantics/issues/common/IssueUtils.java +++ b/bundles/org.simantics.issues.common/src/org/simantics/issues/common/IssueUtils.java @@ -25,7 +25,6 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; -import org.simantics.Logger; import org.simantics.Simantics; import org.simantics.databoard.Bindings; import org.simantics.databoard.util.URIStringUtils; @@ -57,11 +56,14 @@ import org.simantics.layer0.Layer0; import org.simantics.operation.Layer0X; import org.simantics.scl.runtime.function.FunctionImpl2; import org.simantics.utils.datastructures.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * @author Tuukka Lehtonen */ public class IssueUtils { + private static final Logger LOGGER = LoggerFactory.getLogger(IssueUtils.class); public static Resource toSeverityResource(IssueResource ISSUE, Severity severity) { switch (severity) { @@ -113,7 +115,7 @@ public class IssueUtils { }); } } catch (DatabaseException e) { - Logger.defaultLogError(e); + LOGGER.error("Updating issue source failed.", e); } } else { Session session = Simantics.getSession(); @@ -173,7 +175,7 @@ public class IssueUtils { @Override public void exception(ReadGraph graph, Throwable throwable) throws DatabaseException { - Logger.defaultLogError(throwable); + LOGGER.error("IssueValidityListener received an exception.", throwable); } } @@ -207,7 +209,7 @@ public class IssueUtils { @Override public void exception(ReadGraph graph, Throwable t) { - Logger.defaultLogError(t); + LOGGER.error("IssueSourceManagedIssuesListener received an exception.", t); } @Override @@ -263,7 +265,7 @@ public class IssueUtils { @Override public void exception(ReadGraph graph, Throwable t) { - Logger.defaultLogError(t); + LOGGER.error("ActiveIssueSourceListener received an exception.", t); } @Override diff --git a/bundles/org.simantics.logback.configuration/.classpath b/bundles/org.simantics.logback.configuration/.classpath new file mode 100644 index 000000000..eca7bdba8 --- /dev/null +++ b/bundles/org.simantics.logback.configuration/.classpath @@ -0,0 +1,7 @@ + + + + + + + diff --git a/bundles/org.simantics.logback.configuration/.project b/bundles/org.simantics.logback.configuration/.project new file mode 100644 index 000000000..3fdee5609 --- /dev/null +++ b/bundles/org.simantics.logback.configuration/.project @@ -0,0 +1,28 @@ + + + org.simantics.logback.configuration + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.pde.ManifestBuilder + + + + + org.eclipse.pde.SchemaBuilder + + + + + + org.eclipse.pde.PluginNature + org.eclipse.jdt.core.javanature + + diff --git a/bundles/org.simantics.logback.configuration/.settings/org.eclipse.jdt.core.prefs b/bundles/org.simantics.logback.configuration/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 000000000..0c68a61dc --- /dev/null +++ b/bundles/org.simantics.logback.configuration/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,7 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.problem.assertIdentifier=error +org.eclipse.jdt.core.compiler.problem.enumIdentifier=error +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/bundles/org.simantics.logback.configuration/META-INF/MANIFEST.MF b/bundles/org.simantics.logback.configuration/META-INF/MANIFEST.MF new file mode 100644 index 000000000..ae85b8ace --- /dev/null +++ b/bundles/org.simantics.logback.configuration/META-INF/MANIFEST.MF @@ -0,0 +1,9 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: Simantics Logback Configuration +Bundle-SymbolicName: org.simantics.logback.configuration +Bundle-Version: 1.0.0.qualifier +Fragment-Host: ch.qos.logback.classic;bundle-version="1.1.7" +Bundle-RequiredExecutionEnvironment: JavaSE-1.8 +Require-Bundle: ch.qos.logback.core;bundle-version="1.1.7" +Bundle-Vendor: Semantum Oy diff --git a/bundles/org.simantics.logback.configuration/build.properties b/bundles/org.simantics.logback.configuration/build.properties new file mode 100644 index 000000000..f9c250446 --- /dev/null +++ b/bundles/org.simantics.logback.configuration/build.properties @@ -0,0 +1,5 @@ +source.. = src/ +output.. = bin/ +bin.includes = META-INF/,\ + .,\ + logback.xml diff --git a/bundles/org.simantics.logback.configuration/logback.xml b/bundles/org.simantics.logback.configuration/logback.xml new file mode 100644 index 000000000..38dc39338 --- /dev/null +++ b/bundles/org.simantics.logback.configuration/logback.xml @@ -0,0 +1,18 @@ + + + + + + + %-5p [%d] %c: %m%n%rEx + + + + + + + + + + + \ No newline at end of file diff --git a/bundles/org.simantics.logback.configuration/src/.keep b/bundles/org.simantics.logback.configuration/src/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/bundles/org.simantics.ltk.antlr/META-INF/MANIFEST.MF b/bundles/org.simantics.ltk.antlr/META-INF/MANIFEST.MF index 99da36f06..0be675325 100644 --- a/bundles/org.simantics.ltk.antlr/META-INF/MANIFEST.MF +++ b/bundles/org.simantics.ltk.antlr/META-INF/MANIFEST.MF @@ -5,5 +5,5 @@ Bundle-SymbolicName: org.simantics.ltk.antlr Bundle-Version: 1.1.10.qualifier Bundle-RequiredExecutionEnvironment: JavaSE-1.8 Require-Bundle: org.simantics.ltk;bundle-version="1.0.0";visibility:=reexport, - org.antlr.runtime;bundle-version="3.2.0";visibility:=reexport + org.antlr.runtime;bundle-version="[3.2.0,4.0.0)";visibility:=reexport Export-Package: org.simantics.ltk.antlr diff --git a/bundles/org.simantics.modeling.ui/META-INF/MANIFEST.MF b/bundles/org.simantics.modeling.ui/META-INF/MANIFEST.MF index 10ad43d9a..f396a5815 100644 --- a/bundles/org.simantics.modeling.ui/META-INF/MANIFEST.MF +++ b/bundles/org.simantics.modeling.ui/META-INF/MANIFEST.MF @@ -32,7 +32,7 @@ Require-Bundle: org.simantics.project;bundle-version="1.0.0", org.simantics.issues;bundle-version="1.1.0", org.simantics.document;bundle-version="1.0.0", org.simantics.graph.db;bundle-version="1.1.9", - org.bouncycastle;bundle-version="1.47.0", + org.bouncycastle.bcprov-jdk14;bundle-version="1.38.0", org.simantics.image2.ontology;bundle-version="1.1.0", org.simantics.scl.compiler;bundle-version="0.4.0", org.simantics.scl.compiler.dummy;bundle-version="1.0.0", @@ -61,7 +61,8 @@ Require-Bundle: org.simantics.project;bundle-version="1.0.0", org.simantics.db.layer0, org.simantics.silk.ontology;bundle-version="1.1.0", org.simantics.image.ui;bundle-version="1.0.0", - org.simantics.export.core;bundle-version="1.0.0" + org.simantics.export.core;bundle-version="1.0.0", + org.slf4j.api Export-Package: org.simantics.modeling.ui, org.simantics.modeling.ui.actions, org.simantics.modeling.ui.chart.property, diff --git a/bundles/org.simantics.modeling.ui/src/org/simantics/modeling/ui/actions/ModeledActions.java b/bundles/org.simantics.modeling.ui/src/org/simantics/modeling/ui/actions/ModeledActions.java index 47d5643a1..ea50134d7 100644 --- a/bundles/org.simantics.modeling.ui/src/org/simantics/modeling/ui/actions/ModeledActions.java +++ b/bundles/org.simantics.modeling.ui/src/org/simantics/modeling/ui/actions/ModeledActions.java @@ -33,7 +33,6 @@ import org.eclipse.jface.action.ActionContributionItem; import org.eclipse.jface.action.IContributionItem; import org.eclipse.jface.action.MenuManager; import org.eclipse.jface.action.Separator; -import org.simantics.Logger; import org.simantics.browsing.ui.NodeContext; import org.simantics.browsing.ui.common.NodeContextBuilder; import org.simantics.browsing.ui.model.InvalidContribution; @@ -42,13 +41,17 @@ import org.simantics.browsing.ui.model.actions.IActionCategory; import org.simantics.db.ReadGraph; import org.simantics.db.Resource; import org.simantics.db.exception.DatabaseException; +import org.simantics.issues.common.IssueUtils; import org.simantics.modeling.ui.Activator; import org.simantics.project.ontology.ProjectResource; import org.simantics.ui.contribution.DynamicMenuContribution; import org.simantics.ui.selection.WorkbenchSelectionElement; import org.simantics.ui.selection.WorkbenchSelectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class ModeledActions extends DynamicMenuContribution implements IExecutableExtension { + private static final Logger LOGGER = LoggerFactory.getLogger(IssueUtils.class); public static final Set defaultBrowseContexts = Collections.singleton(ProjectResource.URIs.ProjectActionContext); @@ -121,7 +124,7 @@ public class ModeledActions extends DynamicMenuContribution implements IExecutab result.add(NodeContextBuilder.buildWithInput(res)); } } catch (DatabaseException e) { - Logger.defaultLogError(e); + LOGGER.error("Failed to get node contexts for selection.", e); } } } diff --git a/bundles/org.simantics.modeling.ui/src/org/simantics/modeling/ui/componentTypeEditor/ConfigurationPropertiesSection.java b/bundles/org.simantics.modeling.ui/src/org/simantics/modeling/ui/componentTypeEditor/ConfigurationPropertiesSection.java index 1ad4cd539..477a5543b 100644 --- a/bundles/org.simantics.modeling.ui/src/org/simantics/modeling/ui/componentTypeEditor/ConfigurationPropertiesSection.java +++ b/bundles/org.simantics.modeling.ui/src/org/simantics/modeling/ui/componentTypeEditor/ConfigurationPropertiesSection.java @@ -38,7 +38,6 @@ import org.eclipse.ui.PlatformUI; import org.eclipse.ui.forms.widgets.Form; import org.eclipse.ui.forms.widgets.FormToolkit; import org.eclipse.ui.forms.widgets.Section; -import org.simantics.Logger; import org.simantics.Simantics; import org.simantics.databoard.Bindings; import org.simantics.db.ReadGraph; @@ -63,9 +62,13 @@ import org.simantics.modeling.userComponent.ComponentTypeCommands; import org.simantics.selectionview.SelectionViewResources; import org.simantics.structural.stubs.StructuralResource2; import org.simantics.utils.datastructures.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class ConfigurationPropertiesSection implements ComponentTypeViewerSection { + private static final Logger LOGGER = LoggerFactory.getLogger(ConfigurationPropertiesSection.class); + private static final String[] COLUMN_NAMES = new String[] {"Name", "Type", "Default Value", "Unit", "Range", "Label", "Description"}; private static final int[] COLUMN_LENGTHS = @@ -443,7 +446,7 @@ public class ConfigurationPropertiesSection implements ComponentTypeViewerSectio } catch (DatabaseException e1) { - Logger.defaultLogError(e1); + LOGGER.error("Lifting properties failed", e1); return; } @@ -565,7 +568,7 @@ public class ConfigurationPropertiesSection implements ComponentTypeViewerSectio } }); } catch (DatabaseException e) { - Logger.defaultLogError(e); + LOGGER.error("Finding UserDefinedProperties failed.", e); return Collections.emptyMap(); } } diff --git a/bundles/org.simantics.modeling.ui/src/org/simantics/modeling/ui/diagramEditor/OpenSheetAdapter.java b/bundles/org.simantics.modeling.ui/src/org/simantics/modeling/ui/diagramEditor/OpenSheetAdapter.java index 207257baa..0a4ca49b4 100644 --- a/bundles/org.simantics.modeling.ui/src/org/simantics/modeling/ui/diagramEditor/OpenSheetAdapter.java +++ b/bundles/org.simantics.modeling.ui/src/org/simantics/modeling/ui/diagramEditor/OpenSheetAdapter.java @@ -13,7 +13,6 @@ package org.simantics.modeling.ui.diagramEditor; import org.eclipse.ui.PartInitException; import org.eclipse.ui.PlatformUI; -import org.simantics.Logger; import org.simantics.db.ReadGraph; import org.simantics.db.Resource; import org.simantics.db.common.request.PossibleIndexRoot; @@ -28,8 +27,11 @@ import org.simantics.ui.SimanticsUI; import org.simantics.ui.workbench.ResourceEditorInput2; import org.simantics.ui.workbench.editor.AbstractResourceEditorAdapter; import org.simantics.utils.ui.workbench.WorkbenchUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class OpenSheetAdapter extends AbstractResourceEditorAdapter { + private static final Logger LOGGER = LoggerFactory.getLogger(OpenSheetAdapter.class); private static final String EDITOR_ID = "org.simantics.spreadsheet.ui.editor2"; @@ -63,7 +65,7 @@ public class OpenSheetAdapter extends AbstractResourceEditorAdapter { String editorId = getEditorId(); WorkbenchUtils.openEditor(editorId, new ResourceEditorInput2(editorId, r, model, rvi)); } catch (PartInitException e) { - Logger.defaultLogError(e); + LOGGER.error("Failed to open the spreadsheet editor.", e); } } }); diff --git a/bundles/org.simantics.modeling.ui/src/org/simantics/modeling/ui/diagramEditor/PopulateElementMonitorDropParticipant.java b/bundles/org.simantics.modeling.ui/src/org/simantics/modeling/ui/diagramEditor/PopulateElementMonitorDropParticipant.java index 0deeb1be1..959bbbe6f 100644 --- a/bundles/org.simantics.modeling.ui/src/org/simantics/modeling/ui/diagramEditor/PopulateElementMonitorDropParticipant.java +++ b/bundles/org.simantics.modeling.ui/src/org/simantics/modeling/ui/diagramEditor/PopulateElementMonitorDropParticipant.java @@ -11,8 +11,6 @@ *******************************************************************************/ package org.simantics.modeling.ui.diagramEditor; -import gnu.trove.set.hash.THashSet; - import java.awt.datatransfer.Transferable; import java.awt.datatransfer.UnsupportedFlavorException; import java.awt.dnd.DnDConstants; @@ -25,7 +23,6 @@ import java.util.HashMap; import java.util.List; import org.eclipse.jface.viewers.IStructuredSelection; -import org.simantics.Logger; import org.simantics.db.ReadGraph; import org.simantics.db.RequestProcessor; import org.simantics.db.Resource; @@ -64,8 +61,13 @@ import org.simantics.ui.selection.WorkbenchSelectionUtils; import org.simantics.utils.datastructures.Triple; import org.simantics.utils.datastructures.hints.IHintContext; import org.simantics.utils.ui.ISelectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import gnu.trove.set.hash.THashSet; public class PopulateElementMonitorDropParticipant extends PopulateElementDropParticipant implements IDropTargetParticipant { + private static final Logger LOGGER = LoggerFactory.getLogger(PopulateElementMonitorDropParticipant.class); private static final boolean DEBUG = false; @@ -129,14 +131,9 @@ public class PopulateElementMonitorDropParticipant extends PopulateElementDropPa dp.getHints().setHint(DnDHints.KEY_DND_GRID_COLUMNS, Integer.valueOf(1)); } - } catch (UnsupportedFlavorException e) { - Logger.defaultLogError(e); - } catch (IOException e) { - Logger.defaultLogError(e); - } catch (DatabaseException e) { - Logger.defaultLogError(e); + } catch (UnsupportedFlavorException|IOException|DatabaseException e) { + LOGGER.error("dragEnter failed", e); } - } dtde.acceptDrag(DnDConstants.ACTION_COPY); diff --git a/bundles/org.simantics.modeling/META-INF/MANIFEST.MF b/bundles/org.simantics.modeling/META-INF/MANIFEST.MF index 249d9ec7c..5245ed47e 100644 --- a/bundles/org.simantics.modeling/META-INF/MANIFEST.MF +++ b/bundles/org.simantics.modeling/META-INF/MANIFEST.MF @@ -36,7 +36,8 @@ Require-Bundle: org.simantics.simulation;bundle-version="1.0.0", org.simantics.scenegraph.profile;bundle-version="1.0.0", org.simantics.scl.db;bundle-version="0.1.3", org.simantics.selectionview.ontology;bundle-version="1.2.0", - org.simantics.scl.ui;bundle-version="0.5.0" + org.simantics.scl.ui;bundle-version="0.5.0", + org.slf4j.api Export-Package: org.simantics.modeling, org.simantics.modeling.actions, org.simantics.modeling.adapters, diff --git a/bundles/org.simantics.modeling/src/org/simantics/modeling/SCL.java b/bundles/org.simantics.modeling/src/org/simantics/modeling/SCL.java index 52909e018..24e50c0f7 100644 --- a/bundles/org.simantics.modeling/src/org/simantics/modeling/SCL.java +++ b/bundles/org.simantics.modeling/src/org/simantics/modeling/SCL.java @@ -10,7 +10,6 @@ import org.eclipse.core.runtime.Platform; import org.eclipse.core.runtime.jobs.IJobManager; import org.eclipse.core.runtime.jobs.Job; import org.simantics.DatabaseJob; -import org.simantics.Logger; import org.simantics.Simantics; import org.simantics.SimanticsPlatform; import org.simantics.SimanticsPlatform.OntologyRecoveryPolicy; @@ -24,8 +23,12 @@ import org.simantics.db.layer0.util.RemoverUtil; import org.simantics.db.layer0.util.SimanticsClipboard; import org.simantics.db.service.DebugSupport; import org.simantics.db.service.ServiceActivityMonitor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class SCL { + private static final Logger LOGGER = LoggerFactory.getLogger(SCL.class); + public static void killPlatformWrite(WriteGraph graph) throws DatabaseException { // Currently not supported. // Would be relatively easy to support the desired functionality. @@ -113,7 +116,7 @@ public class SCL { try { Thread.sleep(ms); } catch (InterruptedException e) { - Logger.defaultLogError(e); + LOGGER.warn("Sleep was interrupted.", e); } } diff --git a/bundles/org.simantics.modeling/src/org/simantics/modeling/scl/SCLRealm.java b/bundles/org.simantics.modeling/src/org/simantics/modeling/scl/SCLRealm.java index 99ac161a5..8ed8c3e0e 100644 --- a/bundles/org.simantics.modeling/src/org/simantics/modeling/scl/SCLRealm.java +++ b/bundles/org.simantics.modeling/src/org/simantics/modeling/scl/SCLRealm.java @@ -1,7 +1,5 @@ package org.simantics.modeling.scl; -import gnu.trove.map.hash.THashMap; - import java.io.IOException; import java.util.Map; import java.util.concurrent.ExecutorService; @@ -11,7 +9,6 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import org.simantics.Logger; import org.simantics.databoard.Bindings; import org.simantics.databoard.binding.Binding; import org.simantics.databoard.binding.mutable.Variant; @@ -24,8 +21,14 @@ import org.simantics.scl.runtime.function.Function; import org.simantics.scl.runtime.tuple.Tuple0; import org.simantics.simulator.variable.NodeManager; import org.simantics.simulator.variable.Realm; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import gnu.trove.map.hash.THashMap; public class SCLRealm implements Realm { + private static final Logger LOGGER = LoggerFactory.getLogger(SCLRealm.class); + public static final String SCL = "scl"; THashMap contextTypes = new THashMap(); @@ -118,7 +121,7 @@ public class SCLRealm implements Realm { try { runnable.run(); } catch (Throwable t) { - Logger.defaultLogError(t); + LOGGER.error("Runnable failed in syncExec.", t); } finally { } return; @@ -132,7 +135,7 @@ public class SCLRealm implements Realm { try { runnable.run(); } catch (Throwable t) { - Logger.defaultLogError(t); + LOGGER.error("Runnable failed in syncExec.", t); } finally { executorThread = oldThread; endSyncExec.release(); @@ -146,7 +149,7 @@ public class SCLRealm implements Realm { try { runnable.run(); } catch (Throwable t) { - Logger.defaultLogError(t); + LOGGER.error("Runnable failed in asyncExec.", t); } finally { } return; diff --git a/bundles/org.simantics.modeling/src/org/simantics/modeling/typicals/SyncTypicalTemplatesToInstances.java b/bundles/org.simantics.modeling/src/org/simantics/modeling/typicals/SyncTypicalTemplatesToInstances.java index cd3260513..96a33393a 100644 --- a/bundles/org.simantics.modeling/src/org/simantics/modeling/typicals/SyncTypicalTemplatesToInstances.java +++ b/bundles/org.simantics.modeling/src/org/simantics/modeling/typicals/SyncTypicalTemplatesToInstances.java @@ -24,7 +24,6 @@ import java.util.Map; import java.util.Set; import org.eclipse.core.runtime.IProgressMonitor; -import org.simantics.Logger; import org.simantics.Simantics; import org.simantics.databoard.Bindings; import org.simantics.db.ReadGraph; @@ -81,6 +80,8 @@ import org.simantics.utils.datastructures.MapSet; import org.simantics.utils.strings.AlphanumComparator; import org.simantics.utils.strings.EString; import org.simantics.utils.ui.ErrorLogger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import gnu.trove.map.hash.THashMap; import gnu.trove.set.hash.THashSet; @@ -102,6 +103,7 @@ import gnu.trove.set.hash.THashSet; * @see TypicalSynchronizationMetadata */ public class SyncTypicalTemplatesToInstances extends WriteRequest { + private static final Logger LOGGER = LoggerFactory.getLogger(SyncTypicalTemplatesToInstances.class); /** * A constant used as the second argument to @@ -423,8 +425,7 @@ public class SyncTypicalTemplatesToInstances extends WriteRequest { syncInstance(graph, template, instance, templateElements); } } catch (Exception e) { - Logger.defaultLogError(e); - e.printStackTrace(); + LOGGER.error("Template synchronization failed.", e); } finally { this.temporaryDiagram.removeHint(DiagramModelHints.KEY_DIAGRAM_RESOURCE); } diff --git a/bundles/org.simantics.project/META-INF/MANIFEST.MF b/bundles/org.simantics.project/META-INF/MANIFEST.MF index ec287cffc..42e503aab 100644 --- a/bundles/org.simantics.project/META-INF/MANIFEST.MF +++ b/bundles/org.simantics.project/META-INF/MANIFEST.MF @@ -33,12 +33,12 @@ Require-Bundle: org.simantics.db.layer0;bundle-version="[0.8.0,2.0.0)", org.eclipse.equinox.p2.touchpoint.natives;bundle-version="1.0.200", org.eclipse.equinox.p2.transport.ecf;bundle-version="1.0.0", org.simantics.graph.db;bundle-version="1.0.0", - org.apache.log4j;bundle-version="1.2.15", org.simantics.db.procore;bundle-version="1.1.0", org.eclipse.swt;bundle-version="3.6.0";resolution:=optional, org.eclipse.core.resources;bundle-version="3.6.0";resolution:=optional, org.simantics.graph.compiler;bundle-version="1.1.10", - org.simantics.ltk;bundle-version="1.1.10" + org.simantics.ltk;bundle-version="1.1.10", + org.slf4j.api Export-Package: org.eclipse.equinox.internal.p2.ui.query, org.eclipse.equinox.internal.provisional.p2.installer, org.simantics.project, diff --git a/bundles/org.simantics.project/src/org/simantics/project/management/ServerManager.java b/bundles/org.simantics.project/src/org/simantics/project/management/ServerManager.java index 56cf3254c..205706c48 100644 --- a/bundles/org.simantics.project/src/org/simantics/project/management/ServerManager.java +++ b/bundles/org.simantics.project/src/org/simantics/project/management/ServerManager.java @@ -22,7 +22,6 @@ import java.util.HashMap; import java.util.Map; import java.util.Properties; -import org.apache.log4j.Logger; import org.simantics.databoard.util.StreamUtil; import org.simantics.db.Driver; import org.simantics.db.Driver.Management; @@ -40,6 +39,8 @@ import org.simantics.db.service.XSupport; import org.simantics.graph.db.CoreInitialization; import org.simantics.layer0.DatabaseManagementResource; import org.simantics.layer0.Layer0; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Server Manager handles starting and pooling of ProCore server instances. @@ -47,7 +48,8 @@ import org.simantics.layer0.Layer0; * @author Toni Kalajainen */ public class ServerManager { - + private static final Logger LOGGER = LoggerFactory.getLogger(ServerManager.class); + /** Default properties with default user and password */ public static final Properties DEFAULT; @@ -81,8 +83,7 @@ public class ServerManager { */ public Session createDatabase(File databaseDirectory) throws DatabaseException { try { - Logger myLogger = Logger.getLogger(ServerManager.class); - myLogger.debug("Creating database to "+ databaseDirectory); + LOGGER.debug("Creating database to "+ databaseDirectory); Session session = null; ServerEx server1 = getServer(databaseDirectory); @@ -206,8 +207,7 @@ public class ServerManager { if (server.isActive()) server.stop(); } catch (DatabaseException e) { - Logger myLogger = Logger.getLogger(ServerManager.class); - myLogger.error(e); + LOGGER.error("Failed to stop database server.", e); } } servers.clear(); diff --git a/bundles/org.simantics.project/src/org/simantics/project/management/ServerManagerFactory.java b/bundles/org.simantics.project/src/org/simantics/project/management/ServerManagerFactory.java index bbeb7541e..65190614c 100644 --- a/bundles/org.simantics.project/src/org/simantics/project/management/ServerManagerFactory.java +++ b/bundles/org.simantics.project/src/org/simantics/project/management/ServerManagerFactory.java @@ -22,19 +22,21 @@ import java.net.URLDecoder; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; -import org.apache.log4j.Logger; import org.simantics.db.DatabaseUserAgent; import org.simantics.db.Driver; import org.simantics.db.Manager; import org.simantics.db.exception.DatabaseException; -import org.simantics.utils.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class ServerManagerFactory { + private static final Logger LOGGER = LoggerFactory.getLogger(ServerManagerFactory.class); + public static ServerManager create(String databaseDriverId, String address) throws IOException, DatabaseException { Driver driver = Manager.getDriver(databaseDriverId); if (driver == null) - throw new IllegalArgumentException("Database driver for ID " + databaseDriverId + " Could not be found!"); - System.out.println("ServerManagerFactory.create called with databaseId=" + databaseDriverId + " and driver is " + driver.toString()); + throw new IllegalArgumentException("Database driver with ID " + databaseDriverId + " could not be found!"); + LOGGER.info("ServerManagerFactory.create called with id {}, driver is {}.", databaseDriverId, driver); DatabaseUserAgent agent = Manager.getUserAgent(databaseDriverId); if (agent != null) driver.setDatabaseUserAgent(address, agent); @@ -166,7 +168,6 @@ public class ServerManagerFactory { * @throws IOException */ private static void extractZip(InputStream zipInput, File dst) throws IOException { - Logger myLogger = Logger.getLogger(FileUtils.class); byte[] buf = new byte[8192]; ZipInputStream zis = new ZipInputStream(zipInput); ZipEntry entry; @@ -175,7 +176,7 @@ public class ServerManagerFactory { while (entry != null) { // for each entry to be extracted String name = entry.getName(); - myLogger.debug("Extracting "+name); + LOGGER.debug("Extracting "+name); File file = new File(dst, name); if (entry.isDirectory()) diff --git a/bundles/org.simantics.scl.compiler/META-INF/MANIFEST.MF b/bundles/org.simantics.scl.compiler/META-INF/MANIFEST.MF index 389381c96..916e66cfd 100755 --- a/bundles/org.simantics.scl.compiler/META-INF/MANIFEST.MF +++ b/bundles/org.simantics.scl.compiler/META-INF/MANIFEST.MF @@ -10,7 +10,8 @@ Require-Bundle: gnu.trove3;bundle-version="3.0.0", org.objectweb.asm;bundle-version="[5.0.0,6.0.0)", org.objectweb.asm.commons;bundle-version="[5.0.0,6.0.0)", org.objectweb.asm.util;bundle-version="[5.0.0,6.0.0)" -Export-Package: org.simantics.scl.compiler.commands, +Export-Package: org.cojen.classfile, + org.simantics.scl.compiler.commands, org.simantics.scl.compiler.common.datatypes, org.simantics.scl.compiler.common.exceptions, org.simantics.scl.compiler.common.names, @@ -38,12 +39,15 @@ Export-Package: org.simantics.scl.compiler.commands, org.simantics.scl.compiler.environment.filter, org.simantics.scl.compiler.environment.specification, org.simantics.scl.compiler.errors, + org.simantics.scl.compiler.internal.elaboration.constraints2, + org.simantics.scl.compiler.internal.elaboration.subsumption, org.simantics.scl.compiler.internal.parsing, org.simantics.scl.compiler.internal.parsing.exceptions, org.simantics.scl.compiler.internal.parsing.parser, org.simantics.scl.compiler.markdown.html, org.simantics.scl.compiler.markdown.inlines, org.simantics.scl.compiler.markdown.internal, + org.simantics.scl.compiler.markdown.nodes, org.simantics.scl.compiler.module, org.simantics.scl.compiler.module.coverage, org.simantics.scl.compiler.module.options, diff --git a/bundles/org.simantics.scl.compiler/src/org/simantics/scl/compiler/elaboration/expressions/EBlock.java b/bundles/org.simantics.scl.compiler/src/org/simantics/scl/compiler/elaboration/expressions/EBlock.java index 295f3c176..ff1312540 100755 --- a/bundles/org.simantics.scl.compiler/src/org/simantics/scl/compiler/elaboration/expressions/EBlock.java +++ b/bundles/org.simantics.scl.compiler/src/org/simantics/scl/compiler/elaboration/expressions/EBlock.java @@ -1,98 +1,100 @@ -package org.simantics.scl.compiler.elaboration.expressions; - -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; - -import org.simantics.scl.compiler.common.exceptions.InternalCompilerError; -import org.simantics.scl.compiler.elaboration.contexts.TranslationContext; -import org.simantics.scl.compiler.elaboration.expressions.block.GuardStatement; -import org.simantics.scl.compiler.elaboration.expressions.block.LetStatement; -import org.simantics.scl.compiler.elaboration.expressions.block.RuleStatement; -import org.simantics.scl.compiler.elaboration.expressions.block.Statement; -import org.simantics.scl.compiler.errors.Locations; - -public class EBlock extends ASTExpression { - - LinkedList statements = new LinkedList(); - boolean monadic; - - public EBlock() { - } - - public void addStatement(Statement statement) { - statements.add(statement); - } - - public void setMonadic(boolean monadic) { - this.monadic = monadic; - } - - public LinkedList getStatements() { - return statements; - } - - @Override - public Expression resolve(TranslationContext context) { - if(statements.isEmpty()) - throw new InternalCompilerError(); - int i = statements.size()-1; - Statement last = statements.get(i); - if(!(last instanceof GuardStatement)) { - context.getErrorLog().log(last.location, "Block should end with an expression"); - return new EError(location); - } - - Expression in = ((GuardStatement)last).value; - while(--i >= 0) { - Statement cur = statements.get(i); - if(cur instanceof RuleStatement) { - int endId = i+1; - while(i>0 && statements.get(i-1) instanceof RuleStatement) - --i; - in = extractRules(i, endId, in); - } - else if(cur instanceof LetStatement && ((LetStatement)cur).pattern.isFunctionPattern()) { - int endId = i+1; - while(i>0 && (cur = statements.get(i-1)) instanceof LetStatement && - ((LetStatement)cur).pattern.isFunctionPattern()) - --i; - in = extractLet(i, endId, in); - } - else - in = cur.toExpression(context, monadic, in); - } - return in.resolve(context); - } - - private Expression extractRules(int begin, int end, Expression in) { - return new EPreRuleset(statements.subList(begin, end).toArray(new RuleStatement[end-begin]), in); - } - - @SuppressWarnings("unchecked") - private Expression extractLet(int begin, int end, Expression in) { - return new EPreLet((List)(List)statements.subList(begin, end), in); - } - - public static Expression create(ArrayList statements) { - EBlock block = new EBlock(); - for(Expression statement : statements) - block.addStatement(new GuardStatement(statement)); - return block; - } - - @Override - public void setLocationDeep(long loc) { - if(location == Locations.NO_LOCATION) { - location = loc; - for(Statement statement : statements) - statement.setLocationDeep(loc); - } - } - - @Override - public Expression accept(ExpressionTransformer transformer) { - return transformer.transform(this); - } - -} +package org.simantics.scl.compiler.elaboration.expressions; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; + +import org.simantics.scl.compiler.common.exceptions.InternalCompilerError; +import org.simantics.scl.compiler.elaboration.contexts.TranslationContext; +import org.simantics.scl.compiler.elaboration.expressions.block.GuardStatement; +import org.simantics.scl.compiler.elaboration.expressions.block.LetStatement; +import org.simantics.scl.compiler.elaboration.expressions.block.RuleStatement; +import org.simantics.scl.compiler.elaboration.expressions.block.Statement; +import org.simantics.scl.compiler.errors.Locations; + +public class EBlock extends ASTExpression { + + LinkedList statements = new LinkedList(); + boolean monadic; + + public EBlock() { + } + + public void addStatement(Statement statement) { + statements.add(statement); + } + + public void setMonadic(boolean monadic) { + this.monadic = monadic; + } + + public LinkedList getStatements() { + return statements; + } + + @Override + public Expression resolve(TranslationContext context) { + if(statements.isEmpty()) { + context.getErrorLog().log(location, "Block must contain at least one statement."); + return new EError(location); + } + int i = statements.size()-1; + Statement last = statements.get(i); + if(!(last instanceof GuardStatement)) { + context.getErrorLog().log(last.location, "Block should end with an expression"); + return new EError(location); + } + + Expression in = ((GuardStatement)last).value; + while(--i >= 0) { + Statement cur = statements.get(i); + if(cur instanceof RuleStatement) { + int endId = i+1; + while(i>0 && statements.get(i-1) instanceof RuleStatement) + --i; + in = extractRules(i, endId, in); + } + else if(cur instanceof LetStatement && ((LetStatement)cur).pattern.isFunctionPattern()) { + int endId = i+1; + while(i>0 && (cur = statements.get(i-1)) instanceof LetStatement && + ((LetStatement)cur).pattern.isFunctionPattern()) + --i; + in = extractLet(i, endId, in); + } + else + in = cur.toExpression(context, monadic, in); + } + return in.resolve(context); + } + + private Expression extractRules(int begin, int end, Expression in) { + return new EPreRuleset(statements.subList(begin, end).toArray(new RuleStatement[end-begin]), in); + } + + @SuppressWarnings("unchecked") + private Expression extractLet(int begin, int end, Expression in) { + return new EPreLet((List)(List)statements.subList(begin, end), in); + } + + public static Expression create(ArrayList statements) { + EBlock block = new EBlock(); + for(Expression statement : statements) + block.addStatement(new GuardStatement(statement)); + return block; + } + + @Override + public void setLocationDeep(long loc) { + if(location == Locations.NO_LOCATION) { + location = loc; + for(Statement statement : statements) + statement.setLocationDeep(loc); + } + } + + @Override + public Expression accept(ExpressionTransformer transformer) { + return transformer.transform(this); + } + +} diff --git a/bundles/org.simantics.scl.compiler/src/org/simantics/scl/compiler/elaboration/expressions/Expression.java b/bundles/org.simantics.scl.compiler/src/org/simantics/scl/compiler/elaboration/expressions/Expression.java index edf93841b..e01098c12 100755 --- a/bundles/org.simantics.scl.compiler/src/org/simantics/scl/compiler/elaboration/expressions/Expression.java +++ b/bundles/org.simantics.scl.compiler/src/org/simantics/scl/compiler/elaboration/expressions/Expression.java @@ -207,8 +207,6 @@ public abstract class Expression extends Symbol implements Typed { public abstract void collectFreeVariables(THashSet vars); public Expression simplify(SimplificationContext context) { - System.out.println("#############################"); - System.out.println(this); throw new InternalCompilerError(location, getClass().getSimpleName() + " does not support simplify method."); } diff --git a/bundles/org.simantics.scl.compiler/src/org/simantics/scl/compiler/elaboration/relations/TransitiveClosureRelation.java b/bundles/org.simantics.scl.compiler/src/org/simantics/scl/compiler/elaboration/relations/TransitiveClosureRelation.java index 26111e0b4..c9fa469d2 100644 --- a/bundles/org.simantics.scl.compiler/src/org/simantics/scl/compiler/elaboration/relations/TransitiveClosureRelation.java +++ b/bundles/org.simantics.scl.compiler/src/org/simantics/scl/compiler/elaboration/relations/TransitiveClosureRelation.java @@ -67,13 +67,9 @@ public class TransitiveClosureRelation extends AbstractRelation implements Compo type = type.replace(getTypeVariables(), typeParameters); Expression continuation = context.getContinuation(); - System.out.println("continuation = " + continuation + " :: " + continuation.getType()); Variable set = new Variable("set", Types.apply(Types.con("MSet", "T"), type)); Variable f = new Variable("f", Types.functionE(type, Types.PROC, continuation.getType())); Variable innerSolved = new Variable("tcTemp", solved.getType()); - System.out.println("set :: " + set.getType()); - System.out.println("f :: " + f.getType()); - System.out.println("tcTemp :: " + innerSolved.getType()); QueryCompilationContext newContext = context.createSubcontext(new EApply( new EVariable(f), new EVariable(innerSolved) diff --git a/bundles/org.simantics.scl.compiler/src/org/simantics/scl/compiler/top/ExpressionEvaluator.java b/bundles/org.simantics.scl.compiler/src/org/simantics/scl/compiler/top/ExpressionEvaluator.java index 5c8f59bb2..39f0ac721 100644 --- a/bundles/org.simantics.scl.compiler/src/org/simantics/scl/compiler/top/ExpressionEvaluator.java +++ b/bundles/org.simantics.scl.compiler/src/org/simantics/scl/compiler/top/ExpressionEvaluator.java @@ -182,7 +182,7 @@ public class ExpressionEvaluator { final Environment environment = runtimeEnvironment.getEnvironment(); // Parse expression - if(expressionText != null) { + if(expressionText != null && !expressionText.trim().isEmpty()) { try { switch(parseMode) { case BLOCK: { diff --git a/bundles/org.simantics.scl.data/scl/Data/Json.md b/bundles/org.simantics.scl.data/scl/Data/Json.md new file mode 100644 index 000000000..fe8e7218f --- /dev/null +++ b/bundles/org.simantics.scl.data/scl/Data/Json.md @@ -0,0 +1,42 @@ +# Basic functions + +::value[toJsonString, fromJsonString] + +# Supported value types + +This module supports the following value types: + +``` +instance Json String +instance Json Short +instance Json Integer +instance Json Long +instance Json Float +instance Json Double + +instance (Json a) => Json [a] +instance (Json a) => Json (Maybe a) + +instance Json () +instance (Json a, Json b) => Json (a, b) +instance (Json a, Json b, Json c) => Json (a, b, c) +instance (Json a, Json b, Json c, Json d) => Json (a, b, c, d) +instance (Json a, Json b, Json c, Json d, Json e) => Json (a, b, c, d, e) + +instance Json Json +``` + +# Generic JSON Type + +::data[Json, JsonField] + +# Adding support for additional value types + +::data[JsonGenerator, JsonParser] +::class[Json] + +It is enough to implement `toJson` and `fromJson`. + +# Undocumented entities + +::undocumented[] \ No newline at end of file diff --git a/bundles/org.simantics.scl.data/scl/Data/Json.scl.skip b/bundles/org.simantics.scl.data/scl/Data/Json.scl.skip new file mode 100644 index 000000000..fb034cd57 --- /dev/null +++ b/bundles/org.simantics.scl.data/scl/Data/Json.scl.skip @@ -0,0 +1,421 @@ +import "StandardLibrary" +import "Data/Writer" +import "JavaBuiltin" as Java + +importJava "com.fasterxml.jackson.core.JsonGenerator" where + data JsonGenerator + +@private +importJava "com.fasterxml.jackson.core.JsonGenerator" where + writeNull :: JsonGenerator -> () + + writeStartArray :: JsonGenerator -> () + @JavaName writeStartArray + writeStartArrayN :: JsonGenerator -> Integer -> () + writeEndArray :: JsonGenerator -> () + + writeStartObject :: JsonGenerator -> () + writeFieldName :: JsonGenerator -> String -> () + writeEndObject :: JsonGenerator -> () + + writeBoolean :: JsonGenerator -> Boolean -> () + + writeString :: JsonGenerator -> String -> () + + @JavaName writeNumber + writeNumberDouble :: JsonGenerator -> Double -> () + @JavaName writeNumber + writeNumberInteger :: JsonGenerator -> Integer -> () + @JavaName writeNumber + writeNumberLong :: JsonGenerator -> Long -> () + @JavaName writeNumber + writeNumberShort :: JsonGenerator -> Short -> () + @JavaName writeNumber + writeNumberFloat :: JsonGenerator -> Float -> () + + @JavaName close + closeGenerator :: JsonGenerator -> () + +@private +importJava "com.fasterxml.jackson.core.JsonToken" where + data JsonToken + END_ARRAY :: JsonToken + END_OBJECT :: JsonToken + FIELD_NAME :: JsonToken + NOT_AVAILABLE :: JsonToken + START_ARRAY :: JsonToken + START_OBJECT :: JsonToken + VALUE_EMBEDDED_OBJECT :: JsonToken + VALUE_FALSE :: JsonToken + VALUE_NULL :: JsonToken + VALUE_NUMBER_FLOAT :: JsonToken + VALUE_NUMBER_INT :: JsonToken + VALUE_STRING :: JsonToken + VALUE_TRUE :: JsonToken +instance Eq JsonToken where + (==) = Java.equals + +importJava "com.fasterxml.jackson.core.JsonParser" where + data JsonParser + +@private +importJava "com.fasterxml.jackson.core.JsonParser" where + nextToken :: JsonParser -> JsonToken + currentToken :: JsonParser -> JsonToken + getDoubleValue :: JsonParser -> Double + getIntValue :: JsonParser -> Integer + getText :: JsonParser -> String + getShortValue :: JsonParser -> Short + getFloatValue :: JsonParser -> Float + getLongValue :: JsonParser -> Long + nextFieldName :: JsonParser -> Maybe String + +@private +importJava "com.fasterxml.jackson.core.JsonFactory" where + data JsonFactory + + @JavaName "" + createJsonFactory :: JsonFactory + + @JavaName createGenerator + createWriterGenerator :: JsonFactory -> Writer -> JsonGenerator + + @JavaName createParser + createStringParser :: JsonFactory -> String -> JsonParser + +@private +defaultFactory = createJsonFactory + +@private +@inline +assertStartArray :: JsonParser -> () +assertStartArray p = if currentToken p == START_ARRAY + then () + else fail "Expected START_ARRAY token." + +@private +@inline +assertEndArray :: JsonParser -> () +assertEndArray p = if nextToken p == END_ARRAY + then () + else fail "Expected END_ARRAY token." + +// *** Json type class ******************************************************** + +class Json a where + writeJson :: JsonGenerator -> a -> () + readJson :: JsonParser -> a + toJson :: a -> Json + fromJson :: Json -> a + + writeJson g v = writeJson g (toJson v) + readJson p = fromJson (readJson p) + +@private +readNextJson :: Json a => JsonParser -> a +readNextJson p = do + nextToken p + readJson p + +""" +Converts the value to a string encoded with JSON +""" +toJsonString :: Json a => a -> String +toJsonString v = runProc do + writer = createStringWriter + generator = createWriterGenerator defaultFactory (toWriter writer) + writeJson generator v + closeGenerator generator + resultOfStringWriter writer + +""" +Parses a JSON encoded string into a value +""" +fromJsonString :: Json a => String -> a +fromJsonString str = runProc do + parser = createStringParser defaultFactory str + readNextJson parser + +instance Json String where + writeJson = writeString + readJson = getText + toJson = JsonString + fromJson (JsonString value) = value + +instance Json Boolean where + writeJson = writeBoolean + readJson p = + if currentToken p == VALUE_TRUE + then True + else False + toJson = JsonBoolean + fromJson (JsonBoolean value) = value + +instance Json Double where + writeJson = writeNumberDouble + readJson = getDoubleValue + toJson = JsonDouble + fromJson (JsonDouble value) = value + +instance Json Float where + writeJson = writeNumberFloat + readJson = getFloatValue + toJson = JsonDouble . toDouble + fromJson (JsonDouble value) = fromDouble value + +instance Json Integer where + writeJson = writeNumberInteger + readJson = getIntValue + toJson = JsonLong . fromInteger + fromJson (JsonLong value) = Java.l2i value + +instance Json Long where + writeJson = writeNumberLong + readJson = getLongValue + toJson = JsonLong + fromJson (JsonLong value) = value + +instance Json Short where + writeJson = writeNumberShort + readJson = getShortValue + toJson = JsonLong . Java.i2l . Java.s2i + fromJson (JsonLong value) = Java.i2s (Java.l2i value) + +instance (Json a) => Json (Maybe a) where + writeJson g (Just v) = writeJson g v + writeJson g Nothing = writeNull g + readJson p = + if currentToken p == VALUE_NULL + then Nothing + else Just (readJson p) + toJson (Just value) = toJson value + toJson Nothing = JsonNull + fromJson JsonNull = Nothing + fromJson json = Just (fromJson json) + +instance (Json a) => Json [a] where + writeJson g l = do + writeStartArray g + iter (writeJson g) l + writeEndArray g + readJson p = MList.freeze result + where + result = MList.create () + assertStartArray p + while (nextToken p != END_ARRAY) + (MList.add result $ readJson p) + toJson l = JsonArray (map toJson l) + fromJson (JsonArray l) = map fromJson l + +instance Json () where + writeJson g _ = do + writeStartArray g + writeEndArray g + readJson p = do + assertStartArray p + assertEndArray p + () + toJson _ = JsonArray [] + fromJson (JsonArray []) = () + +instance (Json a, Json b) => Json (a, b) where + writeJson g (a, b) = do + writeStartArray g + writeJson g a + writeJson g b + writeEndArray g + readJson p = (a, b) + where + assertStartArray p + a = readNextJson p + b = readNextJson p + assertEndArray p + toJson (a, b) = JsonArray [toJson a, toJson b] + fromJson (JsonArray [a, b]) = (fromJson a, fromJson b) + +instance (Json a, Json b, Json c) => Json (a, b, c) where + writeJson g (a, b, c) = do + writeStartArray g + writeJson g a + writeJson g b + writeJson g c + writeEndArray g + readJson p = (a, b, c) + where + assertStartArray p + a = readNextJson p + b = readNextJson p + c = readNextJson p + assertEndArray p + toJson (a, b, c) = JsonArray [toJson a, toJson b, toJson c] + fromJson (JsonArray [a, b, c]) = (fromJson a, fromJson b, fromJson c) + +instance (Json a, Json b, Json c, Json d) => Json (a, b, c, d) where + writeJson g (a, b, c, d) = do + writeStartArray g + writeJson g a + writeJson g b + writeJson g c + writeJson g d + writeEndArray g + readJson p = (a, b, c, d) + where + assertStartArray p + a = readNextJson p + b = readNextJson p + c = readNextJson p + d = readNextJson p + assertEndArray p + toJson (a, b, c, d) = JsonArray [toJson a, toJson b, toJson c, toJson d] + fromJson (JsonArray [a, b, c, d]) = (fromJson a, fromJson b, fromJson c, fromJson d) + +instance (Json a, Json b, Json c, Json d, Json e) => Json (a, b, c, d, e) where + writeJson g (a, b, c, d, e) = do + writeStartArray g + writeJson g a + writeJson g b + writeJson g c + writeJson g d + writeJson g e + writeEndArray g + readJson p = (a, b, c, d, e) + where + assertStartArray p + a = readNextJson p + b = readNextJson p + c = readNextJson p + d = readNextJson p + e = readNextJson p + assertEndArray p + toJson (a, b, c, d, e) = JsonArray [toJson a, toJson b, toJson c, toJson d, toJson e] + fromJson (JsonArray [a, b, c, d, e]) = (fromJson a, fromJson b, fromJson c, fromJson d, fromJson e) + +data Json = + JsonString String + | JsonDouble Double + | JsonLong Long + | JsonArray [Json] + | JsonBoolean Boolean + | JsonNull + | JsonObject [JsonField] +data JsonField = JsonField String Json + +deriving instance Show Json +deriving instance Eq Json +deriving instance Show JsonField +deriving instance Eq JsonField + +instance Json Json where + writeJson g (JsonString value) = writeString g value + writeJson g (JsonDouble value) = writeNumberDouble g value + writeJson g (JsonLong value) = writeNumberLong g value + writeJson g (JsonBoolean value) = writeBoolean g value + writeJson g JsonNull = writeNull g + writeJson g (JsonArray values) = do + writeStartArray g + iter (writeJson g) values + writeEndArray g + writeJson g (JsonObject fields) = do + writeStartObject g + iter (\(JsonField name value) -> do + writeFieldName g name + writeJson g value) fields + writeEndObject g + + readJson p = do + token = currentToken p + if token == VALUE_STRING + then JsonString (getText p) + else if token == VALUE_NUMBER_FLOAT + then JsonDouble (getDoubleValue p) + else if token == VALUE_NUMBER_INT + then JsonLong (getLongValue p) + else if token == VALUE_TRUE + then JsonBoolean True + else if token == VALUE_FALSE + then JsonBoolean False + else if token == VALUE_NULL + then JsonNull + else if token == START_ARRAY + then do + result = MList.create () + while (nextToken p != END_ARRAY) + (MList.add result $ readJson p) + JsonArray (MList.freeze result) + else if token == START_OBJECT + then do + result = MList.create () + readJsonObjectContents result p + JsonObject (MList.freeze result) + else fail "Unsupported token type." + toJson = id + fromJson = id + +@private +readJsonObjectContents :: MList.T JsonField -> JsonParser -> () +readJsonObjectContents result p = + match nextFieldName p with + Just name -> do + MList.add result $ JsonField name (readNextJson p) + readJsonObjectContents result p + Nothing -> () + +/* +@private +makeTypeEqual :: a -> a -> () +makeTypeEqual _ _ = () + +@private +testValue :: Json a => Show a => Eq a => a -> () +testValue v1 = do + v2 = toJsonString v1 + v3 = fromJsonString v2 + makeTypeEqual v1 v3 + print "\(v1) -> \(v2) -> \(v3)" + if v1 != v3 + then fail "Values differ" + else () + +testGenericJson :: String -> () +testGenericJson v1 = do + v2 = fromJsonString v1 :: Json + v3 = toJsonString v2 + print "\(v1) -> \(v2) -> \(v3)" + if v1 != v3 + then fail "Values differ" + else () + +testIt :: () +testIt = do + testValue "asd" + testValue True + testValue False + testValue (123 :: Short) + testValue (123 :: Integer) + testValue (123 :: Long) + testValue (123 :: Double) + testValue (123 :: Float) + testValue (Nothing :: Maybe String) + testValue (Just "asd") + testValue ["a", "b", "c"] + testValue [[],["a"],["b","c"]] + testValue () + testValue ("a", "b") + testValue ("a", "b", "c") + testValue ("a", "b", "c", "d") + testValue [Just "a", Nothing] + testValue [("a", "b"), ("c", "d")] + testValue (("a", "b"), ("c", "d")) + + testGenericJson "\"asd\"" + testGenericJson "123" + testGenericJson "123.0" + testGenericJson "true" + testGenericJson "false" + testGenericJson "null" + testGenericJson "[1,2,3]" + testGenericJson "[[1],[2,3],[]]" + testGenericJson "{}" + testGenericJson "{\"a\":123,\"b\":[]}" + testGenericJson "{\"a\":{}}" +*/ \ No newline at end of file diff --git a/bundles/org.simantics.scl.data/scl/Data/Writer.scl b/bundles/org.simantics.scl.data/scl/Data/Writer.scl new file mode 100644 index 000000000..4c526f2c1 --- /dev/null +++ b/bundles/org.simantics.scl.data/scl/Data/Writer.scl @@ -0,0 +1,18 @@ +import "JavaBuiltin" as Java + +importJava "java.io.Writer" where + data Writer + +importJava "java.io.StringWriter" where + data StringWriter + + @JavaName "" + createStringWriter :: StringWriter + + @JavaName toString + resultOfStringWriter :: StringWriter -> String + +class WriterLike a where + toWriter :: a -> Writer +instance WriterLike StringWriter where + toWriter = Java.unsafeCoerce \ No newline at end of file diff --git a/bundles/org.simantics.scl.runtime/META-INF/MANIFEST.MF b/bundles/org.simantics.scl.runtime/META-INF/MANIFEST.MF index 64d5ac63d..7274b0bf2 100755 --- a/bundles/org.simantics.scl.runtime/META-INF/MANIFEST.MF +++ b/bundles/org.simantics.scl.runtime/META-INF/MANIFEST.MF @@ -21,4 +21,5 @@ Export-Package: org.simantics.scl.runtime, org.simantics.scl.runtime.xml Require-Bundle: org.junit;bundle-version="4.12.0";resolution:=optional, gnu.trove3;bundle-version="3.0.0", - org.simantics.databoard;bundle-version="0.6.5";visibility:=reexport + org.simantics.databoard;bundle-version="0.6.5";visibility:=reexport, + org.slf4j.api;bundle-version="1.7.20" diff --git a/bundles/org.simantics.scl.runtime/scl/Expressions/Equations.scl b/bundles/org.simantics.scl.runtime/scl/Expressions/Equations.scl deleted file mode 100644 index e695a0089..000000000 --- a/bundles/org.simantics.scl.runtime/scl/Expressions/Equations.scl +++ /dev/null @@ -1,13 +0,0 @@ -effect Equation - "equation" - "org.simantics.scl.runtime.equations.EquationContext" - -importJava "org.simantics.scl.runtime.equations.EquationContext" where - data EquationContext - - listenEquationVariable :: String -> (a -> ()) -> () - setEquationVariable :: String -> a -> () - applyEquationContext :: ( a) -> EquationContext -> a - -importJava "org.simantics.scl.runtime.equations.TestEquationContext" where - solveEquations :: ( a) -> [(String,String)] \ No newline at end of file diff --git a/bundles/org.simantics.scl.runtime/scl/Prelude.scl b/bundles/org.simantics.scl.runtime/scl/Prelude.scl index b29b04a2f..84f8b91b6 100644 --- a/bundles/org.simantics.scl.runtime/scl/Prelude.scl +++ b/bundles/org.simantics.scl.runtime/scl/Prelude.scl @@ -1266,7 +1266,7 @@ instance MonadZero Maybe where mzero = Nothing instance MonadOr Maybe where - morelse (Just a) _ = Just a + morelse a@(Just _) _ = a morelse _ b = b "`execJust v f` executes the function `f` with parameter value `x`, if `v=Just x`. If `v=Nothing`, the function does nothing." diff --git a/bundles/org.simantics.scl.runtime/scl/StandardLibrary.scl b/bundles/org.simantics.scl.runtime/scl/StandardLibrary.scl index 6b43166e3..76ecece9c 100644 --- a/bundles/org.simantics.scl.runtime/scl/StandardLibrary.scl +++ b/bundles/org.simantics.scl.runtime/scl/StandardLibrary.scl @@ -15,7 +15,7 @@ include "MSet" as MSet include "MList" as MList include "MMultiMap" as MMultiMap include "Coercion" -include "Json2" +//include "Json2" include "IterN" as Extra diff --git a/bundles/org.simantics.scl.runtime/src/org/simantics/scl/runtime/equations/EquationContext.java b/bundles/org.simantics.scl.runtime/src/org/simantics/scl/runtime/equations/EquationContext.java deleted file mode 100644 index f20dd6fb3..000000000 --- a/bundles/org.simantics.scl.runtime/src/org/simantics/scl/runtime/equations/EquationContext.java +++ /dev/null @@ -1,20 +0,0 @@ -package org.simantics.scl.runtime.equations; - -import org.simantics.scl.runtime.SCLContext; -import org.simantics.scl.runtime.function.Function; -import org.simantics.scl.runtime.tuple.Tuple0; - -public interface EquationContext { - void listenEquationVariable(String variableName, Function callback); - void setEquationVariable(String variableName, Object value); - - public static Object applyEquationContext(Function f, EquationContext equationContext) { - SCLContext context = SCLContext.getCurrent(); - Object oldEquationContext = context.put("equation", equationContext); - try { - return f.apply(Tuple0.INSTANCE); - } finally { - context.put("equation", oldEquationContext); - } - } -} diff --git a/bundles/org.simantics.scl.runtime/src/org/simantics/scl/runtime/equations/TestEquationContext.java b/bundles/org.simantics.scl.runtime/src/org/simantics/scl/runtime/equations/TestEquationContext.java deleted file mode 100644 index 856ef2870..000000000 --- a/bundles/org.simantics.scl.runtime/src/org/simantics/scl/runtime/equations/TestEquationContext.java +++ /dev/null @@ -1,95 +0,0 @@ -package org.simantics.scl.runtime.equations; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import org.simantics.scl.runtime.SCLContext; -import org.simantics.scl.runtime.function.Function; -import org.simantics.scl.runtime.tuple.Tuple0; -import org.simantics.scl.runtime.tuple.Tuple2; - -import gnu.trove.map.hash.THashMap; -import gnu.trove.procedure.TObjectObjectProcedure; - -public class TestEquationContext implements EquationContext { - - public static final boolean TRACE = true; - - THashMap values = new THashMap(); - THashMap> listenerMap = new THashMap>(); - - @Override - public void listenEquationVariable(String variableName, Function listener) { - if(TRACE) - System.out.println("listenEquationVariable(" + variableName + ", " + listener + ")"); - if(values.containsKey(variableName)) { - Object value = values.get(variableName); - if(TRACE) - System.out.println(" apply " + value); - listener.apply(value); - } - else { - if(TRACE) - System.out.println(" add listener"); - ArrayList listeners = listenerMap.get(variableName); - if(listeners == null) { - listeners = new ArrayList(); - listenerMap.put(variableName, listeners); - } - listeners.add(listener); - } - } - - @Override - public void setEquationVariable(String variableName, Object value) { - if(TRACE) - System.out.println("setEquationVariable(" + variableName + ", " + value + ")"); - if(values.containsKey(variableName)) - throw new IllegalStateException("Value for " + variableName + " already defined (oldValue=" + values.get(variableName) + - ", newValue=" + value + ")."); - values.put(variableName, value); - ArrayList listeners = listenerMap.remove(variableName); - SCLContext context = SCLContext.getCurrent(); - if(listeners != null) { - Object oldEquationContex = context.put("equation", this); - try { - for(Function listener : listeners) { - if(TRACE) - System.out.println(" apply " + listener + " " + value); - listener.apply(value); - } - } finally { - context.put("equation", oldEquationContex); - } - } - } - - public static List solveEquations(Function f) { - TestEquationContext equationContext = new TestEquationContext(); - SCLContext context = SCLContext.getCurrent(); - Object oldEquationContext = context.put("equation", equationContext); - try { - f.apply(Tuple0.INSTANCE); - } finally { - context.put("equation", oldEquationContext); - } - ArrayList result = new ArrayList(equationContext.values.size()); - equationContext.values.forEachEntry(new TObjectObjectProcedure() { - @Override - public boolean execute(String a, Object b) { - result.add(new Tuple2(a, String.valueOf(b))); - return true; - } - }); - Collections.sort(result, (t1, t2) -> { - return ((String)t1.c0).compareTo((String)t2.c0); - }); - return result; - } - - public THashMap getValues() { - return values; - } - -} diff --git a/bundles/org.simantics.scl.runtime/src/org/simantics/scl/runtime/reporting/SCLReportingHandler.java b/bundles/org.simantics.scl.runtime/src/org/simantics/scl/runtime/reporting/SCLReportingHandler.java index a53595646..91e71d652 100755 --- a/bundles/org.simantics.scl.runtime/src/org/simantics/scl/runtime/reporting/SCLReportingHandler.java +++ b/bundles/org.simantics.scl.runtime/src/org/simantics/scl/runtime/reporting/SCLReportingHandler.java @@ -1,5 +1,8 @@ package org.simantics.scl.runtime.reporting; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** *

An interface that is used to handle printing commands from SCL * such as Prelude.print. This interface is typically stored to @@ -11,6 +14,7 @@ package org.simantics.scl.runtime.reporting; * @author Hannu Niemistö */ public interface SCLReportingHandler { + public static final Logger LOGGER = LoggerFactory.getLogger(SCLReportingHandler.class); public static final String REPORTING_HANDLER = "reportingHandler"; public void print(String text); @@ -19,6 +23,34 @@ public interface SCLReportingHandler { public void didWork(double amount); public static final SCLReportingHandler DEFAULT = new AbstractSCLReportingHandler() { + @Override + public void print(String text) { + LOGGER.info(text); + } + + @Override + public void printError(String error) { + LOGGER.error(error); + } + }; + + public static final SCLReportingHandler DEFAULT_WITHOUT_ECHO = new AbstractSCLReportingHandler() { + @Override + public void print(String text) { + LOGGER.info(text); + } + + @Override + public void printError(String error) { + LOGGER.error(error); + } + + @Override + public void printCommand(String command) { + } + }; + + public static final SCLReportingHandler SYSOUT = new AbstractSCLReportingHandler() { @Override public void print(String text) { System.out.println(text); @@ -30,7 +62,7 @@ public interface SCLReportingHandler { } }; - public static final SCLReportingHandler DEFAULT_WITHOUT_ECHO = new AbstractSCLReportingHandler() { + public static final SCLReportingHandler SYSOUT_WITHOUT_ECHO = new AbstractSCLReportingHandler() { @Override public void print(String text) { System.out.println(text); diff --git a/bundles/org.simantics.scl.ui/src/org/simantics/scl/ui/editor2/OpenDeclaration.java b/bundles/org.simantics.scl.ui/src/org/simantics/scl/ui/editor2/OpenDeclaration.java index f8ea0c20d..522b8d3ed 100644 --- a/bundles/org.simantics.scl.ui/src/org/simantics/scl/ui/editor2/OpenDeclaration.java +++ b/bundles/org.simantics.scl.ui/src/org/simantics/scl/ui/editor2/OpenDeclaration.java @@ -46,14 +46,27 @@ public class OpenDeclaration extends AbstractHandler { return text.substring(startPos, endPos); } - public static String extractAt(String text, int caretPos) { + public static String extractIdentifierOrSymbolAt(String text, int caretPos) { String result = extractIdentifierAt(text, caretPos); if(!result.isEmpty()) return result; return extractSymbolAt(text, caretPos); } - + private static String extractLineAt(String text, int caretPos) { + int startPos = caretPos; + while(startPos > 0 && !isNewline(text.charAt(startPos-1))) + --startPos; + int endPos = caretPos; + while(endPos < text.length() && !isNewline(text.charAt(endPos))) + ++endPos; + return text.substring(startPos, endPos); + } + + private static boolean isNewline(char c) { + return c=='\n' || c=='\r'; + } + @Override public Object execute(ExecutionEvent event) throws ExecutionException { IEditorPart editor = @@ -62,14 +75,28 @@ public class OpenDeclaration extends AbstractHandler { return null; SCLModuleEditor2 moduleEditor = (SCLModuleEditor2)editor; StyledText styledText = (StyledText)moduleEditor.getAdapter(Control.class); - String identifierAtCaret = extractAt(styledText.getText(), styledText.getCaretOffset()); - if(identifierAtCaret.isEmpty()) - return null; - SCLTextEditorEnvironment editorEnvironment = moduleEditor.getSCLTextEditorEnvironment(); - editorEnvironment.updateEnvironment(moduleEditor.getDocument()); - SCLValue value = editorEnvironment.getValue(identifierAtCaret); - if(value != null) - OpenSCLDefinition.openDefinition(value); + String text = styledText.getText(); + int caretOffset = styledText.getCaretOffset(); + + // Find the line where the caret is + String lineAtCaret = extractLineAt(text, caretOffset); + if(lineAtCaret.startsWith("import ") || lineAtCaret.startsWith("include ")) { + int p1 = lineAtCaret.indexOf('"', 6); + int p2 = lineAtCaret.indexOf('"', p1+1); + String moduleName = lineAtCaret.substring(p1+1, p2); + OpenSCLModule.openModule(moduleName); + } + else { + // Try to find an identifier at caret + String identifierAtCaret = extractIdentifierOrSymbolAt(text, caretOffset); + if(identifierAtCaret.isEmpty()) + return null; + SCLTextEditorEnvironment editorEnvironment = moduleEditor.getSCLTextEditorEnvironment(); + editorEnvironment.updateEnvironment(moduleEditor.getDocument()); + SCLValue value = editorEnvironment.getValue(identifierAtCaret); + if(value != null) + OpenSCLDefinition.openDefinition(value); + } return null; } diff --git a/bundles/org.simantics.scl.ui/src/org/simantics/scl/ui/editor2/OpenSCLModule.java b/bundles/org.simantics.scl.ui/src/org/simantics/scl/ui/editor2/OpenSCLModule.java index a9792a4d0..a0208e5b2 100644 --- a/bundles/org.simantics.scl.ui/src/org/simantics/scl/ui/editor2/OpenSCLModule.java +++ b/bundles/org.simantics.scl.ui/src/org/simantics/scl/ui/editor2/OpenSCLModule.java @@ -1,31 +1,35 @@ -package org.simantics.scl.ui.editor2; - -import org.eclipse.core.commands.AbstractHandler; -import org.eclipse.core.commands.ExecutionEvent; -import org.eclipse.core.commands.ExecutionException; -import org.eclipse.ui.IWorkbenchPage; -import org.eclipse.ui.PartInitException; -import org.eclipse.ui.PlatformUI; - -public class OpenSCLModule extends AbstractHandler { - - @Override - public Object execute(ExecutionEvent event) throws ExecutionException { - SCLModuleSelectionDialog dialog = new SCLModuleSelectionDialog( - PlatformUI.getWorkbench().getActiveWorkbenchWindow().getShell()); - if(dialog.open() == SCLModuleSelectionDialog.OK) { - String moduleName = (String)dialog.getFirstResult(); - IWorkbenchPage page = PlatformUI.getWorkbench().getActiveWorkbenchWindow().getActivePage(); - if(page == null) - return null; - SCLModuleEditorInput input = new StandardSCLModuleEditorInput(moduleName); - try { - page.openEditor(input, "org.simantics.scl.ui.editor2"); - } catch (PartInitException e) { - e.printStackTrace(); - } - } - return null; - } - -} +package org.simantics.scl.ui.editor2; + +import org.eclipse.core.commands.AbstractHandler; +import org.eclipse.core.commands.ExecutionEvent; +import org.eclipse.core.commands.ExecutionException; +import org.eclipse.ui.IWorkbenchPage; +import org.eclipse.ui.PartInitException; +import org.eclipse.ui.PlatformUI; + +public class OpenSCLModule extends AbstractHandler { + + @Override + public Object execute(ExecutionEvent event) throws ExecutionException { + SCLModuleSelectionDialog dialog = new SCLModuleSelectionDialog( + PlatformUI.getWorkbench().getActiveWorkbenchWindow().getShell()); + if(dialog.open() == SCLModuleSelectionDialog.OK) { + String moduleName = (String)dialog.getFirstResult(); + openModule(moduleName); + } + return null; + } + + public static void openModule(String moduleName) { + IWorkbenchPage page = PlatformUI.getWorkbench().getActiveWorkbenchWindow().getActivePage(); + if(page == null) + return; + SCLModuleEditorInput input = new StandardSCLModuleEditorInput(moduleName); + try { + page.openEditor(input, "org.simantics.scl.ui.editor2"); + } catch (PartInitException e) { + e.printStackTrace(); + } + } + +} diff --git a/bundles/org.simantics.selectionview/META-INF/MANIFEST.MF b/bundles/org.simantics.selectionview/META-INF/MANIFEST.MF index 606078e6c..1652bea8f 100644 --- a/bundles/org.simantics.selectionview/META-INF/MANIFEST.MF +++ b/bundles/org.simantics.selectionview/META-INF/MANIFEST.MF @@ -21,7 +21,8 @@ Require-Bundle: org.eclipse.ui, org.simantics.views.ontology;bundle-version="1.0.0", org.simantics.browsing.ui.model;bundle-version="1.0.0", org.simantics.modeling;bundle-version="1.1.1", - org.simantics.scenegraph.ontology;bundle-version="1.0.0" + org.simantics.scenegraph.ontology;bundle-version="1.0.0", + org.slf4j.api Bundle-RequiredExecutionEnvironment: JavaSE-1.8 Bundle-ActivationPolicy: lazy Export-Package: org.simantics.selectionview diff --git a/bundles/org.simantics.selectionview/src/org/simantics/selectionview/StandardSelectionProcessor.java b/bundles/org.simantics.selectionview/src/org/simantics/selectionview/StandardSelectionProcessor.java index 7b5cdf98c..ccf4eeb9f 100644 --- a/bundles/org.simantics.selectionview/src/org/simantics/selectionview/StandardSelectionProcessor.java +++ b/bundles/org.simantics.selectionview/src/org/simantics/selectionview/StandardSelectionProcessor.java @@ -6,7 +6,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.Set; -import org.simantics.Logger; +import org.simantics.Simantics; import org.simantics.db.ReadGraph; import org.simantics.db.Resource; import org.simantics.db.common.request.PossibleIndexRoot; @@ -21,8 +21,11 @@ import org.simantics.db.layer0.variable.Variables; import org.simantics.layer0.Layer0; import org.simantics.ui.selection.WorkbenchSelectionUtils; import org.simantics.utils.ui.ISelectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class StandardSelectionProcessor implements SelectionProcessor { + private static final Logger LOGGER = LoggerFactory.getLogger(StandardSelectionProcessor.class); private static Resource getIndexRoot(ReadGraph graph, Object selection) throws DatabaseException { @@ -113,10 +116,8 @@ public class StandardSelectionProcessor implements SelectionProcessor { + private static final Logger LOGGER = LoggerFactory.getLogger(CellValue.class); + public CellValue(Variable variable) { super(variable); } @@ -38,8 +41,9 @@ public class CellValue extends VariableRead { Binding binding = Bindings.getBinding( value2.getClass() ); return new Variant(binding, value2); } catch (BindingConstructionException e) { - Logger.defaultLogError(e); - return Variant.ofInstance("Unsupported content " + value2 + " from " + var.getURI(graph)); + String msg = "Unsupported content " + value2 + " from " + var.getURI(graph); + LOGGER.error(msg, e); + return Variant.ofInstance(msg); } } } @@ -48,8 +52,9 @@ public class CellValue extends VariableRead { Binding binding = Bindings.getBinding( value.getClass() ); return new Variant(binding, value); } catch (BindingConstructionException e) { - Logger.defaultLogError(e); - return Variant.ofInstance("Unsupported content " + value + " at " + variable.getURI(graph)); + String msg = "Unsupported content " + value + " at " + variable.getURI(graph); + LOGGER.error(msg, e); + return Variant.ofInstance(msg); } } } diff --git a/bundles/org.simantics.spreadsheet.graph/src/org/simantics/spreadsheet/graph/ExcelImport.java b/bundles/org.simantics.spreadsheet.graph/src/org/simantics/spreadsheet/graph/ExcelImport.java index 81aa67eaa..444054c8b 100644 --- a/bundles/org.simantics.spreadsheet.graph/src/org/simantics/spreadsheet/graph/ExcelImport.java +++ b/bundles/org.simantics.spreadsheet.graph/src/org/simantics/spreadsheet/graph/ExcelImport.java @@ -398,7 +398,7 @@ public class ExcelImport { XSSFColor fontColor = xssfFont.getXSSFColor(); if (fontColor != null) { - byte[] rgb = fontColor.getRgbWithTint(); + byte[] rgb = fontColor.getRGBWithTint(); String ix = fontColor.getARGBHex(); RGB.Integer s = hex2Rgb(ix); diff --git a/bundles/org.simantics.spreadsheet.graph/src/org/simantics/spreadsheet/graph/GraphUI.java b/bundles/org.simantics.spreadsheet.graph/src/org/simantics/spreadsheet/graph/GraphUI.java index 9a3e87326..b2fc404f2 100644 --- a/bundles/org.simantics.spreadsheet.graph/src/org/simantics/spreadsheet/graph/GraphUI.java +++ b/bundles/org.simantics.spreadsheet.graph/src/org/simantics/spreadsheet/graph/GraphUI.java @@ -61,7 +61,6 @@ import org.simantics.spreadsheet.CellEditor; import org.simantics.spreadsheet.ClientModel; import org.simantics.spreadsheet.ClientModel.OperationMode; import org.simantics.spreadsheet.SheetCommands; -import org.simantics.spreadsheet.common.logging.Logger; import org.simantics.spreadsheet.event.model.RemoveCellHandler; import org.simantics.spreadsheet.resource.SpreadsheetResource; import org.simantics.ui.selection.WorkbenchSelectionUtils; @@ -69,6 +68,8 @@ import org.simantics.utils.datastructures.Pair; import org.simantics.utils.strings.AlphanumComparator; import org.simantics.utils.threads.logger.ITask; import org.simantics.utils.threads.logger.ThreadLogger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import gnu.trove.map.hash.THashMap; import gnu.trove.map.hash.TObjectIntHashMap; @@ -102,6 +103,8 @@ class FilteredVariableProperties extends UnaryRead(variable) { + this.cellEditor = processor.sync(new VariableRead>(variable) { @Override - public CellEditor perform(ReadGraph graph) throws DatabaseException { + public CellEditor perform(ReadGraph graph) throws DatabaseException { SpreadsheetResource SHEET = SpreadsheetResource.getInstance(graph); return variable.getPropertyValue(graph, SHEET.cellEditor); } @@ -338,6 +341,8 @@ public class GraphUI implements Adaptable, ListenerSupport, AsyncListenerSupport private static class PropertyListener extends SingleSetSyncListenerDelegate> { + private static final Logger LOGGER = LoggerFactory.getLogger(PropertyListener.class); + private ClientModel client; private String childName; private boolean listenerDisposed; @@ -370,7 +375,7 @@ public class GraphUI implements Adaptable, ListenerSupport, AsyncListenerSupport @Override public void exception(ReadGraph graph, Throwable throwable) throws DatabaseException { - Logger.defaultLogError(throwable); + LOGGER.error("PropertyListener.exception", throwable); String propertyName = property.first; if("content".equals(propertyName)) { @@ -492,7 +497,7 @@ public class GraphUI implements Adaptable, ListenerSupport, AsyncListenerSupport return (T)new CellEditor() { @Override - public void edit(Transaction transaction, String location, String property, T value, Binding binding, Consumer callback) { + public void edit(Transaction transaction, String location, String property, E value, Binding binding, Consumer callback) { if (ClientModel.ITERATION_ENABLED.equals(location)) { Simantics.getSession().asyncRequest(new ReadRequest() { @@ -539,7 +544,7 @@ public class GraphUI implements Adaptable, ListenerSupport, AsyncListenerSupport load(newContext, client); return; } catch (DatabaseException e) { - Logger.defaultLogError(e); + LOGGER.error("edit failed for model key '" + ClientModel.CONTEXT_CURRENT + "'", e); } } } @@ -564,7 +569,7 @@ public class GraphUI implements Adaptable, ListenerSupport, AsyncListenerSupport load(newInput, client); return; } catch (DatabaseException e) { - Logger.defaultLogError(e); + LOGGER.error("edit failed for model key '" + ClientModel.SHEETS_CURRENT + "'", e); } } } @@ -610,7 +615,7 @@ public class GraphUI implements Adaptable, ListenerSupport, AsyncListenerSupport // fullSynchronize(); return; } catch (DatabaseException e) { - Logger.defaultLogError(e); + LOGGER.error("edit failed for model key '" + ClientModel.STATES_CURRENT + "'", e); } } } @@ -664,7 +669,7 @@ public class GraphUI implements Adaptable, ListenerSupport, AsyncListenerSupport } } catch (DatabaseException e) { - Logger.defaultLogError(e); + LOGGER.error("edit failed for model key '" + ClientModel.SOURCES_CURRENT + "'", e); } } return; @@ -796,12 +801,12 @@ public class GraphUI implements Adaptable, ListenerSupport, AsyncListenerSupport @Override public void exception(AsyncReadGraph graph, Throwable t) { - Logger.defaultLogError("Failed to read properties.", t); + LOGGER.error("Failed to read properties.", t); } @Override public void exception(ReadGraph graph, Throwable t) { - Logger.defaultLogError("Failed to read properties.", t); + LOGGER.error("Failed to read properties.", t); } public void dispose() { diff --git a/bundles/org.simantics.structural2/META-INF/MANIFEST.MF b/bundles/org.simantics.structural2/META-INF/MANIFEST.MF index 94409e4d2..ad3c50fac 100644 --- a/bundles/org.simantics.structural2/META-INF/MANIFEST.MF +++ b/bundles/org.simantics.structural2/META-INF/MANIFEST.MF @@ -23,3 +23,4 @@ Export-Package: org.simantics.structural2, Bundle-Vendor: VTT Technical Research Centre of Finland Bundle-ActivationPolicy: lazy Bundle-Activator: org.simantics.structural2.internal.Activator +Import-Package: org.slf4j diff --git a/bundles/org.simantics.structural2/src/org/simantics/structural2/queries/ConnectionPointMapOfResource.java b/bundles/org.simantics.structural2/src/org/simantics/structural2/queries/ConnectionPointMapOfResource.java index 2dff0a740..2924b3dac 100644 --- a/bundles/org.simantics.structural2/src/org/simantics/structural2/queries/ConnectionPointMapOfResource.java +++ b/bundles/org.simantics.structural2/src/org/simantics/structural2/queries/ConnectionPointMapOfResource.java @@ -11,8 +11,6 @@ *******************************************************************************/ package org.simantics.structural2.queries; -import gnu.trove.map.hash.THashMap; - import java.util.Collection; import java.util.Collections; import java.util.Map; @@ -25,9 +23,15 @@ import org.simantics.db.exception.DatabaseException; import org.simantics.db.layer0.request.PropertyInfo; import org.simantics.db.request.RequestFlags; import org.simantics.db.service.QueryControl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import gnu.trove.map.hash.THashMap; public class ConnectionPointMapOfResource extends TransientResourceRead> { + private static final Logger LOGGER = LoggerFactory.getLogger(ConnectionPointMapOfResource.class); + public ConnectionPointMapOfResource(ReadGraph graph, Resource resource) throws DatabaseException { super(graph, resource); } @@ -49,7 +53,7 @@ public class ConnectionPointMapOfResource extends TransientResourceRead(predicates.size()); if (result.put(info.name, predicate) != null) - System.err.println(this + ": The database contains siblings with the same name " + info.name + " (resource=$" + resource.getResourceId() + ")."); + LOGGER.error("The database contains siblings with the same name " + info.name + " (resource=$" + resource.getResourceId() + ")."); } } diff --git a/bundles/org.simantics.ui/META-INF/MANIFEST.MF b/bundles/org.simantics.ui/META-INF/MANIFEST.MF index 73c65c06f..4b043cee1 100644 --- a/bundles/org.simantics.ui/META-INF/MANIFEST.MF +++ b/bundles/org.simantics.ui/META-INF/MANIFEST.MF @@ -29,7 +29,8 @@ Require-Bundle: org.eclipse.core.runtime, org.eclipse.e4.ui.workbench, org.eclipse.e4.ui.di, org.eclipse.e4.ui.bindings, - org.simantics.modeling.ontology;bundle-version="1.2.0" + org.simantics.modeling.ontology;bundle-version="1.2.0", + org.slf4j.api Bundle-ActivationPolicy: lazy Export-Package: org.simantics.ui, org.simantics.ui.colors, diff --git a/bundles/org.simantics.ui/src/org/simantics/ui/workbench/editor/GraphEditorAdapterDescriptor.java b/bundles/org.simantics.ui/src/org/simantics/ui/workbench/editor/GraphEditorAdapterDescriptor.java index 3ffa5b457..b5bb6191c 100644 --- a/bundles/org.simantics.ui/src/org/simantics/ui/workbench/editor/GraphEditorAdapterDescriptor.java +++ b/bundles/org.simantics.ui/src/org/simantics/ui/workbench/editor/GraphEditorAdapterDescriptor.java @@ -5,7 +5,6 @@ import java.util.Collection; import org.eclipse.jface.resource.ImageDescriptor; import org.eclipse.ui.PartInitException; import org.eclipse.ui.PlatformUI; -import org.simantics.Logger; import org.simantics.Simantics; import org.simantics.db.ReadGraph; import org.simantics.db.Resource; @@ -19,9 +18,12 @@ import org.simantics.modeling.ModelingResources; import org.simantics.ui.utils.ResourceAdaptionUtils; import org.simantics.ui.workbench.ResourceEditorInput2; import org.simantics.utils.ui.workbench.WorkbenchUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class GraphEditorAdapterDescriptor implements EditorAdapterDescriptor { - + private static final Logger LOGGER = LoggerFactory.getLogger(GraphEditorAdapterDescriptor.class); + private final String editorId; private final String label; private final ImageDescriptor imageDescriptor; @@ -95,7 +97,7 @@ public class GraphEditorAdapterDescriptor implements EditorAdapterDescriptor { try { WorkbenchUtils.openEditor(editorId, new ResourceEditorInput2(editorId, r, model, rvi)); } catch (PartInitException e) { - Logger.defaultLogError(e); + LOGGER.error("Failed to open the graph editor", e); } } }); diff --git a/bundles/org.simantics.utils.datastructures/build.properties b/bundles/org.simantics.utils.datastructures/build.properties index accbb69ba..05d60ae9b 100644 --- a/bundles/org.simantics.utils.datastructures/build.properties +++ b/bundles/org.simantics.utils.datastructures/build.properties @@ -12,4 +12,4 @@ source.. = src/ output.. = bin/ bin.includes = META-INF/,\ - . \ No newline at end of file + . diff --git a/bundles/org.simantics.utils/META-INF/MANIFEST.MF b/bundles/org.simantics.utils/META-INF/MANIFEST.MF index bb0a1bba0..2e9869928 100644 --- a/bundles/org.simantics.utils/META-INF/MANIFEST.MF +++ b/bundles/org.simantics.utils/META-INF/MANIFEST.MF @@ -14,6 +14,6 @@ Export-Package: org.simantics.utils, org.simantics.utils.xml Bundle-ClassPath: . Require-Bundle: org.junit;bundle-version="4.12.0";resolution:=optional, - org.apache.log4j;bundle-version="1.2.15", - org.simantics.databoard;bundle-version="0.6.2" + org.simantics.databoard;bundle-version="0.6.2", + org.slf4j.api;bundle-version="1.7.20" Bundle-RequiredExecutionEnvironment: JavaSE-1.8 diff --git a/bundles/org.simantics.utils/src/org/simantics/utils/FileUtils.java b/bundles/org.simantics.utils/src/org/simantics/utils/FileUtils.java index aae9666ab..51449d57b 100644 --- a/bundles/org.simantics.utils/src/org/simantics/utils/FileUtils.java +++ b/bundles/org.simantics.utils/src/org/simantics/utils/FileUtils.java @@ -47,7 +47,6 @@ import java.util.zip.ZipException; import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; -import org.apache.log4j.Logger; import org.simantics.databoard.Bindings; import org.simantics.databoard.adapter.AdaptException; import org.simantics.databoard.adapter.Adapter; @@ -56,6 +55,8 @@ import org.simantics.databoard.binding.Binding; import org.simantics.databoard.type.Datatype; import org.simantics.utils.bytes.LEInt; import org.simantics.utils.strings.FileNameUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Utilities for common file operations. @@ -68,6 +69,8 @@ import org.simantics.utils.strings.FileNameUtils; */ public class FileUtils { + private static final Logger LOGGER = LoggerFactory.getLogger(FileUtils.class); + /** * Create escaped filename * @@ -536,8 +539,8 @@ public class FileUtils { * false if some or all failed to be deleted */ public static boolean deleteDir(File dir) { - Logger myLogger = Logger.getLogger(FileUtils.class); - myLogger.debug("Deleting directory "+dir); + if (LOGGER.isDebugEnabled()) + LOGGER.debug("Deleting directory "+dir); boolean result = true; if (!dir.isDirectory()) return false; @@ -555,8 +558,8 @@ public class FileUtils { } public static String deleteDirs(File dir) { - Logger myLogger = Logger.getLogger(FileUtils.class); - myLogger.debug("Deleting directory "+dir); + if (LOGGER.isDebugEnabled()) + LOGGER.debug("Deleting directory "+dir); boolean result = true; if (!dir.isDirectory()) @@ -795,10 +798,8 @@ public class FileUtils { } public static void compressZip(String sourcePath, String zipDir) throws IOException { - Logger myLogger = Logger.getLogger(FileUtils.class); - myLogger.debug("Compressing file " + sourcePath + " to zip " + zipDir + "."); - - System.out.println("Compressing file " + sourcePath + " to zip " + zipDir + "."); + if (LOGGER.isDebugEnabled()) + LOGGER.debug("Compressing file " + sourcePath + " to zip " + zipDir + "."); File filesource = new File(sourcePath); URI base = filesource.toURI(); @@ -821,8 +822,7 @@ public class FileUtils { } } } finally { - myLogger.debug("Filecompression done."); - System.out.println("Filecompression done."); + LOGGER.debug("Filecompression done."); } } @@ -840,8 +840,8 @@ public class FileUtils { * @throws IOException */ public static void extractZip(File zipFile, File dst) throws IOException { - Logger myLogger = Logger.getLogger(FileUtils.class); - myLogger.debug("Extracting zip "+zipFile); + if (LOGGER.isDebugEnabled()) + LOGGER.debug("Extracting zip "+zipFile); try (FileInputStream fis = new FileInputStream(zipFile)) { extractZip(fis, dst); } @@ -855,7 +855,6 @@ public class FileUtils { * @throws IOException */ public static void extractZip(InputStream zipInput, File dst) throws IOException { - Logger myLogger = Logger.getLogger(FileUtils.class); byte[] buf = new byte[8192]; ZipInputStream zis = new ZipInputStream(zipInput); ZipEntry entry; @@ -864,7 +863,8 @@ public class FileUtils { while (entry != null) { // for each entry to be extracted String name = entry.getName(); - myLogger.debug("Extracting "+name); + if (LOGGER.isDebugEnabled()) + LOGGER.debug("Extracting "+name); File file = new File(dst, name); if (entry.isDirectory()) diff --git a/bundles/org.simantics.utils/src/org/simantics/utils/logging/LogManager.java b/bundles/org.simantics.utils/src/org/simantics/utils/logging/LogManager.java deleted file mode 100644 index 1ee88a32a..000000000 --- a/bundles/org.simantics.utils/src/org/simantics/utils/logging/LogManager.java +++ /dev/null @@ -1,151 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2007, 2010 Association for Decentralized Information Management - * in Industry THTH ry. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Eclipse Public License v1.0 - * which accompanies this distribution, and is available at - * http://www.eclipse.org/legal/epl-v10.html - * - * Contributors: - * VTT Technical Research Centre of Finland - initial API and implementation - *******************************************************************************/ -package org.simantics.utils.logging; - -import java.util.Properties; - -import org.apache.log4j.Hierarchy; -import org.apache.log4j.Level; -import org.apache.log4j.Logger; -import org.apache.log4j.PropertyConfigurator; -import org.apache.log4j.spi.LoggerFactory; -import org.apache.log4j.spi.RootLogger; - -/** - * This class encapsulates a Log4J Hierarchy and centralizes all Logger access. - */ -public class LogManager { - - private Hierarchy hierarchy; - - /** - * Creates a new LogManager. Saves the log and state location. - * Creates a new Hierarchy and add a new EventListener to it. - * Configure the hierarchy with the properties passed. Add this object to - * the list of active log managers. - * - * @param properties log configuration properties - */ - public LogManager(Properties properties) { - this.hierarchy = new Hierarchy(new RootLogger(Level.DEBUG)); - new PropertyConfigurator().doConfigure(properties, this.hierarchy); - } - - /** - * Checks if this PluginLogManager is disabled for this level. - * - * @param level level value - * @return boolean true if it is disabled - */ - public boolean isDisabled(int level) { - return this.hierarchy.isDisabled(level); - } - - /** - * Enable logging for logging requests with level l or higher. By default - * all levels are enabled. - * - * @param level level object - */ - public void setThreshold(Level level) { - this.hierarchy.setThreshold(level); - } - - /** - * The string version of setThreshold(Level level) - * - * @param level level string - */ - public void setThreshold(String level) { - this.hierarchy.setThreshold(level); - } - - /** - * Get the repository-wide threshold. - * - * @return Level - */ - public Level getThreshold() { - return this.hierarchy.getThreshold(); - } - - /** - * Returns a new logger instance named as the first parameter using the - * default factory. If a logger of that name already exists, then it will be - * returned. Otherwise, a new logger will be instantiated and then linked - * with its existing ancestors as well as children. - * - * @param clazz the class to get the logger for - * @return Logger - */ - public Logger getLogger(Class clazz) { - return this.hierarchy.getLogger(clazz.getName()); - } - - /** - * Returns a new logger instance named as the first parameter using the - * default factory. If a logger of that name already exists, then it will be - * returned. Otherwise, a new logger will be instantiated and then linked - * with its existing ancestors as well as children. - * - * @param name logger name - * @return Logger - */ - public Logger getLogger(String name) { - return this.hierarchy.getLogger(name); - } - - /** - * The same as getLogger(String name) but using a factory instance instead - * of a default factory. - * - * @param name logger name - * @param factory factory instance - * @return Logger - */ - public Logger getLogger(String name, LoggerFactory factory) { - return this.hierarchy.getLogger(name, factory); - } - - /** - * Returns the root of this hierarchy. - * - * @return Logger - */ - public Logger getRootLogger() { - return this.hierarchy.getRootLogger(); - } - - /** - * Checks if this logger exists. - * - * @return Logger - */ - public Logger exists(String name) { - return this.hierarchy.exists(name); - } - - /** - * Disposes the logger hierarchy - */ - public void shutdown() { - this.hierarchy.shutdown(); - } - - /** - * Resets configuration values to its defaults. - */ - public void resetConfiguration() { - this.hierarchy.resetConfiguration(); - } - -} \ No newline at end of file diff --git a/bundles/org.simantics.utils/src/org/simantics/utils/logging/Logger.java b/bundles/org.simantics.utils/src/org/simantics/utils/logging/Logger.java deleted file mode 100644 index 472f66c7d..000000000 --- a/bundles/org.simantics.utils/src/org/simantics/utils/logging/Logger.java +++ /dev/null @@ -1,108 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2007, 2010 Association for Decentralized Information Management - * in Industry THTH ry. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Eclipse Public License v1.0 - * which accompanies this distribution, and is available at - * http://www.eclipse.org/legal/epl-v10.html - * - * Contributors: - * VTT Technical Research Centre of Finland - initial API and implementation - *******************************************************************************/ -package org.simantics.utils.logging; - -import java.util.Properties; - -abstract public class Logger { - - public static final boolean ECHO = false; - - protected static final Properties defaultProperties(String ... keyValuePairs) { - - assert(keyValuePairs.length % 2 == 0); - - Properties defaultProperties = new Properties(); - defaultProperties.put("log4j.rootCategory", "ERROR, default"); - defaultProperties.put("log4j.appender.default", "org.apache.log4j.FileAppender"); - defaultProperties.put("log4j.appender.default.append", "false"); - defaultProperties.put("log4j.appender.default.layout", "org.apache.log4j.PatternLayout"); - defaultProperties.put("log4j.appender.default.layout.ConversionPattern", "%-6r [%15.15t] %-5p %30.30c - %m%n"); - - for(int i=0;inull - */ - public void logTrace(String message, Throwable exception) { - // Errors are much more useful with a stack trace! - if (exception == null) { - exception = new RuntimeException(); - } - getLogger().trace(message, exception); - } - - /** - * Log an info event. - * - * @param message message of the info - * @param exception the exception, or null - */ - public void logInfo(String message, Throwable exception) { - // Errors are much more useful with a stack trace! - if (exception == null) { - exception = new RuntimeException(); - } - getLogger().info(message, exception); - } - - /** - * Log an error event. - * - * @param message message of the error - * @param exception the exception, or null - */ - public void logError(String message, Throwable exception) { - - // Errors are much more useful with a stack trace! - if (exception == null) { - exception = new RuntimeException(); - } - getLogger().error(message, exception); - } - - /** - * Log an error event. - * - * @param message message of the error - * @param exception the exception, or null - */ - public void logWarning(String message, Throwable exception) { - // Errors are much more useful with a stack trace! - if (exception == null) { - exception = new RuntimeException(); - } - getLogger().error(message, exception); - } - -} diff --git a/bundles/org.simantics.workbench/META-INF/MANIFEST.MF b/bundles/org.simantics.workbench/META-INF/MANIFEST.MF index 76a575878..363faa7c9 100644 --- a/bundles/org.simantics.workbench/META-INF/MANIFEST.MF +++ b/bundles/org.simantics.workbench/META-INF/MANIFEST.MF @@ -6,8 +6,8 @@ Bundle-Version: 1.5.1.qualifier Bundle-Activator: org.simantics.workbench.internal.Activator Bundle-Vendor: VTT Technical Research Centre of Finland Bundle-Localization: plugin -Export-Package: org.simantics.workbench, - org.simantics.workbench.internal, +Export-Package: org.simantics.workbench, + org.simantics.workbench.internal, org.simantics.workbench.internal.contributions.search Require-Bundle: com.ibm.icu, org.eclipse.core.runtime, @@ -44,6 +44,8 @@ Require-Bundle: com.ibm.icu, org.eclipse.e4.core.di;bundle-version="1.5.0", org.eclipse.e4.ui.di;bundle-version="1.1.0", org.eclipse.e4.core.contexts, - org.eclipse.e4.ui.services + org.eclipse.e4.ui.services, + org.slf4j.api;bundle-version="1.7.20" Bundle-RequiredExecutionEnvironment: JavaSE-1.8 Bundle-ActivationPolicy: lazy +Bundle-ClassPath: . diff --git a/bundles/org.simantics.workbench/src/org/simantics/workbench/internal/SimanticsWorkbenchAdvisor.java b/bundles/org.simantics.workbench/src/org/simantics/workbench/internal/SimanticsWorkbenchAdvisor.java index d75f1e476..bce220c9c 100644 --- a/bundles/org.simantics.workbench/src/org/simantics/workbench/internal/SimanticsWorkbenchAdvisor.java +++ b/bundles/org.simantics.workbench/src/org/simantics/workbench/internal/SimanticsWorkbenchAdvisor.java @@ -1,1228 +1,1233 @@ -/******************************************************************************* - * Copyright (c) 2007, 2010 Association for Decentralized Information Management - * in Industry THTH ry. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Eclipse Public License v1.0 - * which accompanies this distribution, and is available at - * http://www.eclipse.org/legal/epl-v10.html - * - * Contributors: - * VTT Technical Research Centre of Finland - initial API and implementation - *******************************************************************************/ -package org.simantics.workbench.internal; - -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.net.URL; -import java.text.Collator; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.Map; -import java.util.TreeMap; - -import org.eclipse.core.internal.resources.Workspace; -import org.eclipse.core.net.proxy.IProxyService; -import org.eclipse.core.resources.IContainer; -import org.eclipse.core.resources.IResource; -import org.eclipse.core.resources.ResourcesPlugin; -import org.eclipse.core.resources.WorkspaceJob; -import org.eclipse.core.runtime.CoreException; -import org.eclipse.core.runtime.FileLocator; -import org.eclipse.core.runtime.IAdaptable; -import org.eclipse.core.runtime.IBundleGroup; -import org.eclipse.core.runtime.IBundleGroupProvider; -import org.eclipse.core.runtime.ILog; -import org.eclipse.core.runtime.IProgressMonitor; -import org.eclipse.core.runtime.IStatus; -import org.eclipse.core.runtime.MultiStatus; -import org.eclipse.core.runtime.Path; -import org.eclipse.core.runtime.Platform; -import org.eclipse.core.runtime.ProgressMonitorWrapper; -import org.eclipse.core.runtime.Status; -import org.eclipse.core.runtime.SubMonitor; -import org.eclipse.core.runtime.jobs.Job; -import org.eclipse.e4.core.contexts.IEclipseContext; -import org.eclipse.e4.ui.internal.workbench.E4Workbench; -import org.eclipse.jface.dialogs.ErrorDialog; -import org.eclipse.jface.dialogs.IDialogSettings; -import org.eclipse.jface.dialogs.MessageDialog; -import org.eclipse.jface.dialogs.TrayDialog; -import org.eclipse.jface.operation.IRunnableWithProgress; -import org.eclipse.jface.preference.IPreferenceStore; -import org.eclipse.jface.resource.ImageDescriptor; -import org.eclipse.jface.util.Policy; -import org.eclipse.jface.window.Window; -import org.eclipse.swt.SWT; -import org.eclipse.swt.events.SelectionAdapter; -import org.eclipse.swt.events.SelectionEvent; -import org.eclipse.swt.widgets.Composite; -import org.eclipse.swt.widgets.Display; -import org.eclipse.swt.widgets.Event; -import org.eclipse.swt.widgets.Listener; -import org.eclipse.swt.widgets.Shell; -import org.eclipse.ui.IPerspectiveDescriptor; -import org.eclipse.ui.PlatformUI; -import org.eclipse.ui.application.IWorkbenchConfigurer; -import org.eclipse.ui.application.IWorkbenchWindowConfigurer; -import org.eclipse.ui.application.WorkbenchAdvisor; -import org.eclipse.ui.application.WorkbenchWindowAdvisor; -import org.eclipse.ui.ide.IDE; -import org.eclipse.ui.internal.ISelectionConversionService; -import org.eclipse.ui.internal.Workbench; -import org.eclipse.ui.internal.ide.AboutInfo; -import org.eclipse.ui.internal.ide.IDEInternalPreferences; -import org.eclipse.ui.internal.ide.IDEInternalWorkbenchImages; -import org.eclipse.ui.internal.ide.IDESelectionConversionService; -import org.eclipse.ui.internal.ide.IDEWorkbenchActivityHelper; -import org.eclipse.ui.internal.ide.IDEWorkbenchErrorHandler; -import org.eclipse.ui.internal.ide.IDEWorkbenchMessages; -import org.eclipse.ui.internal.ide.IDEWorkbenchPlugin; -import org.eclipse.ui.internal.ide.undo.WorkspaceUndoMonitor; -import org.eclipse.ui.internal.progress.ProgressMonitorJobsDialog; -import org.eclipse.ui.keys.IBindingService; -import org.eclipse.ui.progress.IProgressService; -import org.eclipse.ui.statushandlers.AbstractStatusHandler; -import org.osgi.framework.Bundle; -import org.osgi.framework.ServiceReference; -import org.osgi.framework.Version; -import org.simantics.CancelStartupException; -import org.simantics.PlatformException; -import org.simantics.Simantics; -import org.simantics.SimanticsPlatform; -import org.simantics.SimanticsPlatform.OntologyRecoveryPolicy; -import org.simantics.SimanticsPlatform.RecoveryPolicy; -import org.simantics.application.arguments.IArguments; -import org.simantics.application.arguments.SimanticsArguments; -import org.simantics.db.common.Indexing; -import org.simantics.db.indexing.DatabaseIndexing; -import org.simantics.db.procore.server.environment.RebootRequiredException; -import org.simantics.db.procore.server.environment.windows.Product; -import org.simantics.internal.TimedSessionCache; -import org.simantics.project.IProject; -import org.simantics.project.ProjectKeys; -import org.simantics.ui.SimanticsUI; -import org.simantics.ui.jobs.SessionGarbageCollectorJob; -import org.simantics.ui.workbench.PerspectiveBarsActivator; -import org.simantics.ui.workbench.PerspectiveContextActivator; -import org.simantics.utils.logging.TimeLogger; -import org.simantics.utils.threads.ThreadUtils; -import org.simantics.utils.ui.dialogs.ShowError; -import org.simantics.utils.ui.dialogs.ShowMessage; - - -/** - * @author Tuukka Lehtonen - */ -public class SimanticsWorkbenchAdvisor extends WorkbenchAdvisor { - - private static final boolean PROFILE_PLATFORM_STARTUP = false; - - private static final String SHUT_DOWN_TASK = "Shutting down..."; - - private static final String SHUT_DOWN_PLATFORM_TASK = "Shutting down platform..."; - - private static final String WORKBENCH_PREFERENCE_CATEGORY_ID = "org.eclipse.ui.preferencePages.Workbench"; //$NON-NLS-1$ - - /** - * The dialog setting key to access the known installed features since the - * last time the workbench was run. - */ - private static final String INSTALLED_FEATURES = "installedFeatures"; //$NON-NLS-1$ - - /** - * The arguments received by the application. - */ - protected final IArguments args; - - protected final boolean restoredPreviousSession = false; - - /** - * Only true while opening the initial windows during {@link #openWindows()}. - * Used by {@link SimanticsWorkbenchWindowAdvisor#postWindowOpen()} to - * recognize when to skip all one-time initialization. - */ - protected boolean workbenchWindowsInitialized = false; - - /** - * Whether or not to save unsaved database changes before exiting the - * workbench. - */ - protected boolean saveAtExit = false; - - /** - * Ordered map of versioned feature ids -> info that are new for this - * session; null if uninitialized. Key type: - * String, Value type: AboutInfo. - */ - private Map newlyAddedBundleGroups; - - /** - * Array of AboutInfo for all new installed features that - * specify a welcome perspective. - */ - private AboutInfo[] welcomePerspectiveInfos = null; - - /** - * Helper for managing activites in response to workspace changes. - */ - private IDEWorkbenchActivityHelper activityHelper = null; - - /** - * Helper for managing work that is performed when the system is otherwise - * idle. - */ - private IDEIdleHelper idleHelper; - - private Listener settingsChangeListener; - - /** - * Support class for monitoring workspace changes and periodically - * validating the undo history - */ - private WorkspaceUndoMonitor workspaceUndoMonitor; - - /** - * The IDE workbench error handler. - */ - private AbstractStatusHandler ideWorkbenchErrorHandler; - - /** - * Helper class used to process delayed events. - */ - private DelayedEventsProcessor delayedEventsProcessor; - - /** - * Creates a new workbench advisor instance. - * @param processor - */ - public SimanticsWorkbenchAdvisor(IArguments args, DelayedEventsProcessor processor) { - super(); - this.args = args; - this.delayedEventsProcessor = processor; - - Listener closeListener = new Listener() { - public void handleEvent(Event event) { - boolean doExit = SimanticsWorkbenchWindowAdvisor.promptOnExit(null); - event.doit = doExit; - if (!doExit) - event.type = SWT.None; - } - }; - Display.getDefault().addListener(SWT.Close, closeListener); - } - - public IArguments getArguments() { - return args; - } - - public boolean workbenchInitialized() { - return workbenchWindowsInitialized; - } - - public boolean restoredPreviousSession() { - return restoredPreviousSession; - } - - boolean saveAtExit() { - return saveAtExit; - } - - void setSaveAtExit(boolean saveAtExit) { - this.saveAtExit = saveAtExit; - } - - /* - * (non-Javadoc) - * - * @see org.eclipse.ui.application.WorkbenchAdvisor#initialize - */ - @Override - public void initialize(IWorkbenchConfigurer configurer) { - // By default, we always save and restore the workbench state. - configurer.setSaveAndRestore(true); - - checkWorkspaceDatabaseIndexes(); - - // Start tracking the active perspective to activate contexts based on it. - new PerspectiveContextActivator(); - new PerspectiveBarsActivator(); - - // register workspace adapters - IDE.registerAdapters(); - - // register shared images - declareWorkbenchImages(); - - // initialize the activity helper - activityHelper = IDEWorkbenchActivityHelper.getInstance(); - - // initialize idle handler - idleHelper = new IDEIdleHelper(configurer); - - // initialize the workspace undo monitor - workspaceUndoMonitor = WorkspaceUndoMonitor.getInstance(); - - // show Help button in JFace dialogs - TrayDialog.setDialogHelpAvailable(true); - - Policy.setComparator(Collator.getInstance()); - } - - private void checkWorkspaceDatabaseIndexes() { - try { - DatabaseIndexing.validateIndexes(); - } catch (IOException e) { - Activator.logError("Problems encountered while checking database indexes, see exception for details.", e); - } - } - - public WorkbenchWindowAdvisor createWorkbenchWindowAdvisorClass(SimanticsWorkbenchAdvisor advisor, IWorkbenchWindowConfigurer configurer) { - return new SimanticsWorkbenchWindowAdvisor(this, configurer); - } - - @Override - public WorkbenchWindowAdvisor createWorkbenchWindowAdvisor(IWorkbenchWindowConfigurer configurer) { - // Attach database session watchdog. - new SessionWatchdog().attach( Simantics.getSessionContextProvider() ); - - return createWorkbenchWindowAdvisorClass(this, configurer); - } - - /** - * For gaining direct access to super.openWindows() in implementations - * inheriting this one. - */ - public boolean openWindowsSuper() { - return super.openWindows(); - } - - /** - * Sadly we do not know why key bindings are lost and why this helps. But it - * does. Visiting the Keys preference page and pressing OK uses - * this the same call and it seems to salvage the bindings that have been in - * some cases destroyed by BindingToModelProcessor. - * - *

- * Related links: - * https://techblog.ralph-schuster.eu/2013/10/13/eclipsee4-problem-with-key-bindings/comment-page-1/ - * https://www.eclipse.org/forums/index.php/t/550175/ - * https://bugs.eclipse.org/bugs/show_bug.cgi?id=461037 - * - * @see platform issue #6353 - */ - private void fixBindings() { - try { - IBindingService bs = PlatformUI.getWorkbench().getAdapter(IBindingService.class); - bs.savePreferences(bs.getActiveScheme(), bs.getBindings()); - } catch (IOException e) { - Activator.logError(getClass().getSimpleName() + ".fixBindings failed", e); - } - } - - @Override - public boolean openWindows() { - boolean platformOk = startPlatform(); - TimeLogger.log("SimanticsWorkbenchAdvisor.startPlatform finished"); - - if (platformOk) { - // At this point workbenchConfigurer.getSaveAndRestore() - // returns false iff something has gone terribly wrong - // before this. Currently saveAndRestore is always true. - boolean windowsOpened = super.openWindows(); - TimeLogger.log("Opened windows"); - if (windowsOpened) { - workbenchWindowsInitialized = true; - - // Start the database garbage collector after a short while. - SessionGarbageCollectorJob.getInstance().scheduleAfterQuietTime(); - - // Discard database session undo history at this point to prevent - // the user from undoing any initialization operations performed - // by the platform startup. - SimanticsPlatform.INSTANCE.discardSessionUndoHistory(); - TimeLogger.log("Discarded session undo history"); - - // #6353: Workaround for - fixBindings(); - - return true; - } - } - - // Make sure platform shutdown is ran if window opening fails. - try { - platformShutdownRunnable.run(null); - } catch (InvocationTargetException e) { - Activator.logError(getClass().getSimpleName() + ".openWindows failed", e.getCause()); - } catch (InterruptedException e) { - Activator.logError(getClass().getSimpleName() + ".openWindows failed", e); - } - return false; - } - - protected boolean startPlatform() { - // Verify selected perspective - if (args.contains(SimanticsArguments.PERSPECTIVE)) { - String perspectiveId = args.get(SimanticsArguments.PERSPECTIVE); - IPerspectiveDescriptor perspective = PlatformUI.getWorkbench().getPerspectiveRegistry().findPerspectiveWithId(perspectiveId); - if (perspective == null) { - StringBuilder msg = new StringBuilder(); - msg.append("Requested perspective not found: '" + perspectiveId + "'\n"); - msg.append("Valid alternatives are:\n"); - for (IPerspectiveDescriptor pd : PlatformUI.getWorkbench().getPerspectiveRegistry().getPerspectives()) { - msg.append(" " + pd.getId() + "\n"); - } - - ShowMessage.syncShowError("Invalid Perspective", msg.toString()); - return false; - } - } - - ILog log = Platform.getLog(Activator.getDefault().getBundle()); - - try { - // - // - // Create Simantics Platform Helper. - // - // If Simantics is started from Eclipse IDE or with -fixerrors option, - // there is an attempt to fix errors. - // - // On ontology mismatch, there is an attempt to merge new ontology to the - // existing database. With -reinstall, the database is cleaned and - // reinstalled. - // - // - - RecoveryPolicy workspacePolicy = Platform.inDevelopmentMode() ? RecoveryPolicy.FixError : RecoveryPolicy.ThrowError; - OntologyRecoveryPolicy ontologyPolicy = Platform.inDevelopmentMode() ? OntologyRecoveryPolicy.Merge : OntologyRecoveryPolicy.ThrowError; - - if (args.contains(SimanticsArguments.RECOVERY_POLICY_FIX_ERRORS)) { - workspacePolicy = RecoveryPolicy.FixError; - ontologyPolicy = OntologyRecoveryPolicy.Merge; - } - - boolean requireSynchronize = true; - - if (args.contains(SimanticsArguments.ONTOLOGY_RECOVERY_POLICY_REINSTALL)) { - ontologyPolicy = OntologyRecoveryPolicy.ReinstallDatabase; - } - - if (args.contains(SimanticsArguments.DO_NOT_SYNCHRONIZE_ONTOLOGIES)) { - requireSynchronize = false; - } - - if (args.contains(SimanticsArguments.DISABLE_INDEX)) { - Indexing.setDefaultDependenciesIndexingEnabled(false); - } - - if (args.contains(SimanticsArguments.SERVER)) { - String serverAddress = args.get(SimanticsArguments.SERVER); - throw new PlatformException("Argument not supported: " + SimanticsArguments.SERVER + " " + serverAddress); - } - - String databaseDriverId = Simantics.getDefaultDatabaseDriver(); - if (args.contains(SimanticsArguments.DATABASE_ID)) { - databaseDriverId = args.get(SimanticsArguments.DATABASE_ID); - Simantics.setDefaultDatabaseDriver(databaseDriverId); - } - - IProgressMonitor mon = null; - if (PROFILE_PLATFORM_STARTUP) - mon = new TimingProgressMonitor(); - SimanticsPlatform.INSTANCE.startUp(databaseDriverId, mon, workspacePolicy, ontologyPolicy, requireSynchronize, new JFaceUserAgent()); - - // Make sure that the default perspective comes from the project if - // the project has set ProjectKeys#DEFAULT_PERSPECTIVE. - // This might go wrong if project features interact with - // PerspectiveRegistry while configuring themselves, since that will - // cause an invocation to #getInitialWindowPerspectiveId() while - // the project has not yet been properly initialized. - getWorkbenchConfigurer().getWorkbench().getPerspectiveRegistry().setDefaultPerspective(getInitialWindowPerspectiveId()); - TimeLogger.log("Completed setting default perspective"); - - return true; - } catch (CancelStartupException e) { - return false; - } catch (PlatformException e) { - boolean hasStackTrace = e.getStackTrace().length > 0; - Throwable ee = e; - while (ee.getCause() != null) { - ee = ee.getCause(); - hasStackTrace = ee.getStackTrace().length > 0; - } - - log.log(new Status(IStatus.ERROR, Activator.PLUGIN_ID, e.getMessage(), hasStackTrace ? e : null)); - if (hasStackTrace) { - new ShowError("Platform Initialization Failed", "Simantics Platform initialization failed:\n\n" + e.getMessage(), e, true); - } else { - StringBuilder sb = new StringBuilder(256); - sb.append(e.getMessage()); - for (Throwable c=e.getCause(); null != c && null != c.getMessage(); c=c.getCause()) - sb.append("\ncause: ").append(c.getMessage()); - new ShowError("Startup Failed", sb.toString(), (Exception) null, true); - } - - return false; - } catch (Exception e) { - log.log(new Status(IStatus.ERROR, Activator.PLUGIN_ID, e.getMessage(), e)); - - Throwable cause = e.getCause(); - if (cause instanceof RebootRequiredException) { - RebootRequiredException rre = (RebootRequiredException) cause; - StringBuilder msg = new StringBuilder(); - msg.append("The application must be restarted after installing the following products:\n"); - for (Product product : rre.products) - msg.append("\t" + product + "\n"); - msg.append("\nThe application will now close."); - MessageDialog.openInformation(null, "Restart Required", msg.toString()); - } else { - new ShowError("Platform Startup Failed", "Simantics Platform startup failed:\n\n" + e.getMessage(), e, true); - } - return false; - } - - } - - /* - * (non-Javadoc) - * - * @see org.eclipse.ui.application.WorkbenchAdvisor#preStartup() - */ - @Override - public void preStartup() { - - // Suspend background jobs while we startup - Job.getJobManager().suspend(); - - // Register the build actions - IProgressService service = PlatformUI.getWorkbench() - .getProgressService(); - ImageDescriptor newImage = IDEInternalWorkbenchImages - .getImageDescriptor(IDEInternalWorkbenchImages.IMG_ETOOL_BUILD_EXEC); - service.registerIconForFamily(newImage, - ResourcesPlugin.FAMILY_MANUAL_BUILD); - service.registerIconForFamily(newImage, - ResourcesPlugin.FAMILY_AUTO_BUILD); - } - - /* - * (non-Javadoc) - * - * @see org.eclipse.ui.application.WorkbenchAdvisor#postStartup() - */ - @Override - public void postStartup() { - try { - refreshFromLocal(); - activateProxyService(); - ((Workbench) PlatformUI.getWorkbench()).registerService( - ISelectionConversionService.class, - new IDESelectionConversionService()); - - initializeSettingsChangeListener(); - Display.getCurrent().addListener(SWT.Settings, - settingsChangeListener); - } finally {// Resume background jobs after we startup - Job.getJobManager().resume(); - } - } - - /** - * Activate the proxy service by obtaining it. - */ - private void activateProxyService() { - Bundle bundle = Platform.getBundle("org.eclipse.ui.ide"); //$NON-NLS-1$ - Object proxyService = null; - if (bundle != null) { - ServiceReference ref = bundle.getBundleContext().getServiceReference(IProxyService.class.getName()); - if (ref != null) - proxyService = bundle.getBundleContext().getService(ref); - } - if (proxyService == null) { - IDEWorkbenchPlugin.log("Proxy service could not be found."); //$NON-NLS-1$ - } - } - - /** - * Initialize the listener for settings changes. - */ - private void initializeSettingsChangeListener() { - settingsChangeListener = new Listener() { - - boolean currentHighContrast = Display.getCurrent() - .getHighContrast(); - - @Override - public void handleEvent(org.eclipse.swt.widgets.Event event) { - if (Display.getCurrent().getHighContrast() == currentHighContrast) - return; - - currentHighContrast = !currentHighContrast; - - // make sure they really want to do this - if (new MessageDialog(null, - IDEWorkbenchMessages.SystemSettingsChange_title, null, - IDEWorkbenchMessages.SystemSettingsChange_message, - MessageDialog.QUESTION, new String[] { - IDEWorkbenchMessages.SystemSettingsChange_yes, - IDEWorkbenchMessages.SystemSettingsChange_no }, - 1).open() == Window.OK) { - PlatformUI.getWorkbench().restart(); - } - } - }; - - } - - /* - * (non-Javadoc) - * - * @see org.eclipse.ui.application.WorkbenchAdvisor#postShutdown - */ - @Override - public void postShutdown() { - if (activityHelper != null) { - activityHelper.shutdown(); - activityHelper = null; - } - if (idleHelper != null) { - idleHelper.shutdown(); - idleHelper = null; - } - if (workspaceUndoMonitor != null) { - workspaceUndoMonitor.shutdown(); - workspaceUndoMonitor = null; - } - if (IDEWorkbenchPlugin.getPluginWorkspace() != null) { - disconnectFromWorkspace(); - } - } - - /* - * (non-Javadoc) - * - * @see org.eclipse.ui.application.WorkbenchAdvisor#preShutdown() - */ - @Override - public boolean preShutdown() { - Display.getCurrent().removeListener(SWT.Settings, - settingsChangeListener); - return super.preShutdown(); - } - - /** - * Return true if the intro plugin is present and false otherwise. - * - * @return boolean - */ - public boolean hasIntro() { - return getWorkbenchConfigurer().getWorkbench().getIntroManager() - .hasIntro(); - } - - private void refreshFromLocal() { - String[] commandLineArgs = Platform.getCommandLineArgs(); - IPreferenceStore store = IDEWorkbenchPlugin.getDefault() - .getPreferenceStore(); - boolean refresh = store - .getBoolean(IDEInternalPreferences.REFRESH_WORKSPACE_ON_STARTUP); - if (!refresh) { - return; - } - - // Do not refresh if it was already done by core on startup. - for (int i = 0; i < commandLineArgs.length; i++) { - if (commandLineArgs[i].equalsIgnoreCase("-refresh")) { //$NON-NLS-1$ - return; - } - } - - final IContainer root = ResourcesPlugin.getWorkspace().getRoot(); - Job job = new WorkspaceJob(IDEWorkbenchMessages.Workspace_refreshing) { - @Override - public IStatus runInWorkspace(IProgressMonitor monitor) - throws CoreException { - root.refreshLocal(IResource.DEPTH_INFINITE, monitor); - return Status.OK_STATUS; - } - }; - job.setRule(root); - job.schedule(); - } - - private static class CancelableProgressMonitorWrapper extends ProgressMonitorWrapper { - private double total = 0; - private ProgressMonitorJobsDialog dialog; - - CancelableProgressMonitorWrapper(IProgressMonitor monitor, - ProgressMonitorJobsDialog dialog) { - super(monitor); - this.dialog = dialog; - } - - /* - * (non-Javadoc) - * @see org.eclipse.core.runtime.ProgressMonitorWrapper#internalWorked(double) - */ - public void internalWorked(double work) { - super.internalWorked(work); - total += work; - updateProgressDetails(); - } - - /* - * (non-Javadoc) - * @see org.eclipse.core.runtime.ProgressMonitorWrapper#worked(int) - */ - public void worked(int work) { - super.worked(work); - total += work; - updateProgressDetails(); - } - - public void beginTask(String name, int totalWork) { - super.beginTask(name, totalWork); - subTask(IDEWorkbenchMessages.IDEWorkbenchAdvisor_preHistoryCompaction); - } - - private void updateProgressDetails() { - if (!isCanceled() && Math.abs(total - 4.0) < 0.0001 /* right before history compacting */) { - subTask(IDEWorkbenchMessages.IDEWorkbenchAdvisor_cancelHistoryPruning); - dialog.setCancelable(true); - } - if (Math.abs(total - 5.0) < 0.0001 /* history compacting finished */) { - subTask(IDEWorkbenchMessages.IDEWorkbenchAdvisor_postHistoryCompaction); - dialog.setCancelable(false); - } - } - } - - private static class CancelableProgressMonitorJobsDialog extends ProgressMonitorJobsDialog { - - public CancelableProgressMonitorJobsDialog(Shell parent) { - super(parent); - } - - /* - * (non-Javadoc) - * @see org.eclipse.ui.internal.progress.ProgressMonitorJobsDialog#createDetailsButton(org.eclipse.swt.widgets.Composite) - */ - protected void createButtonsForButtonBar(Composite parent) { - super.createButtonsForButtonBar(parent); - registerCancelButtonListener(); - } - - public void registerCancelButtonListener() { - cancel.addSelectionListener(new SelectionAdapter() { - public void widgetSelected(SelectionEvent e) { - subTaskLabel.setText(""); //$NON-NLS-1$ - } - }); - } - } - - - final IRunnableWithProgress platformShutdownRunnable = new IRunnableWithProgress() { - /** - * @param monitor - * the progress monitor to use for reporting progress to the - * user, or null indicating that no progress - * should be reported and the operation cannot be cancelled. - */ - @Override - public void run(IProgressMonitor monitor) { - SubMonitor progress = SubMonitor.convert(monitor, SHUT_DOWN_PLATFORM_TASK, 100); - try { - try { - progress.subTask("Platform"); - SimanticsPlatform.INSTANCE.shutdown(progress.newChild(50)); - } catch (PlatformException e) { - Activator.logError("Problems encountered while shutting down Simantics platform, see exception for details.", e); - } - - progress.subTask("Remaining database connections"); - SimanticsUI.closeSessions(); - progress.worked(20); - TimedSessionCache.close(); - progress.worked(20); - - progress.subTask("Thread pools"); - ThreadUtils.shutdown(); - progress.worked(5); - - progress.subTask("Clear index status"); - try { - // Everything ok, clear index dirty state. - DatabaseIndexing.clearAllDirty(); - } catch (IOException e) { - Activator.logError("Problems encountered while refreshing database index states, see exception for details.", e); - } - progress.worked(5); - - progress.setWorkRemaining(0); - } finally { - if (monitor != null) { - monitor.done(); - } - } - } - }; - - /** - * Disconnect from the workspace and close ProCore sessions. - */ - private void disconnectFromWorkspace() { - // save the workspace - final MultiStatus status = new MultiStatus( - IDEWorkbenchPlugin.IDE_WORKBENCH, 1, - IDEWorkbenchMessages.ProblemSavingWorkbench, null); - - final ProgressMonitorJobsDialog p = new CancelableProgressMonitorJobsDialog( - null); - - final boolean applyPolicy = ResourcesPlugin.getWorkspace() - .getDescription().isApplyFileStatePolicy(); - - final IRunnableWithProgress workspaceShutdownRunnable = new IRunnableWithProgress() { - @Override - public void run(IProgressMonitor monitor) { - try { - status.merge(((Workspace) ResourcesPlugin.getWorkspace()).save(true, true, monitor)); - } catch (CoreException e) { - status.merge(e.getStatus()); - } - } - }; - - IRunnableWithProgress shutdownRunnable = new IRunnableWithProgress() { - @Override - public void run(IProgressMonitor monitor) throws InvocationTargetException, InterruptedException { - if (applyPolicy) - monitor = new CancelableProgressMonitorWrapper( - monitor, p); - - SubMonitor progress = SubMonitor.convert(monitor, SHUT_DOWN_TASK, 2); - try { - workspaceShutdownRunnable.run(progress.newChild(1, SubMonitor.SUPPRESS_NONE)); - platformShutdownRunnable.run(progress.newChild(1, SubMonitor.SUPPRESS_NONE)); - } finally { - monitor.done(); - } - } - }; - - try { - new ProgressMonitorJobsDialog(null).run(true, false, shutdownRunnable); - } catch (InvocationTargetException e) { - status.merge(new Status(IStatus.ERROR, - IDEWorkbenchPlugin.IDE_WORKBENCH, 1, - IDEWorkbenchMessages.InternalError, e.getTargetException())); - } catch (InterruptedException e) { - status.merge(new Status(IStatus.ERROR, - IDEWorkbenchPlugin.IDE_WORKBENCH, 1, - IDEWorkbenchMessages.InternalError, e)); - } - ErrorDialog.openError(null, - IDEWorkbenchMessages.ProblemsSavingWorkspace, null, status, - IStatus.ERROR | IStatus.WARNING); - if (!status.isOK()) { - IDEWorkbenchPlugin.log( - IDEWorkbenchMessages.ProblemsSavingWorkspace, status); - } - } - - /* - * (non-Javadoc) - * - * @see org.eclipse.ui.application.WorkbenchAdvisor#getDefaultPageInput - */ - @Override - public IAdaptable getDefaultPageInput() { - return ResourcesPlugin.getWorkspace().getRoot(); - } - - /* - * (non-Javadoc) - * - * @see org.eclipse.ui.application.WorkbenchAdvisor - */ - @Override - public String getInitialWindowPerspectiveId() { - int index = PlatformUI.getWorkbench().getWorkbenchWindowCount() - 1; - - String perspectiveId = null; - AboutInfo[] welcomeInfos = getWelcomePerspectiveInfos(); - if (index >= 0 && welcomeInfos != null && index < welcomeInfos.length) { - perspectiveId = welcomeInfos[index].getWelcomePerspectiveId(); - } - - if (perspectiveId == null && args.contains(SimanticsArguments.PERSPECTIVE)) { - String id = args.get(SimanticsArguments.PERSPECTIVE); - IPerspectiveDescriptor perspective = PlatformUI.getWorkbench().getPerspectiveRegistry().findPerspectiveWithId(id); - if (perspective != null) - perspectiveId = id; - } - - if (perspectiveId == null) { - IProject project = SimanticsUI.peekProject(); - if (project != null) - perspectiveId = project.getHint(ProjectKeys.DEFAULT_PERSPECTIVE); - } - - //System.out.println("Initial perspective: " + perspectiveId); - - return perspectiveId; - } - - /** - * Returns the map of versioned feature ids -> info object for all installed - * features. The format of the versioned feature id (the key of the map) is - * featureId + ":" + versionId. - * - * @return map of versioned feature ids -> info object (key type: - * String, value type: AboutInfo) - * @since 3.0 - */ - private Map computeBundleGroupMap() { - // use tree map to get predicable order - Map ids = new TreeMap(); - - IBundleGroupProvider[] providers = Platform.getBundleGroupProviders(); - for (int i = 0; i < providers.length; ++i) { - IBundleGroup[] groups = providers[i].getBundleGroups(); - for (int j = 0; j < groups.length; ++j) { - IBundleGroup group = groups[j]; - AboutInfo info = new AboutInfo(group); - - String version = info.getVersionId(); - version = version == null ? "0.0.0" //$NON-NLS-1$ - : new Version(version).toString(); - String versionedFeature = group.getIdentifier() + ":" + version; //$NON-NLS-1$ - - ids.put(versionedFeature, info); - } - } - - return ids; - } - - /** - * Returns the ordered map of versioned feature ids -> AboutInfo that are - * new for this session. - * - * @return ordered map of versioned feature ids (key type: - * String) -> infos (value type: - * AboutInfo). - */ - public Map getNewlyAddedBundleGroups() { - if (newlyAddedBundleGroups == null) { - newlyAddedBundleGroups = createNewBundleGroupsMap(); - } - return newlyAddedBundleGroups; - } - - /** - * Updates the old features setting and returns a map of new features. - */ - private Map createNewBundleGroupsMap() { - // retrieve list of installed bundle groups from last session - IDialogSettings settings = IDEWorkbenchPlugin.getDefault() - .getDialogSettings(); - String[] previousFeaturesArray = settings.getArray(INSTALLED_FEATURES); - - // get a map of currently installed bundle groups and store it for next - // session - Map bundleGroups = computeBundleGroupMap(); - String[] currentFeaturesArray = new String[bundleGroups.size()]; - bundleGroups.keySet().toArray(currentFeaturesArray); - settings.put(INSTALLED_FEATURES, currentFeaturesArray); - - // remove the previously known from the current set - if (previousFeaturesArray != null) { - for (int i = 0; i < previousFeaturesArray.length; ++i) { - bundleGroups.remove(previousFeaturesArray[i]); - } - } - - return bundleGroups; - } - - /** - * Declares all IDE-specific workbench images. This includes both "shared" - * images (named in {@link IDE.SharedImages}) and internal images (named in - * {@link org.eclipse.ui.internal.ide.IDEInternalWorkbenchImages}). - * - * @see IWorkbenchConfigurer#declareImage - */ - private void declareWorkbenchImages() { - - final String ICONS_PATH = "$nl$/icons/full/";//$NON-NLS-1$ - final String PATH_ELOCALTOOL = ICONS_PATH + "elcl16/"; // Enabled //$NON-NLS-1$ - - // toolbar - // icons. - final String PATH_DLOCALTOOL = ICONS_PATH + "dlcl16/"; // Disabled //$NON-NLS-1$ - // //$NON-NLS-1$ - // toolbar - // icons. - final String PATH_ETOOL = ICONS_PATH + "etool16/"; // Enabled toolbar //$NON-NLS-1$ - // //$NON-NLS-1$ - // icons. - final String PATH_DTOOL = ICONS_PATH + "dtool16/"; // Disabled toolbar //$NON-NLS-1$ - // //$NON-NLS-1$ - // icons. - final String PATH_OBJECT = ICONS_PATH + "obj16/"; // Model object //$NON-NLS-1$ - // //$NON-NLS-1$ - // icons - final String PATH_WIZBAN = ICONS_PATH + "wizban/"; // Wizard //$NON-NLS-1$ - // //$NON-NLS-1$ - // icons - - Bundle ideBundle = Platform.getBundle(IDEWorkbenchPlugin.IDE_WORKBENCH); - - declareWorkbenchImage(ideBundle, - IDEInternalWorkbenchImages.IMG_ETOOL_BUILD_EXEC, PATH_ETOOL - + "build_exec.gif", false); //$NON-NLS-1$ - declareWorkbenchImage(ideBundle, - IDEInternalWorkbenchImages.IMG_ETOOL_BUILD_EXEC_HOVER, - PATH_ETOOL + "build_exec.gif", false); //$NON-NLS-1$ - declareWorkbenchImage(ideBundle, - IDEInternalWorkbenchImages.IMG_ETOOL_BUILD_EXEC_DISABLED, - PATH_DTOOL + "build_exec.gif", false); //$NON-NLS-1$ - - declareWorkbenchImage(ideBundle, - IDEInternalWorkbenchImages.IMG_ETOOL_SEARCH_SRC, PATH_ETOOL - + "search_src.gif", false); //$NON-NLS-1$ - declareWorkbenchImage(ideBundle, - IDEInternalWorkbenchImages.IMG_ETOOL_SEARCH_SRC_HOVER, - PATH_ETOOL + "search_src.gif", false); //$NON-NLS-1$ - declareWorkbenchImage(ideBundle, - IDEInternalWorkbenchImages.IMG_ETOOL_SEARCH_SRC_DISABLED, - PATH_DTOOL + "search_src.gif", false); //$NON-NLS-1$ - - declareWorkbenchImage(ideBundle, - IDEInternalWorkbenchImages.IMG_ETOOL_NEXT_NAV, PATH_ETOOL - + "next_nav.gif", false); //$NON-NLS-1$ - - declareWorkbenchImage(ideBundle, - IDEInternalWorkbenchImages.IMG_ETOOL_PREVIOUS_NAV, PATH_ETOOL - + "prev_nav.gif", false); //$NON-NLS-1$ - - declareWorkbenchImage(ideBundle, - IDEInternalWorkbenchImages.IMG_WIZBAN_NEWPRJ_WIZ, PATH_WIZBAN - + "newprj_wiz.png", false); //$NON-NLS-1$ - declareWorkbenchImage(ideBundle, - IDEInternalWorkbenchImages.IMG_WIZBAN_NEWFOLDER_WIZ, - PATH_WIZBAN + "newfolder_wiz.png", false); //$NON-NLS-1$ - declareWorkbenchImage(ideBundle, - IDEInternalWorkbenchImages.IMG_WIZBAN_NEWFILE_WIZ, PATH_WIZBAN - + "newfile_wiz.png", false); //$NON-NLS-1$ - - declareWorkbenchImage(ideBundle, - IDEInternalWorkbenchImages.IMG_WIZBAN_IMPORTDIR_WIZ, - PATH_WIZBAN + "importdir_wiz.png", false); //$NON-NLS-1$ - declareWorkbenchImage(ideBundle, - IDEInternalWorkbenchImages.IMG_WIZBAN_IMPORTZIP_WIZ, - PATH_WIZBAN + "importzip_wiz.png", false); //$NON-NLS-1$ - - declareWorkbenchImage(ideBundle, - IDEInternalWorkbenchImages.IMG_WIZBAN_EXPORTDIR_WIZ, - PATH_WIZBAN + "exportdir_wiz.png", false); //$NON-NLS-1$ - declareWorkbenchImage(ideBundle, - IDEInternalWorkbenchImages.IMG_WIZBAN_EXPORTZIP_WIZ, - PATH_WIZBAN + "exportzip_wiz.png", false); //$NON-NLS-1$ - - declareWorkbenchImage(ideBundle, - IDEInternalWorkbenchImages.IMG_WIZBAN_RESOURCEWORKINGSET_WIZ, - PATH_WIZBAN + "workset_wiz.png", false); //$NON-NLS-1$ - - declareWorkbenchImage(ideBundle, - IDEInternalWorkbenchImages.IMG_DLGBAN_SAVEAS_DLG, PATH_WIZBAN - + "saveas_wiz.png", false); //$NON-NLS-1$ - - declareWorkbenchImage(ideBundle, - IDEInternalWorkbenchImages.IMG_DLGBAN_QUICKFIX_DLG, PATH_WIZBAN - + "quick_fix.png", false); //$NON-NLS-1$ - - declareWorkbenchImage(ideBundle, IDE.SharedImages.IMG_OBJ_PROJECT, - PATH_OBJECT + "prj_obj.gif", true); //$NON-NLS-1$ - declareWorkbenchImage(ideBundle, - IDE.SharedImages.IMG_OBJ_PROJECT_CLOSED, PATH_OBJECT - + "cprj_obj.gif", true); //$NON-NLS-1$ - declareWorkbenchImage(ideBundle, IDE.SharedImages.IMG_OPEN_MARKER, - PATH_ELOCALTOOL + "gotoobj_tsk.gif", true); //$NON-NLS-1$ - - declareWorkbenchImage(ideBundle, - IDEInternalWorkbenchImages.IMG_ELCL_QUICK_FIX_ENABLED, - PATH_ELOCALTOOL + "smartmode_co.gif", true); //$NON-NLS-1$ - - declareWorkbenchImage(ideBundle, - IDEInternalWorkbenchImages.IMG_DLCL_QUICK_FIX_DISABLED, - PATH_DLOCALTOOL + "smartmode_co.gif", true); //$NON-NLS-1$ - - // task objects - // declareRegistryImage(IDEInternalWorkbenchImages.IMG_OBJS_HPRIO_TSK, - // PATH_OBJECT+"hprio_tsk.gif"); - // declareRegistryImage(IDEInternalWorkbenchImages.IMG_OBJS_MPRIO_TSK, - // PATH_OBJECT+"mprio_tsk.gif"); - // declareRegistryImage(IDEInternalWorkbenchImages.IMG_OBJS_LPRIO_TSK, - // PATH_OBJECT+"lprio_tsk.gif"); - - declareWorkbenchImage(ideBundle, IDE.SharedImages.IMG_OBJS_TASK_TSK, - PATH_OBJECT + "taskmrk_tsk.gif", true); //$NON-NLS-1$ - declareWorkbenchImage(ideBundle, IDE.SharedImages.IMG_OBJS_BKMRK_TSK, - PATH_OBJECT + "bkmrk_tsk.gif", true); //$NON-NLS-1$ - - declareWorkbenchImage(ideBundle, - IDEInternalWorkbenchImages.IMG_OBJS_COMPLETE_TSK, PATH_OBJECT - + "complete_tsk.gif", true); //$NON-NLS-1$ - declareWorkbenchImage(ideBundle, - IDEInternalWorkbenchImages.IMG_OBJS_INCOMPLETE_TSK, PATH_OBJECT - + "incomplete_tsk.gif", true); //$NON-NLS-1$ - declareWorkbenchImage(ideBundle, - IDEInternalWorkbenchImages.IMG_OBJS_WELCOME_ITEM, PATH_OBJECT - + "welcome_item.gif", true); //$NON-NLS-1$ - declareWorkbenchImage(ideBundle, - IDEInternalWorkbenchImages.IMG_OBJS_WELCOME_BANNER, PATH_OBJECT - + "welcome_banner.gif", true); //$NON-NLS-1$ - declareWorkbenchImage(ideBundle, - IDEInternalWorkbenchImages.IMG_OBJS_ERROR_PATH, PATH_OBJECT - + "error_tsk.gif", true); //$NON-NLS-1$ - declareWorkbenchImage(ideBundle, - IDEInternalWorkbenchImages.IMG_OBJS_WARNING_PATH, PATH_OBJECT - + "warn_tsk.gif", true); //$NON-NLS-1$ - declareWorkbenchImage(ideBundle, - IDEInternalWorkbenchImages.IMG_OBJS_INFO_PATH, PATH_OBJECT - + "info_tsk.gif", true); //$NON-NLS-1$ - - declareWorkbenchImage(ideBundle, - IDEInternalWorkbenchImages.IMG_LCL_FLAT_LAYOUT, PATH_ELOCALTOOL - + "flatLayout.gif", true); //$NON-NLS-1$ - declareWorkbenchImage(ideBundle, - IDEInternalWorkbenchImages.IMG_LCL_HIERARCHICAL_LAYOUT, - PATH_ELOCALTOOL + "hierarchicalLayout.gif", true); //$NON-NLS-1$ - declareWorkbenchImage(ideBundle, - IDEInternalWorkbenchImages.IMG_ETOOL_PROBLEM_CATEGORY, - PATH_ETOOL + "problem_category.gif", true); //$NON-NLS-1$ - /* - declareWorkbenchImage(ideBundle, - IDEInternalWorkbenchImages.IMG_LCL_LINKTO_HELP, PATH_ELOCALTOOL - + "linkto_help.gif", false); //$NON-NLS-1$ - */ - - // synchronization indicator objects - // declareRegistryImage(IDEInternalWorkbenchImages.IMG_OBJS_WBET_STAT, - // PATH_OVERLAY+"wbet_stat.gif"); - // declareRegistryImage(IDEInternalWorkbenchImages.IMG_OBJS_SBET_STAT, - // PATH_OVERLAY+"sbet_stat.gif"); - // declareRegistryImage(IDEInternalWorkbenchImages.IMG_OBJS_CONFLICT_STAT, - // PATH_OVERLAY+"conflict_stat.gif"); - - // content locality indicator objects - // declareRegistryImage(IDEInternalWorkbenchImages.IMG_OBJS_NOTLOCAL_STAT, - // PATH_STAT+"notlocal_stat.gif"); - // declareRegistryImage(IDEInternalWorkbenchImages.IMG_OBJS_LOCAL_STAT, - // PATH_STAT+"local_stat.gif"); - // declareRegistryImage(IDEInternalWorkbenchImages.IMG_OBJS_FILLLOCAL_STAT, - // PATH_STAT+"filllocal_stat.gif"); - } - - /** - * Declares an IDE-specific workbench image. - * - * @param symbolicName - * the symbolic name of the image - * @param path - * the path of the image file; this path is relative to the base - * of the IDE plug-in - * @param shared - * true if this is a shared image, and - * false if this is not a shared image - * @see IWorkbenchConfigurer#declareImage - */ - private void declareWorkbenchImage(Bundle ideBundle, String symbolicName, - String path, boolean shared) { - URL url = FileLocator.find(ideBundle, new Path(path), null); - ImageDescriptor desc = ImageDescriptor.createFromURL(url); - getWorkbenchConfigurer().declareImage(symbolicName, desc, shared); - } - - /* - * (non-Javadoc) - * - * @see org.eclipse.ui.application.WorkbenchAdvisor#getMainPreferencePageId - */ - @Override - public String getMainPreferencePageId() { - // indicate that we want the Workench preference page to be prominent - return WORKBENCH_PREFERENCE_CATEGORY_ID; - } - - /** - * @return the workspace location string, or null if the - * location is not being shown - */ - public String getWorkspaceLocation() { - // read command line, which has priority - IEclipseContext context = getWorkbenchConfigurer().getWorkbench().getService(IEclipseContext.class); - String location = context != null ? (String) context.get(E4Workbench.FORCED_SHOW_LOCATION) : null; - if (location != null) { - return location; - } - // read the preference - if (IDEWorkbenchPlugin.getDefault().getPreferenceStore().getBoolean(IDEInternalPreferences.SHOW_LOCATION)) { - return Platform.getLocation().toOSString(); - } - return null; - } - - /** - * @return the welcome perspective infos, or null if none or - * if they should be ignored due to the new intro being present - */ - public AboutInfo[] getWelcomePerspectiveInfos() { - if (welcomePerspectiveInfos == null) { - // support old welcome perspectives if intro plugin is not present - if (!hasIntro()) { - Map m = getNewlyAddedBundleGroups(); - ArrayList list = new ArrayList(m.size()); - for (Iterator i = m.values().iterator(); i.hasNext();) { - AboutInfo info = i.next(); - if (info != null && info.getWelcomePerspectiveId() != null - && info.getWelcomePageURL() != null) { - list.add(info); - } - } - welcomePerspectiveInfos = new AboutInfo[list.size()]; - list.toArray(welcomePerspectiveInfos); - } - } - return welcomePerspectiveInfos; - } - - /* - * (non-Javadoc) - * - * @see org.eclipse.ui.application.WorkbenchAdvisor#getWorkbenchErrorHandler() - */ - @Override - public AbstractStatusHandler getWorkbenchErrorHandler() { - if (ideWorkbenchErrorHandler == null) { - ideWorkbenchErrorHandler = new IDEWorkbenchErrorHandler( - getWorkbenchConfigurer()); - } - return ideWorkbenchErrorHandler; - } - - /* (non-Javadoc) - * @see org.eclipse.ui.application.WorkbenchAdvisor#eventLoopIdle(org.eclipse.swt.widgets.Display) - */ - @Override - public void eventLoopIdle(Display display) { - if (delayedEventsProcessor != null) - delayedEventsProcessor.catchUp(display); - super.eventLoopIdle(display); - } - -} +/******************************************************************************* + * Copyright (c) 2007, 2010 Association for Decentralized Information Management + * in Industry THTH ry. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * VTT Technical Research Centre of Finland - initial API and implementation + *******************************************************************************/ +package org.simantics.workbench.internal; + +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.net.URL; +import java.text.Collator; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.Map; +import java.util.TreeMap; + +import org.eclipse.core.internal.resources.Workspace; +import org.eclipse.core.net.proxy.IProxyService; +import org.eclipse.core.resources.IContainer; +import org.eclipse.core.resources.IResource; +import org.eclipse.core.resources.ResourcesPlugin; +import org.eclipse.core.resources.WorkspaceJob; +import org.eclipse.core.runtime.CoreException; +import org.eclipse.core.runtime.FileLocator; +import org.eclipse.core.runtime.IAdaptable; +import org.eclipse.core.runtime.IBundleGroup; +import org.eclipse.core.runtime.IBundleGroupProvider; +import org.eclipse.core.runtime.ILog; +import org.eclipse.core.runtime.IProgressMonitor; +import org.eclipse.core.runtime.IStatus; +import org.eclipse.core.runtime.MultiStatus; +import org.eclipse.core.runtime.Path; +import org.eclipse.core.runtime.Platform; +import org.eclipse.core.runtime.ProgressMonitorWrapper; +import org.eclipse.core.runtime.Status; +import org.eclipse.core.runtime.SubMonitor; +import org.eclipse.core.runtime.jobs.Job; +import org.eclipse.e4.core.contexts.IEclipseContext; +import org.eclipse.e4.ui.internal.workbench.E4Workbench; +import org.eclipse.jface.dialogs.ErrorDialog; +import org.eclipse.jface.dialogs.IDialogSettings; +import org.eclipse.jface.dialogs.MessageDialog; +import org.eclipse.jface.dialogs.TrayDialog; +import org.eclipse.jface.operation.IRunnableWithProgress; +import org.eclipse.jface.preference.IPreferenceStore; +import org.eclipse.jface.resource.ImageDescriptor; +import org.eclipse.jface.util.Policy; +import org.eclipse.jface.window.Window; +import org.eclipse.swt.SWT; +import org.eclipse.swt.events.SelectionAdapter; +import org.eclipse.swt.events.SelectionEvent; +import org.eclipse.swt.widgets.Composite; +import org.eclipse.swt.widgets.Display; +import org.eclipse.swt.widgets.Event; +import org.eclipse.swt.widgets.Listener; +import org.eclipse.swt.widgets.Shell; +import org.eclipse.ui.IPerspectiveDescriptor; +import org.eclipse.ui.PlatformUI; +import org.eclipse.ui.application.IWorkbenchConfigurer; +import org.eclipse.ui.application.IWorkbenchWindowConfigurer; +import org.eclipse.ui.application.WorkbenchAdvisor; +import org.eclipse.ui.application.WorkbenchWindowAdvisor; +import org.eclipse.ui.ide.IDE; +import org.eclipse.ui.internal.ISelectionConversionService; +import org.eclipse.ui.internal.Workbench; +import org.eclipse.ui.internal.ide.AboutInfo; +import org.eclipse.ui.internal.ide.IDEInternalPreferences; +import org.eclipse.ui.internal.ide.IDEInternalWorkbenchImages; +import org.eclipse.ui.internal.ide.IDESelectionConversionService; +import org.eclipse.ui.internal.ide.IDEWorkbenchActivityHelper; +import org.eclipse.ui.internal.ide.IDEWorkbenchErrorHandler; +import org.eclipse.ui.internal.ide.IDEWorkbenchMessages; +import org.eclipse.ui.internal.ide.IDEWorkbenchPlugin; +import org.eclipse.ui.internal.ide.undo.WorkspaceUndoMonitor; +import org.eclipse.ui.internal.progress.ProgressMonitorJobsDialog; +import org.eclipse.ui.keys.IBindingService; +import org.eclipse.ui.progress.IProgressService; +import org.eclipse.ui.statushandlers.AbstractStatusHandler; +import org.osgi.framework.Bundle; +import org.osgi.framework.ServiceReference; +import org.osgi.framework.Version; +import org.simantics.CancelStartupException; +import org.simantics.PlatformException; +import org.simantics.Simantics; +import org.simantics.SimanticsPlatform; +import org.simantics.SimanticsPlatform.OntologyRecoveryPolicy; +import org.simantics.SimanticsPlatform.RecoveryPolicy; +import org.simantics.application.arguments.IArguments; +import org.simantics.application.arguments.SimanticsArguments; +import org.simantics.db.common.Indexing; +import org.simantics.db.indexing.DatabaseIndexing; +import org.simantics.db.procore.server.environment.RebootRequiredException; +import org.simantics.db.procore.server.environment.windows.Product; +import org.simantics.internal.TimedSessionCache; +import org.simantics.project.IProject; +import org.simantics.project.ProjectKeys; +import org.simantics.ui.SimanticsUI; +import org.simantics.ui.jobs.SessionGarbageCollectorJob; +import org.simantics.ui.workbench.PerspectiveBarsActivator; +import org.simantics.ui.workbench.PerspectiveContextActivator; +import org.simantics.utils.logging.TimeLogger; +import org.simantics.utils.threads.ThreadUtils; +import org.simantics.utils.ui.dialogs.ShowError; +import org.simantics.utils.ui.dialogs.ShowMessage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * @author Tuukka Lehtonen + */ +public class SimanticsWorkbenchAdvisor extends WorkbenchAdvisor { + + private static final Logger LOGGER = LoggerFactory.getLogger(SimanticsWorkbenchAdvisor.class); + + private static final boolean PROFILE_PLATFORM_STARTUP = false; + + private static final String SHUT_DOWN_TASK = "Shutting down..."; + + private static final String SHUT_DOWN_PLATFORM_TASK = "Shutting down platform..."; + + private static final String WORKBENCH_PREFERENCE_CATEGORY_ID = "org.eclipse.ui.preferencePages.Workbench"; //$NON-NLS-1$ + + /** + * The dialog setting key to access the known installed features since the + * last time the workbench was run. + */ + private static final String INSTALLED_FEATURES = "installedFeatures"; //$NON-NLS-1$ + + /** + * The arguments received by the application. + */ + protected final IArguments args; + + protected final boolean restoredPreviousSession = false; + + /** + * Only true while opening the initial windows during {@link #openWindows()}. + * Used by {@link SimanticsWorkbenchWindowAdvisor#postWindowOpen()} to + * recognize when to skip all one-time initialization. + */ + protected boolean workbenchWindowsInitialized = false; + + /** + * Whether or not to save unsaved database changes before exiting the + * workbench. + */ + protected boolean saveAtExit = false; + + /** + * Ordered map of versioned feature ids -> info that are new for this + * session; null if uninitialized. Key type: + * String, Value type: AboutInfo. + */ + private Map newlyAddedBundleGroups; + + /** + * Array of AboutInfo for all new installed features that + * specify a welcome perspective. + */ + private AboutInfo[] welcomePerspectiveInfos = null; + + /** + * Helper for managing activites in response to workspace changes. + */ + private IDEWorkbenchActivityHelper activityHelper = null; + + /** + * Helper for managing work that is performed when the system is otherwise + * idle. + */ + private IDEIdleHelper idleHelper; + + private Listener settingsChangeListener; + + /** + * Support class for monitoring workspace changes and periodically + * validating the undo history + */ + private WorkspaceUndoMonitor workspaceUndoMonitor; + + /** + * The IDE workbench error handler. + */ + private AbstractStatusHandler ideWorkbenchErrorHandler; + + /** + * Helper class used to process delayed events. + */ + private DelayedEventsProcessor delayedEventsProcessor; + + /** + * Creates a new workbench advisor instance. + * @param processor + */ + public SimanticsWorkbenchAdvisor(IArguments args, DelayedEventsProcessor processor) { + super(); + this.args = args; + this.delayedEventsProcessor = processor; + + Listener closeListener = new Listener() { + public void handleEvent(Event event) { + boolean doExit = SimanticsWorkbenchWindowAdvisor.promptOnExit(null); + event.doit = doExit; + if (!doExit) + event.type = SWT.None; + } + }; + Display.getDefault().addListener(SWT.Close, closeListener); + } + + public IArguments getArguments() { + return args; + } + + public boolean workbenchInitialized() { + return workbenchWindowsInitialized; + } + + public boolean restoredPreviousSession() { + return restoredPreviousSession; + } + + boolean saveAtExit() { + return saveAtExit; + } + + void setSaveAtExit(boolean saveAtExit) { + this.saveAtExit = saveAtExit; + } + + /* + * (non-Javadoc) + * + * @see org.eclipse.ui.application.WorkbenchAdvisor#initialize + */ + @Override + public void initialize(IWorkbenchConfigurer configurer) { + // By default, we always save and restore the workbench state. + configurer.setSaveAndRestore(true); + + checkWorkspaceDatabaseIndexes(); + + // Start tracking the active perspective to activate contexts based on it. + new PerspectiveContextActivator(); + new PerspectiveBarsActivator(); + + // register workspace adapters + IDE.registerAdapters(); + + // register shared images + declareWorkbenchImages(); + + // initialize the activity helper + activityHelper = IDEWorkbenchActivityHelper.getInstance(); + + // initialize idle handler + idleHelper = new IDEIdleHelper(configurer); + + // initialize the workspace undo monitor + workspaceUndoMonitor = WorkspaceUndoMonitor.getInstance(); + + // show Help button in JFace dialogs + TrayDialog.setDialogHelpAvailable(true); + + Policy.setComparator(Collator.getInstance()); + } + + private void checkWorkspaceDatabaseIndexes() { + try { + DatabaseIndexing.validateIndexes(); + } catch (IOException e) { + Activator.logError("Problems encountered while checking database indexes, see exception for details.", e); + } + } + + public WorkbenchWindowAdvisor createWorkbenchWindowAdvisorClass(SimanticsWorkbenchAdvisor advisor, IWorkbenchWindowConfigurer configurer) { + return new SimanticsWorkbenchWindowAdvisor(this, configurer); + } + + @Override + public WorkbenchWindowAdvisor createWorkbenchWindowAdvisor(IWorkbenchWindowConfigurer configurer) { + // Attach database session watchdog. + new SessionWatchdog().attach( Simantics.getSessionContextProvider() ); + + return createWorkbenchWindowAdvisorClass(this, configurer); + } + + /** + * For gaining direct access to super.openWindows() in implementations + * inheriting this one. + */ + public boolean openWindowsSuper() { + return super.openWindows(); + } + + /** + * Sadly we do not know why key bindings are lost and why this helps. But it + * does. Visiting the Keys preference page and pressing OK uses + * this the same call and it seems to salvage the bindings that have been in + * some cases destroyed by BindingToModelProcessor. + * + *

+ * Related links: + * https://techblog.ralph-schuster.eu/2013/10/13/eclipsee4-problem-with-key-bindings/comment-page-1/ + * https://www.eclipse.org/forums/index.php/t/550175/ + * https://bugs.eclipse.org/bugs/show_bug.cgi?id=461037 + * + * @see platform issue #6353 + */ + private void fixBindings() { + try { + IBindingService bs = PlatformUI.getWorkbench().getAdapter(IBindingService.class); + bs.savePreferences(bs.getActiveScheme(), bs.getBindings()); + } catch (IOException e) { + Activator.logError(getClass().getSimpleName() + ".fixBindings failed", e); + } + } + + @Override + public boolean openWindows() { + boolean platformOk = startPlatform(); + LOGGER.info("startPlatform finished"); + TimeLogger.log("SimanticsWorkbenchAdvisor.startPlatform finished"); + + if (platformOk) { + // At this point workbenchConfigurer.getSaveAndRestore() + // returns false iff something has gone terribly wrong + // before this. Currently saveAndRestore is always true. + boolean windowsOpened = super.openWindows(); + TimeLogger.log("Opened windows"); + if (windowsOpened) { + workbenchWindowsInitialized = true; + + // Start the database garbage collector after a short while. + SessionGarbageCollectorJob.getInstance().scheduleAfterQuietTime(); + + // Discard database session undo history at this point to prevent + // the user from undoing any initialization operations performed + // by the platform startup. + SimanticsPlatform.INSTANCE.discardSessionUndoHistory(); + TimeLogger.log("Discarded session undo history"); + + // #6353: Workaround for + fixBindings(); + + return true; + } + } + + // Make sure platform shutdown is ran if window opening fails. + try { + platformShutdownRunnable.run(null); + } catch (InvocationTargetException e) { + Activator.logError(getClass().getSimpleName() + ".openWindows failed", e.getCause()); + } catch (InterruptedException e) { + Activator.logError(getClass().getSimpleName() + ".openWindows failed", e); + } + return false; + } + + protected boolean startPlatform() { + // Verify selected perspective + if (args.contains(SimanticsArguments.PERSPECTIVE)) { + String perspectiveId = args.get(SimanticsArguments.PERSPECTIVE); + IPerspectiveDescriptor perspective = PlatformUI.getWorkbench().getPerspectiveRegistry().findPerspectiveWithId(perspectiveId); + if (perspective == null) { + StringBuilder msg = new StringBuilder(); + msg.append("Requested perspective not found: '" + perspectiveId + "'\n"); + msg.append("Valid alternatives are:\n"); + for (IPerspectiveDescriptor pd : PlatformUI.getWorkbench().getPerspectiveRegistry().getPerspectives()) { + msg.append(" " + pd.getId() + "\n"); + } + + ShowMessage.syncShowError("Invalid Perspective", msg.toString()); + return false; + } + } + + ILog log = Platform.getLog(Activator.getDefault().getBundle()); + + try { + // + // + // Create Simantics Platform Helper. + // + // If Simantics is started from Eclipse IDE or with -fixerrors option, + // there is an attempt to fix errors. + // + // On ontology mismatch, there is an attempt to merge new ontology to the + // existing database. With -reinstall, the database is cleaned and + // reinstalled. + // + // + + RecoveryPolicy workspacePolicy = Platform.inDevelopmentMode() ? RecoveryPolicy.FixError : RecoveryPolicy.ThrowError; + OntologyRecoveryPolicy ontologyPolicy = Platform.inDevelopmentMode() ? OntologyRecoveryPolicy.Merge : OntologyRecoveryPolicy.ThrowError; + + if (args.contains(SimanticsArguments.RECOVERY_POLICY_FIX_ERRORS)) { + workspacePolicy = RecoveryPolicy.FixError; + ontologyPolicy = OntologyRecoveryPolicy.Merge; + } + + boolean requireSynchronize = true; + + if (args.contains(SimanticsArguments.ONTOLOGY_RECOVERY_POLICY_REINSTALL)) { + ontologyPolicy = OntologyRecoveryPolicy.ReinstallDatabase; + } + + if (args.contains(SimanticsArguments.DO_NOT_SYNCHRONIZE_ONTOLOGIES)) { + requireSynchronize = false; + } + + if (args.contains(SimanticsArguments.DISABLE_INDEX)) { + Indexing.setDefaultDependenciesIndexingEnabled(false); + } + + if (args.contains(SimanticsArguments.SERVER)) { + String serverAddress = args.get(SimanticsArguments.SERVER); + throw new PlatformException("Argument not supported: " + SimanticsArguments.SERVER + " " + serverAddress); + } + + String databaseDriverId = Simantics.getDefaultDatabaseDriver(); + if (args.contains(SimanticsArguments.DATABASE_ID)) { + databaseDriverId = args.get(SimanticsArguments.DATABASE_ID); + Simantics.setDefaultDatabaseDriver(databaseDriverId); + } + + IProgressMonitor mon = null; + if (PROFILE_PLATFORM_STARTUP) + mon = new TimingProgressMonitor(); + SimanticsPlatform.INSTANCE.startUp(databaseDriverId, mon, workspacePolicy, ontologyPolicy, requireSynchronize, new JFaceUserAgent()); + + // Make sure that the default perspective comes from the project if + // the project has set ProjectKeys#DEFAULT_PERSPECTIVE. + // This might go wrong if project features interact with + // PerspectiveRegistry while configuring themselves, since that will + // cause an invocation to #getInitialWindowPerspectiveId() while + // the project has not yet been properly initialized. + getWorkbenchConfigurer().getWorkbench().getPerspectiveRegistry().setDefaultPerspective(getInitialWindowPerspectiveId()); + TimeLogger.log("Completed setting default perspective"); + + return true; + } catch (CancelStartupException e) { + return false; + } catch (PlatformException e) { + boolean hasStackTrace = e.getStackTrace().length > 0; + Throwable ee = e; + while (ee.getCause() != null) { + ee = ee.getCause(); + hasStackTrace = ee.getStackTrace().length > 0; + } + + log.log(new Status(IStatus.ERROR, Activator.PLUGIN_ID, e.getMessage(), hasStackTrace ? e : null)); + if (hasStackTrace) { + new ShowError("Platform Initialization Failed", "Simantics Platform initialization failed:\n\n" + e.getMessage(), e, true); + } else { + StringBuilder sb = new StringBuilder(256); + sb.append(e.getMessage()); + for (Throwable c=e.getCause(); null != c && null != c.getMessage(); c=c.getCause()) + sb.append("\ncause: ").append(c.getMessage()); + new ShowError("Startup Failed", sb.toString(), (Exception) null, true); + } + + return false; + } catch (Exception e) { + log.log(new Status(IStatus.ERROR, Activator.PLUGIN_ID, e.getMessage(), e)); + + Throwable cause = e.getCause(); + if (cause instanceof RebootRequiredException) { + RebootRequiredException rre = (RebootRequiredException) cause; + StringBuilder msg = new StringBuilder(); + msg.append("The application must be restarted after installing the following products:\n"); + for (Product product : rre.products) + msg.append("\t" + product + "\n"); + msg.append("\nThe application will now close."); + MessageDialog.openInformation(null, "Restart Required", msg.toString()); + } else { + new ShowError("Platform Startup Failed", "Simantics Platform startup failed:\n\n" + e.getMessage(), e, true); + } + return false; + } + + } + + /* + * (non-Javadoc) + * + * @see org.eclipse.ui.application.WorkbenchAdvisor#preStartup() + */ + @Override + public void preStartup() { + + // Suspend background jobs while we startup + Job.getJobManager().suspend(); + + // Register the build actions + IProgressService service = PlatformUI.getWorkbench() + .getProgressService(); + ImageDescriptor newImage = IDEInternalWorkbenchImages + .getImageDescriptor(IDEInternalWorkbenchImages.IMG_ETOOL_BUILD_EXEC); + service.registerIconForFamily(newImage, + ResourcesPlugin.FAMILY_MANUAL_BUILD); + service.registerIconForFamily(newImage, + ResourcesPlugin.FAMILY_AUTO_BUILD); + } + + /* + * (non-Javadoc) + * + * @see org.eclipse.ui.application.WorkbenchAdvisor#postStartup() + */ + @Override + public void postStartup() { + try { + refreshFromLocal(); + activateProxyService(); + ((Workbench) PlatformUI.getWorkbench()).registerService( + ISelectionConversionService.class, + new IDESelectionConversionService()); + + initializeSettingsChangeListener(); + Display.getCurrent().addListener(SWT.Settings, + settingsChangeListener); + } finally {// Resume background jobs after we startup + Job.getJobManager().resume(); + } + } + + /** + * Activate the proxy service by obtaining it. + */ + private void activateProxyService() { + Bundle bundle = Platform.getBundle("org.eclipse.ui.ide"); //$NON-NLS-1$ + Object proxyService = null; + if (bundle != null) { + ServiceReference ref = bundle.getBundleContext().getServiceReference(IProxyService.class.getName()); + if (ref != null) + proxyService = bundle.getBundleContext().getService(ref); + } + if (proxyService == null) { + IDEWorkbenchPlugin.log("Proxy service could not be found."); //$NON-NLS-1$ + } + } + + /** + * Initialize the listener for settings changes. + */ + private void initializeSettingsChangeListener() { + settingsChangeListener = new Listener() { + + boolean currentHighContrast = Display.getCurrent() + .getHighContrast(); + + @Override + public void handleEvent(org.eclipse.swt.widgets.Event event) { + if (Display.getCurrent().getHighContrast() == currentHighContrast) + return; + + currentHighContrast = !currentHighContrast; + + // make sure they really want to do this + if (new MessageDialog(null, + IDEWorkbenchMessages.SystemSettingsChange_title, null, + IDEWorkbenchMessages.SystemSettingsChange_message, + MessageDialog.QUESTION, new String[] { + IDEWorkbenchMessages.SystemSettingsChange_yes, + IDEWorkbenchMessages.SystemSettingsChange_no }, + 1).open() == Window.OK) { + PlatformUI.getWorkbench().restart(); + } + } + }; + + } + + /* + * (non-Javadoc) + * + * @see org.eclipse.ui.application.WorkbenchAdvisor#postShutdown + */ + @Override + public void postShutdown() { + if (activityHelper != null) { + activityHelper.shutdown(); + activityHelper = null; + } + if (idleHelper != null) { + idleHelper.shutdown(); + idleHelper = null; + } + if (workspaceUndoMonitor != null) { + workspaceUndoMonitor.shutdown(); + workspaceUndoMonitor = null; + } + if (IDEWorkbenchPlugin.getPluginWorkspace() != null) { + disconnectFromWorkspace(); + } + } + + /* + * (non-Javadoc) + * + * @see org.eclipse.ui.application.WorkbenchAdvisor#preShutdown() + */ + @Override + public boolean preShutdown() { + Display.getCurrent().removeListener(SWT.Settings, + settingsChangeListener); + return super.preShutdown(); + } + + /** + * Return true if the intro plugin is present and false otherwise. + * + * @return boolean + */ + public boolean hasIntro() { + return getWorkbenchConfigurer().getWorkbench().getIntroManager() + .hasIntro(); + } + + private void refreshFromLocal() { + String[] commandLineArgs = Platform.getCommandLineArgs(); + IPreferenceStore store = IDEWorkbenchPlugin.getDefault() + .getPreferenceStore(); + boolean refresh = store + .getBoolean(IDEInternalPreferences.REFRESH_WORKSPACE_ON_STARTUP); + if (!refresh) { + return; + } + + // Do not refresh if it was already done by core on startup. + for (int i = 0; i < commandLineArgs.length; i++) { + if (commandLineArgs[i].equalsIgnoreCase("-refresh")) { //$NON-NLS-1$ + return; + } + } + + final IContainer root = ResourcesPlugin.getWorkspace().getRoot(); + Job job = new WorkspaceJob(IDEWorkbenchMessages.Workspace_refreshing) { + @Override + public IStatus runInWorkspace(IProgressMonitor monitor) + throws CoreException { + root.refreshLocal(IResource.DEPTH_INFINITE, monitor); + return Status.OK_STATUS; + } + }; + job.setRule(root); + job.schedule(); + } + + private static class CancelableProgressMonitorWrapper extends ProgressMonitorWrapper { + private double total = 0; + private ProgressMonitorJobsDialog dialog; + + CancelableProgressMonitorWrapper(IProgressMonitor monitor, + ProgressMonitorJobsDialog dialog) { + super(monitor); + this.dialog = dialog; + } + + /* + * (non-Javadoc) + * @see org.eclipse.core.runtime.ProgressMonitorWrapper#internalWorked(double) + */ + public void internalWorked(double work) { + super.internalWorked(work); + total += work; + updateProgressDetails(); + } + + /* + * (non-Javadoc) + * @see org.eclipse.core.runtime.ProgressMonitorWrapper#worked(int) + */ + public void worked(int work) { + super.worked(work); + total += work; + updateProgressDetails(); + } + + public void beginTask(String name, int totalWork) { + super.beginTask(name, totalWork); + subTask(IDEWorkbenchMessages.IDEWorkbenchAdvisor_preHistoryCompaction); + } + + private void updateProgressDetails() { + if (!isCanceled() && Math.abs(total - 4.0) < 0.0001 /* right before history compacting */) { + subTask(IDEWorkbenchMessages.IDEWorkbenchAdvisor_cancelHistoryPruning); + dialog.setCancelable(true); + } + if (Math.abs(total - 5.0) < 0.0001 /* history compacting finished */) { + subTask(IDEWorkbenchMessages.IDEWorkbenchAdvisor_postHistoryCompaction); + dialog.setCancelable(false); + } + } + } + + private static class CancelableProgressMonitorJobsDialog extends ProgressMonitorJobsDialog { + + public CancelableProgressMonitorJobsDialog(Shell parent) { + super(parent); + } + + /* + * (non-Javadoc) + * @see org.eclipse.ui.internal.progress.ProgressMonitorJobsDialog#createDetailsButton(org.eclipse.swt.widgets.Composite) + */ + protected void createButtonsForButtonBar(Composite parent) { + super.createButtonsForButtonBar(parent); + registerCancelButtonListener(); + } + + public void registerCancelButtonListener() { + cancel.addSelectionListener(new SelectionAdapter() { + public void widgetSelected(SelectionEvent e) { + subTaskLabel.setText(""); //$NON-NLS-1$ + } + }); + } + } + + + final IRunnableWithProgress platformShutdownRunnable = new IRunnableWithProgress() { + /** + * @param monitor + * the progress monitor to use for reporting progress to the + * user, or null indicating that no progress + * should be reported and the operation cannot be cancelled. + */ + @Override + public void run(IProgressMonitor monitor) { + SubMonitor progress = SubMonitor.convert(monitor, SHUT_DOWN_PLATFORM_TASK, 100); + try { + try { + progress.subTask("Platform"); + SimanticsPlatform.INSTANCE.shutdown(progress.newChild(50)); + } catch (PlatformException e) { + Activator.logError("Problems encountered while shutting down Simantics platform, see exception for details.", e); + } + + progress.subTask("Remaining database connections"); + SimanticsUI.closeSessions(); + progress.worked(20); + TimedSessionCache.close(); + progress.worked(20); + + progress.subTask("Thread pools"); + ThreadUtils.shutdown(); + progress.worked(5); + + progress.subTask("Clear index status"); + try { + // Everything ok, clear index dirty state. + DatabaseIndexing.clearAllDirty(); + } catch (IOException e) { + Activator.logError("Problems encountered while refreshing database index states, see exception for details.", e); + } + progress.worked(5); + + progress.setWorkRemaining(0); + } finally { + if (monitor != null) { + monitor.done(); + } + } + } + }; + + /** + * Disconnect from the workspace and close ProCore sessions. + */ + private void disconnectFromWorkspace() { + // save the workspace + final MultiStatus status = new MultiStatus( + IDEWorkbenchPlugin.IDE_WORKBENCH, 1, + IDEWorkbenchMessages.ProblemSavingWorkbench, null); + + final ProgressMonitorJobsDialog p = new CancelableProgressMonitorJobsDialog( + null); + + final boolean applyPolicy = ResourcesPlugin.getWorkspace() + .getDescription().isApplyFileStatePolicy(); + + final IRunnableWithProgress workspaceShutdownRunnable = new IRunnableWithProgress() { + @Override + public void run(IProgressMonitor monitor) { + try { + status.merge(((Workspace) ResourcesPlugin.getWorkspace()).save(true, true, monitor)); + } catch (CoreException e) { + status.merge(e.getStatus()); + } + } + }; + + IRunnableWithProgress shutdownRunnable = new IRunnableWithProgress() { + @Override + public void run(IProgressMonitor monitor) throws InvocationTargetException, InterruptedException { + if (applyPolicy) + monitor = new CancelableProgressMonitorWrapper( + monitor, p); + + SubMonitor progress = SubMonitor.convert(monitor, SHUT_DOWN_TASK, 2); + try { + workspaceShutdownRunnable.run(progress.newChild(1, SubMonitor.SUPPRESS_NONE)); + platformShutdownRunnable.run(progress.newChild(1, SubMonitor.SUPPRESS_NONE)); + } finally { + monitor.done(); + } + } + }; + + try { + new ProgressMonitorJobsDialog(null).run(true, false, shutdownRunnable); + } catch (InvocationTargetException e) { + status.merge(new Status(IStatus.ERROR, + IDEWorkbenchPlugin.IDE_WORKBENCH, 1, + IDEWorkbenchMessages.InternalError, e.getTargetException())); + } catch (InterruptedException e) { + status.merge(new Status(IStatus.ERROR, + IDEWorkbenchPlugin.IDE_WORKBENCH, 1, + IDEWorkbenchMessages.InternalError, e)); + } + ErrorDialog.openError(null, + IDEWorkbenchMessages.ProblemsSavingWorkspace, null, status, + IStatus.ERROR | IStatus.WARNING); + if (!status.isOK()) { + IDEWorkbenchPlugin.log( + IDEWorkbenchMessages.ProblemsSavingWorkspace, status); + } + } + + /* + * (non-Javadoc) + * + * @see org.eclipse.ui.application.WorkbenchAdvisor#getDefaultPageInput + */ + @Override + public IAdaptable getDefaultPageInput() { + return ResourcesPlugin.getWorkspace().getRoot(); + } + + /* + * (non-Javadoc) + * + * @see org.eclipse.ui.application.WorkbenchAdvisor + */ + @Override + public String getInitialWindowPerspectiveId() { + int index = PlatformUI.getWorkbench().getWorkbenchWindowCount() - 1; + + String perspectiveId = null; + AboutInfo[] welcomeInfos = getWelcomePerspectiveInfos(); + if (index >= 0 && welcomeInfos != null && index < welcomeInfos.length) { + perspectiveId = welcomeInfos[index].getWelcomePerspectiveId(); + } + + if (perspectiveId == null && args.contains(SimanticsArguments.PERSPECTIVE)) { + String id = args.get(SimanticsArguments.PERSPECTIVE); + IPerspectiveDescriptor perspective = PlatformUI.getWorkbench().getPerspectiveRegistry().findPerspectiveWithId(id); + if (perspective != null) + perspectiveId = id; + } + + if (perspectiveId == null) { + IProject project = SimanticsUI.peekProject(); + if (project != null) + perspectiveId = project.getHint(ProjectKeys.DEFAULT_PERSPECTIVE); + } + + //System.out.println("Initial perspective: " + perspectiveId); + + return perspectiveId; + } + + /** + * Returns the map of versioned feature ids -> info object for all installed + * features. The format of the versioned feature id (the key of the map) is + * featureId + ":" + versionId. + * + * @return map of versioned feature ids -> info object (key type: + * String, value type: AboutInfo) + * @since 3.0 + */ + private Map computeBundleGroupMap() { + // use tree map to get predicable order + Map ids = new TreeMap(); + + IBundleGroupProvider[] providers = Platform.getBundleGroupProviders(); + for (int i = 0; i < providers.length; ++i) { + IBundleGroup[] groups = providers[i].getBundleGroups(); + for (int j = 0; j < groups.length; ++j) { + IBundleGroup group = groups[j]; + AboutInfo info = new AboutInfo(group); + + String version = info.getVersionId(); + version = version == null ? "0.0.0" //$NON-NLS-1$ + : new Version(version).toString(); + String versionedFeature = group.getIdentifier() + ":" + version; //$NON-NLS-1$ + + ids.put(versionedFeature, info); + } + } + + return ids; + } + + /** + * Returns the ordered map of versioned feature ids -> AboutInfo that are + * new for this session. + * + * @return ordered map of versioned feature ids (key type: + * String) -> infos (value type: + * AboutInfo). + */ + public Map getNewlyAddedBundleGroups() { + if (newlyAddedBundleGroups == null) { + newlyAddedBundleGroups = createNewBundleGroupsMap(); + } + return newlyAddedBundleGroups; + } + + /** + * Updates the old features setting and returns a map of new features. + */ + private Map createNewBundleGroupsMap() { + // retrieve list of installed bundle groups from last session + IDialogSettings settings = IDEWorkbenchPlugin.getDefault() + .getDialogSettings(); + String[] previousFeaturesArray = settings.getArray(INSTALLED_FEATURES); + + // get a map of currently installed bundle groups and store it for next + // session + Map bundleGroups = computeBundleGroupMap(); + String[] currentFeaturesArray = new String[bundleGroups.size()]; + bundleGroups.keySet().toArray(currentFeaturesArray); + settings.put(INSTALLED_FEATURES, currentFeaturesArray); + + // remove the previously known from the current set + if (previousFeaturesArray != null) { + for (int i = 0; i < previousFeaturesArray.length; ++i) { + bundleGroups.remove(previousFeaturesArray[i]); + } + } + + return bundleGroups; + } + + /** + * Declares all IDE-specific workbench images. This includes both "shared" + * images (named in {@link IDE.SharedImages}) and internal images (named in + * {@link org.eclipse.ui.internal.ide.IDEInternalWorkbenchImages}). + * + * @see IWorkbenchConfigurer#declareImage + */ + private void declareWorkbenchImages() { + + final String ICONS_PATH = "$nl$/icons/full/";//$NON-NLS-1$ + final String PATH_ELOCALTOOL = ICONS_PATH + "elcl16/"; // Enabled //$NON-NLS-1$ + + // toolbar + // icons. + final String PATH_DLOCALTOOL = ICONS_PATH + "dlcl16/"; // Disabled //$NON-NLS-1$ + // //$NON-NLS-1$ + // toolbar + // icons. + final String PATH_ETOOL = ICONS_PATH + "etool16/"; // Enabled toolbar //$NON-NLS-1$ + // //$NON-NLS-1$ + // icons. + final String PATH_DTOOL = ICONS_PATH + "dtool16/"; // Disabled toolbar //$NON-NLS-1$ + // //$NON-NLS-1$ + // icons. + final String PATH_OBJECT = ICONS_PATH + "obj16/"; // Model object //$NON-NLS-1$ + // //$NON-NLS-1$ + // icons + final String PATH_WIZBAN = ICONS_PATH + "wizban/"; // Wizard //$NON-NLS-1$ + // //$NON-NLS-1$ + // icons + + Bundle ideBundle = Platform.getBundle(IDEWorkbenchPlugin.IDE_WORKBENCH); + + declareWorkbenchImage(ideBundle, + IDEInternalWorkbenchImages.IMG_ETOOL_BUILD_EXEC, PATH_ETOOL + + "build_exec.gif", false); //$NON-NLS-1$ + declareWorkbenchImage(ideBundle, + IDEInternalWorkbenchImages.IMG_ETOOL_BUILD_EXEC_HOVER, + PATH_ETOOL + "build_exec.gif", false); //$NON-NLS-1$ + declareWorkbenchImage(ideBundle, + IDEInternalWorkbenchImages.IMG_ETOOL_BUILD_EXEC_DISABLED, + PATH_DTOOL + "build_exec.gif", false); //$NON-NLS-1$ + + declareWorkbenchImage(ideBundle, + IDEInternalWorkbenchImages.IMG_ETOOL_SEARCH_SRC, PATH_ETOOL + + "search_src.gif", false); //$NON-NLS-1$ + declareWorkbenchImage(ideBundle, + IDEInternalWorkbenchImages.IMG_ETOOL_SEARCH_SRC_HOVER, + PATH_ETOOL + "search_src.gif", false); //$NON-NLS-1$ + declareWorkbenchImage(ideBundle, + IDEInternalWorkbenchImages.IMG_ETOOL_SEARCH_SRC_DISABLED, + PATH_DTOOL + "search_src.gif", false); //$NON-NLS-1$ + + declareWorkbenchImage(ideBundle, + IDEInternalWorkbenchImages.IMG_ETOOL_NEXT_NAV, PATH_ETOOL + + "next_nav.gif", false); //$NON-NLS-1$ + + declareWorkbenchImage(ideBundle, + IDEInternalWorkbenchImages.IMG_ETOOL_PREVIOUS_NAV, PATH_ETOOL + + "prev_nav.gif", false); //$NON-NLS-1$ + + declareWorkbenchImage(ideBundle, + IDEInternalWorkbenchImages.IMG_WIZBAN_NEWPRJ_WIZ, PATH_WIZBAN + + "newprj_wiz.png", false); //$NON-NLS-1$ + declareWorkbenchImage(ideBundle, + IDEInternalWorkbenchImages.IMG_WIZBAN_NEWFOLDER_WIZ, + PATH_WIZBAN + "newfolder_wiz.png", false); //$NON-NLS-1$ + declareWorkbenchImage(ideBundle, + IDEInternalWorkbenchImages.IMG_WIZBAN_NEWFILE_WIZ, PATH_WIZBAN + + "newfile_wiz.png", false); //$NON-NLS-1$ + + declareWorkbenchImage(ideBundle, + IDEInternalWorkbenchImages.IMG_WIZBAN_IMPORTDIR_WIZ, + PATH_WIZBAN + "importdir_wiz.png", false); //$NON-NLS-1$ + declareWorkbenchImage(ideBundle, + IDEInternalWorkbenchImages.IMG_WIZBAN_IMPORTZIP_WIZ, + PATH_WIZBAN + "importzip_wiz.png", false); //$NON-NLS-1$ + + declareWorkbenchImage(ideBundle, + IDEInternalWorkbenchImages.IMG_WIZBAN_EXPORTDIR_WIZ, + PATH_WIZBAN + "exportdir_wiz.png", false); //$NON-NLS-1$ + declareWorkbenchImage(ideBundle, + IDEInternalWorkbenchImages.IMG_WIZBAN_EXPORTZIP_WIZ, + PATH_WIZBAN + "exportzip_wiz.png", false); //$NON-NLS-1$ + + declareWorkbenchImage(ideBundle, + IDEInternalWorkbenchImages.IMG_WIZBAN_RESOURCEWORKINGSET_WIZ, + PATH_WIZBAN + "workset_wiz.png", false); //$NON-NLS-1$ + + declareWorkbenchImage(ideBundle, + IDEInternalWorkbenchImages.IMG_DLGBAN_SAVEAS_DLG, PATH_WIZBAN + + "saveas_wiz.png", false); //$NON-NLS-1$ + + declareWorkbenchImage(ideBundle, + IDEInternalWorkbenchImages.IMG_DLGBAN_QUICKFIX_DLG, PATH_WIZBAN + + "quick_fix.png", false); //$NON-NLS-1$ + + declareWorkbenchImage(ideBundle, IDE.SharedImages.IMG_OBJ_PROJECT, + PATH_OBJECT + "prj_obj.gif", true); //$NON-NLS-1$ + declareWorkbenchImage(ideBundle, + IDE.SharedImages.IMG_OBJ_PROJECT_CLOSED, PATH_OBJECT + + "cprj_obj.gif", true); //$NON-NLS-1$ + declareWorkbenchImage(ideBundle, IDE.SharedImages.IMG_OPEN_MARKER, + PATH_ELOCALTOOL + "gotoobj_tsk.gif", true); //$NON-NLS-1$ + + declareWorkbenchImage(ideBundle, + IDEInternalWorkbenchImages.IMG_ELCL_QUICK_FIX_ENABLED, + PATH_ELOCALTOOL + "smartmode_co.gif", true); //$NON-NLS-1$ + + declareWorkbenchImage(ideBundle, + IDEInternalWorkbenchImages.IMG_DLCL_QUICK_FIX_DISABLED, + PATH_DLOCALTOOL + "smartmode_co.gif", true); //$NON-NLS-1$ + + // task objects + // declareRegistryImage(IDEInternalWorkbenchImages.IMG_OBJS_HPRIO_TSK, + // PATH_OBJECT+"hprio_tsk.gif"); + // declareRegistryImage(IDEInternalWorkbenchImages.IMG_OBJS_MPRIO_TSK, + // PATH_OBJECT+"mprio_tsk.gif"); + // declareRegistryImage(IDEInternalWorkbenchImages.IMG_OBJS_LPRIO_TSK, + // PATH_OBJECT+"lprio_tsk.gif"); + + declareWorkbenchImage(ideBundle, IDE.SharedImages.IMG_OBJS_TASK_TSK, + PATH_OBJECT + "taskmrk_tsk.gif", true); //$NON-NLS-1$ + declareWorkbenchImage(ideBundle, IDE.SharedImages.IMG_OBJS_BKMRK_TSK, + PATH_OBJECT + "bkmrk_tsk.gif", true); //$NON-NLS-1$ + + declareWorkbenchImage(ideBundle, + IDEInternalWorkbenchImages.IMG_OBJS_COMPLETE_TSK, PATH_OBJECT + + "complete_tsk.gif", true); //$NON-NLS-1$ + declareWorkbenchImage(ideBundle, + IDEInternalWorkbenchImages.IMG_OBJS_INCOMPLETE_TSK, PATH_OBJECT + + "incomplete_tsk.gif", true); //$NON-NLS-1$ + declareWorkbenchImage(ideBundle, + IDEInternalWorkbenchImages.IMG_OBJS_WELCOME_ITEM, PATH_OBJECT + + "welcome_item.gif", true); //$NON-NLS-1$ + declareWorkbenchImage(ideBundle, + IDEInternalWorkbenchImages.IMG_OBJS_WELCOME_BANNER, PATH_OBJECT + + "welcome_banner.gif", true); //$NON-NLS-1$ + declareWorkbenchImage(ideBundle, + IDEInternalWorkbenchImages.IMG_OBJS_ERROR_PATH, PATH_OBJECT + + "error_tsk.gif", true); //$NON-NLS-1$ + declareWorkbenchImage(ideBundle, + IDEInternalWorkbenchImages.IMG_OBJS_WARNING_PATH, PATH_OBJECT + + "warn_tsk.gif", true); //$NON-NLS-1$ + declareWorkbenchImage(ideBundle, + IDEInternalWorkbenchImages.IMG_OBJS_INFO_PATH, PATH_OBJECT + + "info_tsk.gif", true); //$NON-NLS-1$ + + declareWorkbenchImage(ideBundle, + IDEInternalWorkbenchImages.IMG_LCL_FLAT_LAYOUT, PATH_ELOCALTOOL + + "flatLayout.gif", true); //$NON-NLS-1$ + declareWorkbenchImage(ideBundle, + IDEInternalWorkbenchImages.IMG_LCL_HIERARCHICAL_LAYOUT, + PATH_ELOCALTOOL + "hierarchicalLayout.gif", true); //$NON-NLS-1$ + declareWorkbenchImage(ideBundle, + IDEInternalWorkbenchImages.IMG_ETOOL_PROBLEM_CATEGORY, + PATH_ETOOL + "problem_category.gif", true); //$NON-NLS-1$ + /* + declareWorkbenchImage(ideBundle, + IDEInternalWorkbenchImages.IMG_LCL_LINKTO_HELP, PATH_ELOCALTOOL + + "linkto_help.gif", false); //$NON-NLS-1$ + */ + + // synchronization indicator objects + // declareRegistryImage(IDEInternalWorkbenchImages.IMG_OBJS_WBET_STAT, + // PATH_OVERLAY+"wbet_stat.gif"); + // declareRegistryImage(IDEInternalWorkbenchImages.IMG_OBJS_SBET_STAT, + // PATH_OVERLAY+"sbet_stat.gif"); + // declareRegistryImage(IDEInternalWorkbenchImages.IMG_OBJS_CONFLICT_STAT, + // PATH_OVERLAY+"conflict_stat.gif"); + + // content locality indicator objects + // declareRegistryImage(IDEInternalWorkbenchImages.IMG_OBJS_NOTLOCAL_STAT, + // PATH_STAT+"notlocal_stat.gif"); + // declareRegistryImage(IDEInternalWorkbenchImages.IMG_OBJS_LOCAL_STAT, + // PATH_STAT+"local_stat.gif"); + // declareRegistryImage(IDEInternalWorkbenchImages.IMG_OBJS_FILLLOCAL_STAT, + // PATH_STAT+"filllocal_stat.gif"); + } + + /** + * Declares an IDE-specific workbench image. + * + * @param symbolicName + * the symbolic name of the image + * @param path + * the path of the image file; this path is relative to the base + * of the IDE plug-in + * @param shared + * true if this is a shared image, and + * false if this is not a shared image + * @see IWorkbenchConfigurer#declareImage + */ + private void declareWorkbenchImage(Bundle ideBundle, String symbolicName, + String path, boolean shared) { + URL url = FileLocator.find(ideBundle, new Path(path), null); + ImageDescriptor desc = ImageDescriptor.createFromURL(url); + getWorkbenchConfigurer().declareImage(symbolicName, desc, shared); + } + + /* + * (non-Javadoc) + * + * @see org.eclipse.ui.application.WorkbenchAdvisor#getMainPreferencePageId + */ + @Override + public String getMainPreferencePageId() { + // indicate that we want the Workench preference page to be prominent + return WORKBENCH_PREFERENCE_CATEGORY_ID; + } + + /** + * @return the workspace location string, or null if the + * location is not being shown + */ + public String getWorkspaceLocation() { + // read command line, which has priority + IEclipseContext context = getWorkbenchConfigurer().getWorkbench().getService(IEclipseContext.class); + String location = context != null ? (String) context.get(E4Workbench.FORCED_SHOW_LOCATION) : null; + if (location != null) { + return location; + } + // read the preference + if (IDEWorkbenchPlugin.getDefault().getPreferenceStore().getBoolean(IDEInternalPreferences.SHOW_LOCATION)) { + return Platform.getLocation().toOSString(); + } + return null; + } + + /** + * @return the welcome perspective infos, or null if none or + * if they should be ignored due to the new intro being present + */ + public AboutInfo[] getWelcomePerspectiveInfos() { + if (welcomePerspectiveInfos == null) { + // support old welcome perspectives if intro plugin is not present + if (!hasIntro()) { + Map m = getNewlyAddedBundleGroups(); + ArrayList list = new ArrayList(m.size()); + for (Iterator i = m.values().iterator(); i.hasNext();) { + AboutInfo info = i.next(); + if (info != null && info.getWelcomePerspectiveId() != null + && info.getWelcomePageURL() != null) { + list.add(info); + } + } + welcomePerspectiveInfos = new AboutInfo[list.size()]; + list.toArray(welcomePerspectiveInfos); + } + } + return welcomePerspectiveInfos; + } + + /* + * (non-Javadoc) + * + * @see org.eclipse.ui.application.WorkbenchAdvisor#getWorkbenchErrorHandler() + */ + @Override + public AbstractStatusHandler getWorkbenchErrorHandler() { + if (ideWorkbenchErrorHandler == null) { + ideWorkbenchErrorHandler = new IDEWorkbenchErrorHandler( + getWorkbenchConfigurer()); + } + return ideWorkbenchErrorHandler; + } + + /* (non-Javadoc) + * @see org.eclipse.ui.application.WorkbenchAdvisor#eventLoopIdle(org.eclipse.swt.widgets.Display) + */ + @Override + public void eventLoopIdle(Display display) { + if (delayedEventsProcessor != null) + delayedEventsProcessor.catchUp(display); + super.eventLoopIdle(display); + } + +} diff --git a/bundles/org.simantics.workbench/src/org/simantics/workbench/internal/SimanticsWorkbenchApplication.java b/bundles/org.simantics.workbench/src/org/simantics/workbench/internal/SimanticsWorkbenchApplication.java index cc0cc024e..8b0431f38 100644 --- a/bundles/org.simantics.workbench/src/org/simantics/workbench/internal/SimanticsWorkbenchApplication.java +++ b/bundles/org.simantics.workbench/src/org/simantics/workbench/internal/SimanticsWorkbenchApplication.java @@ -1,627 +1,626 @@ -/******************************************************************************* - * Copyright (c) 2007, 2010 Association for Decentralized Information Management - * in Industry THTH ry. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Eclipse Public License v1.0 - * which accompanies this distribution, and is available at - * http://www.eclipse.org/legal/epl-v10.html - * - * Contributors: - * VTT Technical Research Centre of Finland - initial API and implementation - *******************************************************************************/ -package org.simantics.workbench.internal; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.Map; -import java.util.Properties; - -import org.eclipse.core.runtime.IConfigurationElement; -import org.eclipse.core.runtime.IExecutableExtension; -import org.eclipse.core.runtime.IStatus; -import org.eclipse.core.runtime.Platform; -import org.eclipse.core.runtime.Status; -import org.eclipse.equinox.app.IApplication; -import org.eclipse.equinox.app.IApplicationContext; -import org.eclipse.jface.dialogs.Dialog; -import org.eclipse.jface.dialogs.MessageDialog; -import org.eclipse.osgi.service.datalocation.Location; -import org.eclipse.osgi.util.NLS; -import org.eclipse.swt.SWT; -import org.eclipse.swt.widgets.Display; -import org.eclipse.swt.widgets.MessageBox; -import org.eclipse.swt.widgets.Shell; -import org.eclipse.ui.IWorkbench; -import org.eclipse.ui.PlatformUI; -import org.eclipse.ui.application.WorkbenchAdvisor; -import org.eclipse.ui.internal.WorkbenchPlugin; -import org.eclipse.ui.internal.ide.ChooseWorkspaceData; -import org.eclipse.ui.internal.ide.ChooseWorkspaceDialog; -import org.eclipse.ui.internal.ide.IDEWorkbenchMessages; -import org.eclipse.ui.internal.ide.IDEWorkbenchPlugin; -import org.eclipse.ui.internal.ide.StatusUtil; -import org.simantics.application.arguments.ApplicationUtils; -import org.simantics.application.arguments.Arguments; -import org.simantics.application.arguments.IArgumentFactory; -import org.simantics.application.arguments.IArguments; -import org.simantics.application.arguments.SimanticsArguments; -import org.simantics.db.management.ISessionContextProvider; -import org.simantics.db.management.ISessionContextProviderSource; -import org.simantics.db.management.SessionContextProvider; -import org.simantics.db.management.SingleSessionContextProviderSource; -import org.simantics.ui.SimanticsUI; -import org.simantics.ui.WorkbenchWindowSessionContextProviderSource; -import org.simantics.utils.ui.BundleUtils; - - -/** - * The "main program" for the Eclipse IDE. - * - * @since 3.0 - */ -public class SimanticsWorkbenchApplication implements IApplication, IExecutableExtension { - - /** - * The name of the folder containing metadata information for the workspace. - */ - public static final String METADATA_FOLDER = ".metadata"; //$NON-NLS-1$ - - private static final String VERSION_FILENAME = "version.ini"; //$NON-NLS-1$ - - private static final String WORKSPACE_VERSION_KEY = "org.eclipse.core.runtime"; //$NON-NLS-1$ - - private static final String WORKSPACE_VERSION_VALUE = "1"; //$NON-NLS-1$ - - private static final String PROP_EXIT_CODE = "eclipse.exitcode"; //$NON-NLS-1$ - - /** - * A special return code that will be recognized by the launcher and used to - * restart the workbench. - */ - private static final Integer EXIT_RELAUNCH = new Integer(24); - - /** - * A special return code that will be recognized by the PDE launcher and used to - * show an error dialog if the workspace is locked. - */ - private static final Integer EXIT_WORKSPACE_LOCKED = new Integer(15); - - /** - * Creates a new IDE application. - */ - public SimanticsWorkbenchApplication() { - // There is nothing to do for WorkbenchApplication - } - - public WorkbenchAdvisor createWorkbenchAdvisor(IArguments args, DelayedEventsProcessor processor) { - return new SimanticsWorkbenchAdvisor(args, processor); - } - - /* (non-Javadoc) - * @see org.eclipse.equinox.app.IApplication#start(org.eclipse.equinox.app.IApplicationContext context) - */ - @Override - public Object start(IApplicationContext appContext) throws Exception { - ApplicationUtils.loadSystemProperties(BundleUtils.find(Activator.PLUGIN_ID, "system.properties")); - IArguments args = parseArguments((String[]) appContext.getArguments().get(IApplicationContext.APPLICATION_ARGS)); - - Display display = createDisplay(); - // processor must be created before we start event loop - DelayedEventsProcessor processor = new DelayedEventsProcessor(display); - - try { - Object argCheck = verifyArguments(args); - if (argCheck != null) - return argCheck; - - // look and see if there's a splash shell we can parent off of - Shell shell = WorkbenchPlugin.getSplashShell(display); - if (shell != null) { - // should should set the icon and message for this shell to be the - // same as the chooser dialog - this will be the guy that lives in - // the task bar and without these calls you'd have the default icon - // with no message. - shell.setText(ChooseWorkspaceDialog.getWindowTitle()); - shell.setImages(Dialog.getDefaultImages()); - } - - Object instanceLocationCheck = checkInstanceLocation(shell, appContext.getArguments(), args); - if (instanceLocationCheck != null) { - WorkbenchPlugin.unsetSplashShell(display); - Platform.endSplash(); - return instanceLocationCheck; - } - - final ISessionContextProvider provider = new SessionContextProvider(null); - final ISessionContextProviderSource contextProviderSource = new SingleSessionContextProviderSource(provider); - //final ISessionContextProviderSource contextProviderSource = new WorkbenchWindowSessionContextProviderSource(PlatformUI.getWorkbench()); - SimanticsUI.setSessionContextProviderSource(contextProviderSource); - org.simantics.db.layer0.internal.SimanticsInternal.setSessionContextProviderSource(contextProviderSource); - org.simantics.Simantics.setSessionContextProviderSource(contextProviderSource); - - // create the workbench with this advisor and run it until it exits - // N.B. createWorkbench remembers the advisor, and also registers - // the workbench globally so that all UI plug-ins can find it using - // PlatformUI.getWorkbench() or AbstractUIPlugin.getWorkbench() - int returnCode = PlatformUI.createAndRunWorkbench(display, - createWorkbenchAdvisor(args, processor)); - - // the workbench doesn't support relaunch yet (bug 61809) so - // for now restart is used, and exit data properties are checked - // here to substitute in the relaunch return code if needed - if (returnCode != PlatformUI.RETURN_RESTART) { - return EXIT_OK; - } - - // if the exit code property has been set to the relaunch code, then - // return that code now, otherwise this is a normal restart - return EXIT_RELAUNCH.equals(Integer.getInteger(PROP_EXIT_CODE)) ? EXIT_RELAUNCH - : EXIT_RESTART; - } finally { - if (display != null) { - display.dispose(); - } - Location instanceLoc = Platform.getInstanceLocation(); - if (instanceLoc != null) - instanceLoc.release(); - } - } - - /*************************************************************************/ - - private IArguments parseArguments(String[] args) { - IArgumentFactory[] accepted = { - SimanticsArguments.RECOVERY_POLICY_FIX_ERRORS, - SimanticsArguments.ONTOLOGY_RECOVERY_POLICY_REINSTALL, - SimanticsArguments.DEFAULT_WORKSPACE_LOCATION, - SimanticsArguments.WORKSPACE_CHOOSER, - SimanticsArguments.WORKSPACE_NO_REMEMBER, - SimanticsArguments.PERSPECTIVE, - SimanticsArguments.SERVER, - SimanticsArguments.NEW_MODEL, - SimanticsArguments.EXPERIMENT, - SimanticsArguments.DISABLE_INDEX, - SimanticsArguments.DATABASE_ID, - }; - IArguments result = Arguments.parse(args, accepted); - return result; - } - - private Object verifyArguments(IArguments args) { - StringBuilder report = new StringBuilder(); - -// if (args.contains(SimanticsArguments.NEW_PROJECT)) { -// if (args.contains(SimanticsArguments.PROJECT)) { -// exclusiveArguments(report, SimanticsArguments.PROJECT, SimanticsArguments.NEW_PROJECT); -// } -// // Must have a server to checkout from when creating a new -// // project right from the beginning. -// if (!args.contains(SimanticsArguments.SERVER)) { -// missingArgument(report, SimanticsArguments.SERVER); -// } -// } else if (args.contains(SimanticsArguments.PROJECT)) { -// // To load a project, a server must be defined to checkout from -// if (!args.contains(SimanticsArguments.SERVER)) { -// missingArgument(report, SimanticsArguments.SERVER); -// } -// } - - // NEW_MODEL and MODEL arguments are optional - // EXPERIMENT argument is optional - - String result = report.toString(); - boolean valid = result.length() == 0; - - if (!valid) { - String msg = NLS.bind(Messages.Application_1, result); - MessageDialog.openInformation(null, Messages.Application_2, msg); - } - return valid ? null : EXIT_OK; - } - -// private void exclusiveArguments(StringBuilder sb, IArgumentFactory arg1, IArgumentFactory arg2) { -// sb.append(NLS.bind(Messages.Application_3, arg1.getArgument(), arg2.getArgument())); -// sb.append('\n'); -// } -// -// private void missingArgument(StringBuilder sb, IArgumentFactory arg) { -// sb.append(NLS.bind(Messages.Application_0, arg.getArgument())); -// sb.append('\n'); -// } - - /*************************************************************************/ - - /** - * Creates the display used by the application. - * - * @return the display used by the application - */ - protected Display createDisplay() { - return PlatformUI.createDisplay(); - } - - /* (non-Javadoc) - * @see org.eclipse.core.runtime.IExecutableExtension#setInitializationData(org.eclipse.core.runtime.IConfigurationElement, java.lang.String, java.lang.Object) - */ - @Override - public void setInitializationData(IConfigurationElement config, - String propertyName, Object data) { - // There is nothing to do for ProConfApplication - } - - /** - * Return true if a valid workspace path has been set and false otherwise. - * Prompt for and set the path if possible and required. - * @param applicationArguments - * - * @return true if a valid instance location has been set and false - * otherwise - */ - private Object checkInstanceLocation(Shell shell, Map applicationArguments, IArguments args) { - // -data @none was specified but an ide requires workspace - Location instanceLoc = Platform.getInstanceLocation(); - if (instanceLoc == null) { - MessageDialog - .openError( - shell, - IDEWorkbenchMessages.IDEApplication_workspaceMandatoryTitle, - IDEWorkbenchMessages.IDEApplication_workspaceMandatoryMessage); - return EXIT_OK; - } - - // -data "/valid/path", workspace already set - // This information is stored in configuration/.settings/org.eclipse.ui.ide.prefs - if (instanceLoc.isSet()) { - // make sure the meta data version is compatible (or the user has - // chosen to overwrite it). - if (!checkValidWorkspace(shell, instanceLoc.getURL())) { - return EXIT_OK; - } - - // at this point its valid, so try to lock it and update the - // metadata version information if successful - try { - if (instanceLoc.lock()) { - writeWorkspaceVersion(); - return null; - } - - // we failed to create the directory. - // Two possibilities: - // 1. directory is already in use - // 2. directory could not be created - File workspaceDirectory = new File(instanceLoc.getURL().getFile()); - if (workspaceDirectory.exists()) { - if (isDevLaunchMode(applicationArguments)) { - return EXIT_WORKSPACE_LOCKED; - } - MessageDialog.openError( - shell, - IDEWorkbenchMessages.IDEApplication_workspaceCannotLockTitle, - IDEWorkbenchMessages.IDEApplication_workspaceCannotLockMessage); - } else { - MessageDialog.openError( - shell, - IDEWorkbenchMessages.IDEApplication_workspaceCannotBeSetTitle, - IDEWorkbenchMessages.IDEApplication_workspaceCannotBeSetMessage); - } - } catch (IOException e) { - IDEWorkbenchPlugin.log("Could not obtain lock for workspace location", //$NON-NLS-1$ - e); - MessageDialog - .openError( - shell, - IDEWorkbenchMessages.InternalError, - e.getMessage()); - } - return EXIT_OK; - } - - // -data @noDefault or -data not specified, prompt and set - ChooseWorkspaceData launchData = null; - if (args.contains(SimanticsArguments.DEFAULT_WORKSPACE_LOCATION)) { - launchData = new ChooseWorkspaceData(args.get(SimanticsArguments.DEFAULT_WORKSPACE_LOCATION)); - } else { - launchData = new ChooseWorkspaceData(instanceLoc.getDefault()); - } - - boolean force = args.contains(SimanticsArguments.WORKSPACE_CHOOSER); - boolean suppressAskAgain = args.contains(SimanticsArguments.WORKSPACE_NO_REMEMBER); - - while (true) { - URL workspaceUrl = promptForWorkspace(shell, launchData, force, suppressAskAgain); - if (workspaceUrl == null) { - return EXIT_OK; - } - - // if there is an error with the first selection, then force the - // dialog to open to give the user a chance to correct - force = true; - - try { - // the operation will fail if the url is not a valid - // instance data area, so other checking is unneeded - if (instanceLoc.setURL(workspaceUrl, true)) { - launchData.writePersistedData(); - writeWorkspaceVersion(); - return null; - } - } catch (IllegalStateException e) { - MessageDialog - .openError( - shell, - IDEWorkbenchMessages.IDEApplication_workspaceCannotBeSetTitle, - IDEWorkbenchMessages.IDEApplication_workspaceCannotBeSetMessage); - return EXIT_OK; - } - - // by this point it has been determined that the workspace is - // already in use -- force the user to choose again - MessageDialog.openError(shell, IDEWorkbenchMessages.IDEApplication_workspaceInUseTitle, - IDEWorkbenchMessages.IDEApplication_workspaceInUseMessage); - } - } - - private static boolean isDevLaunchMode(Map args) { - // see org.eclipse.pde.internal.core.PluginPathFinder.isDevLaunchMode() - if (Boolean.getBoolean("eclipse.pde.launch")) //$NON-NLS-1$ - return true; - return args.containsKey("-pdelaunch"); //$NON-NLS-1$ - } - - /** - * Open a workspace selection dialog on the argument shell, populating the - * argument data with the user's selection. Perform first level validation - * on the selection by comparing the version information. This method does - * not examine the runtime state (e.g., is the workspace already locked?). - * - * @param shell - * @param launchData - * @param force - * setting to true makes the dialog open regardless of the - * showDialog value - * @return An URL storing the selected workspace or null if the user has - * canceled the launch operation. - */ - private URL promptForWorkspace(Shell shell, ChooseWorkspaceData launchData, - boolean force, boolean suppressAskAgain) { - URL url = null; - do { - // okay to use the shell now - this is the splash shell - new ChooseWorkspaceDialog(shell, launchData, suppressAskAgain, true).prompt(force); - String instancePath = launchData.getSelection(); - if (instancePath == null) { - return null; - } - - // the dialog is not forced on the first iteration, but is on every - // subsequent one -- if there was an error then the user needs to be - // allowed to fix it - force = true; - - // 70576: don't accept empty input - if (instancePath.length() <= 0) { - MessageDialog - .openError( - shell, - IDEWorkbenchMessages.IDEApplication_workspaceEmptyTitle, - IDEWorkbenchMessages.IDEApplication_workspaceEmptyMessage); - continue; - } - - // create the workspace if it does not already exist - File workspace = new File(instancePath); - if (!workspace.exists()) { - workspace.mkdir(); - } - - try { - // Don't use File.toURL() since it adds a leading slash that Platform does not - // handle properly. See bug 54081 for more details. - String path = workspace.getAbsolutePath().replace( - File.separatorChar, '/'); - url = new URL("file", null, path); //$NON-NLS-1$ - } catch (MalformedURLException e) { - MessageDialog - .openError( - shell, - IDEWorkbenchMessages.IDEApplication_workspaceInvalidTitle, - IDEWorkbenchMessages.IDEApplication_workspaceInvalidMessage); - continue; - } - } while (!checkValidWorkspace(shell, url)); - - return url; - } - - /** - * Return true if the argument directory is ok to use as a workspace and - * false otherwise. A version check will be performed, and a confirmation - * box may be displayed on the argument shell if an older version is - * detected. - * - * @return true if the argument URL is ok to use as a workspace and false - * otherwise. - */ - private boolean checkValidWorkspace(Shell shell, URL url) { - // a null url is not a valid workspace - if (url == null) { - return false; - } - - String version = readWorkspaceVersion(url); - - // if the version could not be read, then there is not any existing - // workspace data to trample, e.g., perhaps its a new directory that - // is just starting to be used as a workspace - if (version == null) { - return true; - } - - final int ide_version = Integer.parseInt(WORKSPACE_VERSION_VALUE); - int workspace_version = Integer.parseInt(version); - - // equality test is required since any version difference (newer - // or older) may result in data being trampled - if (workspace_version == ide_version) { - return true; - } - - // At this point workspace has been detected to be from a version - // other than the current ide version -- find out if the user wants - // to use it anyhow. - int severity; - String title; - String message; - if (workspace_version < ide_version) { - // Workspace < IDE. Update must be possible without issues, - // so only inform user about it. - severity = MessageDialog.INFORMATION; - title = IDEWorkbenchMessages.IDEApplication_versionTitle_olderWorkspace; - message = NLS.bind(IDEWorkbenchMessages.IDEApplication_versionMessage_olderWorkspace, url.getFile()); - } else { - // Workspace > IDE. It must have been opened with a newer IDE version. - // Downgrade might be problematic, so warn user about it. - severity = MessageDialog.WARNING; - title = IDEWorkbenchMessages.IDEApplication_versionTitle_newerWorkspace; - message = NLS.bind(IDEWorkbenchMessages.IDEApplication_versionMessage_newerWorkspace, url.getFile()); - } - - MessageBox mbox = new MessageBox(shell, SWT.OK | SWT.CANCEL - | SWT.ICON_WARNING | SWT.APPLICATION_MODAL); - mbox.setText(title); - mbox.setMessage(message); - return mbox.open() == SWT.OK; - } - - /** - * Look at the argument URL for the workspace's version information. Return - * that version if found and null otherwise. - */ - private static String readWorkspaceVersion(URL workspace) { - File versionFile = getVersionFile(workspace, false); - if (versionFile == null || !versionFile.exists()) { - return null; - } - - try { - // Although the version file is not spec'ed to be a Java properties - // file, it happens to follow the same format currently, so using - // Properties to read it is convenient. - Properties props = new Properties(); - FileInputStream is = new FileInputStream(versionFile); - try { - props.load(is); - } finally { - is.close(); - } - - return props.getProperty(WORKSPACE_VERSION_KEY); - } catch (IOException e) { - IDEWorkbenchPlugin.log("Could not read version file", new Status( //$NON-NLS-1$ - IStatus.ERROR, IDEWorkbenchPlugin.IDE_WORKBENCH, - IStatus.ERROR, - e.getMessage() == null ? "" : e.getMessage(), //$NON-NLS-1$, - e)); - return null; - } - } - - /** - * Write the version of the metadata into a known file overwriting any - * existing file contents. Writing the version file isn't really crucial, - * so the function is silent about failure - */ - private static void writeWorkspaceVersion() { - Location instanceLoc = Platform.getInstanceLocation(); - if (instanceLoc == null || instanceLoc.isReadOnly()) { - return; - } - - File versionFile = getVersionFile(instanceLoc.getURL(), true); - if (versionFile == null) { - return; - } - - OutputStream output = null; - try { - String versionLine = WORKSPACE_VERSION_KEY + '=' - + WORKSPACE_VERSION_VALUE; - - output = new FileOutputStream(versionFile); - output.write(versionLine.getBytes("UTF-8")); //$NON-NLS-1$ - } catch (IOException e) { - IDEWorkbenchPlugin.log("Could not write version file", //$NON-NLS-1$ - StatusUtil.newStatus(IStatus.ERROR, e.getMessage(), e)); - } finally { - try { - if (output != null) { - output.close(); - } - } catch (IOException e) { - // do nothing - } - } - } - - /** - * The version file is stored in the metadata area of the workspace. This - * method returns an URL to the file or null if the directory or file does - * not exist (and the create parameter is false). - * - * @param create - * If the directory and file does not exist this parameter - * controls whether it will be created. - * @return An url to the file or null if the version file does not exist or - * could not be created. - */ - private static File getVersionFile(URL workspaceUrl, boolean create) { - if (workspaceUrl == null) { - return null; - } - - try { - // make sure the directory exists - File metaDir = new File(workspaceUrl.getPath(), METADATA_FOLDER); - if (!metaDir.exists() && (!create || !metaDir.mkdir())) { - return null; - } - - // make sure the file exists - File versionFile = new File(metaDir, VERSION_FILENAME); - if (!versionFile.exists() - && (!create || !versionFile.createNewFile())) { - return null; - } - - return versionFile; - } catch (IOException e) { - // cannot log because instance area has not been set - return null; - } - } - - /* (non-Javadoc) - * @see org.eclipse.equinox.app.IApplication#stop() - */ - @Override - public void stop() { - final IWorkbench workbench = PlatformUI.getWorkbench(); - if (workbench == null) - return; - final Display display = workbench.getDisplay(); - display.syncExec(new Runnable() { - @Override - public void run() { - if (!display.isDisposed()) - workbench.close(); - } - }); - } - -} +/******************************************************************************* + * Copyright (c) 2007, 2010 Association for Decentralized Information Management + * in Industry THTH ry. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * VTT Technical Research Centre of Finland - initial API and implementation + *******************************************************************************/ +package org.simantics.workbench.internal; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Map; +import java.util.Properties; + +import org.eclipse.core.runtime.IConfigurationElement; +import org.eclipse.core.runtime.IExecutableExtension; +import org.eclipse.core.runtime.IStatus; +import org.eclipse.core.runtime.Platform; +import org.eclipse.core.runtime.Status; +import org.eclipse.equinox.app.IApplication; +import org.eclipse.equinox.app.IApplicationContext; +import org.eclipse.jface.dialogs.Dialog; +import org.eclipse.jface.dialogs.MessageDialog; +import org.eclipse.osgi.service.datalocation.Location; +import org.eclipse.osgi.util.NLS; +import org.eclipse.swt.SWT; +import org.eclipse.swt.widgets.Display; +import org.eclipse.swt.widgets.MessageBox; +import org.eclipse.swt.widgets.Shell; +import org.eclipse.ui.IWorkbench; +import org.eclipse.ui.PlatformUI; +import org.eclipse.ui.application.WorkbenchAdvisor; +import org.eclipse.ui.internal.WorkbenchPlugin; +import org.eclipse.ui.internal.ide.ChooseWorkspaceData; +import org.eclipse.ui.internal.ide.ChooseWorkspaceDialog; +import org.eclipse.ui.internal.ide.IDEWorkbenchMessages; +import org.eclipse.ui.internal.ide.IDEWorkbenchPlugin; +import org.eclipse.ui.internal.ide.StatusUtil; +import org.simantics.application.arguments.ApplicationUtils; +import org.simantics.application.arguments.Arguments; +import org.simantics.application.arguments.IArgumentFactory; +import org.simantics.application.arguments.IArguments; +import org.simantics.application.arguments.SimanticsArguments; +import org.simantics.db.management.ISessionContextProvider; +import org.simantics.db.management.ISessionContextProviderSource; +import org.simantics.db.management.SessionContextProvider; +import org.simantics.db.management.SingleSessionContextProviderSource; +import org.simantics.ui.SimanticsUI; +import org.simantics.utils.ui.BundleUtils; + + +/** + * The "main program" for the Eclipse IDE. + * + * @since 3.0 + */ +public class SimanticsWorkbenchApplication implements IApplication, IExecutableExtension { + + /** + * The name of the folder containing metadata information for the workspace. + */ + public static final String METADATA_FOLDER = ".metadata"; //$NON-NLS-1$ + + private static final String VERSION_FILENAME = "version.ini"; //$NON-NLS-1$ + + private static final String WORKSPACE_VERSION_KEY = "org.eclipse.core.runtime"; //$NON-NLS-1$ + + private static final String WORKSPACE_VERSION_VALUE = "1"; //$NON-NLS-1$ + + private static final String PROP_EXIT_CODE = "eclipse.exitcode"; //$NON-NLS-1$ + + /** + * A special return code that will be recognized by the launcher and used to + * restart the workbench. + */ + private static final Integer EXIT_RELAUNCH = new Integer(24); + + /** + * A special return code that will be recognized by the PDE launcher and used to + * show an error dialog if the workspace is locked. + */ + private static final Integer EXIT_WORKSPACE_LOCKED = new Integer(15); + + /** + * Creates a new IDE application. + */ + public SimanticsWorkbenchApplication() { + // There is nothing to do for WorkbenchApplication + } + + public WorkbenchAdvisor createWorkbenchAdvisor(IArguments args, DelayedEventsProcessor processor) { + return new SimanticsWorkbenchAdvisor(args, processor); + } + + /* (non-Javadoc) + * @see org.eclipse.equinox.app.IApplication#start(org.eclipse.equinox.app.IApplicationContext context) + */ + @Override + public Object start(IApplicationContext appContext) throws Exception { + ApplicationUtils.loadSystemProperties(BundleUtils.find(Activator.PLUGIN_ID, "system.properties")); + IArguments args = parseArguments((String[]) appContext.getArguments().get(IApplicationContext.APPLICATION_ARGS)); + + Display display = createDisplay(); + // processor must be created before we start event loop + DelayedEventsProcessor processor = new DelayedEventsProcessor(display); + + try { + Object argCheck = verifyArguments(args); + if (argCheck != null) + return argCheck; + + // look and see if there's a splash shell we can parent off of + Shell shell = WorkbenchPlugin.getSplashShell(display); + if (shell != null) { + // should should set the icon and message for this shell to be the + // same as the chooser dialog - this will be the guy that lives in + // the task bar and without these calls you'd have the default icon + // with no message. + shell.setText(ChooseWorkspaceDialog.getWindowTitle()); + shell.setImages(Dialog.getDefaultImages()); + } + + Object instanceLocationCheck = checkInstanceLocation(shell, appContext.getArguments(), args); + if (instanceLocationCheck != null) { + WorkbenchPlugin.unsetSplashShell(display); + Platform.endSplash(); + return instanceLocationCheck; + } + + final ISessionContextProvider provider = new SessionContextProvider(null); + final ISessionContextProviderSource contextProviderSource = new SingleSessionContextProviderSource(provider); + //final ISessionContextProviderSource contextProviderSource = new WorkbenchWindowSessionContextProviderSource(PlatformUI.getWorkbench()); + SimanticsUI.setSessionContextProviderSource(contextProviderSource); + org.simantics.db.layer0.internal.SimanticsInternal.setSessionContextProviderSource(contextProviderSource); + org.simantics.Simantics.setSessionContextProviderSource(contextProviderSource); + + // create the workbench with this advisor and run it until it exits + // N.B. createWorkbench remembers the advisor, and also registers + // the workbench globally so that all UI plug-ins can find it using + // PlatformUI.getWorkbench() or AbstractUIPlugin.getWorkbench() + int returnCode = PlatformUI.createAndRunWorkbench(display, + createWorkbenchAdvisor(args, processor)); + + // the workbench doesn't support relaunch yet (bug 61809) so + // for now restart is used, and exit data properties are checked + // here to substitute in the relaunch return code if needed + if (returnCode != PlatformUI.RETURN_RESTART) { + return EXIT_OK; + } + + // if the exit code property has been set to the relaunch code, then + // return that code now, otherwise this is a normal restart + return EXIT_RELAUNCH.equals(Integer.getInteger(PROP_EXIT_CODE)) ? EXIT_RELAUNCH + : EXIT_RESTART; + } finally { + if (display != null) { + display.dispose(); + } + Location instanceLoc = Platform.getInstanceLocation(); + if (instanceLoc != null) + instanceLoc.release(); + } + } + + /*************************************************************************/ + + private IArguments parseArguments(String[] args) { + IArgumentFactory[] accepted = { + SimanticsArguments.RECOVERY_POLICY_FIX_ERRORS, + SimanticsArguments.ONTOLOGY_RECOVERY_POLICY_REINSTALL, + SimanticsArguments.DEFAULT_WORKSPACE_LOCATION, + SimanticsArguments.WORKSPACE_CHOOSER, + SimanticsArguments.WORKSPACE_NO_REMEMBER, + SimanticsArguments.PERSPECTIVE, + SimanticsArguments.SERVER, + SimanticsArguments.NEW_MODEL, + SimanticsArguments.EXPERIMENT, + SimanticsArguments.DISABLE_INDEX, + SimanticsArguments.DATABASE_ID, + }; + IArguments result = Arguments.parse(args, accepted); + return result; + } + + private Object verifyArguments(IArguments args) { + StringBuilder report = new StringBuilder(); + +// if (args.contains(SimanticsArguments.NEW_PROJECT)) { +// if (args.contains(SimanticsArguments.PROJECT)) { +// exclusiveArguments(report, SimanticsArguments.PROJECT, SimanticsArguments.NEW_PROJECT); +// } +// // Must have a server to checkout from when creating a new +// // project right from the beginning. +// if (!args.contains(SimanticsArguments.SERVER)) { +// missingArgument(report, SimanticsArguments.SERVER); +// } +// } else if (args.contains(SimanticsArguments.PROJECT)) { +// // To load a project, a server must be defined to checkout from +// if (!args.contains(SimanticsArguments.SERVER)) { +// missingArgument(report, SimanticsArguments.SERVER); +// } +// } + + // NEW_MODEL and MODEL arguments are optional + // EXPERIMENT argument is optional + + String result = report.toString(); + boolean valid = result.length() == 0; + + if (!valid) { + String msg = NLS.bind(Messages.Application_1, result); + MessageDialog.openInformation(null, Messages.Application_2, msg); + } + return valid ? null : EXIT_OK; + } + +// private void exclusiveArguments(StringBuilder sb, IArgumentFactory arg1, IArgumentFactory arg2) { +// sb.append(NLS.bind(Messages.Application_3, arg1.getArgument(), arg2.getArgument())); +// sb.append('\n'); +// } +// +// private void missingArgument(StringBuilder sb, IArgumentFactory arg) { +// sb.append(NLS.bind(Messages.Application_0, arg.getArgument())); +// sb.append('\n'); +// } + + /*************************************************************************/ + + /** + * Creates the display used by the application. + * + * @return the display used by the application + */ + protected Display createDisplay() { + return PlatformUI.createDisplay(); + } + + /* (non-Javadoc) + * @see org.eclipse.core.runtime.IExecutableExtension#setInitializationData(org.eclipse.core.runtime.IConfigurationElement, java.lang.String, java.lang.Object) + */ + @Override + public void setInitializationData(IConfigurationElement config, + String propertyName, Object data) { + // There is nothing to do for ProConfApplication + } + + /** + * Return true if a valid workspace path has been set and false otherwise. + * Prompt for and set the path if possible and required. + * @param applicationArguments + * + * @return true if a valid instance location has been set and false + * otherwise + */ + private Object checkInstanceLocation(Shell shell, Map applicationArguments, IArguments args) { + // -data @none was specified but an ide requires workspace + Location instanceLoc = Platform.getInstanceLocation(); + if (instanceLoc == null) { + MessageDialog + .openError( + shell, + IDEWorkbenchMessages.IDEApplication_workspaceMandatoryTitle, + IDEWorkbenchMessages.IDEApplication_workspaceMandatoryMessage); + return EXIT_OK; + } + + // -data "/valid/path", workspace already set + // This information is stored in configuration/.settings/org.eclipse.ui.ide.prefs + if (instanceLoc.isSet()) { + // make sure the meta data version is compatible (or the user has + // chosen to overwrite it). + if (!checkValidWorkspace(shell, instanceLoc.getURL())) { + return EXIT_OK; + } + + // at this point its valid, so try to lock it and update the + // metadata version information if successful + try { + if (instanceLoc.lock()) { + writeWorkspaceVersion(); + return null; + } + + // we failed to create the directory. + // Two possibilities: + // 1. directory is already in use + // 2. directory could not be created + File workspaceDirectory = new File(instanceLoc.getURL().getFile()); + if (workspaceDirectory.exists()) { + if (isDevLaunchMode(applicationArguments)) { + return EXIT_WORKSPACE_LOCKED; + } + MessageDialog.openError( + shell, + IDEWorkbenchMessages.IDEApplication_workspaceCannotLockTitle, + IDEWorkbenchMessages.IDEApplication_workspaceCannotLockMessage); + } else { + MessageDialog.openError( + shell, + IDEWorkbenchMessages.IDEApplication_workspaceCannotBeSetTitle, + IDEWorkbenchMessages.IDEApplication_workspaceCannotBeSetMessage); + } + } catch (IOException e) { + IDEWorkbenchPlugin.log("Could not obtain lock for workspace location", //$NON-NLS-1$ + e); + MessageDialog + .openError( + shell, + IDEWorkbenchMessages.InternalError, + e.getMessage()); + } + return EXIT_OK; + } + + // -data @noDefault or -data not specified, prompt and set + ChooseWorkspaceData launchData = null; + if (args.contains(SimanticsArguments.DEFAULT_WORKSPACE_LOCATION)) { + launchData = new ChooseWorkspaceData(args.get(SimanticsArguments.DEFAULT_WORKSPACE_LOCATION)); + } else { + launchData = new ChooseWorkspaceData(instanceLoc.getDefault()); + } + + boolean force = args.contains(SimanticsArguments.WORKSPACE_CHOOSER); + boolean suppressAskAgain = args.contains(SimanticsArguments.WORKSPACE_NO_REMEMBER); + + while (true) { + URL workspaceUrl = promptForWorkspace(shell, launchData, force, suppressAskAgain); + if (workspaceUrl == null) { + return EXIT_OK; + } + + // if there is an error with the first selection, then force the + // dialog to open to give the user a chance to correct + force = true; + + try { + // the operation will fail if the url is not a valid + // instance data area, so other checking is unneeded + if (instanceLoc.setURL(workspaceUrl, true)) { + launchData.writePersistedData(); + writeWorkspaceVersion(); + return null; + } + } catch (IllegalStateException e) { + MessageDialog + .openError( + shell, + IDEWorkbenchMessages.IDEApplication_workspaceCannotBeSetTitle, + IDEWorkbenchMessages.IDEApplication_workspaceCannotBeSetMessage); + return EXIT_OK; + } + + // by this point it has been determined that the workspace is + // already in use -- force the user to choose again + MessageDialog.openError(shell, IDEWorkbenchMessages.IDEApplication_workspaceInUseTitle, + IDEWorkbenchMessages.IDEApplication_workspaceInUseMessage); + } + } + + private static boolean isDevLaunchMode(Map args) { + // see org.eclipse.pde.internal.core.PluginPathFinder.isDevLaunchMode() + if (Boolean.getBoolean("eclipse.pde.launch")) //$NON-NLS-1$ + return true; + return args.containsKey("-pdelaunch"); //$NON-NLS-1$ + } + + /** + * Open a workspace selection dialog on the argument shell, populating the + * argument data with the user's selection. Perform first level validation + * on the selection by comparing the version information. This method does + * not examine the runtime state (e.g., is the workspace already locked?). + * + * @param shell + * @param launchData + * @param force + * setting to true makes the dialog open regardless of the + * showDialog value + * @return An URL storing the selected workspace or null if the user has + * canceled the launch operation. + */ + private URL promptForWorkspace(Shell shell, ChooseWorkspaceData launchData, + boolean force, boolean suppressAskAgain) { + URL url = null; + do { + // okay to use the shell now - this is the splash shell + new ChooseWorkspaceDialog(shell, launchData, suppressAskAgain, true).prompt(force); + String instancePath = launchData.getSelection(); + if (instancePath == null) { + return null; + } + + // the dialog is not forced on the first iteration, but is on every + // subsequent one -- if there was an error then the user needs to be + // allowed to fix it + force = true; + + // 70576: don't accept empty input + if (instancePath.length() <= 0) { + MessageDialog + .openError( + shell, + IDEWorkbenchMessages.IDEApplication_workspaceEmptyTitle, + IDEWorkbenchMessages.IDEApplication_workspaceEmptyMessage); + continue; + } + + // create the workspace if it does not already exist + File workspace = new File(instancePath); + if (!workspace.exists()) { + workspace.mkdir(); + } + + try { + // Don't use File.toURL() since it adds a leading slash that Platform does not + // handle properly. See bug 54081 for more details. + String path = workspace.getAbsolutePath().replace( + File.separatorChar, '/'); + url = new URL("file", null, path); //$NON-NLS-1$ + } catch (MalformedURLException e) { + MessageDialog + .openError( + shell, + IDEWorkbenchMessages.IDEApplication_workspaceInvalidTitle, + IDEWorkbenchMessages.IDEApplication_workspaceInvalidMessage); + continue; + } + } while (!checkValidWorkspace(shell, url)); + + return url; + } + + /** + * Return true if the argument directory is ok to use as a workspace and + * false otherwise. A version check will be performed, and a confirmation + * box may be displayed on the argument shell if an older version is + * detected. + * + * @return true if the argument URL is ok to use as a workspace and false + * otherwise. + */ + private boolean checkValidWorkspace(Shell shell, URL url) { + // a null url is not a valid workspace + if (url == null) { + return false; + } + + String version = readWorkspaceVersion(url); + + // if the version could not be read, then there is not any existing + // workspace data to trample, e.g., perhaps its a new directory that + // is just starting to be used as a workspace + if (version == null) { + return true; + } + + final int ide_version = Integer.parseInt(WORKSPACE_VERSION_VALUE); + int workspace_version = Integer.parseInt(version); + + // equality test is required since any version difference (newer + // or older) may result in data being trampled + if (workspace_version == ide_version) { + return true; + } + + // At this point workspace has been detected to be from a version + // other than the current ide version -- find out if the user wants + // to use it anyhow. + int severity; + String title; + String message; + if (workspace_version < ide_version) { + // Workspace < IDE. Update must be possible without issues, + // so only inform user about it. + severity = MessageDialog.INFORMATION; + title = IDEWorkbenchMessages.IDEApplication_versionTitle_olderWorkspace; + message = NLS.bind(IDEWorkbenchMessages.IDEApplication_versionMessage_olderWorkspace, url.getFile()); + } else { + // Workspace > IDE. It must have been opened with a newer IDE version. + // Downgrade might be problematic, so warn user about it. + severity = MessageDialog.WARNING; + title = IDEWorkbenchMessages.IDEApplication_versionTitle_newerWorkspace; + message = NLS.bind(IDEWorkbenchMessages.IDEApplication_versionMessage_newerWorkspace, url.getFile()); + } + + MessageBox mbox = new MessageBox(shell, SWT.OK | SWT.CANCEL + | SWT.ICON_WARNING | SWT.APPLICATION_MODAL); + mbox.setText(title); + mbox.setMessage(message); + return mbox.open() == SWT.OK; + } + + /** + * Look at the argument URL for the workspace's version information. Return + * that version if found and null otherwise. + */ + private static String readWorkspaceVersion(URL workspace) { + File versionFile = getVersionFile(workspace, false); + if (versionFile == null || !versionFile.exists()) { + return null; + } + + try { + // Although the version file is not spec'ed to be a Java properties + // file, it happens to follow the same format currently, so using + // Properties to read it is convenient. + Properties props = new Properties(); + FileInputStream is = new FileInputStream(versionFile); + try { + props.load(is); + } finally { + is.close(); + } + + return props.getProperty(WORKSPACE_VERSION_KEY); + } catch (IOException e) { + IDEWorkbenchPlugin.log("Could not read version file", new Status( //$NON-NLS-1$ + IStatus.ERROR, IDEWorkbenchPlugin.IDE_WORKBENCH, + IStatus.ERROR, + e.getMessage() == null ? "" : e.getMessage(), //$NON-NLS-1$, + e)); + return null; + } + } + + /** + * Write the version of the metadata into a known file overwriting any + * existing file contents. Writing the version file isn't really crucial, + * so the function is silent about failure + */ + private static void writeWorkspaceVersion() { + Location instanceLoc = Platform.getInstanceLocation(); + if (instanceLoc == null || instanceLoc.isReadOnly()) { + return; + } + + File versionFile = getVersionFile(instanceLoc.getURL(), true); + if (versionFile == null) { + return; + } + + OutputStream output = null; + try { + String versionLine = WORKSPACE_VERSION_KEY + '=' + + WORKSPACE_VERSION_VALUE; + + output = new FileOutputStream(versionFile); + output.write(versionLine.getBytes("UTF-8")); //$NON-NLS-1$ + } catch (IOException e) { + IDEWorkbenchPlugin.log("Could not write version file", //$NON-NLS-1$ + StatusUtil.newStatus(IStatus.ERROR, e.getMessage(), e)); + } finally { + try { + if (output != null) { + output.close(); + } + } catch (IOException e) { + // do nothing + } + } + } + + /** + * The version file is stored in the metadata area of the workspace. This + * method returns an URL to the file or null if the directory or file does + * not exist (and the create parameter is false). + * + * @param create + * If the directory and file does not exist this parameter + * controls whether it will be created. + * @return An url to the file or null if the version file does not exist or + * could not be created. + */ + private static File getVersionFile(URL workspaceUrl, boolean create) { + if (workspaceUrl == null) { + return null; + } + + try { + // make sure the directory exists + File metaDir = new File(workspaceUrl.getPath(), METADATA_FOLDER); + if (!metaDir.exists() && (!create || !metaDir.mkdir())) { + return null; + } + + // make sure the file exists + File versionFile = new File(metaDir, VERSION_FILENAME); + if (!versionFile.exists() + && (!create || !versionFile.createNewFile())) { + return null; + } + + return versionFile; + } catch (IOException e) { + // cannot log because instance area has not been set + return null; + } + } + + /* (non-Javadoc) + * @see org.eclipse.equinox.app.IApplication#stop() + */ + @Override + public void stop() { + final IWorkbench workbench = PlatformUI.getWorkbench(); + if (workbench == null) + return; + final Display display = workbench.getDisplay(); + display.syncExec(new Runnable() { + @Override + public void run() { + if (!display.isDisposed()) + workbench.close(); + } + }); + } + +} diff --git a/bundles/org.simantics/META-INF/MANIFEST.MF b/bundles/org.simantics/META-INF/MANIFEST.MF index 5cd86db54..90161d598 100644 --- a/bundles/org.simantics/META-INF/MANIFEST.MF +++ b/bundles/org.simantics/META-INF/MANIFEST.MF @@ -19,7 +19,8 @@ Require-Bundle: org.eclipse.core.runtime;visibility:=reexport, org.simantics.scl.osgi;bundle-version="1.0.4", org.simantics.scl.compiler;bundle-version="0.4.0", org.simantics.platform.ui.ontology;bundle-version="1.0.0", - org.simantics.db.procore;bundle-version="1.1.0" + org.simantics.db.procore;bundle-version="1.1.0", + org.slf4j.api Bundle-RequiredExecutionEnvironment: JavaSE-1.8 Bundle-ActivationPolicy: lazy Export-Package: org.simantics, diff --git a/bundles/org.simantics/src/org/simantics/Logger.java b/bundles/org.simantics/src/org/simantics/Logger.java deleted file mode 100644 index 3001de3f6..000000000 --- a/bundles/org.simantics/src/org/simantics/Logger.java +++ /dev/null @@ -1,126 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2007, 2010 Association for Decentralized Information Management - * in Industry THTH ry. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Eclipse Public License v1.0 - * which accompanies this distribution, and is available at - * http://www.eclipse.org/legal/epl-v10.html - * - * Contributors: - * VTT Technical Research Centre of Finland - initial API and implementation - *******************************************************************************/ -package org.simantics; - -import java.util.Properties; - -import org.simantics.internal.Activator; - - - -public class Logger { - public static final boolean ECHO = true; - public static final Properties defaultProperties = new Properties(); - static { - defaultProperties.put("log4j.rootCategory", "INFO, default"); - defaultProperties.put("log4j.appender.default", "org.apache.log4j.FileAppender"); - defaultProperties.put("log4j.appender.default.File", Activator.LOG_FILE_NAME); - defaultProperties.put("log4j.appender.default.append", "false"); - defaultProperties.put("log4j.appender.default.layout", "org.apache.log4j.PatternLayout"); - defaultProperties.put("log4j.appender.default.layout.ConversionPattern", "%d{ISO8601} %-6r [%15.15t] %-5p %30.30c - %m%n"); - } - private static LogManager defaultLogManager = new LogManager(defaultProperties); - private static final Logger defaultErrorLogger = new Logger(LogManager.class); - private org.apache.log4j.Logger logger; - Logger(Class clazz) { - logger = defaultLogManager.getLogger(clazz); - } - - /** - * Log a trace event. - * - * @param message message of the trace - * @param exception the exception, or null - */ - public void logTrace(String message, Throwable exception) { - // Errors are much more useful with a stack trace! - if (exception == null) { - exception = new RuntimeException(); - } - logger.trace(message, exception); - } - - /** - * Log an info event. - * - * @param message message of the info - * @param exception the exception, or null - */ - public void logInfo(String message, Throwable exception) { - // Errors are much more useful with a stack trace! - if (exception == null) { - exception = new RuntimeException(); - } - logger.info(message, exception); - } - - /** - * Log a warning event. - * - * @param message message of the warning - * @param exception the exception, or null - */ - public void logWarning(String message, Throwable exception) { - // Errors are much more useful with a stack trace! - if (exception == null) { - exception = new RuntimeException(); - } - logger.warn(message, exception); - } - - /** - * Log an error event. - * - * @param message message of the error - * @param exception the exception, or null - */ - public void logError(String message, Throwable exception) { - // Errors are much more useful with a stack trace! - if (exception == null) { - exception = new RuntimeException(); - } - logger.error(message, exception); - } - - public static Logger getDefault() { - return defaultErrorLogger; - } - - public static LogManager getDefaultLogManager() { - return defaultLogManager; - } - public static void defaultLogError(Throwable exception) { - getDefault().logError(exception.getLocalizedMessage(), exception); - if(ECHO) exception.printStackTrace(); - } - public static void defaultLogError(String message) { - getDefault().logError(message, null); - if(ECHO) - System.err.println(message); - } - public static void defaultLogError(String message, Throwable exception) { - getDefault().logError(message, exception); - if(ECHO) - System.err.println(message); - } - public static void defaultLogInfo(String message) { - getDefault().logInfo(message, null); - if(ECHO) - System.err.println(message); - } - public static void defaultLogTrace(String message) { - getDefault().logTrace(message, null); - if(ECHO) - System.err.println(message); - } - -} diff --git a/bundles/org.simantics/src/org/simantics/Simantics.java b/bundles/org.simantics/src/org/simantics/Simantics.java index 50110cb03..daa9e7e10 100644 --- a/bundles/org.simantics/src/org/simantics/Simantics.java +++ b/bundles/org.simantics/src/org/simantics/Simantics.java @@ -56,6 +56,8 @@ import org.simantics.utils.FileService; import org.simantics.utils.FileUtils; import org.simantics.utils.TempFiles; import org.simantics.utils.threads.ThreadUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A facade for accessing basic Simantics platform services. Usable without a @@ -67,6 +69,7 @@ import org.simantics.utils.threads.ThreadUtils; * TODO: duplicate of org.simantics.db.layer0.util.Simantics, do something about this!! */ public class Simantics { + private static final Logger LOGGER = LoggerFactory.getLogger(Simantics.class); /** * Default database driver ID @@ -440,7 +443,7 @@ public class Simantics { try { IndexUtils.flushIndexCaches(progress, session); } catch (Exception e) { - Logger.defaultLogError(e); + LOGGER.error("Flushing index caches failed.", e); } } diff --git a/bundles/org.simantics/src/org/simantics/SimanticsPlatform.java b/bundles/org.simantics/src/org/simantics/SimanticsPlatform.java index 7e6bd9dd9..11013eaf7 100644 --- a/bundles/org.simantics/src/org/simantics/SimanticsPlatform.java +++ b/bundles/org.simantics/src/org/simantics/SimanticsPlatform.java @@ -1,1041 +1,1047 @@ -/******************************************************************************* - * Copyright (c) 2007, 2010 Association for Decentralized Information Management - * in Industry THTH ry. - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Eclipse Public License v1.0 - * which accompanies this distribution, and is available at - * http://www.eclipse.org/legal/epl-v10.html - * - * Contributors: - * VTT Technical Research Centre of Finland - initial API and implementation - *******************************************************************************/ -package org.simantics; - -import static org.simantics.db.common.utils.Transaction.commit; -import static org.simantics.db.common.utils.Transaction.endTransaction; -import static org.simantics.db.common.utils.Transaction.readGraph; -import static org.simantics.db.common.utils.Transaction.startTransaction; -import static org.simantics.db.common.utils.Transaction.writeGraph; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Properties; -import java.util.Set; -import java.util.TreeMap; -import java.util.UUID; - -import org.eclipse.core.runtime.ILog; -import org.eclipse.core.runtime.IProduct; -import org.eclipse.core.runtime.IProgressMonitor; -import org.eclipse.core.runtime.IStatus; -import org.eclipse.core.runtime.NullProgressMonitor; -import org.eclipse.core.runtime.Platform; -import org.eclipse.core.runtime.Status; -import org.eclipse.core.runtime.SubMonitor; -import org.eclipse.osgi.service.resolver.BundleDescription; -import org.simantics.databoard.Bindings; -import org.simantics.databoard.Databoard; -import org.simantics.datatypes.literal.Font; -import org.simantics.datatypes.literal.RGB; -import org.simantics.db.Driver; -import org.simantics.db.Driver.Management; -import org.simantics.db.Manager; -import org.simantics.db.ReadGraph; -import org.simantics.db.Resource; -import org.simantics.db.Session; -import org.simantics.db.SessionModel; -import org.simantics.db.UndoContext; -import org.simantics.db.VirtualGraph; -import org.simantics.db.WriteGraph; -import org.simantics.db.common.request.ObjectsWithType; -import org.simantics.db.common.request.Queries; -import org.simantics.db.common.request.WriteRequest; -import org.simantics.db.common.request.WriteResultRequest; -import org.simantics.db.common.utils.Transaction; -import org.simantics.db.exception.ClusterSetExistException; -import org.simantics.db.exception.DatabaseException; -import org.simantics.db.exception.ResourceNotFoundException; -import org.simantics.db.indexing.DatabaseIndexing; -import org.simantics.db.layer0.genericrelation.DependenciesRelation; -import org.simantics.db.layer0.util.SimanticsClipboardImpl; -import org.simantics.db.layer0.util.SimanticsKeys; -import org.simantics.db.layer0.util.TGTransferableGraphSource; -import org.simantics.db.layer0.variable.VariableRepository; -import org.simantics.db.management.SessionContext; -import org.simantics.db.request.Read; -import org.simantics.db.service.LifecycleSupport.LifecycleListener; -import org.simantics.db.service.LifecycleSupport.LifecycleState; -import org.simantics.db.service.QueryControl; -import org.simantics.db.service.UndoRedoSupport; -import org.simantics.db.service.VirtualGraphSupport; -import org.simantics.db.service.XSupport; -import org.simantics.graph.db.GraphDependencyAnalyzer; -import org.simantics.graph.db.GraphDependencyAnalyzer.IU; -import org.simantics.graph.db.GraphDependencyAnalyzer.IdentityNode; -import org.simantics.graph.db.IImportAdvisor; -import org.simantics.graph.db.TransferableGraphs; -import org.simantics.graph.diff.Diff; -import org.simantics.graph.diff.TransferableGraphDelta1; -import org.simantics.internal.Activator; -import org.simantics.internal.startup.StartupExtensions; -import org.simantics.layer0.Layer0; -import org.simantics.operation.Layer0X; -import org.simantics.project.IProject; -import org.simantics.project.ProjectFeatures; -import org.simantics.project.ProjectKeys; -import org.simantics.project.Projects; -import org.simantics.project.exception.ProjectException; -import org.simantics.project.features.registry.GroupReference; -import org.simantics.project.management.DatabaseManagement; -import org.simantics.project.management.GraphBundle; -import org.simantics.project.management.GraphBundleEx; -import org.simantics.project.management.GraphBundleRef; -import org.simantics.project.management.PlatformUtil; -import org.simantics.project.management.ServerManager; -import org.simantics.project.management.ServerManagerFactory; -import org.simantics.project.management.WorkspaceUtil; -import org.simantics.utils.FileUtils; -import org.simantics.utils.datastructures.Pair; -import org.simantics.utils.logging.TimeLogger; - -/** - * SimanticsPlatform performs procedures required in order to get simantics - * workbench into operational state. This consists of the following steps: - *

    - *
  • Asserting there is Database - *
    • - *
  • Starting Database process - *
    • - *
  • Opening a session to Database process - *
    • - *
  • Asserting required ontologies or other transferable graphs are installed in the database - *
    • - *
  • Asserting required project is installed in the database - *
    • - *
  • Asserting Simantics Features are installed in the database - *
    • - *
  • Asserting Simantics Features are installed to the project - *
    • - *
  • Shutdown: Save Session, Close session, Kill Database process - *
    • - *
- * - * @author Toni Kalajainen - */ -public class SimanticsPlatform implements LifecycleListener { - - /** - * The policy is relevant when developing Simantics from Eclipse IDE. - * It is applied when the ontology in the database of a workspace doesn't match - * a newer ontology in the Eclipse workspace. - */ - public static enum OntologyRecoveryPolicy { ThrowError, Merge, ReinstallDatabase } - - /** - * This policy dictates how the Simantics platform startup should react if - * the started workspace is not set up properly. The alternatives are to - * just throw an error and fail or to attempt all possible measures to fix - * the encountered problems. - */ - public static enum RecoveryPolicy { ThrowError, FixError } - - /** Singleton instance, started in SimanticsWorkbenchAdvisor */ - public static final SimanticsPlatform INSTANCE = new SimanticsPlatform(); - - /** Set to true when the Simantics Platform is in good-and-go condition */ - public boolean running; - - /** Database Session */ - public Session session; - private Management databasebManagement; - - /** Database session context */ - public SessionContext sessionContext; - - /** Project identifier in Database */ - public String projectURI; - - /** Project name */ - public String projectName; - - /** Project resource */ - public Resource projectResource; - - /** Session specific bindings */ - public SimanticsBindings simanticsBindings; - public SimanticsBindings simanticsBindings2; - - public Thread mainThread; - - /** - * The {@link IProject} activated by - * {@link #startUp(IProgressMonitor, RecoveryPolicy, OntologyRecoveryPolicy, ServerAddress, PlatformUserAgent)} - */ - private IProject project; - - protected ILog log; - - /** - * Create a new simantics plaform manager in uninitialized state and - * with default policies.

- */ - public SimanticsPlatform() { - log = Platform.getLog(Activator.getBundleContext().getBundle()); - mainThread = Thread.currentThread(); - } - - public String getApplicationClientId() { - IProduct product = Platform.getProduct(); - if(product == null) return "noProduct";//UUID.randomUUID().toString(); - String application = product.getApplication(); - return application != null ? application : UUID.randomUUID().toString(); - } - - private Session setupDatabase(String databaseDriverId, IProgressMonitor progressMonitor, RecoveryPolicy workspacePolicy, PlatformUserAgent userAgent) throws PlatformException { - if (progressMonitor == null) - progressMonitor = new NullProgressMonitor(); - File dbLocation = Platform.getLocation().append("db").toFile(); - ServerManager serverManager; - try { - serverManager = ServerManagerFactory.create(databaseDriverId, dbLocation.getAbsolutePath()); - } catch (DatabaseException | IOException e) { - throw new PlatformException("Failed to initialize Server Manager", e); - } - progressMonitor.beginTask("Setting up Simantics Database", 100); - progressMonitor.setTaskName("Asserting Database is installed."); - String msg = "Failed to initialize Simantics database."; - try { - // Create database - log.log(new Status(IStatus.INFO, Activator.PLUGIN_ID, "Creating database at " + dbLocation)); - progressMonitor.setTaskName("Creating database at " + dbLocation); - databasebManagement = serverManager.getManagement(dbLocation); - databasebManagement.create(); - // Create layer0. - return serverManager.createDatabase(dbLocation); - } catch (DatabaseException e) { - throw new PlatformException(msg, e); - } catch (Throwable e) { - throw new PlatformException(msg, e); - } finally { - progressMonitor.worked(20); - } - } - - public void synchronizeOntologies(IProgressMonitor progressMonitor, OntologyRecoveryPolicy ontologyPolicy, boolean requireSynchronize) throws PlatformException { - - if (progressMonitor == null) progressMonitor = new NullProgressMonitor(); - - final DatabaseManagement mgmt = new DatabaseManagement(); - - PlatformUtil.compileAllDynamicOntologies(); - - progressMonitor.setTaskName("Asserting all ontologies are installed"); - final Map platformTGs = new HashMap(); - try { - - // Get a list of bundles installed into the database - progressMonitor.subTask("find installed bundles from database"); - Map installedTGs = new HashMap(); - for (GraphBundle b : session.syncRequest( mgmt.GraphBundleQuery )) { - installedTGs.put(GraphBundleRef.of(b), GraphBundleEx.extend(b)); - } - - if(!requireSynchronize && installedTGs.size() > 1 && !Platform.inDevelopmentMode()) return; -// if(installedTGs.size() > 1) return; - - // Get a list of all bundles in the platform (Bundle Context) - List tgs = new ArrayList(); - progressMonitor.subTask("load all transferable graphs from platform"); - PlatformUtil.getAllGraphs(tgs); - progressMonitor.subTask("extend bundles to compile versions"); - for (GraphBundle b : tgs) { - GraphBundleEx gbe = GraphBundleEx.extend(b); - gbe.build(); - platformTGs.put(GraphBundleRef.of(b), gbe); - } - - // Compile a list of TGs that need to be installed or reinstalled in the database - progressMonitor.subTask("check bundle reinstallation demand"); - List installTGs = new ArrayList(); - // Create list of TGs to update, - Map reinstallTGs = new TreeMap(); - for (Entry e : platformTGs.entrySet()) { - GraphBundleRef key = e.getKey(); - GraphBundleEx platformBundle = e.getValue(); - GraphBundleEx existingBundle = installedTGs.get(key); - -// System.out.println("GraphBundleRef key=" + key.toString()); - - if (existingBundle == null) { - // Bundle did not exist in the database, put it into list of bundles to install - installTGs.add(platformBundle); - } - else { - // Bundle exists in the database - boolean platformBundleIsNewer = existingBundle.getVersion().compareTo(platformBundle.getVersion())<0; - if (!platformBundleIsNewer) - continue; - // Check hash of transferable graph to know whether to update or not. - if (platformBundle.getHashcode() == existingBundle.getHashcode()) - continue; - //System.out.println("Ontology hashcodes do not match: platform bundle=" - // + platformBundle.getVersionedId() + ", hash=" + platformBundle.getHashcode() - // + "; existing bundle=" + existingBundle.getVersionedId() + ", hash=" + existingBundle.getHashcode()); - reinstallTGs.put(platformBundle, existingBundle); - } - } - // INSTALL - // Database is missing graphs - if (!installTGs.isEmpty() || !reinstallTGs.isEmpty()) { - session.getService(XSupport.class).setServiceMode(true, true); - - // Throw error - if (ontologyPolicy == OntologyRecoveryPolicy.ThrowError) { - StringBuilder sb = new StringBuilder("The following graphs are not installed in the database: "); - if (!installTGs.isEmpty()) { - int i = 0; - for (GraphBundleEx e : installTGs) { - if (i>0) sb.append(", "); - i++; - sb.append(e.toString()); - } - sb.append(" is missing from the database.\n"); - } - if (!reinstallTGs.isEmpty()) { - int i = 0; - for (Entry e : reinstallTGs.entrySet()) { - if (i>0) sb.append(", "); - i++; - sb.append(e.getKey().toString()); - } - sb.append(" Database/Platform Bundle version mismatch.\n"); - } - sb.append("Hint: Use -fixErrors to install the graphs."); - throw new PlatformException(sb.toString()); - } - // Reinstall database - if (ontologyPolicy == OntologyRecoveryPolicy.ReinstallDatabase) { - log.log(new Status(IStatus.INFO, Activator.PLUGIN_ID, "Reinstalling the database.")); - // TODO Install DB - // Stop Session - // Kill Process - // Delete Database - // Create Database - // Start Database - // Open Session - // Install TGs - throw new PlatformException("Reinstalling Database, NOT IMPLEMENTED"); - } - - if (ontologyPolicy == OntologyRecoveryPolicy.Merge) { - progressMonitor.subTask("Merging ontology changes"); - // Sort missing TGs into install order - GraphDependencyAnalyzer analyzer = new GraphDependencyAnalyzer(); - for(GraphBundle tg : installTGs) analyzer.addGraph(tg, tg.getGraph()); - for(GraphBundle tg : reinstallTGs.keySet()) analyzer.addGraph(tg, tg.getGraph()); - if(!analyzer.analyzeDependency()) { - Collection> problems = analyzer.getConflicts(); - StringBuilder sb = new StringBuilder(); - for (Pair problem : problems) { - sb.append("Conflict with "+problem.first+" and "+problem.second+".\n"); - } - throw new PlatformException(sb.toString()); - } - else if(!session.syncRequest( analyzer.queryExternalDependenciesSatisfied )) { - Collection unsatisfiedDependencies = analyzer.getUnsatisfiedDependencies(); - StringBuilder sb = new StringBuilder(); - for (IdentityNode dep: unsatisfiedDependencies) { - sb.append("Unsatisfied Dependency "+dep+". Required by\n"); - for(IU iu : GraphDependencyAnalyzer.toCollection(dep.getRequires())) { - sb.append(" " + ((GraphBundle)iu.getId()).getId() + "\n"); - } - } - throw new PlatformException(sb.toString()); - } - - List sortedBundles = analyzer.getSortedGraphs(); - if(!sortedBundles.isEmpty()) { - - session.syncRequest(new WriteRequest() { - @Override - public void perform(WriteGraph graph) throws DatabaseException { - try { - graph.newClusterSet(graph.getRootLibrary()); - } catch (ClusterSetExistException e) { - // Cluster set exist already, no problem. - } - graph.setClusterSet4NewResource(graph.getRootLibrary()); - graph.flushCluster(); - } - }); - - boolean mergedOntologies = false; - - // Install TGs - for(final GraphBundle tg : sortedBundles) { - - final IImportAdvisor advisor = new OntologyImportAdvisor(tg, mgmt); - final GraphBundle oldTG = reinstallTGs.get(tg); - - if (oldTG==null) { - - boolean createImmutable = tg.getImmutable(); - session.getService(XSupport.class).setServiceMode(true, createImmutable); - - // Install TG - log.log(new Status(IStatus.INFO, Activator.PLUGIN_ID, "Installing "+tg.toString()+" - "+tg.getName())); - TransferableGraphs.importGraph1(session, new TGTransferableGraphSource(tg.getGraph()), advisor, null); - } else { - // Merge TG - startTransaction(session, false); - TransferableGraphDelta1 delta = new Diff(oldTG.getGraph(), tg.getGraph()).diff(); - final long[] oldResources = oldTG.getResourceArray(); - boolean changes = TransferableGraphs.hasChanges(readGraph(), oldResources, delta); - endTransaction(); - if (!changes) { - //log.log(new Status(IStatus.INFO, Activator.PLUGIN_ID, "Nothing to merge for "+tg.toString())); - continue; - } - - log.log(new Status(IStatus.INFO, Activator.PLUGIN_ID, "Merging new version of "+tg.toString())); - - startTransaction(session, true); - - //delta.print(); - try { - - - long[] resourceArray = TransferableGraphs.applyDelta(writeGraph(), oldResources, delta); - tg.setResourceArray(resourceArray); - mgmt.setGraphBundleEntry(tg); - commit(); - mergedOntologies = true; - } catch (Throwable t) { - throw new PlatformException(t); - } finally { - endTransaction(); - } - } - } - - session.syncRequest(new WriteRequest() { - @Override - public void perform(WriteGraph graph) throws DatabaseException { - graph.setClusterSet4NewResource(graph.getRootLibrary()); - graph.flushCluster(); - } - }); - - if (mergedOntologies) - DatabaseIndexing.deleteAllIndexes(); - } - - TimeLogger.log("Ontologies synchronized."); - - } - session.getService(XSupport.class).setServiceMode(false, false); - } - progressMonitor.worked(20); - } catch (IOException e) { - throw new PlatformException(e); - } catch (DatabaseException e) { - throw new PlatformException(e); - } - - } - - public boolean assertConfiguration(IProgressMonitor progressMonitor, RecoveryPolicy workspacePolicy) throws PlatformException { - - if (progressMonitor == null) progressMonitor = new NullProgressMonitor(); - - File workspaceLocation = Platform.getLocation().toFile(); - - boolean installProject = false; - progressMonitor.setTaskName("Asserting simantics.cfg is installed"); - try { - File propertyFile = new File(workspaceLocation, "simantics.cfg"); - Properties properties; - try { - properties = WorkspaceUtil.readProperties(propertyFile); - } catch (IOException e) { - if (workspacePolicy == RecoveryPolicy.ThrowError) throw new PlatformException("Could not load "+propertyFile); - - // Create a project and write Property file - properties = new Properties(); - properties.setProperty("project_uri", "http://Projects/Development%20Project"); - properties.setProperty("project_name", "Development Project"); - WorkspaceUtil.writeProperties(propertyFile, properties); - installProject |= true; - } - projectURI = properties.getProperty("project_uri"); - projectName = properties.getProperty("project_name"); - progressMonitor.worked(10); - } catch (IOException e) { - throw new PlatformException(e); - } - - return installProject; - - } - - public boolean assertProject(IProgressMonitor progressMonitor, RecoveryPolicy workspacePolicy, boolean installProject) throws PlatformException { - - if (progressMonitor == null) progressMonitor = new NullProgressMonitor(); - - final DatabaseManagement mgmt = new DatabaseManagement(); - - progressMonitor.setTaskName("Asserting project resource exists in the database"); - try { - projectResource = session.syncRequest( Queries.resource( projectURI ) ); - } catch (ResourceNotFoundException nfe) { - // Project was not found - if (workspacePolicy == RecoveryPolicy.ThrowError) - throw new PlatformException("Project Resource "+projectURI+" is not found in the database."); - // Create empty project with no features - try { - Transaction.startTransaction(session, true); - try { - // The project needs to be created mutable. - session.getService(XSupport.class).setServiceMode(true, false); - - ArrayList empty = new ArrayList(); - projectResource = mgmt.createProject(projectName, empty); - installProject |= true; - - session.getService(XSupport.class).setServiceMode(false, false); - Transaction.commit(); - } finally { - Transaction.endTransaction(); - } - //session.getService( LifecycleSupport.class ).save(); - } catch (DatabaseException e) { - throw new PlatformException("Failed to create "+projectURI, e); - } - } catch (DatabaseException e) { - throw new PlatformException("Failed to create "+projectURI, e); - } - progressMonitor.worked(10); - - return installProject; - - } - - public void updateInstalledGroups(IProgressMonitor progressMonitor, boolean installProject) throws PlatformException { - - if (installProject) - { - // Attach all feature groups available in platform to created project - progressMonitor.setTaskName("Install all features"); - Set publishedFeatureGroups = ProjectFeatures.getInstallGroupsOfPublishedFeatures(); - Collection groupsWithoutVersion = GroupReference.stripVersions(publishedFeatureGroups); - - // final List Platform_Features = new ArrayList(); - // - // // Convert graph instances - // Collection platformGraphs = new ArrayList(); - // for (GraphBundleEx e : platformTGs.values()) platformGraphs.add( e.getGraph() ); - // IGraph graph = Graphs.createGraph(platformGraphs); - // - // Res PublishedProjectFeatures = UriUtils.uriToPath( ProjectResource.URIs.PublishedProjectFeatures ); - // Path HasFeature = UriUtils.uriToPath( ProjectResource.URIs.HasFeature ); - // for(Res feature : graph.getObjects(PublishedProjectFeatures, HasFeature)) { - // System.out.println("Installing Project Feature: "+feature.toString()); - // Platform_Features.add( feature.toString() ); - // } - - try { - Transaction.startTransaction(session, true); - try { - // for (String feature : Platform_Features) { - // try { - // getResource(feature); - // } catch(ResourceNotFoundException e) { - // System.out.println(feature+" not found"); - // } - // mgmt.installFeature(projectResource, feature); - // } - Projects.setProjectInstalledGroups(writeGraph(), projectResource, groupsWithoutVersion); - Transaction.commit(); - } finally { - Transaction.endTransaction(); - } - //session.getService( LifecycleSupport.class ).save(); - } catch(DatabaseException ae) { - throw new PlatformException("Failed to install features", ae); - } - progressMonitor.worked(10); - } - - } - - public void assertSessionModel(IProgressMonitor progressMonitor) throws PlatformException { - - Properties properties = session.getService(Properties.class); - final String clientId = properties.getProperty("clientId"); - - try { - - // Currently this needs to be done before data becomes available - VirtualGraphSupport support = session.getService(VirtualGraphSupport.class); - VirtualGraph activations = support.getWorkspacePersistent("activations"); - - Resource sessionModel = session.syncRequest(new Read() { - - @Override - public Resource perform(ReadGraph graph) throws DatabaseException { - - Layer0X L0X = Layer0X.getInstance(graph); - for(Resource sessionModel : graph.syncRequest(new ObjectsWithType(graph.getRootLibrary(), L0X.HasSession, L0X.Session))) { - String id = graph.getPossibleRelatedValue(sessionModel, L0X.Session_HasClientId); - if(id != null && id.equals(clientId)) return sessionModel; - } - return null; - - } - - }); - - if(sessionModel == null) { - - sessionModel = session.syncRequest(new WriteResultRequest(activations) { - - @Override - public Resource perform(WriteGraph graph) throws DatabaseException { - Layer0 L0 = Layer0.getInstance(graph); - Layer0X L0X = Layer0X.getInstance(graph); - Resource session = graph.newResource(); - graph.claim(session, L0.InstanceOf, null, L0X.Session); - graph.claim(session, L0X.Session_HasUser, null, graph.getResource("http://Users/AdminUser")); - graph.addLiteral(session, L0X.Session_HasClientId, L0X.Session_HasClientId_Inverse, clientId, Bindings.STRING); - graph.claim(graph.getRootLibrary(), L0X.HasSession, session); - return session; - } - }); - - } - - session.registerService(SessionModel.class, new PlatformSessionModel(sessionModel)); - } catch (DatabaseException e) { - throw new PlatformException(e); - } - - } - - static class PlatformSessionModel implements SessionModel { - private final Resource sessionModel; - - public PlatformSessionModel(Resource model) { - this.sessionModel = model; - } - - @Override - public Resource getResource() { - return sessionModel; - } - } - - public void resetDatabase(IProgressMonitor monitor) throws PlatformException { - File dbLocation = Platform.getLocation().append("db").toFile(); - if(!dbLocation.exists()) return; - try { // Load driver - Driver driver = Manager.getDriver("procore"); - Management management = driver.getManagement(dbLocation.getAbsolutePath(), null); - management.delete(); - } catch (DatabaseException e) { - throw new PlatformException("Failed to remove database at " + dbLocation.getAbsolutePath(), e); - } - // We have created extra files to database folder which have to be deleted also. - // This is an awful idea! Do not create extra files to database folder! - Throwable t = null; - for (int i=0; i<10; ++i) { - try { - FileUtils.deleteAll(dbLocation); - t = null; - break; - } catch (IOException e) { - // Assuming this has been thrown because delete file/folder failed. - t = e; - } - try { - Thread.sleep(200); - } catch (InterruptedException e) { - // Ignoring interrupted exception. - } - } - if (null != t) - throw new PlatformException("Failed to remove database folder at " + dbLocation.getAbsolutePath(), t); - } - public void resetWorkspace(IProgressMonitor monitor, ArrayList fileFilter) throws PlatformException, IllegalStateException, IOException { - File file = Platform.getLocation().toFile(); - if (null != fileFilter) - FileUtils.deleteAllWithFilter(file , fileFilter); - resetDatabase(monitor); - } - - /** - * Start-up the platform. The procedure consists of 8 steps. Once everything - * is up and running, all fields are set property. - *

- * - * If workspacePolicy is FixErrors, there is an attempt to fix unexpected - * errors. It includes installing database files, installing ontologies, and - * installing project features. - *

- * - * In SWB this is handled in SimanticsWorkbenchAdvisor#openWindows(). - *

- * - * If remote server is given, simantics plaform takes connection there - * instead of local server at "db/". - * - * @param workspacePolicy action to take on workspace/database related - * errors - * @param ontologyPolicy action to take on ontology mismatch - * @param progressMonitor optional progress monitor - * @param userAgent interface for resorting to user feedback during platform - * startup or null to resort to default measures - * @throws PlatformException - */ - public SessionContext startUp(String databaseDriverId, IProgressMonitor progressMonitor, RecoveryPolicy workspacePolicy, - OntologyRecoveryPolicy ontologyPolicy, boolean requireSynchronize, PlatformUserAgent userAgent) - throws PlatformException - { - - assert(!running); - TimeLogger.log("Beginning of SimanticsPlatform.startUp"); - - if (progressMonitor == null) progressMonitor = new NullProgressMonitor(); - - // For debugging on what kind of platform automatic tests are running in - // case there are problems. - if ("true".equals(System.getProperty("org.simantics.dumpBundleState"))) - dumpPlatformBundleState(); - - // 0. Consult all startup extensions before doing anything with the workspace. - StartupExtensions.consultStartupExtensions(); - TimeLogger.log("Consulted platform pre-startup extensions"); - - // 0.1. Clear all temporary files - Simantics.clearTemporaryDirectory(); - TimeLogger.log("Cleared temporary directory"); - - // 0.2 Clear VariableRepository.repository static map which holds references to SessionImplDb - VariableRepository.clear(); - - // 1. Assert there is a database at /db - session = setupDatabase(databaseDriverId, progressMonitor, workspacePolicy, userAgent); - TimeLogger.log("Database setup complete"); - - // 1.1 - XSupport support = session.getService(XSupport.class); - if (support.rolledback()) { - try { - DatabaseIndexing.deleteAllIndexes(); - } catch (IOException e) { - throw new PlatformException(e); - } - } - - // 2. Assert all graphs, and correct versions, are installed to the database - synchronizeOntologies(progressMonitor, ontologyPolicy, requireSynchronize); - TimeLogger.log("Synchronized ontologies"); - - // 4. Assert simantics.cfg exists - boolean installProject = assertConfiguration(progressMonitor,workspacePolicy); - - // 5. Assert Project Resource is installed in the database - installProject = assertProject(progressMonitor, workspacePolicy, installProject); - - // 6. Install all features into project, if in debug mode - updateInstalledGroups(progressMonitor, installProject); - TimeLogger.log("Installed all features into project"); - - // 7. Assert L0.Session in database for this session - assertSessionModel(progressMonitor); - - session.getService(XSupport.class).setServiceMode(false, false); - - try { - session.sync(new WriteRequest() { - - @Override - public void perform(WriteGraph graph) throws DatabaseException { - QueryControl qc = graph.getService(QueryControl.class); - qc.flush(graph); - } - - }); - TimeLogger.log("Flushed queries"); - } catch (DatabaseException e) { - Logger.defaultLogError(e); - } - boolean loadProject = true; - try { - - sessionContext = SimanticsPlatform.INSTANCE.createSessionContext(true); - // This must be before setSessionContext since some listeners might query this - sessionContext.setHint(SimanticsKeys.KEY_PROJECT, SimanticsPlatform.INSTANCE.projectResource); - - Simantics.setSessionContext(sessionContext); - - // 1. Put ResourceBinding that throws an exception to General Bindings - simanticsBindings = new SimanticsBindings( null ); - Bindings.classBindingFactory.addFactory( simanticsBindings ); - - - // 2. Create session-specific second Binding context (Databoard) and - // put that to Session as a service - Session session = sessionContext.getSession(); - Databoard sessionDataboard = new Databoard(); - session.registerService(Databoard.class, sessionDataboard); - simanticsBindings2 = new SimanticsBindings( session ); - sessionDataboard.classBindingFactory.addFactory( simanticsBindings2 ); - - // Register datatype bindings - Bindings.defaultBindingFactory.getRepository().put(RGB.Integer.BINDING.type(), RGB.Integer.BINDING); - Bindings.defaultBindingFactory.getRepository().put(Font.BINDING.type(), Font.BINDING); - - if(loadProject) { - - TimeLogger.log("Load projects"); - project = Projects.loadProject(sessionContext.getSession(), SimanticsPlatform.INSTANCE.projectResource); - - sessionContext.setHint(ProjectKeys.KEY_PROJECT, project); - TimeLogger.log("Loading projects complete"); - - project.activate(); - TimeLogger.log("Project activated"); - } - - } catch (DatabaseException e) { - Logger.defaultLogError(e); - throw new PlatformException(e); - } catch (ProjectException e) { - boolean hasStackTrace = e.getStackTrace().length > 0; - if (!hasStackTrace) - throw new PlatformException(e.getMessage(), hasStackTrace); - throw new PlatformException(e, hasStackTrace); - } - - running = true; - - return sessionContext; - - } - - public SessionContext createSessionContext(boolean init) throws PlatformException { - try { - // Construct and initialize SessionContext from Session. - SessionContext sessionContext = SessionContext.create(session, init); - if (init) - sessionContext.registerServices(); - return sessionContext; - } catch (DatabaseException e) { - throw new PlatformException(e); - } - } - -// private static File getIgnorePrerequisitesFile(URL workspaceUrl) { -// if (workspaceUrl == null) -// return null; -// return new File(workspaceUrl.getPath(), ".ignorePrerequisites"); -// } -// -// private void ensurePrerequisites(IProgressMonitor progressMonitor, PlatformUserAgent userAgent) throws PlatformException { -// Location loc = Platform.getInstanceLocation(); -// File ignorePrerequisites = getIgnorePrerequisitesFile(loc.getURL()); -// if (loc.isSet() && ignorePrerequisites != null) { -// if (ignorePrerequisites.exists() || ignorePrerequisites.isFile()) -// return; -// } -// -// try { -// ServerEnvironment.ensureServerDependenciesMet(); -// } catch (ExecutionEnvironmentException e) { -// // Not installed properly, ask user whether to try installation. -// try { -// StringBuilder msg = new StringBuilder(); -// msg.append("Your system seems to be missing the following prerequisites for running this application:\n\n"); -// for (Product product : e.requiredProducts) -// msg.append("\t" + product.getDescription() + "\n"); -// msg.append("\nYou can either install the missing components now or ignore and attempt to start the application without them. Ignore Always will ignore this question for this workspace."); -// msg.append("\n\nSelecting Cancel will close the application."); -// -// int selection = 0; -// if (userAgent != null) { -// selection = userAgent.showPrompt("Missing Prerequisites", msg.toString(), new String[] { -// "Install Pre-requisites", -// "Ignore Now", -// "Ignore Always", -// "Cancel" -// }, selection); -// } -// boolean tryInstall = false; -// switch (selection) { -// case 0: -// tryInstall = true; -// break; -// case 2: -// ignorePrerequisites.createNewFile(); -// case 1: -// break; -// case 3: -// case -1: -// throw new CancelStartupException(); -// } -// -// if (tryInstall) { -// // Try to install it and check for success afterwards. -// ServerEnvironment.tryInstallDependencies(progressMonitor); -// ServerEnvironment.ensureServerDependenciesMet(); -// } -// } catch (InstallException ie) { -// throw new PlatformException(ie); -// } catch (ExecutionEnvironmentException eee) { -// throw new PlatformException(eee); -// } catch (IOException ie) { -// throw new PlatformException(ie); -// } -// } -// } - - /** - * Shutdown Simantics Platform. - * - * In SWB this is handled in SimanticsWorkbenchAdvisor#disconnectFromWorkspace. - * - * @param progressMonitor optional progress monitor - * @throws PlatformException - */ - public void shutdown(IProgressMonitor progressMonitor) throws PlatformException - { - SubMonitor progress = SubMonitor.convert(progressMonitor, 100); - PlatformException platformException = null; - try { - progress.subTask("Close Project"); - if (project != null) { - project.safeDispose(); - } - progress.worked(10); - - running = false; - progress.subTask("Close Database Session"); - Databoard databoard = null; - if (sessionContext != null) { - Session s = sessionContext.peekSession(); - if (s != null) { - databoard = s.peekService(Databoard.class); - - progress.subTask("Flushing Index Caches"); - try { - Simantics.flushIndexCaches(progress.newChild(20), s); - } catch (Throwable t) { - Logger.defaultLogError(t); - } - } - - progress.subTask("Close Database Session"); - sessionContext.safeDispose(); - sessionContext = null; - Simantics.setSessionContext(null); - } - if (simanticsBindings != null) { - Bindings.classBindingFactory.removeFactory( simanticsBindings ); - simanticsBindings = null; - } - if (databoard != null) { - if (simanticsBindings2 != null) { - databoard.classBindingFactory.removeFactory( simanticsBindings2 ); - simanticsBindings2 = null; - } - databoard.clear(); - } - - // Make sure Simantics clipboard doesn't store unwanted session data references. - Simantics.setClipboard(new SimanticsClipboardImpl()); - - progress.worked(30); - - session = null; - projectResource = null; - - DependenciesRelation.assertFinishedTracking(); - - } catch (Exception e) { - platformException = new PlatformException("Failed to shutdown Simantics Platform", e); - } - - progress.worked(10); - progress.subTask("Shutting down database"); - try { - if (null != databasebManagement) - databasebManagement.shutdown(); - } catch (Throwable t) { - Logger.defaultLogError(t); - } - progress.worked(10); - - progress.subTask("Clearing Workspace Temporary Directory"); - try { - Simantics.clearTemporaryDirectory(); - } catch (Throwable t) { - Logger.defaultLogError(t); - } - progress.worked(10); - if (null != platformException) - throw platformException; - } - - // TODO: consider removing this in the future ?? - @Override - public void stateChanged(LifecycleState newState) { - if(newState == LifecycleState.CLOSED) { - if(running) { - if(Platform.isRunning()) { - mainThread.interrupt(); - } - } - } - } - - /** - * @return true if discard was successful, false - * if there was no session, {@link UndoRedoSupport} or - * {@link UndoContext} to discard through - */ - public boolean discardSessionUndoHistory() { - Session s = session; - if (s != null) { - UndoRedoSupport urs = s.peekService(UndoRedoSupport.class); - if (urs != null) { - UndoContext uc = urs.getUndoContext(s); - if (uc != null) { - uc.clear(); - return true; - } - } - } - return false; - } - - public void reconnect(String databaseDriverId) throws Exception { - // Starts database server. - SimanticsPlatform.INSTANCE.startUp(databaseDriverId, null, RecoveryPolicy.ThrowError, OntologyRecoveryPolicy.ThrowError, true, null); - } - - private void dumpPlatformBundleState() { - BundleDescription[] bs = Platform.getPlatformAdmin().getState().getBundles(); - System.out.println("Total bundles: " + bs.length); - for (BundleDescription b : bs) { - System.out.format("%-80s @ %s\n", b.toString(), b.getLocation()); - } - } - -} - +/******************************************************************************* + * Copyright (c) 2007, 2010 Association for Decentralized Information Management + * in Industry THTH ry. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * VTT Technical Research Centre of Finland - initial API and implementation + *******************************************************************************/ +package org.simantics; + +import static org.simantics.db.common.utils.Transaction.commit; +import static org.simantics.db.common.utils.Transaction.endTransaction; +import static org.simantics.db.common.utils.Transaction.readGraph; +import static org.simantics.db.common.utils.Transaction.startTransaction; +import static org.simantics.db.common.utils.Transaction.writeGraph; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Properties; +import java.util.Set; +import java.util.TreeMap; +import java.util.UUID; + +import org.eclipse.core.runtime.ILog; +import org.eclipse.core.runtime.IProduct; +import org.eclipse.core.runtime.IProgressMonitor; +import org.eclipse.core.runtime.IStatus; +import org.eclipse.core.runtime.NullProgressMonitor; +import org.eclipse.core.runtime.Platform; +import org.eclipse.core.runtime.Status; +import org.eclipse.core.runtime.SubMonitor; +import org.eclipse.osgi.service.resolver.BundleDescription; +import org.simantics.databoard.Bindings; +import org.simantics.databoard.Databoard; +import org.simantics.datatypes.literal.Font; +import org.simantics.datatypes.literal.RGB; +import org.simantics.db.Driver; +import org.simantics.db.Driver.Management; +import org.simantics.db.Manager; +import org.simantics.db.ReadGraph; +import org.simantics.db.Resource; +import org.simantics.db.Session; +import org.simantics.db.SessionModel; +import org.simantics.db.UndoContext; +import org.simantics.db.VirtualGraph; +import org.simantics.db.WriteGraph; +import org.simantics.db.common.request.ObjectsWithType; +import org.simantics.db.common.request.Queries; +import org.simantics.db.common.request.WriteRequest; +import org.simantics.db.common.request.WriteResultRequest; +import org.simantics.db.common.utils.Transaction; +import org.simantics.db.exception.ClusterSetExistException; +import org.simantics.db.exception.DatabaseException; +import org.simantics.db.exception.ResourceNotFoundException; +import org.simantics.db.indexing.DatabaseIndexing; +import org.simantics.db.layer0.genericrelation.DependenciesRelation; +import org.simantics.db.layer0.util.SimanticsClipboardImpl; +import org.simantics.db.layer0.util.SimanticsKeys; +import org.simantics.db.layer0.util.TGTransferableGraphSource; +import org.simantics.db.layer0.variable.VariableRepository; +import org.simantics.db.management.SessionContext; +import org.simantics.db.request.Read; +import org.simantics.db.service.LifecycleSupport.LifecycleListener; +import org.simantics.db.service.LifecycleSupport.LifecycleState; +import org.simantics.db.service.QueryControl; +import org.simantics.db.service.UndoRedoSupport; +import org.simantics.db.service.VirtualGraphSupport; +import org.simantics.db.service.XSupport; +import org.simantics.graph.db.GraphDependencyAnalyzer; +import org.simantics.graph.db.GraphDependencyAnalyzer.IU; +import org.simantics.graph.db.GraphDependencyAnalyzer.IdentityNode; +import org.simantics.graph.db.IImportAdvisor; +import org.simantics.graph.db.TransferableGraphs; +import org.simantics.graph.diff.Diff; +import org.simantics.graph.diff.TransferableGraphDelta1; +import org.simantics.internal.Activator; +import org.simantics.internal.startup.StartupExtensions; +import org.simantics.layer0.Layer0; +import org.simantics.operation.Layer0X; +import org.simantics.project.IProject; +import org.simantics.project.ProjectFeatures; +import org.simantics.project.ProjectKeys; +import org.simantics.project.Projects; +import org.simantics.project.exception.ProjectException; +import org.simantics.project.features.registry.GroupReference; +import org.simantics.project.management.DatabaseManagement; +import org.simantics.project.management.GraphBundle; +import org.simantics.project.management.GraphBundleEx; +import org.simantics.project.management.GraphBundleRef; +import org.simantics.project.management.PlatformUtil; +import org.simantics.project.management.ServerManager; +import org.simantics.project.management.ServerManagerFactory; +import org.simantics.project.management.WorkspaceUtil; +import org.simantics.utils.FileUtils; +import org.simantics.utils.datastructures.Pair; +import org.simantics.utils.logging.TimeLogger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * SimanticsPlatform performs procedures required in order to get simantics + * workbench into operational state. This consists of the following steps: + *

    + *
  • Asserting there is Database + *
    • + *
  • Starting Database process + *
    • + *
  • Opening a session to Database process + *
    • + *
  • Asserting required ontologies or other transferable graphs are installed in the database + *
    • + *
  • Asserting required project is installed in the database + *
    • + *
  • Asserting Simantics Features are installed in the database + *
    • + *
  • Asserting Simantics Features are installed to the project + *
    • + *
  • Shutdown: Save Session, Close session, Kill Database process + *
    • + *
+ * + * @author Toni Kalajainen + */ +public class SimanticsPlatform implements LifecycleListener { + + private static final Logger LOGGER = LoggerFactory.getLogger(SimanticsPlatform.class); + + /** + * The policy is relevant when developing Simantics from Eclipse IDE. + * It is applied when the ontology in the database of a workspace doesn't match + * a newer ontology in the Eclipse workspace. + */ + public static enum OntologyRecoveryPolicy { ThrowError, Merge, ReinstallDatabase } + + /** + * This policy dictates how the Simantics platform startup should react if + * the started workspace is not set up properly. The alternatives are to + * just throw an error and fail or to attempt all possible measures to fix + * the encountered problems. + */ + public static enum RecoveryPolicy { ThrowError, FixError } + + /** Singleton instance, started in SimanticsWorkbenchAdvisor */ + public static final SimanticsPlatform INSTANCE = new SimanticsPlatform(); + + /** Set to true when the Simantics Platform is in good-and-go condition */ + public boolean running; + + /** Database Session */ + public Session session; + private Management databasebManagement; + + /** Database session context */ + public SessionContext sessionContext; + + /** Project identifier in Database */ + public String projectURI; + + /** Project name */ + public String projectName; + + /** Project resource */ + public Resource projectResource; + + /** Session specific bindings */ + public SimanticsBindings simanticsBindings; + public SimanticsBindings simanticsBindings2; + + public Thread mainThread; + + /** + * The {@link IProject} activated by + * {@link #startUp(IProgressMonitor, RecoveryPolicy, OntologyRecoveryPolicy, ServerAddress, PlatformUserAgent)} + */ + private IProject project; + + protected ILog log; + + /** + * Create a new simantics plaform manager in uninitialized state and + * with default policies.

+ */ + public SimanticsPlatform() { + log = Platform.getLog(Activator.getBundleContext().getBundle()); + mainThread = Thread.currentThread(); + } + + public String getApplicationClientId() { + IProduct product = Platform.getProduct(); + if(product == null) return "noProduct";//UUID.randomUUID().toString(); + String application = product.getApplication(); + return application != null ? application : UUID.randomUUID().toString(); + } + + private Session setupDatabase(String databaseDriverId, IProgressMonitor progressMonitor, RecoveryPolicy workspacePolicy, PlatformUserAgent userAgent) throws PlatformException { + if (progressMonitor == null) + progressMonitor = new NullProgressMonitor(); + File dbLocation = Platform.getLocation().append("db").toFile(); + ServerManager serverManager; + try { + serverManager = ServerManagerFactory.create(databaseDriverId, dbLocation.getAbsolutePath()); + } catch (DatabaseException | IOException e) { + throw new PlatformException("Failed to initialize Server Manager", e); + } + progressMonitor.beginTask("Setting up Simantics Database", 100); + progressMonitor.setTaskName("Asserting Database is installed."); + String msg = "Failed to initialize Simantics database."; + try { + // Create database + log.log(new Status(IStatus.INFO, Activator.PLUGIN_ID, "Creating database at " + dbLocation)); + progressMonitor.setTaskName("Creating database at " + dbLocation); + databasebManagement = serverManager.getManagement(dbLocation); + databasebManagement.create(); + // Create layer0. + return serverManager.createDatabase(dbLocation); + } catch (DatabaseException e) { + throw new PlatformException(msg, e); + } catch (Throwable e) { + throw new PlatformException(msg, e); + } finally { + progressMonitor.worked(20); + } + } + + public void synchronizeOntologies(IProgressMonitor progressMonitor, OntologyRecoveryPolicy ontologyPolicy, boolean requireSynchronize) throws PlatformException { + + if (progressMonitor == null) progressMonitor = new NullProgressMonitor(); + + final DatabaseManagement mgmt = new DatabaseManagement(); + + PlatformUtil.compileAllDynamicOntologies(); + + progressMonitor.setTaskName("Asserting all ontologies are installed"); + final Map platformTGs = new HashMap(); + try { + + // Get a list of bundles installed into the database + progressMonitor.subTask("find installed bundles from database"); + Map installedTGs = new HashMap(); + for (GraphBundle b : session.syncRequest( mgmt.GraphBundleQuery )) { + installedTGs.put(GraphBundleRef.of(b), GraphBundleEx.extend(b)); + } + + if(!requireSynchronize && installedTGs.size() > 1 && !Platform.inDevelopmentMode()) return; +// if(installedTGs.size() > 1) return; + + // Get a list of all bundles in the platform (Bundle Context) + List tgs = new ArrayList(); + progressMonitor.subTask("load all transferable graphs from platform"); + PlatformUtil.getAllGraphs(tgs); + progressMonitor.subTask("extend bundles to compile versions"); + for (GraphBundle b : tgs) { + GraphBundleEx gbe = GraphBundleEx.extend(b); + gbe.build(); + platformTGs.put(GraphBundleRef.of(b), gbe); + } + + // Compile a list of TGs that need to be installed or reinstalled in the database + progressMonitor.subTask("check bundle reinstallation demand"); + List installTGs = new ArrayList(); + // Create list of TGs to update, + Map reinstallTGs = new TreeMap(); + for (Entry e : platformTGs.entrySet()) { + GraphBundleRef key = e.getKey(); + GraphBundleEx platformBundle = e.getValue(); + GraphBundleEx existingBundle = installedTGs.get(key); + +// System.out.println("GraphBundleRef key=" + key.toString()); + + if (existingBundle == null) { + // Bundle did not exist in the database, put it into list of bundles to install + installTGs.add(platformBundle); + } + else { + // Bundle exists in the database + boolean platformBundleIsNewer = existingBundle.getVersion().compareTo(platformBundle.getVersion())<0; + if (!platformBundleIsNewer) + continue; + // Check hash of transferable graph to know whether to update or not. + if (platformBundle.getHashcode() == existingBundle.getHashcode()) + continue; + //System.out.println("Ontology hashcodes do not match: platform bundle=" + // + platformBundle.getVersionedId() + ", hash=" + platformBundle.getHashcode() + // + "; existing bundle=" + existingBundle.getVersionedId() + ", hash=" + existingBundle.getHashcode()); + reinstallTGs.put(platformBundle, existingBundle); + } + } + // INSTALL + // Database is missing graphs + if (!installTGs.isEmpty() || !reinstallTGs.isEmpty()) { + session.getService(XSupport.class).setServiceMode(true, true); + + // Throw error + if (ontologyPolicy == OntologyRecoveryPolicy.ThrowError) { + StringBuilder sb = new StringBuilder("The following graphs are not installed in the database: "); + if (!installTGs.isEmpty()) { + int i = 0; + for (GraphBundleEx e : installTGs) { + if (i>0) sb.append(", "); + i++; + sb.append(e.toString()); + } + sb.append(" is missing from the database.\n"); + } + if (!reinstallTGs.isEmpty()) { + int i = 0; + for (Entry e : reinstallTGs.entrySet()) { + if (i>0) sb.append(", "); + i++; + sb.append(e.getKey().toString()); + } + sb.append(" Database/Platform Bundle version mismatch.\n"); + } + sb.append("Hint: Use -fixErrors to install the graphs."); + throw new PlatformException(sb.toString()); + } + // Reinstall database + if (ontologyPolicy == OntologyRecoveryPolicy.ReinstallDatabase) { + log.log(new Status(IStatus.INFO, Activator.PLUGIN_ID, "Reinstalling the database.")); + // TODO Install DB + // Stop Session + // Kill Process + // Delete Database + // Create Database + // Start Database + // Open Session + // Install TGs + throw new PlatformException("Reinstalling Database, NOT IMPLEMENTED"); + } + + if (ontologyPolicy == OntologyRecoveryPolicy.Merge) { + progressMonitor.subTask("Merging ontology changes"); + // Sort missing TGs into install order + GraphDependencyAnalyzer analyzer = new GraphDependencyAnalyzer(); + for(GraphBundle tg : installTGs) analyzer.addGraph(tg, tg.getGraph()); + for(GraphBundle tg : reinstallTGs.keySet()) analyzer.addGraph(tg, tg.getGraph()); + if(!analyzer.analyzeDependency()) { + Collection> problems = analyzer.getConflicts(); + StringBuilder sb = new StringBuilder(); + for (Pair problem : problems) { + sb.append("Conflict with "+problem.first+" and "+problem.second+".\n"); + } + throw new PlatformException(sb.toString()); + } + else if(!session.syncRequest( analyzer.queryExternalDependenciesSatisfied )) { + Collection unsatisfiedDependencies = analyzer.getUnsatisfiedDependencies(); + StringBuilder sb = new StringBuilder(); + for (IdentityNode dep: unsatisfiedDependencies) { + sb.append("Unsatisfied Dependency "+dep+". Required by\n"); + for(IU iu : GraphDependencyAnalyzer.toCollection(dep.getRequires())) { + sb.append(" " + ((GraphBundle)iu.getId()).getId() + "\n"); + } + } + throw new PlatformException(sb.toString()); + } + + List sortedBundles = analyzer.getSortedGraphs(); + if(!sortedBundles.isEmpty()) { + + session.syncRequest(new WriteRequest() { + @Override + public void perform(WriteGraph graph) throws DatabaseException { + try { + graph.newClusterSet(graph.getRootLibrary()); + } catch (ClusterSetExistException e) { + // Cluster set exist already, no problem. + } + graph.setClusterSet4NewResource(graph.getRootLibrary()); + graph.flushCluster(); + } + }); + + boolean mergedOntologies = false; + + // Install TGs + for(final GraphBundle tg : sortedBundles) { + + final IImportAdvisor advisor = new OntologyImportAdvisor(tg, mgmt); + final GraphBundle oldTG = reinstallTGs.get(tg); + + if (oldTG==null) { + + boolean createImmutable = tg.getImmutable(); + session.getService(XSupport.class).setServiceMode(true, createImmutable); + + // Install TG + log.log(new Status(IStatus.INFO, Activator.PLUGIN_ID, "Installing "+tg.toString()+" - "+tg.getName())); + TransferableGraphs.importGraph1(session, new TGTransferableGraphSource(tg.getGraph()), advisor, null); + } else { + // Merge TG + startTransaction(session, false); + TransferableGraphDelta1 delta = new Diff(oldTG.getGraph(), tg.getGraph()).diff(); + final long[] oldResources = oldTG.getResourceArray(); + boolean changes = TransferableGraphs.hasChanges(readGraph(), oldResources, delta); + endTransaction(); + if (!changes) { + //log.log(new Status(IStatus.INFO, Activator.PLUGIN_ID, "Nothing to merge for "+tg.toString())); + continue; + } + + log.log(new Status(IStatus.INFO, Activator.PLUGIN_ID, "Merging new version of "+tg.toString())); + + startTransaction(session, true); + + //delta.print(); + try { + + + long[] resourceArray = TransferableGraphs.applyDelta(writeGraph(), oldResources, delta); + tg.setResourceArray(resourceArray); + mgmt.setGraphBundleEntry(tg); + commit(); + mergedOntologies = true; + } catch (Throwable t) { + throw new PlatformException(t); + } finally { + endTransaction(); + } + } + } + + session.syncRequest(new WriteRequest() { + @Override + public void perform(WriteGraph graph) throws DatabaseException { + graph.setClusterSet4NewResource(graph.getRootLibrary()); + graph.flushCluster(); + } + }); + + if (mergedOntologies) + DatabaseIndexing.deleteAllIndexes(); + } + + TimeLogger.log("Ontologies synchronized."); + + } + session.getService(XSupport.class).setServiceMode(false, false); + } + progressMonitor.worked(20); + } catch (IOException e) { + throw new PlatformException(e); + } catch (DatabaseException e) { + throw new PlatformException(e); + } + + } + + public boolean assertConfiguration(IProgressMonitor progressMonitor, RecoveryPolicy workspacePolicy) throws PlatformException { + + if (progressMonitor == null) progressMonitor = new NullProgressMonitor(); + + File workspaceLocation = Platform.getLocation().toFile(); + + boolean installProject = false; + progressMonitor.setTaskName("Asserting simantics.cfg is installed"); + try { + File propertyFile = new File(workspaceLocation, "simantics.cfg"); + Properties properties; + try { + properties = WorkspaceUtil.readProperties(propertyFile); + } catch (IOException e) { + if (workspacePolicy == RecoveryPolicy.ThrowError) throw new PlatformException("Could not load "+propertyFile); + + // Create a project and write Property file + properties = new Properties(); + properties.setProperty("project_uri", "http://Projects/Development%20Project"); + properties.setProperty("project_name", "Development Project"); + WorkspaceUtil.writeProperties(propertyFile, properties); + installProject |= true; + } + projectURI = properties.getProperty("project_uri"); + projectName = properties.getProperty("project_name"); + progressMonitor.worked(10); + } catch (IOException e) { + throw new PlatformException(e); + } + + return installProject; + + } + + public boolean assertProject(IProgressMonitor progressMonitor, RecoveryPolicy workspacePolicy, boolean installProject) throws PlatformException { + + if (progressMonitor == null) progressMonitor = new NullProgressMonitor(); + + final DatabaseManagement mgmt = new DatabaseManagement(); + + progressMonitor.setTaskName("Asserting project resource exists in the database"); + try { + projectResource = session.syncRequest( Queries.resource( projectURI ) ); + } catch (ResourceNotFoundException nfe) { + // Project was not found + if (workspacePolicy == RecoveryPolicy.ThrowError) + throw new PlatformException("Project Resource "+projectURI+" is not found in the database."); + // Create empty project with no features + try { + Transaction.startTransaction(session, true); + try { + // The project needs to be created mutable. + session.getService(XSupport.class).setServiceMode(true, false); + + ArrayList empty = new ArrayList(); + projectResource = mgmt.createProject(projectName, empty); + installProject |= true; + + session.getService(XSupport.class).setServiceMode(false, false); + Transaction.commit(); + } finally { + Transaction.endTransaction(); + } + //session.getService( LifecycleSupport.class ).save(); + } catch (DatabaseException e) { + throw new PlatformException("Failed to create "+projectURI, e); + } + } catch (DatabaseException e) { + throw new PlatformException("Failed to create "+projectURI, e); + } + progressMonitor.worked(10); + + return installProject; + + } + + public void updateInstalledGroups(IProgressMonitor progressMonitor, boolean installProject) throws PlatformException { + + if (installProject) + { + // Attach all feature groups available in platform to created project + progressMonitor.setTaskName("Install all features"); + Set publishedFeatureGroups = ProjectFeatures.getInstallGroupsOfPublishedFeatures(); + Collection groupsWithoutVersion = GroupReference.stripVersions(publishedFeatureGroups); + + // final List Platform_Features = new ArrayList(); + // + // // Convert graph instances + // Collection platformGraphs = new ArrayList(); + // for (GraphBundleEx e : platformTGs.values()) platformGraphs.add( e.getGraph() ); + // IGraph graph = Graphs.createGraph(platformGraphs); + // + // Res PublishedProjectFeatures = UriUtils.uriToPath( ProjectResource.URIs.PublishedProjectFeatures ); + // Path HasFeature = UriUtils.uriToPath( ProjectResource.URIs.HasFeature ); + // for(Res feature : graph.getObjects(PublishedProjectFeatures, HasFeature)) { + // System.out.println("Installing Project Feature: "+feature.toString()); + // Platform_Features.add( feature.toString() ); + // } + + try { + Transaction.startTransaction(session, true); + try { + // for (String feature : Platform_Features) { + // try { + // getResource(feature); + // } catch(ResourceNotFoundException e) { + // System.out.println(feature+" not found"); + // } + // mgmt.installFeature(projectResource, feature); + // } + Projects.setProjectInstalledGroups(writeGraph(), projectResource, groupsWithoutVersion); + Transaction.commit(); + } finally { + Transaction.endTransaction(); + } + //session.getService( LifecycleSupport.class ).save(); + } catch(DatabaseException ae) { + throw new PlatformException("Failed to install features", ae); + } + progressMonitor.worked(10); + } + + } + + public void assertSessionModel(IProgressMonitor progressMonitor) throws PlatformException { + + Properties properties = session.getService(Properties.class); + final String clientId = properties.getProperty("clientId"); + + try { + + // Currently this needs to be done before data becomes available + VirtualGraphSupport support = session.getService(VirtualGraphSupport.class); + VirtualGraph activations = support.getWorkspacePersistent("activations"); + + Resource sessionModel = session.syncRequest(new Read() { + + @Override + public Resource perform(ReadGraph graph) throws DatabaseException { + + Layer0X L0X = Layer0X.getInstance(graph); + for(Resource sessionModel : graph.syncRequest(new ObjectsWithType(graph.getRootLibrary(), L0X.HasSession, L0X.Session))) { + String id = graph.getPossibleRelatedValue(sessionModel, L0X.Session_HasClientId); + if(id != null && id.equals(clientId)) return sessionModel; + } + return null; + + } + + }); + + if(sessionModel == null) { + + sessionModel = session.syncRequest(new WriteResultRequest(activations) { + + @Override + public Resource perform(WriteGraph graph) throws DatabaseException { + Layer0 L0 = Layer0.getInstance(graph); + Layer0X L0X = Layer0X.getInstance(graph); + Resource session = graph.newResource(); + graph.claim(session, L0.InstanceOf, null, L0X.Session); + graph.claim(session, L0X.Session_HasUser, null, graph.getResource("http://Users/AdminUser")); + graph.addLiteral(session, L0X.Session_HasClientId, L0X.Session_HasClientId_Inverse, clientId, Bindings.STRING); + graph.claim(graph.getRootLibrary(), L0X.HasSession, session); + return session; + } + }); + + } + + session.registerService(SessionModel.class, new PlatformSessionModel(sessionModel)); + } catch (DatabaseException e) { + throw new PlatformException(e); + } + + } + + static class PlatformSessionModel implements SessionModel { + private final Resource sessionModel; + + public PlatformSessionModel(Resource model) { + this.sessionModel = model; + } + + @Override + public Resource getResource() { + return sessionModel; + } + } + + public void resetDatabase(IProgressMonitor monitor) throws PlatformException { + File dbLocation = Platform.getLocation().append("db").toFile(); + if(!dbLocation.exists()) return; + try { // Load driver + Driver driver = Manager.getDriver("procore"); + Management management = driver.getManagement(dbLocation.getAbsolutePath(), null); + management.delete(); + } catch (DatabaseException e) { + throw new PlatformException("Failed to remove database at " + dbLocation.getAbsolutePath(), e); + } + // We have created extra files to database folder which have to be deleted also. + // This is an awful idea! Do not create extra files to database folder! + Throwable t = null; + for (int i=0; i<10; ++i) { + try { + FileUtils.deleteAll(dbLocation); + t = null; + break; + } catch (IOException e) { + // Assuming this has been thrown because delete file/folder failed. + t = e; + } + try { + Thread.sleep(200); + } catch (InterruptedException e) { + // Ignoring interrupted exception. + } + } + if (null != t) + throw new PlatformException("Failed to remove database folder at " + dbLocation.getAbsolutePath(), t); + } + public void resetWorkspace(IProgressMonitor monitor, ArrayList fileFilter) throws PlatformException, IllegalStateException, IOException { + File file = Platform.getLocation().toFile(); + if (null != fileFilter) + FileUtils.deleteAllWithFilter(file , fileFilter); + resetDatabase(monitor); + } + + /** + * Start-up the platform. The procedure consists of 8 steps. Once everything + * is up and running, all fields are set property. + *

+ * + * If workspacePolicy is FixErrors, there is an attempt to fix unexpected + * errors. It includes installing database files, installing ontologies, and + * installing project features. + *

+ * + * In SWB this is handled in SimanticsWorkbenchAdvisor#openWindows(). + *

+ * + * If remote server is given, simantics plaform takes connection there + * instead of local server at "db/". + * + * @param workspacePolicy action to take on workspace/database related + * errors + * @param ontologyPolicy action to take on ontology mismatch + * @param progressMonitor optional progress monitor + * @param userAgent interface for resorting to user feedback during platform + * startup or null to resort to default measures + * @throws PlatformException + */ + public SessionContext startUp(String databaseDriverId, IProgressMonitor progressMonitor, RecoveryPolicy workspacePolicy, + OntologyRecoveryPolicy ontologyPolicy, boolean requireSynchronize, PlatformUserAgent userAgent) + throws PlatformException + { + + assert(!running); + TimeLogger.log("Beginning of SimanticsPlatform.startUp"); + + LOGGER.info("Beginning of SimanticsPlatform.startUp"); + + if (progressMonitor == null) progressMonitor = new NullProgressMonitor(); + + // For debugging on what kind of platform automatic tests are running in + // case there are problems. + if ("true".equals(System.getProperty("org.simantics.dumpBundleState"))) + dumpPlatformBundleState(); + + // 0. Consult all startup extensions before doing anything with the workspace. + StartupExtensions.consultStartupExtensions(); + TimeLogger.log("Consulted platform pre-startup extensions"); + + // 0.1. Clear all temporary files + Simantics.clearTemporaryDirectory(); + TimeLogger.log("Cleared temporary directory"); + + // 0.2 Clear VariableRepository.repository static map which holds references to SessionImplDb + VariableRepository.clear(); + + // 1. Assert there is a database at /db + session = setupDatabase(databaseDriverId, progressMonitor, workspacePolicy, userAgent); + TimeLogger.log("Database setup complete"); + + // 1.1 + XSupport support = session.getService(XSupport.class); + if (support.rolledback()) { + try { + DatabaseIndexing.deleteAllIndexes(); + } catch (IOException e) { + throw new PlatformException(e); + } + } + + // 2. Assert all graphs, and correct versions, are installed to the database + synchronizeOntologies(progressMonitor, ontologyPolicy, requireSynchronize); + TimeLogger.log("Synchronized ontologies"); + + // 4. Assert simantics.cfg exists + boolean installProject = assertConfiguration(progressMonitor,workspacePolicy); + + // 5. Assert Project Resource is installed in the database + installProject = assertProject(progressMonitor, workspacePolicy, installProject); + + // 6. Install all features into project, if in debug mode + updateInstalledGroups(progressMonitor, installProject); + TimeLogger.log("Installed all features into project"); + + // 7. Assert L0.Session in database for this session + assertSessionModel(progressMonitor); + + session.getService(XSupport.class).setServiceMode(false, false); + + try { + session.sync(new WriteRequest() { + + @Override + public void perform(WriteGraph graph) throws DatabaseException { + QueryControl qc = graph.getService(QueryControl.class); + qc.flush(graph); + } + + }); + TimeLogger.log("Flushed queries"); + } catch (DatabaseException e) { + LOGGER.error("Flushing queries failed.", e); + } + boolean loadProject = true; + try { + + sessionContext = SimanticsPlatform.INSTANCE.createSessionContext(true); + // This must be before setSessionContext since some listeners might query this + sessionContext.setHint(SimanticsKeys.KEY_PROJECT, SimanticsPlatform.INSTANCE.projectResource); + + Simantics.setSessionContext(sessionContext); + + // 1. Put ResourceBinding that throws an exception to General Bindings + simanticsBindings = new SimanticsBindings( null ); + Bindings.classBindingFactory.addFactory( simanticsBindings ); + + + // 2. Create session-specific second Binding context (Databoard) and + // put that to Session as a service + Session session = sessionContext.getSession(); + Databoard sessionDataboard = new Databoard(); + session.registerService(Databoard.class, sessionDataboard); + simanticsBindings2 = new SimanticsBindings( session ); + sessionDataboard.classBindingFactory.addFactory( simanticsBindings2 ); + + // Register datatype bindings + Bindings.defaultBindingFactory.getRepository().put(RGB.Integer.BINDING.type(), RGB.Integer.BINDING); + Bindings.defaultBindingFactory.getRepository().put(Font.BINDING.type(), Font.BINDING); + + if(loadProject) { + + TimeLogger.log("Load projects"); + project = Projects.loadProject(sessionContext.getSession(), SimanticsPlatform.INSTANCE.projectResource); + + sessionContext.setHint(ProjectKeys.KEY_PROJECT, project); + TimeLogger.log("Loading projects complete"); + + project.activate(); + TimeLogger.log("Project activated"); + } + + } catch (DatabaseException e) { + LOGGER.error("Platform startup failed.", e); + throw new PlatformException(e); + } catch (ProjectException e) { + boolean hasStackTrace = e.getStackTrace().length > 0; + if (!hasStackTrace) + throw new PlatformException(e.getMessage(), hasStackTrace); + throw new PlatformException(e, hasStackTrace); + } + + running = true; + + return sessionContext; + + } + + public SessionContext createSessionContext(boolean init) throws PlatformException { + try { + // Construct and initialize SessionContext from Session. + SessionContext sessionContext = SessionContext.create(session, init); + if (init) + sessionContext.registerServices(); + return sessionContext; + } catch (DatabaseException e) { + throw new PlatformException(e); + } + } + +// private static File getIgnorePrerequisitesFile(URL workspaceUrl) { +// if (workspaceUrl == null) +// return null; +// return new File(workspaceUrl.getPath(), ".ignorePrerequisites"); +// } +// +// private void ensurePrerequisites(IProgressMonitor progressMonitor, PlatformUserAgent userAgent) throws PlatformException { +// Location loc = Platform.getInstanceLocation(); +// File ignorePrerequisites = getIgnorePrerequisitesFile(loc.getURL()); +// if (loc.isSet() && ignorePrerequisites != null) { +// if (ignorePrerequisites.exists() || ignorePrerequisites.isFile()) +// return; +// } +// +// try { +// ServerEnvironment.ensureServerDependenciesMet(); +// } catch (ExecutionEnvironmentException e) { +// // Not installed properly, ask user whether to try installation. +// try { +// StringBuilder msg = new StringBuilder(); +// msg.append("Your system seems to be missing the following prerequisites for running this application:\n\n"); +// for (Product product : e.requiredProducts) +// msg.append("\t" + product.getDescription() + "\n"); +// msg.append("\nYou can either install the missing components now or ignore and attempt to start the application without them. Ignore Always will ignore this question for this workspace."); +// msg.append("\n\nSelecting Cancel will close the application."); +// +// int selection = 0; +// if (userAgent != null) { +// selection = userAgent.showPrompt("Missing Prerequisites", msg.toString(), new String[] { +// "Install Pre-requisites", +// "Ignore Now", +// "Ignore Always", +// "Cancel" +// }, selection); +// } +// boolean tryInstall = false; +// switch (selection) { +// case 0: +// tryInstall = true; +// break; +// case 2: +// ignorePrerequisites.createNewFile(); +// case 1: +// break; +// case 3: +// case -1: +// throw new CancelStartupException(); +// } +// +// if (tryInstall) { +// // Try to install it and check for success afterwards. +// ServerEnvironment.tryInstallDependencies(progressMonitor); +// ServerEnvironment.ensureServerDependenciesMet(); +// } +// } catch (InstallException ie) { +// throw new PlatformException(ie); +// } catch (ExecutionEnvironmentException eee) { +// throw new PlatformException(eee); +// } catch (IOException ie) { +// throw new PlatformException(ie); +// } +// } +// } + + /** + * Shutdown Simantics Platform. + * + * In SWB this is handled in SimanticsWorkbenchAdvisor#disconnectFromWorkspace. + * + * @param progressMonitor optional progress monitor + * @throws PlatformException + */ + public void shutdown(IProgressMonitor progressMonitor) throws PlatformException + { + SubMonitor progress = SubMonitor.convert(progressMonitor, 100); + PlatformException platformException = null; + try { + progress.subTask("Close Project"); + if (project != null) { + project.safeDispose(); + } + progress.worked(10); + + running = false; + progress.subTask("Close Database Session"); + Databoard databoard = null; + if (sessionContext != null) { + Session s = sessionContext.peekSession(); + if (s != null) { + databoard = s.peekService(Databoard.class); + + progress.subTask("Flushing Index Caches"); + try { + Simantics.flushIndexCaches(progress.newChild(20), s); + } catch (Throwable t) { + LOGGER.error("Failed to flush index caches.", t); + } + } + + progress.subTask("Close Database Session"); + sessionContext.safeDispose(); + sessionContext = null; + Simantics.setSessionContext(null); + } + if (simanticsBindings != null) { + Bindings.classBindingFactory.removeFactory( simanticsBindings ); + simanticsBindings = null; + } + if (databoard != null) { + if (simanticsBindings2 != null) { + databoard.classBindingFactory.removeFactory( simanticsBindings2 ); + simanticsBindings2 = null; + } + databoard.clear(); + } + + // Make sure Simantics clipboard doesn't store unwanted session data references. + Simantics.setClipboard(new SimanticsClipboardImpl()); + + progress.worked(30); + + session = null; + projectResource = null; + + DependenciesRelation.assertFinishedTracking(); + + } catch (Exception e) { + platformException = new PlatformException("Failed to shutdown Simantics Platform", e); + } + + progress.worked(10); + progress.subTask("Shutting down database"); + try { + if (null != databasebManagement) + databasebManagement.shutdown(); + } catch (Throwable t) { + LOGGER.error("Database shutdown failed.", t); + } + progress.worked(10); + + progress.subTask("Clearing Workspace Temporary Directory"); + try { + Simantics.clearTemporaryDirectory(); + } catch (Throwable t) { + LOGGER.error("Failed to clear the temporary directory.", t); + } + progress.worked(10); + if (null != platformException) + throw platformException; + } + + // TODO: consider removing this in the future ?? + @Override + public void stateChanged(LifecycleState newState) { + if(newState == LifecycleState.CLOSED) { + if(running) { + if(Platform.isRunning()) { + mainThread.interrupt(); + } + } + } + } + + /** + * @return true if discard was successful, false + * if there was no session, {@link UndoRedoSupport} or + * {@link UndoContext} to discard through + */ + public boolean discardSessionUndoHistory() { + Session s = session; + if (s != null) { + UndoRedoSupport urs = s.peekService(UndoRedoSupport.class); + if (urs != null) { + UndoContext uc = urs.getUndoContext(s); + if (uc != null) { + uc.clear(); + return true; + } + } + } + return false; + } + + public void reconnect(String databaseDriverId) throws Exception { + // Starts database server. + SimanticsPlatform.INSTANCE.startUp(databaseDriverId, null, RecoveryPolicy.ThrowError, OntologyRecoveryPolicy.ThrowError, true, null); + } + + private void dumpPlatformBundleState() { + BundleDescription[] bs = Platform.getPlatformAdmin().getState().getBundles(); + System.out.println("Total bundles: " + bs.length); + for (BundleDescription b : bs) { + System.out.format("%-80s @ %s\n", b.toString(), b.getLocation()); + } + } + +} + diff --git a/bundles/pom.xml b/bundles/pom.xml index 4641096f8..0b2a85da4 100644 --- a/bundles/pom.xml +++ b/bundles/pom.xml @@ -1,203 +1,210 @@ - - - 4.0.0 - org.simantics - org.simantics.root.bundles - 1.0.0-SNAPSHOT - pom - - - org.simantics - org.simantics.root - 1.0.0-SNAPSHOT - - - - - - org.eclipse.tycho - tycho-compiler-plugin - ${tycho.version} - - -err:-forbidden - - - - org.eclipse.tycho - tycho-source-plugin - ${tycho.version} - - - plugin-source - - plugin-source - - - - - - - - - com.famfamfam.silk - org.simantics - org.simantics.action.ontology - org.simantics.annotation.ontology - org.simantics.annotation.ui - org.simantics.application - org.simantics.basicexpression - org.simantics.browsing.ui - org.simantics.browsing.ui.common - org.simantics.browsing.ui.graph - org.simantics.browsing.ui.graph.impl - org.simantics.browsing.ui.model - org.simantics.browsing.ui.ontology - org.simantics.browsing.ui.platform - org.simantics.browsing.ui.swt - org.simantics.charts - org.simantics.charts.ontology - org.simantics.color.ontology - org.simantics.common - org.simantics.compressions - org.simantics.databoard - org.simantics.datatypes - org.simantics.datatypes.ontology - org.simantics.db - org.simantics.db.common - org.simantics.db.impl - org.simantics.db.indexing - org.simantics.db.layer0 - org.simantics.db.management - org.simantics.db.procore - org.simantics.db.procore.server.environment - org.simantics.db.procore.ui - org.simantics.db.server - org.simantics.db.services - org.simantics.debug.browser - org.simantics.debug.browser.ui - org.simantics.debug.graphical - org.simantics.debug.ui - org.simantics.diagram - org.simantics.diagram.connection - org.simantics.diagram.ontology - org.simantics.diagram.profile - org.simantics.document - org.simantics.document.base.ontology - org.simantics.document.linking.ontology - org.simantics.document.linking.ui - org.simantics.document.ontology - org.simantics.document.server - org.simantics.document.server.io - org.simantics.document.swt.core - org.simantics.document.swt.ontology - org.simantics.document.ui - org.simantics.document.ui.ontology - org.simantics.dublincore.ontology - org.simantics.editors - org.simantics.editors.win32 - org.simantics.equation - org.simantics.event - org.simantics.event.ontology - org.simantics.excel - org.simantics.export.core - org.simantics.export.ui - org.simantics.fastlz - org.simantics.g2d - org.simantics.g2d.ontology - org.simantics.graph - org.simantics.graph.compiler - org.simantics.graph.db - org.simantics.graphfile - org.simantics.graphfile.ontology - org.simantics.graphviz - org.simantics.graphviz.ui - org.simantics.history - org.simantics.image.ontology - org.simantics.image.ui - org.simantics.image2.ontology - org.simantics.issues - org.simantics.issues.common - org.simantics.issues.ontology - org.simantics.issues.ui - org.simantics.issues.ui.ontology - org.simantics.layer0 - org.simantics.layer0.utils - org.simantics.layer0x.ontology - org.simantics.ltk - org.simantics.ltk.antlr - org.simantics.lz4 - org.simantics.mapping - org.simantics.message - org.simantics.message.ui - org.simantics.migration.ui - org.simantics.modeling - org.simantics.modeling.ontology - org.simantics.modeling.template2d.ontology - org.simantics.modeling.template2d.ui - org.simantics.modeling.ui - org.simantics.nativemem - org.simantics.objmap2 - org.simantics.platform.ui.ontology - org.simantics.project - org.simantics.project.ontology - org.simantics.scenegraph - org.simantics.scenegraph.loader - org.simantics.scenegraph.ontology - org.simantics.scenegraph.profile - org.simantics.scenegraph.swing - org.simantics.scenegraph.ui - org.simantics.scl.commands - org.simantics.scl.compiler - org.simantics.scl.compiler.dummy - org.simantics.scl.data - org.simantics.scl.db - org.simantics.scl.expressions - org.simantics.scl.osgi - org.simantics.scl.reflection - org.simantics.scl.runtime - org.simantics.scl.ui - org.simantics.scl.ui.editor - org.simantics.selectionview - org.simantics.selectionview.ontology - org.simantics.selectionview.ui.ontology - org.simantics.silk.ontology - org.simantics.simulation - org.simantics.simulation.ontology - org.simantics.simulation.sequences - org.simantics.simulation.ui - org.simantics.simulator.variable - org.simantics.softwareconfiguration.ontology - org.simantics.spreadsheet - org.simantics.spreadsheet.common - org.simantics.spreadsheet.graph - org.simantics.spreadsheet.ontology - org.simantics.spreadsheet.ui - org.simantics.structural.ontology - org.simantics.structural.synchronization - org.simantics.structural.synchronization.client - org.simantics.structural.ui - org.simantics.structural2 - org.simantics.team.ui - org.simantics.threadlog - org.simantics.trend - org.simantics.ui - org.simantics.user.ontology - org.simantics.utils - org.simantics.utils.datastructures - org.simantics.utils.thread - org.simantics.utils.thread.swt - org.simantics.utils.ui - org.simantics.utils.ui.workbench - org.simantics.viewpoint.ontology - org.simantics.views - org.simantics.views.ontology - org.simantics.views.swt - org.simantics.views.swt.client - org.simantics.wiki.ui - org.simantics.workbench - org.simantics.workbench.ontology - org.simantics.workbench.search - winterwell.markdown - - + + + 4.0.0 + org.simantics + org.simantics.root.bundles + 1.0.0-SNAPSHOT + pom + + + org.simantics + org.simantics.root + 1.0.0-SNAPSHOT + + + + + + org.eclipse.tycho + tycho-compiler-plugin + ${tycho.version} + + -err:-forbidden + + + + org.eclipse.tycho + tycho-source-plugin + ${tycho.version} + + + plugin-source + + plugin-source + + + + + + + + + com.famfamfam.silk + org.simantics + org.simantics.action.ontology + org.simantics.acorn + org.simantics.annotation.ontology + org.simantics.annotation.ui + org.simantics.application + org.simantics.backup + org.simantics.backup.db + org.simantics.backup.ontology + org.simantics.basicexpression + org.simantics.browsing.ui + org.simantics.browsing.ui.common + org.simantics.browsing.ui.graph + org.simantics.browsing.ui.graph.impl + org.simantics.browsing.ui.model + org.simantics.browsing.ui.ontology + org.simantics.browsing.ui.platform + org.simantics.browsing.ui.swt + org.simantics.charts + org.simantics.charts.ontology + org.simantics.color.ontology + org.simantics.common + org.simantics.compressions + org.simantics.databoard + org.simantics.datatypes + org.simantics.datatypes.ontology + org.simantics.db + org.simantics.db.common + org.simantics.db.impl + org.simantics.db.indexing + org.simantics.db.layer0 + org.simantics.db.management + org.simantics.db.procore + org.simantics.db.procore.server.environment + org.simantics.db.procore.ui + org.simantics.db.server + org.simantics.db.services + org.simantics.debug.browser + org.simantics.debug.browser.ui + org.simantics.debug.graphical + org.simantics.debug.ui + org.simantics.diagram + org.simantics.diagram.connection + org.simantics.diagram.ontology + org.simantics.diagram.profile + org.simantics.document + org.simantics.document.base.ontology + org.simantics.document.linking.ontology + org.simantics.document.linking.ui + org.simantics.document.ontology + org.simantics.document.server + org.simantics.document.server.io + org.simantics.document.swt.core + org.simantics.document.swt.ontology + org.simantics.document.ui + org.simantics.document.ui.ontology + org.simantics.dublincore.ontology + org.simantics.editors + org.simantics.editors.win32 + org.simantics.equation + org.simantics.event + org.simantics.event.ontology + org.simantics.excel + org.simantics.export.core + org.simantics.export.ui + org.simantics.fastlz + org.simantics.fileimport + org.simantics.fileimport.ui + org.simantics.g2d + org.simantics.g2d.ontology + org.simantics.graph + org.simantics.graph.compiler + org.simantics.graph.db + org.simantics.graphfile + org.simantics.graphfile.ontology + org.simantics.graphviz + org.simantics.graphviz.ui + org.simantics.history + org.simantics.image.ontology + org.simantics.image.ui + org.simantics.image2.ontology + org.simantics.issues + org.simantics.issues.common + org.simantics.issues.ontology + org.simantics.issues.ui + org.simantics.issues.ui.ontology + org.simantics.layer0 + org.simantics.layer0.utils + org.simantics.layer0x.ontology + org.simantics.logback.configuration + org.simantics.ltk + org.simantics.ltk.antlr + org.simantics.lz4 + org.simantics.mapping + org.simantics.message + org.simantics.message.ui + org.simantics.migration.ui + org.simantics.modeling + org.simantics.modeling.ontology + org.simantics.modeling.template2d.ontology + org.simantics.modeling.template2d.ui + org.simantics.modeling.ui + org.simantics.nativemem + org.simantics.objmap2 + org.simantics.platform.ui.ontology + org.simantics.project + org.simantics.project.ontology + org.simantics.scenegraph + org.simantics.scenegraph.loader + org.simantics.scenegraph.ontology + org.simantics.scenegraph.profile + org.simantics.scenegraph.swing + org.simantics.scenegraph.ui + org.simantics.scl.commands + org.simantics.scl.compiler + org.simantics.scl.compiler.dummy + org.simantics.scl.data + org.simantics.scl.db + org.simantics.scl.expressions + org.simantics.scl.osgi + org.simantics.scl.reflection + org.simantics.scl.runtime + org.simantics.scl.ui + org.simantics.scl.ui.editor + org.simantics.selectionview + org.simantics.selectionview.ontology + org.simantics.selectionview.ui.ontology + org.simantics.silk.ontology + org.simantics.simulation + org.simantics.simulation.ontology + org.simantics.simulation.sequences + org.simantics.simulation.ui + org.simantics.simulator.variable + org.simantics.softwareconfiguration.ontology + org.simantics.spreadsheet + org.simantics.spreadsheet.common + org.simantics.spreadsheet.graph + org.simantics.spreadsheet.ontology + org.simantics.spreadsheet.ui + org.simantics.structural.ontology + org.simantics.structural.synchronization + org.simantics.structural.synchronization.client + org.simantics.structural.ui + org.simantics.structural2 + org.simantics.team.ui + org.simantics.threadlog + org.simantics.trend + org.simantics.ui + org.simantics.user.ontology + org.simantics.utils + org.simantics.utils.datastructures + org.simantics.utils.thread + org.simantics.utils.thread.swt + org.simantics.utils.ui + org.simantics.utils.ui.workbench + org.simantics.viewpoint.ontology + org.simantics.views + org.simantics.views.ontology + org.simantics.views.swt + org.simantics.views.swt.client + org.simantics.wiki.ui + org.simantics.workbench + org.simantics.workbench.ontology + org.simantics.workbench.search + winterwell.markdown + + diff --git a/features/com.lowagie.text.feature/.gitignore b/features/com.lowagie.text.feature/.gitignore index 46281dc83..c4130baed 100644 --- a/features/com.lowagie.text.feature/.gitignore +++ b/features/com.lowagie.text.feature/.gitignore @@ -1 +1 @@ -/target/** +/target/** diff --git a/features/org.apache.lucene4.feature/.gitignore b/features/org.apache.lucene4.feature/.gitignore index 46281dc83..c4130baed 100644 --- a/features/org.apache.lucene4.feature/.gitignore +++ b/features/org.apache.lucene4.feature/.gitignore @@ -1 +1 @@ -/target/** +/target/** diff --git a/features/org.simantics.charts.feature/feature.xml b/features/org.simantics.charts.feature/feature.xml index 65acfa390..32d412a74 100644 --- a/features/org.simantics.charts.feature/feature.xml +++ b/features/org.simantics.charts.feature/feature.xml @@ -17,16 +17,12 @@ [Enter License Description here.] - - + + + + diff --git a/features/org.simantics.desktop.feature/feature.xml b/features/org.simantics.desktop.feature/feature.xml index d29ccac34..ee8db3761 100644 --- a/features/org.simantics.desktop.feature/feature.xml +++ b/features/org.simantics.desktop.feature/feature.xml @@ -26,7 +26,7 @@ version="0.0.0"/> diff --git a/features/org.simantics.g2d.feature/feature.xml b/features/org.simantics.g2d.feature/feature.xml index 060e326e0..e3e9eb0d3 100644 --- a/features/org.simantics.g2d.feature/feature.xml +++ b/features/org.simantics.g2d.feature/feature.xml @@ -33,11 +33,11 @@ version="0.0.0"/> - - diff --git a/features/org.simantics.issues.ui.feature/feature.xml b/features/org.simantics.issues.ui.feature/feature.xml index 035aa296c..71b86dfd0 100644 --- a/features/org.simantics.issues.ui.feature/feature.xml +++ b/features/org.simantics.issues.ui.feature/feature.xml @@ -1,6 +1,6 @@ @@ -18,7 +18,7 @@ - - diff --git a/features/org.simantics.platform.ui.feature/feature.xml b/features/org.simantics.platform.ui.feature/feature.xml index c3efa01b5..000aad2c7 100644 --- a/features/org.simantics.platform.ui.feature/feature.xml +++ b/features/org.simantics.platform.ui.feature/feature.xml @@ -32,7 +32,7 @@ reusable components implemented on top of the platform. version="0.0.0"/> + - - - + --> + + + Briefly put, includes Eclipse RCP feature along with other basic framework parts often required by applications with a workbench UI. @@ -29,15 +29,15 @@ To be more precise, the additions to Eclipse RCP are: <li>Apache Commons HTTPClient + dependencies</li> <li>Apache Xerces XML parser + dependencies</li> <li>JUnit 4</li> -</ul> - - - +</ul> + + + Copyright (c) 2007, 2010 Association for Decentralized Information Management in Industry THTH ry.<br/> -All rights reserved. This program and the accompanying materials are made available under the terms of the Eclipse Public License v1.0 which accompanies this distribution, and is available at <a href="http://www.eclipse.org/legal/epl-v10.html">http://www.eclipse.org/legal/epl-v10.html</a> - - - +All rights reserved. This program and the accompanying materials are made available under the terms of the Eclipse Public License v1.0 which accompanies this distribution, and is available at <a href="http://www.eclipse.org/legal/epl-v10.html">http://www.eclipse.org/legal/epl-v10.html</a> + + + Eclipse Public License - v 1.0 THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. @@ -107,573 +107,594 @@ All Recipient's rights under this Agreement shall terminate if it fails to Everyone is permitted to copy and distribute copies of this Agreement, but in order to avoid inconsistency the Agreement is copyrighted and may only be modified in the following manner. The Agreement Steward reserves the right to publish new versions (including revisions) of this Agreement from time to time. No one other than the Agreement Steward has the right to modify this Agreement. The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation may assign the responsibility to serve as the Agreement Steward to a suitable separate entity. Each new version of the Agreement will be given a distinguishing version number. The Program (including Contributions) may always be distributed subject to the version of the Agreement under which it was received. In addition, after a new version of the Agreement is published, Contributor may elect to distribute the Program (including its Contributions) under the new version. Except as expressly stated in Sections 2(a) and 2(b) above, Recipient receives no rights or licenses to the intellectual property of any Contributor under this Agreement, whether expressly, by implication, estoppel or otherwise. All rights in the Program not expressly granted under this Agreement are reserved. -This Agreement is governed by the laws of the State of New York and the intellectual property laws of the United States of America. No party to this Agreement will bring a legal action under this Agreement more than one year after the cause of action arose. Each party waives its rights to a jury trial in any resulting litigation. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +This Agreement is governed by the laws of the State of New York and the intellectual property laws of the United States of America. No party to this Agreement will bring a legal action under this Agreement more than one year after the cause of action arose. Each party waives its rights to a jury trial in any resulting litigation. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/features/org.simantics.sdk.feature/feature.xml b/features/org.simantics.sdk.feature/feature.xml index ee4a899a1..ed48ad311 100644 --- a/features/org.simantics.sdk.feature/feature.xml +++ b/features/org.simantics.sdk.feature/feature.xml @@ -49,7 +49,7 @@ version="0.0.0"/> + + + + + + + + diff --git a/features/org.simantics.utils.feature/feature.xml b/features/org.simantics.utils.feature/feature.xml index 3892f7d08..b81e2e567 100644 --- a/features/org.simantics.utils.feature/feature.xml +++ b/features/org.simantics.utils.feature/feature.xml @@ -1,14 +1,14 @@ - - - @@ -39,28 +35,35 @@ + + diff --git a/features/org.simantics.views.swt.client.feature/feature.xml b/features/org.simantics.views.swt.client.feature/feature.xml index ae293d674..d8b3cc81f 100644 --- a/features/org.simantics.views.swt.client.feature/feature.xml +++ b/features/org.simantics.views.swt.client.feature/feature.xml @@ -1,6 +1,6 @@ diff --git a/features/org.simantics.views.swt.feature/feature.xml b/features/org.simantics.views.swt.feature/feature.xml index f94557de7..e80360cf6 100644 --- a/features/org.simantics.views.swt.feature/feature.xml +++ b/features/org.simantics.views.swt.feature/feature.xml @@ -1,6 +1,6 @@ @@ -17,12 +17,9 @@ [Enter License Description here.] - + - - diff --git a/features/org.simantics.wiki.ui.feature/feature.xml b/features/org.simantics.wiki.ui.feature/feature.xml index 8284fb3c0..5569344e7 100644 --- a/features/org.simantics.wiki.ui.feature/feature.xml +++ b/features/org.simantics.wiki.ui.feature/feature.xml @@ -1,6 +1,6 @@ diff --git a/pom.xml b/pom.xml index eeebff8e9..85a794381 100644 --- a/pom.xml +++ b/pom.xml @@ -17,6 +17,7 @@ bundles features releng + tests diff --git a/releng/org.simantics.sdk.build.p2.site/pom.xml b/releng/org.simantics.sdk.build.p2.site/pom.xml index 4e17e3765..0693c09b7 100644 --- a/releng/org.simantics.sdk.build.p2.site/pom.xml +++ b/releng/org.simantics.sdk.build.p2.site/pom.xml @@ -1,335 +1,361 @@ - - - 4.0.0 - - org.simantics - org.simantics.sdk.build.p2.site - pom - 1.25.0 - - - 2.1.7.b1 - 4.9.0 - 4.9.0.b0003 - org.apache.lucene4 - - - - - - - - - - - org.eclipse.tycho.extras - tycho-p2-extras-plugin - 0.25.0 - - - org.reficio - p2-maven-plugin - 1.2.0-SNAPSHOT - - - default-cli - - - - - - org.antlr:antlr-runtime:3.5.2 - true - - - org.apache.poi:poi:3.14 - true - - - org.apache.poi:poi-ooxml:3.14 - true - - - org.apache.lucene:lucene-core:${lucene.version} - true - - ${lucene.prefix}.core - ${lucene.version.actual} - *;version="${lucene.version}" - - - - org.apache.lucene:lucene-queries:${lucene.version} - true - - ${lucene.prefix}.queries - ${lucene.version.actual} - ${lucene.prefix}.core;bundle-version="${lucene.version}" - *;version="${lucene.version}" - !org.apache.lucene.*,*;resolution:=optional - - - - org.apache.lucene:lucene-sandbox:${lucene.version} - true - - ${lucene.prefix}.sandbox - ${lucene.version.actual} - ${lucene.prefix}.core;bundle-version="${lucene.version}" - *;version="${lucene.version}" - !org.apache.lucene.*,*;resolution:=optional - - - - org.apache.lucene:lucene-analyzers-common:${lucene.version} - true - - ${lucene.prefix}.analyzers-common - ${lucene.version.actual} - ${lucene.prefix}.core;bundle-version="${lucene.version}" - *;version="${lucene.version}" - !org.apache.lucene.*,*;resolution:=optional - - - - org.apache.lucene:lucene-queryparser:${lucene.version} - true - - ${lucene.prefix}.queryparser - ${lucene.version.actual} - ${lucene.prefix}.core;bundle-version="${lucene.version}",${lucene.prefix}.queries;bundle-version="${lucene.version}",${lucene.prefix}.sandbox;bundle-version="${lucene.version}" - *;version="${lucene.version}" - !org.apache.lucene.*,*;resolution:=optional - - - - org.jdom:jdom2:2.0.6 - true - - org.jdom2 - - - - org.jfree:jfreechart:1.0.19 - true - - org.jfree.jchart - - - - - net.sf.supercsv:super-csv:2.4.0 - true - true - - org.supercsv - - - - org.ini4j:ini4j:0.5.4 - true - - - commons-collections:commons-collections:3.2.2 - true - - - org.apache.commons:commons-compress:1.12 - true - - - commons-lang:commons-lang:2.6 - true - - - commons-io:commons-io:1.4 - true - - - org.apache.pdfbox:pdfbox:2.0.2 - true - - - org.apache.pdfbox:fontbox:2.0.2 - true - - - org.apache.pdfbox:xmpbox:2.0.2 - true - - - log4j:log4j:1.2.17 - true - true - - org.apache.log4j - - - - net.sf.ucanaccess:ucanaccess:3.0.6 - true - - net.ucanaccess - - - - it.unimi.dsi:fastutil:7.0.12 - true - - - org.eclipse.collections:eclipse-collections-api:7.1.0 - true - true - - org.eclipse.collections.* - - - - org.eclipse.collections:eclipse-collections:7.1.0 - true - - - net.sf.trove4j:trove4j:2.1.0 - true - - GNU Trove 2 - gnu.trove2 - - - - net.sf.trove4j:trove4j:3.0.3 - true - - GNU Trove 3 - gnu.trove3 - - - - org.freemarker:freemarker:2.3.23 - true - true - - freemarker - - - - com.lowagie:itext:2.1.7 - true - - com.lowagie.text - ${itext.version.actual} - - - - javax.vecmath:vecmath:1.5.2 - false - true - - javax.vecmath - - - - org.mozilla:rhino:1.7.7.1 - true - - - net.java.dev.jna:jna:4.2.2 - true - - - net.java.dev.jna:jna-platform:4.2.2 - true - - - - - - - - - org.mortbay.jetty - jetty-maven-plugin - 8.1.5.v20120716 - - 10 - ${basedir}/target/repository/ - - /site - - - - - - - - - reficio - http://repo.reficio.org/maven/ - - - - + + + 4.0.0 + + org.simantics + org.simantics.sdk.build.p2.site + pom + 1.25.0 + + + 2.1.7.b1 + 4.9.0 + 4.9.0.b0003 + org.apache.lucene4 + + + + + + + + + + + org.eclipse.tycho.extras + tycho-p2-extras-plugin + 0.25.0 + + + org.reficio + p2-maven-plugin + 1.2.0-SNAPSHOT + + + default-cli + + + + + + org.antlr:antlr-runtime:3.5.2 + true + + + org.apache.poi:poi:3.14 + true + + + org.apache.poi:poi-ooxml:3.14 + true + + + org.apache.lucene:lucene-core:${lucene.version} + true + + ${lucene.prefix}.core + ${lucene.version.actual} + *;version="${lucene.version}" + + + + org.apache.lucene:lucene-queries:${lucene.version} + true + + ${lucene.prefix}.queries + ${lucene.version.actual} + ${lucene.prefix}.core;bundle-version="${lucene.version}" + *;version="${lucene.version}" + !org.apache.lucene.*,*;resolution:=optional + + + + org.apache.lucene:lucene-sandbox:${lucene.version} + true + + ${lucene.prefix}.sandbox + ${lucene.version.actual} + ${lucene.prefix}.core;bundle-version="${lucene.version}" + *;version="${lucene.version}" + !org.apache.lucene.*,*;resolution:=optional + + + + org.apache.lucene:lucene-analyzers-common:${lucene.version} + true + + ${lucene.prefix}.analyzers-common + ${lucene.version.actual} + ${lucene.prefix}.core;bundle-version="${lucene.version}" + *;version="${lucene.version}" + !org.apache.lucene.*,*;resolution:=optional + + + + org.apache.lucene:lucene-queryparser:${lucene.version} + true + + ${lucene.prefix}.queryparser + ${lucene.version.actual} + ${lucene.prefix}.core;bundle-version="${lucene.version}",${lucene.prefix}.queries;bundle-version="${lucene.version}",${lucene.prefix}.sandbox;bundle-version="${lucene.version}" + *;version="${lucene.version}" + !org.apache.lucene.*,*;resolution:=optional + + + + org.jdom:jdom2:2.0.6 + true + + org.jdom2 + + + + org.jfree:jfreechart:1.0.19 + true + + org.jfree.jchart + + + + + net.sf.supercsv:super-csv:2.4.0 + true + true + + org.supercsv + + + + org.ini4j:ini4j:0.5.4 + true + + + commons-collections:commons-collections:3.2.2 + true + + + org.apache.commons:commons-compress:1.12 + true + + + commons-lang:commons-lang:2.6 + true + + + commons-io:commons-io:1.4 + true + + + org.apache.pdfbox:pdfbox:2.0.3 + true + + + org.apache.pdfbox:fontbox:2.0.3 + true + + + org.apache.pdfbox:xmpbox:2.0.3 + true + + + log4j:log4j:1.2.17 + true + true + + org.apache.log4j + + + + net.sf.ucanaccess:ucanaccess:3.0.6 + true + + net.ucanaccess + + + + it.unimi.dsi:fastutil:7.0.13 + true + + + org.eclipse.collections:eclipse-collections-api:7.1.0 + true + true + + org.eclipse.collections.* + + + + org.eclipse.collections:eclipse-collections:7.1.0 + true + + + net.sf.trove4j:trove4j:2.1.0 + true + + GNU Trove 2 + gnu.trove2 + + + + net.sf.trove4j:trove4j:3.0.3 + true + + GNU Trove 3 + gnu.trove3 + + + + org.freemarker:freemarker:2.3.23 + true + true + + freemarker + + + + com.lowagie:itext:2.1.7 + true + + com.lowagie.text + ${itext.version.actual} + + + + javax.vecmath:vecmath:1.5.2 + false + true + + javax.vecmath + + + + org.mozilla:rhino:1.7.7.1 + true + + + net.java.dev.jna:jna:4.2.2 + true + + + net.java.dev.jna:jna-platform:4.2.2 + true + + + com.fasterxml.jackson.core:jackson-core:2.8.2 + true + + + org.slf4j:slf4j-api:1.7.20 + true + true + + SLF4J API Module + org.slf4j.api + + + + ch.qos.logback:logback-classic:1.1.7 + true + true + + + com.koloboke:koloboke-compile:0.5.1 + true + + + com.koloboke:koloboke-impl-common-jdk8:1.0.0 + true + + + + + + + + + org.mortbay.jetty + jetty-maven-plugin + 8.1.5.v20120716 + + 10 + ${basedir}/target/repository/ + + /site + + + + + + + + + reficio + http://repo.reficio.org/maven/ + + + + diff --git a/releng/org.simantics.sdk.build.targetdefinition/org.simantics.sdk.build.targetdefinition-semantum.target b/releng/org.simantics.sdk.build.targetdefinition/org.simantics.sdk.build.targetdefinition-semantum.target new file mode 100644 index 000000000..e4a3cb62c --- /dev/null +++ b/releng/org.simantics.sdk.build.targetdefinition/org.simantics.sdk.build.targetdefinition-semantum.target @@ -0,0 +1,200 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/releng/org.simantics.sdk.build.targetdefinition/org.simantics.sdk.build.targetdefinition.target b/releng/org.simantics.sdk.build.targetdefinition/org.simantics.sdk.build.targetdefinition.target index 34bfa95c2..705d64a33 100644 --- a/releng/org.simantics.sdk.build.targetdefinition/org.simantics.sdk.build.targetdefinition.target +++ b/releng/org.simantics.sdk.build.targetdefinition/org.simantics.sdk.build.targetdefinition.target @@ -1,186 +1,206 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/releng/org.simantics.target/simantics-sdk.target b/releng/org.simantics.target/simantics-sdk.target deleted file mode 100644 index ce7cb626b..000000000 --- a/releng/org.simantics.target/simantics-sdk.target +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - - - - - - - - - - - - diff --git a/releng/org.simantics.target/simantics.target b/releng/org.simantics.target/simantics.target index 324e1ea3f..789164029 100644 --- a/releng/org.simantics.target/simantics.target +++ b/releng/org.simantics.target/simantics.target @@ -1,9 +1,10 @@ - + + - - + + diff --git a/sonar-simantics-platform-sdk.properties b/sonar-simantics-platform-sdk.properties new file mode 100644 index 000000000..d92944975 --- /dev/null +++ b/sonar-simantics-platform-sdk.properties @@ -0,0 +1,23 @@ +# required metadata +sonar.projectKey=simantics-platform-sdk +sonar.projectName=Simantics Platform SDK +sonar.projectVersion=1.25 + +# path to source directories (required) +sonar.sources=bundles + +# path to test source directories (optional) +#sonar.tests=testDir1,testDir2 + +# path to project binaries (optional), for example directory of Java bytecode +#sonar.binaries=binDir + +# optional comma-separated list of paths to libraries. Only path to JAR file and path to directory of classes are supported. +#sonar.libraries=path/to/library.jar,path/to/classes/dir + +# Uncomment this line to analyse a project which is not a java project. +# The value of the property must be the key of the language. +#sonar.language=cobol + +# Additional parameters +#sonar.my.property=value \ No newline at end of file diff --git a/tests/org.simantics.scl.compiler.tests/.classpath b/tests/org.simantics.scl.compiler.tests/.classpath new file mode 100644 index 000000000..b862a296d --- /dev/null +++ b/tests/org.simantics.scl.compiler.tests/.classpath @@ -0,0 +1,7 @@ + + + + + + + diff --git a/tests/org.simantics.scl.compiler.tests/.project b/tests/org.simantics.scl.compiler.tests/.project new file mode 100644 index 000000000..9fa2ae131 --- /dev/null +++ b/tests/org.simantics.scl.compiler.tests/.project @@ -0,0 +1,28 @@ + + + org.simantics.scl.compiler.tests + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.pde.ManifestBuilder + + + + + org.eclipse.pde.SchemaBuilder + + + + + + org.eclipse.pde.PluginNature + org.eclipse.jdt.core.javanature + + diff --git a/tests/org.simantics.scl.compiler.tests/.settings/org.eclipse.jdt.core.prefs b/tests/org.simantics.scl.compiler.tests/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 000000000..295926d96 --- /dev/null +++ b/tests/org.simantics.scl.compiler.tests/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,7 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.problem.assertIdentifier=error +org.eclipse.jdt.core.compiler.problem.enumIdentifier=error +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/tests/org.simantics.scl.compiler.tests/META-INF/MANIFEST.MF b/tests/org.simantics.scl.compiler.tests/META-INF/MANIFEST.MF new file mode 100644 index 000000000..5caaeecbd --- /dev/null +++ b/tests/org.simantics.scl.compiler.tests/META-INF/MANIFEST.MF @@ -0,0 +1,12 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: Tests +Bundle-SymbolicName: org.simantics.scl.compiler.tests +Bundle-Version: 1.0.0.qualifier +Bundle-RequiredExecutionEnvironment: JavaSE-1.8 +Require-Bundle: org.junit;bundle-version="4.12.0", + org.simantics.scl.compiler;bundle-version="0.5.0", + org.simantics.scl.osgi;bundle-version="1.0.4", + gnu.trove3, + org.objectweb.asm.util, + org.eclipse.equinox.ds;bundle-version="1.4.300" diff --git a/tests/org.simantics.scl.compiler.tests/build.properties b/tests/org.simantics.scl.compiler.tests/build.properties new file mode 100644 index 000000000..41eb6ade2 --- /dev/null +++ b/tests/org.simantics.scl.compiler.tests/build.properties @@ -0,0 +1,4 @@ +source.. = src/ +output.. = bin/ +bin.includes = META-INF/,\ + . diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/ActiveTests.java b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/ActiveTests.java new file mode 100644 index 000000000..f9533efee --- /dev/null +++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/ActiveTests.java @@ -0,0 +1,24 @@ +package org.simantics.scl.compiler.tests; + +import org.junit.Ignore; +import org.junit.Test; + +public class ActiveTests extends TestBase { + + public ActiveTests() { super("scl"); } +/* + @Test public void Equations1() { test(); } + @Test public void MarketModel2() { test(); } + @Test public void Overloading2() { test(); } + @Test public void Overloading3() { test(); } + //@Ignore + @Test public void PatternError() { test(); } + @Test public void Serialization() { test(); } + @Ignore + @Test public void TypeClass2() { test(); } + @Test public void TypeClassBug2() { test(); } + */ + + //@Test public void CityoptSetup() { test(); } + @Test public void EmptyLet() { test(); } +} diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/FindAllowedChars.java b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/FindAllowedChars.java new file mode 100644 index 000000000..1e4078670 --- /dev/null +++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/FindAllowedChars.java @@ -0,0 +1,66 @@ +package org.simantics.scl.compiler.tests; + +import java.lang.reflect.Method; + +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; + +public class FindAllowedChars { + public static class MyClassLoader extends ClassLoader { + final String className; + final byte[] classBytes; + + public MyClassLoader(ClassLoader parent, String className, byte[] classBytes) { + super(parent); + this.className = className; + this.classBytes = classBytes; + } + + public MyClassLoader(String className, byte[] classBytes) { + this.className = className; + this.classBytes = classBytes; + } + + @Override + protected Class findClass(String name) throws ClassNotFoundException { + if(name.equals(name)) + return defineClass(name, classBytes, 0, classBytes.length); + else + return super.findClass(name); + } + } + + public static void test(String className, String methodName) throws Exception { + ClassWriter classWriter = new ClassWriter(ClassWriter.COMPUTE_MAXS); + classWriter.visit(Opcodes.V1_5, Opcodes.ACC_PUBLIC, className, null, "java/lang/Object", null); + + MethodVisitor methodVisitor = classWriter.visitMethod(Opcodes.ACC_PUBLIC | Opcodes.ACC_STATIC, methodName, "()V", null, null); + /*methodVisitor.visitFieldInsn(Opcodes.GETSTATIC, "java/lang/System", "out", "Ljava/io/PrintStream;"); + methodVisitor.visitLdcInsn("Hello world!"); + methodVisitor.visitMethodInsn(Opcodes.INVOKEVIRTUAL, "java/io/PrintStream", "println", "(Ljava/lang/String;)V", false);*/ + methodVisitor.visitInsn(Opcodes.RETURN); + methodVisitor.visitMaxs(0, 0); + methodVisitor.visitEnd(); + classWriter.visitEnd(); + + ClassLoader loader = new MyClassLoader(className, classWriter.toByteArray()); + Class clazz = loader.loadClass(className); + Method method = clazz.getMethod(methodName); + method.invoke(null); + } + + public static void main(String[] args) throws Exception { + for(int a=Character.MIN_VALUE;a maybeModule = PRELUDE_MODULE_REPOSITORY.getModule("Prelude"); + if(!maybeModule.didSucceed()) + return; + Module module = maybeModule.getResult(); + ModuleCoverage coverage = CoverageUtils.getCoverage(module); + if(coverage == null) + return; + coverage.print(System.out); + printCoverageTree(module.getBranchPoints().get("lookup"), 0); + } + + private static void printCoverageTree(BranchPoint[] branchPoints, int ind) { + for(BranchPoint bp : branchPoints) { + for(int i=0;i auxModuleNameList = new ArrayList(); + while(j < testParts.length) { + String part = testParts[j]; + if(part.startsWith("// module ")) + auxModuleNameList.add(part.substring(10).split("\\n", 2)[0].trim()); + else + break; + ++j; + } + int mainId = j; + String[] moduleNames = new String[mainId+1]; + String[] moduleTexts = new String[mainId+1]; + for(int i=0;i result = testEnvironment.getModule(moduleNames[lastId]); + if(!result.didSucceed()) + return ((Failure)result).toString(moduleTexts[lastId]); + else { + Object main = testEnvironment.getRuntimeModule(moduleNames[lastId]).getResult().getValue("main"); + return String.valueOf(main); + } + } + + private String[] readTestParts(String testPath) throws IOException { + InputStream stream = getClass().getResourceAsStream(testPath); + try { + byte[] buffer = new byte[1024]; + int pos = 0; + while(true) { + int c = stream.read(buffer, pos, buffer.length-pos); + if(c <= 0) + break; + pos += c; + if(pos < buffer.length) + break; + buffer = Arrays.copyOf(buffer, pos*2); + } + String text = new String(buffer, 0, pos, UTF8); + String[] result = TEST_SEPARATOR.split(text); + for(int i=1;i print i) [1,2,3]"); + session.execute("iter (\\i -> print i) [(),(),()]"); + } + +} diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/TestExpressionEvaluator.java b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/TestExpressionEvaluator.java new file mode 100644 index 000000000..e143c6ab9 --- /dev/null +++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/TestExpressionEvaluator.java @@ -0,0 +1,197 @@ +package org.simantics.scl.compiler.tests; + +import java.util.Arrays; + +import org.junit.Before; +import org.junit.Test; +import org.simantics.scl.compiler.elaboration.expressions.EVariable; +import org.simantics.scl.compiler.elaboration.expressions.Expression; +import org.simantics.scl.compiler.elaboration.expressions.Variable; +import org.simantics.scl.compiler.environment.AbstractLocalEnvironment; +import org.simantics.scl.compiler.environment.Environment; +import org.simantics.scl.compiler.environment.LocalEnvironment; +import org.simantics.scl.compiler.environment.specification.EnvironmentSpecification; +import org.simantics.scl.compiler.errors.CompilationErrorFormatter; +import org.simantics.scl.compiler.module.repository.ImportFailure; +import org.simantics.scl.compiler.module.repository.ImportFailureException; +import org.simantics.scl.compiler.module.repository.ModuleRepository; +import org.simantics.scl.compiler.runtime.RuntimeEnvironment; +import org.simantics.scl.compiler.source.repository.CompositeModuleSourceRepository; +import org.simantics.scl.compiler.source.repository.SourceRepositories; +import org.simantics.scl.compiler.top.ExpressionEvaluator; +import org.simantics.scl.compiler.top.SCLExpressionCompilationException; +import org.simantics.scl.compiler.types.Type; +import org.simantics.scl.compiler.types.Types; +import org.simantics.scl.runtime.function.Function; +import org.simantics.scl.runtime.tuple.Tuple0; + +import junit.framework.Assert; + +public class TestExpressionEvaluator { + + public static final boolean TIMING = false; + public static final int COUNT = 10000; + + ModuleRepository moduleRepository; + + RuntimeEnvironment runtimeEnvironment; + + @Before + public void initialize() throws Exception { + moduleRepository = InitialRepository.getInitialRepository(); + + // Environment for compiling expressions + EnvironmentSpecification environmentSpecification = new EnvironmentSpecification(); + environmentSpecification.importModule("Builtin", ""); + environmentSpecification.importModule("Prelude", ""); + + try { + runtimeEnvironment = moduleRepository.createRuntimeEnvironment(environmentSpecification, + getClass().getClassLoader()); + } catch(ImportFailureException e) { + for(ImportFailure failure : e.failures) + System.err.println("Failed to import " + failure.moduleName); + throw e; + } + } + + private void testExpression0(String expressionText, + Object expectedValue, + Type expectedType) throws Exception { + // Compiling and running expression + try { + Object result = new ExpressionEvaluator(runtimeEnvironment, expressionText) + .expectedType(expectedType) + .eval(); + if(expectedValue != null) + Assert.assertEquals(expectedValue, result); + } catch(SCLExpressionCompilationException e) { + System.out.println(CompilationErrorFormatter.toString(expressionText, e.getErrors())); + throw e; + } + } + + private void testExpression(String expressionText, + Object expectedValue, + Type expectedType) throws Exception { + if(TIMING) { + System.out.println(expressionText); + long beginTime = System.nanoTime(); + for(int i=0;i x) [(1,2),(2,3)]", + Arrays.asList(2.0, 3.0), + Types.list(Types.DOUBLE)); + testExpression("map (\\x -> snd x) [(1,2),(2,3)]", + Arrays.asList(2.0, 3.0), + Types.list(Types.DOUBLE)); + testExpression("let f x = x+1 in (f . f . f) 3", + Double.valueOf(6.0), + Types.DOUBLE); + if(!TIMING) + testExpression("print \"Hello world!\"", + Tuple0.INSTANCE, + Types.UNIT); + testExpression("[1,2+3,4+5]", + Arrays.asList(1,5,9), + Types.list(Types.INTEGER)); + testExpression("let a = 5.3 in let f x = x+a in f 3", + Double.valueOf(8.3), + Types.DOUBLE); + testExpression("let mm x y = if x < y then x else y in mm 2 (mm 1 3)", + Double.valueOf(1.0), + Types.DOUBLE); + } + + @Test + public void testLocalEnvironment() throws Exception { + String expressionText = "a + b"; + LocalEnvironment localEnvironment = new AbstractLocalEnvironment() { + Variable[] localParameters = new Variable[] { + new Variable("a", Types.DOUBLE), + new Variable("b", Types.DOUBLE), + }; + + @Override + public Expression resolve(Environment environment, String localName) { + if(localName.equals("a")) + return new EVariable(localParameters[0]); + else if(localName.equals("b")) + return new EVariable(localParameters[1]); + else + return null; + } + + @Override + protected Variable[] getContextVariables() { + return localParameters; + } + }; + try { + Object result = new ExpressionEvaluator(runtimeEnvironment, expressionText) + .localEnvironment(localEnvironment) + .expectedType(Types.DOUBLE) + .eval(); + Assert.assertEquals( + Double.valueOf(15.0), + ((Function)result).apply(7.0, 8.0)); + } catch(SCLExpressionCompilationException e) { + System.out.println(CompilationErrorFormatter.toString(expressionText, e.getErrors())); + throw e; + } + } + + @Test + public void testArities() throws Exception { + for(int arity=1;arity<50;++arity) { + // Build expressions + StringBuilder b = new StringBuilder(); + b.append('\\'); + for(int i=0;i "); + for(int i=0;i 0) + b.append(" + "); + b.append("v" + i); + } + //System.out.println(b.toString()); + + // Compile + Type expectedType = Types.INTEGER; + for(int i=0;i", "(Ljava/lang/Object;)V", false); + + methodWriter.visitVarInsn(Opcodes.ALOAD, 1); + methodWriter.visitVarInsn(Opcodes.ALOAD, 2); + methodWriter.visitMethodInsn(Opcodes.INVOKESPECIAL, "java/util/concurrent/atomic/AtomicReference", "", "(Ljava/lang/Object;)V", false); + + methodWriter.visitVarInsn(Opcodes.ALOAD, 2); + methodWriter.visitInsn(Opcodes.ARETURN); + methodWriter.visitMaxs(0, 0); + methodWriter.visitEnd(); + + byte[] bytes = classWriter.toByteArray(); + ClassLoader classLoader = new ClassLoader() { + @Override + protected Class findClass(String name) throws ClassNotFoundException { + if(!name.equals("test.Test")) + throw new ClassNotFoundException(); + return defineClass("test.Test", bytes, 0, bytes.length); + } + }; + Class clazz = classLoader.loadClass("test.Test"); + Method method = clazz.getMethod("main"); + AtomicReference result = (AtomicReference)method.invoke(null); + System.out.println(result.hashCode()); + //System.out.println(result.get().hashCode()); + } + +} diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/experimentation/TestEquals.java b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/experimentation/TestEquals.java new file mode 100644 index 000000000..022ae572f --- /dev/null +++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/experimentation/TestEquals.java @@ -0,0 +1,15 @@ +package org.simantics.scl.compiler.tests.experimentation; + +public class TestEquals { + public static boolean eqInt(int a, int b) { + return a==b; + } + + public static boolean eqDouble(double a, double b) { + return a==b; + } + + public static boolean eqObject(Object a, Object b) { + return a.equals(b); + } +} diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/experimentation/TestTypeDesc.java b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/experimentation/TestTypeDesc.java new file mode 100644 index 000000000..71538898d --- /dev/null +++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/experimentation/TestTypeDesc.java @@ -0,0 +1,12 @@ +package org.simantics.scl.compiler.tests.experimentation; + +import org.cojen.classfile.TypeDesc; + +public class TestTypeDesc { + public static void main(String[] args) { + System.out.println(TypeDesc.forClass(String.class).getFullName()); + System.out.println(TypeDesc.forClass(String.class).getDescriptor()); + System.out.println(TypeDesc.forClass(String.class).getRootName()); + + } +} diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/imports/Maybe4Imports.java b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/imports/Maybe4Imports.java new file mode 100644 index 000000000..30fe62719 --- /dev/null +++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/imports/Maybe4Imports.java @@ -0,0 +1,15 @@ +package org.simantics.scl.compiler.tests.imports; + +import java.util.Random; + +public class Maybe4Imports { + + public static Object toMaybeDouble(Random r, String s) { + try { + return new Double(s); + } catch(Exception e) { + return null; + } + } + +} diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/markdown/RunMarkdownTests.java b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/markdown/RunMarkdownTests.java new file mode 100644 index 000000000..9971a020b --- /dev/null +++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/markdown/RunMarkdownTests.java @@ -0,0 +1,100 @@ +package org.simantics.scl.compiler.tests.markdown; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.StringReader; +import java.nio.charset.Charset; + +import org.simantics.scl.compiler.markdown.internal.MarkdownParser; +import org.simantics.scl.compiler.markdown.nodes.Node; + +public class RunMarkdownTests { + + public static int FAILED = 0; + public static int SUCCEEDED = 1; + public static int SKIPPED = 2; + + public static void main(String[] args) throws IOException { + BufferedReader reader = new BufferedReader( + new InputStreamReader( + RunMarkdownTests.class.getResourceAsStream("spec.txt"), + Charset.forName("UTF-8"))); + + StringBuilder in = new StringBuilder(); + StringBuilder out = new StringBuilder(); + int state = 0; + int testId = 0; + int passed = 0; + int failed = 0; + int skipped = 0; + while(true) { + String line = reader.readLine(); + if(line == null) + break; + switch(state) { + case 0: + if(line.equals("```````````````````````````````` example")) + ++state; + break; + case 1: + if(line.equals(".")) + ++state; + else { + if(in.length() > 0) + in.append('\n'); + in.append(line); + } + break; + case 2: + if(line.equals("````````````````````````````````")) { + ++testId; + int status = test(testId, in.toString(), out.toString()); + if(status == SUCCEEDED) + ++passed; + else if(status == FAILED) + ++failed; + else + ++skipped; + in = new StringBuilder(); + out = new StringBuilder(); + state = 0; + } + else { + if(out.length() > 0) + out.append('\n'); + out.append(line); + } + break; + } + } + + System.out.println("Passed: " + passed + "/" + testId); + System.out.println("Failed: " + failed + "/" + testId); + System.out.println("Skipped: " + skipped + "/" + testId); + } + + public static int test(int id, String in, String out) throws IOException { + MarkdownParser parser = new MarkdownParser(); + Node node = parser.parseDocument(new StringReader(in.replace('\u2192', '\t'))); + + String result = node.toHtml().replace('\t', '\u2192'); + + boolean passed = result.equals(out); + + if(!passed) { + System.out.println("Example " + id); // + (passed ? " passed" : " failed")); + System.out.println("---- in --------------------------------------------------------------------"); + System.out.println(in); + System.out.println("---- expected --------------------------------------------------------------"); + System.out.println(out); + System.out.println("---- actual ----------------------------------------------------------------"); + System.out.println(result); + System.out.println("----------------------------------------------------------------------------"); + System.out.println(); + } + + return passed ? SUCCEEDED : FAILED; + } + +} diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/markdown/spec.txt b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/markdown/spec.txt new file mode 100644 index 000000000..bdaed436d --- /dev/null +++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/markdown/spec.txt @@ -0,0 +1,9252 @@ +--- +title: CommonMark Spec +author: John MacFarlane +version: 0.25 +date: '2016-03-24' +license: '[CC-BY-SA 4.0](http://creativecommons.org/licenses/by-sa/4.0/)' +... + +# Introduction + +## What is Markdown? + +Markdown is a plain text format for writing structured documents, +based on conventions used for indicating formatting in email and +usenet posts. It was developed in 2004 by John Gruber, who wrote +the first Markdown-to-HTML converter in perl, and it soon became +widely used in websites. By 2014 there were dozens of +implementations in many languages. Some of them extended basic +Markdown syntax with conventions for footnotes, definition lists, +tables, and other constructs, and some allowed output not just in +HTML but in LaTeX and many other formats. + +## Why is a spec needed? + +John Gruber's [canonical description of Markdown's +syntax](http://daringfireball.net/projects/markdown/syntax) +does not specify the syntax unambiguously. Here are some examples of +questions it does not answer: + +1. How much indentation is needed for a sublist? The spec says that + continuation paragraphs need to be indented four spaces, but is + not fully explicit about sublists. It is natural to think that + they, too, must be indented four spaces, but `Markdown.pl` does + not require that. This is hardly a "corner case," and divergences + between implementations on this issue often lead to surprises for + users in real documents. (See [this comment by John + Gruber](http://article.gmane.org/gmane.text.markdown.general/1997).) + +2. Is a blank line needed before a block quote or heading? + Most implementations do not require the blank line. However, + this can lead to unexpected results in hard-wrapped text, and + also to ambiguities in parsing (note that some implementations + put the heading inside the blockquote, while others do not). + (John Gruber has also spoken [in favor of requiring the blank + lines](http://article.gmane.org/gmane.text.markdown.general/2146).) + +3. Is a blank line needed before an indented code block? + (`Markdown.pl` requires it, but this is not mentioned in the + documentation, and some implementations do not require it.) + + ``` markdown + paragraph + code? + ``` + +4. What is the exact rule for determining when list items get + wrapped in `

` tags? Can a list be partially "loose" and partially + "tight"? What should we do with a list like this? + + ``` markdown + 1. one + + 2. two + 3. three + ``` + + Or this? + + ``` markdown + 1. one + - a + + - b + 2. two + ``` + + (There are some relevant comments by John Gruber + [here](http://article.gmane.org/gmane.text.markdown.general/2554).) + +5. Can list markers be indented? Can ordered list markers be right-aligned? + + ``` markdown + 8. item 1 + 9. item 2 + 10. item 2a + ``` + +6. Is this one list with a thematic break in its second item, + or two lists separated by a thematic break? + + ``` markdown + * a + * * * * * + * b + ``` + +7. When list markers change from numbers to bullets, do we have + two lists or one? (The Markdown syntax description suggests two, + but the perl scripts and many other implementations produce one.) + + ``` markdown + 1. fee + 2. fie + - foe + - fum + ``` + +8. What are the precedence rules for the markers of inline structure? + For example, is the following a valid link, or does the code span + take precedence ? + + ``` markdown + [a backtick (`)](/url) and [another backtick (`)](/url). + ``` + +9. What are the precedence rules for markers of emphasis and strong + emphasis? For example, how should the following be parsed? + + ``` markdown + *foo *bar* baz* + ``` + +10. What are the precedence rules between block-level and inline-level + structure? For example, how should the following be parsed? + + ``` markdown + - `a long code span can contain a hyphen like this + - and it can screw things up` + ``` + +11. Can list items include section headings? (`Markdown.pl` does not + allow this, but does allow blockquotes to include headings.) + + ``` markdown + - # Heading + ``` + +12. Can list items be empty? + + ``` markdown + * a + * + * b + ``` + +13. Can link references be defined inside block quotes or list items? + + ``` markdown + > Blockquote [foo]. + > + > [foo]: /url + ``` + +14. If there are multiple definitions for the same reference, which takes + precedence? + + ``` markdown + [foo]: /url1 + [foo]: /url2 + + [foo][] + ``` + +In the absence of a spec, early implementers consulted `Markdown.pl` +to resolve these ambiguities. But `Markdown.pl` was quite buggy, and +gave manifestly bad results in many cases, so it was not a +satisfactory replacement for a spec. + +Because there is no unambiguous spec, implementations have diverged +considerably. As a result, users are often surprised to find that +a document that renders one way on one system (say, a github wiki) +renders differently on another (say, converting to docbook using +pandoc). To make matters worse, because nothing in Markdown counts +as a "syntax error," the divergence often isn't discovered right away. + +## About this document + +This document attempts to specify Markdown syntax unambiguously. +It contains many examples with side-by-side Markdown and +HTML. These are intended to double as conformance tests. An +accompanying script `spec_tests.py` can be used to run the tests +against any Markdown program: + + python test/spec_tests.py --spec spec.txt --program PROGRAM + +Since this document describes how Markdown is to be parsed into +an abstract syntax tree, it would have made sense to use an abstract +representation of the syntax tree instead of HTML. But HTML is capable +of representing the structural distinctions we need to make, and the +choice of HTML for the tests makes it possible to run the tests against +an implementation without writing an abstract syntax tree renderer. + +This document is generated from a text file, `spec.txt`, written +in Markdown with a small extension for the side-by-side tests. +The script `tools/makespec.py` can be used to convert `spec.txt` into +HTML or CommonMark (which can then be converted into other formats). + +In the examples, the `→` character is used to represent tabs. + +# Preliminaries + +## Characters and lines + +Any sequence of [characters] is a valid CommonMark +document. + +A [character](@) is a Unicode code point. Although some +code points (for example, combining accents) do not correspond to +characters in an intuitive sense, all code points count as characters +for purposes of this spec. + +This spec does not specify an encoding; it thinks of lines as composed +of [characters] rather than bytes. A conforming parser may be limited +to a certain encoding. + +A [line](@) is a sequence of zero or more [characters] +other than newline (`U+000A`) or carriage return (`U+000D`), +followed by a [line ending] or by the end of file. + +A [line ending](@) is a newline (`U+000A`), a carriage return +(`U+000D`) not followed by a newline, or a carriage return and a +following newline. + +A line containing no characters, or a line containing only spaces +(`U+0020`) or tabs (`U+0009`), is called a [blank line](@). + +The following definitions of character classes will be used in this spec: + +A [whitespace character](@) is a space +(`U+0020`), tab (`U+0009`), newline (`U+000A`), line tabulation (`U+000B`), +form feed (`U+000C`), or carriage return (`U+000D`). + +[Whitespace](@) is a sequence of one or more [whitespace +characters]. + +A [Unicode whitespace character](@) is +any code point in the Unicode `Zs` class, or a tab (`U+0009`), +carriage return (`U+000D`), newline (`U+000A`), or form feed +(`U+000C`). + +[Unicode whitespace](@) is a sequence of one +or more [Unicode whitespace characters]. + +A [space](@) is `U+0020`. + +A [non-whitespace character](@) is any character +that is not a [whitespace character]. + +An [ASCII punctuation character](@) +is `!`, `"`, `#`, `$`, `%`, `&`, `'`, `(`, `)`, +`*`, `+`, `,`, `-`, `.`, `/`, `:`, `;`, `<`, `=`, `>`, `?`, `@`, +`[`, `\`, `]`, `^`, `_`, `` ` ``, `{`, `|`, `}`, or `~`. + +A [punctuation character](@) is an [ASCII +punctuation character] or anything in +the Unicode classes `Pc`, `Pd`, `Pe`, `Pf`, `Pi`, `Po`, or `Ps`. + +## Tabs + +Tabs in lines are not expanded to [spaces]. However, +in contexts where indentation is significant for the +document's structure, tabs behave as if they were replaced +by spaces with a tab stop of 4 characters. + +```````````````````````````````` example +→foo→baz→→bim +. +

foo→baz→→bim
+
+```````````````````````````````` + + +```````````````````````````````` example + →foo→baz→→bim +. +
foo→baz→→bim
+
+```````````````````````````````` + + +```````````````````````````````` example + a→a + ὐ→a +. +
a→a
+ὐ→a
+
+```````````````````````````````` + + +```````````````````````````````` example + - foo + +→bar +. +
    +
  • +

    foo

    +

    bar

    +
    • +
+```````````````````````````````` + +```````````````````````````````` example +- foo + +→→bar +. +
    +
  • +

    foo

    +
      bar
+
    +
    • +
+```````````````````````````````` + +```````````````````````````````` example +>→→foo +. +
+
  foo
+
+
+```````````````````````````````` + +```````````````````````````````` example +-→→foo +. +
    +
  • +
      foo
+
    +
    • +
+```````````````````````````````` + + +```````````````````````````````` example + foo +→bar +. +
foo
+bar
+
+```````````````````````````````` + +```````````````````````````````` example + - foo + - bar +→ - baz +. +
    +
  • foo +
      +
    • bar +
        +
      • baz
        • +
      +
      • +
    +
    • +
+```````````````````````````````` + + + +## Insecure characters + +For security reasons, the Unicode character `U+0000` must be replaced +with the REPLACEMENT CHARACTER (`U+FFFD`). + +# Blocks and inlines + +We can think of a document as a sequence of +[blocks](@)---structural elements like paragraphs, block +quotations, lists, headings, rules, and code blocks. Some blocks (like +block quotes and list items) contain other blocks; others (like +headings and paragraphs) contain [inline](@) content---text, +links, emphasized text, images, code, and so on. + +## Precedence + +Indicators of block structure always take precedence over indicators +of inline structure. So, for example, the following is a list with +two items, not a list with one item containing a code span: + +```````````````````````````````` example +- `one +- two` +. +
    +
  • `one
    • +
  • two`
    • +
+```````````````````````````````` + + +This means that parsing can proceed in two steps: first, the block +structure of the document can be discerned; second, text lines inside +paragraphs, headings, and other block constructs can be parsed for inline +structure. The second step requires information about link reference +definitions that will be available only at the end of the first +step. Note that the first step requires processing lines in sequence, +but the second can be parallelized, since the inline parsing of +one block element does not affect the inline parsing of any other. + +## Container blocks and leaf blocks + +We can divide blocks into two types: +[container block](@)s, +which can contain other blocks, and [leaf block](@)s, +which cannot. + +# Leaf blocks + +This section describes the different kinds of leaf block that make up a +Markdown document. + +## Thematic breaks + +A line consisting of 0-3 spaces of indentation, followed by a sequence +of three or more matching `-`, `_`, or `*` characters, each followed +optionally by any number of spaces, forms a +[thematic break](@). + +```````````````````````````````` example +*** +--- +___ +. +
+
+
+```````````````````````````````` + + +Wrong characters: + +```````````````````````````````` example ++++ +. +

+++

+```````````````````````````````` + + +```````````````````````````````` example +=== +. +

===

+```````````````````````````````` + + +Not enough characters: + +```````````````````````````````` example +-- +** +__ +. +

-- +** +__

+```````````````````````````````` + + +One to three spaces indent are allowed: + +```````````````````````````````` example + *** + *** + *** +. +
+
+
+```````````````````````````````` + + +Four spaces is too many: + +```````````````````````````````` example + *** +. +
***
+
+```````````````````````````````` + + +```````````````````````````````` example +Foo + *** +. +

Foo +***

+```````````````````````````````` + + +More than three characters may be used: + +```````````````````````````````` example +_____________________________________ +. +
+```````````````````````````````` + + +Spaces are allowed between the characters: + +```````````````````````````````` example + - - - +. +
+```````````````````````````````` + + +```````````````````````````````` example + ** * ** * ** * ** +. +
+```````````````````````````````` + + +```````````````````````````````` example +- - - - +. +
+```````````````````````````````` + + +Spaces are allowed at the end: + +```````````````````````````````` example +- - - - +. +
+```````````````````````````````` + + +However, no other characters may occur in the line: + +```````````````````````````````` example +_ _ _ _ a + +a------ + +---a--- +. +

_ _ _ _ a

+

a------

+

---a---

+```````````````````````````````` + + +It is required that all of the [non-whitespace characters] be the same. +So, this is not a thematic break: + +```````````````````````````````` example + *-* +. +

-

+```````````````````````````````` + + +Thematic breaks do not need blank lines before or after: + +```````````````````````````````` example +- foo +*** +- bar +. +
    +
  • foo
    • +
+
+
    +
  • bar
    • +
+```````````````````````````````` + + +Thematic breaks can interrupt a paragraph: + +```````````````````````````````` example +Foo +*** +bar +. +

Foo

+
+

bar

+```````````````````````````````` + + +If a line of dashes that meets the above conditions for being a +thematic break could also be interpreted as the underline of a [setext +heading], the interpretation as a +[setext heading] takes precedence. Thus, for example, +this is a setext heading, not a paragraph followed by a thematic break: + +```````````````````````````````` example +Foo +--- +bar +. +

Foo

+

bar

+```````````````````````````````` + + +When both a thematic break and a list item are possible +interpretations of a line, the thematic break takes precedence: + +```````````````````````````````` example +* Foo +* * * +* Bar +. +
    +
  • Foo
    • +
+
+
    +
  • Bar
    • +
+```````````````````````````````` + + +If you want a thematic break in a list item, use a different bullet: + +```````````````````````````````` example +- Foo +- * * * +. +
    +
  • Foo
    • +
  • +
    +
    • +
+```````````````````````````````` + + +## ATX headings + +An [ATX heading](@) +consists of a string of characters, parsed as inline content, between an +opening sequence of 1--6 unescaped `#` characters and an optional +closing sequence of any number of unescaped `#` characters. +The opening sequence of `#` characters must be followed by a +[space] or by the end of line. The optional closing sequence of `#`s must be +preceded by a [space] and may be followed by spaces only. The opening +`#` character may be indented 0-3 spaces. The raw contents of the +heading are stripped of leading and trailing spaces before being parsed +as inline content. The heading level is equal to the number of `#` +characters in the opening sequence. + +Simple headings: + +```````````````````````````````` example +# foo +## foo +### foo +#### foo +##### foo +###### foo +. +

foo

+

foo

+

foo

+

foo

+
foo
+
foo
+```````````````````````````````` + + +More than six `#` characters is not a heading: + +```````````````````````````````` example +####### foo +. +

####### foo

+```````````````````````````````` + + +At least one space is required between the `#` characters and the +heading's contents, unless the heading is empty. Note that many +implementations currently do not require the space. However, the +space was required by the +[original ATX implementation](http://www.aaronsw.com/2002/atx/atx.py), +and it helps prevent things like the following from being parsed as +headings: + +```````````````````````````````` example +#5 bolt + +#hashtag +. +

#5 bolt

+

#hashtag

+```````````````````````````````` + + +A tab will not work: + +```````````````````````````````` example +#→foo +. +

#→foo

+```````````````````````````````` + + +This is not a heading, because the first `#` is escaped: + +```````````````````````````````` example +\## foo +. +

## foo

+```````````````````````````````` + + +Contents are parsed as inlines: + +```````````````````````````````` example +# foo *bar* \*baz\* +. +

foo bar *baz*

+```````````````````````````````` + + +Leading and trailing blanks are ignored in parsing inline content: + +```````````````````````````````` example +# foo +. +

foo

+```````````````````````````````` + + +One to three spaces indentation are allowed: + +```````````````````````````````` example + ### foo + ## foo + # foo +. +

foo

+

foo

+

foo

+```````````````````````````````` + + +Four spaces are too much: + +```````````````````````````````` example + # foo +. +
# foo
+
+```````````````````````````````` + + +```````````````````````````````` example +foo + # bar +. +

foo +# bar

+```````````````````````````````` + + +A closing sequence of `#` characters is optional: + +```````````````````````````````` example +## foo ## + ### bar ### +. +

foo

+

bar

+```````````````````````````````` + + +It need not be the same length as the opening sequence: + +```````````````````````````````` example +# foo ################################## +##### foo ## +. +

foo

+
foo
+```````````````````````````````` + + +Spaces are allowed after the closing sequence: + +```````````````````````````````` example +### foo ### +. +

foo

+```````````````````````````````` + + +A sequence of `#` characters with anything but [spaces] following it +is not a closing sequence, but counts as part of the contents of the +heading: + +```````````````````````````````` example +### foo ### b +. +

foo ### b

+```````````````````````````````` + + +The closing sequence must be preceded by a space: + +```````````````````````````````` example +# foo# +. +

foo#

+```````````````````````````````` + + +Backslash-escaped `#` characters do not count as part +of the closing sequence: + +```````````````````````````````` example +### foo \### +## foo #\## +# foo \# +. +

foo ###

+

foo ###

+

foo #

+```````````````````````````````` + + +ATX headings need not be separated from surrounding content by blank +lines, and they can interrupt paragraphs: + +```````````````````````````````` example +**** +## foo +**** +. +
+

foo

+
+```````````````````````````````` + + +```````````````````````````````` example +Foo bar +# baz +Bar foo +. +

Foo bar

+

baz

+

Bar foo

+```````````````````````````````` + + +ATX headings can be empty: + +```````````````````````````````` example +## +# +### ### +. +

+

+

+```````````````````````````````` + + +## Setext headings + +A [setext heading](@) consists of one or more +lines of text, each containing at least one [non-whitespace +character], with no more than 3 spaces indentation, followed by +a [setext heading underline]. The lines of text must be such +that, were they not followed by the setext heading underline, +they would be interpreted as a paragraph: they cannot be +interpretable as a [code fence], [ATX heading][ATX headings], +[block quote][block quotes], [thematic break][thematic breaks], +[list item][list items], or [HTML block][HTML blocks]. + +A [setext heading underline](@) is a sequence of +`=` characters or a sequence of `-` characters, with no more than 3 +spaces indentation and any number of trailing spaces. If a line +containing a single `-` can be interpreted as an +empty [list items], it should be interpreted this way +and not as a [setext heading underline]. + +The heading is a level 1 heading if `=` characters are used in +the [setext heading underline], and a level 2 heading if `-` +characters are used. The contents of the heading are the result +of parsing the preceding lines of text as CommonMark inline +content. + +In general, a setext heading need not be preceded or followed by a +blank line. However, it cannot interrupt a paragraph, so when a +setext heading comes after a paragraph, a blank line is needed between +them. + +Simple examples: + +```````````````````````````````` example +Foo *bar* +========= + +Foo *bar* +--------- +. +

Foo bar

+

Foo bar

+```````````````````````````````` + + +The content of the header may span more than one line: + +```````````````````````````````` example +Foo *bar +baz* +==== +. +

Foo bar +baz

+```````````````````````````````` + + +The underlining can be any length: + +```````````````````````````````` example +Foo +------------------------- + +Foo += +. +

Foo

+

Foo

+```````````````````````````````` + + +The heading content can be indented up to three spaces, and need +not line up with the underlining: + +```````````````````````````````` example + Foo +--- + + Foo +----- + + Foo + === +. +

Foo

+

Foo

+

Foo

+```````````````````````````````` + + +Four spaces indent is too much: + +```````````````````````````````` example + Foo + --- + + Foo +--- +. +
Foo
+---
+
+Foo
+
+
+```````````````````````````````` + + +The setext heading underline can be indented up to three spaces, and +may have trailing spaces: + +```````````````````````````````` example +Foo + ---- +. +

Foo

+```````````````````````````````` + + +Four spaces is too much: + +```````````````````````````````` example +Foo + --- +. +

Foo +---

+```````````````````````````````` + + +The setext heading underline cannot contain internal spaces: + +```````````````````````````````` example +Foo += = + +Foo +--- - +. +

Foo += =

+

Foo

+
+```````````````````````````````` + + +Trailing spaces in the content line do not cause a line break: + +```````````````````````````````` example +Foo +----- +. +

Foo

+```````````````````````````````` + + +Nor does a backslash at the end: + +```````````````````````````````` example +Foo\ +---- +. +

Foo\

+```````````````````````````````` + + +Since indicators of block structure take precedence over +indicators of inline structure, the following are setext headings: + +```````````````````````````````` example +`Foo +---- +` + + +. +

`Foo

+

`

+

<a title="a lot

+

of dashes"/>

+```````````````````````````````` + + +The setext heading underline cannot be a [lazy continuation +line] in a list item or block quote: + +```````````````````````````````` example +> Foo +--- +. +
+

Foo

+
+
+```````````````````````````````` + + +```````````````````````````````` example +> foo +bar +=== +. +
+

foo +bar +===

+
+```````````````````````````````` + + +```````````````````````````````` example +- Foo +--- +. +
    +
  • Foo
    • +
+
+```````````````````````````````` + + +A blank line is needed between a paragraph and a following +setext heading, since otherwise the paragraph becomes part +of the heading's content: + +```````````````````````````````` example +Foo +Bar +--- +. +

Foo +Bar

+```````````````````````````````` + + +But in general a blank line is not required before or after +setext headings: + +```````````````````````````````` example +--- +Foo +--- +Bar +--- +Baz +. +
+

Foo

+

Bar

+

Baz

+```````````````````````````````` + + +Setext headings cannot be empty: + +```````````````````````````````` example + +==== +. +

====

+```````````````````````````````` + + +Setext heading text lines must not be interpretable as block +constructs other than paragraphs. So, the line of dashes +in these examples gets interpreted as a thematic break: + +```````````````````````````````` example +--- +--- +. +
+
+```````````````````````````````` + + +```````````````````````````````` example +- foo +----- +. +
    +
  • foo
    • +
+
+```````````````````````````````` + + +```````````````````````````````` example + foo +--- +. +
foo
+
+
+```````````````````````````````` + + +```````````````````````````````` example +> foo +----- +. +
+

foo

+
+
+```````````````````````````````` + + +If you want a heading with `> foo` as its literal text, you can +use backslash escapes: + +```````````````````````````````` example +\> foo +------ +. +

> foo

+```````````````````````````````` + + +**Compatibility note:** Most existing Markdown implementations +do not allow the text of setext headings to span multiple lines. +But there is no consensus about how to interpret + +``` markdown +Foo +bar +--- +baz +``` + +One can find four different interpretations: + +1. paragraph "Foo", heading "bar", paragraph "baz" +2. paragraph "Foo bar", thematic break, paragraph "baz" +3. paragraph "Foo bar --- baz" +4. heading "Foo bar", paragraph "baz" + +We find interpretation 4 most natural, and interpretation 4 +increases the expressive power of CommonMark, by allowing +multiline headings. Authors who want interpretation 1 can +put a blank line after the first paragraph: + +```````````````````````````````` example +Foo + +bar +--- +baz +. +

Foo

+

bar

+

baz

+```````````````````````````````` + + +Authors who want interpretation 2 can put blank lines around +the thematic break, + +```````````````````````````````` example +Foo +bar + +--- + +baz +. +

Foo +bar

+
+

baz

+```````````````````````````````` + + +or use a thematic break that cannot count as a [setext heading +underline], such as + +```````````````````````````````` example +Foo +bar +* * * +baz +. +

Foo +bar

+
+

baz

+```````````````````````````````` + + +Authors who want interpretation 3 can use backslash escapes: + +```````````````````````````````` example +Foo +bar +\--- +baz +. +

Foo +bar +--- +baz

+```````````````````````````````` + + +## Indented code blocks + +An [indented code block](@) is composed of one or more +[indented chunks] separated by blank lines. +An [indented chunk](@) is a sequence of non-blank lines, +each indented four or more spaces. The contents of the code block are +the literal contents of the lines, including trailing +[line endings], minus four spaces of indentation. +An indented code block has no [info string]. + +An indented code block cannot interrupt a paragraph, so there must be +a blank line between a paragraph and a following indented code block. +(A blank line is not needed, however, between a code block and a following +paragraph.) + +```````````````````````````````` example + a simple + indented code block +. +
a simple
+  indented code block
+
+```````````````````````````````` + + +If there is any ambiguity between an interpretation of indentation +as a code block and as indicating that material belongs to a [list +item][list items], the list item interpretation takes precedence: + +```````````````````````````````` example + - foo + + bar +. +
    +
  • +

    foo

    +

    bar

    +
    • +
+```````````````````````````````` + + +```````````````````````````````` example +1. foo + + - bar +. +
    +
  1. +

    foo

    +
      +
    • bar
      • +
    +
    2. +
+```````````````````````````````` + + + +The contents of a code block are literal text, and do not get parsed +as Markdown: + +```````````````````````````````` example + + *hi* + + - one +. +
<a/>
+*hi*
+
+- one
+
+```````````````````````````````` + + +Here we have three chunks separated by blank lines: + +```````````````````````````````` example + chunk1 + + chunk2 + + + + chunk3 +. +
chunk1
+
+chunk2
+
+
+
+chunk3
+
+```````````````````````````````` + + +Any initial spaces beyond four will be included in the content, even +in interior blank lines: + +```````````````````````````````` example + chunk1 + + chunk2 +. +
chunk1
+  
+  chunk2
+
+```````````````````````````````` + + +An indented code block cannot interrupt a paragraph. (This +allows hanging indents and the like.) + +```````````````````````````````` example +Foo + bar + +. +

Foo +bar

+```````````````````````````````` + + +However, any non-blank line with fewer than four leading spaces ends +the code block immediately. So a paragraph may occur immediately +after indented code: + +```````````````````````````````` example + foo +bar +. +
foo
+
+

bar

+```````````````````````````````` + + +And indented code can occur immediately before and after other kinds of +blocks: + +```````````````````````````````` example +# Heading + foo +Heading +------ + foo +---- +. +

Heading

+
foo
+
+

Heading

+
foo
+
+
+```````````````````````````````` + + +The first line can be indented more than four spaces: + +```````````````````````````````` example + foo + bar +. +
    foo
+bar
+
+```````````````````````````````` + + +Blank lines preceding or following an indented code block +are not included in it: + +```````````````````````````````` example + + + foo + + +. +
foo
+
+```````````````````````````````` + + +Trailing spaces are included in the code block's content: + +```````````````````````````````` example + foo +. +
foo  
+
+```````````````````````````````` + + + +## Fenced code blocks + +A [code fence](@) is a sequence +of at least three consecutive backtick characters (`` ` ``) or +tildes (`~`). (Tildes and backticks cannot be mixed.) +A [fenced code block](@) +begins with a code fence, indented no more than three spaces. + +The line with the opening code fence may optionally contain some text +following the code fence; this is trimmed of leading and trailing +spaces and called the [info string](@). +The [info string] may not contain any backtick +characters. (The reason for this restriction is that otherwise +some inline code would be incorrectly interpreted as the +beginning of a fenced code block.) + +The content of the code block consists of all subsequent lines, until +a closing [code fence] of the same type as the code block +began with (backticks or tildes), and with at least as many backticks +or tildes as the opening code fence. If the leading code fence is +indented N spaces, then up to N spaces of indentation are removed from +each line of the content (if present). (If a content line is not +indented, it is preserved unchanged. If it is indented less than N +spaces, all of the indentation is removed.) + +The closing code fence may be indented up to three spaces, and may be +followed only by spaces, which are ignored. If the end of the +containing block (or document) is reached and no closing code fence +has been found, the code block contains all of the lines after the +opening code fence until the end of the containing block (or +document). (An alternative spec would require backtracking in the +event that a closing code fence is not found. But this makes parsing +much less efficient, and there seems to be no real down side to the +behavior described here.) + +A fenced code block may interrupt a paragraph, and does not require +a blank line either before or after. + +The content of a code fence is treated as literal text, not parsed +as inlines. The first word of the [info string] is typically used to +specify the language of the code sample, and rendered in the `class` +attribute of the `code` tag. However, this spec does not mandate any +particular treatment of the [info string]. + +Here is a simple example with backticks: + +```````````````````````````````` example +``` +< + > +``` +. +
<
+ >
+
+```````````````````````````````` + + +With tildes: + +```````````````````````````````` example +~~~ +< + > +~~~ +. +
<
+ >
+
+```````````````````````````````` + + +The closing code fence must use the same character as the opening +fence: + +```````````````````````````````` example +``` +aaa +~~~ +``` +. +
aaa
+~~~
+
+```````````````````````````````` + + +```````````````````````````````` example +~~~ +aaa +``` +~~~ +. +
aaa
+```
+
+```````````````````````````````` + + +The closing code fence must be at least as long as the opening fence: + +```````````````````````````````` example +```` +aaa +``` +`````` +. +
aaa
+```
+
+```````````````````````````````` + + +```````````````````````````````` example +~~~~ +aaa +~~~ +~~~~ +. +
aaa
+~~~
+
+```````````````````````````````` + + +Unclosed code blocks are closed by the end of the document +(or the enclosing [block quote][block quotes] or [list item][list items]): + +```````````````````````````````` example +``` +. +
+```````````````````````````````` + + +```````````````````````````````` example +````` + +``` +aaa +. +

+```
+aaa
+
+```````````````````````````````` + + +```````````````````````````````` example +> ``` +> aaa + +bbb +. +
+
aaa
+
+
+

bbb

+```````````````````````````````` + + +A code block can have all empty lines as its content: + +```````````````````````````````` example +``` + + +``` +. +

+  
+
+```````````````````````````````` + + +A code block can be empty: + +```````````````````````````````` example +``` +``` +. +
+```````````````````````````````` + + +Fences can be indented. If the opening fence is indented, +content lines will have equivalent opening indentation removed, +if present: + +```````````````````````````````` example + ``` + aaa +aaa +``` +. +
aaa
+aaa
+
+```````````````````````````````` + + +```````````````````````````````` example + ``` +aaa + aaa +aaa + ``` +. +
aaa
+aaa
+aaa
+
+```````````````````````````````` + + +```````````````````````````````` example + ``` + aaa + aaa + aaa + ``` +. +
aaa
+ aaa
+aaa
+
+```````````````````````````````` + + +Four spaces indentation produces an indented code block: + +```````````````````````````````` example + ``` + aaa + ``` +. +
```
+aaa
+```
+
+```````````````````````````````` + + +Closing fences may be indented by 0-3 spaces, and their indentation +need not match that of the opening fence: + +```````````````````````````````` example +``` +aaa + ``` +. +
aaa
+
+```````````````````````````````` + + +```````````````````````````````` example + ``` +aaa + ``` +. +
aaa
+
+```````````````````````````````` + + +This is not a closing fence, because it is indented 4 spaces: + +```````````````````````````````` example +``` +aaa + ``` +. +
aaa
+    ```
+
+```````````````````````````````` + + + +Code fences (opening and closing) cannot contain internal spaces: + +```````````````````````````````` example +``` ``` +aaa +. +

+aaa

+```````````````````````````````` + + +```````````````````````````````` example +~~~~~~ +aaa +~~~ ~~ +. +
aaa
+~~~ ~~
+
+```````````````````````````````` + + +Fenced code blocks can interrupt paragraphs, and can be followed +directly by paragraphs, without a blank line between: + +```````````````````````````````` example +foo +``` +bar +``` +baz +. +

foo

+
bar
+
+

baz

+```````````````````````````````` + + +Other blocks can also occur before and after fenced code blocks +without an intervening blank line: + +```````````````````````````````` example +foo +--- +~~~ +bar +~~~ +# baz +. +

foo

+
bar
+
+

baz

+```````````````````````````````` + + +An [info string] can be provided after the opening code fence. +Opening and closing spaces will be stripped, and the first word, prefixed +with `language-`, is used as the value for the `class` attribute of the +`code` element within the enclosing `pre` element. + +```````````````````````````````` example +```ruby +def foo(x) + return 3 +end +``` +. +
def foo(x)
+  return 3
+end
+
+```````````````````````````````` + + +```````````````````````````````` example +~~~~ ruby startline=3 $%@#$ +def foo(x) + return 3 +end +~~~~~~~ +. +
def foo(x)
+  return 3
+end
+
+```````````````````````````````` + + +```````````````````````````````` example +````; +```` +. +
+```````````````````````````````` + + +[Info strings] for backtick code blocks cannot contain backticks: + +```````````````````````````````` example +``` aa ``` +foo +. +

aa +foo

+```````````````````````````````` + + +Closing code fences cannot have [info strings]: + +```````````````````````````````` example +``` +``` aaa +``` +. +
``` aaa
+
+```````````````````````````````` + + + +## HTML blocks + +An [HTML block](@) is a group of lines that is treated +as raw HTML (and will not be escaped in HTML output). + +There are seven kinds of [HTML block], which can be defined +by their start and end conditions. The block begins with a line that +meets a [start condition](@) (after up to three spaces +optional indentation). It ends with the first subsequent line that +meets a matching [end condition](@), or the last line of +the document, if no line is encountered that meets the +[end condition]. If the first line meets both the [start condition] +and the [end condition], the block will contain just that line. + +1. **Start condition:** line begins with the string ``, or the end of the line.\ +**End condition:** line contains an end tag +``, ``, or `` (case-insensitive; it +need not match the start tag). + +2. **Start condition:** line begins with the string ``. + +3. **Start condition:** line begins with the string ``. + +4. **Start condition:** line begins with the string ``. + +5. **Start condition:** line begins with the string +``. + +6. **Start condition:** line begins the string `<` or ``, or +the string `/>`.\ +**End condition:** line is followed by a [blank line]. + +7. **Start condition:** line begins with a complete [open tag] +or [closing tag] (with any [tag name] other than `script`, +`style`, or `pre`) followed only by [whitespace] +or the end of the line.\ +**End condition:** line is followed by a [blank line]. + +All types of [HTML blocks] except type 7 may interrupt +a paragraph. Blocks of type 7 may not interrupt a paragraph. +(This restriction is intended to prevent unwanted interpretation +of long tags inside a wrapped paragraph as starting HTML blocks.) + +Some simple examples follow. Here are some basic HTML blocks +of type 6: + +```````````````````````````````` example + + + + +
+ hi +
+ +okay. +. + + + + +
+ hi +
+

okay.

+```````````````````````````````` + + +```````````````````````````````` example + +*foo* +```````````````````````````````` + + +Here we have two HTML blocks with a Markdown paragraph between them: + +```````````````````````````````` example +
+ +*Markdown* + +
+. +
+

Markdown

+
+```````````````````````````````` + + +The tag on the first line can be partial, as long +as it is split where there would be whitespace: + +```````````````````````````````` example +
+
+. +
+
+```````````````````````````````` + + +```````````````````````````````` example +
+
+. +
+
+```````````````````````````````` + + +An open tag need not be closed: +```````````````````````````````` example +
+*foo* + +*bar* +. +
+*foo* +

bar

+```````````````````````````````` + + + +A partial tag need not even be completed (garbage +in, garbage out): + +```````````````````````````````` example + +. + +```````````````````````````````` + + +```````````````````````````````` example +
+foo +
+. +
+foo +
+```````````````````````````````` + + +Everything until the next blank line or end of document +gets included in the HTML block. So, in the following +example, what looks like a Markdown code block +is actually part of the HTML block, which continues until a blank +line or the end of the document is reached: + +```````````````````````````````` example +
+``` c +int x = 33; +``` +. +
+``` c +int x = 33; +``` +```````````````````````````````` + + +To start an [HTML block] with a tag that is *not* in the +list of block-level tags in (6), you must put the tag by +itself on the first line (and it must be complete): + +```````````````````````````````` example + +*bar* + +. + +*bar* + +```````````````````````````````` + + +In type 7 blocks, the [tag name] can be anything: + +```````````````````````````````` example + +*bar* + +. + +*bar* + +```````````````````````````````` + + +```````````````````````````````` example + +*bar* + +. + +*bar* + +```````````````````````````````` + + +```````````````````````````````` example + +*bar* +. + +*bar* +```````````````````````````````` + + +These rules are designed to allow us to work with tags that +can function as either block-level or inline-level tags. +The `` tag is a nice example. We can surround content with +`` tags in three different ways. In this case, we get a raw +HTML block, because the `` tag is on a line by itself: + +```````````````````````````````` example + +*foo* + +. + +*foo* + +```````````````````````````````` + + +In this case, we get a raw HTML block that just includes +the `` tag (because it ends with the following blank +line). So the contents get interpreted as CommonMark: + +```````````````````````````````` example + + +*foo* + + +. + +

foo

+ +```````````````````````````````` + + +Finally, in this case, the `` tags are interpreted +as [raw HTML] *inside* the CommonMark paragraph. (Because +the tag is not on a line by itself, we get inline HTML +rather than an [HTML block].) + +```````````````````````````````` example +*foo* +. +

foo

+```````````````````````````````` + + +HTML tags designed to contain literal content +(`script`, `style`, `pre`), comments, processing instructions, +and declarations are treated somewhat differently. +Instead of ending at the first blank line, these blocks +end at the first line containing a corresponding end tag. +As a result, these blocks can contain blank lines: + +A pre tag (type 1): + +```````````````````````````````` example +

+import Text.HTML.TagSoup
+
+main :: IO ()
+main = print $ parseTags tags
+
+. +

+import Text.HTML.TagSoup
+
+main :: IO ()
+main = print $ parseTags tags
+
+```````````````````````````````` + + +A script tag (type 1): + +```````````````````````````````` example + +. + +```````````````````````````````` + + +A style tag (type 1): + +```````````````````````````````` example + +. + +```````````````````````````````` + + +If there is no matching end tag, the block will end at the +end of the document (or the enclosing [block quote][block quotes] +or [list item][list items]): + +```````````````````````````````` example + +*foo* +. + +

foo

+```````````````````````````````` + + +```````````````````````````````` example +*bar* +*baz* +. +*bar* +

baz

+```````````````````````````````` + + +Note that anything on the last line after the +end tag will be included in the [HTML block]: + +```````````````````````````````` example +1. *bar* +. +1. *bar* +```````````````````````````````` + + +A comment (type 2): + +```````````````````````````````` example + +. + +```````````````````````````````` + + + +A processing instruction (type 3): + +```````````````````````````````` example +'; + +?> +. +'; + +?> +```````````````````````````````` + + +A declaration (type 4): + +```````````````````````````````` example + +. + +```````````````````````````````` + + +CDATA (type 5): + +```````````````````````````````` example + +. + +```````````````````````````````` + + +The opening tag can be indented 1-3 spaces, but not 4: + +```````````````````````````````` example + + + +. + +
<!-- foo -->
+
+```````````````````````````````` + + +```````````````````````````````` example +
+ +
+. +
+
<div>
+
+```````````````````````````````` + + +An HTML block of types 1--6 can interrupt a paragraph, and need not be +preceded by a blank line. + +```````````````````````````````` example +Foo +
+bar +
+. +

Foo

+
+bar +
+```````````````````````````````` + + +However, a following blank line is needed, except at the end of +a document, and except for blocks of types 1--5, above: + +```````````````````````````````` example +
+bar +
+*foo* +. +
+bar +
+*foo* +```````````````````````````````` + + +HTML blocks of type 7 cannot interrupt a paragraph: + +```````````````````````````````` example +Foo + +baz +. +

Foo + +baz

+```````````````````````````````` + + +This rule differs from John Gruber's original Markdown syntax +specification, which says: + +> The only restrictions are that block-level HTML elements — +> e.g. `
`, ``, `
`, `
`, etc. — must be separated from
+> surrounding content by blank lines, and the start and end tags of the
+> block should not be indented with tabs or spaces.
+
+In some ways Gruber's rule is more restrictive than the one given
+here:
+
+- It requires that an HTML block be preceded by a blank line.
+- It does not allow the start tag to be indented.
+- It requires a matching end tag, which it also does not allow to
+  be indented.
+
+Most Markdown implementations (including some of Gruber's own) do not
+respect all of these restrictions.
+
+There is one respect, however, in which Gruber's rule is more liberal
+than the one given here, since it allows blank lines to occur inside
+an HTML block.  There are two reasons for disallowing them here.
+First, it removes the need to parse balanced tags, which is
+expensive and can require backtracking from the end of the document
+if no matching end tag is found. Second, it provides a very simple
+and flexible way of including Markdown content inside HTML tags:
+simply separate the Markdown from the HTML using blank lines:
+
+Compare:
+
+```````````````````````````````` example
+

+
+*Emphasized* text.
+
+

+.
+

+
Emphasized text.

+

+````````````````````````````````
+
+
+```````````````````````````````` example
+

+*Emphasized* text.
+

+.
+

+*Emphasized* text.
+

+````````````````````````````````
+
+
+Some Markdown implementations have adopted a convention of
+interpreting content inside tags as text if the open tag has
+the attribute `markdown=1`.  The rule given above seems a simpler and
+more elegant way of achieving the same expressive power, which is also
+much simpler to parse.
+
+The main potential drawback is that one can no longer paste HTML
+blocks into Markdown documents with 100% reliability.  However,
+*in most cases* this will work fine, because the blank lines in
+HTML are usually followed by HTML block tags.  For example:
+
+```````````````````````````````` example
+
+ + + + + + + +
+Hi +
+. + + + + +
+Hi +
+```````````````````````````````` + + +There are problems, however, if the inner tags are indented +*and* separated by spaces, as then they will be interpreted as +an indented code block: + +```````````````````````````````` example + + + + + + + + +
+ Hi +
+. + + +
<td>
+  Hi
+</td>
+
+ +
+```````````````````````````````` + + +Fortunately, blank lines are usually not necessary and can be +deleted. The exception is inside `
` tags, but as described
+above, raw HTML blocks starting with `
` *can* contain blank
+lines.
+
+## Link reference definitions
+
+A [link reference definition](@)
+consists of a [link label], indented up to three spaces, followed
+by a colon (`:`), optional [whitespace] (including up to one
+[line ending]), a [link destination],
+optional [whitespace] (including up to one
+[line ending]), and an optional [link
+title], which if it is present must be separated
+from the [link destination] by [whitespace].
+No further [non-whitespace characters] may occur on the line.
+
+A [link reference definition]
+does not correspond to a structural element of a document.  Instead, it
+defines a label which can be used in [reference links]
+and reference-style [images] elsewhere in the document.  [Link
+reference definitions] can come either before or after the links that use
+them.
+
+```````````````````````````````` example
+[foo]: /url "title"
+
+[foo]
+.
+
foo

+````````````````````````````````
+
+
+```````````````````````````````` example
+   [foo]: 
+      /url  
+           'the title'  
+
+[foo]
+.
+
foo

+````````````````````````````````
+
+
+```````````````````````````````` example
+[Foo*bar\]]:my_(url) 'title (with parens)'
+
+[Foo*bar\]]
+.
+
Foo*bar]

+````````````````````````````````
+
+
+```````````````````````````````` example
+[Foo bar]:
+
+'title'
+
+[Foo bar]
+.
+
Foo bar

+````````````````````````````````
+
+
+The title may extend over multiple lines:
+
+```````````````````````````````` example
+[foo]: /url '
+title
+line1
+line2
+'
+
+[foo]
+.
+
foo

+````````````````````````````````
+
+
+However, it may not contain a [blank line]:
+
+```````````````````````````````` example
+[foo]: /url 'title
+
+with blank line'
+
+[foo]
+.
+
[foo]: /url 'title

+
with blank line'

+
[foo]

+````````````````````````````````
+
+
+The title may be omitted:
+
+```````````````````````````````` example
+[foo]:
+/url
+
+[foo]
+.
+
foo

+````````````````````````````````
+
+
+The link destination may not be omitted:
+
+```````````````````````````````` example
+[foo]:
+
+[foo]
+.
+
[foo]:

+
[foo]

+````````````````````````````````
+
+
+Both title and destination can contain backslash escapes
+and literal backslashes:
+
+```````````````````````````````` example
+[foo]: /url\bar\*baz "foo\"bar\baz"
+
+[foo]
+.
+
foo

+````````````````````````````````
+
+
+A link can come before its corresponding definition:
+
+```````````````````````````````` example
+[foo]
+
+[foo]: url
+.
+
foo

+````````````````````````````````
+
+
+If there are several matching definitions, the first one takes
+precedence:
+
+```````````````````````````````` example
+[foo]
+
+[foo]: first
+[foo]: second
+.
+
foo

+````````````````````````````````
+
+
+As noted in the section on [Links], matching of labels is
+case-insensitive (see [matches]).
+
+```````````````````````````````` example
+[FOO]: /url
+
+[Foo]
+.
+
Foo

+````````````````````````````````
+
+
+```````````````````````````````` example
+[ΑΓΩ]: /φου
+
+[αγω]
+.
+
αγω

+````````````````````````````````
+
+
+Here is a link reference definition with no corresponding link.
+It contributes nothing to the document.
+
+```````````````````````````````` example
+[foo]: /url
+.
+````````````````````````````````
+
+
+Here is another one:
+
+```````````````````````````````` example
+[
+foo
+]: /url
+bar
+.
+
bar

+````````````````````````````````
+
+
+This is not a link reference definition, because there are
+[non-whitespace characters] after the title:
+
+```````````````````````````````` example
+[foo]: /url "title" ok
+.
+
[foo]: /url "title" ok

+````````````````````````````````
+
+
+This is a link reference definition, but it has no title:
+
+```````````````````````````````` example
+[foo]: /url
+"title" ok
+.
+
"title" ok

+````````````````````````````````
+
+
+This is not a link reference definition, because it is indented
+four spaces:
+
+```````````````````````````````` example
+    [foo]: /url "title"
+
+[foo]
+.
+
[foo]: /url "title"
+

+
[foo]

+````````````````````````````````
+
+
+This is not a link reference definition, because it occurs inside
+a code block:
+
+```````````````````````````````` example
+```
+[foo]: /url
+```
+
+[foo]
+.
+
[foo]: /url
+

+
[foo]

+````````````````````````````````
+
+
+A [link reference definition] cannot interrupt a paragraph.
+
+```````````````````````````````` example
+Foo
+[bar]: /baz
+
+[bar]
+.
+
Foo
+[bar]: /baz

+
[bar]

+````````````````````````````````
+
+
+However, it can directly follow other block elements, such as headings
+and thematic breaks, and it need not be followed by a blank line.
+
+```````````````````````````````` example
+# [Foo]
+[foo]: /url
+> bar
+.
+
Foo

+

+
bar

+

+````````````````````````````````
+
+
+Several [link reference definitions]
+can occur one after another, without intervening blank lines.
+
+```````````````````````````````` example
+[foo]: /foo-url "foo"
+[bar]: /bar-url
+  "bar"
+[baz]: /baz-url
+
+[foo],
+[bar],
+[baz]
+.
+
foo,
+bar,
+baz

+````````````````````````````````
+
+
+[Link reference definitions] can occur
+inside block containers, like lists and block quotations.  They
+affect the entire document, not just the container in which they
+are defined:
+
+```````````````````````````````` example
+[foo]
+
+> [foo]: /url
+.
+
foo

+

+

+````````````````````````````````
+
+
+
+## Paragraphs
+
+A sequence of non-blank lines that cannot be interpreted as other
+kinds of blocks forms a [paragraph](@).
+The contents of the paragraph are the result of parsing the
+paragraph's raw content as inlines.  The paragraph's raw content
+is formed by concatenating the lines and removing initial and final
+[whitespace].
+
+A simple example with two paragraphs:
+
+```````````````````````````````` example
+aaa
+
+bbb
+.
+
aaa

+
bbb

+````````````````````````````````
+
+
+Paragraphs can contain multiple lines, but no blank lines:
+
+```````````````````````````````` example
+aaa
+bbb
+
+ccc
+ddd
+.
+
aaa
+bbb

+
ccc
+ddd

+````````````````````````````````
+
+
+Multiple blank lines between paragraph have no effect:
+
+```````````````````````````````` example
+aaa
+
+
+bbb
+.
+
aaa

+
bbb

+````````````````````````````````
+
+
+Leading spaces are skipped:
+
+```````````````````````````````` example
+  aaa
+ bbb
+.
+
aaa
+bbb

+````````````````````````````````
+
+
+Lines after the first may be indented any amount, since indented
+code blocks cannot interrupt paragraphs.
+
+```````````````````````````````` example
+aaa
+             bbb
+                                       ccc
+.
+
aaa
+bbb
+ccc

+````````````````````````````````
+
+
+However, the first line may be indented at most three spaces,
+or an indented code block will be triggered:
+
+```````````````````````````````` example
+   aaa
+bbb
+.
+
aaa
+bbb

+````````````````````````````````
+
+
+```````````````````````````````` example
+    aaa
+bbb
+.
+
aaa
+

+
bbb

+````````````````````````````````
+
+
+Final spaces are stripped before inline parsing, so a paragraph
+that ends with two or more spaces will not end with a [hard line
+break]:
+
+```````````````````````````````` example
+aaa     
+bbb     
+.
+
aaa

+bbb

+````````````````````````````````
+
+
+## Blank lines
+
+[Blank lines] between block-level elements are ignored,
+except for the role they play in determining whether a [list]
+is [tight] or [loose].
+
+Blank lines at the beginning and end of the document are also ignored.
+
+```````````````````````````````` example
+  
+
+aaa
+  
+
+# aaa
+
+  
+.
+
aaa

+
aaa

+````````````````````````````````
+
+
+
+# Container blocks
+
+A [container block] is a block that has other
+blocks as its contents.  There are two basic kinds of container blocks:
+[block quotes] and [list items].
+[Lists] are meta-containers for [list items].
+
+We define the syntax for container blocks recursively.  The general
+form of the definition is:
+
+> If X is a sequence of blocks, then the result of
+> transforming X in such-and-such a way is a container of type Y
+> with these blocks as its content.
+
+So, we explain what counts as a block quote or list item by explaining
+how these can be *generated* from their contents. This should suffice
+to define the syntax, although it does not give a recipe for *parsing*
+these constructions.  (A recipe is provided below in the section entitled
+[A parsing strategy](#appendix-a-parsing-strategy).)
+
+## Block quotes
+
+A [block quote marker](@)
+consists of 0-3 spaces of initial indent, plus (a) the character `>` together
+with a following space, or (b) a single character `>` not followed by a space.
+
+The following rules define [block quotes]:
+
+1.  **Basic case.**  If a string of lines *Ls* constitute a sequence
+    of blocks *Bs*, then the result of prepending a [block quote
+    marker] to the beginning of each line in *Ls*
+    is a [block quote](#block-quotes) containing *Bs*.
+
+2.  **Laziness.**  If a string of lines *Ls* constitute a [block
+    quote](#block-quotes) with contents *Bs*, then the result of deleting
+    the initial [block quote marker] from one or
+    more lines in which the next [non-whitespace character] after the [block
+    quote marker] is [paragraph continuation
+    text] is a block quote with *Bs* as its content.
+    [Paragraph continuation text](@) is text
+    that will be parsed as part of the content of a paragraph, but does
+    not occur at the beginning of the paragraph.
+
+3.  **Consecutiveness.**  A document cannot contain two [block
+    quotes] in a row unless there is a [blank line] between them.
+
+Nothing else counts as a [block quote](#block-quotes).
+
+Here is a simple example:
+
+```````````````````````````````` example
+> # Foo
+> bar
+> baz
+.
+

+
Foo

+
bar
+baz

+

+````````````````````````````````
+
+
+The spaces after the `>` characters can be omitted:
+
+```````````````````````````````` example
+># Foo
+>bar
+> baz
+.
+

+
Foo

+
bar
+baz

+

+````````````````````````````````
+
+
+The `>` characters can be indented 1-3 spaces:
+
+```````````````````````````````` example
+   > # Foo
+   > bar
+ > baz
+.
+

+
Foo

+
bar
+baz

+

+````````````````````````````````
+
+
+Four spaces gives us a code block:
+
+```````````````````````````````` example
+    > # Foo
+    > bar
+    > baz
+.
+
> # Foo
+> bar
+> baz
+

+````````````````````````````````
+
+
+The Laziness clause allows us to omit the `>` before a
+paragraph continuation line:
+
+```````````````````````````````` example
+> # Foo
+> bar
+baz
+.
+

+
Foo

+
bar
+baz

+

+````````````````````````````````
+
+
+A block quote can contain some lazy and some non-lazy
+continuation lines:
+
+```````````````````````````````` example
+> bar
+baz
+> foo
+.
+

+
bar
+baz
+foo

+

+````````````````````````````````
+
+
+Laziness only applies to lines that would have been continuations of
+paragraphs had they been prepended with [block quote markers].
+For example, the `> ` cannot be omitted in the second line of
+
+``` markdown
+> foo
+> ---
+```
+
+without changing the meaning:
+
+```````````````````````````````` example
+> foo
+---
+.
+

+
foo

+

+

+````````````````````````````````
+
+
+Similarly, if we omit the `> ` in the second line of
+
+``` markdown
+> - foo
+> - bar
+```
+
+then the block quote ends after the first line:
+
+```````````````````````````````` example
+> - foo
+- bar
+.
+

+
    
+
  • foo
    • 
+

+

+
    
+
  • bar
    • 
+

+````````````````````````````````
+
+
+For the same reason, we can't omit the `> ` in front of
+subsequent lines of an indented or fenced code block:
+
+```````````````````````````````` example
+>     foo
+    bar
+.
+

+
foo
+

+

+
bar
+

+````````````````````````````````
+
+
+```````````````````````````````` example
+> ```
+foo
+```
+.
+

+

+

+
foo

+

+````````````````````````````````
+
+
+Note that in the following case, we have a paragraph
+continuation line:
+
+```````````````````````````````` example
+> foo
+    - bar
+.
+

+
foo
+- bar

+

+````````````````````````````````
+
+
+To see why, note that in
+
+```markdown
+> foo
+>     - bar
+```
+
+the `- bar` is indented too far to start a list, and can't
+be an indented code block because indented code blocks cannot
+interrupt paragraphs, so it is a [paragraph continuation line].
+
+A block quote can be empty:
+
+```````````````````````````````` example
+>
+.
+

+

+````````````````````````````````
+
+
+```````````````````````````````` example
+>
+>  
+> 
+.
+

+

+````````````````````````````````
+
+
+A block quote can have initial or final blank lines:
+
+```````````````````````````````` example
+>
+> foo
+>  
+.
+

+
foo

+

+````````````````````````````````
+
+
+A blank line always separates block quotes:
+
+```````````````````````````````` example
+> foo
+
+> bar
+.
+

+
foo

+

+

+
bar

+

+````````````````````````````````
+
+
+(Most current Markdown implementations, including John Gruber's
+original `Markdown.pl`, will parse this example as a single block quote
+with two paragraphs.  But it seems better to allow the author to decide
+whether two block quotes or one are wanted.)
+
+Consecutiveness means that if we put these block quotes together,
+we get a single block quote:
+
+```````````````````````````````` example
+> foo
+> bar
+.
+

+
foo
+bar

+

+````````````````````````````````
+
+
+To get a block quote with two paragraphs, use:
+
+```````````````````````````````` example
+> foo
+>
+> bar
+.
+

+
foo

+
bar

+

+````````````````````````````````
+
+
+Block quotes can interrupt paragraphs:
+
+```````````````````````````````` example
+foo
+> bar
+.
+
foo

+

+
bar

+

+````````````````````````````````
+
+
+In general, blank lines are not needed before or after block
+quotes:
+
+```````````````````````````````` example
+> aaa
+***
+> bbb
+.
+

+
aaa

+

+

+

+
bbb

+

+````````````````````````````````
+
+
+However, because of laziness, a blank line is needed between
+a block quote and a following paragraph:
+
+```````````````````````````````` example
+> bar
+baz
+.
+

+
bar
+baz

+

+````````````````````````````````
+
+
+```````````````````````````````` example
+> bar
+
+baz
+.
+

+
bar

+

+
baz

+````````````````````````````````
+
+
+```````````````````````````````` example
+> bar
+>
+baz
+.
+

+
bar

+

+
baz

+````````````````````````````````
+
+
+It is a consequence of the Laziness rule that any number
+of initial `>`s may be omitted on a continuation line of a
+nested block quote:
+
+```````````````````````````````` example
+> > > foo
+bar
+.
+

+

+

+
foo
+bar

+

+

+

+````````````````````````````````
+
+
+```````````````````````````````` example
+>>> foo
+> bar
+>>baz
+.
+

+

+

+
foo
+bar
+baz

+

+

+

+````````````````````````````````
+
+
+When including an indented code block in a block quote,
+remember that the [block quote marker] includes
+both the `>` and a following space.  So *five spaces* are needed after
+the `>`:
+
+```````````````````````````````` example
+>     code
+
+>    not code
+.
+

+
code
+

+

+

+
not code

+

+````````````````````````````````
+
+
+
+## List items
+
+A [list marker](@) is a
+[bullet list marker] or an [ordered list marker].
+
+A [bullet list marker](@)
+is a `-`, `+`, or `*` character.
+
+An [ordered list marker](@)
+is a sequence of 1--9 arabic digits (`0-9`), followed by either a
+`.` character or a `)` character.  (The reason for the length
+limit is that with 10 digits we start seeing integer overflows
+in some browsers.)
+
+The following rules define [list items]:
+
+1.  **Basic case.**  If a sequence of lines *Ls* constitute a sequence of
+    blocks *Bs* starting with a [non-whitespace character] and not separated
+    from each other by more than one blank line, and *M* is a list
+    marker of width *W* followed by 0 < *N* < 5 spaces, then the result
+    of prepending *M* and the following spaces to the first line of
+    *Ls*, and indenting subsequent lines of *Ls* by *W + N* spaces, is a
+    list item with *Bs* as its contents.  The type of the list item
+    (bullet or ordered) is determined by the type of its list marker.
+    If the list item is ordered, then it is also assigned a start
+    number, based on the ordered list marker.
+
+For example, let *Ls* be the lines
+
+```````````````````````````````` example
+A paragraph
+with two lines.
+
+    indented code
+
+> A block quote.
+.
+
A paragraph
+with two lines.

+
indented code
+

+

+
A block quote.

+

+````````````````````````````````
+
+
+And let *M* be the marker `1.`, and *N* = 2.  Then rule #1 says
+that the following is an ordered list item with start number 1,
+and the same contents as *Ls*:
+
+```````````````````````````````` example
+1.  A paragraph
+    with two lines.
+
+        indented code
+
+    > A block quote.
+.
+
    
+
  1. 
+
    A paragraph
+with two lines.
    
+
    indented code
+
    
+
    
+
    A block quote.
    
+
    
+
    2. 
+

+````````````````````````````````
+
+
+The most important thing to notice is that the position of
+the text after the list marker determines how much indentation
+is needed in subsequent blocks in the list item.  If the list
+marker takes up two spaces, and there are three spaces between
+the list marker and the next [non-whitespace character], then blocks
+must be indented five spaces in order to fall under the list
+item.
+
+Here are some examples showing how far content must be indented to be
+put under the list item:
+
+```````````````````````````````` example
+- one
+
+ two
+.
+
    
+
  • one
    • 
+

+
two

+````````````````````````````````
+
+
+```````````````````````````````` example
+- one
+
+  two
+.
+
    
+
  • 
+
    one
    
+
    two
    
+
    • 
+

+````````````````````````````````
+
+
+```````````````````````````````` example
+ -    one
+
+     two
+.
+
    
+
  • one
    • 
+

+
 two
+

+````````````````````````````````
+
+
+```````````````````````````````` example
+ -    one
+
+      two
+.
+
    
+
  • 
+
    one
    
+
    two
    
+
    • 
+

+````````````````````````````````
+
+
+It is tempting to think of this in terms of columns:  the continuation
+blocks must be indented at least to the column of the first
+[non-whitespace character] after the list marker. However, that is not quite right.
+The spaces after the list marker determine how much relative indentation
+is needed.  Which column this indentation reaches will depend on
+how the list item is embedded in other constructions, as shown by
+this example:
+
+```````````````````````````````` example
+   > > 1.  one
+>>
+>>     two
+.
+

+

+
    
+
  1. 
+
    one
    
+
    two
    
+
    2. 
+

+

+

+````````````````````````````````
+
+
+Here `two` occurs in the same column as the list marker `1.`,
+but is actually contained in the list item, because there is
+sufficient indentation after the last containing blockquote marker.
+
+The converse is also possible.  In the following example, the word `two`
+occurs far to the right of the initial text of the list item, `one`, but
+it is not considered part of the list item, because it is not indented
+far enough past the blockquote marker:
+
+```````````````````````````````` example
+>>- one
+>>
+  >  > two
+.
+

+

+
    
+
  • one
    • 
+

+
two

+

+

+````````````````````````````````
+
+
+Note that at least one space is needed between the list marker and
+any following content, so these are not list items:
+
+```````````````````````````````` example
+-one
+
+2.two
+.
+
-one

+
2.two

+````````````````````````````````
+
+
+A list item may not contain blocks that are separated by more than
+one blank line.  Thus, two blank lines will end a list, unless the
+two blanks are contained in a [fenced code block].
+
+```````````````````````````````` example
+- foo
+
+  bar
+
+- foo
+
+
+  bar
+
+- ```
+  foo
+
+
+  bar
+  ```
+
+- baz
+
+  + ```
+    foo
+
+
+    bar
+    ```
+.
+
    
+
  • 
+
    foo
    
+
    bar
    
+
    • 
+
  • 
+
    foo
    
+
    • 
+

+
bar

+
    
+
  • 
+
    foo
+
+
+bar
+
    
+
    • 
+
  • 
+
    baz
    
+
      
+
    • 
+
      foo
+
+
+bar
+
      
+
      • 
+
    
+
    • 
+

+````````````````````````````````
+
+
+A list item may contain any kind of block:
+
+```````````````````````````````` example
+1.  foo
+
+    ```
+    bar
+    ```
+
+    baz
+
+    > bam
+.
+
    
+
  1. 
+
    foo
    
+
    bar
+
    
+
    baz
    
+
    
+
    bam
    
+
    
+
    2. 
+

+````````````````````````````````
+
+
+A list item that contains an indented code block will preserve
+empty lines within the code block verbatim, unless there are two
+or more empty lines in a row (since as described above, two
+blank lines end the list):
+
+```````````````````````````````` example
+- Foo
+
+      bar
+
+      baz
+.
+
    
+
  • 
+
    Foo
    
+
    bar
+
+baz
+
    
+
    • 
+

+````````````````````````````````
+
+
+```````````````````````````````` example
+- Foo
+
+      bar
+
+
+      baz
+.
+
    
+
  • 
+
    Foo
    
+
    bar
+
    
+
    • 
+

+
  baz
+

+````````````````````````````````
+
+
+Note that ordered list start numbers must be nine digits or less:
+
+```````````````````````````````` example
+123456789. ok
+.
+
    
+
  1. ok
    2. 
+

+````````````````````````````````
+
+
+```````````````````````````````` example
+1234567890. not ok
+.
+
1234567890. not ok

+````````````````````````````````
+
+
+A start number may begin with 0s:
+
+```````````````````````````````` example
+0. ok
+.
+
    
+
  1. ok
    2. 
+

+````````````````````````````````
+
+
+```````````````````````````````` example
+003. ok
+.
+
    
+
  1. ok
    2. 
+

+````````````````````````````````
+
+
+A start number may not be negative:
+
+```````````````````````````````` example
+-1. not ok
+.
+
-1. not ok

+````````````````````````````````
+
+
+
+2.  **Item starting with indented code.**  If a sequence of lines *Ls*
+    constitute a sequence of blocks *Bs* starting with an indented code
+    block and not separated from each other by more than one blank line,
+    and *M* is a list marker of width *W* followed by
+    one space, then the result of prepending *M* and the following
+    space to the first line of *Ls*, and indenting subsequent lines of
+    *Ls* by *W + 1* spaces, is a list item with *Bs* as its contents.
+    If a line is empty, then it need not be indented.  The type of the
+    list item (bullet or ordered) is determined by the type of its list
+    marker.  If the list item is ordered, then it is also assigned a
+    start number, based on the ordered list marker.
+
+An indented code block will have to be indented four spaces beyond
+the edge of the region where text will be included in the list item.
+In the following case that is 6 spaces:
+
+```````````````````````````````` example
+- foo
+
+      bar
+.
+
    
+
  • 
+
    foo
    
+
    bar
+
    
+
    • 
+

+````````````````````````````````
+
+
+And in this case it is 11 spaces:
+
+```````````````````````````````` example
+  10.  foo
+
+           bar
+.
+
    
+
  1. 
+
    foo
    
+
    bar
+
    
+
    2. 
+

+````````````````````````````````
+
+
+If the *first* block in the list item is an indented code block,
+then by rule #2, the contents must be indented *one* space after the
+list marker:
+
+```````````````````````````````` example
+    indented code
+
+paragraph
+
+    more code
+.
+
indented code
+

+
paragraph

+
more code
+

+````````````````````````````````
+
+
+```````````````````````````````` example
+1.     indented code
+
+   paragraph
+
+       more code
+.
+
    
+
  1. 
+
    indented code
+
    
+
    paragraph
    
+
    more code
+
    
+
    2. 
+

+````````````````````````````````
+
+
+Note that an additional space indent is interpreted as space
+inside the code block:
+
+```````````````````````````````` example
+1.      indented code
+
+   paragraph
+
+       more code
+.
+
    
+
  1. 
+
     indented code
+
    
+
    paragraph
    
+
    more code
+
    
+
    2. 
+

+````````````````````````````````
+
+
+Note that rules #1 and #2 only apply to two cases:  (a) cases
+in which the lines to be included in a list item begin with a
+[non-whitespace character], and (b) cases in which
+they begin with an indented code
+block.  In a case like the following, where the first block begins with
+a three-space indent, the rules do not allow us to form a list item by
+indenting the whole thing and prepending a list marker:
+
+```````````````````````````````` example
+   foo
+
+bar
+.
+
foo

+
bar

+````````````````````````````````
+
+
+```````````````````````````````` example
+-    foo
+
+  bar
+.
+
    
+
  • foo
    • 
+

+
bar

+````````````````````````````````
+
+
+This is not a significant restriction, because when a block begins
+with 1-3 spaces indent, the indentation can always be removed without
+a change in interpretation, allowing rule #1 to be applied.  So, in
+the above case:
+
+```````````````````````````````` example
+-  foo
+
+   bar
+.
+
    
+
  • 
+
    foo
    
+
    bar
    
+
    • 
+

+````````````````````````````````
+
+
+3.  **Item starting with a blank line.**  If a sequence of lines *Ls*
+    starting with a single [blank line] constitute a (possibly empty)
+    sequence of blocks *Bs*, not separated from each other by more than
+    one blank line, and *M* is a list marker of width *W*,
+    then the result of prepending *M* to the first line of *Ls*, and
+    indenting subsequent lines of *Ls* by *W + 1* spaces, is a list
+    item with *Bs* as its contents.
+    If a line is empty, then it need not be indented.  The type of the
+    list item (bullet or ordered) is determined by the type of its list
+    marker.  If the list item is ordered, then it is also assigned a
+    start number, based on the ordered list marker.
+
+Here are some list items that start with a blank line but are not empty:
+
+```````````````````````````````` example
+-
+  foo
+-
+  ```
+  bar
+  ```
+-
+      baz
+.
+
    
+
  • foo
    • 
+
  • 
+
    bar
+
    
+
    • 
+
  • 
+
    baz
+
    
+
    • 
+

+````````````````````````````````
+
+When the list item starts with a blank line, the number of spaces
+following the list marker doesn't change the required indentation:
+
+```````````````````````````````` example
+-   
+  foo
+.
+
    
+
  • foo
    • 
+

+````````````````````````````````
+
+
+A list item can begin with at most one blank line.
+In the following example, `foo` is not part of the list
+item:
+
+```````````````````````````````` example
+-
+
+  foo
+.
+
    
+
  • 
+

+
foo

+````````````````````````````````
+
+
+Here is an empty bullet list item:
+
+```````````````````````````````` example
+- foo
+-
+- bar
+.
+
    
+
  • foo
    • 
+
  • 
+
  • bar
    • 
+

+````````````````````````````````
+
+
+It does not matter whether there are spaces following the [list marker]:
+
+```````````````````````````````` example
+- foo
+-   
+- bar
+.
+
    
+
  • foo
    • 
+
  • 
+
  • bar
    • 
+

+````````````````````````````````
+
+
+Here is an empty ordered list item:
+
+```````````````````````````````` example
+1. foo
+2.
+3. bar
+.
+
    
+
  1. foo
    2. 
+
  2. 
+
  3. bar
    4. 
+

+````````````````````````````````
+
+
+A list may start or end with an empty list item:
+
+```````````````````````````````` example
+*
+.
+
    
+
  • 
+

+````````````````````````````````
+
+
+
+4.  **Indentation.**  If a sequence of lines *Ls* constitutes a list item
+    according to rule #1, #2, or #3, then the result of indenting each line
+    of *Ls* by 1-3 spaces (the same for each line) also constitutes a
+    list item with the same contents and attributes.  If a line is
+    empty, then it need not be indented.
+
+Indented one space:
+
+```````````````````````````````` example
+ 1.  A paragraph
+     with two lines.
+
+         indented code
+
+     > A block quote.
+.
+
    
+
  1. 
+
    A paragraph
+with two lines.
    
+
    indented code
+
    
+
    
+
    A block quote.
    
+
    
+
    2. 
+

+````````````````````````````````
+
+
+Indented two spaces:
+
+```````````````````````````````` example
+  1.  A paragraph
+      with two lines.
+
+          indented code
+
+      > A block quote.
+.
+
    
+
  1. 
+
    A paragraph
+with two lines.
    
+
    indented code
+
    
+
    
+
    A block quote.
    
+
    
+
    2. 
+

+````````````````````````````````
+
+
+Indented three spaces:
+
+```````````````````````````````` example
+   1.  A paragraph
+       with two lines.
+
+           indented code
+
+       > A block quote.
+.
+
    
+
  1. 
+
    A paragraph
+with two lines.
    
+
    indented code
+
    
+
    
+
    A block quote.
    
+
    
+
    2. 
+

+````````````````````````````````
+
+
+Four spaces indent gives a code block:
+
+```````````````````````````````` example
+    1.  A paragraph
+        with two lines.
+
+            indented code
+
+        > A block quote.
+.
+
1.  A paragraph
+    with two lines.
+
+        indented code
+
+    > A block quote.
+

+````````````````````````````````
+
+
+
+5.  **Laziness.**  If a string of lines *Ls* constitute a [list
+    item](#list-items) with contents *Bs*, then the result of deleting
+    some or all of the indentation from one or more lines in which the
+    next [non-whitespace character] after the indentation is
+    [paragraph continuation text] is a
+    list item with the same contents and attributes.  The unindented
+    lines are called
+    [lazy continuation line](@)s.
+
+Here is an example with [lazy continuation lines]:
+
+```````````````````````````````` example
+  1.  A paragraph
+with two lines.
+
+          indented code
+
+      > A block quote.
+.
+
    
+
  1. 
+
    A paragraph
+with two lines.
    
+
    indented code
+
    
+
    
+
    A block quote.
    
+
    
+
    2. 
+

+````````````````````````````````
+
+
+Indentation can be partially deleted:
+
+```````````````````````````````` example
+  1.  A paragraph
+    with two lines.
+.
+
    
+
  1. A paragraph
+with two lines.
    2. 
+

+````````````````````````````````
+
+
+These examples show how laziness can work in nested structures:
+
+```````````````````````````````` example
+> 1. > Blockquote
+continued here.
+.
+

+
    
+
  1. 
+
    
+
    Blockquote
+continued here.
    
+
    
+
    2. 
+

+

+````````````````````````````````
+
+
+```````````````````````````````` example
+> 1. > Blockquote
+> continued here.
+.
+

+
    
+
  1. 
+
    
+
    Blockquote
+continued here.
    
+
    
+
    2. 
+

+

+````````````````````````````````
+
+
+
+6.  **That's all.** Nothing that is not counted as a list item by rules
+    #1--5 counts as a [list item](#list-items).
+
+The rules for sublists follow from the general rules above.  A sublist
+must be indented the same number of spaces a paragraph would need to be
+in order to be included in the list item.
+
+So, in this case we need two spaces indent:
+
+```````````````````````````````` example
+- foo
+  - bar
+    - baz
+.
+
    
+
  • foo
+
      
+
    • bar
+
        
+
      • baz
        • 
+
      
+
      • 
+
    
+
    • 
+

+````````````````````````````````
+
+
+One is not enough:
+
+```````````````````````````````` example
+- foo
+ - bar
+  - baz
+.
+
    
+
  • foo
    • 
+
  • bar
    • 
+
  • baz
    • 
+

+````````````````````````````````
+
+
+Here we need four, because the list marker is wider:
+
+```````````````````````````````` example
+10) foo
+    - bar
+.
+
    
+
  1. foo
+
      
+
    • bar
      • 
+
    
+
    2. 
+

+````````````````````````````````
+
+
+Three is not enough:
+
+```````````````````````````````` example
+10) foo
+   - bar
+.
+
    
+
  1. foo
    2. 
+

+
    
+
  • bar
    • 
+

+````````````````````````````````
+
+
+A list may be the first block in a list item:
+
+```````````````````````````````` example
+- - foo
+.
+
    
+
  • 
+
      
+
    • foo
      • 
+
    
+
    • 
+

+````````````````````````````````
+
+
+```````````````````````````````` example
+1. - 2. foo
+.
+
    
+
  1. 
+
      
+
    • 
+
        
+
      1. foo
        2. 
+
      
+
      • 
+
    
+
    2. 
+

+````````````````````````````````
+
+
+A list item can contain a heading:
+
+```````````````````````````````` example
+- # Foo
+- Bar
+  ---
+  baz
+.
+
    
+
  • 
+
    Foo
    
+
    • 
+
  • 
+
    Bar
    
+baz
    • 
+

+````````````````````````````````
+
+
+### Motivation
+
+John Gruber's Markdown spec says the following about list items:
+
+1. "List markers typically start at the left margin, but may be indented
+   by up to three spaces. List markers must be followed by one or more
+   spaces or a tab."
+
+2. "To make lists look nice, you can wrap items with hanging indents....
+   But if you don't want to, you don't have to."
+
+3. "List items may consist of multiple paragraphs. Each subsequent
+   paragraph in a list item must be indented by either 4 spaces or one
+   tab."
+
+4. "It looks nice if you indent every line of the subsequent paragraphs,
+   but here again, Markdown will allow you to be lazy."
+
+5. "To put a blockquote within a list item, the blockquote's `>`
+   delimiters need to be indented."
+
+6. "To put a code block within a list item, the code block needs to be
+   indented twice — 8 spaces or two tabs."
+
+These rules specify that a paragraph under a list item must be indented
+four spaces (presumably, from the left margin, rather than the start of
+the list marker, but this is not said), and that code under a list item
+must be indented eight spaces instead of the usual four.  They also say
+that a block quote must be indented, but not by how much; however, the
+example given has four spaces indentation.  Although nothing is said
+about other kinds of block-level content, it is certainly reasonable to
+infer that *all* block elements under a list item, including other
+lists, must be indented four spaces.  This principle has been called the
+*four-space rule*.
+
+The four-space rule is clear and principled, and if the reference
+implementation `Markdown.pl` had followed it, it probably would have
+become the standard.  However, `Markdown.pl` allowed paragraphs and
+sublists to start with only two spaces indentation, at least on the
+outer level.  Worse, its behavior was inconsistent: a sublist of an
+outer-level list needed two spaces indentation, but a sublist of this
+sublist needed three spaces.  It is not surprising, then, that different
+implementations of Markdown have developed very different rules for
+determining what comes under a list item.  (Pandoc and python-Markdown,
+for example, stuck with Gruber's syntax description and the four-space
+rule, while discount, redcarpet, marked, PHP Markdown, and others
+followed `Markdown.pl`'s behavior more closely.)
+
+Unfortunately, given the divergences between implementations, there
+is no way to give a spec for list items that will be guaranteed not
+to break any existing documents.  However, the spec given here should
+correctly handle lists formatted with either the four-space rule or
+the more forgiving `Markdown.pl` behavior, provided they are laid out
+in a way that is natural for a human to read.
+
+The strategy here is to let the width and indentation of the list marker
+determine the indentation necessary for blocks to fall under the list
+item, rather than having a fixed and arbitrary number.  The writer can
+think of the body of the list item as a unit which gets indented to the
+right enough to fit the list marker (and any indentation on the list
+marker).  (The laziness rule, #5, then allows continuation lines to be
+unindented if needed.)
+
+This rule is superior, we claim, to any rule requiring a fixed level of
+indentation from the margin.  The four-space rule is clear but
+unnatural. It is quite unintuitive that
+
+``` markdown
+- foo
+
+  bar
+
+  - baz
+```
+
+should be parsed as two lists with an intervening paragraph,
+
+``` html
+
    
+
  • foo
    • 
+

+
bar

+
    
+
  • baz
    • 
+

+```
+
+as the four-space rule demands, rather than a single list,
+
+``` html
+
    
+
  • 
+
    foo
    
+
    bar
    
+
      
+
    • baz
      • 
+
    
+
    • 
+

+```
+
+The choice of four spaces is arbitrary.  It can be learned, but it is
+not likely to be guessed, and it trips up beginners regularly.
+
+Would it help to adopt a two-space rule?  The problem is that such
+a rule, together with the rule allowing 1--3 spaces indentation of the
+initial list marker, allows text that is indented *less than* the
+original list marker to be included in the list item. For example,
+`Markdown.pl` parses
+
+``` markdown
+   - one
+
+  two
+```
+
+as a single list item, with `two` a continuation paragraph:
+
+``` html
+
    
+
  • 
+
    one
    
+
    two
    
+
    • 
+

+```
+
+and similarly
+
+``` markdown
+>   - one
+>
+>  two
+```
+
+as
+
+``` html
+

+
    
+
  • 
+
    one
    
+
    two
    
+
    • 
+

+

+```
+
+This is extremely unintuitive.
+
+Rather than requiring a fixed indent from the margin, we could require
+a fixed indent (say, two spaces, or even one space) from the list marker (which
+may itself be indented).  This proposal would remove the last anomaly
+discussed.  Unlike the spec presented above, it would count the following
+as a list item with a subparagraph, even though the paragraph `bar`
+is not indented as far as the first paragraph `foo`:
+
+``` markdown
+ 10. foo
+
+   bar  
+```
+
+Arguably this text does read like a list item with `bar` as a subparagraph,
+which may count in favor of the proposal.  However, on this proposal indented
+code would have to be indented six spaces after the list marker.  And this
+would break a lot of existing Markdown, which has the pattern:
+
+``` markdown
+1.  foo
+
+        indented code
+```
+
+where the code is indented eight spaces.  The spec above, by contrast, will
+parse this text as expected, since the code block's indentation is measured
+from the beginning of `foo`.
+
+The one case that needs special treatment is a list item that *starts*
+with indented code.  How much indentation is required in that case, since
+we don't have a "first paragraph" to measure from?  Rule #2 simply stipulates
+that in such cases, we require one space indentation from the list marker
+(and then the normal four spaces for the indented code).  This will match the
+four-space rule in cases where the list marker plus its initial indentation
+takes four spaces (a common case), but diverge in other cases.
+
+## Lists
+
+A [list](@) is a sequence of one or more
+list items [of the same type].  The list items
+may be separated by single [blank lines], but two
+blank lines end all containing lists.
+
+Two list items are [of the same type](@)
+if they begin with a [list marker] of the same type.
+Two list markers are of the
+same type if (a) they are bullet list markers using the same character
+(`-`, `+`, or `*`) or (b) they are ordered list numbers with the same
+delimiter (either `.` or `)`).
+
+A list is an [ordered list](@)
+if its constituent list items begin with
+[ordered list markers], and a
+[bullet list](@) if its constituent list
+items begin with [bullet list markers].
+
+The [start number](@)
+of an [ordered list] is determined by the list number of
+its initial list item.  The numbers of subsequent list items are
+disregarded.
+
+A list is [loose](@) if any of its constituent
+list items are separated by blank lines, or if any of its constituent
+list items directly contain two block-level elements with a blank line
+between them.  Otherwise a list is [tight](@).
+(The difference in HTML output is that paragraphs in a loose list are
+wrapped in `
` tags, while paragraphs in a tight list are not.)
+
+Changing the bullet or ordered list delimiter starts a new list:
+
+```````````````````````````````` example
+- foo
+- bar
++ baz
+.
+
    
+
  • foo
    • 
+
  • bar
    • 
+

+
    
+
  • baz
    • 
+

+````````````````````````````````
+
+
+```````````````````````````````` example
+1. foo
+2. bar
+3) baz
+.
+
    
+
  1. foo
    2. 
+
  2. bar
    3. 
+

+
    
+
  1. baz
    2. 
+

+````````````````````````````````
+
+
+In CommonMark, a list can interrupt a paragraph. That is,
+no blank line is needed to separate a paragraph from a following
+list:
+
+```````````````````````````````` example
+Foo
+- bar
+- baz
+.
+
Foo

+
    
+
  • bar
    • 
+
  • baz
    • 
+

+````````````````````````````````
+
+
+`Markdown.pl` does not allow this, through fear of triggering a list
+via a numeral in a hard-wrapped line:
+
+```````````````````````````````` example
+The number of windows in my house is
+14.  The number of doors is 6.
+.
+
The number of windows in my house is

+
    
+
  1. The number of doors is 6.
    2. 
+

+````````````````````````````````
+
+
+Oddly, `Markdown.pl` *does* allow a blockquote to interrupt a paragraph,
+even though the same considerations might apply.  We think that the two
+cases should be treated the same.  Here are two reasons for allowing
+lists to interrupt paragraphs:
+
+First, it is natural and not uncommon for people to start lists without
+blank lines:
+
+    I need to buy
+    - new shoes
+    - a coat
+    - a plane ticket
+
+Second, we are attracted to a
+
+> [principle of uniformity](@):
+> if a chunk of text has a certain
+> meaning, it will continue to have the same meaning when put into a
+> container block (such as a list item or blockquote).
+
+(Indeed, the spec for [list items] and [block quotes] presupposes
+this principle.) This principle implies that if
+
+      * I need to buy
+        - new shoes
+        - a coat
+        - a plane ticket
+
+is a list item containing a paragraph followed by a nested sublist,
+as all Markdown implementations agree it is (though the paragraph
+may be rendered without `
` tags, since the list is "tight"),
+then
+
+    I need to buy
+    - new shoes
+    - a coat
+    - a plane ticket
+
+by itself should be a paragraph followed by a nested sublist.
+
+Our adherence to the [principle of uniformity]
+thus inclines us to think that there are two coherent packages:
+
+1.  Require blank lines before *all* lists and blockquotes,
+    including lists that occur as sublists inside other list items.
+
+2.  Require blank lines in none of these places.
+
+[reStructuredText](http://docutils.sourceforge.net/rst.html) takes
+the first approach, for which there is much to be said.  But the second
+seems more consistent with established practice with Markdown.
+
+There can be blank lines between items, but two blank lines end
+a list:
+
+```````````````````````````````` example
+- foo
+
+- bar
+
+
+- baz
+.
+
    
+
  • 
+
    foo
    
+
    • 
+
  • 
+
    bar
    
+
    • 
+

+
    
+
  • baz
    • 
+

+````````````````````````````````
+
+
+As illustrated above in the section on [list items],
+two blank lines between blocks *within* a list item will also end a
+list:
+
+```````````````````````````````` example
+- foo
+
+
+  bar
+- baz
+.
+
    
+
  • foo
    • 
+

+
bar

+
    
+
  • baz
    • 
+

+````````````````````````````````
+
+
+Indeed, two blank lines will end *all* containing lists:
+
+```````````````````````````````` example
+- foo
+  - bar
+    - baz
+
+
+      bim
+.
+
    
+
  • foo
+
      
+
    • bar
+
        
+
      • baz
        • 
+
      
+
      • 
+
    
+
    • 
+

+
  bim
+

+````````````````````````````````
+
+
+Thus, two blank lines can be used to separate consecutive lists of
+the same type, or to separate a list from an indented code block
+that would otherwise be parsed as a subparagraph of the final list
+item:
+
+```````````````````````````````` example
+- foo
+- bar
+
+
+- baz
+- bim
+.
+
    
+
  • foo
    • 
+
  • bar
    • 
+

+
    
+
  • baz
    • 
+
  • bim
    • 
+

+````````````````````````````````
+
+
+```````````````````````````````` example
+-   foo
+
+    notcode
+
+-   foo
+
+
+    code
+.
+
    
+
  • 
+
    foo
    
+
    notcode
    
+
    • 
+
  • 
+
    foo
    
+
    • 
+

+
code
+

+````````````````````````````````
+
+
+List items need not be indented to the same level.  The following
+list items will be treated as items at the same list level,
+since none is indented enough to belong to the previous list
+item:
+
+```````````````````````````````` example
+- a
+ - b
+  - c
+   - d
+    - e
+   - f
+  - g
+ - h
+- i
+.
+
    
+
  • a
    • 
+
  • b
    • 
+
  • c
    • 
+
  • d
    • 
+
  • e
    • 
+
  • f
    • 
+
  • g
    • 
+
  • h
    • 
+
  • i
    • 
+

+````````````````````````````````
+
+
+```````````````````````````````` example
+1. a
+
+  2. b
+
+    3. c
+.
+
    
+
  1. 
+
    a
    
+
    2. 
+
  2. 
+
    b
    
+
    3. 
+
  3. 
+
    c
    
+
    4. 
+

+````````````````````````````````
+
+
+This is a loose list, because there is a blank line between
+two of the list items:
+
+```````````````````````````````` example
+- a
+- b
+
+- c
+.
+
    
+
  • 
+
    a
    
+
    • 
+
  • 
+
    b
    
+
    • 
+
  • 
+
    c
    
+
    • 
+

+````````````````````````````````
+
+
+So is this, with a empty second item:
+
+```````````````````````````````` example
+* a
+*
+
+* c
+.
+
    
+
  • 
+
    a
    
+
    • 
+
  • 
+
  • 
+
    c
    
+
    • 
+

+````````````````````````````````
+
+
+These are loose lists, even though there is no space between the items,
+because one of the items directly contains two block-level elements
+with a blank line between them:
+
+```````````````````````````````` example
+- a
+- b
+
+  c
+- d
+.
+
    
+
  • 
+
    a
    
+
    • 
+
  • 
+
    b
    
+
    c
    
+
    • 
+
  • 
+
    d
    
+
    • 
+

+````````````````````````````````
+
+
+```````````````````````````````` example
+- a
+- b
+
+  [ref]: /url
+- d
+.
+
    
+
  • 
+
    a
    
+
    • 
+
  • 
+
    b
    
+
    • 
+
  • 
+
    d
    
+
    • 
+

+````````````````````````````````
+
+
+This is a tight list, because the blank lines are in a code block:
+
+```````````````````````````````` example
+- a
+- ```
+  b
+
+
+  ```
+- c
+.
+
    
+
  • a
    • 
+
  • 
+
    b
+
+
+
    
+
    • 
+
  • c
    • 
+

+````````````````````````````````
+
+
+This is a tight list, because the blank line is between two
+paragraphs of a sublist.  So the sublist is loose while
+the outer list is tight:
+
+```````````````````````````````` example
+- a
+  - b
+
+    c
+- d
+.
+
    
+
  • a
+
      
+
    • 
+
      b
      
+
      c
      
+
      • 
+
    
+
    • 
+
  • d
    • 
+

+````````````````````````````````
+
+
+This is a tight list, because the blank line is inside the
+block quote:
+
+```````````````````````````````` example
+* a
+  > b
+  >
+* c
+.
+
    
+
  • a
+
    
+
    b
    
+
    
+
    • 
+
  • c
    • 
+

+````````````````````````````````
+
+
+This list is tight, because the consecutive block elements
+are not separated by blank lines:
+
+```````````````````````````````` example
+- a
+  > b
+  ```
+  c
+  ```
+- d
+.
+
    
+
  • a
+
    
+
    b
    
+
    
+
    c
+
    
+
    • 
+
  • d
    • 
+

+````````````````````````````````
+
+
+A single-paragraph list is tight:
+
+```````````````````````````````` example
+- a
+.
+
    
+
  • a
    • 
+

+````````````````````````````````
+
+
+```````````````````````````````` example
+- a
+  - b
+.
+
    
+
  • a
+
      
+
    • b
      • 
+
    
+
    • 
+

+````````````````````````````````
+
+
+This list is loose, because of the blank line between the
+two block elements in the list item:
+
+```````````````````````````````` example
+1. ```
+   foo
+   ```
+
+   bar
+.
+
    
+
  1. 
+
    foo
+
    
+
    bar
    
+
    2. 
+

+````````````````````````````````
+
+
+Here the outer list is loose, the inner list tight:
+
+```````````````````````````````` example
+* foo
+  * bar
+
+  baz
+.
+
    
+
  • 
+
    foo
    
+
      
+
    • bar
      • 
+
    
+
    baz
    
+
    • 
+

+````````````````````````````````
+
+
+```````````````````````````````` example
+- a
+  - b
+  - c
+
+- d
+  - e
+  - f
+.
+
    
+
  • 
+
    a
    
+
      
+
    • b
      • 
+
    • c
      • 
+
    
+
    • 
+
  • 
+
    d
    
+
      
+
    • e
      • 
+
    • f
      • 
+
    
+
    • 
+

+````````````````````````````````
+
+
+# Inlines
+
+Inlines are parsed sequentially from the beginning of the character
+stream to the end (left to right, in left-to-right languages).
+Thus, for example, in
+
+```````````````````````````````` example
+`hi`lo`
+.
+
hilo`

+````````````````````````````````
+
+
+`hi` is parsed as code, leaving the backtick at the end as a literal
+backtick.
+
+## Backslash escapes
+
+Any ASCII punctuation character may be backslash-escaped:
+
+```````````````````````````````` example
+\!\"\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\=\>\?\@\[\\\]\^\_\`\{\|\}\~
+.
+
!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~

+````````````````````````````````
+
+
+Backslashes before other characters are treated as literal
+backslashes:
+
+```````````````````````````````` example
+\→\A\a\ \3\φ\«
+.
+
\→\A\a\ \3\φ\«

+````````````````````````````````
+
+
+Escaped characters are treated as regular characters and do
+not have their usual Markdown meanings:
+
+```````````````````````````````` example
+\*not emphasized*
+\
 not a tag
+\[not a link](/foo)
+\`not code`
+1\. not a list
+\* not a list
+\# not a heading
+\[foo]: /url "not a reference"
+.
+
*not emphasized*
+<br/> not a tag
+[not a link](/foo)
+`not code`
+1. not a list
+* not a list
+# not a heading
+[foo]: /url "not a reference"

+````````````````````````````````
+
+
+If a backslash is itself escaped, the following character is not:
+
+```````````````````````````````` example
+\\*emphasis*
+.
+
\emphasis

+````````````````````````````````
+
+
+A backslash at the end of the line is a [hard line break]:
+
+```````````````````````````````` example
+foo\
+bar
+.
+
foo

+bar

+````````````````````````````````
+
+
+Backslash escapes do not work in code blocks, code spans, autolinks, or
+raw HTML:
+
+```````````````````````````````` example
+`` \[\` ``
+.
+
\[\`

+````````````````````````````````
+
+
+```````````````````````````````` example
+    \[\]
+.
+
\[\]
+

+````````````````````````````````
+
+
+```````````````````````````````` example
+~~~
+\[\]
+~~~
+.
+
\[\]
+

+````````````````````````````````
+
+
+```````````````````````````````` example
+
+.
+
http://example.com?find=\*

+````````````````````````````````
+
+
+```````````````````````````````` example
+
+.
+
+````````````````````````````````
+
+
+But they work in all other contexts, including URLs and link titles,
+link references, and [info strings] in [fenced code blocks]:
+
+```````````````````````````````` example
+[foo](/bar\* "ti\*tle")
+.
+
foo

+````````````````````````````````
+
+
+```````````````````````````````` example
+[foo]
+
+[foo]: /bar\* "ti\*tle"
+.
+
foo

+````````````````````````````````
+
+
+```````````````````````````````` example
+``` foo\+bar
+foo
+```
+.
+
foo
+

+````````````````````````````````
+
+
+
+## Entity and numeric character references
+
+All valid HTML entity references and numeric character
+references, except those occuring in code blocks and code spans,
+are recognized as such and treated as equivalent to the
+corresponding Unicode characters.  Conforming CommonMark parsers
+need not store information about whether a particular character
+was represented in the source using a Unicode character or
+an entity reference.
+
+[Entity references](@) consist of `&` + any of the valid
+HTML5 entity names + `;`. The
+document 
+is used as an authoritative source for the valid entity
+references and their corresponding code points.
+
+```````````````````````````````` example
+  & © Æ Ď
+¾ ℋ ⅆ
+∲ ≧̸
+.
+
  & © Æ Ď
+¾ ℋ ⅆ
+∲ ≧̸

+````````````````````````````````
+
+
+[Decimal numeric character
+references](@)
+consist of `&#` + a string of 1--8 arabic digits + `;`. A
+numeric character reference is parsed as the corresponding
+Unicode character. Invalid Unicode code points will be replaced by
+the REPLACEMENT CHARACTER (`U+FFFD`).  For security reasons,
+the code point `U+0000` will also be replaced by `U+FFFD`.
+
+```````````````````````````````` example
+# Ӓ Ϡ � �
+.
+
# Ӓ Ϡ � �

+````````````````````````````````
+
+
+[Hexadecimal numeric character
+references](@) consist of `&#` +
+either `X` or `x` + a string of 1-8 hexadecimal digits + `;`.
+They too are parsed as the corresponding Unicode character (this
+time specified with a hexadecimal numeral instead of decimal).
+
+```````````````````````````````` example
+" ആ ಫ
+.
+
" ആ ಫ

+````````````````````````````````
+
+
+Here are some nonentities:
+
+```````````````````````````````` example
+  &x; &#; &#x;
+&ThisIsNotDefined; &hi?;
+.
+
&nbsp &x; &#; &#x;
+&ThisIsNotDefined; &hi?;

+````````````````````````````````
+
+
+Although HTML5 does accept some entity references
+without a trailing semicolon (such as `©`), these are not
+recognized here, because it makes the grammar too ambiguous:
+
+```````````````````````````````` example
+©
+.
+
&copy

+````````````````````````````````
+
+
+Strings that are not on the list of HTML5 named entities are not
+recognized as entity references either:
+
+```````````````````````````````` example
+&MadeUpEntity;
+.
+
&MadeUpEntity;

+````````````````````````````````
+
+
+Entity and numeric character references are recognized in any
+context besides code spans or code blocks, including
+URLs, [link titles], and [fenced code block][] [info strings]:
+
+```````````````````````````````` example
+
+.
+
+````````````````````````````````
+
+
+```````````````````````````````` example
+[foo](/föö "föö")
+.
+
foo

+````````````````````````````````
+
+
+```````````````````````````````` example
+[foo]
+
+[foo]: /föö "föö"
+.
+
foo

+````````````````````````````````
+
+
+```````````````````````````````` example
+``` föö
+foo
+```
+.
+
foo
+

+````````````````````````````````
+
+
+Entity and numeric character references are treated as literal
+text in code spans and code blocks:
+
+```````````````````````````````` example
+`föö`
+.
+
f&ouml;&ouml;

+````````````````````````````````
+
+
+```````````````````````````````` example
+    föfö
+.
+
f&ouml;f&ouml;
+

+````````````````````````````````
+
+
+## Code spans
+
+A [backtick string](@)
+is a string of one or more backtick characters (`` ` ``) that is neither
+preceded nor followed by a backtick.
+
+A [code span](@) begins with a backtick string and ends with
+a backtick string of equal length.  The contents of the code span are
+the characters between the two backtick strings, with leading and
+trailing spaces and [line endings] removed, and
+[whitespace] collapsed to single spaces.
+
+This is a simple code span:
+
+```````````````````````````````` example
+`foo`
+.
+
foo

+````````````````````````````````
+
+
+Here two backticks are used, because the code contains a backtick.
+This example also illustrates stripping of leading and trailing spaces:
+
+```````````````````````````````` example
+`` foo ` bar  ``
+.
+
foo ` bar

+````````````````````````````````
+
+
+This example shows the motivation for stripping leading and trailing
+spaces:
+
+```````````````````````````````` example
+` `` `
+.
+
``

+````````````````````````````````
+
+
+[Line endings] are treated like spaces:
+
+```````````````````````````````` example
+``
+foo
+``
+.
+
foo

+````````````````````````````````
+
+
+Interior spaces and [line endings] are collapsed into
+single spaces, just as they would be by a browser:
+
+```````````````````````````````` example
+`foo   bar
+  baz`
+.
+
foo bar baz

+````````````````````````````````
+
+
+Q: Why not just leave the spaces, since browsers will collapse them
+anyway?  A:  Because we might be targeting a non-HTML format, and we
+shouldn't rely on HTML-specific rendering assumptions.
+
+(Existing implementations differ in their treatment of internal
+spaces and [line endings].  Some, including `Markdown.pl` and
+`showdown`, convert an internal [line ending] into a
+`
` tag.  But this makes things difficult for those who like to
+hard-wrap their paragraphs, since a line break in the midst of a code
+span will cause an unintended line break in the output.  Others just
+leave internal spaces as they are, which is fine if only HTML is being
+targeted.)
+
+```````````````````````````````` example
+`foo `` bar`
+.
+
foo `` bar

+````````````````````````````````
+
+
+Note that backslash escapes do not work in code spans. All backslashes
+are treated literally:
+
+```````````````````````````````` example
+`foo\`bar`
+.
+
foo\bar`

+````````````````````````````````
+
+
+Backslash escapes are never needed, because one can always choose a
+string of *n* backtick characters as delimiters, where the code does
+not contain any strings of exactly *n* backtick characters.
+
+Code span backticks have higher precedence than any other inline
+constructs except HTML tags and autolinks.  Thus, for example, this is
+not parsed as emphasized text, since the second `*` is part of a code
+span:
+
+```````````````````````````````` example
+*foo`*`
+.
+
*foo*

+````````````````````````````````
+
+
+And this is not parsed as a link:
+
+```````````````````````````````` example
+[not a `link](/foo`)
+.
+
[not a link](/foo)

+````````````````````````````````
+
+
+Code spans, HTML tags, and autolinks have the same precedence.
+Thus, this is code:
+
+```````````````````````````````` example
+``
+.
+
<a href="">`

+````````````````````````````````
+
+
+But this is an HTML tag:
+
+```````````````````````````````` example
+`
+.
+
`

+````````````````````````````````
+
+
+And this is code:
+
+```````````````````````````````` example
+``
+.
+
<http://foo.bar.baz>`

+````````````````````````````````
+
+
+But this is an autolink:
+
+```````````````````````````````` example
+`
+.
+
http://foo.bar.`baz`

+````````````````````````````````
+
+
+When a backtick string is not closed by a matching backtick string,
+we just have literal backticks:
+
+```````````````````````````````` example
+```foo``
+.
+
```foo``

+````````````````````````````````
+
+
+```````````````````````````````` example
+`foo
+.
+
`foo

+````````````````````````````````
+
+
+## Emphasis and strong emphasis
+
+John Gruber's original [Markdown syntax
+description](http://daringfireball.net/projects/markdown/syntax#em) says:
+
+> Markdown treats asterisks (`*`) and underscores (`_`) as indicators of
+> emphasis. Text wrapped with one `*` or `_` will be wrapped with an HTML
+> `` tag; double `*`'s or `_`'s will be wrapped with an HTML ``
+> tag.
+
+This is enough for most users, but these rules leave much undecided,
+especially when it comes to nested emphasis.  The original
+`Markdown.pl` test suite makes it clear that triple `***` and
+`___` delimiters can be used for strong emphasis, and most
+implementations have also allowed the following patterns:
+
+``` markdown
+***strong emph***
+***strong** in emph*
+***emph* in strong**
+**in strong *emph***
+*in emph **strong***
+```
+
+The following patterns are less widely supported, but the intent
+is clear and they are useful (especially in contexts like bibliography
+entries):
+
+``` markdown
+*emph *with emph* in it*
+**strong **with strong** in it**
+```
+
+Many implementations have also restricted intraword emphasis to
+the `*` forms, to avoid unwanted emphasis in words containing
+internal underscores.  (It is best practice to put these in code
+spans, but users often do not.)
+
+``` markdown
+internal emphasis: foo*bar*baz
+no emphasis: foo_bar_baz
+```
+
+The rules given below capture all of these patterns, while allowing
+for efficient parsing strategies that do not backtrack.
+
+First, some definitions.  A [delimiter run](@) is either
+a sequence of one or more `*` characters that is not preceded or
+followed by a `*` character, or a sequence of one or more `_`
+characters that is not preceded or followed by a `_` character.
+
+A [left-flanking delimiter run](@) is
+a [delimiter run] that is (a) not followed by [Unicode whitespace],
+and (b) either not followed by a [punctuation character], or
+preceded by [Unicode whitespace] or a [punctuation character].
+For purposes of this definition, the beginning and the end of
+the line count as Unicode whitespace.
+
+A [right-flanking delimiter run](@) is
+a [delimiter run] that is (a) not preceded by [Unicode whitespace],
+and (b) either not preceded by a [punctuation character], or
+followed by [Unicode whitespace] or a [punctuation character].
+For purposes of this definition, the beginning and the end of
+the line count as Unicode whitespace.
+
+Here are some examples of delimiter runs.
+
+  - left-flanking but not right-flanking:
+
+    ```
+    ***abc
+      _abc
+    **"abc"
+     _"abc"
+    ```
+
+  - right-flanking but not left-flanking:
+
+    ```
+     abc***
+     abc_
+    "abc"**
+    "abc"_
+    ```
+
+  - Both left and right-flanking:
+
+    ```
+     abc***def
+    "abc"_"def"
+    ```
+
+  - Neither left nor right-flanking:
+
+    ```
+    abc *** def
+    a _ b
+    ```
+
+(The idea of distinguishing left-flanking and right-flanking
+delimiter runs based on the character before and the character
+after comes from Roopesh Chander's
+[vfmd](http://www.vfmd.org/vfmd-spec/specification/#procedure-for-identifying-emphasis-tags).
+vfmd uses the terminology "emphasis indicator string" instead of "delimiter
+run," and its rules for distinguishing left- and right-flanking runs
+are a bit more complex than the ones given here.)
+
+The following rules define emphasis and strong emphasis:
+
+1.  A single `*` character [can open emphasis](@)
+    iff (if and only if) it is part of a [left-flanking delimiter run].
+
+2.  A single `_` character [can open emphasis] iff
+    it is part of a [left-flanking delimiter run]
+    and either (a) not part of a [right-flanking delimiter run]
+    or (b) part of a [right-flanking delimiter run]
+    preceded by punctuation.
+
+3.  A single `*` character [can close emphasis](@)
+    iff it is part of a [right-flanking delimiter run].
+
+4.  A single `_` character [can close emphasis] iff
+    it is part of a [right-flanking delimiter run]
+    and either (a) not part of a [left-flanking delimiter run]
+    or (b) part of a [left-flanking delimiter run]
+    followed by punctuation.
+
+5.  A double `**` [can open strong emphasis](@)
+    iff it is part of a [left-flanking delimiter run].
+
+6.  A double `__` [can open strong emphasis] iff
+    it is part of a [left-flanking delimiter run]
+    and either (a) not part of a [right-flanking delimiter run]
+    or (b) part of a [right-flanking delimiter run]
+    preceded by punctuation.
+
+7.  A double `**` [can close strong emphasis](@)
+    iff it is part of a [right-flanking delimiter run].
+
+8.  A double `__` [can close strong emphasis]
+    it is part of a [right-flanking delimiter run]
+    and either (a) not part of a [left-flanking delimiter run]
+    or (b) part of a [left-flanking delimiter run]
+    followed by punctuation.
+
+9.  Emphasis begins with a delimiter that [can open emphasis] and ends
+    with a delimiter that [can close emphasis], and that uses the same
+    character (`_` or `*`) as the opening delimiter.  There must
+    be a nonempty sequence of inlines between the open delimiter
+    and the closing delimiter; these form the contents of the emphasis
+    inline.
+
+10. Strong emphasis begins with a delimiter that
+    [can open strong emphasis] and ends with a delimiter that
+    [can close strong emphasis], and that uses the same character
+    (`_` or `*`) as the opening delimiter.
+    There must be a nonempty sequence of inlines between the open
+    delimiter and the closing delimiter; these form the contents of
+    the strong emphasis inline.
+
+11. A literal `*` character cannot occur at the beginning or end of
+    `*`-delimited emphasis or `**`-delimited strong emphasis, unless it
+    is backslash-escaped.
+
+12. A literal `_` character cannot occur at the beginning or end of
+    `_`-delimited emphasis or `__`-delimited strong emphasis, unless it
+    is backslash-escaped.
+
+Where rules 1--12 above are compatible with multiple parsings,
+the following principles resolve ambiguity:
+
+13. The number of nestings should be minimized. Thus, for example,
+    an interpretation `...` is always preferred to
+    `...`.
+
+14. An interpretation `...` is always
+    preferred to `..`.
+
+15. When two potential emphasis or strong emphasis spans overlap,
+    so that the second begins before the first ends and ends after
+    the first ends, the first takes precedence. Thus, for example,
+    `*foo _bar* baz_` is parsed as `foo _bar baz_` rather
+    than `*foo bar* baz`.  For the same reason,
+    `**foo*bar**` is parsed as `foobar*`
+    rather than `foo*bar`.
+
+16. When there are two potential emphasis or strong emphasis spans
+    with the same closing delimiter, the shorter one (the one that
+    opens later) takes precedence. Thus, for example,
+    `**foo **bar baz**` is parsed as `**foo bar baz`
+    rather than `foo **bar baz`.
+
+17. Inline code spans, links, images, and HTML tags group more tightly
+    than emphasis.  So, when there is a choice between an interpretation
+    that contains one of these elements and one that does not, the
+    former always wins.  Thus, for example, `*[foo*](bar)` is
+    parsed as `*foo*` rather than as
+    `[foo](bar)`.
+
+These rules can be illustrated through a series of examples.
+
+Rule 1:
+
+```````````````````````````````` example
+*foo bar*
+.
+
foo bar

+````````````````````````````````
+
+
+This is not emphasis, because the opening `*` is followed by
+whitespace, and hence not part of a [left-flanking delimiter run]:
+
+```````````````````````````````` example
+a * foo bar*
+.
+
a * foo bar*

+````````````````````````````````
+
+
+This is not emphasis, because the opening `*` is preceded
+by an alphanumeric and followed by punctuation, and hence
+not part of a [left-flanking delimiter run]:
+
+```````````````````````````````` example
+a*"foo"*
+.
+
a*"foo"*

+````````````````````````````````
+
+
+Unicode nonbreaking spaces count as whitespace, too:
+
+```````````````````````````````` example
+* a *
+.
+
* a *

+````````````````````````````````
+
+
+Intraword emphasis with `*` is permitted:
+
+```````````````````````````````` example
+foo*bar*
+.
+
foobar

+````````````````````````````````
+
+
+```````````````````````````````` example
+5*6*78
+.
+
5678

+````````````````````````````````
+
+
+Rule 2:
+
+```````````````````````````````` example
+_foo bar_
+.
+
foo bar

+````````````````````````````````
+
+
+This is not emphasis, because the opening `_` is followed by
+whitespace:
+
+```````````````````````````````` example
+_ foo bar_
+.
+
_ foo bar_

+````````````````````````````````
+
+
+This is not emphasis, because the opening `_` is preceded
+by an alphanumeric and followed by punctuation:
+
+```````````````````````````````` example
+a_"foo"_
+.
+
a_"foo"_

+````````````````````````````````
+
+
+Emphasis with `_` is not allowed inside words:
+
+```````````````````````````````` example
+foo_bar_
+.
+
foo_bar_

+````````````````````````````````
+
+
+```````````````````````````````` example
+5_6_78
+.
+
5_6_78

+````````````````````````````````
+
+
+```````````````````````````````` example
+пристаням_стремятся_
+.
+
пристаням_стремятся_

+````````````````````````````````
+
+
+Here `_` does not generate emphasis, because the first delimiter run
+is right-flanking and the second left-flanking:
+
+```````````````````````````````` example
+aa_"bb"_cc
+.
+
aa_"bb"_cc

+````````````````````````````````
+
+
+This is emphasis, even though the opening delimiter is
+both left- and right-flanking, because it is preceded by
+punctuation:
+
+```````````````````````````````` example
+foo-_(bar)_
+.
+
foo-(bar)

+````````````````````````````````
+
+
+Rule 3:
+
+This is not emphasis, because the closing delimiter does
+not match the opening delimiter:
+
+```````````````````````````````` example
+_foo*
+.
+
_foo*

+````````````````````````````````
+
+
+This is not emphasis, because the closing `*` is preceded by
+whitespace:
+
+```````````````````````````````` example
+*foo bar *
+.
+
*foo bar *

+````````````````````````````````
+
+
+A newline also counts as whitespace:
+
+```````````````````````````````` example
+*foo bar
+*
+.
+
*foo bar

+
    
+
  • 
+

+````````````````````````````````
+
+
+This is not emphasis, because the second `*` is
+preceded by punctuation and followed by an alphanumeric
+(hence it is not part of a [right-flanking delimiter run]:
+
+```````````````````````````````` example
+*(*foo)
+.
+
*(*foo)

+````````````````````````````````
+
+
+The point of this restriction is more easily appreciated
+with this example:
+
+```````````````````````````````` example
+*(*foo*)*
+.
+
(foo)

+````````````````````````````````
+
+
+Intraword emphasis with `*` is allowed:
+
+```````````````````````````````` example
+*foo*bar
+.
+
foobar

+````````````````````````````````
+
+
+
+Rule 4:
+
+This is not emphasis, because the closing `_` is preceded by
+whitespace:
+
+```````````````````````````````` example
+_foo bar _
+.
+
_foo bar _

+````````````````````````````````
+
+
+This is not emphasis, because the second `_` is
+preceded by punctuation and followed by an alphanumeric:
+
+```````````````````````````````` example
+_(_foo)
+.
+
_(_foo)

+````````````````````````````````
+
+
+This is emphasis within emphasis:
+
+```````````````````````````````` example
+_(_foo_)_
+.
+
(foo)

+````````````````````````````````
+
+
+Intraword emphasis is disallowed for `_`:
+
+```````````````````````````````` example
+_foo_bar
+.
+
_foo_bar

+````````````````````````````````
+
+
+```````````````````````````````` example
+_пристаням_стремятся
+.
+
_пристаням_стремятся

+````````````````````````````````
+
+
+```````````````````````````````` example
+_foo_bar_baz_
+.
+
foo_bar_baz

+````````````````````````````````
+
+
+This is emphasis, even though the closing delimiter is
+both left- and right-flanking, because it is followed by
+punctuation:
+
+```````````````````````````````` example
+_(bar)_.
+.
+
(bar).

+````````````````````````````````
+
+
+Rule 5:
+
+```````````````````````````````` example
+**foo bar**
+.
+
foo bar

+````````````````````````````````
+
+
+This is not strong emphasis, because the opening delimiter is
+followed by whitespace:
+
+```````````````````````````````` example
+** foo bar**
+.
+
** foo bar**

+````````````````````````````````
+
+
+This is not strong emphasis, because the opening `**` is preceded
+by an alphanumeric and followed by punctuation, and hence
+not part of a [left-flanking delimiter run]:
+
+```````````````````````````````` example
+a**"foo"**
+.
+
a**"foo"**

+````````````````````````````````
+
+
+Intraword strong emphasis with `**` is permitted:
+
+```````````````````````````````` example
+foo**bar**
+.
+
foobar

+````````````````````````````````
+
+
+Rule 6:
+
+```````````````````````````````` example
+__foo bar__
+.
+
foo bar

+````````````````````````````````
+
+
+This is not strong emphasis, because the opening delimiter is
+followed by whitespace:
+
+```````````````````````````````` example
+__ foo bar__
+.
+
__ foo bar__

+````````````````````````````````
+
+
+A newline counts as whitespace:
+```````````````````````````````` example
+__
+foo bar__
+.
+
__
+foo bar__

+````````````````````````````````
+
+
+This is not strong emphasis, because the opening `__` is preceded
+by an alphanumeric and followed by punctuation:
+
+```````````````````````````````` example
+a__"foo"__
+.
+
a__"foo"__

+````````````````````````````````
+
+
+Intraword strong emphasis is forbidden with `__`:
+
+```````````````````````````````` example
+foo__bar__
+.
+
foo__bar__

+````````````````````````````````
+
+
+```````````````````````````````` example
+5__6__78
+.
+
5__6__78

+````````````````````````````````
+
+
+```````````````````````````````` example
+пристаням__стремятся__
+.
+
пристаням__стремятся__

+````````````````````````````````
+
+
+```````````````````````````````` example
+__foo, __bar__, baz__
+.
+
foo, bar, baz

+````````````````````````````````
+
+
+This is strong emphasis, even though the opening delimiter is
+both left- and right-flanking, because it is preceded by
+punctuation:
+
+```````````````````````````````` example
+foo-__(bar)__
+.
+
foo-(bar)

+````````````````````````````````
+
+
+
+Rule 7:
+
+This is not strong emphasis, because the closing delimiter is preceded
+by whitespace:
+
+```````````````````````````````` example
+**foo bar **
+.
+
**foo bar **

+````````````````````````````````
+
+
+(Nor can it be interpreted as an emphasized `*foo bar *`, because of
+Rule 11.)
+
+This is not strong emphasis, because the second `**` is
+preceded by punctuation and followed by an alphanumeric:
+
+```````````````````````````````` example
+**(**foo)
+.
+
**(**foo)

+````````````````````````````````
+
+
+The point of this restriction is more easily appreciated
+with these examples:
+
+```````````````````````````````` example
+*(**foo**)*
+.
+
(foo)

+````````````````````````````````
+
+
+```````````````````````````````` example
+**Gomphocarpus (*Gomphocarpus physocarpus*, syn.
+*Asclepias physocarpa*)**
+.
+
Gomphocarpus (Gomphocarpus physocarpus, syn.
+Asclepias physocarpa)

+````````````````````````````````
+
+
+```````````````````````````````` example
+**foo "*bar*" foo**
+.
+
foo "bar" foo

+````````````````````````````````
+
+
+Intraword emphasis:
+
+```````````````````````````````` example
+**foo**bar
+.
+
foobar

+````````````````````````````````
+
+
+Rule 8:
+
+This is not strong emphasis, because the closing delimiter is
+preceded by whitespace:
+
+```````````````````````````````` example
+__foo bar __
+.
+
__foo bar __

+````````````````````````````````
+
+
+This is not strong emphasis, because the second `__` is
+preceded by punctuation and followed by an alphanumeric:
+
+```````````````````````````````` example
+__(__foo)
+.
+
__(__foo)

+````````````````````````````````
+
+
+The point of this restriction is more easily appreciated
+with this example:
+
+```````````````````````````````` example
+_(__foo__)_
+.
+
(foo)

+````````````````````````````````
+
+
+Intraword strong emphasis is forbidden with `__`:
+
+```````````````````````````````` example
+__foo__bar
+.
+
__foo__bar

+````````````````````````````````
+
+
+```````````````````````````````` example
+__пристаням__стремятся
+.
+
__пристаням__стремятся

+````````````````````````````````
+
+
+```````````````````````````````` example
+__foo__bar__baz__
+.
+
foo__bar__baz

+````````````````````````````````
+
+
+This is strong emphasis, even though the closing delimiter is
+both left- and right-flanking, because it is followed by
+punctuation:
+
+```````````````````````````````` example
+__(bar)__.
+.
+
(bar).

+````````````````````````````````
+
+
+Rule 9:
+
+Any nonempty sequence of inline elements can be the contents of an
+emphasized span.
+
+```````````````````````````````` example
+*foo [bar](/url)*
+.
+
foo bar

+````````````````````````````````
+
+
+```````````````````````````````` example
+*foo
+bar*
+.
+
foo
+bar

+````````````````````````````````
+
+
+In particular, emphasis and strong emphasis can be nested
+inside emphasis:
+
+```````````````````````````````` example
+_foo __bar__ baz_
+.
+
foo bar baz

+````````````````````````````````
+
+
+```````````````````````````````` example
+_foo _bar_ baz_
+.
+
foo bar baz

+````````````````````````````````
+
+
+```````````````````````````````` example
+__foo_ bar_
+.
+
foo bar

+````````````````````````````````
+
+
+```````````````````````````````` example
+*foo *bar**
+.
+
foo bar

+````````````````````````````````
+
+
+```````````````````````````````` example
+*foo **bar** baz*
+.
+
foo bar baz

+````````````````````````````````
+
+
+But note:
+
+```````````````````````````````` example
+*foo**bar**baz*
+.
+
foobarbaz

+````````````````````````````````
+
+
+The difference is that in the preceding case, the internal delimiters
+[can close emphasis], while in the cases with spaces, they cannot.
+
+```````````````````````````````` example
+***foo** bar*
+.
+
foo bar

+````````````````````````````````
+
+
+```````````````````````````````` example
+*foo **bar***
+.
+
foo bar

+````````````````````````````````
+
+
+Note, however, that in the following case we get no strong
+emphasis, because the opening delimiter is closed by the first
+`*` before `bar`:
+
+```````````````````````````````` example
+*foo**bar***
+.
+
foobar**

+````````````````````````````````
+
+
+
+Indefinite levels of nesting are possible:
+
+```````````````````````````````` example
+*foo **bar *baz* bim** bop*
+.
+
foo bar baz bim bop

+````````````````````````````````
+
+
+```````````````````````````````` example
+*foo [*bar*](/url)*
+.
+
foo bar

+````````````````````````````````
+
+
+There can be no empty emphasis or strong emphasis:
+
+```````````````````````````````` example
+** is not an empty emphasis
+.
+
** is not an empty emphasis

+````````````````````````````````
+
+
+```````````````````````````````` example
+**** is not an empty strong emphasis
+.
+
**** is not an empty strong emphasis

+````````````````````````````````
+
+
+
+Rule 10:
+
+Any nonempty sequence of inline elements can be the contents of an
+strongly emphasized span.
+
+```````````````````````````````` example
+**foo [bar](/url)**
+.
+
foo bar

+````````````````````````````````
+
+
+```````````````````````````````` example
+**foo
+bar**
+.
+
foo
+bar

+````````````````````````````````
+
+
+In particular, emphasis and strong emphasis can be nested
+inside strong emphasis:
+
+```````````````````````````````` example
+__foo _bar_ baz__
+.
+
foo bar baz

+````````````````````````````````
+
+
+```````````````````````````````` example
+__foo __bar__ baz__
+.
+
foo bar baz

+````````````````````````````````
+
+
+```````````````````````````````` example
+____foo__ bar__
+.
+
foo bar

+````````````````````````````````
+
+
+```````````````````````````````` example
+**foo **bar****
+.
+
foo bar

+````````````````````````````````
+
+
+```````````````````````````````` example
+**foo *bar* baz**
+.
+
foo bar baz

+````````````````````````````````
+
+
+But note:
+
+```````````````````````````````` example
+**foo*bar*baz**
+.
+
foobarbaz**

+````````````````````````````````
+
+
+The difference is that in the preceding case, the internal delimiters
+[can close emphasis], while in the cases with spaces, they cannot.
+
+```````````````````````````````` example
+***foo* bar**
+.
+
foo bar

+````````````````````````````````
+
+
+```````````````````````````````` example
+**foo *bar***
+.
+
foo bar

+````````````````````````````````
+
+
+Indefinite levels of nesting are possible:
+
+```````````````````````````````` example
+**foo *bar **baz**
+bim* bop**
+.
+
foo bar baz
+bim bop

+````````````````````````````````
+
+
+```````````````````````````````` example
+**foo [*bar*](/url)**
+.
+
foo bar

+````````````````````````````````
+
+
+There can be no empty emphasis or strong emphasis:
+
+```````````````````````````````` example
+__ is not an empty emphasis
+.
+
__ is not an empty emphasis

+````````````````````````````````
+
+
+```````````````````````````````` example
+____ is not an empty strong emphasis
+.
+
____ is not an empty strong emphasis

+````````````````````````````````
+
+
+
+Rule 11:
+
+```````````````````````````````` example
+foo ***
+.
+
foo ***

+````````````````````````````````
+
+
+```````````````````````````````` example
+foo *\**
+.
+
foo *

+````````````````````````````````
+
+
+```````````````````````````````` example
+foo *_*
+.
+
foo _

+````````````````````````````````
+
+
+```````````````````````````````` example
+foo *****
+.
+
foo *****

+````````````````````````````````
+
+
+```````````````````````````````` example
+foo **\***
+.
+
foo *

+````````````````````````````````
+
+
+```````````````````````````````` example
+foo **_**
+.
+
foo _

+````````````````````````````````
+
+
+Note that when delimiters do not match evenly, Rule 11 determines
+that the excess literal `*` characters will appear outside of the
+emphasis, rather than inside it:
+
+```````````````````````````````` example
+**foo*
+.
+
*foo

+````````````````````````````````
+
+
+```````````````````````````````` example
+*foo**
+.
+
foo*

+````````````````````````````````
+
+
+```````````````````````````````` example
+***foo**
+.
+
*foo

+````````````````````````````````
+
+
+```````````````````````````````` example
+****foo*
+.
+
***foo

+````````````````````````````````
+
+
+```````````````````````````````` example
+**foo***
+.
+
foo*

+````````````````````````````````
+
+
+```````````````````````````````` example
+*foo****
+.
+
foo***

+````````````````````````````````
+
+
+
+Rule 12:
+
+```````````````````````````````` example
+foo ___
+.
+
foo ___

+````````````````````````````````
+
+
+```````````````````````````````` example
+foo _\__
+.
+
foo _

+````````````````````````````````
+
+
+```````````````````````````````` example
+foo _*_
+.
+
foo *

+````````````````````````````````
+
+
+```````````````````````````````` example
+foo _____
+.
+
foo _____

+````````````````````````````````
+
+
+```````````````````````````````` example
+foo __\___
+.
+
foo _

+````````````````````````````````
+
+
+```````````````````````````````` example
+foo __*__
+.
+
foo *

+````````````````````````````````
+
+
+```````````````````````````````` example
+__foo_
+.
+
_foo

+````````````````````````````````
+
+
+Note that when delimiters do not match evenly, Rule 12 determines
+that the excess literal `_` characters will appear outside of the
+emphasis, rather than inside it:
+
+```````````````````````````````` example
+_foo__
+.
+
foo_

+````````````````````````````````
+
+
+```````````````````````````````` example
+___foo__
+.
+
_foo

+````````````````````````````````
+
+
+```````````````````````````````` example
+____foo_
+.
+
___foo

+````````````````````````````````
+
+
+```````````````````````````````` example
+__foo___
+.
+
foo_

+````````````````````````````````
+
+
+```````````````````````````````` example
+_foo____
+.
+
foo___

+````````````````````````````````
+
+
+Rule 13 implies that if you want emphasis nested directly inside
+emphasis, you must use different delimiters:
+
+```````````````````````````````` example
+**foo**
+.
+
foo

+````````````````````````````````
+
+
+```````````````````````````````` example
+*_foo_*
+.
+
foo

+````````````````````````````````
+
+
+```````````````````````````````` example
+__foo__
+.
+
foo

+````````````````````````````````
+
+
+```````````````````````````````` example
+_*foo*_
+.
+
foo

+````````````````````````````````
+
+
+However, strong emphasis within strong emphasis is possible without
+switching delimiters:
+
+```````````````````````````````` example
+****foo****
+.
+
foo

+````````````````````````````````
+
+
+```````````````````````````````` example
+____foo____
+.
+
foo

+````````````````````````````````
+
+
+
+Rule 13 can be applied to arbitrarily long sequences of
+delimiters:
+
+```````````````````````````````` example
+******foo******
+.
+
foo

+````````````````````````````````
+
+
+Rule 14:
+
+```````````````````````````````` example
+***foo***
+.
+
foo

+````````````````````````````````
+
+
+```````````````````````````````` example
+_____foo_____
+.
+
foo

+````````````````````````````````
+
+
+Rule 15:
+
+```````````````````````````````` example
+*foo _bar* baz_
+.
+
foo _bar baz_

+````````````````````````````````
+
+
+```````````````````````````````` example
+**foo*bar**
+.
+
foobar*

+````````````````````````````````
+
+
+```````````````````````````````` example
+*foo __bar *baz bim__ bam*
+.
+
foo bar *baz bim bam

+````````````````````````````````
+
+
+Rule 16:
+
+```````````````````````````````` example
+**foo **bar baz**
+.
+
**foo bar baz

+````````````````````````````````
+
+
+```````````````````````````````` example
+*foo *bar baz*
+.
+
*foo bar baz

+````````````````````````````````
+
+
+Rule 17:
+
+```````````````````````````````` example
+*[bar*](/url)
+.
+
*bar*

+````````````````````````````````
+
+
+```````````````````````````````` example
+_foo [bar_](/url)
+.
+
_foo bar_

+````````````````````````````````
+
+
+```````````````````````````````` example
+*
+.
+
*

+````````````````````````````````
+
+
+```````````````````````````````` example
+**
+.
+
**

+````````````````````````````````
+
+
+```````````````````````````````` example
+__
+.
+
__

+````````````````````````````````
+
+
+```````````````````````````````` example
+*a `*`*
+.
+
a *

+````````````````````````````````
+
+
+```````````````````````````````` example
+_a `_`_
+.
+
a _

+````````````````````````````````
+
+
+```````````````````````````````` example
+**a
+.
+
**ahttp://foo.bar/?q=**

+````````````````````````````````
+
+
+```````````````````````````````` example
+__a
+.
+
__ahttp://foo.bar/?q=__

+````````````````````````````````
+
+
+
+## Links
+
+A link contains [link text] (the visible text), a [link destination]
+(the URI that is the link destination), and optionally a [link title].
+There are two basic kinds of links in Markdown.  In [inline links] the
+destination and title are given immediately after the link text.  In
+[reference links] the destination and title are defined elsewhere in
+the document.
+
+A [link text](@) consists of a sequence of zero or more
+inline elements enclosed by square brackets (`[` and `]`).  The
+following rules apply:
+
+- Links may not contain other links, at any level of nesting. If
+  multiple otherwise valid link definitions appear nested inside each
+  other, the inner-most definition is used.
+
+- Brackets are allowed in the [link text] only if (a) they
+  are backslash-escaped or (b) they appear as a matched pair of brackets,
+  with an open bracket `[`, a sequence of zero or more inlines, and
+  a close bracket `]`.
+
+- Backtick [code spans], [autolinks], and raw [HTML tags] bind more tightly
+  than the brackets in link text.  Thus, for example,
+  `` [foo`]` `` could not be a link text, since the second `]`
+  is part of a code span.
+
+- The brackets in link text bind more tightly than markers for
+  [emphasis and strong emphasis]. Thus, for example, `*[foo*](url)` is a link.
+
+A [link destination](@) consists of either
+
+- a sequence of zero or more characters between an opening `<` and a
+  closing `>` that contains no spaces, line breaks, or unescaped
+  `<` or `>` characters, or
+
+- a nonempty sequence of characters that does not include
+  ASCII space or control characters, and includes parentheses
+  only if (a) they are backslash-escaped or (b) they are part of
+  a balanced pair of unescaped parentheses that is not itself
+  inside a balanced pair of unescaped parentheses.
+
+A [link title](@)  consists of either
+
+- a sequence of zero or more characters between straight double-quote
+  characters (`"`), including a `"` character only if it is
+  backslash-escaped, or
+
+- a sequence of zero or more characters between straight single-quote
+  characters (`'`), including a `'` character only if it is
+  backslash-escaped, or
+
+- a sequence of zero or more characters between matching parentheses
+  (`(...)`), including a `)` character only if it is backslash-escaped.
+
+Although [link titles] may span multiple lines, they may not contain
+a [blank line].
+
+An [inline link](@) consists of a [link text] followed immediately
+by a left parenthesis `(`, optional [whitespace], an optional
+[link destination], an optional [link title] separated from the link
+destination by [whitespace], optional [whitespace], and a right
+parenthesis `)`. The link's text consists of the inlines contained
+in the [link text] (excluding the enclosing square brackets).
+The link's URI consists of the link destination, excluding enclosing
+`<...>` if present, with backslash-escapes in effect as described
+above.  The link's title consists of the link title, excluding its
+enclosing delimiters, with backslash-escapes in effect as described
+above.
+
+Here is a simple inline link:
+
+```````````````````````````````` example
+[link](/uri "title")
+.
+
link

+````````````````````````````````
+
+
+The title may be omitted:
+
+```````````````````````````````` example
+[link](/uri)
+.
+
link

+````````````````````````````````
+
+
+Both the title and the destination may be omitted:
+
+```````````````````````````````` example
+[link]()
+.
+
link

+````````````````````````````````
+
+
+```````````````````````````````` example
+[link](<>)
+.
+
link

+````````````````````````````````
+
+
+The destination cannot contain spaces or line breaks,
+even if enclosed in pointy brackets:
+
+```````````````````````````````` example
+[link](/my uri)
+.
+
[link](/my uri)

+````````````````````````````````
+
+
+```````````````````````````````` example
+[link]()
+.
+
[link](</my uri>)

+````````````````````````````````
+
+
+```````````````````````````````` example
+[link](foo
+bar)
+.
+
[link](foo
+bar)

+````````````````````````````````
+
+
+```````````````````````````````` example
+[link]()
+.
+
[link]()

+````````````````````````````````
+
+Parentheses inside the link destination may be escaped:
+
+```````````````````````````````` example
+[link](\(foo\))
+.
+
link

+````````````````````````````````
+
+One level of balanced parentheses is allowed without escaping:
+
+```````````````````````````````` example
+[link]((foo)and(bar))
+.
+
link

+````````````````````````````````
+
+However, if you have parentheses within parentheses, you need to escape
+or use the `<...>` form:
+
+```````````````````````````````` example
+[link](foo(and(bar)))
+.
+
[link](foo(and(bar)))

+````````````````````````````````
+
+
+```````````````````````````````` example
+[link](foo(and\(bar\)))
+.
+
link

+````````````````````````````````
+
+
+```````````````````````````````` example
+[link]()
+.
+
link

+````````````````````````````````
+
+
+Parentheses and other symbols can also be escaped, as usual
+in Markdown:
+
+```````````````````````````````` example
+[link](foo\)\:)
+.
+
link

+````````````````````````````````
+
+
+A link can contain fragment identifiers and queries:
+
+```````````````````````````````` example
+[link](#fragment)
+
+[link](http://example.com#fragment)
+
+[link](http://example.com?foo=3#frag)
+.
+
link

+
link

+
link

+````````````````````````````````
+
+
+Note that a backslash before a non-escapable character is
+just a backslash:
+
+```````````````````````````````` example
+[link](foo\bar)
+.
+
link

+````````````````````````````````
+
+
+URL-escaping should be left alone inside the destination, as all
+URL-escaped characters are also valid URL characters. Entity and
+numerical character references in the destination will be parsed
+into the corresponding Unicode code points, as usual.  These may
+be optionally URL-escaped when written as HTML, but this spec
+does not enforce any particular policy for rendering URLs in
+HTML or other formats.  Renderers may make different decisions
+about how to escape or normalize URLs in the output.
+
+```````````````````````````````` example
+[link](foo%20bä)
+.
+
link

+````````````````````````````````
+
+
+Note that, because titles can often be parsed as destinations,
+if you try to omit the destination and keep the title, you'll
+get unexpected results:
+
+```````````````````````````````` example
+[link]("title")
+.
+
link

+````````````````````````````````
+
+
+Titles may be in single quotes, double quotes, or parentheses:
+
+```````````````````````````````` example
+[link](/url "title")
+[link](/url 'title')
+[link](/url (title))
+.
+
link
+link
+link

+````````````````````````````````
+
+
+Backslash escapes and entity and numeric character references
+may be used in titles:
+
+```````````````````````````````` example
+[link](/url "title \""")
+.
+
link

+````````````````````````````````
+
+
+Nested balanced quotes are not allowed without escaping:
+
+```````````````````````````````` example
+[link](/url "title "and" title")
+.
+
[link](/url "title "and" title")

+````````````````````````````````
+
+
+But it is easy to work around this by using a different quote type:
+
+```````````````````````````````` example
+[link](/url 'title "and" title')
+.
+
link

+````````````````````````````````
+
+
+(Note:  `Markdown.pl` did allow double quotes inside a double-quoted
+title, and its test suite included a test demonstrating this.
+But it is hard to see a good rationale for the extra complexity this
+brings, since there are already many ways---backslash escaping,
+entity and numeric character references, or using a different
+quote type for the enclosing title---to write titles containing
+double quotes.  `Markdown.pl`'s handling of titles has a number
+of other strange features.  For example, it allows single-quoted
+titles in inline links, but not reference links.  And, in
+reference links but not inline links, it allows a title to begin
+with `"` and end with `)`.  `Markdown.pl` 1.0.1 even allows
+titles with no closing quotation mark, though 1.0.2b8 does not.
+It seems preferable to adopt a simple, rational rule that works
+the same way in inline links and link reference definitions.)
+
+[Whitespace] is allowed around the destination and title:
+
+```````````````````````````````` example
+[link](   /uri
+  "title"  )
+.
+
link

+````````````````````````````````
+
+
+But it is not allowed between the link text and the
+following parenthesis:
+
+```````````````````````````````` example
+[link] (/uri)
+.
+
[link] (/uri)

+````````````````````````````````
+
+
+The link text may contain balanced brackets, but not unbalanced ones,
+unless they are escaped:
+
+```````````````````````````````` example
+[link [foo [bar]]](/uri)
+.
+
link [foo [bar]]

+````````````````````````````````
+
+
+```````````````````````````````` example
+[link] bar](/uri)
+.
+
[link] bar](/uri)

+````````````````````````````````
+
+
+```````````````````````````````` example
+[link [bar](/uri)
+.
+
[link bar

+````````````````````````````````
+
+
+```````````````````````````````` example
+[link \[bar](/uri)
+.
+
link [bar

+````````````````````````````````
+
+
+The link text may contain inline content:
+
+```````````````````````````````` example
+[link *foo **bar** `#`*](/uri)
+.
+
link foo bar #

+````````````````````````````````
+
+
+```````````````````````````````` example
+[![moon](moon.jpg)](/uri)
+.
+
moon

+````````````````````````````````
+
+
+However, links may not contain other links, at any level of nesting.
+
+```````````````````````````````` example
+[foo [bar](/uri)](/uri)
+.
+
[foo bar](/uri)

+````````````````````````````````
+
+
+```````````````````````````````` example
+[foo *[bar [baz](/uri)](/uri)*](/uri)
+.
+
[foo [bar baz](/uri)](/uri)

+````````````````````````````````
+
+
+```````````````````````````````` example
+![[[foo](uri1)](uri2)](uri3)
+.
+
[foo](uri2)

+````````````````````````````````
+
+
+These cases illustrate the precedence of link text grouping over
+emphasis grouping:
+
+```````````````````````````````` example
+*[foo*](/uri)
+.
+
*foo*

+````````````````````````````````
+
+
+```````````````````````````````` example
+[foo *bar](baz*)
+.
+
foo *bar

+````````````````````````````````
+
+
+Note that brackets that *aren't* part of links do not take
+precedence:
+
+```````````````````````````````` example
+*foo [bar* baz]
+.
+
foo [bar baz]

+````````````````````````````````
+
+
+These cases illustrate the precedence of HTML tags, code spans,
+and autolinks over link grouping:
+
+```````````````````````````````` example
+[foo 
+.
+
[foo 

+````````````````````````````````
+
+
+```````````````````````````````` example
+[foo`](/uri)`
+.
+
[foo](/uri)

+````````````````````````````````
+
+
+```````````````````````````````` example
+[foo
+.
+
[foohttp://example.com/?search=](uri)

+````````````````````````````````
+
+
+There are three kinds of [reference link](@)s:
+[full](#full-reference-link), [collapsed](#collapsed-reference-link),
+and [shortcut](#shortcut-reference-link).
+
+A [full reference link](@)
+consists of a [link text] immediately followed by a [link label]
+that [matches] a [link reference definition] elsewhere in the document.
+
+A [link label](@)  begins with a left bracket (`[`) and ends
+with the first right bracket (`]`) that is not backslash-escaped.
+Between these brackets there must be at least one [non-whitespace character].
+Unescaped square bracket characters are not allowed in
+[link labels].  A link label can have at most 999
+characters inside the square brackets.
+
+One label [matches](@)
+another just in case their normalized forms are equal.  To normalize a
+label, perform the *Unicode case fold* and collapse consecutive internal
+[whitespace] to a single space.  If there are multiple
+matching reference link definitions, the one that comes first in the
+document is used.  (It is desirable in such cases to emit a warning.)
+
+The contents of the first link label are parsed as inlines, which are
+used as the link's text.  The link's URI and title are provided by the
+matching [link reference definition].
+
+Here is a simple example:
+
+```````````````````````````````` example
+[foo][bar]
+
+[bar]: /url "title"
+.
+
foo

+````````````````````````````````
+
+
+The rules for the [link text] are the same as with
+[inline links].  Thus:
+
+The link text may contain balanced brackets, but not unbalanced ones,
+unless they are escaped:
+
+```````````````````````````````` example
+[link [foo [bar]]][ref]
+
+[ref]: /uri
+.
+
link [foo [bar]]

+````````````````````````````````
+
+
+```````````````````````````````` example
+[link \[bar][ref]
+
+[ref]: /uri
+.
+
link [bar

+````````````````````````````````
+
+
+The link text may contain inline content:
+
+```````````````````````````````` example
+[link *foo **bar** `#`*][ref]
+
+[ref]: /uri
+.
+
link foo bar #

+````````````````````````````````
+
+
+```````````````````````````````` example
+[![moon](moon.jpg)][ref]
+
+[ref]: /uri
+.
+
moon

+````````````````````````````````
+
+
+However, links may not contain other links, at any level of nesting.
+
+```````````````````````````````` example
+[foo [bar](/uri)][ref]
+
+[ref]: /uri
+.
+
[foo bar]ref

+````````````````````````````````
+
+
+```````````````````````````````` example
+[foo *bar [baz][ref]*][ref]
+
+[ref]: /uri
+.
+
[foo bar baz]ref

+````````````````````````````````
+
+
+(In the examples above, we have two [shortcut reference links]
+instead of one [full reference link].)
+
+The following cases illustrate the precedence of link text grouping over
+emphasis grouping:
+
+```````````````````````````````` example
+*[foo*][ref]
+
+[ref]: /uri
+.
+
*foo*

+````````````````````````````````
+
+
+```````````````````````````````` example
+[foo *bar][ref]
+
+[ref]: /uri
+.
+
foo *bar

+````````````````````````````````
+
+
+These cases illustrate the precedence of HTML tags, code spans,
+and autolinks over link grouping:
+
+```````````````````````````````` example
+[foo 
+
+[ref]: /uri
+.
+
[foo 

+````````````````````````````````
+
+
+```````````````````````````````` example
+[foo`][ref]`
+
+[ref]: /uri
+.
+
[foo][ref]

+````````````````````````````````
+
+
+```````````````````````````````` example
+[foo
+
+[ref]: /uri
+.
+
[foohttp://example.com/?search=][ref]

+````````````````````````````````
+
+
+Matching is case-insensitive:
+
+```````````````````````````````` example
+[foo][BaR]
+
+[bar]: /url "title"
+.
+
foo

+````````````````````````````````
+
+
+Unicode case fold is used:
+
+```````````````````````````````` example
+[Толпой][Толпой] is a Russian word.
+
+[ТОЛПОЙ]: /url
+.
+
Толпой is a Russian word.

+````````````````````````````````
+
+
+Consecutive internal [whitespace] is treated as one space for
+purposes of determining matching:
+
+```````````````````````````````` example
+[Foo
+  bar]: /url
+
+[Baz][Foo bar]
+.
+
Baz

+````````````````````````````````
+
+
+No [whitespace] is allowed between the [link text] and the
+[link label]:
+
+```````````````````````````````` example
+[foo] [bar]
+
+[bar]: /url "title"
+.
+
[foo] bar

+````````````````````````````````
+
+
+```````````````````````````````` example
+[foo]
+[bar]
+
+[bar]: /url "title"
+.
+
[foo]
+bar

+````````````````````````````````
+
+
+This is a departure from John Gruber's original Markdown syntax
+description, which explicitly allows whitespace between the link
+text and the link label.  It brings reference links in line with
+[inline links], which (according to both original Markdown and
+this spec) cannot have whitespace after the link text.  More
+importantly, it prevents inadvertent capture of consecutive
+[shortcut reference links]. If whitespace is allowed between the
+link text and the link label, then in the following we will have
+a single reference link, not two shortcut reference links, as
+intended:
+
+``` markdown
+[foo]
+[bar]
+
+[foo]: /url1
+[bar]: /url2
+```
+
+(Note that [shortcut reference links] were introduced by Gruber
+himself in a beta version of `Markdown.pl`, but never included
+in the official syntax description.  Without shortcut reference
+links, it is harmless to allow space between the link text and
+link label; but once shortcut references are introduced, it is
+too dangerous to allow this, as it frequently leads to
+unintended results.)
+
+When there are multiple matching [link reference definitions],
+the first is used:
+
+```````````````````````````````` example
+[foo]: /url1
+
+[foo]: /url2
+
+[bar][foo]
+.
+
bar

+````````````````````````````````
+
+
+Note that matching is performed on normalized strings, not parsed
+inline content.  So the following does not match, even though the
+labels define equivalent inline content:
+
+```````````````````````````````` example
+[bar][foo\!]
+
+[foo!]: /url
+.
+
[bar][foo!]

+````````````````````````````````
+
+
+[Link labels] cannot contain brackets, unless they are
+backslash-escaped:
+
+```````````````````````````````` example
+[foo][ref[]
+
+[ref[]: /uri
+.
+
[foo][ref[]

+
[ref[]: /uri

+````````````````````````````````
+
+
+```````````````````````````````` example
+[foo][ref[bar]]
+
+[ref[bar]]: /uri
+.
+
[foo][ref[bar]]

+
[ref[bar]]: /uri

+````````````````````````````````
+
+
+```````````````````````````````` example
+[[[foo]]]
+
+[[[foo]]]: /url
+.
+
[[[foo]]]

+
[[[foo]]]: /url

+````````````````````````````````
+
+
+```````````````````````````````` example
+[foo][ref\[]
+
+[ref\[]: /uri
+.
+
foo

+````````````````````````````````
+
+
+Note that in this example `]` is not backslash-escaped:
+
+```````````````````````````````` example
+[bar\\]: /uri
+
+[bar\\]
+.
+
bar\

+````````````````````````````````
+
+
+A [link label] must contain at least one [non-whitespace character]:
+
+```````````````````````````````` example
+[]
+
+[]: /uri
+.
+
[]

+
[]: /uri

+````````````````````````````````
+
+
+```````````````````````````````` example
+[
+ ]
+
+[
+ ]: /uri
+.
+
[
+]

+
[
+]: /uri

+````````````````````````````````
+
+
+A [collapsed reference link](@)
+consists of a [link label] that [matches] a
+[link reference definition] elsewhere in the
+document, followed by the string `[]`.
+The contents of the first link label are parsed as inlines,
+which are used as the link's text.  The link's URI and title are
+provided by the matching reference link definition.  Thus,
+`[foo][]` is equivalent to `[foo][foo]`.
+
+```````````````````````````````` example
+[foo][]
+
+[foo]: /url "title"
+.
+
foo

+````````````````````````````````
+
+
+```````````````````````````````` example
+[*foo* bar][]
+
+[*foo* bar]: /url "title"
+.
+
foo bar

+````````````````````````````````
+
+
+The link labels are case-insensitive:
+
+```````````````````````````````` example
+[Foo][]
+
+[foo]: /url "title"
+.
+
Foo

+````````````````````````````````
+
+
+
+As with full reference links, [whitespace] is not
+allowed between the two sets of brackets:
+
+```````````````````````````````` example
+[foo] 
+[]
+
+[foo]: /url "title"
+.
+
foo
+[]

+````````````````````````````````
+
+
+A [shortcut reference link](@)
+consists of a [link label] that [matches] a
+[link reference definition] elsewhere in the
+document and is not followed by `[]` or a link label.
+The contents of the first link label are parsed as inlines,
+which are used as the link's text.  the link's URI and title
+are provided by the matching link reference definition.
+Thus, `[foo]` is equivalent to `[foo][]`.
+
+```````````````````````````````` example
+[foo]
+
+[foo]: /url "title"
+.
+
foo

+````````````````````````````````
+
+
+```````````````````````````````` example
+[*foo* bar]
+
+[*foo* bar]: /url "title"
+.
+
foo bar

+````````````````````````````````
+
+
+```````````````````````````````` example
+[[*foo* bar]]
+
+[*foo* bar]: /url "title"
+.
+
[foo bar]

+````````````````````````````````
+
+
+```````````````````````````````` example
+[[bar [foo]
+
+[foo]: /url
+.
+
[[bar foo

+````````````````````````````````
+
+
+The link labels are case-insensitive:
+
+```````````````````````````````` example
+[Foo]
+
+[foo]: /url "title"
+.
+
Foo

+````````````````````````````````
+
+
+A space after the link text should be preserved:
+
+```````````````````````````````` example
+[foo] bar
+
+[foo]: /url
+.
+
foo bar

+````````````````````````````````
+
+
+If you just want bracketed text, you can backslash-escape the
+opening bracket to avoid links:
+
+```````````````````````````````` example
+\[foo]
+
+[foo]: /url "title"
+.
+
[foo]

+````````````````````````````````
+
+
+Note that this is a link, because a link label ends with the first
+following closing bracket:
+
+```````````````````````````````` example
+[foo*]: /url
+
+*[foo*]
+.
+
*foo*

+````````````````````````````````
+
+
+Full references take precedence over shortcut references:
+
+```````````````````````````````` example
+[foo][bar]
+
+[foo]: /url1
+[bar]: /url2
+.
+
foo

+````````````````````````````````
+
+
+In the following case `[bar][baz]` is parsed as a reference,
+`[foo]` as normal text:
+
+```````````````````````````````` example
+[foo][bar][baz]
+
+[baz]: /url
+.
+
[foo]bar

+````````````````````````````````
+
+
+Here, though, `[foo][bar]` is parsed as a reference, since
+`[bar]` is defined:
+
+```````````````````````````````` example
+[foo][bar][baz]
+
+[baz]: /url1
+[bar]: /url2
+.
+
foobaz

+````````````````````````````````
+
+
+Here `[foo]` is not parsed as a shortcut reference, because it
+is followed by a link label (even though `[bar]` is not defined):
+
+```````````````````````````````` example
+[foo][bar][baz]
+
+[baz]: /url1
+[foo]: /url2
+.
+
[foo]bar

+````````````````````````````````
+
+
+
+## Images
+
+Syntax for images is like the syntax for links, with one
+difference. Instead of [link text], we have an
+[image description](@).  The rules for this are the
+same as for [link text], except that (a) an
+image description starts with `![` rather than `[`, and
+(b) an image description may contain links.
+An image description has inline elements
+as its contents.  When an image is rendered to HTML,
+this is standardly used as the image's `alt` attribute.
+
+```````````````````````````````` example
+![foo](/url "title")
+.
+
foo

+````````````````````````````````
+
+
+```````````````````````````````` example
+![foo *bar*]
+
+[foo *bar*]: train.jpg "train & tracks"
+.
+
foo bar

+````````````````````````````````
+
+
+```````````````````````````````` example
+![foo ![bar](/url)](/url2)
+.
+
foo bar

+````````````````````````````````
+
+
+```````````````````````````````` example
+![foo [bar](/url)](/url2)
+.
+
foo bar

+````````````````````````````````
+
+
+Though this spec is concerned with parsing, not rendering, it is
+recommended that in rendering to HTML, only the plain string content
+of the [image description] be used.  Note that in
+the above example, the alt attribute's value is `foo bar`, not `foo
+[bar](/url)` or `foo bar`.  Only the plain string
+content is rendered, without formatting.
+
+```````````````````````````````` example
+![foo *bar*][]
+
+[foo *bar*]: train.jpg "train & tracks"
+.
+
foo bar

+````````````````````````````````
+
+
+```````````````````````````````` example
+![foo *bar*][foobar]
+
+[FOOBAR]: train.jpg "train & tracks"
+.
+
foo bar

+````````````````````````````````
+
+
+```````````````````````````````` example
+![foo](train.jpg)
+.
+
foo

+````````````````````````````````
+
+
+```````````````````````````````` example
+My ![foo bar](/path/to/train.jpg  "title"   )
+.
+
My foo bar

+````````````````````````````````
+
+
+```````````````````````````````` example
+![foo]()
+.
+
foo

+````````````````````````````````
+
+
+```````````````````````````````` example
+![](/url)
+.
+

+````````````````````````````````
+
+
+Reference-style:
+
+```````````````````````````````` example
+![foo][bar]
+
+[bar]: /url
+.
+
foo

+````````````````````````````````
+
+
+```````````````````````````````` example
+![foo][bar]
+
+[BAR]: /url
+.
+
foo

+````````````````````````````````
+
+
+Collapsed:
+
+```````````````````````````````` example
+![foo][]
+
+[foo]: /url "title"
+.
+
foo

+````````````````````````````````
+
+
+```````````````````````````````` example
+![*foo* bar][]
+
+[*foo* bar]: /url "title"
+.
+
foo bar

+````````````````````````````````
+
+
+The labels are case-insensitive:
+
+```````````````````````````````` example
+![Foo][]
+
+[foo]: /url "title"
+.
+
Foo

+````````````````````````````````
+
+
+As with reference links, [whitespace] is not allowed
+between the two sets of brackets:
+
+```````````````````````````````` example
+![foo] 
+[]
+
+[foo]: /url "title"
+.
+
foo
+[]

+````````````````````````````````
+
+
+Shortcut:
+
+```````````````````````````````` example
+![foo]
+
+[foo]: /url "title"
+.
+
foo

+````````````````````````````````
+
+
+```````````````````````````````` example
+![*foo* bar]
+
+[*foo* bar]: /url "title"
+.
+
foo bar

+````````````````````````````````
+
+
+Note that link labels cannot contain unescaped brackets:
+
+```````````````````````````````` example
+![[foo]]
+
+[[foo]]: /url "title"
+.
+
![[foo]]

+
[[foo]]: /url "title"

+````````````````````````````````
+
+
+The link labels are case-insensitive:
+
+```````````````````````````````` example
+![Foo]
+
+[foo]: /url "title"
+.
+
Foo

+````````````````````````````````
+
+
+If you just want bracketed text, you can backslash-escape the
+opening `!` and `[`:
+
+```````````````````````````````` example
+\!\[foo]
+
+[foo]: /url "title"
+.
+
![foo]

+````````````````````````````````
+
+
+If you want a link after a literal `!`, backslash-escape the
+`!`:
+
+```````````````````````````````` example
+\![foo]
+
+[foo]: /url "title"
+.
+
!foo

+````````````````````````````````
+
+
+## Autolinks
+
+[Autolink](@)s are absolute URIs and email addresses inside
+`<` and `>`. They are parsed as links, with the URL or email address
+as the link label.
+
+A [URI autolink](@) consists of `<`, followed by an
+[absolute URI] not containing `<`, followed by `>`.  It is parsed as
+a link to the URI, with the URI as the link's label.
+
+An [absolute URI](@),
+for these purposes, consists of a [scheme] followed by a colon (`:`)
+followed by zero or more characters other than ASCII
+[whitespace] and control characters, `<`, and `>`.  If
+the URI includes these characters, they must be percent-encoded
+(e.g. `%20` for a space).
+
+For purposes of this spec, a [scheme](@) is any sequence
+of 2--32 characters beginning with an ASCII letter and followed
+by any combination of ASCII letters, digits, or the symbols plus
+("+"), period ("."), or hyphen ("-").
+
+Here are some valid autolinks:
+
+```````````````````````````````` example
+
+.
+
http://foo.bar.baz

+````````````````````````````````
+
+
+```````````````````````````````` example
+
+.
+
http://foo.bar.baz/test?q=hello&id=22&boolean

+````````````````````````````````
+
+
+```````````````````````````````` example
+
+.
+
irc://foo.bar:2233/baz

+````````````````````````````````
+
+
+Uppercase is also fine:
+
+```````````````````````````````` example
+
+.
+
MAILTO:FOO@BAR.BAZ

+````````````````````````````````
+
+
+Note that many strings that count as [absolute URIs] for
+purposes of this spec are not valid URIs, because their
+schemes are not registered or because of other problems
+with their syntax:
+
+```````````````````````````````` example
+
+.
+
a+b+c:d

+````````````````````````````````
+
+
+```````````````````````````````` example
+
+.
+
made-up-scheme://foo,bar

+````````````````````````````````
+
+
+```````````````````````````````` example
+
+.
+
http://../

+````````````````````````````````
+
+
+```````````````````````````````` example
+
+.
+
localhost:5001/foo

+````````````````````````````````
+
+
+Spaces are not allowed in autolinks:
+
+```````````````````````````````` example
+
+.
+
<http://foo.bar/baz bim>

+````````````````````````````````
+
+
+Backslash-escapes do not work inside autolinks:
+
+```````````````````````````````` example
+
+.
+
http://example.com/\[\

+````````````````````````````````
+
+
+An [email autolink](@)
+consists of `<`, followed by an [email address],
+followed by `>`.  The link's label is the email address,
+and the URL is `mailto:` followed by the email address.
+
+An [email address](@),
+for these purposes, is anything that matches
+the [non-normative regex from the HTML5
+spec](https://html.spec.whatwg.org/multipage/forms.html#e-mail-state-(type=email)):
+
+    /^[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?
+    (?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$/
+
+Examples of email autolinks:
+
+```````````````````````````````` example
+
+.
+
foo@bar.example.com

+````````````````````````````````
+
+
+```````````````````````````````` example
+
+.
+
foo+special@Bar.baz-bar0.com

+````````````````````````````````
+
+
+Backslash-escapes do not work inside email autolinks:
+
+```````````````````````````````` example
+
+.
+
<foo+@bar.example.com>

+````````````````````````````````
+
+
+These are not autolinks:
+
+```````````````````````````````` example
+<>
+.
+
<>

+````````````````````````````````
+
+
+```````````````````````````````` example
+< http://foo.bar >
+.
+
< http://foo.bar >

+````````````````````````````````
+
+
+```````````````````````````````` example
+
+.
+
<m:abc>

+````````````````````````````````
+
+
+```````````````````````````````` example
+
+.
+
<foo.bar.baz>

+````````````````````````````````
+
+
+```````````````````````````````` example
+http://example.com
+.
+
http://example.com

+````````````````````````````````
+
+
+```````````````````````````````` example
+foo@bar.example.com
+.
+
foo@bar.example.com

+````````````````````````````````
+
+
+## Raw HTML
+
+Text between `<` and `>` that looks like an HTML tag is parsed as a
+raw HTML tag and will be rendered in HTML without escaping.
+Tag and attribute names are not limited to current HTML tags,
+so custom tags (and even, say, DocBook tags) may be used.
+
+Here is the grammar for tags:
+
+A [tag name](@) consists of an ASCII letter
+followed by zero or more ASCII letters, digits, or
+hyphens (`-`).
+
+An [attribute](@) consists of [whitespace],
+an [attribute name], and an optional
+[attribute value specification].
+
+An [attribute name](@)
+consists of an ASCII letter, `_`, or `:`, followed by zero or more ASCII
+letters, digits, `_`, `.`, `:`, or `-`.  (Note:  This is the XML
+specification restricted to ASCII.  HTML5 is laxer.)
+
+An [attribute value specification](@)
+consists of optional [whitespace],
+a `=` character, optional [whitespace], and an [attribute
+value].
+
+An [attribute value](@)
+consists of an [unquoted attribute value],
+a [single-quoted attribute value], or a [double-quoted attribute value].
+
+An [unquoted attribute value](@)
+is a nonempty string of characters not
+including spaces, `"`, `'`, `=`, `<`, `>`, or `` ` ``.
+
+A [single-quoted attribute value](@)
+consists of `'`, zero or more
+characters not including `'`, and a final `'`.
+
+A [double-quoted attribute value](@)
+consists of `"`, zero or more
+characters not including `"`, and a final `"`.
+
+An [open tag](@) consists of a `<` character, a [tag name],
+zero or more [attributes], optional [whitespace], an optional `/`
+character, and a `>` character.
+
+A [closing tag](@) consists of the string ``.
+
+An [HTML comment](@) consists of ``,
+where *text* does not start with `>` or `->`, does not end with `-`,
+and does not contain `--`.  (See the
+[HTML5 spec](http://www.w3.org/TR/html5/syntax.html#comments).)
+
+A [processing instruction](@)
+consists of the string ``, and the string
+`?>`.
+
+A [declaration](@) consists of the
+string ``, and the character `>`.
+
+A [CDATA section](@) consists of
+the string ``, and the string `]]>`.
+
+An [HTML tag](@) consists of an [open tag], a [closing tag],
+an [HTML comment], a [processing instruction], a [declaration],
+or a [CDATA section].
+
+Here are some simple open tags:
+
+```````````````````````````````` example
+
+.
+

+````````````````````````````````
+
+
+Empty elements:
+
+```````````````````````````````` example
+
+.
+

+````````````````````````````````
+
+
+[Whitespace] is allowed:
+
+```````````````````````````````` example
+
+.
+

+````````````````````````````````
+
+
+With attributes:
+
+```````````````````````````````` example
+
+.
+

+````````````````````````````````
+
+
+Custom tag names can be used:
+
+```````````````````````````````` example
+Foo 
+.
+
Foo 

+````````````````````````````````
+
+
+Illegal tag names, not parsed as HTML:
+
+```````````````````````````````` example
+<33> <__>
+.
+
<33> <__>

+````````````````````````````````
+
+
+Illegal attribute names:
+
+```````````````````````````````` example
+
+.
+
<a h*#ref="hi">

+````````````````````````````````
+
+
+Illegal attribute values:
+
+```````````````````````````````` example
+
+.
+
</a href="foo">

+````````````````````````````````
+
+
+Comments:
+
+```````````````````````````````` example
+foo 
+.
+
foo 

+````````````````````````````````
+
+
+```````````````````````````````` example
+foo 
+.
+
foo <!-- not a comment -- two hyphens -->

+````````````````````````````````
+
+
+Not comments:
+
+```````````````````````````````` example
+foo  foo -->
+
+foo 
+.
+
foo <!--> foo -->

+
foo <!-- foo--->

+````````````````````````````````
+
+
+Processing instructions:
+
+```````````````````````````````` example
+foo 
+.
+
foo 

+````````````````````````````````
+
+
+Declarations:
+
+```````````````````````````````` example
+foo 
+.
+
foo 

+````````````````````````````````
+
+
+CDATA sections:
+
+```````````````````````````````` example
+foo &<]]>
+.
+
foo &<]]>

+````````````````````````````````
+
+
+Entity and numeric character references are preserved in HTML
+attributes:
+
+```````````````````````````````` example
+foo 
+.
+
foo 

+````````````````````````````````
+
+
+Backslash escapes do not work in HTML attributes:
+
+```````````````````````````````` example
+foo 
+.
+
foo 

+````````````````````````````````
+
+
+```````````````````````````````` example
+
+.
+
<a href=""">

+````````````````````````````````
+
+
+## Hard line breaks
+
+A line break (not in a code span or HTML tag) that is preceded
+by two or more spaces and does not occur at the end of a block
+is parsed as a [hard line break](@) (rendered
+in HTML as a `
` tag):
+
+```````````````````````````````` example
+foo  
+baz
+.
+
foo

+baz

+````````````````````````````````
+
+
+For a more visible alternative, a backslash before the
+[line ending] may be used instead of two spaces:
+
+```````````````````````````````` example
+foo\
+baz
+.
+
foo

+baz

+````````````````````````````````
+
+
+More than two spaces can be used:
+
+```````````````````````````````` example
+foo       
+baz
+.
+
foo

+baz

+````````````````````````````````
+
+
+Leading spaces at the beginning of the next line are ignored:
+
+```````````````````````````````` example
+foo  
+     bar
+.
+
foo

+bar

+````````````````````````````````
+
+
+```````````````````````````````` example
+foo\
+     bar
+.
+
foo

+bar

+````````````````````````````````
+
+
+Line breaks can occur inside emphasis, links, and other constructs
+that allow inline content:
+
+```````````````````````````````` example
+*foo  
+bar*
+.
+
foo

+bar

+````````````````````````````````
+
+
+```````````````````````````````` example
+*foo\
+bar*
+.
+
foo

+bar

+````````````````````````````````
+
+
+Line breaks do not occur inside code spans
+
+```````````````````````````````` example
+`code  
+span`
+.
+
code span

+````````````````````````````````
+
+
+```````````````````````````````` example
+`code\
+span`
+.
+
code\ span

+````````````````````````````````
+
+
+or HTML tags:
+
+```````````````````````````````` example
+
+.
+

+````````````````````````````````
+
+
+```````````````````````````````` example
+
+.
+

+````````````````````````````````
+
+
+Hard line breaks are for separating inline content within a block.
+Neither syntax for hard line breaks works at the end of a paragraph or
+other block element:
+
+```````````````````````````````` example
+foo\
+.
+
foo\

+````````````````````````````````
+
+
+```````````````````````````````` example
+foo  
+.
+
foo

+````````````````````````````````
+
+
+```````````````````````````````` example
+### foo\
+.
+
foo\

+````````````````````````````````
+
+
+```````````````````````````````` example
+### foo  
+.
+
foo

+````````````````````````````````
+
+
+## Soft line breaks
+
+A regular line break (not in a code span or HTML tag) that is not
+preceded by two or more spaces or a backslash is parsed as a
+softbreak.  (A softbreak may be rendered in HTML either as a
+[line ending] or as a space. The result will be the same in
+browsers. In the examples here, a [line ending] will be used.)
+
+```````````````````````````````` example
+foo
+baz
+.
+
foo
+baz

+````````````````````````````````
+
+
+Spaces at the end of the line and beginning of the next line are
+removed:
+
+```````````````````````````````` example
+foo 
+ baz
+.
+
foo
+baz

+````````````````````````````````
+
+
+A conforming parser may render a soft line break in HTML either as a
+line break or as a space.
+
+A renderer may also provide an option to render soft line breaks
+as hard line breaks.
+
+## Textual content
+
+Any characters not given an interpretation by the above rules will
+be parsed as plain textual content.
+
+```````````````````````````````` example
+hello $.;'there
+.
+
hello $.;'there

+````````````````````````````````
+
+
+```````````````````````````````` example
+Foo χρῆν
+.
+
Foo χρῆν

+````````````````````````````````
+
+
+Internal spaces are preserved verbatim:
+
+```````````````````````````````` example
+Multiple     spaces
+.
+
Multiple     spaces

+````````````````````````````````
+
+
+
+
+# Appendix: A parsing strategy
+
+In this appendix we describe some features of the parsing strategy
+used in the CommonMark reference implementations.
+
+## Overview
+
+Parsing has two phases:
+
+1. In the first phase, lines of input are consumed and the block
+structure of the document---its division into paragraphs, block quotes,
+list items, and so on---is constructed.  Text is assigned to these
+blocks but not parsed. Link reference definitions are parsed and a
+map of links is constructed.
+
+2. In the second phase, the raw text contents of paragraphs and headings
+are parsed into sequences of Markdown inline elements (strings,
+code spans, links, emphasis, and so on), using the map of link
+references constructed in phase 1.
+
+At each point in processing, the document is represented as a tree of
+**blocks**.  The root of the tree is a `document` block.  The `document`
+may have any number of other blocks as **children**.  These children
+may, in turn, have other blocks as children.  The last child of a block
+is normally considered **open**, meaning that subsequent lines of input
+can alter its contents.  (Blocks that are not open are **closed**.)
+Here, for example, is a possible document tree, with the open blocks
+marked by arrows:
+
+``` tree
+-> document
+  -> block_quote
+       paragraph
+         "Lorem ipsum dolor\nsit amet."
+    -> list (type=bullet tight=true bullet_char=-)
+         list_item
+           paragraph
+             "Qui *quodsi iracundia*"
+      -> list_item
+        -> paragraph
+             "aliquando id"
+```
+
+## Phase 1: block structure
+
+Each line that is processed has an effect on this tree.  The line is
+analyzed and, depending on its contents, the document may be altered
+in one or more of the following ways:
+
+1. One or more open blocks may be closed.
+2. One or more new blocks may be created as children of the
+   last open block.
+3. Text may be added to the last (deepest) open block remaining
+   on the tree.
+
+Once a line has been incorporated into the tree in this way,
+it can be discarded, so input can be read in a stream.
+
+For each line, we follow this procedure:
+
+1. First we iterate through the open blocks, starting with the
+root document, and descending through last children down to the last
+open block.  Each block imposes a condition that the line must satisfy
+if the block is to remain open.  For example, a block quote requires a
+`>` character.  A paragraph requires a non-blank line.
+In this phase we may match all or just some of the open
+blocks.  But we cannot close unmatched blocks yet, because we may have a
+[lazy continuation line].
+
+2.  Next, after consuming the continuation markers for existing
+blocks, we look for new block starts (e.g. `>` for a block quote.
+If we encounter a new block start, we close any blocks unmatched
+in step 1 before creating the new block as a child of the last
+matched block.
+
+3.  Finally, we look at the remainder of the line (after block
+markers like `>`, list markers, and indentation have been consumed).
+This is text that can be incorporated into the last open
+block (a paragraph, code block, heading, or raw HTML).
+
+Setext headings are formed when we see a line of a paragraph
+that is a [setext heading underline].
+
+Reference link definitions are detected when a paragraph is closed;
+the accumulated text lines are parsed to see if they begin with
+one or more reference link definitions.  Any remainder becomes a
+normal paragraph.
+
+We can see how this works by considering how the tree above is
+generated by four lines of Markdown:
+
+``` markdown
+> Lorem ipsum dolor
+sit amet.
+> - Qui *quodsi iracundia*
+> - aliquando id
+```
+
+At the outset, our document model is just
+
+``` tree
+-> document
+```
+
+The first line of our text,
+
+``` markdown
+> Lorem ipsum dolor
+```
+
+causes a `block_quote` block to be created as a child of our
+open `document` block, and a `paragraph` block as a child of
+the `block_quote`.  Then the text is added to the last open
+block, the `paragraph`:
+
+``` tree
+-> document
+  -> block_quote
+    -> paragraph
+         "Lorem ipsum dolor"
+```
+
+The next line,
+
+``` markdown
+sit amet.
+```
+
+is a "lazy continuation" of the open `paragraph`, so it gets added
+to the paragraph's text:
+
+``` tree
+-> document
+  -> block_quote
+    -> paragraph
+         "Lorem ipsum dolor\nsit amet."
+```
+
+The third line,
+
+``` markdown
+> - Qui *quodsi iracundia*
+```
+
+causes the `paragraph` block to be closed, and a new `list` block
+opened as a child of the `block_quote`.  A `list_item` is also
+added as a child of the `list`, and a `paragraph` as a child of
+the `list_item`.  The text is then added to the new `paragraph`:
+
+``` tree
+-> document
+  -> block_quote
+       paragraph
+         "Lorem ipsum dolor\nsit amet."
+    -> list (type=bullet tight=true bullet_char=-)
+      -> list_item
+        -> paragraph
+             "Qui *quodsi iracundia*"
+```
+
+The fourth line,
+
+``` markdown
+> - aliquando id
+```
+
+causes the `list_item` (and its child the `paragraph`) to be closed,
+and a new `list_item` opened up as child of the `list`.  A `paragraph`
+is added as a child of the new `list_item`, to contain the text.
+We thus obtain the final tree:
+
+``` tree
+-> document
+  -> block_quote
+       paragraph
+         "Lorem ipsum dolor\nsit amet."
+    -> list (type=bullet tight=true bullet_char=-)
+         list_item
+           paragraph
+             "Qui *quodsi iracundia*"
+      -> list_item
+        -> paragraph
+             "aliquando id"
+```
+
+## Phase 2: inline structure
+
+Once all of the input has been parsed, all open blocks are closed.
+
+We then "walk the tree," visiting every node, and parse raw
+string contents of paragraphs and headings as inlines.  At this
+point we have seen all the link reference definitions, so we can
+resolve reference links as we go.
+
+``` tree
+document
+  block_quote
+    paragraph
+      str "Lorem ipsum dolor"
+      softbreak
+      str "sit amet."
+    list (type=bullet tight=true bullet_char=-)
+      list_item
+        paragraph
+          str "Qui "
+          emph
+            str "quodsi iracundia"
+      list_item
+        paragraph
+          str "aliquando id"
+```
+
+Notice how the [line ending] in the first paragraph has
+been parsed as a `softbreak`, and the asterisks in the first list item
+have become an `emph`.
+
+### An algorithm for parsing nested emphasis and links
+
+By far the trickiest part of inline parsing is handling emphasis,
+strong emphasis, links, and images.  This is done using the following
+algorithm.
+
+When we're parsing inlines and we hit either
+
+- a run of `*` or `_` characters, or
+- a `[` or `![`
+
+we insert a text node with these symbols as its literal content, and we
+add a pointer to this text node to the [delimiter stack](@).
+
+The [delimiter stack] is a doubly linked list.  Each
+element contains a pointer to a text node, plus information about
+
+- the type of delimiter (`[`, `![`, `*`, `_`)
+- the number of delimiters,
+- whether the delimiter is "active" (all are active to start), and
+- whether the delimiter is a potential opener, a potential closer,
+  or both (which depends on what sort of characters precede
+  and follow the delimiters).
+
+When we hit a `]` character, we call the *look for link or image*
+procedure (see below).
+
+When we hit the end of the input, we call the *process emphasis*
+procedure (see below), with `stack_bottom` = NULL.
+
+#### *look for link or image*
+
+Starting at the top of the delimiter stack, we look backwards
+through the stack for an opening `[` or `![` delimiter.
+
+- If we don't find one, we return a literal text node `]`.
+
+- If we do find one, but it's not *active*, we remove the inactive
+  delimiter from the stack, and return a literal text node `]`.
+
+- If we find one and it's active, then we parse ahead to see if
+  we have an inline link/image, reference link/image, compact reference
+  link/image, or shortcut reference link/image.
+
+  + If we don't, then we remove the opening delimiter from the
+    delimiter stack and return a literal text node `]`.
+
+  + If we do, then
+
+    * We return a link or image node whose children are the inlines
+      after the text node pointed to by the opening delimiter.
+
+    * We run *process emphasis* on these inlines, with the `[` opener
+      as `stack_bottom`.
+
+    * We remove the opening delimiter.
+
+    * If we have a link (and not an image), we also set all
+      `[` delimiters before the opening delimiter to *inactive*.  (This
+      will prevent us from getting links within links.)
+
+#### *process emphasis*
+
+Parameter `stack_bottom` sets a lower bound to how far we
+descend in the [delimiter stack].  If it is NULL, we can
+go all the way to the bottom.  Otherwise, we stop before
+visiting `stack_bottom`.
+
+Let `current_position` point to the element on the [delimiter stack]
+just above `stack_bottom` (or the first element if `stack_bottom`
+is NULL).
+
+We keep track of the `openers_bottom` for each delimiter
+type (`*`, `_`).  Initialize this to `stack_bottom`.
+
+Then we repeat the following until we run out of potential
+closers:
+
+- Move `current_position` forward in the delimiter stack (if needed)
+  until we find the first potential closer with delimiter `*` or `_`.
+  (This will be the potential closer closest
+  to the beginning of the input -- the first one in parse order.)
+
+- Now, look back in the stack (staying above `stack_bottom` and
+  the `openers_bottom` for this delimiter type) for the
+  first matching potential opener ("matching" means same delimiter).
+
+- If one is found:
+
+  + Figure out whether we have emphasis or strong emphasis:
+    if both closer and opener spans have length >= 2, we have
+    strong, otherwise regular.
+
+  + Insert an emph or strong emph node accordingly, after
+    the text node corresponding to the opener.
+
+  + Remove any delimiters between the opener and closer from
+    the delimiter stack.
+
+  + Remove 1 (for regular emph) or 2 (for strong emph) delimiters
+    from the opening and closing text nodes.  If they become empty
+    as a result, remove them and remove the corresponding element
+    of the delimiter stack.  If the closing node is removed, reset
+    `current_position` to the next element in the stack.
+
+- If none in found:
+
+  + Set `openers_bottom` to the element before `current_position`.
+    (We know that there are no openers for this kind of closer up to and
+    including this point, so this puts a lower bound on future searches.)
+
+  + If the closer at `current_position` is not a potential opener,
+    remove it from the delimiter stack (since we know it can't
+    be a closer either).
+
+  + Advance `current_position` to the next element in the stack.
+
+After we're done, we remove all delimiters above `stack_bottom` from the
+delimiter stack.
+
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/performance/SumOfInverses.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/performance/SumOfInverses.scl
new file mode 100644
index 000000000..a3f940760
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/performance/SumOfInverses.scl
@@ -0,0 +1,8 @@
+import "Prelude"
+
+main :: Double -> Double
+main m = loop 1 0
+  where
+    loop x cur = if x > m
+                 then cur
+                 else loop (x+1) (cur + 1 / x)
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/performance/SumOfInverses2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/performance/SumOfInverses2.scl
new file mode 100644
index 000000000..7745029bf
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/performance/SumOfInverses2.scl
@@ -0,0 +1,4 @@
+import "Prelude"
+
+main :: Integer -> Double
+main m = sum [1/(fromInteger x) | x <- [1..m]]
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/AmbiguousType.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/AmbiguousType.scl
new file mode 100644
index 000000000..f4871d7d5
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/AmbiguousType.scl
@@ -0,0 +1,12 @@
+
+
+class Show a where
+    show :: a -> String
+    
+class Read a where
+    read :: String -> a
+  
+combine x = show (read x)
+--
+9:13-9:17: Constrain Show a contains free variables not mentioned in the type of the value.
+9:19-9:23: Constrain Read a contains free variables not mentioned in the type of the value.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ApplicationOfNunfunction.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ApplicationOfNunfunction.scl
new file mode 100644
index 000000000..13167bee6
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ApplicationOfNunfunction.scl
@@ -0,0 +1,3 @@
+main = "Hello" "world!"
+--
+1:8-1:24: Application of non-function.
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Arity1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Arity1.scl
new file mode 100644
index 000000000..fa48e47c1
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Arity1.scl
@@ -0,0 +1,9 @@
+
+data Foo a = Foo a
+
+f :: Foo a -> a
+f = \Foo a -> a 
+
+main = "Not to be executed"
+--
+5:6-5:11: Arity is 1 but 2 patterns have been given.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/AsPattern.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/AsPattern.scl
new file mode 100644
index 000000000..0e3337099
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/AsPattern.scl
@@ -0,0 +1,8 @@
+import "JavaBuiltin" as Java
+
+sort p@(a,b) | Java.icmpgt a b = (b,a)
+             | True            = p
+               
+main = sort (sort (2 :: Integer,1 :: Integer))
+--
+(1,2)
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/BigContext.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/BigContext.scl
new file mode 100644
index 000000000..fc70cc23c
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/BigContext.scl
@@ -0,0 +1,28 @@
+import "Prelude"
+
+main = do 
+  a1 = 1
+  a2 = 1
+  a3 = 1
+  a4 = 1
+  a5 = 1
+  a6 = 1
+  a7 = 1
+  a8 = 1
+  a9 = 1
+  a10 = 1
+  a11 = 1
+  a12 = 1
+  a13 = 1
+  a14 = 1
+  a15 = 1
+  a16 = 1
+  a17 = 1
+  a18 = 1
+  a19 = 1
+  a20 = 1
+  f x = x + a1 + a2 + a3 + a4 + a5 + a6 + a7 + a8 + a9 + a10 + 
+            a11 + a12 + a13 + a14 + a15 + a16 + a17 + a18 + a19 + a20
+  f 10
+--
+30
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/BigFunction.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/BigFunction.scl
new file mode 100644
index 000000000..102bc1d7e
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/BigFunction.scl
@@ -0,0 +1,7 @@
+import "Prelude"
+
+f a b c d e f g h = a + b + c + d + e + f + g + h
+
+main = (id f) 1 2 3 4 5 6 7 8
+--
+36
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/BigInstances.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/BigInstances.scl
new file mode 100644
index 000000000..c64b1b703
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/BigInstances.scl
@@ -0,0 +1,16 @@
+
+class Foo a where
+    foo :: a
+    
+class (Foo a) => Bar a where
+    bar :: a
+    
+instance Bar Double where
+    foo x = x
+    bar x = x
+    
+main = foo 3.0 + bar 4.0
+--
+7.0
+
+    
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/BinaryOperators1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/BinaryOperators1.scl
new file mode 100644
index 000000000..86263c9aa
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/BinaryOperators1.scl
@@ -0,0 +1,25 @@
+import "Prelude"
+
+data E = E String
+
+instance Additive E where
+    zero = E "0"
+    E a + E b = E ("(" + a + "+" + b + ")")
+    
+instance Ring E where
+    one = E "1"
+    neg (E a) = E ("(-" + a + ")")
+    E a - E b = E ("(" + a + "-" + b + ")")
+    E a * E b = E ("(" + a + "*" + b + ")") 
+    fromInteger x = E (show x)
+
+eToString (E a) = a
+
+a = E "a"
+b = E "b"
+c = E "c"
+d = E "d"
+
+main = eToString (-a + b + (-c*d))
+--
+(((-a)+b)+(-(c*d))) 
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/BlankExpression.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/BlankExpression.scl
new file mode 100644
index 000000000..f31e144e8
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/BlankExpression.scl
@@ -0,0 +1,3 @@
+main = _
+--
+???
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/BooleanId.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/BooleanId.scl
new file mode 100644
index 000000000..2044c1c9b
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/BooleanId.scl
@@ -0,0 +1,12 @@
+@private
+@inline
+not True = False
+not False = True
+
+@private
+@inline
+id x = not (not x)
+
+main = id True
+--
+true
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Bug4450.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Bug4450.scl
new file mode 100644
index 000000000..54e65c2e9
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Bug4450.scl
@@ -0,0 +1,15 @@
+import "Prelude"
+
+csvWrite :: String -> [[String]] ->  ()
+csvWrite fname rows = ()
+
+/// Like writeEntries but with a transformer function also for values.
+/// kfun key + vfun value should have the length of header.
+writeEntries' :: (k -> [String]) -> (v -> [String])
+    -> String -> [String] -> [(k, v)] ->  ()
+writeEntries' kfun vfun fname header rows =
+    csvWrite fname $ [header] + [kfun k + vfun v | (k, v) <- rows]
+    
+main = "OK"
+--
+OK
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Character1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Character1.scl
new file mode 100644
index 000000000..b08d2aef3
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Character1.scl
@@ -0,0 +1,5 @@
+import "Prelude"
+
+main = subChar '6' '0'
+--
+6
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ClashingClass.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ClashingClass.scl
new file mode 100644
index 000000000..dee1e8d43
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ClashingClass.scl
@@ -0,0 +1,8 @@
+
+class A a where
+    foo :: a
+    
+class A a where
+    bar :: a
+--
+5:1-6:13: Class A has already been defined in this module.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ClashingData.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ClashingData.scl
new file mode 100644
index 000000000..60245db07
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ClashingData.scl
@@ -0,0 +1,5 @@
+
+data A = A
+data A = B
+--
+3:1-3:11: Type A has already been defined in this module.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ClashingInstance.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ClashingInstance.scl
new file mode 100644
index 000000000..72178db3a
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ClashingInstance.scl
@@ -0,0 +1,14 @@
+import "JavaBuiltin" as Java
+
+infixl 6 (+)
+
+class Additive a where 
+    (+)  :: a -> a -> a    
+    
+instance Additive Double where
+    (+) = Java.dadd
+
+instance Additive Double where
+    (+) = Java.dadd
+--
+11:1-12:20: Duplicate definition of the instance Additive Double.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ClashingValueDefinition.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ClashingValueDefinition.scl
new file mode 100644
index 000000000..f7ec72a43
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ClashingValueDefinition.scl
@@ -0,0 +1,5 @@
+a = 1
+a = 2
+main = "Should not be executed."
+--
+???
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ClashingValueType.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ClashingValueType.scl
new file mode 100644
index 000000000..d248d20f2
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ClashingValueType.scl
@@ -0,0 +1,6 @@
+
+id :: Integer -> Integer
+id :: Double -> Double
+id x = x
+--
+3:1-3:23: Type of id has already been declared in this module.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Collaz.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Collaz.scl
new file mode 100644
index 000000000..2e4552d0d
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Collaz.scl
@@ -0,0 +1,16 @@
+import "Prelude"
+
+f x = if x `mod` 2 == 0
+      then x `div` 2
+      else 3*x + 1
+fd x = unfoldr (\x -> do
+                   r = f x 
+                   if x == 1
+                   then Nothing
+                   else Just (x,r)
+                ) x
+
+//main :: [Integer]                
+main = fd 7
+--
+[7, 22, 11, 34, 17, 52, 26, 13, 40, 20, 10, 5, 16, 8, 4, 2]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Compose.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Compose.scl
new file mode 100644
index 000000000..fad8b9b67
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Compose.scl
@@ -0,0 +1,20 @@
+import "JavaBuiltin" as Java
+
+(+) = Java.iadd
+(-) = Java.isub
+(*) = Java.imul
+
+data List a = Nil | Cons a (List a) 
+
+compose :: List (a -> a) -> a -> a
+compose Nil        x = x
+compose (Cons h t) x = compose t (h x)
+
+succ x = x + 1
+prec x = x - 1
+double x = x * 2
+
+f = compose (Cons succ (Cons double (Cons prec Nil)))
+main = f 13
+--
+27
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Composition.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Composition.scl
new file mode 100644
index 000000000..80b1c06a4
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Composition.scl
@@ -0,0 +1,17 @@
+infixr 9 (.)
+
+@private
+@inline
+(f . g) = \x -> f (g x)
+
+@private
+@inline
+flip (x,y) = (y,x)
+
+@private
+@inline
+flip4 = flip . flip . flip . flip
+
+main = flip4 ("a", "b")
+--
+(a,b)
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ConjunctionMacro.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ConjunctionMacro.scl
new file mode 100644
index 000000000..99188f8bb
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ConjunctionMacro.scl
@@ -0,0 +1,6 @@
+(&&) :: Boolean -> ( Boolean) ->  Boolean
+a && b = if a then b else False
+
+main = False && fail "Should not be evaluated!"
+--
+false
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Constant.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Constant.scl
new file mode 100644
index 000000000..7ea0367bd
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Constant.scl
@@ -0,0 +1,6 @@
+
+a = 14 :: Integer
+
+main = a
+--
+14
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ConstructorNameClash.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ConstructorNameClash.scl
new file mode 100644
index 000000000..52826a538
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ConstructorNameClash.scl
@@ -0,0 +1,7 @@
+
+data Vec2 = Vec2 Double Double
+data Vec3 = Vec2 Double Double Double
+
+main = "Not to be executed."
+--
+3:13-3:38: Value Vec2 is already defined.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/DefaultMethods1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/DefaultMethods1.scl
new file mode 100644
index 000000000..142dd30a5
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/DefaultMethods1.scl
@@ -0,0 +1,13 @@
+import "JavaBuiltin" as Java
+
+class Ord a where
+    (<) :: a -> a -> Boolean
+    min :: a -> a -> a
+    min x y = if x < y then x else y
+    
+instance Ord Integer where
+    (<) = Java.icmplt
+
+main = min (43 :: Integer) (69 :: Integer) 
+--
+43
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Deriving3.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Deriving3.scl
new file mode 100644
index 000000000..ec2f5b7bd
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Deriving3.scl
@@ -0,0 +1,29 @@
+import "Prelude"
+
+a,b,c,d :: Either Boolean Boolean
+a = Left False
+b = Left True 
+c = Right False
+d = Right True
+
+main = a == a
+    && a < b
+    && a < c
+    && a < d
+    
+    && b > a
+    && b == b
+    && b < c
+    && b < d
+
+    && c > a
+    && c > b
+    && c == c
+    && c < d
+
+    && d > a
+    && d > b
+    && d > c
+    && d == d
+--
+true
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Deriving4.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Deriving4.scl
new file mode 100644
index 000000000..03ea0d609
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Deriving4.scl
@@ -0,0 +1,9 @@
+import "Prelude"
+
+data Foo = Foo Integer Long Double Float
+
+deriving instance Show Foo
+
+main = show (Foo (-1) (-1) (-1) (-1))
+--
+Foo (-1) (-1) (-1.0) (-1.0)
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/DifferentBranchTypes.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/DifferentBranchTypes.scl
new file mode 100644
index 000000000..2a0925fe9
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/DifferentBranchTypes.scl
@@ -0,0 +1,19 @@
+import "Prelude"
+
+foo n = n
+  where
+    if n > 0
+    then 1
+    else "asd"
+
+bar n = n
+  where
+    do 
+       c = n+1
+       if c > 0
+       then 1
+       else "asd"
+    
+main = foo 3 + bar 3
+--
+6
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Div.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Div.scl
new file mode 100644
index 000000000..4a8e1ca24
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Div.scl
@@ -0,0 +1,5 @@
+import "Prelude"
+
+main = 7 `div` 3
+--
+2
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/DoubleConversion.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/DoubleConversion.scl
new file mode 100644
index 000000000..31e95edb3
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/DoubleConversion.scl
@@ -0,0 +1,17 @@
+import "Prelude"
+
+f :: Float
+f = 1.0
+
+d :: Double
+d = 1.0
+
+d2f :: Float
+d2f = fromDouble d
+
+f2d :: Double
+f2d = toDouble f
+
+main = show (d+f2d, f+d2f)
+--
+(2.0, 2.0)
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/DoubleEffect.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/DoubleEffect.scl
new file mode 100644
index 000000000..b0615106f
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/DoubleEffect.scl
@@ -0,0 +1,8 @@
+app :: (Integer ->  Integer) ->  ((Integer ->  Integer) ->  Integer)
+app f = do
+  a = f 1
+  \g -> g a
+  
+main = app (\x -> x) (\x -> x)
+--
+1
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Effects1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Effects1.scl
new file mode 100644
index 000000000..11a3303d4
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Effects1.scl
@@ -0,0 +1,18 @@
+import "JavaBuiltin" as Java
+
+importJava "java.util.regex.Pattern" where
+    data Pattern
+
+    compile :: String -> Pattern
+    matcher :: Pattern -> String ->  Matcher
+
+importJava "java.util.regex.Matcher" where
+    data Matcher
+
+    matches :: Matcher ->  Boolean
+
+doMatch pattern text = matches (matcher pattern text)
+
+main = doMatch (compile ".*xxx.*") "fffxxooxxxlll"
+--
+true
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Effects2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Effects2.scl
new file mode 100644
index 000000000..d595ee938
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Effects2.scl
@@ -0,0 +1,21 @@
+import "JavaBuiltin" as Java
+
+importJava "java.util.List" where
+    data List a
+
+    add :: List a -> a ->  Boolean
+
+importJava "java.util.ArrayList" where
+    @JavaName ""
+    arrayList :: Integer ->  List a
+
+singleton :: a ->  List a
+singleton el = result
+  where
+    result = arrayList 1
+    r = add result el
+
+main :: List Integer
+main = runProc (singleton (13 :: Integer))
+--
+[13]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Effects3.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Effects3.scl
new file mode 100644
index 000000000..718482938
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Effects3.scl
@@ -0,0 +1,12 @@
+effectfulId :: a ->  a
+effectfulId x = x
+
+(.) :: (b ->  c) -> (a ->  b) -> a ->  c
+(f . g) x = f (g x)
+
+doubleId = effectfulId . effectfulId
+
+main :: Integer
+main = runProc (doubleId (13 :: Integer))
+--
+13
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Effects4.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Effects4.scl
new file mode 100644
index 000000000..85138a68a
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Effects4.scl
@@ -0,0 +1,12 @@
+effectfulId :: a ->  a
+effectfulId x = x
+
+//double :: (a ->  a) -> a ->  a
+double f x = f (f x)
+
+doubleId = double effectfulId
+
+main :: Integer
+main = runProc (doubleId (13 :: Integer))
+--
+13
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Effects5.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Effects5.scl
new file mode 100644
index 000000000..595b6346b
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Effects5.scl
@@ -0,0 +1,13 @@
+effectfulId :: a ->  a
+effectfulId x = x
+
+(.) :: (b ->  c) -> (a ->  b) -> a ->  c
+(f . g) x = f (g x)
+
+//doubleId :: a ->  a
+doubleId = effectfulId . effectfulId
+
+main :: Integer
+main = runProc (doubleId (13 :: Integer))
+--
+13
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Effects6.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Effects6.scl
new file mode 100644
index 000000000..e99dd4fcb
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Effects6.scl
@@ -0,0 +1,13 @@
+effectfulId :: a ->  a
+effectfulId x = x
+
+(.) :: (b ->  c) -> (a ->  b) -> a ->  c
+(f . g) x = f (g x)
+
+doubleId :: a ->  a
+doubleId = effectfulId . effectfulId
+
+main :: Integer
+main = runProc (doubleId (13 :: Integer))
+--
+13
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/EmptyLet.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/EmptyLet.scl
new file mode 100644
index 000000000..68c662378
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/EmptyLet.scl
@@ -0,0 +1,9 @@
+import "Prelude"
+
+ex1 = let in 1
+ex2 = let {} in 2
+ex3 = let {a=3} in a 
+
+main = ex1 + ex2 + ex3
+--
+6
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Equality.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Equality.scl
new file mode 100644
index 000000000..acf00e936
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Equality.scl
@@ -0,0 +1,41 @@
+import "Prelude"
+
+main = """
+\(newEq () ())
+\(newEq True True)
+\(newEq False False)
+\(newEq True False)
+\(newEq False True)
+\(newEq (1::Integer) (1::Integer))
+\(newEq (1::Integer) (2::Integer))
+\(newEq (1::Long) (1::Long))
+\(newEq (1::Long) (2::Long))
+\(newEq (1::Double) (1::Double))
+\(newEq (1::Double) (2::Double))
+\(newEq "a" "a")
+\(newEq "a" "b")
+\(newEq (Just "a") (Just "a"))
+\(newEq (Just "a") (Just "b"))
+\(newEq Nothing Nothing)
+\(newEq (Just "a") Nothing)
+\(newEq Nothing (Just "a"))
+"""
+--
+True
+True
+True
+False
+False
+True
+False
+True
+False
+True
+False
+True
+False
+True
+False
+True
+False
+False
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Equations1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Equations1.scl
new file mode 100644
index 000000000..613d56bff
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Equations1.scl
@@ -0,0 +1,13 @@
+import "Prelude"
+import "Expressions/Equations"
+
+main = v
+  where
+    v = solveEquations do
+        listenEquationVariable "a" $ \a -> setEquationVariable "c" (a-1)
+        setEquationVariable "a" 123
+        listenEquationVariable "a" $ \a -> setEquationVariable "b" (a+1)
+    print v 
+--
+[(a,123), (b,124), (c,122)]
+
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ExistentialData.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ExistentialData.scl
new file mode 100644
index 000000000..62130b49f
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ExistentialData.scl
@@ -0,0 +1,23 @@
+import "JavaBuiltin" as Java
+
+(+) = Java.iadd
+
+data Thunk a = /* forall s. */ Thunk s (s -> a)
+
+id :: a -> a
+id x = x
+
+runThunk :: Thunk a -> a
+runThunk (Thunk s f) = f s
+
+makeThunk :: a -> Thunk a
+makeThunk x = Thunk x id
+
+mapThunk :: (a -> b) -> Thunk a -> Thunk b
+mapThunk f (Thunk s g) = Thunk s (\x -> f (g x))
+
+a = makeThunk (13 :: Integer)
+b = mapThunk (\x -> x+1) a
+main = runThunk b
+--
+14
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ExistentialData2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ExistentialData2.scl
new file mode 100644
index 000000000..b0299691e
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ExistentialData2.scl
@@ -0,0 +1,7 @@
+data Thunk a = /* forall s. */ Thunk s (s -> a)
+
+mixThunks (Thunk s0 f0) (Thunk s1 f1) = f0 s1
+
+main = "Not to be executed!"
+--
+3:44-3:46: Expected  got .
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ExpressionParsing.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ExpressionParsing.scl
new file mode 100644
index 000000000..6719bf935
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ExpressionParsing.scl
@@ -0,0 +1,27 @@
+import "Prelude"
+
+data Exp = Exp String
+
+expToString :: Exp -> String
+expToString (Exp s) = s
+
+instance Additive Exp where
+    zero = Exp "0"
+    Exp a + Exp b = Exp ("(" + a + " + " + b + ")")
+
+instance Ring Exp where
+    one = Exp "1"
+    neg (Exp a) = Exp ("(-" + a + ")")
+    Exp a * Exp b = Exp ("(" + a + " * " + b + ")")
+    Exp a - Exp b = Exp ("(" + a + " - " + b + ")")
+    fromInteger x = Exp (show x)
+    
+a = Exp "a"
+b = Exp "b"
+c = Exp "c"
+d = Exp "d"
+e = Exp "e"
+
+main = expToString (a + b*c + d*e)
+--
+((a + (b * c)) + (d * e)) 
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FaultyRecursion.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FaultyRecursion.scl
new file mode 100644
index 000000000..523b40877
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FaultyRecursion.scl
@@ -0,0 +1,3 @@
+fib n = fib
+--
+1:9-1:12: Expected  got   a>.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Fibonacci.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Fibonacci.scl
new file mode 100644
index 000000000..b7b56f50c
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Fibonacci.scl
@@ -0,0 +1,14 @@
+import "JavaBuiltin" as Java
+
+(+) = Java.iadd
+(-) = Java.isub
+(<=) = Java.icmple
+
+fibonacci x = if x <= (1 :: Integer) 
+              then 1 :: Integer 
+              else fibonacci (x - 1) 
+                 + fibonacci (x - 2)
+
+main = fibonacci 10
+--
+89
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Fibonacci2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Fibonacci2.scl
new file mode 100644
index 000000000..229145fbc
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Fibonacci2.scl
@@ -0,0 +1,11 @@
+import "Prelude"
+
+fibonacci :: Integer -> Integer
+fibonacci x = if x <= 1 
+              then 1 
+              else fibonacci (x - 1) 
+                 + fibonacci (x - 2)
+
+main = fibonacci 10
+--
+89
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Fibonacci3.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Fibonacci3.scl
new file mode 100644
index 000000000..d5ce91c60
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Fibonacci3.scl
@@ -0,0 +1,20 @@
+import "JavaBuiltin" as Java
+ 
+class Num a where
+    (+) :: a -> a -> a
+    (-) :: a -> a -> a
+
+instance Num Integer where
+    (+) = Java.iadd
+    (-) = Java.isub
+
+(<=) = Java.icmple
+
+fibonacci x = if x <= (1 :: Integer) 
+              then 1 :: Integer
+              else fibonacci (x - (1 :: Integer)) 
+                 + fibonacci (x - (2 :: Integer))
+
+main = fibonacci 10
+--
+89
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FingerTree.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FingerTree.scl
new file mode 100644
index 000000000..b81f642f6
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FingerTree.scl
@@ -0,0 +1,173 @@
+import "JavaBuiltin" as Java
+
+main = foldl Java.iadd (0 :: Integer) (concat (concat (Single (1 :: Integer)) 
+      (Single (2 :: Integer))) (Single (3 :: Integer))) 
+
+data Digit a = Digit1 a
+             | Digit2 a a
+             | Digit3 a a a
+             | Digit4 a a a a
+data Node a = Node2 a a | Node3 a a a
+data FingerTree a = Empty | Single a | Deep (Digit a) (FingerTree (Node a)) (Digit a)
+
+insertL :: a -> FingerTree a -> FingerTree a
+insertL a Empty      = Single a
+insertL a (Single b) = Deep (Digit1 a) Empty (Digit1 b)
+insertL a (Deep (Digit1 b) m r) = Deep (Digit2 a b) m r
+insertL a (Deep (Digit2 b c) m r) = Deep (Digit3 a b c) m r
+insertL a (Deep (Digit3 b c d) m r) = Deep (Digit4 a b c d) m r
+insertL a (Deep (Digit4 b c d e) m r) = Deep (Digit2 a b) (insertL (Node3 c d e) m) r
+
+insertR :: FingerTree a -> a -> FingerTree a
+insertR Empty a      = Single a
+insertR (Single a) b = Deep (Digit1 a) Empty (Digit1 b)
+insertR (Deep l m (Digit1 a)) b = Deep l m (Digit2 a b)
+insertR (Deep l m (Digit2 a b)) c = Deep l m (Digit3 a b c)
+insertR (Deep l m (Digit3 a b c)) d = Deep l m (Digit4 a b c d)
+insertR (Deep l m (Digit4 a b c d)) e = Deep l (insertR m (Node3 a b c)) (Digit2 d e)
+
+foldl :: (a -> b -> a) -> a -> FingerTree b -> a
+foldl f init Empty = init
+foldl f init (Single x) = f init x
+foldl f init (Deep l m r) = foldlD (foldl foldlN (foldlD init l) m) r
+  where
+    foldlD init (Digit1 a) = f init a
+    foldlD init (Digit2 a b) = f (f init a) b
+    foldlD init (Digit3 a b c) = f (f (f init a) b) c
+    foldlD init (Digit4 a b c d) = f (f (f (f init a) b) c) d
+    
+    foldlN init (Node2 a b) = f (f init a) b
+    foldlN init (Node3 a b c) = f (f (f init a) b) c
+
+data View a = Nil | Cons a (FingerTree a)
+
+viewL :: FingerTree a -> View a
+viewL Empty = Nil
+viewL (Single a) = Cons a Empty
+viewL (Deep (Digit1 a) m r) = Cons a tail
+  where
+    tail = match viewL m with
+        Nil -> digitToFingerTree r
+        Cons h t -> Deep (nodeToDigit h) t r
+viewL (Deep (Digit2 a b) m r) = Cons a (Deep (Digit1 a) m r)
+viewL (Deep (Digit3 a b c) m r) = Cons a (Deep (Digit2 a b) m r)
+viewL (Deep (Digit4 a b c d) m r) = Cons a (Deep (Digit3 a b c) m r)
+
+concat :: FingerTree a -> FingerTree a -> FingerTree a
+concat Empty a = a
+concat a Empty = a
+concat (Single a) b = insertL a b
+concat a (Single b) = insertR a b
+concat (Deep l1 m1 r1) (Deep l2 m2 r2) = Deep l1 mm r2
+  where
+    mm = concatAux m1 (digitsToNodes r1 l2) m2
+    
+// --- Implementation details -------------------------------------------------
+
+digitToFingerTree :: Digit a -> FingerTree a
+digitToFingerTree (Digit1 a)       = Single a
+digitToFingerTree (Digit2 a b)     = Deep (Digit1 a)   Empty (Digit1 b)
+digitToFingerTree (Digit3 a b c)   = Deep (Digit2 a b) Empty (Digit1 c)
+digitToFingerTree (Digit4 a b c d) = Deep (Digit2 a b) Empty (Digit2 c d)
+
+nodeToDigit :: Node a -> Digit a
+nodeToDigit (Node2 a b)   = Digit2 a b
+nodeToDigit (Node3 a b c) = Digit3 a b c
+    
+concatAux :: FingerTree a -> Digit a -> FingerTree a -> FingerTree a
+concatAux Empty ds a = insertLD ds a
+concatAux a ds Empty = insertRD a ds
+concatAux (Single a) ds b = insertL a (insertLD ds b)
+concatAux a ds (Single b) = insertR (insertRD a ds) b
+concatAux (Deep l1 m1 r1) ds (Deep l2 m2 r2) = Deep l1 mm r2
+  where
+    mm = concatAux m1 (digitsToNodes3 r1 ds r2) m2
+
+insertLD :: Digit a -> FingerTree a -> FingerTree a
+insertLD (Digit1 a) t = insertL a t
+insertLD (Digit2 a b) t = insertL a (insertL b t)
+insertLD (Digit3 a b c) t = insertL a (insertL b (insertL c t))
+insertLD (Digit4 a b c d) t = insertL a (insertL b (insertL c (insertL d t)))
+
+insertRD :: FingerTree a -> Digit a -> FingerTree a
+insertRD t (Digit1 a) = insertR t a
+insertRD t (Digit2 a b) = insertR (insertR t a) b
+insertRD t (Digit3 a b c) = insertR (insertR (insertR t a) b) c
+insertRD t (Digit4 a b c d) = insertR (insertR (insertR (insertR t a) b) c) d
+    
+digitsToNodes :: Digit a -> Digit a -> Digit (Node a)
+digitsToNodes (Digit1 a) x = dd1 a x
+digitsToNodes (Digit2 a b) x = dd2 a b x 
+digitsToNodes (Digit3 a b c) x = dd3 a b c x
+digitsToNodes (Digit4 a b c d) x = dd4 a b c d x 
+
+digitsToNodes3 :: Digit a -> Digit a -> Digit a -> Digit (Node a)
+digitsToNodes3 (Digit1 a) x y = ddd1 a x y
+digitsToNodes3 (Digit2 a b) x y = ddd2 a b x y
+digitsToNodes3 (Digit3 a b c) x y = ddd3 a b c x y
+digitsToNodes3 (Digit4 a b c d) x y = ddd4 a b c d x y   
+    
+d2 a b = Digit1 (Node2 a b)
+d3 a b c = Digit1 (Node3 a b c)
+d4 a b c d = Digit2 (Node2 a b) (Node2 c d)
+d5 a b c d e = Digit2 (Node3 a b c) (Node2 d e)
+d6 a b c d e f = Digit2 (Node3 a b c) (Node3 d e f)
+d7 a b c d e f g = Digit3 (Node3 a b c) (Node2 d e) (Node2 f g)
+d8 a b c d e f g h = Digit3 (Node3 a b c) (Node3 d e f) (Node2 g h)
+d9 a b c d e f g h i = Digit3 (Node3 a b c) (Node3 d e f) (Node3 g h i)
+d10 a b c d e f g h i j = Digit4 (Node3 a b c) (Node3 d e f) (Node2 g h) (Node2 i j)
+d11 a b c d e f g h i j k = Digit4 (Node3 a b c) (Node3 d e f) (Node3 g h i) (Node2 j k)
+d12 a b c d e f g h i j k l = Digit4 (Node3 a b c) (Node3 d e f) (Node3 g h i) (Node3 j k l)
+
+dd1 a (Digit1 b) = d2 a b
+dd1 a (Digit2 b c) = d3 a b c
+dd1 a (Digit3 b c d) = d4 a b c d
+dd1 a (Digit4 b c d e) = d5 a b c d e
+dd2 a b (Digit1 c) = d3 a b c
+dd2 a b (Digit2 c d) = d4 a b c d
+dd2 a b (Digit3 c d e) = d5 a b c d e
+dd2 a b (Digit4 c d e f) = d6 a b c d e f
+dd3 a b c (Digit1 d) = d4 a b c d
+dd3 a b c (Digit2 d e) = d5 a b c d e
+dd3 a b c (Digit3 d e f) = d6 a b c d e f
+dd3 a b c (Digit4 d e f g) = d7 a b c d e f g
+dd4 a b c d (Digit1 e) = d5 a b c d e
+dd4 a b c d (Digit2 e f) = d6 a b c d e f
+dd4 a b c d (Digit3 e f g) = d7 a b c d e f g
+dd4 a b c d (Digit4 e f g h) = d8 a b c d e f g h
+dd5 a b c d e (Digit1 f) = d6 a b c d e f
+dd5 a b c d e (Digit2 f g) = d7 a b c d e f g
+dd5 a b c d e (Digit3 f g h) = d8 a b c d e f g h
+dd5 a b c d e (Digit4 f g h i) = d9 a b c d e f g h i
+dd6 a b c d e f (Digit1 g) = d7 a b c d e f g
+dd6 a b c d e f (Digit2 g h) = d8 a b c d e f g h
+dd6 a b c d e f (Digit3 g h i) = d9 a b c d e f g h i
+dd6 a b c d e f (Digit4 g h i j) = d10 a b c d e f g h i j
+dd7 a b c d e f g (Digit1 h) = d8 a b c d e f g h
+dd7 a b c d e f g (Digit2 h i) = d9 a b c d e f g h i
+dd7 a b c d e f g (Digit3 h i j) = d10 a b c d e f g h i j
+dd7 a b c d e f g (Digit4 h i j k) = d11 a b c d e f g h i j k
+dd8 a b c d e f g h (Digit1 i) = d9 a b c d e f g h i
+dd8 a b c d e f g h (Digit2 i j) = d10 a b c d e f g h i j
+dd8 a b c d e f g h (Digit3 i j k) = d11 a b c d e f g h i j k
+dd8 a b c d e f g h (Digit4 i j k l) = d12 a b c d e f g h i j k l
+
+ddd1 a (Digit1 b) y = dd2 a b y
+ddd1 a (Digit2 b c) y = dd3 a b c y
+ddd1 a (Digit3 b c d) y = dd4 a b c d y
+ddd1 a (Digit4 b c d e) y = dd5 a b c d e y
+ddd2 a b (Digit1 c) y = dd3 a b c y
+ddd2 a b (Digit2 c d) y = dd4 a b c d y
+ddd2 a b (Digit3 c d e) y = dd5 a b c d e y
+ddd2 a b (Digit4 c d e f) y = dd6 a b c d e f y
+ddd3 a b c (Digit1 d) y = dd4 a b c d y
+ddd3 a b c (Digit2 d e) y = dd5 a b c d e y
+ddd3 a b c (Digit3 d e f) y = dd6 a b c d e f y
+ddd3 a b c (Digit4 d e f g) y = dd7 a b c d e f g y
+ddd4 a b c d (Digit1 e) y = dd5 a b c d e y
+ddd4 a b c d (Digit2 e f) y = dd6 a b c d e f y
+ddd4 a b c d (Digit3 e f g) y = dd7 a b c d e f g y
+ddd4 a b c d (Digit4 e f g h) y = dd8 a b c d e f g h y
+
+--
+6
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FoldMissingInitialValue.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FoldMissingInitialValue.scl
new file mode 100644
index 000000000..affa2e66f
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FoldMissingInitialValue.scl
@@ -0,0 +1,6 @@
+import "Prelude"
+
+f p l = (foldl (+) (map ((+)p) l)) + p
+--
+3:1-3:39: Couldn't simplify all effect subsumptions away. The current compiler cannot handle this situation. Try adding more type annotations.
+3:25-3:31: Type [a b] ->  a b is not a subtype of a.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FoldlBuild1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FoldlBuild1.scl
new file mode 100644
index 000000000..21dfdea64
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FoldlBuild1.scl
@@ -0,0 +1,18 @@
+import "Prelude"
+
+inc :: Ref Integer ->  Integer
+inc r = do 
+    v = getRef r
+    newV = v+1
+    r := newV
+    newV
+
+main = do
+    r = ref 0
+    // Because both map and for get side-effectful functions
+    // as parameters, the fusion is not allowed.
+    l = map (\_ -> inc r) [1..4]    
+    for l (\i -> r := i+1)
+    getRef r
+--
+5
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FoldlBuild2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FoldlBuild2.scl
new file mode 100644
index 000000000..2078fb7c2
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FoldlBuild2.scl
@@ -0,0 +1,16 @@
+import "Prelude"
+
+inc :: Ref Integer ->  Integer
+inc r = do 
+    v = getRef r
+    newV = v+1
+    r := newV
+    newV
+
+main = do
+    r = ref 0
+    l = map (\_ -> inc r) [1..4]
+    r := 4 // Map must be executed before this statement
+    foldl (+) 0 l
+--
+10
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Forall1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Forall1.scl
new file mode 100644
index 000000000..32e51a716
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Forall1.scl
@@ -0,0 +1,8 @@
+     
+id :: forall a. a -> a
+   // ^ not usually needed, but we test just that this is possible
+id x = x
+
+main = id (3 :: Integer)
+--
+3
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Forall2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Forall2.scl
new file mode 100644
index 000000000..97d1c3510
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Forall2.scl
@@ -0,0 +1,9 @@
+
+data List a = Nil | Cons a (List a)
+
+build :: (forall l. l -> (a -> l -> l) -> l) -> List a
+build f = f Nil Cons
+
+main = build (\nil cons -> cons (1 :: Integer) (cons (2 :: Integer) nil))
+-- 
+(Cons 1 (Cons 2 Nil))
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Forall3.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Forall3.scl
new file mode 100644
index 000000000..018a2a0be
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Forall3.scl
@@ -0,0 +1,11 @@
+
+data List a = Nil | Cons a (List a)
+
+build :: (forall l. l -> (a -> l -> l) -> l) -> List a
+build f = f Nil Cons
+
+main = build (\nil cons -> cons (1 :: Integer) (Cons (2 :: Integer) nil))
+                                             // ^^^^
+-- 
+7:48-7:73: Expected  got .
+7:69-7:72: Expected  got .
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Formula.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Formula.scl
new file mode 100644
index 000000000..10eafb061
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Formula.scl
@@ -0,0 +1,131 @@
+import "Prelude"
+
+data Formula a = TrueF
+               | FalseF
+               | XorF (Formula a) (Formula a)
+               | AndF (Formula a) (Formula a)
+               | ConditionF a
+               | NextF (Formula a)
+               | UntilF (Formula a) (Formula a)
+
+deriving instance (Ord a) => Ord (Formula a)
+
+instance (Show a) => Show (Formula a) where
+    sb <+ TrueF = sb << "true"
+    sb <+ FalseF = sb << "false"
+    sb <+ XorF a b = sb << "(" <+ a << " `XorF` " <+ b << ")"
+    sb <+ AndF a b = sb  << "(" <+ a << " &&& " <+ b << ")"
+    sb <+ ConditionF c = sb  <+ c
+    sb <+ NextF a = sb  << "(next " <+ a << ")"
+    sb <+ UntilF a b = sb << "(" <+ a << " `UntilF` " <+ b << ")"
+
+xorF FalseF f2 = f2
+xorF f1 FalseF = f1             
+xorF f1@(XorF h1 t1) f2@(XorF h2 t2) = 
+    let cmp = compare h1 h2 
+    in  if cmp < 0
+        then XorF h1 (xorF t1 f2)
+        else if cmp > 0
+        then XorF h2 (xorF f1 t2)
+        else xorF t1 t2
+xorF f1@(XorF h1 t1) f2 = 
+    let cmp = compare h1 f2
+    in  if cmp < 0
+        then XorF h1 (xorF t1 f2)
+        else if cmp > 0
+        then XorF f2 f1
+        else t1
+xorF f1 f2@(XorF h2 t2) =
+    let cmp = compare f1 h2
+    in  if cmp < 0
+        then XorF f1 f2
+        else if cmp > 0
+        then XorF h2 (xorF f1 t2)
+        else t2
+xorF f1 f2 =
+    let cmp = compare f1 f2
+    in  if cmp < 0
+        then XorF f1 f2
+        else if cmp > 0
+        then XorF f2 f1
+        else TrueF
+        
+notF f = xorF TrueF f
+
+TrueF &&& f2 = f2
+f1 &&& TrueF = f1
+FalseF &&& _ = FalseF
+_ &&& FalseF = FalseF
+XorF h1 t1 &&& f2 = xorF (h1 &&& f2) (t1 &&& f2)
+f1 &&& XorF h2 t2 = xorF (f1 &&& h2) (f1 &&& t2)
+f1@(AndF h1 t1) &&& f2@(AndF h2 t2) = 
+    let cmp = compare h1 h2 
+    in  if cmp < 0
+        then AndF h1 (t1 &&& f2)
+        else if cmp > 0
+        then AndF h2 (f1 &&& t2)
+        else AndF h1 (t1 &&& t2)
+f1@(AndF h1 t1) &&& f2 = 
+    let cmp = compare h1 f2
+    in  if cmp < 0
+        then AndF h1 (t1 &&& f2)
+        else if cmp > 0
+        then AndF f2 f1
+        else f1
+f1 &&& f2@(AndF h2 t2) =
+    let cmp = compare f1 h2
+    in  if cmp < 0
+        then AndF f1 f2
+        else if cmp > 0
+        then AndF h2 (f1 &&& t2)
+        else f2
+f1 &&& f2 =
+    let cmp = compare f1 f2
+    in  if cmp < 0
+        then AndF f1 f2
+        else if cmp > 0
+        then AndF f2 f1
+        else f1
+
+f1 ||| f2 = xorF (xorF f1 f2) (f1 &&& f2)
+        
+eval :: Ord a => (a ->  Boolean) -> Formula a ->  Formula a
+eval s TrueF = TrueF
+eval s FalseF = FalseF
+eval s (XorF f1 f2) = xorF (eval s f1) (eval s f2)
+eval s (AndF f1 f2) = eval s f1 &&& eval s f2
+eval s (ConditionF c) = if s c then TrueF else FalseF
+eval s (NextF f) = f
+eval s (UntilF f1 f2) = eval s f2 ||| (eval s f1 &&& UntilF f1 f2)
+
+// Concrete conditions
+
+data V = V String (Ref Boolean)
+
+instance Ord V where
+    compare (V a _) (V b _) = compare a b
+instance Show V where
+    sb <+ V a _ = sb <+ a 
+
+cond :: String -> Ref Boolean -> Formula V
+cond name ref = ConditionF (V name ref) 
+
+// Testing
+
+x = ref True
+y = ref False
+
+f = cond "x" x `UntilF` cond "y" y
+
+evalV = eval (\(V _ c) -> getRef c) 
+
+main = do
+    print (evalV f)
+    x := False
+    print (evalV f)
+    y = ref True
+    print (evalV f)
+    x := True
+    print (evalV f)
+--
+()
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FromDynamic.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FromDynamic.scl
new file mode 100644
index 000000000..1ac2e7704
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FromDynamic.scl
@@ -0,0 +1,5 @@
+import "Prelude"
+
+main = (fromDynamic (toDynamic 42) :: String) + "foo"
+--
+42foo
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FromDynamic2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FromDynamic2.scl
new file mode 100644
index 000000000..1b6c1d99d
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FromDynamic2.scl
@@ -0,0 +1,5 @@
+import "Prelude"
+
+main = (fromDynamic (toDynamic "42") :: Integer) - 2
+--
+40
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FromDynamic3.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FromDynamic3.scl
new file mode 100644
index 000000000..a3318d4de
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FromDynamic3.scl
@@ -0,0 +1,5 @@
+import "Prelude"
+
+main = arrayToList (fromDynamic (toDynamic ["42"]) :: (Array Integer))
+--
+[42]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FromDynamic4.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FromDynamic4.scl
new file mode 100644
index 000000000..fb4c32088
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FromDynamic4.scl
@@ -0,0 +1,6 @@
+import "Prelude"
+import "Vector"
+
+main = (fromDynamic (toDynamic ["42"]) :: (Vector Double))!0
+--
+42.0
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FromDynamic5.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FromDynamic5.scl
new file mode 100644
index 000000000..d0d0fd3a2
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FromDynamic5.scl
@@ -0,0 +1,6 @@
+import "Prelude"
+import "Vector"
+
+main = (fromDynamic (toDynamic (mvector [42.0]))) :: [Integer]
+--
+[42]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FunctionFunctor.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FunctionFunctor.scl
new file mode 100644
index 000000000..f63481032
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FunctionFunctor.scl
@@ -0,0 +1,9 @@
+import "Prelude"
+
+//               vvvvvv  mistake, parantheses missing
+instance Functor (->) a where
+    map f g x = f (g x)
+    
+main = "Not to be executed."
+--
+4:1-5:24: Wrong number of parameters to type class Functor.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FunctionalDependencies1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FunctionalDependencies1.scl
new file mode 100644
index 000000000..203de1dba
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FunctionalDependencies1.scl
@@ -0,0 +1,11 @@
+import "Prelude"
+
+class Container c a b | c -> a, c -> b where
+    get :: c -> a -> b
+
+instance Container [a] Integer a where
+    get = (!)
+
+main = get [1,2,3,4,5] 3
+--
+4
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FunctionalDependencies2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FunctionalDependencies2.scl
new file mode 100644
index 000000000..707bb9e2e
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FunctionalDependencies2.scl
@@ -0,0 +1,12 @@
+import "Prelude"
+
+class Mul a b c | a b -> c where
+    mul :: a -> b -> c
+
+instance Mul Integer Integer Integer where
+    mul = (*)
+
+main :: Integer
+main = mul (mul (1 :: Integer) (2 :: Integer)) (3 :: Integer)
+--
+6
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Functor.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Functor.scl
new file mode 100644
index 000000000..6bbd35979
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Functor.scl
@@ -0,0 +1,23 @@
+import "JavaBuiltin" as Java
+
+class Functor f where
+    map :: (a -> b) -> f a -> f b
+    
+data Foo a = Foo a
+
+instance Functor Foo where
+    map f (Foo x) = Foo (f x)
+    
+instance Functor Maybe where
+    map f Nothing = Nothing
+    map f (Just x) = Just (f x)
+    
+data List a = Nil | Cons a (List a)
+
+instance Functor List where
+    map f Nil = Nil
+    map f (Cons h t) = Cons (f h) (map f t)
+    
+main = map (map (Java.iadd 1)) (Cons Nothing (Cons (Just (1 :: Integer)) Nil))
+--
+(Cons null (Cons 2 Nil))
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FunctorM1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FunctorM1.scl
new file mode 100644
index 000000000..5e1874e31
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/FunctorM1.scl
@@ -0,0 +1,11 @@
+import "Prelude"
+
+main = sequence [[1,2], [3,4], [5,6]]
+/*
+  x <- [1,2]
+  y <- [3,4]
+  z <- [5,6]
+  return [x,y,z]
+*/  
+--
+[[1, 3, 5], [1, 3, 6], [1, 4, 5], [1, 4, 6], [2, 3, 5], [2, 3, 6], [2, 4, 5], [2, 4, 6]]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Generalization.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Generalization.scl
new file mode 100644
index 000000000..e5503349f
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Generalization.scl
@@ -0,0 +1,13 @@
+import "JavaBuiltin" as Java
+
+(+) = Java.iadd
+
+data Foo = Foo Integer
+
+escapeFoo (Foo x) = x
+
+id x = x
+
+main = id (3 :: Integer) + escapeFoo (id (Foo (4 :: Integer)))
+--
+7
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/GenericMutualRecursion.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/GenericMutualRecursion.scl
new file mode 100644
index 000000000..24c5858d6
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/GenericMutualRecursion.scl
@@ -0,0 +1,12 @@
+// Idea of this test is to ensure that generic type variables
+// and constraints are handled correctly with mutually recursive
+// functions
+
+import "Prelude"
+
+deepId count x = deepId2 count x
+deepId2 count x = if count <= 0 then x else deepId (count-1) x
+
+main = deepId 5 "FOO"
+--
+FOO
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/GlobalVariables.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/GlobalVariables.scl
new file mode 100644
index 000000000..c6dc99edf
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/GlobalVariables.scl
@@ -0,0 +1,14 @@
+import "Prelude"
+
+global :: Ref Integer
+global = ref 0
+
+inc ::  ()
+inc = global := getRef global + 1
+
+main = do
+    inc
+    inc
+    getRef global
+--
+2
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/GraphPrelude.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/GraphPrelude.scl
new file mode 100644
index 000000000..ddb2d58be
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/GraphPrelude.scl
@@ -0,0 +1,41 @@
+
+/**
+ * Databoard
+ */
+data Binding // Private
+class Serializable a where
+    getBinding :: Binding
+
+/**
+ * Reading graph
+ */
+data Resource
+data ReadGraph // Private
+class ReadTransaction where
+    getGraph :: ReadGraph
+
+resource :: ReadTransaction => String -> Resource
+(#) :: ReadTransaction => Resource -> Resource -> [Resource]
+valueOf :: ReadTransaction => Serializable a => Resource -> a
+
+/**
+ * Writing graph
+ */
+data WriteGraph // Private
+data Graph a = Graph (WriteGraph -> a)
+
+instance Monad Graph where
+    return x = Graph (\_ -> x)
+    Graph g >>= f = \wg -> f (g wg) wg
+    map f (Graph g) = Graph (f . g)
+
+newResource :: Graph Resource
+newResource = Graph __WriteGraph_newResource
+newLiteral :: Serializable a => a -> Graph Resource
+newLiteral = Graph (\wg -> 
+    literal = __WriteGraph_newResource wg
+    _ = __WriteGraph_claimValue wg literal a getBinding
+    literal 
+)
+statement :: Resource -> Resource -> Resource -> Graph ()
+statement s p o = Graph (\wg -> __WriteGraph_claim wg s p o)
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/GuardedExpressionBug.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/GuardedExpressionBug.scl
new file mode 100644
index 000000000..c0d45f161
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/GuardedExpressionBug.scl
@@ -0,0 +1,9 @@
+tableIndex :: Double -> Double
+tableIndex x
+    | True = x
+  where
+    f _ = x
+
+main = "Foo"
+--
+Foo
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Guards1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Guards1.scl
new file mode 100644
index 000000000..db3b43833
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Guards1.scl
@@ -0,0 +1,9 @@
+import "Prelude"
+
+fib :: Integer -> Integer
+fib x | x <= 2 = 1
+      | True   = fib (x-1) + fib (x-2)
+
+main = fib 13
+--
+233
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Guards2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Guards2.scl
new file mode 100644
index 000000000..88734c1ac
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Guards2.scl
@@ -0,0 +1,10 @@
+import "Prelude"
+
+fib :: Integer -> Integer
+fib x = match x with
+  v | v <= 2 -> 1
+    | True   -> fib (v-1) + fib (v-2)
+
+main = fib 13
+--
+233
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/IdAsOperator.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/IdAsOperator.scl
new file mode 100644
index 000000000..576ff86f1
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/IdAsOperator.scl
@@ -0,0 +1,9 @@
+import "JavaBuiltin" as Java
+
+infixl 5 minus
+
+minus = Java.isub
+
+main = 5 `minus` 3 `minus` 2
+--
+0
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/IllegalChar.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/IllegalChar.scl
new file mode 100644
index 000000000..371492bb7
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/IllegalChar.scl
@@ -0,0 +1,3 @@
+a = ¤
+--
+1:5-1:6: Illegal character '¤'.
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ImportJavaConstructor.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ImportJavaConstructor.scl
new file mode 100644
index 000000000..8c45e438b
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ImportJavaConstructor.scl
@@ -0,0 +1,18 @@
+
+@JavaType "org.simantics.scl.compiler.elaboration.expressions.Expression"
+data Expression = 
+    @JavaType "org/simantics/scl/compiler/elaboration/expressions/EIntegerLiteral"
+    @FieldNames [value]
+    EIntegerLiteral String
+  | @JavaType "org.simantics.scl.compiler.elaboration.expressions.ERealLiteral"
+    @FieldNames [value]
+    ERealLiteral String
+
+changeType :: Expression -> Expression
+changeType (EIntegerLiteral value) = ERealLiteral value
+changeType (ERealLiteral value) = EIntegerLiteral value
+
+main :: Expression
+main = changeType (EIntegerLiteral "123") 
+--
+123
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ImportRef.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ImportRef.scl
new file mode 100644
index 000000000..9fb6f62d2
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ImportRef.scl
@@ -0,0 +1,18 @@
+importJava "org.simantics.scl.runtime.procedure.Ref" where
+    data Ref a
+    
+    @JavaName ""
+    ref :: a ->  (Ref a)
+    
+    @JavaName "value"
+    getRef :: Ref a ->  a
+    
+    @JavaName "value"
+    (:=) :: Ref a -> a ->  ()
+
+main = do
+    r = ref (13 :: Integer)
+    r := (14 :: Integer)
+    getRef r
+--
+14
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InconsistentArity.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InconsistentArity.scl
new file mode 100644
index 000000000..82a966892
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InconsistentArity.scl
@@ -0,0 +1,4 @@
+f a b = a
+f x = x
+--
+2:1-2:4: Inconsistent arity. This case has arity 1 while previous cases had arity 2.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InconsistentIndentation.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InconsistentIndentation.scl
new file mode 100644
index 000000000..bd732cf21
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InconsistentIndentation.scl
@@ -0,0 +1,5 @@
+class Foo a where
+    x :: a
+  y :: a
+--
+3:3-3:4: Unexpected token 'y' (ID). Expected one of EOF, RBRACE, SEMICOLON.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/IndentationAndParenthesis.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/IndentationAndParenthesis.scl
new file mode 100644
index 000000000..3a8dc3de7
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/IndentationAndParenthesis.scl
@@ -0,0 +1,16 @@
+// The last character of the program is an extra closing parenthesis
+data List a = Nil | Cons a (List a)
+
+first Nil = 0
+first (Cons x _) = x
+
+reverse l = reverseAux Nil l 
+  where
+    reverseAux accum Nil = accum
+    reverseAux accum (Cons h t) = reverseAux (Cons h accum) t    
+
+main = first (reverse l) 
+  where
+    l = Cons 1 (Cons 2 (Cons 3 Nil)))
+--
+14:37-14:38: No corresponding opening parenthesis for ')'.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Index.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Index.scl
new file mode 100644
index 000000000..8a98aec03
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Index.scl
@@ -0,0 +1,8 @@
+import "Prelude"
+
+f :: Integer -> Maybe Integer
+f = index [(1,2),(2,4),(3,6),(4,8)]
+
+main = f 3
+--
+6
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Inline1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Inline1.scl
new file mode 100644
index 000000000..118abc4c5
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Inline1.scl
@@ -0,0 +1,5 @@
+main = id "Foo" 
+  where
+    id x = x
+--
+Foo
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InlineLoop.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InlineLoop.scl
new file mode 100644
index 000000000..df8433c5c
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InlineLoop.scl
@@ -0,0 +1,9 @@
+data SList a = Nil | Cons a (SList a)
+
+@inline
+copy (Cons h t) = Cons h (copy t)
+copy l = l
+
+main = "OK"
+--
+OK
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InstanceHierarchy.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InstanceHierarchy.scl
new file mode 100644
index 000000000..1c751f397
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InstanceHierarchy.scl
@@ -0,0 +1,26 @@
+import "Prelude" hiding (zero, one)
+
+class MyAdditive a where
+    zero :: a
+    
+class (MyAdditive a) => MyRing a where
+    one :: a
+
+instance MyAdditive Integer where
+    zero = 0
+
+instance MyRing Integer where
+    one = 1
+    
+data Poly a = Poly [a]
+
+instance (MyAdditive a) => MyAdditive (Poly a) where
+    zero = Poly []
+
+instance (MyRing a) => MyRing (Poly a) where
+    one = Poly [one]
+    
+main :: Poly Integer
+main = one
+--
+[1]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InstanceIsTypoedAsClass.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InstanceIsTypoedAsClass.scl
new file mode 100644
index 000000000..f3973cccd
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InstanceIsTypoedAsClass.scl
@@ -0,0 +1,9 @@
+class Functor f where
+    map :: (a -> b) -> f a -> f b
+    
+data Foo a = Foo a
+
+class Functor Foo where
+    map f (Foo x) = Foo (f x)
+--
+6:1-7:30: Class Functor has already been defined in this module.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InstanceTypeVariables.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InstanceTypeVariables.scl
new file mode 100644
index 000000000..e4ee646a9
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InstanceTypeVariables.scl
@@ -0,0 +1,10 @@
+class Foo a b where
+    foo :: a -> b
+
+instance Foo a a where
+    foo x = x :: a
+
+main = "OK"
+--
+OK
+
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidClass1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidClass1.scl
new file mode 100644
index 000000000..796f92c29
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidClass1.scl
@@ -0,0 +1,4 @@
+class Foo a where
+    infix 3 (+)
+--
+2:5-2:16: Invalid declaration under class definition.    
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidEncoding.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidEncoding.scl
new file mode 100644
index 000000000..513b36f58
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidEncoding.scl
@@ -0,0 +1,3 @@
+main = "Hello w�rld!"
+--
+1:16-1:17: Character does not conform to UTF-8 encoding.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidInstance1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidInstance1.scl
new file mode 100644
index 000000000..35064c645
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidInstance1.scl
@@ -0,0 +1,8 @@
+class Foo a where
+    (+) :: a -> a -> a
+
+instance Foo Double where
+    infix 3 (+)
+    x + y = x
+--
+5:5-5:16: Invalid declaration under instance definition.    
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidJavaTypeAnnotation.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidJavaTypeAnnotation.scl
new file mode 100644
index 000000000..9f7270cd1
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidJavaTypeAnnotation.scl
@@ -0,0 +1,6 @@
+@JavaType 123 "sdf"
+data Foo
+
+main = "Not to be executed"
+--
+1:1-1:20: Invalid parameters. Expected @JavaType "className".
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidKinds.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidKinds.scl
new file mode 100644
index 000000000..a18c062ee
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidKinds.scl
@@ -0,0 +1,7 @@
+
+data List a = Nil | Cons a (List a)
+
+foo :: List
+foo = foo
+--
+4:8-4:12: Expected a type with kind * but got * -> *.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidKinds2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidKinds2.scl
new file mode 100644
index 000000000..ffdf0a9f6
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidKinds2.scl
@@ -0,0 +1,9 @@
+class Functor f where
+    map :: (a -> b) -> f a -> f b
+    
+instance Functor Integer where
+    map = fail "Not implemented."
+
+main = "Not to be executed."
+--
+4:18-4:25: Expected a type with kind * -> * but got *.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidKinds3.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidKinds3.scl
new file mode 100644
index 000000000..445dd7ead
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidKinds3.scl
@@ -0,0 +1,5 @@
+
+data List a = Nil | Cons a List
+
+--
+2:28-2:32: Expected a type with kind * but got ?a -> *.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidLambda.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidLambda.scl
new file mode 100644
index 000000000..8897d43dd
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidLambda.scl
@@ -0,0 +1,4 @@
+
+main = \ /* no parameters */ -> 3
+--
+2:30-2:32: Unexpected token '->' (ARROW). Expected one of ATTACHED_HASH, BEGIN_STRING, BLANK, CHAR, DO, ENFORCE, EQ, ESCAPED_SYMBOL, FLOAT, ID, IF, INTEGER, LAMBDA, LBRACKET, LET, LPAREN, MATCH, MDO, SELECT, SELECT_DISTINCT, SELECT_FIRST, TRANSFORMATION, WHEN.
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidModule.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidModule.scl
new file mode 100644
index 000000000..7cf5f959f
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidModule.scl
@@ -0,0 +1,3 @@
+import "INVALID_MODULE" as Foo
+--
+1:1-1:31: Failed to import INVALID_MODULE, because it does not exist.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidPattern1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidPattern1.scl
new file mode 100644
index 000000000..5ba56cffe
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidPattern1.scl
@@ -0,0 +1,7 @@
+id x = x
+
+f (\x -> x) y = y
+
+main = f id "Foo"
+--
+3:3-3:12: Pattern was expected here.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidPattern2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidPattern2.scl
new file mode 100644
index 000000000..2bf9613e6
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidPattern2.scl
@@ -0,0 +1,5 @@
+if x then y else z = x
+
+main = "Not to be executed."
+--
+1:1-1:19: Illegal left hand side of the definition.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidPattern3.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidPattern3.scl
new file mode 100644
index 000000000..460a9bdfe
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidPattern3.scl
@@ -0,0 +1,8 @@
+foo x = b
+  where
+    if a then b else c = x
+
+main = "Not to be executed."
+--
+1:9-1:10: Couldn't resolve variable b.
+3:5-3:23: Pattern was expected here.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidPattern4.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidPattern4.scl
new file mode 100644
index 000000000..11dfd0702
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidPattern4.scl
@@ -0,0 +1,7 @@
+import "Prelude"
+
+a # b # c = a + b + c
+
+main = "Not to be executed."
+--
+3:1-3:10: Illegal left hand side of the definition.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidRuleset1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidRuleset1.scl
new file mode 100644
index 000000000..e53520b28
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidRuleset1.scl
@@ -0,0 +1,10 @@
+data Fact = Foo | Bar
+
+rs = ruleset
+  Foo => Bar
+
+main = "Not to be executed."
+--
+4:10-4:13: Expected
+    <[Fact]> got
+    .
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidRuleset2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidRuleset2.scl
new file mode 100644
index 000000000..675ee4985
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidRuleset2.scl
@@ -0,0 +1,11 @@
+data Fact = Foo1 | Foo2
+data Fact2 = Bar
+
+rs = ruleset
+  Foo1, Foo2 <=> [Bar]
+
+main = "Not to be executed."
+--
+5:19-5:22: Expected
+     got
+    .
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidTypeClassInstance1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidTypeClassInstance1.scl
new file mode 100644
index 000000000..8860c3b01
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/InvalidTypeClassInstance1.scl
@@ -0,0 +1,9 @@
+
+class Foo a where
+    foo :: a -> Integer
+    
+instance Foo Long where
+    foo x = x
+        
+--
+6:13-6:14: Expected  got .
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/JavaAccess1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/JavaAccess1.scl
new file mode 100644
index 000000000..0576f4385
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/JavaAccess1.scl
@@ -0,0 +1,8 @@
+import "JavaBuiltin" as Java
+
+importJava "java.lang.String" where
+    substring :: String -> Integer -> Integer -> String
+
+main = substring "01234" 1 4
+--
+123
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/JavaConstructors.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/JavaConstructors.scl
new file mode 100644
index 000000000..d3ee020f9
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/JavaConstructors.scl
@@ -0,0 +1,11 @@
+
+@JavaType "org.simantics.scl.runtime.tuple.Tuple3"
+data Tuple3 a b c =
+    @FieldNames [c0, c1, c2] 
+    Tuple3 a b c
+    
+toTuple (Tuple3 x y z) = (x, y, z)
+
+main = toTuple (Tuple3 "x" "y" "z")
+--
+(x,y,z)
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/JavaMethods.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/JavaMethods.scl
new file mode 100644
index 000000000..d51bd4bb8
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/JavaMethods.scl
@@ -0,0 +1,9 @@
+import "JavaBuiltin" as Java
+
+importJava "java.lang.Integer" where
+    @JavaName "parseInt"
+    stringToInteger :: String -> Integer
+
+main = stringToInteger "13"
+--
+13
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/JavaTypes.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/JavaTypes.scl
new file mode 100644
index 000000000..830c5a426
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/JavaTypes.scl
@@ -0,0 +1,14 @@
+import "JavaBuiltin" as Java
+
+importJava "java.math.BigInteger" where
+    data BigInteger
+    
+    @JavaName "add"
+    (+) :: BigInteger -> BigInteger -> BigInteger
+
+    @JavaName ""
+    fromString :: String -> BigInteger
+
+main = fromString "123" + fromString "234"
+--
+357
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Kinds1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Kinds1.scl
new file mode 100644
index 000000000..2f2f0d1d1
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Kinds1.scl
@@ -0,0 +1,15 @@
+class Functor f where
+    map :: (a -> b) -> f a -> f b
+
+data Either a b = Left a | Right b
+
+instance Functor (Either a) where
+    map _ (Left x)  = Left x
+    map f (Right y) = Right (f y)
+
+id :: Integer -> Integer
+id x = x
+
+main = map id (Left (12 :: Integer))
+--
+(Left 12)
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/LP.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/LP.scl
new file mode 100644
index 000000000..0c3ddc121
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/LP.scl
@@ -0,0 +1,86 @@
+import "Prelude"
+
+importJava "gnu.trove.map.hash.TIntFloatHashMap" where
+    data LMap
+    
+    @JavaName adjustOrPutValue
+    adjustLMap_ :: LMap -> Integer -> Float -> Float ->  ()
+    
+@inline
+adjustLMap :: LMap -> Integer -> Float ->  ()
+adjustLMap m k v = adjustLMap_ m k v v
+
+data LPTerm = LPTerm (LMap -> Float ->  ())
+
+instance Additive LPTerm where
+    @inline
+    zero = LPTerm (\_ _ -> ())
+    @inline
+    LPTerm a + LPTerm b = LPTerm (\m s -> do a m s ; b m s)
+    sum ts = LPTerm (\m s -> for ts (\(LPTerm t) -> t m s))
+
+instance Ring LPTerm where
+    @inline
+    neg (LPTerm a) = LPTerm (\m s -> a m (-s))
+    @inline
+    LPTerm a - LPTerm b = LPTerm (\m s -> do a m s ; b m (-s))
+    @inline
+    fromInteger c = LPTerm (\m s -> adjustLMap m (-1) (fromInteger c*s))
+    @inline
+    one = LPTerm (\m s -> adjustLMap m (-1) s)
+    _ * _ = fail "Multiplication is not supported."
+
+data LPProblem = LPProblem (Ref Integer)
+
+newProblem :: () ->  LPProblem
+newProblem _ = LPProblem (ref 0)
+
+newVar :: LPProblem ->  LPTerm
+newVar (LPProblem varCounter) = do
+    curId = getRef varCounter
+    varCounter := curId + 1
+    LPTerm (\m s -> adjustLMap m curId s)
+
+infixl 7 (**)
+
+@inline
+(**) :: Float -> LPTerm -> LPTerm
+s0 ** LPTerm t = LPTerm (\m s -> t m (s0*s))
+
+/*
+data LPTerm = LPTerm Double (Map.T String Double)
+
+instance Additive LPTerm where
+    zero = LPTerm 0 Map.empty
+    LPTerm c1 m1 + LPTerm c2 m2 = LPTerm (c1+c2) (Map.merge (+) m1 m2)
+
+instance Ring LPTerm where
+    one = LPTerm 1 Map.empty
+    neg (LPTerm c m) = LPTerm (-c) (map neg m)
+    LPTerm c1 m1 - LPTerm c2 m2 = LPTerm (c1-c2) (Map.merge (-) m1 m2)
+    
+    LPTerm c1 [] * LPTerm c2 m2 = LPTerm (c1*c2) (Map.merge (\x -> c1*x) m2)
+    LPTerm c1 m1 * LPTerm c2 [] = LPTerm (c1*c2) (Map.merge (\x -> c2*x) m1)
+    _ * _ = fail "Invalid expression: not linear."
+    
+    fromInteger i = LPTerm (fromInteger i) Map.empty
+
+data LPConstraint = LPConstraint String LPTerm
+
+(>==) :: LPTerm -> LPTerm -> String -> [LPConstraint]
+(a >== b) name = [LPConstraint name (a-b)] 
+
+(<==) :: LPTerm -> LPTerm -> String -> [LPConstraint]
+(a <== b) name = [LPConstraint name (b-a)]
+
+*/
+
+testi () = do
+    problem = newProblem ()
+    a = newVar problem
+    b = newVar problem
+    3 ** a + 4 ** b + 15
+
+main = "OK"
+--
+OK
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Lambda.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Lambda.scl
new file mode 100644
index 000000000..2316b30b4
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Lambda.scl
@@ -0,0 +1,13 @@
+
+data List a = Nil | Cons a (List a)
+
+map :: (a -> b) -> List a -> List b
+map f Nil        = Nil
+map f (Cons h t) = Cons (f h) (map f t)
+
+constMap :: a -> List b -> List a
+constMap c = map (\x -> c)
+
+main = constMap (5 :: Integer) (Cons (1 :: Integer) (Cons (2 :: Integer) (Cons (3 :: Integer) Nil)))
+--
+(Cons 5 (Cons 5 (Cons 5 Nil)))
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Layout1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Layout1.scl
new file mode 100644
index 000000000..f68e9a6a7
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Layout1.scl
@@ -0,0 +1,12 @@
+import "Prelude"
+
+foobar x = match x with
+    Left lt -> foo lt
+    Right rt -> bar rt
+  where
+    foo x = "left " + show x
+    bar x = "right " + show x
+
+main = foobar (Left "ASD" :: Either String String)
+--
+left "ASD"
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/List.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/List.scl
new file mode 100644
index 000000000..b746c31ca
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/List.scl
@@ -0,0 +1,7 @@
+
+data List a = Nil | Cons a (List a)
+
+main :: List Integer
+main = Cons (3 :: Integer) Nil
+--
+(Cons 3 Nil)
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListError1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListError1.scl
new file mode 100644
index 000000000..0fece992c
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListError1.scl
@@ -0,0 +1,7 @@
+a :: [Integer]
+a = a
+
+main :: [Double]
+main = [x | x <- a]
+--
+5:9-5:10: Expected  got .
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListError2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListError2.scl
new file mode 100644
index 000000000..135e2704b
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListError2.scl
@@ -0,0 +1,7 @@
+a :: Integer
+a = a
+
+main :: [Integer]
+main = [x | x <- a]
+--
+5:18-5:19: Expected <[a]> got .
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax.scl
new file mode 100644
index 000000000..1747a6f62
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax.scl
@@ -0,0 +1,7 @@
+import "Prelude"
+
+a = [1,2]
+b = [5,9]
+main = [(x,y) | x <- a, y <- b]
+--
+[(1,5), (1,9), (2,5), (2,9)]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax10.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax10.scl
new file mode 100644
index 000000000..726dba5c8
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax10.scl
@@ -0,0 +1,12 @@
+import "Prelude"
+
+dists l = sum [sqrt (dx*dx + dy*dy) 
+              | i <- [0..length l-2]
+              , (x1,y1) = l!i
+              , (x2,y2) = l!(i+1)
+              , dx = x1-x2
+              , dy = y1-y2 ]
+              
+main = dists [(0,0),(1,1),(2,0)]
+--            
+2.8284271247461903
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax11.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax11.scl
new file mode 100644
index 000000000..15f044190
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax11.scl
@@ -0,0 +1,8 @@
+import "Prelude"
+
+dists l = sum [sqrt (x*x+y*y) 
+              | (x,y) <- l ]
+              
+main = dists [(0,0),(1,1),(2,0)]
+--            
+3.414213562373095
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax12.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax12.scl
new file mode 100644
index 000000000..8b18eb3d0
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax12.scl
@@ -0,0 +1,5 @@
+import "Prelude"
+              
+main = foldl (+) 0 [x | y <- [1..10], x <- [1..y]]
+--            
+220
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax2.scl
new file mode 100644
index 000000000..e020731a7
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax2.scl
@@ -0,0 +1,7 @@
+import "Prelude"
+
+a = [1,2]
+b = [5,9]
+main = [(x, y) | x <- a, x==2, y <- b]
+--
+[(2,5), (2,9)]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax3.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax3.scl
new file mode 100644
index 000000000..c6cbbca7d
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax3.scl
@@ -0,0 +1,5 @@
+import "Prelude"
+
+main = [1,2,3,4]
+--
+[1, 2, 3, 4]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax4.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax4.scl
new file mode 100644
index 000000000..a95063523
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax4.scl
@@ -0,0 +1,6 @@
+import "Prelude"
+
+main :: [Integer]
+main = [y | x <- [1..3], y = x+4]
+--
+[5, 6, 7]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax5.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax5.scl
new file mode 100644
index 000000000..6ba833078
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax5.scl
@@ -0,0 +1,8 @@
+import "Prelude"
+
+a = [1,2,3,4]
+
+main :: [Integer]
+main = [x+y | x <- a, y <- a, x!=y]
+--
+[3, 4, 5, 3, 5, 6, 4, 5, 7, 5, 6, 7]
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax6.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax6.scl
new file mode 100644
index 000000000..a3e08c47d
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax6.scl
@@ -0,0 +1,6 @@
+import "Prelude"
+
+main :: [Integer]
+main = [x | x <- [1..10], then take 3]
+--
+[1, 2, 3]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax7.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax7.scl
new file mode 100644
index 000000000..54cf21c54
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax7.scl
@@ -0,0 +1,6 @@
+import "Prelude"
+
+main :: [Integer]
+main = [x | x <- [2,4,3,5,4,6], then sortBy by x]
+--
+[2, 3, 4, 4, 5, 6]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax8.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax8.scl
new file mode 100644
index 000000000..ac5ce5765
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax8.scl
@@ -0,0 +1,12 @@
+import "Prelude"
+
+a :: [Either Integer Integer]
+a = [Left 1, Right 2, Left 3, Right 4]
+
+lefts :: [Either a b] -> [a]
+lefts l = [x | Left x <- l] 
+
+main :: [Integer]
+main = lefts a
+--
+[1, 3]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax9.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax9.scl
new file mode 100644
index 000000000..83de25518
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntax9.scl
@@ -0,0 +1,12 @@
+import "Prelude"
+
+a :: [Either Integer Integer]
+a = [Left 1, Right 2, Left 3, Right 4]
+
+lefts :: [Either a b] -> [a]
+lefts l = [x | y <- l, Left x = y] 
+
+main :: [Integer]
+main = lefts a
+--
+[1, 3]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntaxWithoutPrelude.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntaxWithoutPrelude.scl
new file mode 100644
index 000000000..8806f93a3
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ListSyntaxWithoutPrelude.scl
@@ -0,0 +1,3 @@
+main = [1 :: Integer,2 :: Integer,3 :: Integer]
+--
+[1, 2, 3]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/LocalDefinitions.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/LocalDefinitions.scl
new file mode 100644
index 000000000..0e4690a65
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/LocalDefinitions.scl
@@ -0,0 +1,12 @@
+
+data List a = Nil | Cons a (List a)
+
+reverse :: List a -> List a
+reverse l = do 
+    reverseAux accum Nil        = accum
+    reverseAux accum (Cons h t) = reverseAux (Cons h accum) t
+    reverseAux Nil l
+    
+main = reverse (Cons (1 :: Integer) (Cons (2 :: Integer) (Cons (3 :: Integer) (Cons (4 :: Integer) Nil))))
+--
+(Cons 4 (Cons 3 (Cons 2 (Cons 1 Nil))))
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/LocalDefinitions2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/LocalDefinitions2.scl
new file mode 100644
index 000000000..f17a8beee
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/LocalDefinitions2.scl
@@ -0,0 +1,14 @@
+
+data List a = Nil | Cons a (List a)
+
+hasEvenLength :: List a -> Boolean
+hasEvenLength l = do 
+    even Nil        = True
+    even (Cons _ t) = odd t
+    odd  Nil        = False
+    odd  (Cons _ t) = even t
+    even l
+
+main = hasEvenLength (Cons (1 :: Integer) (Cons (2 :: Integer) (Cons (3 :: Integer) (Cons (4 :: Integer) Nil))))
+--
+true
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/LocalDefinitions3.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/LocalDefinitions3.scl
new file mode 100644
index 000000000..71a193f6e
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/LocalDefinitions3.scl
@@ -0,0 +1,10 @@
+
+data Step s = Skip s
+
+next :: (s -> Step s) -> s -> Step s 
+next next0 ss = match next0 ss with 
+    Skip ss -> Skip ss
+    
+main = next (\x -> Skip x) (3 :: Integer)
+--
+3
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/LocalDefinitions4.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/LocalDefinitions4.scl
new file mode 100644
index 000000000..fe07e4742
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/LocalDefinitions4.scl
@@ -0,0 +1,8 @@
+
+first p = x
+  where 
+    (x,y) = p
+    
+main = first (3 :: Integer,4 :: Integer)
+--
+3
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/LocalDefinitions5.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/LocalDefinitions5.scl
new file mode 100644
index 000000000..97cf72c2f
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/LocalDefinitions5.scl
@@ -0,0 +1,13 @@
+import "Prelude"
+
+data Foo = Foo Double Double
+
+a = Foo 1 2
+
+b = y
+  where
+    Foo x y = a
+    
+main = show b
+--
+2.0
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Macros1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Macros1.scl
new file mode 100644
index 000000000..37b4e053e
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Macros1.scl
@@ -0,0 +1,8 @@
+@macro
+(||) :: Boolean -> Boolean -> Boolean 
+a || b = if a then True else b
+
+main :: Boolean
+main = True || (fail "This should not be executed")
+--
+true
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Macros2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Macros2.scl
new file mode 100644
index 000000000..d1379cd8e
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Macros2.scl
@@ -0,0 +1,13 @@
+import "JavaBuiltin" as Java
+
+(==) = Java.icmpeq
+
+@macro
+(&<&) :: Integer -> Integer -> Integer
+a &<& b = if cmp == 0 then b else cmp
+  where   
+    cmp = a
+
+main = 3 &<& 7
+--
+3
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Macros3.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Macros3.scl
new file mode 100644
index 000000000..ec1571dc0
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Macros3.scl
@@ -0,0 +1,16 @@
+import "JavaBuiltin" as Java
+
+@JavaType "java.util.Collection"
+data Collection a
+
+@macro
+collectionToList :: Collection a -> [a]
+collectionToList = Java.unsafeCoerce
+
+singleton :: a -> Collection a
+singleton = Java.staticMethod "java.util.Collections.singletonList"
+
+main :: [Integer]
+main = collectionToList (singleton 15)
+--
+[15]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Macros4.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Macros4.scl
new file mode 100644
index 000000000..87e41e0af
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Macros4.scl
@@ -0,0 +1,12 @@
+@macro
+@inline
+($) :: (a ->  b) -> a ->  b
+f $ x = f x
+
+justExecute :: (() -> a) -> a
+justExecute f = f ()
+
+main :: Integer
+main = justExecute $ \() -> (13 :: Integer)
+--
+13
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Map1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Map1.scl
new file mode 100644
index 000000000..5d9f38c99
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Map1.scl
@@ -0,0 +1,18 @@
+data List a = Nil | Cons a (List a)
+
+map f Nil = Nil
+map f (Cons h t) = Cons (f h) (map f t)
+
+map2 f l = run l
+  where
+    run Nil = Nil
+    run (Cons h t) = Cons (f h) (run t)
+
+map3 f l = run l
+  where
+    run Nil = Nil
+    run (Cons h t) = Cons (f h) (map f t)
+    
+main = "Foo"
+--
+Foo
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MarketModel.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MarketModel.scl
new file mode 100644
index 000000000..da27b041d
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MarketModel.scl
@@ -0,0 +1,11 @@
+import "Prelude"
+
+l :: [Integer]
+l = [2,3]
+
+f :: Integer ->  [Integer]
+f n = [n] + [0 | i <- l]
+    
+main = f 1
+--
+[1, 0, 0]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MarketModel2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MarketModel2.scl
new file mode 100644
index 000000000..bcd576bb1
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MarketModel2.scl
@@ -0,0 +1,12 @@
+import "IterN"
+import "Random"
+
+foo :: Integer ->  Double
+foo n = runRandom $ foldlN addRandom 0.0 n
+
+addRandom :: Double -> Integer ->  Double
+addRandom v i = v + randomDouble
+
+main = "Foo"
+--
+Foo
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Matching.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Matching.scl
new file mode 100644
index 000000000..f240a1c2b
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Matching.scl
@@ -0,0 +1,19 @@
+
+data List a = Nil | Cons a (List a)
+
+first :: List Integer -> Integer
+//first Nil = 0 :: Integer
+first (Cons x _) = x
+
+reverse :: List a -> List a
+reverse l = reverseAux Nil l 
+  where
+    reverseAux accum Nil = accum
+    reverseAux accum (Cons h t) = reverseAux (Cons h accum) t    
+
+main :: Integer
+main = first (reverse l) 
+  where
+    l = Cons (1 :: Integer) (Cons (2 :: Integer) (Cons (3 :: Integer) Nil))
+--
+3
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Matching2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Matching2.scl
new file mode 100644
index 000000000..c71b98bf8
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Matching2.scl
@@ -0,0 +1,9 @@
+
+main = match (5 :: Integer) with
+  (1 :: Integer) -> "wrong"
+  (2 :: Integer) -> "wrong"
+  (5 :: Integer) -> "right"
+  (6 :: Integer) -> "wrong"
+  _ -> "wrong"
+--
+right
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Matching3.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Matching3.scl
new file mode 100644
index 000000000..904a84397
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Matching3.scl
@@ -0,0 +1,10 @@
+
+a = 5 :: Long
+main = match a with
+  1 -> "wrong"
+  2 -> "wrong"
+  5 -> "right"
+  6 -> "wrong"
+  _ -> "wrong"
+--
+right
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Matching3b.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Matching3b.scl
new file mode 100644
index 000000000..486434583
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Matching3b.scl
@@ -0,0 +1,8 @@
+import "StandardLibrary"
+
+foo 1 = "one"
+foo 2 = "two"
+
+main = if True then foo (2 :: Long) else foo (1 :: Integer)
+--
+two
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Matching4.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Matching4.scl
new file mode 100644
index 000000000..8c8812b50
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Matching4.scl
@@ -0,0 +1,11 @@
+import "Prelude"
+
+dummySum [] = 0
+dummySum [a] = a
+dummySum [a,b] = a + b
+dummySum [a,b,c] = a + b + c
+dummySum l = sum l
+
+main = dummySum [] + dummySum [1] + dummySum [1,2] + dummySum [1,2,3]
+--
+10
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Matching5.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Matching5.scl
new file mode 100644
index 000000000..61723e8f8
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Matching5.scl
@@ -0,0 +1,5 @@
+main = do 
+    match () with
+        () -> "Hello"
+--
+Hello
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MatchingWithMissingParameter.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MatchingWithMissingParameter.scl
new file mode 100644
index 000000000..33a681fd3
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MatchingWithMissingParameter.scl
@@ -0,0 +1,10 @@
+data Foo = Foo Integer Integer
+         | Bar
+         
+isFoo (Foo _) = True
+isFoo _ = False
+
+main = "Hello world!"
+--
+4:7-4:14: The function is applied with too few parameters.
+
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MatchingWithoutTypeAnnotations.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MatchingWithoutTypeAnnotations.scl
new file mode 100644
index 000000000..80d147773
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MatchingWithoutTypeAnnotations.scl
@@ -0,0 +1,10 @@
+
+data List a = Nil | Cons a (List a)
+
+//first :: List Integer -> Integer
+first Nil = 0 :: Integer
+first (Cons x _) = x
+
+main = first (Cons (9 :: Integer) (Cons (8 :: Integer) Nil))
+--
+9
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MaximumBy.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MaximumBy.scl
new file mode 100644
index 000000000..c70c9dae8
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MaximumBy.scl
@@ -0,0 +1,10 @@
+import "Prelude" hiding (maximumBy)
+
+maximumBy :: Ord b => (a -> b) -> [a] -> a
+maximumBy f = snd . foldl1 maxF . map (\x -> (f x, x))
+  where
+    maxF (a @ (aV,_)) (b @ (bV,_)) = if aV >= bV then a else b
+  
+main = maximumBy (`mod` 10) [1::Integer, 14, 23, 9, 14, 67]  
+--
+9
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Maybe1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Maybe1.scl
new file mode 100644
index 000000000..98a7d71ae
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Maybe1.scl
@@ -0,0 +1,6 @@
+a = Nothing
+main = match a with
+    Nothing -> "Correct"
+    Just x -> "Incorrect"
+--
+Correct
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Maybe2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Maybe2.scl
new file mode 100644
index 000000000..0bc50fb5e
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Maybe2.scl
@@ -0,0 +1,6 @@
+a = Just True
+main = match a with
+    Nothing -> False
+    Just x -> x
+--
+true
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Maybe3.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Maybe3.scl
new file mode 100644
index 000000000..8a0f12070
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Maybe3.scl
@@ -0,0 +1,6 @@
+a = Just "ABC"
+main = match a with
+    Nothing -> "Incorrect"
+    Just x -> x
+--
+ABC
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Maybe4.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Maybe4.scl
new file mode 100644
index 000000000..53e3ed3d7
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Maybe4.scl
@@ -0,0 +1,18 @@
+import "Prelude" hiding (fromMaybe)
+import "Random"
+
+importJava "org.simantics.scl.compiler.tests.imports.Maybe4Imports" where
+    toMaybeDouble :: String ->  Maybe a
+
+fromMaybe :: a -> Maybe a -> a
+fromMaybe _ (Just v) = v
+fromMaybe def _ = def
+
+f x = do
+    a = fromMaybe (-1.0) (toMaybeDouble x)
+    b = fromMaybe (-1.0) (toMaybeDouble ("1" + x))
+    a+b
+
+main = withSeed 123 (f "2.0")
+--
+14.0
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MissingEffect.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MissingEffect.scl
new file mode 100644
index 000000000..9a5ea98c4
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MissingEffect.scl
@@ -0,0 +1,15 @@
+import "Prelude" hiding (findFirst)
+
+findFirst :: (a ->  Maybe b) -> [a] -> Maybe b
+findFirst f l = loop 0
+  where
+    len = length l
+    loop i                         
+        | i >= len  = Nothing
+        | otherwise = match f (l!i) with
+                        s @ (Just _) -> s
+                        Nothing      -> loop (i+1)
+
+main = "Not to be executed"
+--
+9:29-9:36: Side-effect a is forbidden here.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MissingMethod.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MissingMethod.scl
new file mode 100644
index 000000000..d5b1b27f8
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MissingMethod.scl
@@ -0,0 +1,12 @@
+class FooBar a where
+    foo :: a -> a
+    bar :: a -> a
+    
+instance FooBar Integer where
+    foo x = x
+
+main = "Not to be executed."
+--
+5:1-6:14: Method bar is not defined.
+    
+
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MissingTypeParameter.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MissingTypeParameter.scl
new file mode 100644
index 000000000..fc40ab9cd
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MissingTypeParameter.scl
@@ -0,0 +1,6 @@
+importJava "java.util.Arrays" where
+    toString :: MVector /*Double*/ -> String
+
+main = "Not to be executed."
+--
+???
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ModuleInitialization.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ModuleInitialization.scl
new file mode 100644
index 000000000..bd3dfa10b
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ModuleInitialization.scl
@@ -0,0 +1,11 @@
+import "Prelude"
+
+d = map (+1) c
+a = [1,2,3]
+c = map (+1) b
+b = map (+1) a
+e = map (+1) d
+
+main = e
+--
+[5, 6, 7]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MonadBug1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MonadBug1.scl
new file mode 100644
index 000000000..44ddf00e6
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MonadBug1.scl
@@ -0,0 +1,10 @@
+class Monad m where
+   (>>=) :: m a -> (a -> m b) -> m b
+
+@macro
+(>>) :: Monad m => m a -> m b -> m b
+ma >> mb = ma >>= (\_ -> mb)
+
+main = "OK"
+--
+OK
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MonadSyntax1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MonadSyntax1.scl
new file mode 100644
index 000000000..94774a4f2
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MonadSyntax1.scl
@@ -0,0 +1,11 @@
+import "Prelude"
+
+a = [1, 2]
+b = [5, 6]
+
+main = mdo 
+  x <- a
+  y <- b
+  return (x+y) 
+--
+[6, 7, 7, 8]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MonadSyntax2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MonadSyntax2.scl
new file mode 100644
index 000000000..6244a6162
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/MonadSyntax2.scl
@@ -0,0 +1,12 @@
+import "Prelude"
+
+a = [1, 2]
+b = [5, 6]
+
+main = do 
+  x <- a
+  y <- b
+  return (x+y) 
+--
+7:3-7:9: Bind statements are allowed only in mdo-blocks.
+8:3-8:9: Bind statements are allowed only in mdo-blocks.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Monads1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Monads1.scl
new file mode 100644
index 000000000..90115d3ff
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Monads1.scl
@@ -0,0 +1,8 @@
+import "Prelude"
+
+a = [1, 2]
+b = [5, 6]
+
+main = a >>= \x -> b >>= \y -> return (x+y) 
+--
+[6, 7, 7, 8]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/NoDefinitionErrorMessage.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/NoDefinitionErrorMessage.scl
new file mode 100644
index 000000000..8597a78ac
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/NoDefinitionErrorMessage.scl
@@ -0,0 +1,7 @@
+// Idea of this test is that the missing definition should only cause
+// one error message pointing to the type declaration.
+a :: Double
+b :: Double
+b = a
+--
+3:1-3:12: a is not defined.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/NoInstance.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/NoInstance.scl
new file mode 100644
index 000000000..4bc5cabba
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/NoInstance.scl
@@ -0,0 +1,9 @@
+
+class Foo a where
+    foo :: a
+
+x :: Double
+x = foo
+
+--
+6:5-6:8: Constraint  is not given and cannot be derived.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/NoInstance2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/NoInstance2.scl
new file mode 100644
index 000000000..9490a822d
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/NoInstance2.scl
@@ -0,0 +1,10 @@
+
+class Foo a where
+    foo :: a
+
+y :: Double
+y = y
+    
+x = if True then foo else y
+--
+8:18-8:21: There is no instance for .
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/NonassociativeOperator.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/NonassociativeOperator.scl
new file mode 100644
index 000000000..f2b473ae9
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/NonassociativeOperator.scl
@@ -0,0 +1,7 @@
+infix 3 (+)
+
+a + b = a
+
+threeTimes x = x + x + x
+--
+5:22-5:23: Operator + is not associative.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/NonexistentTypeClassInAnnotation.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/NonexistentTypeClassInAnnotation.scl
new file mode 100644
index 000000000..40306395a
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/NonexistentTypeClassInAnnotation.scl
@@ -0,0 +1,9 @@
+import "Prelude"
+
+deepId :: FooBar b => b -> a -> a
+deepId count x = deepId2 count x
+deepId2 count x = if count <= 0 then x else deepId (count-1) x
+
+main = deepId (5 :: Integer) "FOO"
+--
+3:11-3:17: Unresolved type class FooBar.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/NonexistingEffect.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/NonexistingEffect.scl
new file mode 100644
index 000000000..8ed51f912
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/NonexistingEffect.scl
@@ -0,0 +1,6 @@
+importJava "java.lang.System" where
+    nanoTime :: () ->  Long
+
+main = nanoTime ()
+--
+2:24-2:28: Didn't find effect constructor Pred.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/OneLineMatch.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/OneLineMatch.scl
new file mode 100644
index 000000000..3f16b0783
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/OneLineMatch.scl
@@ -0,0 +1,6 @@
+// Tests that match cases can be given in the same line as the scrutinee
+data FooBar a = Foo a | Bar a
+extract x = match x with Foo v -> v ; Bar v -> v
+main = extract (Foo (3 :: Integer))
+--
+3
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/OpenString1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/OpenString1.scl
new file mode 100644
index 000000000..e8a7c4c76
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/OpenString1.scl
@@ -0,0 +1,3 @@
+"asd 
+--
+1:1-1:6: Unclosed string literal.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/OpenString2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/OpenString2.scl
new file mode 100644
index 000000000..31953cc01
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/OpenString2.scl
@@ -0,0 +1,6 @@
+"asd 
+
+
+asd
+--
+1:1-1:6: Unclosed string literal.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/OverloadedArithmetic1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/OverloadedArithmetic1.scl
new file mode 100644
index 000000000..de083e814
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/OverloadedArithmetic1.scl
@@ -0,0 +1,17 @@
+import "JavaBuiltin" as Java
+
+infixl 6 (+)
+
+class Additive a where 
+    (+)  :: a -> a -> a    
+    
+instance Additive Double where
+    x + y = Java.dadd x y
+
+instance Additive Integer where
+    x + y = Java.iadd x y
+    
+main = ((1.0 :: Double)+(2.0 :: Double),(3::Integer)+(4::Integer))
+--
+(3.0,7)
+
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/OverloadedArithmetic2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/OverloadedArithmetic2.scl
new file mode 100644
index 000000000..1619f585d
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/OverloadedArithmetic2.scl
@@ -0,0 +1,20 @@
+import "JavaBuiltin" as Java
+
+infixl 6 (+)
+
+class Additive a where 
+    (+)  :: a -> a -> a    
+    
+instance Additive Double where
+    x + y = Java.dadd x y
+
+instance Additive Integer where
+    x + y = Java.iadd x y
+    
+instance (Additive a, Additive b) => Additive (a,b) where
+    (x1,y1) + (x2,y2) = (x1+x2,y1+y2)
+    
+main = (1.0::Double,3::Integer) + (2.0::Double,4::Integer)
+--
+(3.0,7)
+
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/OverloadedArithmetic3.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/OverloadedArithmetic3.scl
new file mode 100644
index 000000000..230e486cf
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/OverloadedArithmetic3.scl
@@ -0,0 +1,27 @@
+import "JavaBuiltin" as Java
+
+infixl 7 (*)
+infixl 6 (+)
+
+class Additive a where 
+    (+) :: a -> a -> a    
+
+class (Additive a) => Ring a where
+    (*) :: a -> a -> a
+
+instance Additive Double where
+    x + y = Java.dadd x y
+
+instance Ring Double where
+    x * y = Java.dmul x y
+
+instance Additive Integer where
+    x + y = Java.iadd x y
+
+instance Ring Integer where
+    x * y = Java.imul x y
+
+main = (1.0 :: Double)*(2.0 :: Double)+(3.0 :: Double)*(4.0 :: Double)
+--
+14.0
+
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/OverloadedLiterals2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/OverloadedLiterals2.scl
new file mode 100644
index 000000000..c1670972a
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/OverloadedLiterals2.scl
@@ -0,0 +1,9 @@
+import "Prelude"
+
+inc :: Ring a => a -> a
+inc x = x + 1
+
+main :: Double
+main = inc 34
+--
+35.0
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Overloading1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Overloading1.scl
new file mode 100644
index 000000000..dfb607934
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Overloading1.scl
@@ -0,0 +1,19 @@
+// module Foo1
+import "Prelude"
+
+foo :: Integer -> Boolean
+foo i = i == 5
+--
+// module Foo2
+import "Prelude"
+
+foo :: Integer -> Integer -> Boolean
+foo i j = i == j
+--
+import "Prelude"
+import "Foo1"
+import "Foo2"
+
+main = foo 5 && foo 5 4
+--
+false
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Overloading2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Overloading2.scl
new file mode 100644
index 000000000..e7a28de00
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Overloading2.scl
@@ -0,0 +1,20 @@
+// module Max1
+import "Prelude"
+
+myMax :: Ord a => a -> a -> a
+myMax = max
+--
+// module Max2
+import "Prelude"
+
+myMax :: Ord a => a -> a -> a -> a
+myMax a b c = max a (max b c)
+--
+import "Prelude"
+import "Max1"
+import "Max2"
+
+main = myMax (1 :: Integer) 2 3 + myMax (3 :: Integer) 2 1 + myMax 4 2 :: Integer
+--
+10
+
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Overloading3.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Overloading3.scl
new file mode 100644
index 000000000..9385ce7bc
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Overloading3.scl
@@ -0,0 +1,23 @@
+// module M1
+import "Prelude"
+
+foo :: Ring a => Boolean -> a -> a
+foo True v = v+1
+foo False v = v+1 
+--
+// module M2
+import "Prelude"
+
+foo :: Ring a => String -> a -> a
+foo cond v = if cond=="true"
+             then v+1
+             else v-1 
+--
+import "Prelude"
+import "M1"
+import "M2"
+
+main = foo False (foo "True" 10) :: Integer
+--
+10
+
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Parsing.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Parsing.scl
new file mode 100644
index 000000000..0ca342a34
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Parsing.scl
@@ -0,0 +1,62 @@
+import "Prelude"
+
+"""
+Parser is a function from a string and a position in the
+string to a possible semantics of a substring and the 
+end of the substring.
+"""
+data Parser a = Parser (String -> Integer -> Maybe (a, Integer))
+
+runParser :: Parser a -> String -> Integer -> Maybe (a, Integer)
+runParser (Parser f) = f 
+
+instance Functor Parser where
+    fmap f (Parser p) = Parser (\input pos -> match p input pos with
+        Nothing -> Nothing
+        Just (a, newPos) -> Just (f a, newPos)
+    )
+
+instance Monad Parser where
+    return x = Parser (\_ pos -> Just (x, pos))    
+    (pa >>= f) = Parser (\input pos -> match runParser pa input pos with
+        Nothing -> Nothing
+        Just (a, newPos) -> runParser (f a) input newPos
+    )  
+
+(|||) :: Parser a -> Parser a -> Parser a
+Parser a ||| Parser b = Parser (\input pos -> match a input pos with
+    Nothing -> b input pos
+    Just x -> Just x
+) 
+
+keyword :: String -> Parser ()
+keyword word = Parser (\input pos ->
+    if regionMatches word 0 input pos (length word)
+      then Just ((), pos + (length word))
+      else Nothing
+)
+
+data List a = Nil | Cons a (List a)
+
+listSepL :: Parser () -> Parser a -> Parser (List a)
+listSepL sep el = mdo
+    head <- el
+    tail <- (sep >> listSepL sep el) ||| return Nil
+    return (Cons head tail) 
+
+fromList :: List a -> [a]
+fromList = unfoldr gen
+  where
+    gen Nil        = Nothing
+    gen (Cons h t) = Just (h, t)
+    
+listSep :: Parser () -> Parser a -> Parser [a]    
+listSep sep el = fmap fromList (listSepL sep el) 
+
+aOrB = (keyword "a" >> return "a") ||| (keyword "b" >> return "b")
+
+myParser = listSep (keyword ",") aOrB
+
+main = show (runParser myParser "a,b,b,a" 0)
+--
+Just (["a", "b", "b", "a"], 7)
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/PatternError.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/PatternError.scl
new file mode 100644
index 000000000..ce417affc
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/PatternError.scl
@@ -0,0 +1,22 @@
+import "Prelude"
+
+data CP = CP Integer String
+
+//cpName :: CP -> String
+cpName CP cp name = name
+
+connectionPoints :: Integer -> [(CP, CP)]
+connectionPoints n = []
+
+hasTerminalProblems :: Integer -> Boolean
+hasTerminalProblems uc =
+    let cps = connectionPoints uc
+        cpCount = length cps
+        dcps = map snd cps
+        dnames = map cpName dcps
+        dcpNameCount = length $ unique $ sort dnames
+    in cpCount != dcpNameCount
+
+main = "Not OK"
+--
+6:8-6:10: ???
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/PolymorphicRecursion.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/PolymorphicRecursion.scl
new file mode 100644
index 000000000..ed84193b7
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/PolymorphicRecursion.scl
@@ -0,0 +1,16 @@
+// Idea here is to test the following property:
+// When a function is called recursively it has locally a monomorphic type.
+// Therefore the definition of cons does not work even if it works
+// with a proper type annotation.
+
+data Vec a = Nil | Zero (Vec (a,a)) | One a (Vec (a,a))
+
+// cons :: a -> Vec a -> Vec a
+cons x Nil        = One x Nil
+cons x (Zero ps)  = One x ps
+cons x (One y ps) = Zero (cons (x, y) ps)
+--
+11:21-11:42: Expected  got .
+11:33-11:34: Type (a, a) is not a subtype of a.
+11:36-11:37: Type (a, a) is not a subtype of a.
+11:39-11:41: Expected  got .
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/PolymorphicRecursion2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/PolymorphicRecursion2.scl
new file mode 100644
index 000000000..44738e055
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/PolymorphicRecursion2.scl
@@ -0,0 +1,12 @@
+
+
+class Foo a where
+    foo :: a
+    
+data List a = Nil | Cons a (List a)
+
+r x = Cons foo (r x)
+
+main = (1.0 :: Double)
+--
+1.0
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Polynomials.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Polynomials.scl
new file mode 100644
index 000000000..5e35b9424
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Polynomials.scl
@@ -0,0 +1,50 @@
+import "Prelude"
+
+data Poly a = Poly [a]
+
+normalize l = go (length l)
+  where
+    go i = if i > 0 && l!(i-1)==zero
+           then go (i-1)
+           else take i l 
+
+instance (Eq a, Additive a) => Additive (Poly a) where
+    zero = Poly []
+    Poly a + Poly b = 
+        Poly ( 
+            normalize (
+               zipWith (+) a b + 
+                 if la > lb
+                 then drop lb a
+                 else drop la b
+            )
+        )
+          where
+            la = length a
+            lb = length b        
+
+instance (Eq a, Ring a) => Ring (Poly a) where
+    one = Poly [one]
+    neg (Poly l) = Poly (map neg l)
+    a - b = a + (neg b)
+    Poly a * Poly b = 
+        Poly ( if aDeg < bDeg
+               then [ segSum n 0        n    | n <- [0     ..aDeg]   ]
+                  + [ segSum n 0        aDeg | n <- [aDeg+1..bDeg]   ]
+                  + [ segSum n (n-bDeg) aDeg | n <- [bDeg+1..sumDeg] ]
+               else [ segSum n 0        n    | n <- [0     ..bDeg]   ]
+                  + [ segSum n (n-bDeg) n    | n <- [bDeg+1..aDeg]   ]
+                  + [ segSum n (n-bDeg) aDeg | n <- [aDeg+1..sumDeg] ]
+             )
+          where 
+            aDeg = length a - 1
+            bDeg = length b - 1
+            sumDeg = aDeg + bDeg
+            segSum n low high = sum [ a!i * b!(n-i) | i <- [low..high] ]
+    fromInteger x = Poly [fromInteger x]
+            
+a = Poly [4.0,5.0,8.0,3.0,2.0,1.0]
+b = Poly [1.0,0.0,2.0,1.0]
+main = a * a + a * b + b * a + b * b - (a+b)*(a+b)
+--
+[]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/PrecedenceOfNonoperators.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/PrecedenceOfNonoperators.scl
new file mode 100644
index 000000000..3321e8958
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/PrecedenceOfNonoperators.scl
@@ -0,0 +1,9 @@
+infixr 2 l
+infixr 1 r
+
+l a b = a
+r a b = b
+
+main = (1 :: Integer) `l` (2 :: Integer) `r` (3 :: Integer) `l` (4 :: Integer)
+--
+3
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Primes.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Primes.scl
new file mode 100644
index 000000000..830186ba7
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Primes.scl
@@ -0,0 +1,24 @@
+import "JavaBuiltin" as Java
+
+infixl 7  (%)
+infixl 6  (+)
+infix  4  (==), (<)
+
+(+) = Java.iadd
+(%) = Java.irem
+(<) = Java.icmplt
+(==) = Java.icmpeq
+
+isPrime p = isPrimeAux (2 :: Integer) p
+  where
+    isPrimeAux d p = if d == p then True
+                     else if p % d == 0 then False
+                     else isPrimeAux (d+1) p
+                   
+nextPrime p = if isPrime p 
+              then p
+              else nextPrime (p+(1 :: Integer))                   
+                     
+main = nextPrime 32
+--
+37
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Proc1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Proc1.scl
new file mode 100644
index 000000000..b91eed34f
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Proc1.scl
@@ -0,0 +1,8 @@
+import "Prelude"
+   
+main = do
+    r = ref 13
+    r := 14
+    getRef r
+--
+14
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Proc2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Proc2.scl
new file mode 100644
index 000000000..bd8021b13
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Proc2.scl
@@ -0,0 +1,29 @@
+import "Prelude"
+
+data RealWorld = RealWorld
+data IO a = IO (RealWorld -> (RealWorld, a))
+
+@inline
+unIO (IO m) = m 
+
+instance Functor IO where
+    @inline
+    fmap f x = x >>= (return . f)
+
+instance Monad IO where
+    @inline
+    return x = IO (\s -> (s, x))
+    
+    @inline
+    (IO m) >>= f = IO (\s -> do
+        (newS, v) = m s
+        unIO (f v) newS 
+    )
+
+@inline
+runIO :: IO a -> a
+runIO m = snd (unIO m RealWorld)
+
+main = runIO (return (13 :: Integer))
+--
+13
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Proc3.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Proc3.scl
new file mode 100644
index 000000000..4a17fe480
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Proc3.scl
@@ -0,0 +1,16 @@
+import "Prelude"
+
+@private
+repeat :: Integer -> (() ->  a) ->  ()
+repeat n proc = 
+    if n > 0 then do
+        proc ()
+        repeat (n-1) proc
+    else () 
+
+main = do
+    a = ref 1 
+    repeat 3 (\() -> a := 2)
+    getRef a
+--
+2
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Pythagoras.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Pythagoras.scl
new file mode 100644
index 000000000..95a5b9b78
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Pythagoras.scl
@@ -0,0 +1,24 @@
+import "JavaBuiltin" as Java
+
+infixl 7 (*)
+infixl 6 (+)
+
+(+) :: Double -> Double -> Double
+(+) = Java.dadd
+(*) :: Double -> Double -> Double
+(*) = Java.dmul
+
+importJava "java.lang.Math" where
+    sqrt :: Double -> Double
+    sin :: Double -> Double
+    cos :: Double -> Double
+
+square x = x * x
+
+length x y = sqrt (square x + square y)
+
+pythagoras a = length (cos a) (sin a)
+
+main = pythagoras 2.0
+--
+1.0
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Random1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Random1.scl
new file mode 100644
index 000000000..66922eadb
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Random1.scl
@@ -0,0 +1,19 @@
+import "StandardLibrary"
+
+// Actual program
+
+"""This function returns either 0 or 1 such that
+the expected value is pi/4"""
+approximatePi :: () ->  Double
+approximatePi () = if x*x + y*y < 1 then 1 else 0
+  where
+    x = randomDouble
+    y = randomDouble
+
+averageOfNRepeats n f = sum [f () | n <- [1..n]] / fromInteger n
+
+betterApproximatePi () = averageOfNRepeats 1000 approximatePi * 4
+
+main = withSeed 13 (betterApproximatePi ())
+--
+3.068
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/RangeSyntax.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/RangeSyntax.scl
new file mode 100644
index 000000000..662255248
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/RangeSyntax.scl
@@ -0,0 +1,5 @@
+import "Prelude"
+
+main = [5..9]
+--
+[5, 6, 7, 8, 9]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Record1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Record1.scl
new file mode 100644
index 000000000..fcfbf6a25
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Record1.scl
@@ -0,0 +1,9 @@
+import "Prelude"
+
+data XYS = XYS { x :: Double, y :: Double, s :: String }
+
+len XYS {x = x, y = y} = sqrt (x*x + y*y)
+
+main = len (XYS { x = 4, y = 3, s = "Hello world!" })
+-- 
+5.0
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/RecordShorthand.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/RecordShorthand.scl
new file mode 100644
index 000000000..d3af73e02
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/RecordShorthand.scl
@@ -0,0 +1,11 @@
+import "Prelude"
+
+data Vec = Vec { x :: Double, y :: Double }
+deriving instance Show Vec
+
+createVec x y = Vec {x, y}
+sumVec Vec { x1, y1 } Vec { x2, y2 } = Vec { x = x1+x2, y = y1 + y2 }
+
+main = sumVec (createVec 1 2) (createVec 3 4)
+-- 
+(Vec 4.0 6.0)
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/RecursiveContext.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/RecursiveContext.scl
new file mode 100644
index 000000000..741286211
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/RecursiveContext.scl
@@ -0,0 +1,23 @@
+import "JavaBuiltin" as Java
+
+infixl 6  (+), (-)
+
+class Num a where 
+    (+) :: a -> a -> a
+    (-) :: a -> a -> a
+    isZero :: a -> Boolean
+    one :: a     
+    
+instance Num Integer where
+    x + y = Java.iadd x y
+    x - y = Java.isub x y
+    isZero x = Java.icmpeq Java.iconst_0 x
+    one = Java.iconst_1
+  
+even x = if isZero x then True else odd (x - one)
+odd x = if isZero x then False else even (x - one)   
+    
+main = odd (8 :: Integer)
+--
+false
+
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/RecursiveValues.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/RecursiveValues.scl
new file mode 100644
index 000000000..554852045
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/RecursiveValues.scl
@@ -0,0 +1,6 @@
+a = b
+b = a
+
+main = a
+--
+???: Variables defined recursively must all be functions.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/RecursiveValues2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/RecursiveValues2.scl
new file mode 100644
index 000000000..7291bb990
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/RecursiveValues2.scl
@@ -0,0 +1,5 @@
+f g = a
+  where 
+    a = g a
+--
+3:11-3:12: Couldn't resolve variable a.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/RecursiveValues3.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/RecursiveValues3.scl
new file mode 100644
index 000000000..020a967d5
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/RecursiveValues3.scl
@@ -0,0 +1,12 @@
+data Nat = O | S Nat
+
+// It is important for this test that even and or are not annotated
+even O = True
+even (S x) = odd x
+
+odd O = False
+odd (S x) = even x
+
+main = even (S (S (S (S (S O)))))
+--
+false
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/RecursiveValues4.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/RecursiveValues4.scl
new file mode 100644
index 000000000..0315be34a
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/RecursiveValues4.scl
@@ -0,0 +1,5 @@
+(foo, bar) = (foo, "Hello world!")
+
+main = "ERROR"
+--
+1:1-1:11: Illegal left hand side of the definition.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/RedBlackTrees.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/RedBlackTrees.scl
new file mode 100644
index 000000000..6f47bd013
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/RedBlackTrees.scl
@@ -0,0 +1,101 @@
+import "Prelude"
+
+// Version 1, 'untyped'
+data Color = R | B 
+
+deriving instance Show Color
+
+data RB a = E | T Color (RB a) a (RB a)
+
+rbToList :: RB a -> [a]
+rbToList E = []
+rbToList (T _ l a r) = rbToList l + [a] + rbToList r
+
+deriving instance (Show a) => Show (RB a)
+
+// Insertion and membership test as by Okasaki
+insert :: Ord a => a -> RB a -> RB a
+insert x s = (match ins s with T _ a z b -> T B a z b) 
+    where
+        ins E = T R E x E
+        ins s = match s with
+          T B a y b ->
+              if xy 
+              then balance a y (ins b)
+              else s
+          T R a y b ->
+              if xy 
+              then T R a y (ins b)
+              else s
+
+member :: Ord a => a -> RB a -> Boolean
+member x E = False
+member x (T _ a y b)
+    | xy = member x b
+    | otherwise = True
+
+// balance: first equation is new, to make it work with a weaker invariant
+balance :: RB a -> a -> RB a -> RB a
+balance (T R a x b) y (T R c z d) = T R (T B a x b) y (T B c z d)
+balance (T R (T R a x b) y c) z d = T R (T B a x b) y (T B c z d)
+balance (T R a x (T R b y c)) z d = T R (T B a x b) y (T B c z d)
+balance a x (T R b y (T R c z d)) = T R (T B a x b) y (T B c z d)
+balance a x (T R (T R b y c) z d) = T R (T B a x b) y (T B c z d)
+balance a x b = T B a x b
+
+// deletion a la SMK 
+delete :: Ord a => a -> RB a -> RB a
+delete x t = (match del t with 
+               T _ a y b -> T B a y b
+               _ -> E)
+      where
+        del E = E
+        del (T _ a y b)
+            | xy = delformRight a y b
+            | otherwise = app a b
+        delformLeft a y b= match a with
+            T B _ _ _ -> balleft (del a) y b
+            _         -> T R (del a) y b
+        delformRight a y b = match b with
+            T B _ _ _ -> balright a y (del b)
+            _         -> T R a y (del b)
+
+balleft :: RB a -> a -> RB a -> RB a
+balleft (T R a x b) y c = T R (T B a x b) y c
+balleft bl x (T B a y b) = balance bl x (T R a y b)
+balleft bl x (T R (T B a y b) z c) = T R (T B bl x a) y (balance b z (sub1 c))
+
+balright :: RB a -> a -> RB a -> RB a
+balright a x (T R b y c) = T R a x (T B b y c)
+balright (T B a x b) y bl = balance (T R a x b) y bl
+balright (T R a x (T B b y c)) z bl = T R (balance (sub1 a) x b) y (T B c z bl)
+
+sub1 :: RB a -> RB a
+sub1 (T B a x b) = T R a x b
+sub1 _ = fail "invariance violation"
+
+app :: RB a -> RB a -> RB a
+app E x = x
+app x E = x
+app (T R a x b) (T R c y d) =
+    match app b c with
+        T R b' z c' -> T R(T R a x b') z (T R c' y d)
+        bc -> T R a x (T R bc y d)
+app (T B a x b) (T B c y d) = 
+    match app b c with
+        T R b' z c' -> T R(T B a x b') z (T B c' y d)
+        bc -> balleft a x (T B bc y d)
+app a (T R b x c) = T R (app a b) x c
+app (T R a x b) c = T R a x (app b c)
+
+testList = [4,6,2,7,4,7,2,5]
+
+main = rbToList (foldl (flip insert) E testList)
+--
+[2, 4, 5, 6, 7]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Relations1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Relations1.scl
new file mode 100644
index 000000000..b15c1b861
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Relations1.scl
@@ -0,0 +1,18 @@
+import "StandardLibrary"
+
+Concat ?x ?y ?z :-    
+    @bbf 1
+    ?z = ?x + ?y
+    
+    @bfb 0.5
+    startsWith ?z ?x
+    ?y = drop (length ?x) ?z
+    
+    @fbb 0.5
+    endsWith ?z ?y
+    ?x = take (length ?z - length ?y) ?z
+
+main = select ?y where 
+    Concat "Hello " ?y "Hello world!"
+--
+[world!]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Relations2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Relations2.scl
new file mode 100644
index 000000000..825685edc
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Relations2.scl
@@ -0,0 +1,12 @@
+import "StandardLibrary"
+
+MyExecute ?procedure :-
+    @enforce
+    Execute (?procedure "Foo")
+
+main = do
+    v = ref ""
+    enforce MyExecute (v :=)
+    getRef v 
+--
+Foo
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/RepeatedVariableInPattern.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/RepeatedVariableInPattern.scl
new file mode 100644
index 000000000..8e5cf084c
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/RepeatedVariableInPattern.scl
@@ -0,0 +1,6 @@
+invalidProject :: (a,a) -> a
+invalidProject (a,a) = a
+
+main = "Not to be executed!"
+--
+2:19-2:20: Repeated variable a in pattern.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/SSATypingBug.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/SSATypingBug.scl
new file mode 100644
index 000000000..15ff01275
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/SSATypingBug.scl
@@ -0,0 +1,11 @@
+import "Prelude"
+
+test :: () -> [Integer]
+test _ = do
+  header = [0]
+  rows = map (\foo -> map (\bar -> bar) [foo]) [1..5]
+  foldl (+) header rows
+  
+main = test ()
+--
+[0, 1, 2, 3, 4, 5]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Scanl.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Scanl.scl
new file mode 100644
index 000000000..67821ddef
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Scanl.scl
@@ -0,0 +1,14 @@
+import "Prelude" hiding (scanl)
+
+scanl :: (b -> a ->  b) -> b -> [a] ->  [b]
+scanl f initial l = build (loop initial 0)
+  where
+    len = length l
+    loop cur i accum cons = let nl = cons accum cur
+                            in if i==len
+                               then nl
+                               else loop (f cur (l!i)) (i+1) nl cons
+                               
+main = scanl (+) 0 [1,2,3]
+--
+[0, 1, 3, 6]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Search.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Search.scl
new file mode 100644
index 000000000..352e92c74
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Search.scl
@@ -0,0 +1,47 @@
+import "Prelude" hiding (findFirst)
+
+infinity = 1e9
+
+@inline
+findFirst :: (a ->  Maybe b) -> [a] ->  Maybe b
+findFirst f l = loop 0
+  where
+    len = length l
+    loop i                         
+        | i >= len  = Nothing
+        | otherwise = match f (l!i) with
+                        s @ (Just _) -> s
+                        Nothing      -> loop (i+1)
+
+dfsFirst :: (a ->  Boolean) -> (a ->  [a]) -> [a] ->  (Maybe a)
+dfsFirst acceptable successors initial = tryAll initial
+  where
+    tryAll l = findFirst loop l
+    loop p 
+        | acceptable p = Just p
+        | otherwise    = tryAll (successors p)
+
+data Weighted a = Weighted a Double
+
+//type SearchAlgorithm e a =
+//    (a ->  Boolean) -> (a ->  [Weighted a]) -> [Weighted a] ->  Weighted (Maybe a)
+    
+//dfs :: SearchAlgorithm e a
+dfs :: (a ->  Boolean) -> (a ->  [Weighted a]) -> [Weighted a] ->  Weighted (Maybe a)
+dfs acceptable successors initial = foldl loop (Weighted Nothing infinity) initial
+  where
+    loop best@(Weighted _ bestW) (Weighted p w)
+        | w >= bestW   = best
+        | acceptable p = Weighted (Just p) w
+        | otherwise    = foldl loop best
+                       $ map (\(Weighted p' w') -> Weighted p' (w+w'))
+                       $ successors p
+
+/*
+bfs :: SearchAlgorithm e a
+
+aStar :: (a ->  Double) -> SearchAlgorithm e a
+*/
+main = "Hello"
+--
+Hello
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Sections.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Sections.scl
new file mode 100644
index 000000000..2d7e57677
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Sections.scl
@@ -0,0 +1,5 @@
+import "Prelude"
+
+main = map (1/) $ map (*3) [1,2,3]
+--
+[0.3333333333333333, 0.16666666666666666, 0.1111111111111111]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Select1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Select1.scl
new file mode 100644
index 000000000..f8d0359ec
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Select1.scl
@@ -0,0 +1,11 @@
+import "StandardLibrary"
+
+main = let
+    l = [1..3] 
+  in
+    select (?x,?y) where 
+        ?x <- l
+        ?y <- l
+        ?x = ?y
+--
+[(1,1), (2,2), (3,3)]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Select2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Select2.scl
new file mode 100644
index 000000000..a4928238c
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Select2.scl
@@ -0,0 +1,20 @@
+import "StandardLibrary"
+import "Minigraph"
+
+main = withGraph do
+    a = resource "a"
+    b = map resource ["b0", "b1", "b2", "b3", "b4"]
+    r = map resource ["r0", "r1"]
+    
+    enforce
+        Statement a (r!0) (b!0)
+        Statement a (r!0) (b!1)
+        Statement a (r!1) (b!4)
+        Statement a (r!1) (b!3)
+        Statement (b!1) (r!0) (b!2)
+    R a :- {}
+    R ?x :- R ?y ; Statement ?y (r!0) ?x
+    S ?x :- R ?x ; Statement ?x (r!0) _
+    sort $ map uriOf $ select ?x where S ?x
+--
+[a, b1]
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Select3.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Select3.scl
new file mode 100644
index 000000000..a3e4f44e8
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Select3.scl
@@ -0,0 +1,18 @@
+import "StandardLibrary"
+import "Minigraph"
+
+main = withGraph do
+    a = resource "a"
+    b = map resource ["b0", "b1", "b2", "b3", "b4"]
+    r = map resource ["r0", "r1"]
+    
+    enforce
+        Statement a (r!0) (b!0)
+        Statement a (r!0) (b!1)
+        Statement a (r!1) (b!4)
+        Statement (b!1) (r!1) (b!3)
+        Statement (b!1) (r!0) (b!2)
+    sort $ map uriOf $ select ?x where 
+        Statement a (r!0) (_ : Resource { #r1 = ?x })
+--
+[b3]
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Select4.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Select4.scl
new file mode 100644
index 000000000..3199e9545
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Select4.scl
@@ -0,0 +1,18 @@
+import "StandardLibrary"
+import "Minigraph"
+
+main = withGraph do
+    a = resource "a"
+    b = map resource ["b0", "b1", "b2"]
+    r = map resource ["r0", "r1"]
+    
+    enforce
+        Statement a (r!0) (b!0)
+        Statement a (r!0) (b!1)
+        Statement a (r!1) (b!1)
+        Statement a (r!1) (b!2)
+    sort $ map uriOf $ select ?x where 
+        <|> Statement a (r!0) ?x
+            Statement a (r!1) ?x
+--
+[b0, b1, b1, b2]
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Select5.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Select5.scl
new file mode 100644
index 000000000..18172473f
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Select5.scl
@@ -0,0 +1,11 @@
+import "StandardLibrary"
+import "Minigraph"
+
+main = withGraph do
+    r = map resource ["r0", "r1"]
+    sort $ map uriOf $ select ?x where 
+        Statement ?y (r!0) ?x
+        Statement ?x (r!1) ?y
+--
+6:24-8:30: Failed to compile the query.
+Unsolved variables: ?x, ?y
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Select6.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Select6.scl
new file mode 100644
index 000000000..84e4df30e
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Select6.scl
@@ -0,0 +1,6 @@
+import "StandardLibrary"
+
+main = select ?x where
+    (?x,?y) = (1,2)
+--
+[1]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Select7.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Select7.scl
new file mode 100644
index 000000000..b0be01ade
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Select7.scl
@@ -0,0 +1,6 @@
+import "StandardLibrary"
+
+main = select ?x where
+    (?x, 3) <- [(1,2),(2,3),(3,4)]
+--
+[2]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Select8.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Select8.scl
new file mode 100644
index 000000000..787704445
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Select8.scl
@@ -0,0 +1,10 @@
+import "StandardLibrary"
+
+Foo ?a ?b ?l :-
+    @ffb
+    (?a,?b) <- ?l
+
+main = select ?x where
+    Foo ?x 3 [(1,2),(2,3),(3,4)]
+--
+[2]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Select9.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Select9.scl
new file mode 100644
index 000000000..56806793c
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Select9.scl
@@ -0,0 +1,12 @@
+import "StandardLibrary"
+
+s1 = [1,2,3]
+s2 = [1,3] 
+    
+main = do
+    C ?x :-
+        ?x <- s1
+         ?x <- s2
+    select ?x where C ?x
+--
+[2]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/SelfReferringContextInTypeClass.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/SelfReferringContextInTypeClass.scl
new file mode 100644
index 000000000..d2f1bd7a5
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/SelfReferringContextInTypeClass.scl
@@ -0,0 +1,6 @@
+class Ord a where
+    min :: Ord a => a -> a -> a
+
+main = "Not to be executed."
+--
+2:12-2:15: Unresolved type class Ord.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Serialization.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Serialization.scl
new file mode 100644
index 000000000..e0d44e357
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Serialization.scl
@@ -0,0 +1,14 @@
+import "Prelude"
+
+rt :: Serializable a => a -> a
+rt v = deserialize (serialize v)
+
+main :: String
+main = show (
+  (rt "Hello", rt ()),
+  (rt 1.2 :: Double, rt 1.2 :: Float, 3 :: Integer, 4 :: Long),
+  rt (Just (1 :: Integer)),
+  (rt [1::Integer,2,3], rt [[1::Integer,2],[3,4]], fromDoubleArray (rt (toDoubleArray [3::Double,2,1])))
+  )
+--
+(("Hello", ()), (1.2, 1.2, 3, 4), Just 1, ([1, 2, 3], [[1, 2], [3, 4]], [3.0, 2.0, 1.0]))
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Serialization2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Serialization2.scl
new file mode 100644
index 000000000..8384527e7
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Serialization2.scl
@@ -0,0 +1,14 @@
+import "Prelude"
+import "Serialization"
+
+rt :: IO a => a -> a
+rt v = readByteArray (writeByteArray v)
+
+main = show (
+  (rt "Hello", rt ()),
+  (rt 1.2 :: Double, rt 1.2 :: Float, 3 :: Integer, 4 :: Long),
+  rt (Just (1 :: Integer)),
+  (rt [1::Integer,2,3], rt [[1::Integer,2],[3,4]], fromDoubleArray (rt (toDoubleArray [3,2,1])))
+  )
+--
+(("Hello", ()), (1.2, 1.2, 3, 4), Just 1, ([1, 2, 3], [[1, 2], [3, 4]], [3.0, 2.0, 1.0]))
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Serialization3.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Serialization3.scl
new file mode 100644
index 000000000..d883dfea7
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Serialization3.scl
@@ -0,0 +1,22 @@
+import "Prelude"
+import "Serialization" as Serialization
+
+rt :: Serialization.IO a => a -> a
+rt v = Serialization.readByteArray (Serialization.writeByteArray v)
+
+data FooBar a = Foo Integer | Bar a
+
+deriving instance (Show a) => Show (FooBar a)
+deriving instance (Serialization.IO a) => Serialization.IO (FooBar a)
+
+/*
+instance IO FooBar where
+    read s = match Serialization.read s :: Integer with
+             0 -> Foo (Serialization.read s)
+             1 -> Bar (Serialization.read s)
+    write s (Foo x) = do Serialization.write s (0 :: Integer) ; Serialization.write s x
+    write s (Bar x) = do Serialization.write s (1 :: Integer) ; Serialization.write s x
+*/
+main = show (rt (Foo 3 :: FooBar Double))
+--
+Foo 3
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Set1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Set1.scl
new file mode 100644
index 000000000..b868ecf89
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Set1.scl
@@ -0,0 +1,9 @@
+import "StandardLibrary"
+
+s1 = Set.set [3,2,1,4]
+s2 = Set.set [9,8,3,2]
+s1s2 = Set.union s1 s2
+
+main = show s1s2
+--
+set [1, 2, 3, 4, 8, 9]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/SharedTypeVariable.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/SharedTypeVariable.scl
new file mode 100644
index 000000000..19fd565e3
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/SharedTypeVariable.scl
@@ -0,0 +1,8 @@
+// Idea here is that the type variable in the top level type annotation
+// and the type annotation in the expression should be the same type.
+id :: a -> a
+id x = (x :: a)
+
+main = id "OK"
+--
+OK
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ShortcutFusion.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ShortcutFusion.scl
new file mode 100644
index 000000000..6423ecbef
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ShortcutFusion.scl
@@ -0,0 +1,25 @@
+data List a = Nil | Cons a (List a)
+
+@private
+@inline
+build :: (forall a. a -> (b -> a -> a) -> a) -> List b
+build f = f Nil Cons
+
+@private
+foldr :: (a -> b -> b) -> b -> List a -> b
+foldr cons nil Nil = nil
+foldr cons nil (Cons h t) = cons h (foldr cons nil t)
+
+@private
+@inline
+singleton :: a -> List a
+singleton x = build (\nil cons -> cons x nil)
+
+@private
+@inline
+last :: List a -> a -> a
+last l def = foldr (\x _ -> x) def l
+
+main = last (singleton "Hello") "Foo"
+--
+Hello
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Show1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Show1.scl
new file mode 100644
index 000000000..649f3cbce
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Show1.scl
@@ -0,0 +1,5 @@
+import "Prelude"
+
+main = show [(1,2),(3,4),(5,6)]
+--
+[(1, 2), (3, 4), (5, 6)]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Signals.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Signals.scl
new file mode 100644
index 000000000..726a3f5be
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Signals.scl
@@ -0,0 +1,34 @@
+import "Prelude"
+
+// --- Signals ------------------------------------------------------
+
+data Signal =
+    SigSum [Signal]
+  | SigConst Double
+  | SigNeg Signal
+  | SigMul [Signal]
+
+deriving instance Eq Signal
+deriving instance Hashable Signal
+deriving instance Show Signal
+
+instance Additive Signal where
+    zero = SigConst 0
+    a + b = SigSum [a,b]
+    sum l = SigSum l
+
+instance Ring Signal where
+    one = SigConst 1
+    neg a = SigNeg a
+    fromInteger i = SigConst (fromInteger i)
+    a * b = SigMul [a,b]
+
+/*
+instance Real Signal where
+    fromDouble d = SigConst d
+*/
+
+main :: Signal
+main = 1 + 2 * 3 - 4 
+--
+???
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/SinConst1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/SinConst1.scl
new file mode 100644
index 000000000..66621b13b
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/SinConst1.scl
@@ -0,0 +1,5 @@
+import "Prelude"
+
+main = (sin + const 1) 0
+--
+1.0
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Sort.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Sort.scl
new file mode 100644
index 000000000..7efdf8941
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Sort.scl
@@ -0,0 +1,6 @@
+import "Prelude"
+
+main :: [Integer]
+main = sort [1,5,2,4,3]
+--
+[1, 2, 3, 4, 5]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Sort2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Sort2.scl
new file mode 100644
index 000000000..8518264ec
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Sort2.scl
@@ -0,0 +1,19 @@
+import "JavaBuiltin" as Java
+
+importJava "org.simantics.scl.runtime.Lists" where
+    sortWith :: (a -> a -> Integer) -> [a] -> [a]
+
+//@JavaStaticMethod "org.simantics.scl.runtime.Lists.sortWith"
+//sortWith :: (a -> a -> Integer) -> [a] -> [a]
+
+
+//sortWith = Java.staticMethod "org.simantics.scl.runtime.Lists.sortWith"
+
+dumbCompare :: a -> a -> Integer
+dumbCompare x y = 0
+
+dumbSort = sortWith dumbCompare 
+
+main = "Foo"
+--
+Foo
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/SpecConstr1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/SpecConstr1.scl
new file mode 100644
index 000000000..4ff7d476f
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/SpecConstr1.scl
@@ -0,0 +1,25 @@
+data Either a b = Left a | Right b
+
+data List a = Nil | Cons a (List a)
+
+data Nat = Zero | Succ Nat
+
+sum Zero a = a
+sum a Zero = a
+sum (Succ a) (Succ b) = Succ (Succ (sum a b))
+
+sum_append xs ys
+ = go Zero (Left xs)
+ where
+    go z (Left xs)
+      = match xs with
+        Nil -> go z (Right ys)
+        Cons x xs' -> go (sum x z) (Left xs')
+    go z (Right ys)
+      = match ys with
+        Nil -> z
+        Cons y ys' -> go (sum y z) (Right ys')
+
+main = "Hello world!"
+--
+Hello world!
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/StackTrace.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/StackTrace.scl
new file mode 100644
index 000000000..fe30fa3af
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/StackTrace.scl
@@ -0,0 +1,3 @@
+main = fail "badly"
+--
+???
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/StreamFusion.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/StreamFusion.scl
new file mode 100644
index 000000000..6d1864a22
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/StreamFusion.scl
@@ -0,0 +1,139 @@
+import "JavaBuiltin" as Java
+
+data Either a b = Left a | Right b
+
+data Stream a = Stream (s -> Step a s) s
+data Step a s = Done | Yield a s | Skip s
+
+// stream :: [a] -> Stream a
+// unstream :: Stream a -> [a] 
+// streamRev :: [a] -> Stream a
+// unstreamRev :: Stream a -> [a]
+
+//toStream :: [a] -> Stream a
+//toStream l = Stream (\i -> if i >= length l then Done else Yield (get l i) (i+1)) 0
+
+filterS :: (a -> Boolean) -> Stream a -> Stream a
+filterS p (Stream next0 s0) = Stream next s0
+  where
+    next s = match next s with
+        Done -> Done
+        Skip s -> Skip s        
+        Yield a s -> if p a then Yield a s else Skip s
+
+mapS :: (a -> b) -> Stream a -> Stream b
+mapS f (Stream next0 s0) = Stream next s0
+  where
+    next s = match next0 s with
+        Done -> Done
+        Skip s -> Skip s        
+        Yield a s -> Yield (f a) s
+      
+appendS :: Stream a -> Stream a -> Stream a
+appendS (Stream next1 s1) (Stream next2 s2) = Stream next (Left s1)
+  where
+    next (Left s) = match next1 s with
+        Done -> Skip (Right s2)
+        Skip s -> Skip (Left s)         
+        Yield a s -> Yield a (Left s)
+    next (Right s) = match next2 s with
+        Done -> Done
+        Skip s -> Skip (Right s)    
+        Yield a s -> Yield a (Right s)      
+
+decomposeS :: Stream a -> Maybe (a, Stream a)
+decomposeS (Stream next s0) = loop s0
+  where
+    loop s = match next s with
+        Done -> Nothing
+        Skip s -> loop s
+        Yield a s -> Just (a, Stream next s)    
+
+returnS :: a -> Stream a
+returnS x = Stream next True
+  where 
+    next True = Yield x False
+    next False = Done
+    
+isEmptyS :: Stream a -> Boolean 
+isEmptyS (Stream next s0) = loop s0
+  where
+    loop s = match next s with
+        Done -> True
+        Skip s -> loop s     
+        Yield _ _ -> False
+      
+foldlS :: (a -> b -> a) -> a -> Stream b -> a
+foldlS f init (Stream next s0) = go init s0
+  where
+    go cur s = match next s with
+        Done -> cur
+        Skip s -> go cur s
+        Yield x s -> go (f cur x) s
+      
+scanlS :: (a -> b -> a) -> a -> Stream b -> Stream a
+scanlS f init (Stream next0 s0) = Stream next (s0, init)
+  where
+    next (s,v) = match next0 s with
+        Done -> Done
+        Skip s -> Skip (s, v)
+        Yield x s -> Yield v (s, f v x)    
+            
+concatMapS :: (a -> Stream b) -> Stream a -> Stream b
+concatMapS f (Stream next0 s0) = Stream next (s0, Nothing)
+  where
+    next (s, Nothing) = match next0 s with
+        Done -> Done
+        Skip s -> Skip (s, Nothing)
+        Yield x s -> Skip (s, Just (f x))
+    next (s0, Just (Stream next1 s1)) = match next1 s1 with
+        Done -> Skip (s0, Nothing)
+        Skip s -> Skip (s0, Just (Stream next1 s))
+        Yield x s -> Yield x (s0, Just (Stream next1 s))
+
+zipWithS :: (a -> b -> c) -> Stream a -> Stream b -> Stream c
+zipWithS f (Stream next0 s0) (Stream next1 s1) = Stream next (s0, s1, Nothing)
+  where
+    next (s0, s1, Nothing) = match next0 s0 with
+        Done -> Done
+        Skip s -> Skip (s, s1, Nothing)
+        Yield x s -> Skip (s, s1, Just x)
+    next (s0, s1, Just x) = match next1 s1 with
+        Done -> Done
+        Skip s -> Skip (s0, s, Just x)
+        Yield y s -> Yield (f x y) (s0, s, Nothing)
+
+guardS :: Boolean -> Stream a -> Stream a
+guardS b (Stream next0 s0) = Stream next (b, s0)
+  where
+    next (False, _) = Done
+    next (True, s) = match next0 s with
+        Done -> Done
+        Skip s -> Skip (True, s)
+        Yield x s -> Yield x (True, s)
+/*
+takeS :: Integer -> Stream a -> Stream a
+takeS count (Stream next0 s0) = Stream next (count, s0)
+  where
+    next (count, s) = if count <= 0 then Done
+                      else match next0 s with
+                          Done -> Done
+                        | Skip s -> Skip (count, s)
+                        | Yield x s -> Yield x (count, s)
+*/
+repeatS :: a -> Stream a
+repeatS v = Stream next ()
+  where
+    next () = Yield v ()
+
+iterateS :: (a -> a) -> a -> Stream a
+iterateS f v = Stream next v
+  where
+    next v = Yield v (f v)
+  
+main :: Integer
+main = foldlS Java.iadd (0 :: Integer) 
+              (appendS (appendS (returnS (1 :: Integer)) 
+              (returnS (2 :: Integer))) (returnS (3 :: Integer)))
+--
+6
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/StringEscape.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/StringEscape.scl
new file mode 100644
index 000000000..51a75d3cb
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/StringEscape.scl
@@ -0,0 +1,12 @@
+main = "a\nb\"c\'d"
+--
+a
+b"c'd
+--
+main = "a\u0053"
+--
+aS
+--
+main = "a\xb"
+--
+1:8-1:10: Illegal string escape character.
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/StringInterpolation1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/StringInterpolation1.scl
new file mode 100644
index 000000000..eabad781a
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/StringInterpolation1.scl
@@ -0,0 +1,9 @@
+import "Prelude"
+
+f x = x + 1
+
+g a b = """\(a) + \(b) = \(a+b)"""
+
+main = "f 3 = \(f 3), " + g 1 2
+--
+f 3 = 4, 1 + 2 = 3
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/StringInterpolation2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/StringInterpolation2.scl
new file mode 100644
index 000000000..9d219430b
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/StringInterpolation2.scl
@@ -0,0 +1,6 @@
+stringSum :: String -> Integer
+stringSum "(\(a),\(b),\(c))" = a + b + c
+
+main = stringSum "(1,2,3)"
+--
+6
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/StringMatching1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/StringMatching1.scl
new file mode 100644
index 000000000..5865926af
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/StringMatching1.scl
@@ -0,0 +1,7 @@
+main :: Integer
+main = match "Bar" with
+  "Foo" -> 1
+  "Bar" -> 2
+  _ -> 3
+--
+2
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/SumOfInverses2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/SumOfInverses2.scl
new file mode 100644
index 000000000..d3cacbae2
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/SumOfInverses2.scl
@@ -0,0 +1,8 @@
+import "Prelude"
+
+f :: Integer -> Double
+f m = sum [1/(fromInteger x) | x <- [1..m]]
+
+main = f 100
+--
+5.187377517639621
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Timing.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Timing.scl
new file mode 100644
index 000000000..deb6d507c
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Timing.scl
@@ -0,0 +1,10 @@
+import "StandardLibrary"
+
+main = do 
+    (r,t) = Debug.time (foldl (+) 0 
+                   $ map (\(_,_,z) -> z) 
+                   $ [(a,b,c) | a <- [1..50], b <- [1..50], c <- [1..50], a*a + b*b == c*c])
+    print t
+    r
+--
+1172
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TooManyParametersToSin.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TooManyParametersToSin.scl
new file mode 100644
index 000000000..bfaf3db56
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TooManyParametersToSin.scl
@@ -0,0 +1,6 @@
+import "Prelude"
+
+main = sin 1 2
+--
+3:8-3:11: Constrain Real (a -> b) contains free variables not mentioned in the type of the value.
+3:14-3:15: Constrain Ring a contains free variables not mentioned in the type of the value.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Transformation1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Transformation1.scl
new file mode 100644
index 000000000..144b03155
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Transformation1.scl
@@ -0,0 +1,59 @@
+import "StandardLibrary"
+import "Minigraph"
+import "MMap" as MMap
+
+consistsOf ::  Integer
+consistsOf = resource "consistsOf"
+instanceOf ::  Integer
+instanceOf = resource "instanceOf"
+areConnected ::  Integer
+areConnected = resource "areConnected"
+from ::  Integer
+from = resource "from"
+to ::  Integer
+to = resource "to"
+
+element ::  Integer
+element = resource "Element"
+connection ::  Integer
+connection = resource "Connection"
+
+mapping relation MapDiagrams Integer Integer
+
+rule ElementsRule where
+    @when
+    MapDiagrams ?dA ?dB
+    
+    @from
+    Statement ?dA consistsOf ?elA
+    Statement ?elA instanceOf element
+    
+    @to
+    Statement ?dB consistsOf ?elB
+    Statement ?elB instanceOf element
+    
+    @where
+    MapElements ?elA ?elB
+
+rule ConnectionsRule where
+    @when
+    MapElements ?elA1 ?elB1
+    MapElements ?elA2 ?elB2
+    
+    @from
+    Statement ?elA1 areConnected ?elA2
+    
+    @to
+    Statement ?conn instanceOf connection
+    Statement ?conn from ?elB1
+    Statement ?conn to   ?elB2
+
+foo ::  ()
+foo = transformation OneShotForward where
+    MapDiagrams 0 0
+
+main = withGraph do
+    foo
+    "OK"
+--
+OK
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Transformation2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Transformation2.scl
new file mode 100644
index 000000000..dbb1609a9
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Transformation2.scl
@@ -0,0 +1,25 @@
+import "StandardLibrary"
+
+mapping relation Fib Integer Integer
+
+rule FibRecurrence where
+    @when
+    Fib ?n ?a
+    Fib (?n+1) ?b
+    ?n < 20
+    
+    @where
+    Fib (?n+2) (?a + 1)
+/*
+rule PrintIt where
+    @when
+    Fib ?n ?a
+    
+    @to
+    Execute (print "\(?n) -> \(?a)")
+*/
+main = transformation OneShotForward where
+    Fib 0 1
+    Fib 1 1
+--
+()
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Transformation3.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Transformation3.scl
new file mode 100644
index 000000000..f9b995cee
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Transformation3.scl
@@ -0,0 +1,32 @@
+import "StandardLibrary"
+
+mapping relation Fib Integer Integer
+
+rule FibRecurrence where
+    @when
+    Fib ?n (?a + ?b)
+    ?n >= 2
+    
+    @where
+    Fib (?n-1) ?b
+    Fib (?n-2) ?a
+
+rule Init where
+    @when
+    Fib ?x 1
+    ?x < 2
+    
+rule Seed where
+    @where
+    Fib 20 ?hmm
+
+rule PrintIt where
+    @when
+    Fib ?n ?a
+    
+    @to
+    Execute (print "\(?n) -> \(?a)")
+
+main = transformation OneShotForward where
+--
+()
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Transformation4.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Transformation4.scl
new file mode 100644
index 000000000..5a889e1e0
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Transformation4.scl
@@ -0,0 +1,27 @@
+import "StandardLibrary"
+
+mapping relation M Integer (Integer,Integer)
+
+rule Mix where
+    @when
+    M ?a (?b, ?c)
+    ?a <= 10
+    
+    @where
+    M (?a + 1) (?c, ?b)
+
+rule PrintIt where
+    @when
+    M ?a (?b,?c)
+    
+    @to
+    Execute (print "\(?a) -> \(?b), \(?c)")
+    
+rule Seed where
+    @where
+    M 0 (1,?x)
+    M 10 (?y,2)
+
+main = transformation OneShotForward where
+--
+()
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Transformation5.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Transformation5.scl
new file mode 100644
index 000000000..e041430ee
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Transformation5.scl
@@ -0,0 +1,22 @@
+import "StandardLibrary"
+
+Foo ?x :-
+    @enforce 1
+    Execute (?x := (1 :: Integer))
+
+Bar ?x :-
+    @enforce 2
+    Execute (if getRef ?x == 1 then () else fail "Test failed.")
+    
+rule DoIt where
+    @from
+    ?x = ref (0 :: Integer)
+    
+    @to
+    Bar ?x
+    Foo ?x
+
+main :: ()
+main = transformation OneShotForward where
+--
+()
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Transformation6.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Transformation6.scl
new file mode 100644
index 000000000..74aed46dc
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Transformation6.scl
@@ -0,0 +1,15 @@
+import "StandardLibrary"
+
+mapping relation Foo String String
+    
+rule DoIt where
+    @when
+    Foo ?a ?b
+    
+    @to
+    Foo ?b "c"
+
+main = transformation OneShotForward where
+    Foo "a" "b"
+--
+10:9-10:11: Cannot resolve the variable ?b using the source patterns.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Transformation7.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Transformation7.scl
new file mode 100644
index 000000000..cce48f6ed
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Transformation7.scl
@@ -0,0 +1,14 @@
+import "StandardLibrary"
+
+Foo ?x :- 
+    @enforce
+    Execute (iterList printString [?x :: String])
+
+    
+rule DoIt where
+    @to
+    Foo "Hello world!"
+
+main = transformation OneShotForward where
+--
+()
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TransformationOrder.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TransformationOrder.scl
new file mode 100644
index 000000000..36c08c7b7
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TransformationOrder.scl
@@ -0,0 +1,17 @@
+import "StandardLibrary"
+
+rule A where
+    @to
+    Execute (print "A")
+
+rule B where
+    @to
+    Execute (print "B")
+    
+rule C where
+    @to
+    Execute (print "C")
+    
+main = transformation OneShotForward where
+--
+()
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Tuples.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Tuples.scl
new file mode 100644
index 000000000..5f70e4bea
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Tuples.scl
@@ -0,0 +1,18 @@
+id0 () = ()
+
+id0A :: () -> ()
+id0A () = ()
+
+id2 (a,b) = (a,b)
+
+id2A :: (a,b) -> (a,b)
+id2A (a,b) = (a,b)
+
+id3 (a,b,c) = (a,b,c)
+
+id3A :: (a,b,c) -> (a,b,c)
+id3A (a,b,c) = (a,b,c)
+
+main = id3 (1 :: Integer,id0 (),3 :: Integer)
+--
+(1,(),3)
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Tuples2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Tuples2.scl
new file mode 100644
index 000000000..75b478513
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Tuples2.scl
@@ -0,0 +1,6 @@
+// This is just a restricted version of Tuples test
+id0 () = ()
+
+main = id0 ()
+--
+()
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeAlias1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeAlias1.scl
new file mode 100644
index 000000000..3aad9bf4f
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeAlias1.scl
@@ -0,0 +1,9 @@
+
+type Id = Integer
+
+incrementId :: Id -> Id
+incrementId x = x
+
+main = incrementId 3
+--
+3
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeAlias2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeAlias2.scl
new file mode 100644
index 000000000..61925153b
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeAlias2.scl
@@ -0,0 +1,8 @@
+type List a = [a]
+
+convert :: List Integer -> List Double
+convert v = v
+
+main = "Not to be executed."
+--
+4:13-4:14: Expected <[Double]> got <[Integer]>.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeAlias3.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeAlias3.scl
new file mode 100644
index 000000000..b25a8abfe
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeAlias3.scl
@@ -0,0 +1,9 @@
+type IntegerList = [Integer]
+type DoubleList = [Double]
+
+convert :: IntegerList -> DoubleList
+convert v = v
+
+main = "Not to be executed."
+--
+5:13-5:14: Expected <[Double]> got <[Integer]>.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeAliasRefsToTypeAlias.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeAliasRefsToTypeAlias.scl
new file mode 100644
index 000000000..7ebb60636
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeAliasRefsToTypeAlias.scl
@@ -0,0 +1,22 @@
+type Foo = Bar
+type Bar = Integer
+main = 1 :: Foo
+--
+1
+--
+type Bar = Integer
+type Foo = Bar
+main = 1 :: Foo
+--
+1
+--
+type Foo = Foo
+main = 1 :: Foo
+--
+1:1-1:15: Type alias has a self reference.
+--
+type Foo = Bar
+type Bar = Foo
+main = 1 :: Foo
+--
+1:1-1:15: Recursively defined type alias (Foo, Bar).
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeAnnotation1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeAnnotation1.scl
new file mode 100644
index 000000000..5ba565f12
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeAnnotation1.scl
@@ -0,0 +1,5 @@
+id x = x :: Integer
+
+main = id "foo"
+--
+3:11-3:16: Expected  got .
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeAnnotation2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeAnnotation2.scl
new file mode 100644
index 000000000..ab855f8dc
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeAnnotation2.scl
@@ -0,0 +1,5 @@
+id (x :: Integer) = x
+
+main = id "foo"
+--
+3:11-3:16: Expected  got .
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeClass.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeClass.scl
new file mode 100644
index 000000000..540d44e0e
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeClass.scl
@@ -0,0 +1,10 @@
+
+class Foo a where
+    foo :: a -> Integer
+    
+instance Foo Integer where
+    foo x = x
+    
+main = foo (13 :: Integer)
+--
+13
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeClass2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeClass2.scl
new file mode 100644
index 000000000..5bd09350b
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeClass2.scl
@@ -0,0 +1,28 @@
+import "JavaBuiltin" as Java
+
+(+) = Java.iadd
+
+class Foo a where
+    foo :: a -> Integer
+    /*
+class (Foo a) => Bar a where
+    bar :: a -> Integer
+    */
+instance Foo Integer where
+    foo x = x+1
+  /*  
+instance Bar Integer where
+    bar x = x+2
+*/
+data X a = X a
+
+instance (Foo a) => Foo (X a) where
+    foo (X a) = foo a
+    /*
+instance (Bar a) => Bar (X a) where
+    bar (X a) = bar a*/
+    
+main = foo (X (1 :: Integer))
+// + bar (X (2 :: Integer))
+--
+2
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeClassBug1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeClassBug1.scl
new file mode 100644
index 000000000..2ea51bef6
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeClassBug1.scl
@@ -0,0 +1,19 @@
+import "JavaBuiltin" as Java
+
+importJava "org.simantics.scl.runtime.Lists" where
+    foldl :: (a -> b -> a) -> a -> [b] -> a
+
+class Additive a where
+    zero :: a
+    (+)  :: a -> a -> a    
+    
+    sum  :: [a] -> a
+    sum = foldl (+) zero    
+    
+instance (Additive a, Additive b, Additive c) => Additive (a, b, c) where
+    zero = (zero, zero, zero)
+    (a0, b0, c0) + (a1, b1, c1) = (a0+a1, b0+b1, c0+c1)
+    
+main = "OK"
+--
+OK
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeClassBug2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeClassBug2.scl
new file mode 100644
index 000000000..a341ba6b2
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeClassBug2.scl
@@ -0,0 +1,54 @@
+data Foo1 a = Foo1 a
+data Foo2 a = Foo2 a
+
+foo1 :: Foo1 a -> a
+foo1 (Foo1 x) = x
+
+foo2 :: Foo2 a -> a
+foo2 (Foo2 x) = x
+
+class Makeable s where
+    make :: a -> s a
+
+instance Makeable Foo1 where
+    make = Foo1
+
+instance Makeable Foo2 where
+    make = Foo2
+
+class (Makeable f) => Foo f where
+    foo :: f a -> a
+
+class (Makeable b) => Bar b where
+    bar :: b a -> a
+
+class (Makeable b) => Baz b where
+    baz :: b a -> a
+
+class (Makeable b) => Bim b where
+    bim :: b a -> a
+
+instance Foo Foo1 where
+    foo = foo1
+    
+instance Bar Foo2 where
+    bar = foo2
+    
+instance (Bar b) => Baz b where
+    baz = bar
+
+instance Bim Foo1 where
+    bim = foo1
+
+instance (Baz b) => Bim b where
+    bim = baz
+
+doFoo1 (Foo1 x) = x
+doFoo2 (Foo2 x) = x
+
+useBim :: Bim b => (forall a. b a -> a) -> a -> [a]
+useBim doit x = [doit (make x), bim (make x :: Foo1 a)]
+
+main = "OK"
+--
+"OK"
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeInferenceBug2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeInferenceBug2.scl
new file mode 100644
index 000000000..4fa08a52d
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeInferenceBug2.scl
@@ -0,0 +1,9 @@
+import "Prelude"
+
+distance (x1,y1) (x2,y2) = let dx = x1-x2
+                               dy = y1-y2
+                           in sqrt (dx*dx + dy*dy) print x1
+--
+5:31-5:35: Constrain Real ((a ->  ()) -> ((c ->  ()) ->  e ->  f) ->  i) contains free variables not mentioned in the type of the value.
+5:52-5:57: Constrain Show a contains free variables not mentioned in the type of the value.
+5:58-5:60: Unification of types failed.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeOf1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeOf1.scl
new file mode 100644
index 000000000..933a29b79
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypeOf1.scl
@@ -0,0 +1,5 @@
+import "Prelude"
+
+main = typeOf (\(a,b) -> a+b :: Double)
+--
+(Double, Double) -> Double
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypingBug1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypingBug1.scl
new file mode 100644
index 000000000..3e69ebf52
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypingBug1.scl
@@ -0,0 +1,4 @@
+id x = x
+main = id id (5 :: Integer)
+--
+5
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypingError1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypingError1.scl
new file mode 100644
index 000000000..9f39243c4
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypingError1.scl
@@ -0,0 +1,9 @@
+a :: Double
+a = a
+
+id :: Integer -> Integer
+id x = x
+
+b = id a
+--
+7:8-7:9: Expected  got .
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypingError2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypingError2.scl
new file mode 100644
index 000000000..7632ffcef
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/TypingError2.scl
@@ -0,0 +1,9 @@
+a :: Integer
+a = a
+
+id :: Integer -> Integer
+id x = x
+
+b = id a a
+--
+7:5-7:11: Function of arity 1 is applied with 2 parameters.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/UnaryMinus.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/UnaryMinus.scl
new file mode 100644
index 000000000..ff0e51271
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/UnaryMinus.scl
@@ -0,0 +1,5 @@
+import "Prelude"
+
+main = -5
+--
+-5
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/UndefinedValue.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/UndefinedValue.scl
new file mode 100644
index 000000000..02de8b645
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/UndefinedValue.scl
@@ -0,0 +1,6 @@
+
+foo :: Integer
+// foo = 0
+
+--
+2:1-2:15: foo is not defined.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/UnexpectedToken.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/UnexpectedToken.scl
new file mode 100644
index 000000000..270d5751b
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/UnexpectedToken.scl
@@ -0,0 +1,4 @@
+a = =
+b = 4
+--
+1:5-1:6: Unexpected token '=' (EQUALS). Expected one of ATTACHED_HASH, BEGIN_STRING, BLANK, CHAR, DO, ENFORCE, EQ, ESCAPED_SYMBOL, FLOAT, ID, IF, INTEGER, LAMBDA, LBRACKET, LET, LPAREN, MATCH, MDO, MINUS, SELECT, SELECT_DISTINCT, SELECT_FIRST, TRANSFORMATION, WHEN.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Unification1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Unification1.scl
new file mode 100644
index 000000000..a174362f4
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Unification1.scl
@@ -0,0 +1,28 @@
+import "JavaBuiltin" as Java
+import "StandardLibrary"
+import "Unification"
+
+pair :: Default a => Default b => UTag (a, b) (Unifiable a, Unifiable b)
+pair = uTag 0 (\(ua, ub) -> (extract ua, extract ub)) Java.unsafeCoerce  
+
+/*triple :: Default a => Default b => Default c =>
+    UTag (a, b, c) (Unifiable a, Unifiable b, Unifiable c)*/
+triple = uTag 0 (\(ua, ub, uc) -> (extract ua, extract ub, extract uc)) Java.unsafeCoerce  
+
+main :: (Integer,Integer,Integer)
+main = do
+    um1 = createUMap
+    um2 = createUMap
+    v1 = uVar
+    v2 = uVar
+    v3 = uVar
+    vX = uVar
+    putUMap um1 "a" (uCons triple (v1, v2, v3))
+    putUMap um1 "a" (uCons triple (v2, v3, v1))
+    putUMap um1 "a" (uCons triple (vX, uVar, uVar))
+    putUMap um2 "b" vX
+    putUMapC um2 "b" 12
+
+    getUMap um1 "a"
+--
+(12,12,12)
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/UnknownAnnotation.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/UnknownAnnotation.scl
new file mode 100644
index 000000000..c1b5ef8ea
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/UnknownAnnotation.scl
@@ -0,0 +1,5 @@
+
+@sdlfkmsdlfkm
+main = "Not to be executed"
+--
+2:1-2:14: Unknown annotation.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/UnresolvedClass.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/UnresolvedClass.scl
new file mode 100644
index 000000000..0e1765c4d
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/UnresolvedClass.scl
@@ -0,0 +1,8 @@
+
+data List a = Nil | Cons a (List a)
+
+instance Monoid List where
+    zero = Nil
+    
+--
+4:10-4:16: Couldn't resolve class Monoid.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/UnresolvedTypeInAnnotation.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/UnresolvedTypeInAnnotation.scl
new file mode 100644
index 000000000..18e075104
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/UnresolvedTypeInAnnotation.scl
@@ -0,0 +1,4 @@
+main :: List Integer
+main = 13
+--
+1:9-1:13: Didn't find type constructor List.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/UnresolvedTypeInInstance.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/UnresolvedTypeInInstance.scl
new file mode 100644
index 000000000..1b5400cb0
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/UnresolvedTypeInInstance.scl
@@ -0,0 +1,9 @@
+class Functor f where
+    map :: (a -> b) -> f a -> f b
+
+data Iddd a = Idd a
+
+instance Functor Idd where
+    map (Idd x) = x
+--
+6:18-6:21: Didn't find type constructor Idd.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/UnresolvedVariable.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/UnresolvedVariable.scl
new file mode 100644
index 000000000..ad28907ee
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/UnresolvedVariable.scl
@@ -0,0 +1,4 @@
+a = b
+
+--
+1:5-1:6: Couldn't resolve variable b.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/UnresolvedVariable2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/UnresolvedVariable2.scl
new file mode 100644
index 000000000..32438cda2
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/UnresolvedVariable2.scl
@@ -0,0 +1,7 @@
+
+a = a
+b = b
+c = a + b
+
+--
+4:7-4:8: Couldn't resolve variable +.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ValueAsOperator.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ValueAsOperator.scl
new file mode 100644
index 000000000..37235370f
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ValueAsOperator.scl
@@ -0,0 +1,8 @@
+
+data List a = Nil | Cons a (List a)
+
+data Foo = Foo
+
+main = Foo `Cons` (Foo `Cons` Nil)
+--
+(Cons Foo (Cons Foo Nil))
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ValueConversion.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ValueConversion.scl
new file mode 100644
index 000000000..3afda1b32
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/ValueConversion.scl
@@ -0,0 +1,6 @@
+import "Prelude"
+import "Vector"
+
+main = fromDynamic (toDynamic (vector [3 :: Double])) :: [Double]
+--
+[3.0]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Vector1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Vector1.scl
new file mode 100644
index 000000000..33b6c86df
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Vector1.scl
@@ -0,0 +1,8 @@
+import "StandardLibrary"
+
+a :: Vector Double
+a = vector [1,2,3]
+
+main = map (a !) [0..length a-1]
+--
+[1.0, 2.0, 3.0]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Vector2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Vector2.scl
new file mode 100644
index 000000000..43c0c5359
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Vector2.scl
@@ -0,0 +1,8 @@
+import "StandardLibrary"
+
+convertDataset :: [Double] -> [Double] -> Vector (Vector Double)
+convertDataset xs ys = vector [vector xs, vector ys]
+
+main = "Foo"
+--
+Foo
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Void1.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Void1.scl
new file mode 100644
index 000000000..40a710fa2
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Void1.scl
@@ -0,0 +1,16 @@
+importJava "java.util.List" where
+    data List a
+    
+    @JavaName add
+    addList :: List a -> a ->  Boolean
+
+importJava "java.util.ArrayList" where
+    @JavaName ""
+    newList :: () ->  (List a)
+
+main = do
+    l = newList ()
+    addList l (3 :: Integer)
+    l
+--
+[3]
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Void2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Void2.scl
new file mode 100644
index 000000000..f953aaf58
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Void2.scl
@@ -0,0 +1,13 @@
+import "JavaBuiltin" as Java
+
+class Foo a where
+    foo :: a -> () -> a
+    
+instance Foo Integer where
+    foo x () = x
+    
+idWithFoo x = foo x ()    
+    
+main = idWithFoo (13 :: Integer)
+--
+13
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Void3.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Void3.scl
new file mode 100644
index 000000000..0530c8a11
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/Void3.scl
@@ -0,0 +1,12 @@
+import "JavaBuiltin" as Java
+
+data Foo = Foo ()
+
+
+ff = Foo ()
+
+gg () = 3 :: Integer
+
+main = gg (match ff with Foo a -> a)
+--
+3
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/While.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/While.scl
new file mode 100644
index 000000000..3e450439b
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/While.scl
@@ -0,0 +1,10 @@
+
+while :: ( Boolean) -> ( a) ->  ()
+while cond body = loop ()
+  where loop _ = if cond
+                 then do body ; loop ()
+                 else ()
+
+main = "FOO"
+--
+FOO
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/While2.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/While2.scl
new file mode 100644
index 000000000..6567c22ab
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/While2.scl
@@ -0,0 +1,12 @@
+import "Prelude" hiding (while)
+
+while :: Maybe Boolean -> Maybe a -> Maybe ()
+while condM bodyM = mdo
+    cond <- condM
+    if cond 
+    then bodyM >> while condM bodyM
+    else return ()
+
+main = "FOO"
+--
+FOO
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/While3.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/While3.scl
new file mode 100644
index 000000000..052076958
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/While3.scl
@@ -0,0 +1,11 @@
+import "Prelude"
+
+main = do
+    a = ref 1
+    b = ref 0
+    while (getRef a < 5) do
+        b := getRef b + 1
+        a := getRef a + 1
+    getRef b             
+--
+4
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/WrongDefaultMethod.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/WrongDefaultMethod.scl
new file mode 100644
index 000000000..d94046b4a
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/WrongDefaultMethod.scl
@@ -0,0 +1,8 @@
+
+class Foo a where
+    foo :: a
+    bar = foo
+    
+main = "Not to be executed."
+--
+4:5-4:14: Method bar is not defined in this class.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/WrongInstanceMethod.scl b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/WrongInstanceMethod.scl
new file mode 100644
index 000000000..2cdc4e86b
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scl/WrongInstanceMethod.scl
@@ -0,0 +1,11 @@
+
+class Foo a where
+    foo :: a
+    
+instance Foo Integer where
+    foo = foo
+    bar = foo
+    
+main = "Not to be executed."
+--
+7:5-7:14: Method bar is not defined in the type class Foo.
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scripts/Arithmetic.sts b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scripts/Arithmetic.sts
new file mode 100644
index 000000000..e8f7dce07
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scripts/Arithmetic.sts
@@ -0,0 +1,12 @@
+> 1+1
+2
+> 1-1
+0
+> 1+1.0
+2.0
+> 2*5
+10
+> 4/3
+1.3333333333333333
+> 4 `div` 3
+1
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scripts/Functions.sts b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scripts/Functions.sts
new file mode 100644
index 000000000..76348b876
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scripts/Functions.sts
@@ -0,0 +1,20 @@
+> id x = x
+> id 2
+2
+> id "Hello!"
+"Hello!"
+
+> id = \x -> x
+> id 2
+2
+
+> fib 0 = 1
+> fib 1 = 1
+> fib n = fib (n-1) + fib (n-2)
+
+> fib 2
+2
+> fib 3
+3
+> fib 4
+5
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scripts/Functions2.sts b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scripts/Functions2.sts
new file mode 100644
index 000000000..9879931b7
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scripts/Functions2.sts
@@ -0,0 +1,5 @@
+> f x = x + 1
+> f 3
+4
+> f 4.5
+5.5
\ No newline at end of file
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scripts/Lists.sts b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scripts/Lists.sts
new file mode 100644
index 000000000..024db2467
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/scripts/Lists.sts
@@ -0,0 +1,5 @@
+> [1,2,3]
+[1, 2, 3]
+
+> [5..9]
+[5, 6, 7, 8, 9]
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/unit/TestNamespaceFilter.java b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/unit/TestNamespaceFilter.java
new file mode 100644
index 000000000..fa3bd9f65
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/unit/TestNamespaceFilter.java
@@ -0,0 +1,89 @@
+package org.simantics.scl.compiler.tests.unit;
+
+import java.util.Collection;
+
+import org.junit.Test;
+import org.simantics.scl.compiler.environment.filter.AcceptAllNamespaceFilter;
+import org.simantics.scl.compiler.environment.filter.NamespaceFilter;
+import org.simantics.scl.compiler.environment.filter.NamespaceFilters;
+import org.simantics.scl.compiler.environment.filter.NegativeNamespaceFilter;
+import org.simantics.scl.compiler.environment.filter.PositiveNamespaceFilter;
+
+import gnu.trove.set.hash.THashSet;
+import junit.framework.Assert;
+
+public class TestNamespaceFilter {
+    
+    private void testBooleanOperations(Collection all, NamespaceFilter a, NamespaceFilter b) {
+        {
+            NamespaceFilter c = NamespaceFilters.union(a, b);
+            //System.out.println("union(" + a + ", " + b + ") = " + c);
+            for(String name : all)
+                Assert.assertEquals(
+                        a.isValueIncluded(name) || b.isValueIncluded(name),
+                        c.isValueIncluded(name));
+        }
+        {
+            NamespaceFilter c = NamespaceFilters.intersection(a, b);
+            //System.out.println("intersection(" + a + ", " + b + ") = " + c);
+            for(String name : all)
+                Assert.assertEquals(
+                        a.isValueIncluded(name) && b.isValueIncluded(name),
+                        c.isValueIncluded(name));
+        }
+    }
+
+    private void testBooleanOperations(THashSet a, THashSet b) {
+        THashSet all = new THashSet();
+        all.addAll(a);
+        all.addAll(b);
+        all.add("dummy");
+        
+        PositiveNamespaceFilter pa = new PositiveNamespaceFilter(a);
+        NegativeNamespaceFilter na = new NegativeNamespaceFilter(a);
+        PositiveNamespaceFilter pb = new PositiveNamespaceFilter(b);
+        NegativeNamespaceFilter nb = new NegativeNamespaceFilter(b);
+        testBooleanOperations(all, pa, pb);
+        testBooleanOperations(all, na, pb);
+        testBooleanOperations(all, pa, nb);
+        testBooleanOperations(all, na, nb);
+    }
+    
+    private void testBooleanOperations(THashSet a) {
+        THashSet all = new THashSet();
+        all.addAll(a);
+        all.add("dummy");
+        
+        PositiveNamespaceFilter pa = new PositiveNamespaceFilter(a);
+        NegativeNamespaceFilter na = new NegativeNamespaceFilter(a);
+        testBooleanOperations(all, pa, AcceptAllNamespaceFilter.INSTANCE);
+        testBooleanOperations(all, na, AcceptAllNamespaceFilter.INSTANCE);
+        testBooleanOperations(all, AcceptAllNamespaceFilter.INSTANCE, pa);
+        testBooleanOperations(all, AcceptAllNamespaceFilter.INSTANCE, na);
+        testBooleanOperations(all, AcceptAllNamespaceFilter.INSTANCE, AcceptAllNamespaceFilter.INSTANCE);
+    }
+    
+    @Test
+    public void testBooleanOperations() {
+        for(int p=0;p<8;++p) {
+            THashSet a = new THashSet();
+            for(int i=0;i<3;++i)
+                if(((p >> i) & 1) == 1)
+                    a.add(String.valueOf(i));
+            testBooleanOperations(a);
+        }
+        
+        for(int p=0;p<64;++p) {
+            THashSet a = new THashSet();
+            THashSet b = new THashSet();
+            for(int i=0;i<3;++i) {
+                if(((p >> i) & 1) == 1)
+                    a.add(String.valueOf(i));
+                if(((p >> (i+3)) & 1) == 1)
+                    b.add(String.valueOf(i));
+            }
+            testBooleanOperations(a, b);
+        }
+    }
+    
+}
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/unit/TestSubSolver.java b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/unit/TestSubSolver.java
new file mode 100644
index 000000000..87b9eeb4f
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/unit/TestSubSolver.java
@@ -0,0 +1,37 @@
+package org.simantics.scl.compiler.tests.unit;
+
+import java.util.ArrayList;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.simantics.scl.compiler.errors.ErrorLog;
+import org.simantics.scl.compiler.internal.elaboration.subsumption.SubSolver;
+import org.simantics.scl.compiler.internal.elaboration.subsumption.Subsumption;
+import org.simantics.scl.compiler.types.TMetaVar;
+import org.simantics.scl.compiler.types.Type;
+import org.simantics.scl.compiler.types.Types;
+import org.simantics.scl.compiler.types.kinds.Kinds;
+import org.simantics.scl.compiler.types.util.Polarity;
+
+
+public class TestSubSolver {
+
+    @Test
+    public void testBipolarBounded() {
+        ErrorLog errorLog = new ErrorLog();
+        ArrayList subsumptions = new ArrayList();
+        ArrayList potentialSingletonEffects = new ArrayList();
+        
+        TMetaVar in = Types.metaVar(Kinds.EFFECT);
+        TMetaVar out = Types.metaVar(Kinds.EFFECT);
+        in.addPolarity(Polarity.NEGATIVE);
+        out.addPolarity(Polarity.POSITIVE);
+        subsumptions.add(new Subsumption(0, Types.READ_GRAPH, out));
+        subsumptions.add(new Subsumption(0, in, out));
+        
+        SubSolver solver = new SubSolver(errorLog, subsumptions, potentialSingletonEffects, 0);
+        solver.solve();
+        Assert.assertEquals("", errorLog.getErrorsAsString());
+    }
+    
+}
diff --git a/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/unit/TestTypeParser.java b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/unit/TestTypeParser.java
new file mode 100644
index 000000000..5ce369042
--- /dev/null
+++ b/tests/org.simantics.scl.compiler.tests/src/org/simantics/scl/compiler/tests/unit/TestTypeParser.java
@@ -0,0 +1,25 @@
+package org.simantics.scl.compiler.tests.unit;
+
+import org.junit.Test;
+import org.simantics.scl.compiler.environment.Environment;
+import org.simantics.scl.compiler.environment.Environments;
+import org.simantics.scl.compiler.environment.specification.EnvironmentSpecification;
+import org.simantics.scl.compiler.runtime.RuntimeEnvironment;
+import org.simantics.scl.compiler.tests.TestBase;
+import org.simantics.scl.compiler.types.Type;
+
+public class TestTypeParser {
+    
+    @Test
+    public void testTypeParser() throws Exception {
+        EnvironmentSpecification spec = new EnvironmentSpecification();
+        spec.importModule("Builtin", "");
+        spec.importModule("Prelude", "");
+        RuntimeEnvironment runtimeEnvironment =
+                TestBase.PRELUDE_MODULE_REPOSITORY.createRuntimeEnvironment(spec, getClass().getClassLoader());
+        Environment environment = runtimeEnvironment.getEnvironment();
+        Type type = Environments.getType(environment, "String ->  ()");
+        System.out.println(type);
+    }
+    
+}
diff --git a/tests/org.simantics.scl.osgi.tests/.classpath b/tests/org.simantics.scl.osgi.tests/.classpath
new file mode 100644
index 000000000..b862a296d
--- /dev/null
+++ b/tests/org.simantics.scl.osgi.tests/.classpath
@@ -0,0 +1,7 @@
+
+
+	
+	
+	
+	
+
diff --git a/tests/org.simantics.scl.osgi.tests/.project b/tests/org.simantics.scl.osgi.tests/.project
new file mode 100644
index 000000000..d9af653fc
--- /dev/null
+++ b/tests/org.simantics.scl.osgi.tests/.project
@@ -0,0 +1,28 @@
+
+
+	org.simantics.scl.osgi.tests
+	
+	
+	
+	
+		
+			org.eclipse.jdt.core.javabuilder
+			
+			
+		
+		
+			org.eclipse.pde.ManifestBuilder
+			
+			
+		
+		
+			org.eclipse.pde.SchemaBuilder
+			
+			
+		
+	
+	
+		org.eclipse.pde.PluginNature
+		org.eclipse.jdt.core.javanature
+	
+
diff --git a/tests/org.simantics.scl.osgi.tests/.settings/org.eclipse.jdt.core.prefs b/tests/org.simantics.scl.osgi.tests/.settings/org.eclipse.jdt.core.prefs
new file mode 100644
index 000000000..295926d96
--- /dev/null
+++ b/tests/org.simantics.scl.osgi.tests/.settings/org.eclipse.jdt.core.prefs
@@ -0,0 +1,7 @@
+eclipse.preferences.version=1
+org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
+org.eclipse.jdt.core.compiler.compliance=1.8
+org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
+org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
+org.eclipse.jdt.core.compiler.source=1.8
diff --git a/tests/org.simantics.scl.osgi.tests/META-INF/MANIFEST.MF b/tests/org.simantics.scl.osgi.tests/META-INF/MANIFEST.MF
new file mode 100644
index 000000000..da7e1c6fd
--- /dev/null
+++ b/tests/org.simantics.scl.osgi.tests/META-INF/MANIFEST.MF
@@ -0,0 +1,16 @@
+Manifest-Version: 1.0
+Bundle-ManifestVersion: 2
+Bundle-Name: Tests
+Bundle-SymbolicName: org.simantics.scl.osgi.tests
+Bundle-Version: 1.0.0.qualifier
+Bundle-RequiredExecutionEnvironment: JavaSE-1.8
+Require-Bundle: org.junit;bundle-version="4.12.0",
+ org.simantics.scl.compiler;bundle-version="0.5.0",
+ org.simantics.scl.osgi;bundle-version="1.0.4",
+ gnu.trove3,
+ org.objectweb.asm.util,
+ org.eclipse.equinox.ds;bundle-version="1.4.300",
+ org.simantics;bundle-version="1.0.0",
+ org.simantics.acorn;bundle-version="1.1.2",
+ org.simantics.workbench;bundle-version="1.5.1",
+ org.simantics.application
diff --git a/tests/org.simantics.scl.osgi.tests/build.properties b/tests/org.simantics.scl.osgi.tests/build.properties
new file mode 100644
index 000000000..41eb6ade2
--- /dev/null
+++ b/tests/org.simantics.scl.osgi.tests/build.properties
@@ -0,0 +1,4 @@
+source.. = src/
+output.. = bin/
+bin.includes = META-INF/,\
+               .
diff --git a/tests/org.simantics.scl.osgi.tests/pom.xml b/tests/org.simantics.scl.osgi.tests/pom.xml
new file mode 100644
index 000000000..f22ff3334
--- /dev/null
+++ b/tests/org.simantics.scl.osgi.tests/pom.xml
@@ -0,0 +1,42 @@
+
+
+    4.0.0
+    org.simantics
+    org.simantics.scl.osgi.tests
+    1.0.0-SNAPSHOT
+    eclipse-test-plugin
+
+    
+        org.simantics
+        org.simantics.root.tests
+        1.0.0-SNAPSHOT
+    
+
+    
+        
+            
+                org.eclipse.tycho
+                target-platform-configuration
+                
+                    
+                        
+                            
+                                eclipse-feature
+                                org.simantics.sdk
+                                0.0.0
+                            
+                        
+                    
+                
+            
+            
+                org.eclipse.tycho
+                tycho-surefire-plugin
+                ${tycho.version}
+                
+                
+            
+        
+    
+
diff --git a/tests/org.simantics.scl.osgi.tests/src/org/simantics/scl/osgi/tests/TestSCLOsgi.java b/tests/org.simantics.scl.osgi.tests/src/org/simantics/scl/osgi/tests/TestSCLOsgi.java
new file mode 100644
index 000000000..455429427
--- /dev/null
+++ b/tests/org.simantics.scl.osgi.tests/src/org/simantics/scl/osgi/tests/TestSCLOsgi.java
@@ -0,0 +1,76 @@
+package org.simantics.scl.osgi.tests;
+
+import java.util.ArrayList;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.core.runtime.NullProgressMonitor;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.simantics.PlatformException;
+import org.simantics.Simantics;
+import org.simantics.application.arguments.Arguments;
+import org.simantics.application.arguments.IArgumentFactory;
+import org.simantics.application.arguments.IArguments;
+import org.simantics.application.arguments.SimanticsArguments;
+import org.simantics.scl.compiler.errors.DoesNotExist;
+import org.simantics.scl.compiler.errors.Failable;
+import org.simantics.scl.compiler.module.Module;
+import org.simantics.scl.osgi.SCLOsgi;
+
+import gnu.trove.procedure.TObjectProcedure;
+
+
+public class TestSCLOsgi {
+    private static IProgressMonitor progress = new NullProgressMonitor();
+    
+    @BeforeClass
+    public static void setupDatabase() throws PlatformException {
+        String[] args = new String[0];
+        IArgumentFactory[] accepted = {
+                SimanticsArguments.RECOVERY_POLICY_FIX_ERRORS,
+                SimanticsArguments.ONTOLOGY_RECOVERY_POLICY_REINSTALL,
+                SimanticsArguments.SERVER,
+                SimanticsArguments.LOCAL_SERVER_PORT,
+        };
+        IArguments arguments = Arguments.parse(args, accepted);
+        Simantics.setDefaultDatabaseDriver("acorn");
+        Simantics.startUpHeadless(arguments, progress);
+    }
+    
+    @AfterClass
+    public static void teardownDatabase() throws PlatformException {
+        Simantics.shutdown(progress);
+    }
+    
+    @Test
+    public void testDataJsonExists() {
+        ArrayList modulesWithErrors = new ArrayList(); 
+        SCLOsgi.SOURCE_REPOSITORY.forAllModules(new TObjectProcedure() {
+            @Override
+            public boolean execute(String moduleName) {
+                System.out.print(moduleName);
+                System.out.print(" - ");
+                Failable module = SCLOsgi.MODULE_REPOSITORY.getModule(moduleName);
+                if(module.didSucceed())
+                    System.out.println("succeeded");
+                else if(module == DoesNotExist.INSTANCE)
+                    System.out.println("does not exist"); // should not happen
+                else {
+                    System.out.println("error");
+                    modulesWithErrors.add(moduleName);
+                }
+                return true;
+            }
+        });
+        if(!modulesWithErrors.isEmpty()) {
+            StringBuilder b = new StringBuilder();
+            b.append("Some SCL modules failed to compile:");
+            for(String module : modulesWithErrors)
+                b.append(' ').append(module);
+            Assert.fail(b.toString());
+        }
+    }
+}
diff --git a/tests/pom.xml b/tests/pom.xml
new file mode 100644
index 000000000..20d4c4832
--- /dev/null
+++ b/tests/pom.xml
@@ -0,0 +1,33 @@
+
+
+    4.0.0
+    org.simantics
+    org.simantics.root.tests
+    1.0.0-SNAPSHOT
+    pom
+
+    
+        org.simantics
+        org.simantics.root
+        1.0.0-SNAPSHOT
+    
+
+    
+        
+            
+                org.eclipse.tycho
+                tycho-compiler-plugin
+                ${tycho.version}
+                
+                    -err:-forbidden
+                
+            
+        
+    
+
+    
+        org.simantics.scl.compiler.tests
+        org.simantics.scl.osgi.tests
+    
+