(refs #7250) CHR rules modularization (first working version)

[simantics/platform.git] / bundles / org.simantics.scl.compiler / src / org / simantics / scl / compiler / internal / parsing / parser / SCLPostLexer.java

diff --git a/bundles/org.simantics.scl.compiler/src/org/simantics/scl/compiler/internal/parsing/parser/SCLPostLexer.java b/bundles/org.simantics.scl.compiler/src/org/simantics/scl/compiler/internal/parsing/parser/SCLPostLexer.java
index 3d3b7b2f512033d90e13119b8edbfa8960cbaafd..a2253d8b7bc001cc7d0889a5ab69a0a5885e7922 100644 (file)
--- a/bundles/org.simantics.scl.compiler/src/org/simantics/scl/compiler/internal/parsing/parser/SCLPostLexer.java
+++ b/bundles/org.simantics.scl.compiler/src/org/simantics/scl/compiler/internal/parsing/parser/SCLPostLexer.java
@@ -3,6 +3,7 @@ package org.simantics.scl.compiler.internal.parsing.parser;
 import java.io.IOException;
 import java.util.Arrays;
 
+import org.simantics.scl.compiler.compilation.CompilationContext;
 import org.simantics.scl.compiler.errors.Locations;
 import org.simantics.scl.compiler.internal.parsing.Token;
 import org.simantics.scl.compiler.internal.parsing.exceptions.SCLSyntaxErrorException;
@@ -16,6 +17,9 @@ import gnu.trove.set.hash.TIntHashSet;
  * @author Hannu Niemistö
  */
 public class SCLPostLexer {
+    
+    private static final int PATCH_SIZE = 16;
+    private static final int INITIAL_QUEUE_SIZE = 32;
         
     public static TIntHashSet INDENTABLE = new TIntHashSet();
     public static TIntHashSet NO_SEMICOLON_BEFORE = new TIntHashSet();
@@ -37,6 +41,7 @@ public class SCLPostLexer {
         NO_SEMICOLON_BEFORE.add(SCLTerminals.EOF);
         NO_SEMICOLON_BEFORE.add(SCLTerminals.SYMBOL);
         NO_SEMICOLON_BEFORE.add(SCLTerminals.THEN);
+        NO_SEMICOLON_BEFORE.add(SCLTerminals.ELSE);
         NO_SEMICOLON_BEFORE.add(SCLTerminals.IN);
         NO_SEMICOLON_BEFORE.add(SCLTerminals.RBRACE);
         NO_SEMICOLON_BEFORE.add(SCLTerminals.RBRACKET);
@@ -48,7 +53,7 @@ public class SCLPostLexer {
     }
     
     SCLLexer lexer;
-    Token[] queue = new Token[16];
+    Token[] queue = new Token[INITIAL_QUEUE_SIZE];
     int queuePos=0, queueSize=0;
     TIntArrayList indentations = new TIntArrayList();
     TIntArrayList indentationTokens = new TIntArrayList();
@@ -56,6 +61,13 @@ public class SCLPostLexer {
     int lineStart = 0;
     boolean firstTokenOfLine = true;
     private SCLParserOptions options;
+    private boolean isFirstToken = true;
+    private CompilationContext context;
+    
+    /**
+     * We are parsing a module header and therefore should process tokens one by one and not by patches.
+     */
+    private boolean isInsideModule = false; 
             
     {
         indentations.add(0);
@@ -69,6 +81,11 @@ public class SCLPostLexer {
     public SCLPostLexer(java.io.Reader in) {
         this(new SCLLexer(in));
     }
+    
+    public void setCompilationContext(CompilationContext context) {
+        lexer.setCompilationContext(context);
+        this.context = context;
+    }
 
     public Token nextToken() throws Exception {
         while(queuePos == queueSize)
@@ -96,8 +113,15 @@ public class SCLPostLexer {
         queuePos = 0;
         queueSize = 0;
         
-        for(int i=0;i<8;++i)
+        for(int i=0;i<PATCH_SIZE;++i) {
             handleToken(lexer.nextToken());
+            if(isInsideModule) {
+                if(context.header == null)
+                    break;
+                else
+                    isInsideModule = false;
+            }
+        }
     }
     
     private SCLSyntaxErrorException error(int start, int end, String description) {
@@ -145,11 +169,19 @@ public class SCLPostLexer {
                         int loc = Locations.endOf(prevToken.location);
                         push(new Token(SCLTerminals.RBRACE, loc, loc, "implicit }"));
                     }
-                    if(indentations.get(indentations.size()-1) == level && symbolId != SCLTerminals.ELSE)
+                    if(indentations.get(indentations.size()-1) == level)
                         push(new Token(SCLTerminals.SEMICOLON, symbolStart, symbolStart, "implicit ;"));
                 }
             }
             firstTokenOfLine = false;
+            if(isFirstToken) {
+                isFirstToken = false;
+                if(symbol.id == SCLTerminals.ID && symbol.text.equals("module") && options != null && options.isModule) {
+                    push(new Token(SCLTerminals.MODULE, symbol.location, symbol.text));
+                    isInsideModule = true;
+                    return;
+                }
+            }
         }
         
         switch(symbolId) {
@@ -165,8 +197,8 @@ public class SCLPostLexer {
             return;
         case SCLTerminals.THEN:
             /*for(int tt : indentationTokens.toArray())
-                System.out.print(SCLParser.TERMINAL_NAMES[tt] + " ");
-            System.out.println();*/
+                    System.out.print(SCLParser.TERMINAL_NAMES[tt] + " ");
+                System.out.println();*/
             if(prevTokenId == SCLTerminals.COMMA) {
                 // for list comprehension syntax
                 push(symbol);
@@ -175,7 +207,7 @@ public class SCLPostLexer {
         case SCLTerminals.RBRACE:
         case SCLTerminals.RPAREN:
         case SCLTerminals.RBRACKET:
-        //case SCLTerminals.ELSE:
+        case SCLTerminals.ELSE:
         case SCLTerminals.IN:
             int removedToken = SCLTerminals.EOF;
             while(!indentations.isEmpty()) {
@@ -188,11 +220,15 @@ public class SCLPostLexer {
             }
             if(indentations.isEmpty())
                 throw error(symbolStart, symbolEnd, "No corresponding opening parenthesis for '" + symbol.text + "'.");
-            if(symbolId == SCLTerminals.THEN && removedToken == SCLTerminals.WHEN)
-                curToken = symbol = new Token(SCLTerminals.THEN_AFTER_WHEN, symbol.location, symbol.text);
+            if(symbolId == SCLTerminals.THEN) {
+                if(removedToken == SCLTerminals.WHEN)
+                    curToken = symbol = new Token(SCLTerminals.THEN_AFTER_WHEN, symbol.location, symbol.text);
+                else {
+                    indentations.add(-1);
+                    indentationTokens.add(SCLTerminals.THEN);
+                }
+            }   
             push(symbol);
-            //if(symbolId == SCLTerminals.THEN)
-            //    indentations.add(-1);
             return;
         case SCLTerminals.EOF:
             while(indentations.size() > 1 && indentations.get(indentations.size()-1) >= 0) {