bundles/org.simantics.scl.compiler/src/org/simantics/scl/compiler/internal/parsing/parser/SCLPostLexer.java

   1 package org.simantics.scl.compiler.internal.parsing.parser;
   2
   3 import java.io.IOException;
   4 import java.util.Arrays;
   5
   6 import org.simantics.scl.compiler.errors.Locations;
   7 import org.simantics.scl.compiler.internal.parsing.Token;
   8 import org.simantics.scl.compiler.internal.parsing.exceptions.SCLSyntaxErrorException;
   9
  10 import gnu.trove.list.array.TIntArrayList;
  11 import gnu.trove.set.hash.TIntHashSet;
  12
  13 /**
  14  *
  15  * <http://www.haskell.org/onlinereport/haskell2010/haskellch10.html#x17-17800010.3>
  16  * @author Hannu Niemistö
  17  */
  18 public class SCLPostLexer {
  19
  20     public static TIntHashSet INDENTABLE = new TIntHashSet();
  21     public static TIntHashSet NO_SEMICOLON_BEFORE = new TIntHashSet();
  22     public static TIntHashSet NO_SEMICOLON_AFTER = new TIntHashSet();
  23     static {
  24         INDENTABLE.add(SCLTerminals.WHERE);
  25         INDENTABLE.add(SCLTerminals.QUERY_OP);
  26         INDENTABLE.add(SCLTerminals.WITH);
  27         INDENTABLE.add(SCLTerminals.DO);
  28         INDENTABLE.add(SCLTerminals.MDO);
  29         INDENTABLE.add(SCLTerminals.LET);
  30         INDENTABLE.add(SCLTerminals.ENFORCE);
  31         INDENTABLE.add(SCLTerminals.WHEN);
  32         INDENTABLE.add(SCLTerminals.FOLLOWS);
  33         INDENTABLE.add(SCLTerminals.EQ);
  34
  35         NO_SEMICOLON_BEFORE.add(SCLTerminals.EOF);
  36         NO_SEMICOLON_BEFORE.add(SCLTerminals.SYMBOL);
  37         NO_SEMICOLON_BEFORE.add(SCLTerminals.THEN);
  38         NO_SEMICOLON_BEFORE.add(SCLTerminals.ELSE);
  39         NO_SEMICOLON_BEFORE.add(SCLTerminals.IN);
  40         NO_SEMICOLON_BEFORE.add(SCLTerminals.RBRACE);
  41         NO_SEMICOLON_BEFORE.add(SCLTerminals.RBRACKET);
  42         NO_SEMICOLON_BEFORE.add(SCLTerminals.RPAREN);
  43
  44         NO_SEMICOLON_AFTER.add(SCLTerminals.EOF);
  45         NO_SEMICOLON_AFTER.add(SCLTerminals.SYMBOL);
  46     }
  47
  48     SCLLexer lexer;
  49     Token[] queue = new Token[16];
  50     int queuePos=0, queueSize=0;
  51     TIntArrayList indentations = new TIntArrayList();
  52     Token curToken = null;
  53     int lineStart = 0;
  54     boolean firstTokenOfLine = true;
  55     private SCLParserOptions options;
  56
  57     {
  58         indentations.add(0);
  59     }
  60
  61     public SCLPostLexer(SCLLexer lexer) {
  62         this.lexer = lexer;
  63     }
  64
  65     public SCLPostLexer(java.io.Reader in) {
  66         this(new SCLLexer(in));
  67     }
  68
  69     public Token nextToken() throws Exception {
  70         while(queuePos == queueSize)
  71             fillQueue();
  72         return queue[queuePos++];
  73     }
  74
  75     public Token peekToken() throws Exception {
  76         while(queuePos == queueSize)
  77             fillQueue();
  78         return queue[queuePos];
  79     }
  80
  81     private void push(Token symbol) {
  82         /*System.out.println("TOKEN " + symbol.text + " (" + SCLParser.TERMINAL_NAMES[symbol.id] + ")" +
  83                 " ["
  84                 + Locations.beginOf(symbol.location) + ".."
  85                 + Locations.endOf(symbol.location) + "]");*/
  86         if(queueSize == queue.length)
  87             queue = Arrays.copyOf(queue, queueSize*2);
  88         queue[queueSize++] = symbol;
  89     }
  90
  91     private void fillQueue() throws Exception {
  92         queuePos = 0;
  93         queueSize = 0;
  94
  95         for(int i=0;i<8;++i)
  96             handleToken(lexer.nextToken());
  97     }
  98
  99     private SCLSyntaxErrorException error(int start, int end, String description) {
 100         return new SCLSyntaxErrorException(Locations.location(start, end), description);
 101     }
 102
 103     private void handleToken(Token symbol) throws IOException {
 104         int symbolId = symbol.id;
 105         if(symbolId == SCLTerminals.EOL) {
 106             lineStart = Locations.endOf(symbol.location);
 107             firstTokenOfLine = true;
 108             return;
 109         }
 110
 111         if(symbolId == SCLTerminals.COMMENT) {
 112             firstTokenOfLine = false;
 113             return;
 114         }
 115
 116         Token prevToken = curToken;
 117         int prevTokenId = prevToken == null ? SCLTerminals.EOF : prevToken.id;
 118         curToken = symbol;
 119
 120         int symbolStart = Locations.beginOf(symbol.location);
 121         int symbolEnd = Locations.endOf(symbol.location);
 122
 123         if(INDENTABLE.contains(prevTokenId) && symbolId != SCLTerminals.LBRACE) {
 124             push(new Token(SCLTerminals.LBRACE, symbolStart, symbolStart, "implicit {"));
 125             int symbolIndentation = symbolStart-lineStart;
 126             //System.out.println("symbolIndentation = " + symbolIndentation);
 127             indentations.add(symbolIndentation);
 128             firstTokenOfLine = false;
 129         }
 130         else if(firstTokenOfLine) {
 131             if(NO_SEMICOLON_AFTER.contains(prevTokenId) || NO_SEMICOLON_BEFORE.contains(symbolId))
 132                 ;
 133             else {
 134                 int level = symbolStart - lineStart;
 135                 //System.out.println("level = " + level);
 136                 if(indentations.get(indentations.size()-1) >= level) {
 137                     while(indentations.get(indentations.size()-1) > level) {
 138                         indentations.removeAt(indentations.size()-1);
 139                         int loc = Locations.endOf(prevToken.location);
 140                         push(new Token(SCLTerminals.RBRACE, loc, loc, "implicit }"));
 141                     }
 142                     if(indentations.get(indentations.size()-1) == level)
 143                         push(new Token(SCLTerminals.SEMICOLON, symbolStart, symbolStart, "implicit ;"));
 144                 }
 145             }
 146             firstTokenOfLine = false;
 147         }
 148
 149         switch(symbolId) {
 150         case SCLTerminals.LBRACE:
 151         case SCLTerminals.LPAREN:
 152         case SCLTerminals.LBRACKET:
 153         case SCLTerminals.IF:
 154         case SCLTerminals.LET:
 155             indentations.add(-1);
 156             push(symbol);
 157             return;
 158         case SCLTerminals.RBRACE:
 159         case SCLTerminals.RPAREN:
 160         case SCLTerminals.RBRACKET:
 161         case SCLTerminals.ELSE:
 162         case SCLTerminals.IN:
 163             while(!indentations.isEmpty() && indentations.removeAt(indentations.size()-1) >= 0) {
 164                 int loc = Locations.endOf(prevToken.location);
 165                 push(new Token(SCLTerminals.RBRACE, loc, loc, "implicit }"));
 166             }
 167             if(indentations.isEmpty())
 168                 throw error(symbolStart, symbolEnd, "No corresponding opening parenthesis for '" + symbol.text + "'.");
 169             push(symbol);
 170             return;
 171         case SCLTerminals.THEN: // 'then' both closes and opens a block
 172             while(!indentations.isEmpty() && indentations.removeAt(indentations.size()-1) >= 0) {
 173                 int loc = Locations.endOf(prevToken.location);
 174                 push(new Token(SCLTerminals.RBRACE, loc, loc, "implicit }"));
 175             }
 176             if(indentations.isEmpty())
 177                 throw error(symbolStart, symbolEnd, "No corresponding opening parenthesis for '" + symbol.text + "'.");
 178             push(symbol);
 179             indentations.add(-1);
 180             return;
 181         case SCLTerminals.EOF:
 182             while(indentations.size() > 1 && indentations.get(indentations.size()-1) >= 0) {
 183                 int loc = Locations.endOf(prevToken.location);
 184                 push(new Token(SCLTerminals.RBRACE, loc, loc, "implicit }"));
 185                 indentations.removeAt(indentations.size()-1);
 186             }
 187             if(indentations.size() > 1)
 188                 throw error(symbolStart, symbolEnd, "Unclosed parentheses.");
 189             push(symbol);
 190             return;
 191         default:
 192             push(symbol);
 193             return;
 194         }
 195     }
 196
 197     public void setParserOptions(SCLParserOptions options) {
 198         this.options = options;
 199         lexer.options = options;
 200     }
 201
 202 }