bundles/org.simantics.scl.compiler/src/org/simantics/scl/compiler/internal/parsing/parser/SCLPostLexer.java

   1 package org.simantics.scl.compiler.internal.parsing.parser;
   2
   3 import java.io.IOException;
   4 import java.util.Arrays;
   5
   6 import org.simantics.scl.compiler.errors.Locations;
   7 import org.simantics.scl.compiler.internal.parsing.Token;
   8 import org.simantics.scl.compiler.internal.parsing.exceptions.SCLSyntaxErrorException;
   9
  10 import gnu.trove.list.array.TIntArrayList;
  11 import gnu.trove.set.hash.TIntHashSet;
  12
  13 /**
  14  *
  15  * <http://www.haskell.org/onlinereport/haskell2010/haskellch10.html#x17-17800010.3>
  16  * @author Hannu Niemistö
  17  */
  18 public class SCLPostLexer {
  19
  20     public static TIntHashSet INDENTABLE = new TIntHashSet();
  21     public static TIntHashSet NO_SEMICOLON_BEFORE = new TIntHashSet();
  22     public static TIntHashSet NO_SEMICOLON_AFTER = new TIntHashSet();
  23     static {
  24         INDENTABLE.add(SCLTerminals.WHERE);
  25         INDENTABLE.add(SCLTerminals.QUERY_OP);
  26         INDENTABLE.add(SCLTerminals.WITH);
  27         INDENTABLE.add(SCLTerminals.DO);
  28         INDENTABLE.add(SCLTerminals.MDO);
  29         INDENTABLE.add(SCLTerminals.LET);
  30         INDENTABLE.add(SCLTerminals.ENFORCE);
  31         INDENTABLE.add(SCLTerminals.WHEN);
  32         INDENTABLE.add(SCLTerminals.FOLLOWS);
  33         INDENTABLE.add(SCLTerminals.EQ);
  34
  35         NO_SEMICOLON_BEFORE.add(SCLTerminals.EOF);
  36         NO_SEMICOLON_BEFORE.add(SCLTerminals.SYMBOL);
  37         NO_SEMICOLON_BEFORE.add(SCLTerminals.THEN);
  38         NO_SEMICOLON_BEFORE.add(SCLTerminals.ELSE);
  39         NO_SEMICOLON_BEFORE.add(SCLTerminals.IN);
  40         NO_SEMICOLON_BEFORE.add(SCLTerminals.RBRACE);
  41         NO_SEMICOLON_BEFORE.add(SCLTerminals.RBRACKET);
  42         NO_SEMICOLON_BEFORE.add(SCLTerminals.RPAREN);
  43
  44         NO_SEMICOLON_AFTER.add(SCLTerminals.EOF);
  45         NO_SEMICOLON_AFTER.add(SCLTerminals.SYMBOL);
  46     }
  47
  48     SCLLexer lexer;
  49     Token[] queue = new Token[16];
  50     int queuePos=0, queueSize=0;
  51     TIntArrayList indentations = new TIntArrayList();
  52     Token curToken = null;
  53     int lineStart = 0;
  54     boolean firstTokenOfLine = true;
  55     private SCLParserOptions options;
  56
  57     {
  58         indentations.add(0);
  59     }
  60
  61     public SCLPostLexer(SCLLexer lexer) {
  62         this.lexer = lexer;
  63     }
  64
  65     public SCLPostLexer(java.io.Reader in) {
  66         this(new SCLLexer(in));
  67     }
  68
  69     public Token nextToken() throws Exception {
  70         while(queuePos == queueSize)
  71             fillQueue();
  72         return queue[queuePos++];
  73     }
  74
  75     public Token peekToken() throws Exception {
  76         while(queuePos == queueSize)
  77             fillQueue();
  78         return queue[queuePos];
  79     }
  80
  81     private void push(Token symbol) {
  82         if(queueSize == queue.length)
  83             queue = Arrays.copyOf(queue, queueSize*2);
  84         queue[queueSize++] = symbol;
  85     }
  86
  87     private void fillQueue() throws Exception {
  88         queuePos = 0;
  89         queueSize = 0;
  90
  91         for(int i=0;i<8;++i)
  92             handleToken(lexer.nextToken());
  93     }
  94
  95     private SCLSyntaxErrorException error(int start, int end, String description) {
  96         return new SCLSyntaxErrorException(Locations.location(start, end), description);
  97     }
  98
  99     private void handleToken(Token symbol) throws IOException {
 100         int symbolId = symbol.id;
 101         if(symbolId == SCLTerminals.EOL) {
 102             lineStart = Locations.endOf(symbol.location);
 103             firstTokenOfLine = true;
 104             return;
 105         }
 106
 107         if(symbolId == SCLTerminals.COMMENT) {
 108             firstTokenOfLine = false;
 109             return;
 110         }
 111
 112         Token prevToken = curToken;
 113         int prevTokenId = prevToken == null ? SCLTerminals.EOF : prevToken.id;
 114         curToken = symbol;
 115
 116         int symbolStart = Locations.beginOf(symbol.location);
 117         int symbolEnd = Locations.endOf(symbol.location);
 118
 119         if(INDENTABLE.contains(prevTokenId) && symbolId != SCLTerminals.LBRACE) {
 120             if(prevTokenId == SCLTerminals.LET)
 121                 indentations.add(-1);
 122             push(new Token(SCLTerminals.LBRACE, symbolStart, symbolStart, "implicit {"));
 123             int symbolIndentation = symbolStart-lineStart;
 124             //System.out.println("symbolIndentation = " + symbolIndentation);
 125             indentations.add(symbolIndentation);
 126             firstTokenOfLine = false;
 127         }
 128         else if(firstTokenOfLine) {
 129             if(NO_SEMICOLON_AFTER.contains(prevTokenId) || NO_SEMICOLON_BEFORE.contains(symbolId))
 130                 ;
 131             else {
 132                 int level = symbolStart - lineStart;
 133                 //System.out.println("level = " + level);
 134                 if(indentations.get(indentations.size()-1) >= level) {
 135                     while(indentations.get(indentations.size()-1) > level) {
 136                         indentations.removeAt(indentations.size()-1);
 137                         int loc = Locations.endOf(prevToken.location);
 138                         push(new Token(SCLTerminals.RBRACE, loc, loc, "implicit }"));
 139                     }
 140                     if(indentations.get(indentations.size()-1) == level)
 141                         push(new Token(SCLTerminals.SEMICOLON, symbolStart, symbolStart, "implicit ;"));
 142                 }
 143             }
 144             firstTokenOfLine = false;
 145         }
 146
 147         switch(symbolId) {
 148         case SCLTerminals.LBRACE:
 149         case SCLTerminals.LPAREN:
 150         case SCLTerminals.LBRACKET:
 151         case SCLTerminals.IF:
 152             indentations.add(-1);
 153             push(symbol);
 154             return;
 155         case SCLTerminals.RBRACE:
 156         case SCLTerminals.RPAREN:
 157         case SCLTerminals.RBRACKET:
 158         case SCLTerminals.ELSE:
 159         case SCLTerminals.IN:
 160             while(!indentations.isEmpty() && indentations.removeAt(indentations.size()-1) >= 0) {
 161                 int loc = Locations.endOf(prevToken.location);
 162                 push(new Token(SCLTerminals.RBRACE, loc, loc, "implicit }"));
 163             }
 164             if(indentations.isEmpty())
 165                 throw error(symbolStart, symbolEnd, "No corresponding opening parenthesis for '" + symbol.text + "'.");
 166             push(symbol);
 167             return;
 168         case SCLTerminals.THEN: // 'then' both closes and opens a block
 169             while(!indentations.isEmpty() && indentations.removeAt(indentations.size()-1) >= 0) {
 170                 int loc = Locations.endOf(prevToken.location);
 171                 push(new Token(SCLTerminals.RBRACE, loc, loc, "implicit }"));
 172             }
 173             if(indentations.isEmpty())
 174                 throw error(symbolStart, symbolEnd, "No corresponding opening parenthesis for '" + symbol.text + "'.");
 175             push(symbol);
 176             indentations.add(-1);
 177             return;
 178         case SCLTerminals.EOF:
 179             while(indentations.size() > 1 && indentations.get(indentations.size()-1) >= 0) {
 180                 int loc = Locations.endOf(prevToken.location);
 181                 push(new Token(SCLTerminals.RBRACE, loc, loc, "implicit }"));
 182                 indentations.removeAt(indentations.size()-1);
 183             }
 184             if(indentations.size() > 1)
 185                 throw error(symbolStart, symbolEnd, "Unclosed parentheses.");
 186             push(symbol);
 187             return;
 188         default:
 189             push(symbol);
 190             return;
 191         }
 192     }
 193
 194     public void setParserOptions(SCLParserOptions options) {
 195         this.options = options;
 196         lexer.options = options;
 197     }
 198
 199 }