Merge remote-tracking branch 'origin/svn'
[simantics/platform.git] / bundles / org.simantics.scl.compiler / src / org / simantics / scl / compiler / internal / parsing / parser / SCLPostLexer.java
1 package org.simantics.scl.compiler.internal.parsing.parser;
2
3 import java.io.IOException;
4 import java.util.Arrays;
5
6 import org.simantics.scl.compiler.errors.Locations;
7 import org.simantics.scl.compiler.internal.parsing.Token;
8 import org.simantics.scl.compiler.internal.parsing.exceptions.SCLSyntaxErrorException;
9
10 import gnu.trove.list.array.TIntArrayList;
11 import gnu.trove.set.hash.TIntHashSet;
12
13 /**
14  * 
15  * <http://www.haskell.org/onlinereport/haskell2010/haskellch10.html#x17-17800010.3>
16  * @author Hannu Niemist√∂
17  */
18 public class SCLPostLexer {
19         
20     public static TIntHashSet INDENTABLE = new TIntHashSet();
21     public static TIntHashSet NO_SEMICOLON_BEFORE = new TIntHashSet();
22     public static TIntHashSet NO_SEMICOLON_AFTER = new TIntHashSet();
23     static {
24         INDENTABLE.add(SCLTerminals.WHERE);
25         INDENTABLE.add(SCLTerminals.QUERY_OP);
26         INDENTABLE.add(SCLTerminals.WITH);
27         INDENTABLE.add(SCLTerminals.DO);
28         INDENTABLE.add(SCLTerminals.MDO);
29         INDENTABLE.add(SCLTerminals.LET);
30         INDENTABLE.add(SCLTerminals.ENFORCE);
31         INDENTABLE.add(SCLTerminals.WHEN);
32         INDENTABLE.add(SCLTerminals.FOLLOWS);
33         INDENTABLE.add(SCLTerminals.EQ);
34         
35         NO_SEMICOLON_BEFORE.add(SCLTerminals.EOF);
36         NO_SEMICOLON_BEFORE.add(SCLTerminals.SYMBOL);
37         NO_SEMICOLON_BEFORE.add(SCLTerminals.THEN);
38         NO_SEMICOLON_BEFORE.add(SCLTerminals.ELSE);
39         NO_SEMICOLON_BEFORE.add(SCLTerminals.IN);
40         NO_SEMICOLON_BEFORE.add(SCLTerminals.RBRACE);
41         NO_SEMICOLON_BEFORE.add(SCLTerminals.RBRACKET);
42         NO_SEMICOLON_BEFORE.add(SCLTerminals.RPAREN);
43         
44         NO_SEMICOLON_AFTER.add(SCLTerminals.EOF);
45         NO_SEMICOLON_AFTER.add(SCLTerminals.SYMBOL);
46     }
47     
48     SCLLexer lexer;
49     Token[] queue = new Token[16];
50     int queuePos=0, queueSize=0;
51     TIntArrayList indentations = new TIntArrayList();
52     Token curToken = null;
53     int lineStart = 0;
54     boolean firstTokenOfLine = true;
55     private SCLParserOptions options;
56             
57     {
58         indentations.add(0);
59     }
60     
61     public SCLPostLexer(SCLLexer lexer) {
62         this.lexer = lexer;
63     }
64     
65     public SCLPostLexer(java.io.Reader in) {
66         this(new SCLLexer(in));
67     }
68
69     public Token nextToken() throws Exception {
70         while(queuePos == queueSize)
71             fillQueue();
72         return queue[queuePos++];
73     }
74     
75     public Token peekToken() throws Exception {
76         while(queuePos == queueSize)
77             fillQueue();
78         return queue[queuePos];
79     }
80     
81     private void push(Token symbol) {
82         /*System.out.println("TOKEN " + symbol.text + " (" + SCLParser.TERMINAL_NAMES[symbol.id] + ")" +
83                 " [" 
84                 + Locations.beginOf(symbol.location) + ".." 
85                 + Locations.endOf(symbol.location) + "]");*/
86         if(queueSize == queue.length)
87             queue = Arrays.copyOf(queue, queueSize*2);
88         queue[queueSize++] = symbol;
89     }
90     
91     private void fillQueue() throws Exception {
92         queuePos = 0;
93         queueSize = 0;
94         
95         for(int i=0;i<8;++i)
96             handleToken(lexer.nextToken());
97     }
98     
99     private SCLSyntaxErrorException error(int start, int end, String description) {
100         return new SCLSyntaxErrorException(Locations.location(start, end), description);
101     }
102     
103     private void handleToken(Token symbol) throws IOException {
104         int symbolId = symbol.id;
105         if(symbolId == SCLTerminals.EOL) {
106             lineStart = Locations.endOf(symbol.location);
107             firstTokenOfLine = true;
108             return;
109         }
110         
111         if(symbolId == SCLTerminals.COMMENT) {
112             firstTokenOfLine = false;
113             return;
114         }
115         
116         Token prevToken = curToken;
117         int prevTokenId = prevToken == null ? SCLTerminals.EOF : prevToken.id;
118         curToken = symbol;
119         
120         int symbolStart = Locations.beginOf(symbol.location);
121         int symbolEnd = Locations.endOf(symbol.location);
122         
123         if(INDENTABLE.contains(prevTokenId) && symbolId != SCLTerminals.LBRACE) {
124             push(new Token(SCLTerminals.LBRACE, symbolStart, symbolStart, "implicit {"));
125             int symbolIndentation = symbolStart-lineStart;
126             //System.out.println("symbolIndentation = " + symbolIndentation);
127             indentations.add(symbolIndentation);
128             firstTokenOfLine = false;
129         }
130         else if(firstTokenOfLine) {
131             if(NO_SEMICOLON_AFTER.contains(prevTokenId) || NO_SEMICOLON_BEFORE.contains(symbolId))
132                 ;
133             else {
134                 int level = symbolStart - lineStart;
135                 //System.out.println("level = " + level);
136                 if(indentations.get(indentations.size()-1) >= level) {
137                     while(indentations.get(indentations.size()-1) > level) {
138                         indentations.removeAt(indentations.size()-1);
139                         int loc = Locations.endOf(prevToken.location);
140                         push(new Token(SCLTerminals.RBRACE, loc, loc, "implicit }"));
141                     }
142                     if(indentations.get(indentations.size()-1) == level)
143                         push(new Token(SCLTerminals.SEMICOLON, symbolStart, symbolStart, "implicit ;"));
144                 }
145             }
146             firstTokenOfLine = false;
147         }
148         
149         switch(symbolId) {
150         case SCLTerminals.LBRACE:
151         case SCLTerminals.LPAREN:
152         case SCLTerminals.LBRACKET:
153         case SCLTerminals.IF:
154         case SCLTerminals.LET:
155             indentations.add(-1);
156             push(symbol);
157             return;
158         case SCLTerminals.RBRACE:
159         case SCLTerminals.RPAREN:
160         case SCLTerminals.RBRACKET:
161         case SCLTerminals.ELSE:
162         case SCLTerminals.IN:
163             while(!indentations.isEmpty() && indentations.removeAt(indentations.size()-1) >= 0) {
164                 int loc = Locations.endOf(prevToken.location);
165                 push(new Token(SCLTerminals.RBRACE, loc, loc, "implicit }"));
166             }
167             if(indentations.isEmpty())
168                 throw error(symbolStart, symbolEnd, "No corresponding opening parenthesis for '" + symbol.text + "'.");
169             push(symbol);
170             return;
171         case SCLTerminals.THEN: // 'then' both closes and opens a block
172             while(!indentations.isEmpty() && indentations.removeAt(indentations.size()-1) >= 0) {
173                 int loc = Locations.endOf(prevToken.location);
174                 push(new Token(SCLTerminals.RBRACE, loc, loc, "implicit }"));
175             }
176             if(indentations.isEmpty())
177                 throw error(symbolStart, symbolEnd, "No corresponding opening parenthesis for '" + symbol.text + "'.");
178             push(symbol);
179             indentations.add(-1);
180             return;
181         case SCLTerminals.EOF:
182             while(indentations.size() > 1 && indentations.get(indentations.size()-1) >= 0) {
183                 int loc = Locations.endOf(prevToken.location);
184                 push(new Token(SCLTerminals.RBRACE, loc, loc, "implicit }"));
185                 indentations.removeAt(indentations.size()-1);
186             }
187             if(indentations.size() > 1)
188                 throw error(symbolStart, symbolEnd, "Unclosed parentheses.");
189             push(symbol);
190             return;
191         default:
192             push(symbol);
193             return;
194         }
195     }
196
197     public void setParserOptions(SCLParserOptions options) {
198         this.options = options;
199         lexer.options = options;
200     }
201
202 }