1 package org.simantics.scl.compiler.internal.parsing.parser;
3 import java.io.IOException;
4 import java.util.Arrays;
6 import org.simantics.scl.compiler.errors.Locations;
7 import org.simantics.scl.compiler.internal.parsing.Token;
8 import org.simantics.scl.compiler.internal.parsing.exceptions.SCLSyntaxErrorException;
10 import gnu.trove.list.array.TIntArrayList;
11 import gnu.trove.set.hash.TIntHashSet;
15 * <http://www.haskell.org/onlinereport/haskell2010/haskellch10.html#x17-17800010.3>
16 * @author Hannu Niemistö
18 public class SCLPostLexer {
20 public static TIntHashSet INDENTABLE = new TIntHashSet();
21 public static TIntHashSet NO_SEMICOLON_BEFORE = new TIntHashSet();
22 public static TIntHashSet NO_SEMICOLON_AFTER = new TIntHashSet();
24 INDENTABLE.add(SCLTerminals.WHERE);
25 INDENTABLE.add(SCLTerminals.QUERY_OP);
26 INDENTABLE.add(SCLTerminals.WITH);
27 INDENTABLE.add(SCLTerminals.DO);
28 INDENTABLE.add(SCLTerminals.MDO);
29 INDENTABLE.add(SCLTerminals.LET);
30 INDENTABLE.add(SCLTerminals.ENFORCE);
31 INDENTABLE.add(SCLTerminals.WHEN);
32 INDENTABLE.add(SCLTerminals.FOLLOWS);
33 INDENTABLE.add(SCLTerminals.EQ);
34 INDENTABLE.add(SCLTerminals.LAMBDA_MATCH);
35 INDENTABLE.add(SCLTerminals.THEN_AFTER_WHEN);
37 NO_SEMICOLON_BEFORE.add(SCLTerminals.EOF);
38 NO_SEMICOLON_BEFORE.add(SCLTerminals.SYMBOL);
39 NO_SEMICOLON_BEFORE.add(SCLTerminals.THEN);
40 NO_SEMICOLON_BEFORE.add(SCLTerminals.ELSE);
41 NO_SEMICOLON_BEFORE.add(SCLTerminals.IN);
42 NO_SEMICOLON_BEFORE.add(SCLTerminals.RBRACE);
43 NO_SEMICOLON_BEFORE.add(SCLTerminals.RBRACKET);
44 NO_SEMICOLON_BEFORE.add(SCLTerminals.RPAREN);
45 NO_SEMICOLON_BEFORE.add(SCLTerminals.SEMICOLON);
47 NO_SEMICOLON_AFTER.add(SCLTerminals.EOF);
48 NO_SEMICOLON_AFTER.add(SCLTerminals.SYMBOL);
52 Token[] queue = new Token[16];
53 int queuePos=0, queueSize=0;
54 TIntArrayList indentations = new TIntArrayList();
55 TIntArrayList indentationTokens = new TIntArrayList();
56 Token curToken = null;
58 boolean firstTokenOfLine = true;
59 private SCLParserOptions options;
63 indentationTokens.add(SCLTerminals.EOF);
66 public SCLPostLexer(SCLLexer lexer) {
70 public SCLPostLexer(java.io.Reader in) {
71 this(new SCLLexer(in));
74 public Token nextToken() throws Exception {
75 while(queuePos == queueSize)
77 return queue[queuePos++];
80 public Token peekToken() throws Exception {
81 while(queuePos == queueSize)
83 return queue[queuePos];
86 private void push(Token symbol) {
87 /*System.out.println("TOKEN " + symbol.text + " (" + SCLParser.TERMINAL_NAMES[symbol.id] + ")" +
89 + Locations.beginOf(symbol.location) + ".."
90 + Locations.endOf(symbol.location) + "]");*/
91 if(queueSize == queue.length)
92 queue = Arrays.copyOf(queue, queueSize*2);
93 queue[queueSize++] = symbol;
96 private void fillQueue() throws Exception {
101 handleToken(lexer.nextToken());
104 private SCLSyntaxErrorException error(int start, int end, String description) {
105 return new SCLSyntaxErrorException(Locations.location(start, end), description);
108 private void handleToken(Token symbol) throws IOException {
109 int symbolId = symbol.id;
110 if(symbolId == SCLTerminals.EOL) {
111 lineStart = Locations.endOf(symbol.location);
112 firstTokenOfLine = true;
116 if(symbolId == SCLTerminals.COMMENT) {
117 firstTokenOfLine = false;
121 Token prevToken = curToken;
122 int prevTokenId = prevToken == null ? SCLTerminals.EOF : prevToken.id;
125 int symbolStart = Locations.beginOf(symbol.location);
126 int symbolEnd = Locations.endOf(symbol.location);
128 if(INDENTABLE.contains(prevTokenId) && symbolId != SCLTerminals.LBRACE) {
129 push(new Token(SCLTerminals.LBRACE, symbolStart, symbolStart, "implicit {"));
130 int symbolIndentation = symbolStart-lineStart;
131 //System.out.println("symbolIndentation = " + symbolIndentation);
132 indentations.add(symbolIndentation);
133 indentationTokens.add(prevTokenId);
134 firstTokenOfLine = false;
136 else if(firstTokenOfLine) {
137 if(NO_SEMICOLON_AFTER.contains(prevTokenId) || NO_SEMICOLON_BEFORE.contains(symbolId))
140 int level = symbolStart - lineStart;
141 //System.out.println("level = " + level);
142 if(indentations.get(indentations.size()-1) >= level) {
143 while(indentations.get(indentations.size()-1) > level) {
144 indentationTokens.removeAt(indentations.size()-1);
145 indentations.removeAt(indentations.size()-1);
146 int loc = Locations.endOf(prevToken.location);
147 push(new Token(SCLTerminals.RBRACE, loc, loc, "implicit }"));
149 if(indentations.get(indentations.size()-1) == level)
150 push(new Token(SCLTerminals.SEMICOLON, symbolStart, symbolStart, "implicit ;"));
153 firstTokenOfLine = false;
157 case SCLTerminals.LBRACE:
158 case SCLTerminals.LPAREN:
159 case SCLTerminals.LBRACKET:
160 case SCLTerminals.IF:
161 case SCLTerminals.WHEN:
162 case SCLTerminals.LET:
163 indentations.add(-1);
164 indentationTokens.add(symbolId);
167 case SCLTerminals.THEN:
168 /*for(int tt : indentationTokens.toArray())
169 System.out.print(SCLParser.TERMINAL_NAMES[tt] + " ");
170 System.out.println();*/
171 if(prevTokenId == SCLTerminals.COMMA) {
172 // for list comprehension syntax
176 case SCLTerminals.RBRACE:
177 case SCLTerminals.RPAREN:
178 case SCLTerminals.RBRACKET:
179 case SCLTerminals.ELSE:
180 case SCLTerminals.IN:
181 int removedToken = SCLTerminals.EOF;
182 while(!indentations.isEmpty()) {
183 removedToken = indentationTokens.removeAt(indentations.size()-1);
184 //System.out.println(" removed " + SCLParser.TERMINAL_NAMES[removedToken]);
185 if(indentations.removeAt(indentations.size()-1) < 0)
187 long loc = prevToken != null ? Locations.location(Locations.endOf(prevToken.location), Locations.endOf(prevToken.location)) : symbol.location;
188 push(new Token(SCLTerminals.RBRACE, loc, "implicit }"));
190 if(indentations.isEmpty())
191 throw error(symbolStart, symbolEnd, "No corresponding opening parenthesis for '" + symbol.text + "'.");
192 if(symbolId == SCLTerminals.THEN) {
193 if(removedToken == SCLTerminals.WHEN)
194 curToken = symbol = new Token(SCLTerminals.THEN_AFTER_WHEN, symbol.location, symbol.text);
196 indentations.add(-1);
197 indentationTokens.add(SCLTerminals.THEN);
202 case SCLTerminals.EOF:
203 while(indentations.size() > 1 && indentations.get(indentations.size()-1) >= 0) {
204 long loc = prevToken != null ? Locations.location(Locations.endOf(prevToken.location), Locations.endOf(prevToken.location)) : symbol.location;
205 push(new Token(SCLTerminals.RBRACE, loc, "implicit }"));
206 indentationTokens.removeAt(indentations.size()-1);
207 indentations.removeAt(indentations.size()-1);
209 if(indentations.size() > 1)
210 throw error(symbolStart, symbolEnd, "Unclosed parentheses.");
219 public void setParserOptions(SCLParserOptions options) {
220 this.options = options;
221 lexer.options = options;