Merge "Small improvements to statement collision reporting."
[simantics/platform.git] / bundles / org.simantics.scl.compiler / src / org / simantics / scl / compiler / internal / parsing / parser / SCLPostLexer.java
1 package org.simantics.scl.compiler.internal.parsing.parser;
2
3 import java.io.IOException;
4 import java.util.Arrays;
5
6 import org.simantics.scl.compiler.errors.Locations;
7 import org.simantics.scl.compiler.internal.parsing.Token;
8 import org.simantics.scl.compiler.internal.parsing.exceptions.SCLSyntaxErrorException;
9
10 import gnu.trove.list.array.TIntArrayList;
11 import gnu.trove.set.hash.TIntHashSet;
12
13 /**
14  * 
15  * <http://www.haskell.org/onlinereport/haskell2010/haskellch10.html#x17-17800010.3>
16  * @author Hannu Niemistö
17  */
18 public class SCLPostLexer {
19         
20     public static TIntHashSet INDENTABLE = new TIntHashSet();
21     public static TIntHashSet NO_SEMICOLON_BEFORE = new TIntHashSet();
22     public static TIntHashSet NO_SEMICOLON_AFTER = new TIntHashSet();
23     static {
24         INDENTABLE.add(SCLTerminals.WHERE);
25         INDENTABLE.add(SCLTerminals.QUERY_OP);
26         INDENTABLE.add(SCLTerminals.WITH);
27         INDENTABLE.add(SCLTerminals.DO);
28         INDENTABLE.add(SCLTerminals.MDO);
29         INDENTABLE.add(SCLTerminals.LET);
30         INDENTABLE.add(SCLTerminals.ENFORCE);
31         INDENTABLE.add(SCLTerminals.WHEN);
32         INDENTABLE.add(SCLTerminals.FOLLOWS);
33         INDENTABLE.add(SCLTerminals.EQ);
34         
35         NO_SEMICOLON_BEFORE.add(SCLTerminals.EOF);
36         NO_SEMICOLON_BEFORE.add(SCLTerminals.SYMBOL);
37         NO_SEMICOLON_BEFORE.add(SCLTerminals.THEN);
38         NO_SEMICOLON_BEFORE.add(SCLTerminals.ELSE);
39         NO_SEMICOLON_BEFORE.add(SCLTerminals.IN);
40         NO_SEMICOLON_BEFORE.add(SCLTerminals.RBRACE);
41         NO_SEMICOLON_BEFORE.add(SCLTerminals.RBRACKET);
42         NO_SEMICOLON_BEFORE.add(SCLTerminals.RPAREN);
43         
44         NO_SEMICOLON_AFTER.add(SCLTerminals.EOF);
45         NO_SEMICOLON_AFTER.add(SCLTerminals.SYMBOL);
46     }
47     
48     SCLLexer lexer;
49     Token[] queue = new Token[16];
50     int queuePos=0, queueSize=0;
51     TIntArrayList indentations = new TIntArrayList();
52     Token curToken = null;
53     int lineStart = 0;
54     boolean firstTokenOfLine = true;
55     private SCLParserOptions options;
56             
57     {
58         indentations.add(0);
59     }
60     
61     public SCLPostLexer(SCLLexer lexer) {
62         this.lexer = lexer;
63     }
64     
65     public SCLPostLexer(java.io.Reader in) {
66         this(new SCLLexer(in));
67     }
68
69     public Token nextToken() throws Exception {
70         while(queuePos == queueSize)
71             fillQueue();
72         return queue[queuePos++];
73     }
74     
75     public Token peekToken() throws Exception {
76         while(queuePos == queueSize)
77             fillQueue();
78         return queue[queuePos];
79     }
80     
81     private void push(Token symbol) {
82         if(queueSize == queue.length)
83             queue = Arrays.copyOf(queue, queueSize*2);
84         queue[queueSize++] = symbol;
85     }
86     
87     private void fillQueue() throws Exception {
88         queuePos = 0;
89         queueSize = 0;
90         
91         for(int i=0;i<8;++i)
92             handleToken(lexer.nextToken());
93     }
94     
95     private SCLSyntaxErrorException error(int start, int end, String description) {
96         return new SCLSyntaxErrorException(Locations.location(start, end), description);
97     }
98     
99     private void handleToken(Token symbol) throws IOException {
100         int symbolId = symbol.id;
101         if(symbolId == SCLTerminals.EOL) {
102             lineStart = Locations.endOf(symbol.location);
103             firstTokenOfLine = true;
104             return;
105         }
106         
107         if(symbolId == SCLTerminals.COMMENT) {
108             firstTokenOfLine = false;
109             return;
110         }
111         
112         Token prevToken = curToken;
113         int prevTokenId = prevToken == null ? SCLTerminals.EOF : prevToken.id;
114         curToken = symbol;
115         
116         int symbolStart = Locations.beginOf(symbol.location);
117         int symbolEnd = Locations.endOf(symbol.location);
118         
119         if(INDENTABLE.contains(prevTokenId) && symbolId != SCLTerminals.LBRACE) {
120             if(prevTokenId == SCLTerminals.LET)
121                 indentations.add(-1);
122             push(new Token(SCLTerminals.LBRACE, symbolStart, symbolStart, "implicit {"));
123             int symbolIndentation = symbolStart-lineStart;
124             //System.out.println("symbolIndentation = " + symbolIndentation);
125             indentations.add(symbolIndentation);
126             firstTokenOfLine = false;
127         }
128         else if(firstTokenOfLine) {
129             if(NO_SEMICOLON_AFTER.contains(prevTokenId) || NO_SEMICOLON_BEFORE.contains(symbolId))
130                 ;
131             else {
132                 int level = symbolStart - lineStart;
133                 //System.out.println("level = " + level);
134                 if(indentations.get(indentations.size()-1) >= level) {
135                     while(indentations.get(indentations.size()-1) > level) {
136                         indentations.removeAt(indentations.size()-1);
137                         int loc = Locations.endOf(prevToken.location);
138                         push(new Token(SCLTerminals.RBRACE, loc, loc, "implicit }"));
139                     }
140                     if(indentations.get(indentations.size()-1) == level)
141                         push(new Token(SCLTerminals.SEMICOLON, symbolStart, symbolStart, "implicit ;"));
142                 }
143             }
144             firstTokenOfLine = false;
145         }
146         
147         switch(symbolId) {
148         case SCLTerminals.LBRACE:
149         case SCLTerminals.LPAREN:
150         case SCLTerminals.LBRACKET:
151         case SCLTerminals.IF:
152             indentations.add(-1);
153             push(symbol);
154             return;
155         case SCLTerminals.RBRACE:
156         case SCLTerminals.RPAREN:
157         case SCLTerminals.RBRACKET:
158         case SCLTerminals.ELSE:
159         case SCLTerminals.IN:
160             while(!indentations.isEmpty() && indentations.removeAt(indentations.size()-1) >= 0) {
161                 int loc = Locations.endOf(prevToken.location);
162                 push(new Token(SCLTerminals.RBRACE, loc, loc, "implicit }"));
163             }
164             if(indentations.isEmpty())
165                 throw error(symbolStart, symbolEnd, "No corresponding opening parenthesis for '" + symbol.text + "'.");
166             push(symbol);
167             return;
168         case SCLTerminals.THEN: // 'then' both closes and opens a block
169             while(!indentations.isEmpty() && indentations.removeAt(indentations.size()-1) >= 0) {
170                 int loc = Locations.endOf(prevToken.location);
171                 push(new Token(SCLTerminals.RBRACE, loc, loc, "implicit }"));
172             }
173             if(indentations.isEmpty())
174                 throw error(symbolStart, symbolEnd, "No corresponding opening parenthesis for '" + symbol.text + "'.");
175             push(symbol);
176             indentations.add(-1);
177             return;
178         case SCLTerminals.EOF:
179             while(indentations.size() > 1 && indentations.get(indentations.size()-1) >= 0) {
180                 int loc = Locations.endOf(prevToken.location);
181                 push(new Token(SCLTerminals.RBRACE, loc, loc, "implicit }"));
182                 indentations.removeAt(indentations.size()-1);
183             }
184             if(indentations.size() > 1)
185                 throw error(symbolStart, symbolEnd, "Unclosed parentheses.");
186             push(symbol);
187             return;
188         default:
189             push(symbol);
190             return;
191         }
192     }
193
194     public void setParserOptions(SCLParserOptions options) {
195         this.options = options;
196         lexer.options = options;
197     }
198
199 }