]> gerrit.simantics Code Review - simantics/platform.git/blob
a2253d8b7bc001cc7d0889a5ab69a0a5885e7922
[simantics/platform.git] /
1 package org.simantics.scl.compiler.internal.parsing.parser;
2
3 import java.io.IOException;
4 import java.util.Arrays;
5
6 import org.simantics.scl.compiler.compilation.CompilationContext;
7 import org.simantics.scl.compiler.errors.Locations;
8 import org.simantics.scl.compiler.internal.parsing.Token;
9 import org.simantics.scl.compiler.internal.parsing.exceptions.SCLSyntaxErrorException;
10
11 import gnu.trove.list.array.TIntArrayList;
12 import gnu.trove.set.hash.TIntHashSet;
13
14 /**
15  * 
16  * <http://www.haskell.org/onlinereport/haskell2010/haskellch10.html#x17-17800010.3>
17  * @author Hannu Niemistö
18  */
19 public class SCLPostLexer {
20     
21     private static final int PATCH_SIZE = 16;
22     private static final int INITIAL_QUEUE_SIZE = 32;
23         
24     public static TIntHashSet INDENTABLE = new TIntHashSet();
25     public static TIntHashSet NO_SEMICOLON_BEFORE = new TIntHashSet();
26     public static TIntHashSet NO_SEMICOLON_AFTER = new TIntHashSet();
27     static {
28         INDENTABLE.add(SCLTerminals.WHERE);
29         INDENTABLE.add(SCLTerminals.QUERY_OP);
30         INDENTABLE.add(SCLTerminals.WITH);
31         INDENTABLE.add(SCLTerminals.DO);
32         INDENTABLE.add(SCLTerminals.MDO);
33         INDENTABLE.add(SCLTerminals.LET);
34         INDENTABLE.add(SCLTerminals.ENFORCE);
35         INDENTABLE.add(SCLTerminals.WHEN);
36         INDENTABLE.add(SCLTerminals.FOLLOWS);
37         INDENTABLE.add(SCLTerminals.EQ);
38         INDENTABLE.add(SCLTerminals.LAMBDA_MATCH);
39         INDENTABLE.add(SCLTerminals.THEN_AFTER_WHEN);
40         
41         NO_SEMICOLON_BEFORE.add(SCLTerminals.EOF);
42         NO_SEMICOLON_BEFORE.add(SCLTerminals.SYMBOL);
43         NO_SEMICOLON_BEFORE.add(SCLTerminals.THEN);
44         NO_SEMICOLON_BEFORE.add(SCLTerminals.ELSE);
45         NO_SEMICOLON_BEFORE.add(SCLTerminals.IN);
46         NO_SEMICOLON_BEFORE.add(SCLTerminals.RBRACE);
47         NO_SEMICOLON_BEFORE.add(SCLTerminals.RBRACKET);
48         NO_SEMICOLON_BEFORE.add(SCLTerminals.RPAREN);
49         NO_SEMICOLON_BEFORE.add(SCLTerminals.SEMICOLON);
50         
51         NO_SEMICOLON_AFTER.add(SCLTerminals.EOF);
52         NO_SEMICOLON_AFTER.add(SCLTerminals.SYMBOL);
53     }
54     
55     SCLLexer lexer;
56     Token[] queue = new Token[INITIAL_QUEUE_SIZE];
57     int queuePos=0, queueSize=0;
58     TIntArrayList indentations = new TIntArrayList();
59     TIntArrayList indentationTokens = new TIntArrayList();
60     Token curToken = null;
61     int lineStart = 0;
62     boolean firstTokenOfLine = true;
63     private SCLParserOptions options;
64     private boolean isFirstToken = true;
65     private CompilationContext context;
66     
67     /**
68      * We are parsing a module header and therefore should process tokens one by one and not by patches.
69      */
70     private boolean isInsideModule = false; 
71             
72     {
73         indentations.add(0);
74         indentationTokens.add(SCLTerminals.EOF);
75     }
76     
77     public SCLPostLexer(SCLLexer lexer) {
78         this.lexer = lexer;
79     }
80     
81     public SCLPostLexer(java.io.Reader in) {
82         this(new SCLLexer(in));
83     }
84     
85     public void setCompilationContext(CompilationContext context) {
86         lexer.setCompilationContext(context);
87         this.context = context;
88     }
89
90     public Token nextToken() throws Exception {
91         while(queuePos == queueSize)
92             fillQueue();
93         return queue[queuePos++];
94     }
95     
96     public Token peekToken() throws Exception {
97         while(queuePos == queueSize)
98             fillQueue();
99         return queue[queuePos];
100     }
101     
102     private void push(Token symbol) {
103         /*System.out.println("TOKEN " + symbol.text + " (" + SCLParser.TERMINAL_NAMES[symbol.id] + ")" +
104                 " [" 
105                 + Locations.beginOf(symbol.location) + ".." 
106                 + Locations.endOf(symbol.location) + "]");*/
107         if(queueSize == queue.length)
108             queue = Arrays.copyOf(queue, queueSize*2);
109         queue[queueSize++] = symbol;
110     }
111     
112     private void fillQueue() throws Exception {
113         queuePos = 0;
114         queueSize = 0;
115         
116         for(int i=0;i<PATCH_SIZE;++i) {
117             handleToken(lexer.nextToken());
118             if(isInsideModule) {
119                 if(context.header == null)
120                     break;
121                 else
122                     isInsideModule = false;
123             }
124         }
125     }
126     
127     private SCLSyntaxErrorException error(int start, int end, String description) {
128         return new SCLSyntaxErrorException(Locations.location(start, end), description);
129     }
130     
131     private void handleToken(Token symbol) throws IOException {
132         int symbolId = symbol.id;
133         if(symbolId == SCLTerminals.EOL) {
134             lineStart = Locations.endOf(symbol.location);
135             firstTokenOfLine = true;
136             return;
137         }
138         
139         if(symbolId == SCLTerminals.COMMENT) {
140             firstTokenOfLine = false;
141             return;
142         }
143         
144         Token prevToken = curToken;
145         int prevTokenId = prevToken == null ? SCLTerminals.EOF : prevToken.id;
146         curToken = symbol;
147         
148         int symbolStart = Locations.beginOf(symbol.location);
149         int symbolEnd = Locations.endOf(symbol.location);
150         
151         if(INDENTABLE.contains(prevTokenId) && symbolId != SCLTerminals.LBRACE) {
152             push(new Token(SCLTerminals.LBRACE, symbolStart, symbolStart, "implicit {"));
153             int symbolIndentation = symbolStart-lineStart;
154             //System.out.println("symbolIndentation = " + symbolIndentation);
155             indentations.add(symbolIndentation);
156             indentationTokens.add(prevTokenId);
157             firstTokenOfLine = false;
158         }
159         else if(firstTokenOfLine) {
160             if(NO_SEMICOLON_AFTER.contains(prevTokenId) || NO_SEMICOLON_BEFORE.contains(symbolId))
161                 ;
162             else {
163                 int level = symbolStart - lineStart;
164                 //System.out.println("level = " + level);
165                 if(indentations.get(indentations.size()-1) >= level) {
166                     while(indentations.get(indentations.size()-1) > level) {
167                         indentationTokens.removeAt(indentations.size()-1);
168                         indentations.removeAt(indentations.size()-1);
169                         int loc = Locations.endOf(prevToken.location);
170                         push(new Token(SCLTerminals.RBRACE, loc, loc, "implicit }"));
171                     }
172                     if(indentations.get(indentations.size()-1) == level)
173                         push(new Token(SCLTerminals.SEMICOLON, symbolStart, symbolStart, "implicit ;"));
174                 }
175             }
176             firstTokenOfLine = false;
177             if(isFirstToken) {
178                 isFirstToken = false;
179                 if(symbol.id == SCLTerminals.ID && symbol.text.equals("module") && options != null && options.isModule) {
180                     push(new Token(SCLTerminals.MODULE, symbol.location, symbol.text));
181                     isInsideModule = true;
182                     return;
183                 }
184             }
185         }
186         
187         switch(symbolId) {
188         case SCLTerminals.LBRACE:
189         case SCLTerminals.LPAREN:
190         case SCLTerminals.LBRACKET:
191         case SCLTerminals.IF:
192         case SCLTerminals.WHEN:
193         case SCLTerminals.LET:
194             indentations.add(-1);
195             indentationTokens.add(symbolId);
196             push(symbol);
197             return;
198         case SCLTerminals.THEN:
199             /*for(int tt : indentationTokens.toArray())
200                     System.out.print(SCLParser.TERMINAL_NAMES[tt] + " ");
201                 System.out.println();*/
202             if(prevTokenId == SCLTerminals.COMMA) {
203                 // for list comprehension syntax
204                 push(symbol);
205                 break;
206             }
207         case SCLTerminals.RBRACE:
208         case SCLTerminals.RPAREN:
209         case SCLTerminals.RBRACKET:
210         case SCLTerminals.ELSE:
211         case SCLTerminals.IN:
212             int removedToken = SCLTerminals.EOF;
213             while(!indentations.isEmpty()) {
214                 removedToken = indentationTokens.removeAt(indentations.size()-1);
215                 //System.out.println("    removed " + SCLParser.TERMINAL_NAMES[removedToken]);
216                 if(indentations.removeAt(indentations.size()-1) < 0)
217                     break;
218                 long loc = prevToken != null ? Locations.location(Locations.endOf(prevToken.location), Locations.endOf(prevToken.location)) : symbol.location;
219                 push(new Token(SCLTerminals.RBRACE, loc, "implicit }"));
220             }
221             if(indentations.isEmpty())
222                 throw error(symbolStart, symbolEnd, "No corresponding opening parenthesis for '" + symbol.text + "'.");
223             if(symbolId == SCLTerminals.THEN) {
224                 if(removedToken == SCLTerminals.WHEN)
225                     curToken = symbol = new Token(SCLTerminals.THEN_AFTER_WHEN, symbol.location, symbol.text);
226                 else {
227                     indentations.add(-1);
228                     indentationTokens.add(SCLTerminals.THEN);
229                 }
230             }   
231             push(symbol);
232             return;
233         case SCLTerminals.EOF:
234             while(indentations.size() > 1 && indentations.get(indentations.size()-1) >= 0) {
235                 long loc = prevToken != null ? Locations.location(Locations.endOf(prevToken.location), Locations.endOf(prevToken.location)) : symbol.location;
236                 push(new Token(SCLTerminals.RBRACE, loc, "implicit }"));
237                 indentationTokens.removeAt(indentations.size()-1);
238                 indentations.removeAt(indentations.size()-1);
239             }
240             if(indentations.size() > 1)
241                 throw error(symbolStart, symbolEnd, "Unclosed parentheses.");
242             push(symbol);
243             return;
244         default:
245             push(symbol);
246             return;
247         }
248     }
249
250     public void setParserOptions(SCLParserOptions options) {
251         this.options = options;
252         lexer.options = options;
253     }
254
255 }