import java.io.IOException;
import java.util.Arrays;
+import org.simantics.scl.compiler.compilation.CompilationContext;
import org.simantics.scl.compiler.errors.Locations;
import org.simantics.scl.compiler.internal.parsing.Token;
import org.simantics.scl.compiler.internal.parsing.exceptions.SCLSyntaxErrorException;
* @author Hannu Niemistö
*/
public class SCLPostLexer {
+
+ private static final int PATCH_SIZE = 16;
+ private static final int INITIAL_QUEUE_SIZE = 32;
public static TIntHashSet INDENTABLE = new TIntHashSet();
public static TIntHashSet NO_SEMICOLON_BEFORE = new TIntHashSet();
INDENTABLE.add(SCLTerminals.WITH);
INDENTABLE.add(SCLTerminals.DO);
INDENTABLE.add(SCLTerminals.MDO);
+ INDENTABLE.add(SCLTerminals.EDO);
INDENTABLE.add(SCLTerminals.LET);
INDENTABLE.add(SCLTerminals.ENFORCE);
INDENTABLE.add(SCLTerminals.WHEN);
INDENTABLE.add(SCLTerminals.FOLLOWS);
INDENTABLE.add(SCLTerminals.EQ);
+ INDENTABLE.add(SCLTerminals.LAMBDA_MATCH);
+ INDENTABLE.add(SCLTerminals.THEN_AFTER_WHEN);
NO_SEMICOLON_BEFORE.add(SCLTerminals.EOF);
NO_SEMICOLON_BEFORE.add(SCLTerminals.SYMBOL);
NO_SEMICOLON_BEFORE.add(SCLTerminals.RBRACE);
NO_SEMICOLON_BEFORE.add(SCLTerminals.RBRACKET);
NO_SEMICOLON_BEFORE.add(SCLTerminals.RPAREN);
+ NO_SEMICOLON_BEFORE.add(SCLTerminals.SEMICOLON);
NO_SEMICOLON_AFTER.add(SCLTerminals.EOF);
NO_SEMICOLON_AFTER.add(SCLTerminals.SYMBOL);
}
SCLLexer lexer;
- Token[] queue = new Token[16];
+ Token[] queue = new Token[INITIAL_QUEUE_SIZE];
int queuePos=0, queueSize=0;
TIntArrayList indentations = new TIntArrayList();
+ TIntArrayList indentationTokens = new TIntArrayList();
Token curToken = null;
int lineStart = 0;
boolean firstTokenOfLine = true;
private SCLParserOptions options;
+ private boolean isFirstToken = true;
+ private CompilationContext context;
+
+ /**
+ * We are parsing a module header and therefore should process tokens one by one and not by patches.
+ */
+ private boolean isInsideModule = false;
{
indentations.add(0);
+ indentationTokens.add(SCLTerminals.EOF);
}
public SCLPostLexer(SCLLexer lexer) {
public SCLPostLexer(java.io.Reader in) {
this(new SCLLexer(in));
}
+
+ public void setCompilationContext(CompilationContext context) {
+ lexer.setCompilationContext(context);
+ this.context = context;
+ }
public Token nextToken() throws Exception {
while(queuePos == queueSize)
queuePos = 0;
queueSize = 0;
- for(int i=0;i<8;++i)
+ for(int i=0;i<PATCH_SIZE;++i) {
handleToken(lexer.nextToken());
+ if(isInsideModule) {
+ if(context.header == null)
+ break;
+ else
+ isInsideModule = false;
+ }
+ }
}
private SCLSyntaxErrorException error(int start, int end, String description) {
int symbolIndentation = symbolStart-lineStart;
//System.out.println("symbolIndentation = " + symbolIndentation);
indentations.add(symbolIndentation);
+ indentationTokens.add(prevTokenId);
firstTokenOfLine = false;
}
else if(firstTokenOfLine) {
//System.out.println("level = " + level);
if(indentations.get(indentations.size()-1) >= level) {
while(indentations.get(indentations.size()-1) > level) {
+ indentationTokens.removeAt(indentations.size()-1);
indentations.removeAt(indentations.size()-1);
int loc = Locations.endOf(prevToken.location);
push(new Token(SCLTerminals.RBRACE, loc, loc, "implicit }"));
}
}
firstTokenOfLine = false;
+ if(isFirstToken) {
+ isFirstToken = false;
+ if(symbol.id == SCLTerminals.ID && symbol.text.equals("module") && options != null && options.isModule) {
+ push(new Token(SCLTerminals.MODULE, symbol.location, symbol.text));
+ isInsideModule = true;
+ return;
+ }
+ }
}
switch(symbolId) {
case SCLTerminals.LPAREN:
case SCLTerminals.LBRACKET:
case SCLTerminals.IF:
+ case SCLTerminals.WHEN:
case SCLTerminals.LET:
indentations.add(-1);
+ indentationTokens.add(symbolId);
push(symbol);
return;
+ case SCLTerminals.THEN:
+ /*for(int tt : indentationTokens.toArray())
+ System.out.print(SCLParser.TERMINAL_NAMES[tt] + " ");
+ System.out.println();*/
+ if(prevTokenId == SCLTerminals.COMMA) {
+ // for list comprehension syntax
+ push(symbol);
+ break;
+ }
case SCLTerminals.RBRACE:
case SCLTerminals.RPAREN:
case SCLTerminals.RBRACKET:
case SCLTerminals.ELSE:
case SCLTerminals.IN:
- while(!indentations.isEmpty() && indentations.removeAt(indentations.size()-1) >= 0) {
- int loc = Locations.endOf(prevToken.location);
- push(new Token(SCLTerminals.RBRACE, loc, loc, "implicit }"));
- }
- if(indentations.isEmpty())
- throw error(symbolStart, symbolEnd, "No corresponding opening parenthesis for '" + symbol.text + "'.");
- push(symbol);
- return;
- case SCLTerminals.THEN: // 'then' both closes and opens a block
- while(!indentations.isEmpty() && indentations.removeAt(indentations.size()-1) >= 0) {
- int loc = Locations.endOf(prevToken.location);
- push(new Token(SCLTerminals.RBRACE, loc, loc, "implicit }"));
+ int removedToken = SCLTerminals.EOF;
+ while(!indentations.isEmpty()) {
+ removedToken = indentationTokens.removeAt(indentations.size()-1);
+ //System.out.println(" removed " + SCLParser.TERMINAL_NAMES[removedToken]);
+ if(indentations.removeAt(indentations.size()-1) < 0)
+ break;
+ long loc = prevToken != null ? Locations.location(Locations.endOf(prevToken.location), Locations.endOf(prevToken.location)) : symbol.location;
+ push(new Token(SCLTerminals.RBRACE, loc, "implicit }"));
}
if(indentations.isEmpty())
throw error(symbolStart, symbolEnd, "No corresponding opening parenthesis for '" + symbol.text + "'.");
+ if(symbolId == SCLTerminals.THEN) {
+ if(removedToken == SCLTerminals.WHEN)
+ curToken = symbol = new Token(SCLTerminals.THEN_AFTER_WHEN, symbol.location, symbol.text);
+ else {
+ indentations.add(-1);
+ indentationTokens.add(SCLTerminals.THEN);
+ }
+ }
push(symbol);
- indentations.add(-1);
return;
case SCLTerminals.EOF:
while(indentations.size() > 1 && indentations.get(indentations.size()-1) >= 0) {
- int loc = Locations.endOf(prevToken.location);
- push(new Token(SCLTerminals.RBRACE, loc, loc, "implicit }"));
+ long loc = prevToken != null ? Locations.location(Locations.endOf(prevToken.location), Locations.endOf(prevToken.location)) : symbol.location;
+ push(new Token(SCLTerminals.RBRACE, loc, "implicit }"));
+ indentationTokens.removeAt(indentations.size()-1);
indentations.removeAt(indentations.size()-1);
}
if(indentations.size() > 1)