import java.io.IOException;
import java.util.Arrays;
+import org.simantics.scl.compiler.compilation.CompilationContext;
import org.simantics.scl.compiler.errors.Locations;
import org.simantics.scl.compiler.internal.parsing.Token;
import org.simantics.scl.compiler.internal.parsing.exceptions.SCLSyntaxErrorException;
* @author Hannu Niemistö
*/
public class SCLPostLexer {
+
+ private static final int PATCH_SIZE = 16;
+ private static final int INITIAL_QUEUE_SIZE = 32;
public static TIntHashSet INDENTABLE = new TIntHashSet();
public static TIntHashSet NO_SEMICOLON_BEFORE = new TIntHashSet();
NO_SEMICOLON_BEFORE.add(SCLTerminals.EOF);
NO_SEMICOLON_BEFORE.add(SCLTerminals.SYMBOL);
NO_SEMICOLON_BEFORE.add(SCLTerminals.THEN);
+ NO_SEMICOLON_BEFORE.add(SCLTerminals.ELSE);
NO_SEMICOLON_BEFORE.add(SCLTerminals.IN);
NO_SEMICOLON_BEFORE.add(SCLTerminals.RBRACE);
NO_SEMICOLON_BEFORE.add(SCLTerminals.RBRACKET);
}
SCLLexer lexer;
- Token[] queue = new Token[16];
+ Token[] queue = new Token[INITIAL_QUEUE_SIZE];
int queuePos=0, queueSize=0;
TIntArrayList indentations = new TIntArrayList();
TIntArrayList indentationTokens = new TIntArrayList();
int lineStart = 0;
boolean firstTokenOfLine = true;
private SCLParserOptions options;
+ private boolean isFirstToken = true;
+ private CompilationContext context;
+
+ /**
+ * We are parsing a module header and therefore should process tokens one by one and not by patches.
+ */
+ private boolean isInsideModule = false;
{
indentations.add(0);
public SCLPostLexer(java.io.Reader in) {
this(new SCLLexer(in));
}
+
+ public void setCompilationContext(CompilationContext context) {
+ lexer.setCompilationContext(context);
+ this.context = context;
+ }
public Token nextToken() throws Exception {
while(queuePos == queueSize)
queuePos = 0;
queueSize = 0;
- for(int i=0;i<8;++i)
+ for(int i=0;i<PATCH_SIZE;++i) {
handleToken(lexer.nextToken());
+ if(isInsideModule) {
+ if(context.header == null)
+ break;
+ else
+ isInsideModule = false;
+ }
+ }
}
private SCLSyntaxErrorException error(int start, int end, String description) {
int loc = Locations.endOf(prevToken.location);
push(new Token(SCLTerminals.RBRACE, loc, loc, "implicit }"));
}
- if(indentations.get(indentations.size()-1) == level && symbolId != SCLTerminals.ELSE)
+ if(indentations.get(indentations.size()-1) == level)
push(new Token(SCLTerminals.SEMICOLON, symbolStart, symbolStart, "implicit ;"));
}
}
firstTokenOfLine = false;
+ if(isFirstToken) {
+ isFirstToken = false;
+ if(symbol.id == SCLTerminals.ID && symbol.text.equals("module") && options != null && options.isModule) {
+ push(new Token(SCLTerminals.MODULE, symbol.location, symbol.text));
+ isInsideModule = true;
+ return;
+ }
+ }
}
switch(symbolId) {
return;
case SCLTerminals.THEN:
/*for(int tt : indentationTokens.toArray())
- System.out.print(SCLParser.TERMINAL_NAMES[tt] + " ");
- System.out.println();*/
+ System.out.print(SCLParser.TERMINAL_NAMES[tt] + " ");
+ System.out.println();*/
if(prevTokenId == SCLTerminals.COMMA) {
// for list comprehension syntax
push(symbol);
case SCLTerminals.RBRACE:
case SCLTerminals.RPAREN:
case SCLTerminals.RBRACKET:
- //case SCLTerminals.ELSE:
+ case SCLTerminals.ELSE:
case SCLTerminals.IN:
int removedToken = SCLTerminals.EOF;
while(!indentations.isEmpty()) {
}
if(indentations.isEmpty())
throw error(symbolStart, symbolEnd, "No corresponding opening parenthesis for '" + symbol.text + "'.");
- if(symbolId == SCLTerminals.THEN && removedToken == SCLTerminals.WHEN)
- curToken = symbol = new Token(SCLTerminals.THEN_AFTER_WHEN, symbol.location, symbol.text);
+ if(symbolId == SCLTerminals.THEN) {
+ if(removedToken == SCLTerminals.WHEN)
+ curToken = symbol = new Token(SCLTerminals.THEN_AFTER_WHEN, symbol.location, symbol.text);
+ else {
+ indentations.add(-1);
+ indentationTokens.add(SCLTerminals.THEN);
+ }
+ }
push(symbol);
- //if(symbolId == SCLTerminals.THEN)
- // indentations.add(-1);
return;
case SCLTerminals.EOF:
while(indentations.size() > 1 && indentations.get(indentations.size()-1) >= 0) {