import java.io.IOException;
import java.util.Arrays;
+import org.simantics.scl.compiler.compilation.CompilationContext;
import org.simantics.scl.compiler.errors.Locations;
import org.simantics.scl.compiler.internal.parsing.Token;
import org.simantics.scl.compiler.internal.parsing.exceptions.SCLSyntaxErrorException;
* @author Hannu Niemistö
*/
public class SCLPostLexer {
+
+ private static final int PATCH_SIZE = 16;
+ private static final int INITIAL_QUEUE_SIZE = 32;
public static TIntHashSet INDENTABLE = new TIntHashSet();
public static TIntHashSet NO_SEMICOLON_BEFORE = new TIntHashSet();
INDENTABLE.add(SCLTerminals.WITH);
INDENTABLE.add(SCLTerminals.DO);
INDENTABLE.add(SCLTerminals.MDO);
+ INDENTABLE.add(SCLTerminals.EDO);
INDENTABLE.add(SCLTerminals.LET);
INDENTABLE.add(SCLTerminals.ENFORCE);
INDENTABLE.add(SCLTerminals.WHEN);
}
SCLLexer lexer;
- Token[] queue = new Token[16];
+ Token[] queue = new Token[INITIAL_QUEUE_SIZE];
int queuePos=0, queueSize=0;
TIntArrayList indentations = new TIntArrayList();
TIntArrayList indentationTokens = new TIntArrayList();
int lineStart = 0;
boolean firstTokenOfLine = true;
private SCLParserOptions options;
+ private boolean isFirstToken = true;
+ private CompilationContext context;
+
+ /**
+ * We are parsing a module header and therefore should process tokens one by one and not by patches.
+ */
+ private boolean isInsideModule = false;
{
indentations.add(0);
public SCLPostLexer(java.io.Reader in) {
this(new SCLLexer(in));
}
+
+ public void setCompilationContext(CompilationContext context) {
+ lexer.setCompilationContext(context);
+ this.context = context;
+ }
public Token nextToken() throws Exception {
while(queuePos == queueSize)
queuePos = 0;
queueSize = 0;
- for(int i=0;i<8;++i)
+ for(int i=0;i<PATCH_SIZE;++i) {
handleToken(lexer.nextToken());
+ if(isInsideModule) {
+ if(context.header == null)
+ break;
+ else
+ isInsideModule = false;
+ }
+ }
}
private SCLSyntaxErrorException error(int start, int end, String description) {
}
}
firstTokenOfLine = false;
+ if(isFirstToken) {
+ isFirstToken = false;
+ if(symbol.id == SCLTerminals.ID && symbol.text.equals("module") && options != null && options.isModule) {
+ push(new Token(SCLTerminals.MODULE, symbol.location, symbol.text));
+ isInsideModule = true;
+ return;
+ }
+ }
}
switch(symbolId) {
return;
case SCLTerminals.THEN:
/*for(int tt : indentationTokens.toArray())
- System.out.print(SCLParser.TERMINAL_NAMES[tt] + " ");
- System.out.println();*/
+ System.out.print(SCLParser.TERMINAL_NAMES[tt] + " ");
+ System.out.println();*/
if(prevTokenId == SCLTerminals.COMMA) {
// for list comprehension syntax
push(symbol);