/// <remarks> It is an error to access the current TokenType until <see cref="NextToken"/> has been called. </remarks> public Lexer(string input) { _tokenizer = new Tokenizer(input); for (int i = 0; i < LookAheadCount; i++) _tokens[i] = new LexerToken(); for (int i = 0; i < (LookAheadCount - 1); i++) if (!ReadNext(i)) break; }
public void SetPosition(LexerToken token) { Line = token.Line; LinePos = token.LinePos; }
public void SetEndPosition(LexerToken token) { EndLine = token.Line; EndLinePos = token.LinePos + (token.Token == null ? 0 : token.Token.Length); }