/// <inheritdoc/>
public void Advance()
{
token = la;
if(la == Token.EOF)
return;
LookAheadToken();
}
/// <summary>
/// Tells the lexer to read the next token and to store it on the internal buffer.
/// </summary>
public void Advance()
{
if(la != null){
token = la;
}else{
token = Token.EOF;
return;
}
LookAheadToken();
}
/// <inheritdoc/>
public void SetInitialLocation(int line, int column)
{
if(token != null)
throw new InvalidOperationException("SetInitialLocation should not be called after Advance is called for the first time");
line_num = line;
column_num = column;
la = new Token(line, column, la.Literal, la.Kind);
}
void LookAheadToken()
{
SkipWhitespace();
char ch = PeekChar();
switch(ch){
case EOF:
la = Token.EOF;
break;
case '(':
case ')':
case ',':
case '.':
case ';':
case ':':
case '[':
case ']':
case '=':
GetChar();
la = new Token(line_num, column_num, ch.ToString(), TokenKind.SyntaxToken);
++column_num;
break;
case '-':
la = GetIdOrNumber();
break;
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9':
la = GetNumber(true);
break;
default:
la = GetIdOrKeyword();
break;
}
}
Token GetIdOrKeyword(char first)
{
Debug.Assert(first != EOF && !IsIdTerminator(first), "Really meant an id?");
var sb = new StringBuilder(first.ToString());
// See if there's more chars to Id
char ch = PeekChar();
while(ch != EOF && (!IsIdTerminator(ch))){
sb.Append(ch);
GetChar();
ch = PeekChar();
}
var str = sb.ToString();
var tmp = new Token(line_num, column_num, str, (str == "let") ? TokenKind.KeywordToken : TokenKind.Identifier);
column_num += sb.Length;
return tmp;
}
void LookAheadToken()
{
SkipWhitespace();
char ch = PeekChar();
switch(ch){
case EOF:
la = Token.EOF;
break;
case ',':
case ':':
GetChar();
la = new Token(line_num, column_num, ch.ToString(), TokenKind.SyntaxToken);
++column_num;
break;
case '\n':
GetChar();
la = Token.GetEOLToken(line_num, column_num);
++line_num;
column_num = 1;
break;
case '-':
GetChar();
la = GetStringOrNumber(true);
break;
default:
la = GetStringOrNumber(false);
break;
}
}