// TODO: maintain a list of token types provided by parser so that scanner can remain unbiased
// TODO: maintain a list of special token types
public Token GetToken()
{
SkipWhitespace();
if (currType == CharType.Alpha)
{
WordToken token = WordToken.GetToken(this);
// TODO: check for reserved words
return(token);
}
else if (currType == CharType.Numeric)
{
return(NumberToken.GetToken(this));
}
else if (currType == CharType.Quote)
{
return(StringToken.GetToken(this));
}
else if (currType == CharType.EOF)
{
return(new EndOfFileToken());
}
else if (currType == CharType.Special)
{
SpecialToken token = SpecialToken.GetToken(this);
// TODO: check for special token types
return(token);
}
return(null);
}
// TODO: maintain a list of token types provided by parser so that scanner can remain unbiased
// TODO: maintain a list of special token types
public Token GetToken(bool skipWhitespace = true)
{
if (skipWhitespace)
{
SkipWhitespace();
}
if (currType == CharType.Whitespace)
{
return(new Token()
{
Type = TokenType.Whitespace
});
}
else if (currType == CharType.Alpha)
{
WordToken token = WordToken.GetToken(this);
// TODO: check for reserved words
return(token);
}
else if (currType == CharType.Numeric)
{
// TODO: need a scanner options class that lets me specify things like
// skipping whitespace, treating numbers etc.
// because here an IP address will be treated as a decimal number
// 192.0[.2.1]
return(NumberToken.GetToken(this));
}
else if (currType == CharType.Quote)
{
return(StringToken.GetToken(this));
}
else if (currType == CharType.EOF)
{
return(new EndOfFileToken());
}
else if (currType == CharType.Special)
{
SpecialToken token = SpecialToken.GetToken(this);
// TODO: check for special token types
return(token);
}
return(null);
}
