private Token NextTokenFromScanner(Scanner scanner)
{
this.RemoveWhitespacePrefixFromScanner(scanner);
if (scanner.HasNextChar())
{
char symbol = scanner.NextCharNoEx();
string content = String.Empty;
int startPos = scanner.CursorPosition.characterNumber;
int startLine = scanner.CursorPosition.lineNumber + 1;
// ID?
if (symbol.IsLetter())
{
do
{
content += symbol;
symbol = scanner.NextCharNoEx();
} while (symbol.IsLetterOrDigitOrUnderscore());
if (this.OutputToErrorList)
{
ErrorManager.Add($"Invalid symbol in identifier sequence: '{symbol}'", (startLine, scanner.CursorPosition.characterNumber));
}
scanner.GoBack();
if (Language.Keywords.Contains(content))
{
return(new Token()
{
Type = TokenType.Keyword, TokenContent = content, SourceLocation = (startLine, startPos)
});