public Token scan()
{
for (; ; peek = (char)Console.Read())
{
if (peek == ' ' || peek == '\t')
{
continue;
}
else if (peek == '\n')
{
++line;
}
else
{
break;
}
}
if (Char.IsDigit(peek))
{
int v = 0;
do
{
v = 10 * v + int.Parse(peek.ToString()); //this is dumb is there a better way?
peek = (char)Console.Read();
} while (Char.IsDigit(peek));
return new IntegerToken(v);
}
if (Char.IsLetter(peek))
{
string wordValue = "";
do
{
wordValue += peek;
peek = (char)Console.Read();
} while (Char.IsLetterOrDigit(peek));
if (words.ContainsKey(wordValue))
{
return words[wordValue];
}
else
{
Word w = new Word(Tag.IDENTIFIER, wordValue);
words[wordValue] = w;
return w;
}
}
Token t = new Token(peek);
peek = ' ';
return t;
}
protected virtual Token BuildToken()
{
if (EndOfSource())
{
return null;
}
if (Char.IsDigit(peek))
{
return Integer();
}
if (Char.IsLetter(peek))
{
return Word();
}
Token t = new Token(peek);
peek = ' ';
return t;
}