/// <summary>
/// Lexical analysis on the input string
/// </summary>
/// <param name="str">The text representation of the term</param>
/// <returns>List of tokens contained</returns>
/// <exception cref="ReadException">Thrown if lexical error found</exception>
static LinkedList<Token> Tokenize(string str)
{
LinkedList<Token> result = new LinkedList<Token>();
Buffer b = new Buffer(str);
b.EatWhitespace();
while (!b.EndOfFile())
{
bool error;
Token/*?*/ token = Scan(b, out error);
if (error)
{
string msg = "At position " + (b.Position - 1).ToString() + ", "
+ (b.EndOfFile() ? "end reached while scanning string: " :
"lexical error in string: ");
throw new ReadException(msg + str);
}
// assert token != null;
result.AddLast(token);
b.EatWhitespace();
}
result.AddLast(new Token(Token.Kind.EOF, "eof", b.Position));
return result;
}