public Tuple <PythonCollection <Token>, CompilationError> MakeTokens()
{
PythonCollection <Token> tokens = new PythonCollection <Token>();
while (current_char != null)
{
if (string.IsNullOrWhiteSpace(current_char))
{
Advance();
}
else if (Digits.Contains(current_char))
{
tokens.Append(GetNumber());
}
else if (Letters.Contains(current_char))
{
tokens.Append(MakeIdentifier());
}
else if (current_char == "/")
{
tokens.Append(new Token(TokenType.Div, pos_start: pos));
Advance();
}
else if (current_char == "*")
{
tokens.Append(new Token(TokenType.Mul, pos_start: pos));
Advance();
}
else if (current_char == "+")
{
tokens.Append(new Token(TokenType.Plus, pos_start: pos));
Advance();
}
else if (current_char == "-")
{
tokens.Append(new Token(TokenType.Minus, pos_start: pos));
Advance();
}
else if (current_char == "*")
{
tokens.Append(new Token(TokenType.Mul, pos_start: pos));
Advance();
}
else if (current_char == "^")
{
tokens.Append(new Token(TokenType.Pow, pos_start: pos));
Advance();
}
else if (current_char == "(")
{
tokens.Append(new Token(TokenType.LParen, pos_start: pos));
Advance();
}
else if (current_char == ")")
{
tokens.Append(new Token(TokenType.RParen, pos_start: pos));
Advance();
}
else if (current_char == "!")
{
var result = MakeNotEqualTo();
if (result.Item2 != null)
{
return(Tuple.Create(null as PythonCollection <Token>, result.Item2));
}
tokens.Append(result.Item1);
}
else if (current_char == "=")
{
tokens.Append(MakeEqualTo());
}
else if (current_char == "<")
{
tokens.Append(MakeLessThan());
}
else if (current_char == ">")
{
tokens.Append(MakeGreaterThan());
}
else
{
var cur_char = current_char;
var start = pos.Copy();
Advance();
return(new Tuple <PythonCollection <Token>, CompilationError>(null, new InvalidCharError("Invalid char '" + cur_char + "'", start, pos)));
}
}
tokens.Append(new Token(TokenType.EOF, pos_start: pos));
return(new Tuple <PythonCollection <Token>, CompilationError>(tokens, null));
}
public Parser(PythonCollection <Token> tokens)
{
Tokens = tokens;
Tok_idx = -1;
Advance();
}