/// <summary>
/// Get string tokens.
/// </summary>
/// <returns></returns>
public TokensContainer Tokenize()
{
TokensContainer tokens = new TokensContainer();
while (reader.HasNext())
{
(TokenIdentifier.TokenType type, int advance) = TokenIdentifier.Identify(reader);
if (reader.PeekForward() == ' ')
{
reader.Forward();
}
else
{
for (int i = 0; i < advance; i++)
{
reader.Forward();
}
switch (type)
{
case TokenIdentifier.TokenType.AndOperator:
tokens.AddToken(new Token(TokenIdentifier.TokenType.AndOperator));
break;
case TokenIdentifier.TokenType.DivisionOperator:
tokens.AddToken(new Token(TokenIdentifier.TokenType.DivisionOperator));
break;
case TokenIdentifier.TokenType.Identifier:
tokens.AddToken(IdentifierToken.Parse(reader));
break;
case TokenIdentifier.TokenType.LeftParenthesis:
tokens.AddToken(new Token(TokenIdentifier.TokenType.LeftParenthesis));
break;
case TokenIdentifier.TokenType.MinusOperator:
tokens.AddToken(new Token(TokenIdentifier.TokenType.MinusOperator));
break;
case TokenIdentifier.TokenType.ModuloOperator:
tokens.AddToken(new Token(TokenIdentifier.TokenType.ModuloOperator));
break;
case TokenIdentifier.TokenType.MultiplicationOperator:
tokens.AddToken(new Token(TokenIdentifier.TokenType.MultiplicationOperator));
break;
case TokenIdentifier.TokenType.NegationOperator:
tokens.AddToken(new Token(TokenIdentifier.TokenType.NegationOperator));
break;
case TokenIdentifier.TokenType.Number:
tokens.AddToken(NumberToken.Parse(reader));
break;
case TokenIdentifier.TokenType.BinaryNumber:
tokens.AddToken(BinaryNumberToken.Parse(reader));
break;
case TokenIdentifier.TokenType.OctalNumber:
tokens.AddToken(OctalNumberToken.Parse(reader));
break;
case TokenIdentifier.TokenType.HexNumber:
tokens.AddToken(HexNumberToken.Parse(reader));
break;
case TokenIdentifier.TokenType.OrOperator:
tokens.AddToken(new Token(TokenIdentifier.TokenType.OrOperator));
break;
case TokenIdentifier.TokenType.PlussOperator:
tokens.AddToken(new Token(TokenIdentifier.TokenType.PlussOperator));
break;
case TokenIdentifier.TokenType.PowerOperator:
tokens.AddToken(new Token(TokenIdentifier.TokenType.PowerOperator));
break;
case TokenIdentifier.TokenType.RightParenthesis:
tokens.AddToken(new Token(TokenIdentifier.TokenType.RightParenthesis));
break;
case TokenIdentifier.TokenType.ShiftLeftOperator:
tokens.AddToken(new Token(TokenIdentifier.TokenType.ShiftLeftOperator));
break;
case TokenIdentifier.TokenType.ShiftRightOperator:
tokens.AddToken(new Token(TokenIdentifier.TokenType.ShiftRightOperator));
break;
case TokenIdentifier.TokenType.XorOperator:
tokens.AddToken(new Token(TokenIdentifier.TokenType.XorOperator));
break;
case TokenIdentifier.TokenType.ParamSeparator:
tokens.AddToken(new Token(TokenIdentifier.TokenType.ParamSeparator));
break;
case TokenIdentifier.TokenType.FactorialOperator:
tokens.AddToken(new Token(TokenIdentifier.TokenType.FactorialOperator));
break;
default: throw new Exception("Invalid token at " + reader.Position);
}
}
}
return(tokens);
}