public List<Token> Tokenize(string text)
{
List<Token> tokens = new List<Token>();
Token currentToken = null;
char currentChar;
for (int i = 0; i < text.Length; i++)
{
currentChar = text[i];
if (WhitespaceChars.Contains(currentChar)) // Whitespace
{
CheckIdentifier(currentToken);
if (KeepWhitespace)
{
currentToken = new Token(TokenType.Whitespace, currentChar.ToString(), i);
tokens.Add(currentToken);
}
else
{
currentToken = new Token();
}
}
else if (SymbolChars.Contains(currentChar)) // Symbol
{
CheckIdentifier(currentToken);
currentToken = new Token(TokenType.Symbol, currentChar.ToString(), i);
tokens.Add(currentToken);
}
else if (LiteralDelimiters.Contains(currentChar)) // Literal
{
CheckIdentifier(currentToken);
currentToken = new Token(TokenType.Literal, currentChar.ToString(), i);
tokens.Add(currentToken);
char delimeter = currentChar;
for (i++; i < text.Length; i++)
{
currentChar = text[i];
currentToken.Text += currentChar;
if (currentChar == delimeter && !IsEscaped(currentToken.Text))
{
if (AutoParseLiteral)
{
currentToken.Text = ParseString(currentToken.Text);
}
break;
}
}
}
else // Identifier, Numeric, Keyword
{
if (currentToken != null && currentToken.Type == TokenType.Identifier)
{
currentToken.Text += currentChar;
}
else
{
currentToken = new Token(TokenType.Identifier, currentChar.ToString(), i);
tokens.Add(currentToken);
}
if (i + 1 >= text.Length) // EOF
{
CheckIdentifier(currentToken);
}
}
}
return tokens;
}
private void CheckIdentifier(Token token)
{
if (token != null && token.Type == TokenType.Identifier)
{
double result;
if (double.TryParse(token.Text, out result))
{
token.Type = TokenType.Numeric;
}
else if (Keywords.Contains(token.Text))
{
token.Type = TokenType.Keyword;
}
}
}
private void CheckIdentifier(Token token)
{
if (token != null && token.Type == TokenType.Identifier)
{
double result;
if (double.TryParse(token.Text, out result))
{
token.Type = TokenType.Numeric;
if (tokens.Count > 1)
{
Token previousToken = tokens[tokens.Count - 2];
if (previousToken != null && previousToken.Type == TokenType.Symbol && previousToken.Text.Equals("-", StringComparison.InvariantCulture))
{
tokens.Remove(previousToken);
token.Position--;
token.Text = "-" + token.Text;
}
}
}
else if (Keywords.Contains(token.Text))
{
token.Type = TokenType.Keyword;
}
}
}
