public ITokenizerState Exec(char currentChar, int position, string statement, ref StringBuilder token, List <Token> tokens)
{
if (char.IsWhiteSpace(currentChar) || char.IsControl(currentChar))
{
return(this); // stay in Whitespace state
}
if (InStringState.IsStringDelimiter(currentChar))
{
return(new InStringState());
}
ITokenizerState defState = new DefaultState();
defState.Exec(currentChar, position, statement, ref token, tokens);
return(defState);
}
internal Tokenizer(string statement)
{
char currChar;
ITokenizerState state = new DefaultState();
StringBuilder token = new StringBuilder();
char[] chars = statement.ToCharArray();
for (int i = 0; i < statement.Length; i++)
{
currChar = chars[i];
state = state.Exec(currChar, i, statement, ref token, mTokens);
Context.CheckCancelled();
}
if (token.Length > 0)
{
mTokens.Add(new Token(token.ToString()));
}
}
