public ITokenizerState Exec(char currentChar, int position, string statement, ref StringBuilder token, List <Token> tokens)
{
if (InStringState.IsStringDelimiter(currentChar))
{
tokens.Add(new Token(token.ToString(), TokenType.String));
token = new StringBuilder();
return(new DefaultState());
}
token.Append(currentChar);
return(this); // stay in string state
}
public ITokenizerState Exec(char currentChar, int position, string statement, ref StringBuilder token, List <Token> tokens)
{
if (char.IsWhiteSpace(currentChar) || char.IsControl(currentChar))
{
return(this); // stay in Whitespace state
}
if (InStringState.IsStringDelimiter(currentChar))
{
return(new InStringState());
}
ITokenizerState defState = new DefaultState();
defState.Exec(currentChar, position, statement, ref token, tokens);
return(defState);
}
public ITokenizerState Exec(char currentChar, int position, string statement, ref StringBuilder token, List <Token> tokens)
{
if (char.IsWhiteSpace(currentChar))
{
if (token.Length > 0)
{
tokens.Add(new Token(token.ToString()));
token = new StringBuilder();
}
return(new WhiteSpaceState());
}
if (InStringState.IsStringDelimiter(currentChar))
{
return(new InStringState());
}
if (InCommentState.IsCommentStart(currentChar, position > 0?statement[position - 1]:'\x0000'))
{
return(new InCommentState(statement.Substring(position - 1, 2)));
}
switch (currentChar)
{
case '(':
case ')':
case '[':
case ']':
case '=': // an equals sign that is not inside string quotes should trigger a new token
case ',': // add commas to their own token
if (token.Length > 0)
{
tokens.Add(new Token(token.ToString()));
token = new StringBuilder();
}
tokens.Add(new Token(currentChar.ToString(), TokenType.Comma));
break;
default:
//else add the current char to the string builder
token.Append(currentChar);
break;
}
return(this); // stay in default state
}
