public ITokenizerState Exec(char currentChar, int position, string statement, ref StringBuilder token, List <Token> tokens) { if (char.IsWhiteSpace(currentChar) || char.IsControl(currentChar)) { return(this); // stay in Whitespace state } if (InStringState.IsStringDelimiter(currentChar)) { return(new InStringState()); } ITokenizerState defState = new DefaultState(); defState.Exec(currentChar, position, statement, ref token, tokens); return(defState); }
internal Tokenizer(string statement) { char currChar; ITokenizerState state = new DefaultState(); StringBuilder token = new StringBuilder(); char[] chars = statement.ToCharArray(); for (int i = 0; i < statement.Length; i++) { currChar = chars[i]; state = state.Exec(currChar, i, statement, ref token, mTokens); Context.CheckCancelled(); } if (token.Length > 0) { mTokens.Add(new Token(token.ToString())); } }