public void TokenizeBasic()
{
m_Tokenizer.Tokenize("test param1 123 arg=value");
List <Token> tokens = m_Tokenizer.Flush();
Assert.AreEqual(9, tokens.Count);
TestHelper.CheckToken(tokens[0], false, "test");
TestHelper.CheckToken(tokens[1], true, " ");
TestHelper.CheckToken(tokens[2], false, "param1");
TestHelper.CheckToken(tokens[3], true, " ");
TestHelper.CheckToken(tokens[4], false, "123");
TestHelper.CheckToken(tokens[5], true, " ");
TestHelper.CheckToken(tokens[6], false, "arg");
TestHelper.CheckToken(tokens[7], true, "=");
TestHelper.CheckToken(tokens[8], false, "value");
}
protected IReadOnlyCollection <string> Run(string query)
{
var config = new QueryTokenizerConfig(
new []
{
new ContainerTokens("\"", "\""),
new ContainerTokens("'", "'"),
},
new [] { "=", "!=", ">", ">=", "<", "<=", "~", "!~" }
);
var sut = new QueryTokenizer(config);
return(sut.Tokenize(query));
}