public void TokenizeBasic()
{
m_Tokenizer.Tokenize("test param1 123 arg=value");
List <Token> tokens = m_Tokenizer.Flush();
Assert.AreEqual(9, tokens.Count);
TestHelper.CheckToken(tokens[0], false, "test");
TestHelper.CheckToken(tokens[1], true, " ");
TestHelper.CheckToken(tokens[2], false, "param1");
TestHelper.CheckToken(tokens[3], true, " ");
TestHelper.CheckToken(tokens[4], false, "123");
TestHelper.CheckToken(tokens[5], true, " ");
TestHelper.CheckToken(tokens[6], false, "arg");
TestHelper.CheckToken(tokens[7], true, "=");
TestHelper.CheckToken(tokens[8], false, "value");
}