public void TokensTest(string text, params string[] tokenNames)
{
var tokenizer = new Tokenizer();
var tokens = new List<Token>(tokenizer.Tokenize(text));
for (int i = 0; i < tokenNames.Length; i++)
{
Assert.AreEqual(tokenNames[i], tokens[i].Name);
}
}
public void TokenCountTest(string str, int count)
{
var tokenizer = new Tokenizer();
var tokens = new List<Token>(tokenizer.Tokenize(str));
Assert.AreEqual(count, tokens.Count);
}
public void EmptyStringNoTokensTest()
{
var tokenizer = new Tokenizer();
var tokens = new List<Token>(tokenizer.Tokenize(string.Empty));
Assert.IsEmpty(tokens);
}
