Example #1
0
        public void TokensTest(string text, params string[] tokenNames)
        {
            var tokenizer = new Tokenizer();
            var tokens = new List<Token>(tokenizer.Tokenize(text));

            for (int i = 0; i < tokenNames.Length; i++)
            {
                Assert.AreEqual(tokenNames[i], tokens[i].Name);
            }
        }
Example #2
0
 public void TokenCountTest(string str, int count)
 {
     var tokenizer = new Tokenizer();
     var tokens = new List<Token>(tokenizer.Tokenize(str));
     Assert.AreEqual(count, tokens.Count);
 }
Example #3
0
 public void EmptyStringNoTokensTest()
 {
     var tokenizer = new Tokenizer();
     var tokens = new List<Token>(tokenizer.Tokenize(string.Empty));
     Assert.IsEmpty(tokens);
 }