public IList<string> TestNuveTokenizerReturnDelimsFalse(string text) { var tokenizer = new NuveTokenizer(); IList<string> tokens = tokenizer.Tokenize(text); return tokens; }
public IList<string> TestNuveTokenizerReturnDelimsTrue(string text) { var tokenizer = new NuveTokenizer(true); IList<string> tokens = tokenizer.Tokenize(text); int length = string.Join("", tokens).Length; Assert.AreEqual(length, text.Length); return tokens; }