public IList<string> TestNuveTokenizerReturnDelimsFalse(string text)
{
var tokenizer = new NuveTokenizer();
IList<string> tokens = tokenizer.Tokenize(text);
return tokens;
}
public IList<string> TestNuveTokenizerReturnDelimsTrue(string text)
{
var tokenizer = new NuveTokenizer(true);
IList<string> tokens = tokenizer.Tokenize(text);
int length = string.Join("", tokens).Length;
Assert.AreEqual(length, text.Length);
return tokens;
}