public void TestTokenizer()
{
var model = TokenizerTestUtil.CreateMaxentTokenModel();
var tokenizer = new TokenizerME(model);
TestTokenizer(tokenizer);
}
public void TestTokenizerSimpleModel()
{
var model = TokenizerTestUtil.CreateMaxentTokenModel();
var tokenizer = new TokenizerME(model);
var tokens = tokenizer.Tokenize("test,");
Assert.AreEqual(2, tokens.Length);
Assert.AreEqual("test", tokens[0]);
Assert.AreEqual(",", tokens[1]);
}
