public List<string> Tokenize()
{
var model = new TokenizerModel(File.OpenRead(BaseFolder + "en-token.bin"));
var tokenizer = new TokenizerME(model);
return tokenizer.Tokenize(this.Text).ToList();
}
public void TestTokenizerSimpleModel() {
var model = TokenizerTestUtil.CreateMaxentTokenModel();
var tokenizer = new TokenizerME(model);
var tokens = tokenizer.Tokenize("test,");
Assert.AreEqual(2, tokens.Length);
Assert.AreEqual("test", tokens[0]);
Assert.AreEqual(",", tokens[1]);
}
