public List<string> Tokenize() { var model = new TokenizerModel(File.OpenRead(BaseFolder + "en-token.bin")); var tokenizer = new TokenizerME(model); return tokenizer.Tokenize(this.Text).ToList(); }
public void TestTokenizerSimpleModel() { var model = TokenizerTestUtil.CreateMaxentTokenModel(); var tokenizer = new TokenizerME(model); var tokens = tokenizer.Tokenize("test,"); Assert.AreEqual(2, tokens.Length); Assert.AreEqual("test", tokens[0]); Assert.AreEqual(",", tokens[1]); }