C# (CSharp) SharpNL.Tests.Tokenize TokenizerTestUtil.CreateMaxentTokenModel Exemples

Exemple #1

        public void TestTokenizer()
        {
            var model     = TokenizerTestUtil.CreateMaxentTokenModel();
            var tokenizer = new TokenizerME(model);

            TestTokenizer(tokenizer);
        }

Exemple #2

        public void TestTokenizerSimpleModel()
        {
            var model     = TokenizerTestUtil.CreateMaxentTokenModel();
            var tokenizer = new TokenizerME(model);
            var tokens    = tokenizer.Tokenize("test,");

            Assert.AreEqual(2, tokens.Length);
            Assert.AreEqual("test", tokens[0]);
            Assert.AreEqual(",", tokens[1]);
        }