C# (CSharp) NLPTokenizer.TokenizeReducedSentenceの例

コード例 #1

        public void TestSimplePhrase()
        {
            var words = NLPTokenizer.Tokenize("田中さんはどこへ行きますか？");

            Assert.AreEqual(8, words.Count);

            words = NLPTokenizer.TokenizeReducedSentence();
            Assert.AreEqual(9, words.Count);

            words = NLPTokenizer.TokenizeReducedSentence();
            Assert.AreEqual(10, words.Count);

            words = NLPTokenizer.TokenizeReducedSentence();
            Assert.AreEqual(10, words.Count);

            words = NLPTokenizer.TokenizeReducedSentence();
            Assert.AreEqual(10, words.Count);

            words = NLPTokenizer.UndoTokenizeReducedSentence();
            Assert.AreEqual(10, words.Count);

            words = NLPTokenizer.UndoTokenizeReducedSentence();
            Assert.AreEqual(9, words.Count);

            words = NLPTokenizer.UndoTokenizeReducedSentence();
            Assert.AreEqual(8, words.Count);

            words = NLPTokenizer.UndoTokenizeReducedSentence();
            Assert.AreEqual(8, words.Count);

            words = NLPTokenizer.UndoTokenizeReducedSentence();
            Assert.AreEqual(8, words.Count);
        }

コード例 #2

ファイル: SearchPage.xaml.cs プロジェクト: zeta1999/NLP-Japanese-Dictionary

 private void OnReTokenizeClicked(object sender, RoutedEventArgs e)
 {
     dictionaryWordView.ViewModel.DictionaryWords.Clear();
     tokenizer.TokenizeReducedSentence();
     AddNewTokenizedWordsToViewModel();
 }