public void TestSimplePhrase() { var words = NLPTokenizer.Tokenize("田中さんはどこへ行きますか?"); Assert.AreEqual(8, words.Count); words = NLPTokenizer.TokenizeReducedSentence(); Assert.AreEqual(9, words.Count); words = NLPTokenizer.TokenizeReducedSentence(); Assert.AreEqual(10, words.Count); words = NLPTokenizer.TokenizeReducedSentence(); Assert.AreEqual(10, words.Count); words = NLPTokenizer.TokenizeReducedSentence(); Assert.AreEqual(10, words.Count); words = NLPTokenizer.UndoTokenizeReducedSentence(); Assert.AreEqual(10, words.Count); words = NLPTokenizer.UndoTokenizeReducedSentence(); Assert.AreEqual(9, words.Count); words = NLPTokenizer.UndoTokenizeReducedSentence(); Assert.AreEqual(8, words.Count); words = NLPTokenizer.UndoTokenizeReducedSentence(); Assert.AreEqual(8, words.Count); words = NLPTokenizer.UndoTokenizeReducedSentence(); Assert.AreEqual(8, words.Count); }
private void OnUndoReTokenizeClicked(object sender, RoutedEventArgs e) { dictionaryWordView.ViewModel.DictionaryWords.Clear(); tokenizer.UndoTokenizeReducedSentence(); AddNewTokenizedWordsToViewModel(); }