public void CreateNull() { WordsTokenizerFactory tokenizerFactory = new WordsTokenizerFactory( WordsTokenizerFactory.Grouped, new SimpleWordItemFactory(Global.PosTagger, Global.Raw), new CombinedPipeline <string>(), new CombinedPipeline <WordEx>()); IWordsTokenizer tokenizer = tokenizerFactory.Create(null); Assert.IsInstanceOf <NullWordsTokenizer>(tokenizer); }
public void Create() { WordsTokenizerFactory tokenizerFactory = new WordsTokenizerFactory( WordsTokenizerFactory.Grouped, new SimpleWordItemFactory(Global.PosTagger, Global.Raw), new CombinedPipeline <string>(), new CombinedPipeline <WordEx>()); IWordsTokenizer tokenizer = tokenizerFactory.Create("Test words"); string[] words = tokenizer.GetWords().ToArray(); Assert.AreEqual(2, words.Length); Assert.AreEqual("Test", words[0]); Assert.AreEqual("words", words[1]); }