public virtual void TestReset() { CharArraySet dict = makeDictionary("Rind", "Fleisch", "Draht", "Schere", "Gesetz", "Aufgabe", "Überwachung"); Tokenizer wsTokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("Rindfleischüberwachungsgesetz")); DictionaryCompoundWordTokenFilter tf = new DictionaryCompoundWordTokenFilter(TEST_VERSION_CURRENT, wsTokenizer, dict, CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE, CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE, CompoundWordTokenFilterBase.DEFAULT_MAX_SUBWORD_SIZE, false); ICharTermAttribute termAtt = tf.GetAttribute <ICharTermAttribute>(); tf.Reset(); assertTrue(tf.IncrementToken()); assertEquals("Rindfleischüberwachungsgesetz", termAtt.ToString()); assertTrue(tf.IncrementToken()); assertEquals("Rind", termAtt.ToString()); tf.End(); tf.Dispose(); wsTokenizer.SetReader(new StringReader("Rindfleischüberwachungsgesetz")); tf.Reset(); assertTrue(tf.IncrementToken()); assertEquals("Rindfleischüberwachungsgesetz", termAtt.ToString()); }
public virtual void TestRetainMockAttribute() { CharArraySet dict = makeDictionary("abc", "d", "efg"); Tokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("abcdefg")); TokenStream stream = new MockRetainAttributeFilter(tokenizer); stream = new DictionaryCompoundWordTokenFilter(TEST_VERSION_CURRENT, stream, dict, CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE, CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE, CompoundWordTokenFilterBase.DEFAULT_MAX_SUBWORD_SIZE, false); IMockRetainAttribute retAtt = stream.AddAttribute <IMockRetainAttribute>(); stream.Reset(); while (stream.IncrementToken()) { assertTrue("Custom attribute value was lost", retAtt.Retain); } }
public virtual void TestRetainMockAttribute() { CharArraySet dict = makeDictionary("abc", "d", "efg"); Tokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("abcdefg")); TokenStream stream = new MockRetainAttributeFilter(tokenizer); stream = new DictionaryCompoundWordTokenFilter(TEST_VERSION_CURRENT, stream, dict, CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE, CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE, CompoundWordTokenFilterBase.DEFAULT_MAX_SUBWORD_SIZE, false); IMockRetainAttribute retAtt = stream.AddAttribute<IMockRetainAttribute>(); stream.Reset(); while (stream.IncrementToken()) { assertTrue("Custom attribute value was lost", retAtt.Retain); } }
public virtual void TestReset() { CharArraySet dict = makeDictionary("Rind", "Fleisch", "Draht", "Schere", "Gesetz", "Aufgabe", "Überwachung"); Tokenizer wsTokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("Rindfleischüberwachungsgesetz")); DictionaryCompoundWordTokenFilter tf = new DictionaryCompoundWordTokenFilter(TEST_VERSION_CURRENT, wsTokenizer, dict, CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE, CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE, CompoundWordTokenFilterBase.DEFAULT_MAX_SUBWORD_SIZE, false); ICharTermAttribute termAtt = tf.GetAttribute<ICharTermAttribute>(); tf.Reset(); assertTrue(tf.IncrementToken()); assertEquals("Rindfleischüberwachungsgesetz", termAtt.ToString()); assertTrue(tf.IncrementToken()); assertEquals("Rind", termAtt.ToString()); tf.End(); tf.Dispose(); wsTokenizer.Reader = new StringReader("Rindfleischüberwachungsgesetz"); tf.Reset(); assertTrue(tf.IncrementToken()); assertEquals("Rindfleischüberwachungsgesetz", termAtt.ToString()); }