A TokenFilter which filters out Tokens at the same position and Term text as the previous token in the stream.
Inheritance: TokenFilter
RemoveDuplicatesTokenFilter Class Documentation
Esempio n. 1
0
        public virtual void TestDups(string expected, params Token[] tokens)
        {
            IEnumerator <Token> toks = ((IEnumerable <Token>)tokens).GetEnumerator();
            TokenStream         ts   = new RemoveDuplicatesTokenFilter((new TokenStreamAnonymousClass(toks)));

            AssertTokenStreamContents(ts, Regex.Split(expected, "\\s").TrimEnd());
        }
Esempio n. 2
0
        public virtual void TestDups(string expected, params Token[] tokens)
        {
            IEnumerator <Token> toks = Arrays.AsList(tokens).GetEnumerator();
            TokenStream         ts   = new RemoveDuplicatesTokenFilter((new TokenStreamAnonymousInnerClassHelper(this, toks)));

            AssertTokenStreamContents(ts, Regex.Split(expected, "\\s").TrimEnd());
        }
Esempio n. 3
0
        public virtual void TestDups(string expected, params Token[] tokens)
        {

            IEnumerator<Token> toks = Arrays.AsList(tokens).GetEnumerator();
            TokenStream ts = new RemoveDuplicatesTokenFilter((new TokenStreamAnonymousInnerClassHelper(this, toks)));

            AssertTokenStreamContents(ts, Regex.Split(expected, "\\s"));
        }
Esempio n. 4
0
        public virtual void TestComposition()
        {
            TokenStream ts = new RemoveDuplicatesTokenFilter(new SnowballFilter(new KeywordRepeatFilter(new KeywordRepeatFilter(new MockTokenizer(new StringReader("the birds are flying"), MockTokenizer.WHITESPACE, false))), "English"));

            AssertTokenStreamContents(ts, new string[] { "the", "birds", "bird", "are", "flying", "fli" }, new int[] { 1, 1, 0, 1, 1, 0 });
        }