public virtual void TestDups(string expected, params Token[] tokens) { IEnumerator <Token> toks = ((IEnumerable <Token>)tokens).GetEnumerator(); TokenStream ts = new RemoveDuplicatesTokenFilter((new TokenStreamAnonymousClass(toks))); AssertTokenStreamContents(ts, Regex.Split(expected, "\\s").TrimEnd()); }
public virtual void TestDups(string expected, params Token[] tokens) { IEnumerator <Token> toks = Arrays.AsList(tokens).GetEnumerator(); TokenStream ts = new RemoveDuplicatesTokenFilter((new TokenStreamAnonymousInnerClassHelper(this, toks))); AssertTokenStreamContents(ts, Regex.Split(expected, "\\s").TrimEnd()); }
public virtual void TestDups(string expected, params Token[] tokens) { IEnumerator<Token> toks = Arrays.AsList(tokens).GetEnumerator(); TokenStream ts = new RemoveDuplicatesTokenFilter((new TokenStreamAnonymousInnerClassHelper(this, toks))); AssertTokenStreamContents(ts, Regex.Split(expected, "\\s")); }
public virtual void TestComposition() { TokenStream ts = new RemoveDuplicatesTokenFilter(new SnowballFilter(new KeywordRepeatFilter(new KeywordRepeatFilter(new MockTokenizer(new StringReader("the birds are flying"), MockTokenizer.WHITESPACE, false))), "English")); AssertTokenStreamContents(ts, new string[] { "the", "birds", "bird", "are", "flying", "fli" }, new int[] { 1, 1, 0, 1, 1, 0 }); }