Like Analysis.Core.StopFilter except it will not remove the last token if that token was not followed by some token separator. For example, a query 'find the' would preserve the 'the' since it was not followed by a space or punctuation or something, and mark it KEYWORD so future stemmers won't touch it either while a query like "find the popsicle' would remove 'the' as a stopword.

Normally you'd use the ordinary Analysis.Core.StopFilter in your indexAnalyzer and then this class in your queryAnalyzer, when using one of the analyzing suggesters.

Inheritance: Lucene.Net.Analysis.TokenFilter
SuggestStopFilter Class Documentation
Exemplo n.º 1
0
 public void TestEndNotStopWord()
 {
     CharArraySet stopWords = StopFilter.MakeStopSet(TEST_VERSION_CURRENT, "to");
     TokenStream stream = new MockTokenizer(new StringReader("go to"));
     TokenStream filter = new SuggestStopFilter(stream, stopWords);
     AssertTokenStreamContents(filter,
                               new string[] { "go", "to" },
                               new int[] { 0, 3 },
                               new int[] { 2, 5 },
                               null,
                               new int[] { 1, 1 },
                               null,
                               5,
                               new bool[] { false, true },
                               true);
 }
Exemplo n.º 2
0
        public void TestEndNotStopWord()
        {
            CharArraySet stopWords = StopFilter.MakeStopSet(TEST_VERSION_CURRENT, "to");
            TokenStream  stream    = new MockTokenizer(new StringReader("go to"));
            TokenStream  filter    = new SuggestStopFilter(stream, stopWords);

            AssertTokenStreamContents(filter,
                                      new string[] { "go", "to" },
                                      new int[] { 0, 3 },
                                      new int[] { 2, 5 },
                                      null,
                                      new int[] { 1, 1 },
                                      null,
                                      5,
                                      new bool[] { false, true },
                                      true);
        }
Exemplo n.º 3
0
        public void TestMultipleStopWords()
        {
            CharArraySet stopWords = StopFilter.MakeStopSet(TEST_VERSION_CURRENT, "to", "the", "a");
            TokenStream  stream    = new MockTokenizer(new StringReader("go to a the school"));
            TokenStream  filter    = new SuggestStopFilter(stream, stopWords);

            filter = new SuggestStopFilter(stream, stopWords);
            AssertTokenStreamContents(filter,
                                      new String[] { "go", "school" },
                                      new int[] { 0, 12 },
                                      new int[] { 2, 18 },
                                      null,
                                      new int[] { 1, 4 },
                                      null,
                                      18,
                                      new bool[] { false, false },
                                      true);
        }
Exemplo n.º 4
0
        public void TestMidStopWord()
        {

            CharArraySet stopWords = StopFilter.MakeStopSet(TEST_VERSION_CURRENT, "to");
            TokenStream stream = new MockTokenizer(new StringReader("go to school"));
            TokenStream filter = new SuggestStopFilter(stream, stopWords);

            filter = new SuggestStopFilter(stream, stopWords);
            AssertTokenStreamContents(filter,
                                      new String[] { "go", "school" },
                                      new int[] { 0, 6 },
                                      new int[] { 2, 12 },
                                      null,
                                      new int[] { 1, 2 },
                                      null,
                                      12,
                                      new bool[] { false, false },
                                      true);
        }
Exemplo n.º 5
0
        public void TestMultipleStopWordsEnd2()
        {

            CharArraySet stopWords = StopFilter.MakeStopSet(TEST_VERSION_CURRENT, "to", "the", "a");
            TokenStream stream = new MockTokenizer(new StringReader("go to a the "));
            TokenStream filter = new SuggestStopFilter(stream, stopWords);

            filter = new SuggestStopFilter(stream, stopWords);
            AssertTokenStreamContents(filter,
                                      new String[] { "go" },
                                      new int[] { 0 },
                                      new int[] { 2 },
                                      null,
                                      new int[] { 1 },
                                      null,
                                      12,
                                      new bool[] { false },
                                      true);
        }