IncrementToken() public method

Inserts bigrams for common words into a token stream. For each input token, output the token. If the token and/or the following token are in the list of common words also output a bigram with position increment 0 and type="gram" TODO:Consider adding an option to not emit unigram stopwords as in CDL XTF BigramStopFilter, CommonGramsQueryFilter would need to be changed to work with this. TODO: Consider optimizing for the case of three commongrams i.e "man of the year" normally produces 3 bigrams: "man-of", "of-the", "the-year" but with proper management of positions we could eliminate the middle bigram "of-the"and save a disk seek and a whole set of position lookups.
public IncrementToken ( ) : bool
return bool
CommonGramsFilter Class Documentation
Example #1
0
        public virtual void TestReset()
        {
            const string input = "How the s a brown s cow d like A B thing?";
            WhitespaceTokenizer wt = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(input));
            CommonGramsFilter cgf = new CommonGramsFilter(TEST_VERSION_CURRENT, wt, commonWords);

            ICharTermAttribute term = cgf.AddAttribute<ICharTermAttribute>();
            cgf.Reset();
            assertTrue(cgf.IncrementToken());
            assertEquals("How", term.ToString());
            assertTrue(cgf.IncrementToken());
            assertEquals("How_the", term.ToString());
            assertTrue(cgf.IncrementToken());
            assertEquals("the", term.ToString());
            assertTrue(cgf.IncrementToken());
            assertEquals("the_s", term.ToString());
            cgf.Dispose();

            wt.Reader = new StringReader(input);
            cgf.Reset();
            assertTrue(cgf.IncrementToken());
            assertEquals("How", term.ToString());
        }
Example #2
0
        public virtual void TestReset()
        {
            const string        input = "How the s a brown s cow d like A B thing?";
            WhitespaceTokenizer wt    = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(input));
            CommonGramsFilter   cgf   = new CommonGramsFilter(TEST_VERSION_CURRENT, wt, commonWords);

            ICharTermAttribute term = cgf.AddAttribute <ICharTermAttribute>();

            cgf.Reset();
            assertTrue(cgf.IncrementToken());
            assertEquals("How", term.ToString());
            assertTrue(cgf.IncrementToken());
            assertEquals("How_the", term.ToString());
            assertTrue(cgf.IncrementToken());
            assertEquals("the", term.ToString());
            assertTrue(cgf.IncrementToken());
            assertEquals("the_s", term.ToString());
            cgf.Dispose();

            wt.SetReader(new StringReader(input));
            cgf.Reset();
            assertTrue(cgf.IncrementToken());
            assertEquals("How", term.ToString());
        }