C# (CSharp) Lucene.Net.Analysis.NGram EdgeNGramTokenizer Examples

Example #1

File: EdgeNGramTokenizerTest.cs Project: ywscr/lucenenet

        public virtual void TestRandomStrings()
        {
            for (int i = 0; i < 10; i++)
            {
                int min = TestUtil.NextInt32(Random, 2, 10);
                int max = TestUtil.NextInt32(Random, min, 20);

                Analyzer a = Analyzer.NewAnonymous(createComponents: (fieldName, reader) =>
                {
                    Tokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, reader, min, max);
                    return(new TokenStreamComponents(tokenizer, tokenizer));
                });
                CheckRandomData(Random, a, 100 * RandomMultiplier, 20);
                CheckRandomData(Random, a, 10 * RandomMultiplier, 8192);
            }

            Analyzer b = Analyzer.NewAnonymous(createComponents: (fieldName, reader) =>
            {
#pragma warning disable 612, 618
                Tokenizer tokenizer = new Lucene43EdgeNGramTokenizer(Version.LUCENE_43, reader, Lucene43EdgeNGramTokenizer.Side.BACK, 2, 4);
#pragma warning restore 612, 618
                return(new TokenStreamComponents(tokenizer, tokenizer));
            });

            CheckRandomData(Random, b, 1000 * RandomMultiplier, 20, false, false);
            CheckRandomData(Random, b, 100 * RandomMultiplier, 8192, false, false);
        }

Example #2

        public virtual void TestReset()
        {
            EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, input, 1, 3);

            AssertTokenStreamContents(tokenizer, new string[] { "a", "ab", "abc" }, new int[] { 0, 0, 0 }, new int[] { 1, 2, 3 }, 5); // abcde
            tokenizer.SetReader(new StringReader("abcde"));
            AssertTokenStreamContents(tokenizer, new string[] { "a", "ab", "abc" }, new int[] { 0, 0, 0 }, new int[] { 1, 2, 3 }, 5); // abcde
        }

Example #3

        public virtual void TestTokenizerPositions()
        {
#pragma warning disable 612, 618
            Tokenizer tokenizer = new Lucene43EdgeNGramTokenizer(Version.LUCENE_43, input, Lucene43EdgeNGramTokenizer.Side.FRONT, 1, 3);
#pragma warning restore 612, 618
            AssertTokenStreamContents(tokenizer, new string[] { "a", "ab", "abc" }, new int[] { 0, 0, 0 }, new int[] { 1, 2, 3 }, null, new int[] { 1, 0, 0 }, null, null, false);

            tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, new StringReader("abcde"), 1, 3);
            AssertTokenStreamContents(tokenizer, new string[] { "a", "ab", "abc" }, new int[] { 0, 0, 0 }, new int[] { 1, 2, 3 }, null, new int[] { 1, 1, 1 }, null, null, false);
        }

Example #4

File: TestEdgeNGramTokenizer.cs Project: hanabi1224/lucene.net

 public void TestBackUnigram()
 {
     EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(input, Side.BACK, 1, 1);
     AssertTokenStreamContents(tokenizer, new String[] { "e" }, new int[] { 4 }, new int[] { 5 }, 5 /* abcde */);
 }

Example #5

File: TestEdgeNGramTokenizer.cs Project: hanabi1224/lucene.net

 public void TestFrontUnigram()
 {
     EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(input, Side.FRONT, 1, 1);
     AssertTokenStreamContents(tokenizer, new String[] { "a" }, new int[] { 0 }, new int[] { 1 }, 5 /* abcde */);
 }

Example #6

File: TestEdgeNGramTokenizer.cs Project: hanabi1224/lucene.net

 public void TestReset()
 {
     EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(input, Side.FRONT, 1, 3);
     AssertTokenStreamContents(tokenizer, new String[] { "a", "ab", "abc" }, new int[] { 0, 0, 0 }, new int[] { 1, 2, 3 }, 5 /* abcde */);
     tokenizer.Reset(new StringReader("abcde"));
     AssertTokenStreamContents(tokenizer, new String[] { "a", "ab", "abc" }, new int[] { 0, 0, 0 }, new int[] { 1, 2, 3 }, 5 /* abcde */);
 }

Example #7

File: TestEdgeNGramTokenizer.cs Project: hanabi1224/lucene.net

 public void TestBackRangeOfNgrams()
 {
     EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(input, Side.BACK, 1, 3);
     AssertTokenStreamContents(tokenizer, new String[] { "e", "de", "cde" }, new int[] { 4, 3, 2 }, new int[] { 5, 5, 5 }, 5 /* abcde */);
 }

Example #8

File: TestEdgeNGramTokenizer.cs Project: hanabi1224/lucene.net

 public void TestFrontRangeOfNgrams()
 {
     EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(input, Side.FRONT, 1, 3);
     AssertTokenStreamContents(tokenizer, new String[] { "a", "ab", "abc" }, new int[] { 0, 0, 0 }, new int[] { 1, 2, 3 }, 5 /* abcde */);
 }

Example #9

File: TestEdgeNGramTokenizer.cs Project: hanabi1224/lucene.net

 public void TestOversizedNgrams()
 {
     EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(input, Side.FRONT, 6, 6);
     AssertTokenStreamContents(tokenizer, new String[0], new int[0], new int[0], 5 /* abcde */);
 }

Example #10

        public virtual void TestFrontUnigram()
        {
            EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, input, 1, 1);

            AssertTokenStreamContents(tokenizer, new string[] { "a" }, new int[] { 0 }, new int[] { 1 }, 5); // abcde
        }

Example #11

            protected internal override TokenStreamComponents CreateComponents(string fieldName, Reader reader)
            {
                Tokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, reader, min, max);

                return(new TokenStreamComponents(tokenizer, tokenizer));
            }

Example #12

        public virtual void TestFrontRangeOfNgrams()
        {
            EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, input, 1, 3);

            AssertTokenStreamContents(tokenizer, new string[] { "a", "ab", "abc" }, new int[] { 0, 0, 0 }, new int[] { 1, 2, 3 }, 5); // abcde
        }

Example #13

        public virtual void TestOversizedNgrams()
        {
            EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, input, 6, 6);

            AssertTokenStreamContents(tokenizer, new string[0], new int[0], new int[0], 5); // abcde
        }