Exemplo n.º 1
0
        public virtual void TestCrossPlaneNormalization2()
        {
            var analyzer = new AnalyzerAnonymousInnerClassHelper2();
            var num      = 1000 * RandomMultiplier;

            for (var i = 0; i < num; i++)
            {
                var s  = TestUtil.RandomUnicodeString(Random);
                var ts = analyzer.GetTokenStream("foo", s);
                try
                {
                    ts.Reset();
                    var offsetAtt = ts.AddAttribute <IOffsetAttribute>();
                    while (ts.IncrementToken())
                    {
                        string highlightedText = s.Substring(offsetAtt.StartOffset, offsetAtt.EndOffset - offsetAtt.StartOffset);
                        for (int j = 0, cp = 0; j < highlightedText.Length; j += Character.CharCount(cp))
                        {
                            cp = highlightedText.CodePointAt(j);
                            assertTrue("non-letter:" + cp.ToString("x"), Character.IsLetter(cp));
                        }
                    }
                    ts.End();
                }
                finally
                {
                    IOUtils.DisposeWhileHandlingException(ts);
                }
            }
            // just for fun
            CheckRandomData(Random, analyzer, num);
        }
        public virtual void TestLetterAsciiHuge()
        {
            Random       random    = Random;
            int          maxLength = 8192; // CharTokenizer.IO_BUFFER_SIZE*2
            MockAnalyzer left      = new MockAnalyzer(random, jvmLetter, false);

            left.MaxTokenLength = 255; // match CharTokenizer's max token length
            Analyzer right         = new AnalyzerAnonymousInnerClassHelper2(this);
            int      numIterations = AtLeast(50);

            for (int i = 0; i < numIterations; i++)
            {
                string s = TestUtil.RandomSimpleString(random, maxLength);
                assertEquals(s, left.GetTokenStream("foo", newStringReader(s)), right.GetTokenStream("foo", newStringReader(s)));
            }
        }