public virtual void TestCrossPlaneNormalization2() { var analyzer = new AnalyzerAnonymousInnerClassHelper2(); var num = 1000 * RANDOM_MULTIPLIER; for (var i = 0; i < num; i++) { var s = TestUtil.RandomUnicodeString(Random()); var ts = analyzer.TokenStream("foo", s); try { ts.Reset(); var offsetAtt = ts.AddAttribute<IOffsetAttribute>(); while (ts.IncrementToken()) { string highlightedText = s.Substring(offsetAtt.StartOffset(), offsetAtt.EndOffset() - offsetAtt.StartOffset()); for (int j = 0, cp = 0; j < highlightedText.Length; j += Character.CharCount(cp)) { cp = char.ConvertToUtf32(highlightedText, j); assertTrue("non-letter:" + cp.ToString("x"), Character.IsLetter(cp)); } } ts.End(); } finally { IOUtils.CloseWhileHandlingException(ts); } } // just for fun CheckRandomData(Random(), analyzer, num); }
public virtual void TestCrossPlaneNormalization2() { var analyzer = new AnalyzerAnonymousInnerClassHelper2(); var num = 1000 * RANDOM_MULTIPLIER; for (var i = 0; i < num; i++) { var s = TestUtil.RandomUnicodeString(Random()); var ts = analyzer.TokenStream("foo", s); try { ts.Reset(); var offsetAtt = ts.AddAttribute <IOffsetAttribute>(); while (ts.IncrementToken()) { string highlightedText = s.Substring(offsetAtt.StartOffset(), offsetAtt.EndOffset() - offsetAtt.StartOffset()); for (int j = 0, cp = 0; j < highlightedText.Length; j += Character.CharCount(cp)) { cp = char.ConvertToUtf32(highlightedText, j); assertTrue("non-letter:" + cp.ToString("x"), Character.IsLetter(cp)); } } ts.End(); } finally { IOUtils.CloseWhileHandlingException(ts); } } // just for fun CheckRandomData(Random(), analyzer, num); }
public virtual void TestLetterAsciiHuge() { Random random = Random(); int maxLength = 8192; // CharTokenizer.IO_BUFFER_SIZE*2 MockAnalyzer left = new MockAnalyzer(random, jvmLetter, false); left.MaxTokenLength = 255; // match CharTokenizer's max token length Analyzer right = new AnalyzerAnonymousInnerClassHelper2(this); int numIterations = AtLeast(50); for (int i = 0; i < numIterations; i++) { string s = TestUtil.RandomSimpleString(random, maxLength); assertEquals(s, left.TokenStream("foo", newStringReader(s)), right.TokenStream("foo", newStringReader(s))); } }