public virtual void TestCrossPlaneNormalization2()
{
var analyzer = new AnalyzerAnonymousInnerClassHelper2();
var num = 1000 * RANDOM_MULTIPLIER;
for (var i = 0; i < num; i++)
{
var s = TestUtil.RandomUnicodeString(Random());
var ts = analyzer.TokenStream("foo", s);
try
{
ts.Reset();
var offsetAtt = ts.AddAttribute<IOffsetAttribute>();
while (ts.IncrementToken())
{
string highlightedText = s.Substring(offsetAtt.StartOffset(), offsetAtt.EndOffset() - offsetAtt.StartOffset());
for (int j = 0, cp = 0; j < highlightedText.Length; j += Character.CharCount(cp))
{
cp = char.ConvertToUtf32(highlightedText, j);
assertTrue("non-letter:" + cp.ToString("x"), Character.IsLetter(cp));
}
}
ts.End();
}
finally
{
IOUtils.CloseWhileHandlingException(ts);
}
}
// just for fun
CheckRandomData(Random(), analyzer, num);
}
public virtual void TestCrossPlaneNormalization2()
{
var analyzer = new AnalyzerAnonymousInnerClassHelper2();
var num = 1000 * RANDOM_MULTIPLIER;
for (var i = 0; i < num; i++)
{
var s = TestUtil.RandomUnicodeString(Random());
var ts = analyzer.TokenStream("foo", s);
try
{
ts.Reset();
var offsetAtt = ts.AddAttribute <IOffsetAttribute>();
while (ts.IncrementToken())
{
string highlightedText = s.Substring(offsetAtt.StartOffset(), offsetAtt.EndOffset() - offsetAtt.StartOffset());
for (int j = 0, cp = 0; j < highlightedText.Length; j += Character.CharCount(cp))
{
cp = char.ConvertToUtf32(highlightedText, j);
assertTrue("non-letter:" + cp.ToString("x"), Character.IsLetter(cp));
}
}
ts.End();
}
finally
{
IOUtils.CloseWhileHandlingException(ts);
}
}
// just for fun
CheckRandomData(Random(), analyzer, num);
}
public virtual void TestLetterAsciiHuge()
{
Random random = Random();
int maxLength = 8192; // CharTokenizer.IO_BUFFER_SIZE*2
MockAnalyzer left = new MockAnalyzer(random, jvmLetter, false);
left.MaxTokenLength = 255; // match CharTokenizer's max token length
Analyzer right = new AnalyzerAnonymousInnerClassHelper2(this);
int numIterations = AtLeast(50);
for (int i = 0; i < numIterations; i++)
{
string s = TestUtil.RandomSimpleString(random, maxLength);
assertEquals(s, left.TokenStream("foo", newStringReader(s)), right.TokenStream("foo", newStringReader(s)));
}
}
public virtual void TestLetterAsciiHuge()
{
Random random = Random();
int maxLength = 8192; // CharTokenizer.IO_BUFFER_SIZE*2
MockAnalyzer left = new MockAnalyzer(random, jvmLetter, false);
left.MaxTokenLength = 255; // match CharTokenizer's max token length
Analyzer right = new AnalyzerAnonymousInnerClassHelper2(this);
int numIterations = AtLeast(50);
for (int i = 0; i < numIterations; i++)
{
string s = TestUtil.RandomSimpleString(random, maxLength);
assertEquals(s, left.TokenStream("foo", newStringReader(s)), right.TokenStream("foo", newStringReader(s)));
}
}