public virtual void TestTokenReuse()
{
Analyzer analyzer = new AnalyzerAnonymousClass2(this);
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer));
Document doc = new Document();
doc.Add(NewTextField("f1", "a 5 a a", Field.Store.YES));
writer.AddDocument(doc);
writer.Commit();
SegmentCommitInfo info = writer.NewestSegment();
writer.Dispose();
SegmentReader reader = new SegmentReader(info, DirectoryReader.DEFAULT_TERMS_INDEX_DIVISOR, NewIOContext(Random));
DocsAndPositionsEnum termPositions = MultiFields.GetTermPositionsEnum(reader, reader.LiveDocs, "f1", new BytesRef("a"));
Assert.IsTrue(termPositions.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
int freq = termPositions.Freq;
Assert.AreEqual(3, freq);
Assert.AreEqual(0, termPositions.NextPosition());
Assert.IsNotNull(termPositions.GetPayload());
Assert.AreEqual(6, termPositions.NextPosition());
Assert.IsNull(termPositions.GetPayload());
Assert.AreEqual(7, termPositions.NextPosition());
Assert.IsNull(termPositions.GetPayload());
reader.Dispose();
}
public virtual void TestCrossPlaneNormalization2()
{
var analyzer = new AnalyzerAnonymousClass2();
var num = 1000 * RandomMultiplier;
for (var i = 0; i < num; i++)
{
var s = TestUtil.RandomUnicodeString(Random);
var ts = analyzer.GetTokenStream("foo", s);
try
{
ts.Reset();
var offsetAtt = ts.AddAttribute <IOffsetAttribute>();
while (ts.IncrementToken())
{
string highlightedText = s.Substring(offsetAtt.StartOffset, offsetAtt.EndOffset - offsetAtt.StartOffset);
for (int j = 0, cp = 0; j < highlightedText.Length; j += Character.CharCount(cp))
{
cp = highlightedText.CodePointAt(j);
assertTrue("non-letter:" + cp.ToString("x"), Character.IsLetter(cp));
}
}
ts.End();
}
finally
{
IOUtils.DisposeWhileHandlingException(ts);
}
}
// just for fun
CheckRandomData(Random, analyzer, num);
}
public TokenFilterAnonymousClass(AnalyzerAnonymousClass2 outerInstance, Tokenizer tokenizer)
: base(tokenizer)
{
this.outerInstance = outerInstance;
first = true;
termAtt = AddAttribute <ICharTermAttribute>();
payloadAtt = AddAttribute <IPayloadAttribute>();
posIncrAtt = AddAttribute <IPositionIncrementAttribute>();
}
