public virtual void TestTokenReuse() { Analyzer analyzer = new AnalyzerAnonymousClass2(this); IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer)); Document doc = new Document(); doc.Add(NewTextField("f1", "a 5 a a", Field.Store.YES)); writer.AddDocument(doc); writer.Commit(); SegmentCommitInfo info = writer.NewestSegment(); writer.Dispose(); SegmentReader reader = new SegmentReader(info, DirectoryReader.DEFAULT_TERMS_INDEX_DIVISOR, NewIOContext(Random)); DocsAndPositionsEnum termPositions = MultiFields.GetTermPositionsEnum(reader, reader.LiveDocs, "f1", new BytesRef("a")); Assert.IsTrue(termPositions.NextDoc() != DocIdSetIterator.NO_MORE_DOCS); int freq = termPositions.Freq; Assert.AreEqual(3, freq); Assert.AreEqual(0, termPositions.NextPosition()); Assert.IsNotNull(termPositions.GetPayload()); Assert.AreEqual(6, termPositions.NextPosition()); Assert.IsNull(termPositions.GetPayload()); Assert.AreEqual(7, termPositions.NextPosition()); Assert.IsNull(termPositions.GetPayload()); reader.Dispose(); }
public virtual void TestCrossPlaneNormalization2() { var analyzer = new AnalyzerAnonymousClass2(); var num = 1000 * RandomMultiplier; for (var i = 0; i < num; i++) { var s = TestUtil.RandomUnicodeString(Random); var ts = analyzer.GetTokenStream("foo", s); try { ts.Reset(); var offsetAtt = ts.AddAttribute <IOffsetAttribute>(); while (ts.IncrementToken()) { string highlightedText = s.Substring(offsetAtt.StartOffset, offsetAtt.EndOffset - offsetAtt.StartOffset); for (int j = 0, cp = 0; j < highlightedText.Length; j += Character.CharCount(cp)) { cp = highlightedText.CodePointAt(j); assertTrue("non-letter:" + cp.ToString("x"), Character.IsLetter(cp)); } } ts.End(); } finally { IOUtils.DisposeWhileHandlingException(ts); } } // just for fun CheckRandomData(Random, analyzer, num); }
public TokenFilterAnonymousClass(AnalyzerAnonymousClass2 outerInstance, Tokenizer tokenizer) : base(tokenizer) { this.outerInstance = outerInstance; first = true; termAtt = AddAttribute <ICharTermAttribute>(); payloadAtt = AddAttribute <IPayloadAttribute>(); posIncrAtt = AddAttribute <IPositionIncrementAttribute>(); }