public virtual void TestWickedLongTerm()
{
using RAMDirectory dir = new RAMDirectory();
char[] chars = new char[IndexWriter.MAX_TERM_LENGTH];
Arrays.Fill(chars, 'x');
string bigTerm = new string(chars);
Document doc = new Document();
using (IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new ClassicAnalyzer(TEST_VERSION_CURRENT))))
{
// This produces a too-long term:
string contents = "abc xyz x" + bigTerm + " another term";
doc.Add(new TextField("content", contents, Field.Store.NO));
writer.AddDocument(doc);
// Make sure we can add another normal document
doc = new Document();
doc.Add(new TextField("content", "abc bbb ccc", Field.Store.NO));
writer.AddDocument(doc);
}
#pragma warning disable 612, 618
using (IndexReader reader = IndexReader.Open(dir))
#pragma warning restore 612, 618
{
// Make sure all terms < max size were indexed
assertEquals(2, reader.DocFreq(new Term("content", "abc")));
assertEquals(1, reader.DocFreq(new Term("content", "bbb")));
assertEquals(1, reader.DocFreq(new Term("content", "term")));
assertEquals(1, reader.DocFreq(new Term("content", "another")));
// Make sure position is still incremented when
// massive term is skipped:
DocsAndPositionsEnum tps = MultiFields.GetTermPositionsEnum(reader, MultiFields.GetLiveDocs(reader), "content", new BytesRef("another"));
assertTrue(tps.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
assertEquals(1, tps.Freq);
assertEquals(3, tps.NextPosition());
// Make sure the doc that has the massive term is in
// the index:
assertEquals("document with wicked long term should is not in the index!", 2, reader.NumDocs);
}
// Make sure we can add a document with exactly the
// maximum length term, and search on that term:
doc = new Document();
doc.Add(new TextField("content", bigTerm, Field.Store.NO));
ClassicAnalyzer sa = new ClassicAnalyzer(TEST_VERSION_CURRENT);
sa.MaxTokenLength = 100000;
using (var writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, sa)))
{
writer.AddDocument(doc);
}
#pragma warning disable 612, 618
using (var reader = IndexReader.Open(dir))
#pragma warning restore 612, 618
{
assertEquals(1, reader.DocFreq(new Term("content", bigTerm)));
}
}
public virtual void TestCaching()
{
Directory dir = new RAMDirectory();
RandomIndexWriter writer = new RandomIndexWriter(
#if FEATURE_INSTANCE_TESTDATA_INITIALIZATION
this,
#endif
Random, dir);
Document doc = new Document();
TokenStream stream = new TokenStreamAnonymousInnerClassHelper(this);
stream = new CachingTokenFilter(stream);
doc.Add(new TextField("preanalyzed", stream));
// 1) we consume all tokens twice before we add the doc to the index
CheckTokens(stream);
stream.Reset();
CheckTokens(stream);
// 2) now add the document to the index and verify if all tokens are indexed
// don't reset the stream here, the DocumentWriter should do that implicitly
writer.AddDocument(doc);
IndexReader reader = writer.GetReader();
DocsAndPositionsEnum termPositions = MultiFields.GetTermPositionsEnum(reader, MultiFields.GetLiveDocs(reader), "preanalyzed", new BytesRef("term1"));
Assert.IsTrue(termPositions.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(1, termPositions.Freq);
Assert.AreEqual(0, termPositions.NextPosition());
termPositions = MultiFields.GetTermPositionsEnum(reader, MultiFields.GetLiveDocs(reader), "preanalyzed", new BytesRef("term2"));
Assert.IsTrue(termPositions.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(2, termPositions.Freq);
Assert.AreEqual(1, termPositions.NextPosition());
Assert.AreEqual(3, termPositions.NextPosition());
termPositions = MultiFields.GetTermPositionsEnum(reader, MultiFields.GetLiveDocs(reader), "preanalyzed", new BytesRef("term3"));
Assert.IsTrue(termPositions.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
Assert.AreEqual(1, termPositions.Freq);
Assert.AreEqual(2, termPositions.NextPosition());
reader.Dispose();
writer.Dispose();
// 3) reset stream and consume tokens again
stream.Reset();
CheckTokens(stream);
dir.Dispose();
}
/// <summary>
/// Read the parents of the new categories
/// </summary>
private void InitParents(IndexReader reader, int first)
{
if (reader.MaxDoc == first)
{
return;
}
// it's ok to use MultiFields because we only iterate on one posting list.
// breaking it to loop over the leaves() only complicates code for no
// apparent gain.
DocsAndPositionsEnum positions = MultiFields.GetTermPositionsEnum(reader, null, Consts.FIELD_PAYLOADS, Consts.PAYLOAD_PARENT_BYTES_REF, DocsAndPositionsFlags.PAYLOADS);
// shouldn't really happen, if it does, something's wrong
if (positions == null || positions.Advance(first) == DocIdSetIterator.NO_MORE_DOCS)
{
throw new CorruptIndexException("Missing parent data for category " + first);
}
int num = reader.MaxDoc;
for (int i = first; i < num; i++)
{
if (positions.DocID == i)
{
if (positions.Freq == 0) // shouldn't happen
{
throw new CorruptIndexException("Missing parent data for category " + i);
}
parents[i] = positions.NextPosition();
if (positions.NextDoc() == DocIdSetIterator.NO_MORE_DOCS)
{
if (i + 1 < num)
{
throw new CorruptIndexException("Missing parent data for category " + (i + 1));
}
break;
}
} // this shouldn't happen
else
{
throw new CorruptIndexException("Missing parent data for category " + i);
}
}
}
public virtual void TestSetPosition()
{
Analyzer analyzer = new AnalyzerAnonymousClass(this);
Directory store = NewDirectory();
RandomIndexWriter writer = new RandomIndexWriter(Random, store, analyzer);
Document d = new Document();
d.Add(NewTextField("field", "bogus", Field.Store.YES));
writer.AddDocument(d);
IndexReader reader = writer.GetReader();
writer.Dispose();
IndexSearcher searcher = NewSearcher(reader);
DocsAndPositionsEnum pos = MultiFields.GetTermPositionsEnum(searcher.IndexReader, MultiFields.GetLiveDocs(searcher.IndexReader), "field", new BytesRef("1"));
pos.NextDoc();
// first token should be at position 0
Assert.AreEqual(0, pos.NextPosition());
pos = MultiFields.GetTermPositionsEnum(searcher.IndexReader, MultiFields.GetLiveDocs(searcher.IndexReader), "field", new BytesRef("2"));
pos.NextDoc();
// second token should be at position 2
Assert.AreEqual(2, pos.NextPosition());
PhraseQuery q;
ScoreDoc[] hits;
q = new PhraseQuery();
q.Add(new Term("field", "1"));
q.Add(new Term("field", "2"));
hits = searcher.Search(q, null, 1000).ScoreDocs;
Assert.AreEqual(0, hits.Length);
// same as previous, just specify positions explicitely.
q = new PhraseQuery();
q.Add(new Term("field", "1"), 0);
q.Add(new Term("field", "2"), 1);
hits = searcher.Search(q, null, 1000).ScoreDocs;
Assert.AreEqual(0, hits.Length);
// specifying correct positions should find the phrase.
q = new PhraseQuery();
q.Add(new Term("field", "1"), 0);
q.Add(new Term("field", "2"), 2);
hits = searcher.Search(q, null, 1000).ScoreDocs;
Assert.AreEqual(1, hits.Length);
q = new PhraseQuery();
q.Add(new Term("field", "2"));
q.Add(new Term("field", "3"));
hits = searcher.Search(q, null, 1000).ScoreDocs;
Assert.AreEqual(1, hits.Length);
q = new PhraseQuery();
q.Add(new Term("field", "3"));
q.Add(new Term("field", "4"));
hits = searcher.Search(q, null, 1000).ScoreDocs;
Assert.AreEqual(0, hits.Length);
// phrase query would find it when correct positions are specified.
q = new PhraseQuery();
q.Add(new Term("field", "3"), 0);
q.Add(new Term("field", "4"), 0);
hits = searcher.Search(q, null, 1000).ScoreDocs;
Assert.AreEqual(1, hits.Length);
// phrase query should fail for non existing searched term
// even if there exist another searched terms in the same searched position.
q = new PhraseQuery();
q.Add(new Term("field", "3"), 0);
q.Add(new Term("field", "9"), 0);
hits = searcher.Search(q, null, 1000).ScoreDocs;
Assert.AreEqual(0, hits.Length);
// multi-phrase query should succed for non existing searched term
// because there exist another searched terms in the same searched position.
MultiPhraseQuery mq = new MultiPhraseQuery();
mq.Add(new Term[] { new Term("field", "3"), new Term("field", "9") }, 0);
hits = searcher.Search(mq, null, 1000).ScoreDocs;
Assert.AreEqual(1, hits.Length);
q = new PhraseQuery();
q.Add(new Term("field", "2"));
q.Add(new Term("field", "4"));
hits = searcher.Search(q, null, 1000).ScoreDocs;
Assert.AreEqual(1, hits.Length);
q = new PhraseQuery();
q.Add(new Term("field", "3"));
q.Add(new Term("field", "5"));
hits = searcher.Search(q, null, 1000).ScoreDocs;
Assert.AreEqual(1, hits.Length);
q = new PhraseQuery();
q.Add(new Term("field", "4"));
q.Add(new Term("field", "5"));
hits = searcher.Search(q, null, 1000).ScoreDocs;
Assert.AreEqual(1, hits.Length);
q = new PhraseQuery();
q.Add(new Term("field", "2"));
q.Add(new Term("field", "5"));
hits = searcher.Search(q, null, 1000).ScoreDocs;
Assert.AreEqual(0, hits.Length);
reader.Dispose();
store.Dispose();
}