public void RestDocsAndPositionsEnumStart()
{
Analyzer analyzer = new MockAnalyzer(Random);
int numIters = AtLeast(3);
MemoryIndex memory = new MemoryIndex(true, Random.nextInt(50) * 1024 * 1024);
for (int i = 0; i < numIters; i++)
{ // check reuse
memory.AddField("foo", "bar", analyzer);
AtomicReader reader = (AtomicReader)memory.CreateSearcher().IndexReader;
assertEquals(1, reader.GetTerms("foo").SumTotalTermFreq);
DocsAndPositionsEnum disi = reader.GetTermPositionsEnum(new Term("foo", "bar"));
int docid = disi.DocID;
assertEquals(-1, docid);
assertTrue(disi.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
assertEquals(0, disi.NextPosition());
assertEquals(0, disi.StartOffset);
assertEquals(3, disi.EndOffset);
// now reuse and check again
TermsEnum te = reader.GetTerms("foo").GetEnumerator();
assertTrue(te.SeekExact(new BytesRef("bar")));
disi = te.DocsAndPositions(null, disi);
docid = disi.DocID;
assertEquals(-1, docid);
assertTrue(disi.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
reader.Dispose();
memory.Reset();
}
}
/**
* Build a randomish document for both RAMDirectory and MemoryIndex,
* and run all the queries against it.
*/
public void AssertAgainstRAMDirectory(MemoryIndex memory)
{
memory.Reset();
StringBuilder fooField = new StringBuilder();
StringBuilder termField = new StringBuilder();
// add up to 250 terms to field "foo"
int numFooTerms = Random.nextInt(250 * RandomMultiplier);
for (int i = 0; i < numFooTerms; i++)
{
fooField.append(" ");
fooField.append(RandomTerm());
}
// add up to 250 terms to field "term"
int numTermTerms = Random.nextInt(250 * RandomMultiplier);
for (int i = 0; i < numTermTerms; i++)
{
termField.append(" ");
termField.append(RandomTerm());
}
Store.Directory ramdir = new RAMDirectory();
Analyzer analyzer = RandomAnalyzer();
IndexWriter writer = new IndexWriter(ramdir,
new IndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetCodec(TestUtil.AlwaysPostingsFormat(new Lucene41PostingsFormat())));
Document doc = new Document();
Field field1 = NewTextField("foo", fooField.toString(), Field.Store.NO);
Field field2 = NewTextField("term", termField.toString(), Field.Store.NO);
doc.Add(field1);
doc.Add(field2);
writer.AddDocument(doc);
writer.Dispose();
memory.AddField("foo", fooField.toString(), analyzer);
memory.AddField("term", termField.toString(), analyzer);
if (Verbose)
{
Console.WriteLine("Random MemoryIndex:\n" + memory.toString());
Console.WriteLine("Same index as RAMDirectory: " +
RamUsageEstimator.HumanReadableUnits(RamUsageEstimator.SizeOf(ramdir)));
Console.WriteLine();
}
else
{
assertTrue(memory.GetMemorySize() > 0L);
}
AtomicReader reader = (AtomicReader)memory.CreateSearcher().IndexReader;
DirectoryReader competitor = DirectoryReader.Open(ramdir);
DuellReaders(competitor, reader);
IOUtils.Dispose(reader, competitor);
AssertAllQueries(memory, ramdir, analyzer);
ramdir.Dispose();
}
public void TestDuellMemIndex()
{
LineFileDocs lineFileDocs = new LineFileDocs(Random);
int numDocs = AtLeast(10);
MemoryIndex memory = new MemoryIndex(Random.nextBoolean(), Random.nextInt(50) * 1024 * 1024);
for (int i = 0; i < numDocs; i++)
{
Store.Directory dir = NewDirectory();
MockAnalyzer mockAnalyzer = new MockAnalyzer(Random);
mockAnalyzer.MaxTokenLength = (TestUtil.NextInt32(Random, 1, IndexWriter.MAX_TERM_LENGTH));
IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(Random, TEST_VERSION_CURRENT, mockAnalyzer));
Document nextDoc = lineFileDocs.NextDoc();
Document doc = new Document();
foreach (IIndexableField field in nextDoc.Fields)
{
if (field.IndexableFieldType.IsIndexed)
{
doc.Add(field);
if (Random.nextInt(3) == 0)
{
doc.Add(field); // randomly add the same field twice
}
}
}
writer.AddDocument(doc);
writer.Dispose();
foreach (IIndexableField field in doc.Fields)
{
memory.AddField(field.Name, ((Field)field).GetStringValue(), mockAnalyzer);
}
DirectoryReader competitor = DirectoryReader.Open(dir);
AtomicReader memIndexReader = (AtomicReader)memory.CreateSearcher().IndexReader;
DuellReaders(competitor, memIndexReader);
IOUtils.Dispose(competitor, memIndexReader);
memory.Reset();
dir.Dispose();
}
lineFileDocs.Dispose();
}
public void TestSameFieldAddedMultipleTimes()
{
MemoryIndex mindex = new MemoryIndex(Random.nextBoolean(), Random.nextInt(50) * 1024 * 1024);
MockAnalyzer mockAnalyzer = new MockAnalyzer(Random);
mindex.AddField("field", "the quick brown fox", mockAnalyzer);
mindex.AddField("field", "jumps over the", mockAnalyzer);
AtomicReader reader = (AtomicReader)mindex.CreateSearcher().IndexReader;
assertEquals(7, reader.GetTerms("field").SumTotalTermFreq);
PhraseQuery query = new PhraseQuery();
query.Add(new Term("field", "fox"));
query.Add(new Term("field", "jumps"));
assertTrue(mindex.Search(query) > 0.1);
mindex.Reset();
mockAnalyzer.SetPositionIncrementGap(1 + Random.nextInt(10));
mindex.AddField("field", "the quick brown fox", mockAnalyzer);
mindex.AddField("field", "jumps over the", mockAnalyzer);
assertEquals(0, mindex.Search(query), 0.00001f);
query.Slop = (10);
assertTrue("posGap" + mockAnalyzer.GetPositionIncrementGap("field"), mindex.Search(query) > 0.0001);
}
