public void RestDocsAndPositionsEnumStart() { Analyzer analyzer = new MockAnalyzer(Random); int numIters = AtLeast(3); MemoryIndex memory = new MemoryIndex(true, Random.nextInt(50) * 1024 * 1024); for (int i = 0; i < numIters; i++) { // check reuse memory.AddField("foo", "bar", analyzer); AtomicReader reader = (AtomicReader)memory.CreateSearcher().IndexReader; assertEquals(1, reader.GetTerms("foo").SumTotalTermFreq); DocsAndPositionsEnum disi = reader.GetTermPositionsEnum(new Term("foo", "bar")); int docid = disi.DocID; assertEquals(-1, docid); assertTrue(disi.NextDoc() != DocIdSetIterator.NO_MORE_DOCS); assertEquals(0, disi.NextPosition()); assertEquals(0, disi.StartOffset); assertEquals(3, disi.EndOffset); // now reuse and check again TermsEnum te = reader.GetTerms("foo").GetEnumerator(); assertTrue(te.SeekExact(new BytesRef("bar"))); disi = te.DocsAndPositions(null, disi); docid = disi.DocID; assertEquals(-1, docid); assertTrue(disi.NextDoc() != DocIdSetIterator.NO_MORE_DOCS); reader.Dispose(); memory.Reset(); } }
/** * Build a randomish document for both RAMDirectory and MemoryIndex, * and run all the queries against it. */ public void AssertAgainstRAMDirectory(MemoryIndex memory) { memory.Reset(); StringBuilder fooField = new StringBuilder(); StringBuilder termField = new StringBuilder(); // add up to 250 terms to field "foo" int numFooTerms = Random.nextInt(250 * RandomMultiplier); for (int i = 0; i < numFooTerms; i++) { fooField.append(" "); fooField.append(RandomTerm()); } // add up to 250 terms to field "term" int numTermTerms = Random.nextInt(250 * RandomMultiplier); for (int i = 0; i < numTermTerms; i++) { termField.append(" "); termField.append(RandomTerm()); } Store.Directory ramdir = new RAMDirectory(); Analyzer analyzer = RandomAnalyzer(); IndexWriter writer = new IndexWriter(ramdir, new IndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetCodec(TestUtil.AlwaysPostingsFormat(new Lucene41PostingsFormat()))); Document doc = new Document(); Field field1 = NewTextField("foo", fooField.toString(), Field.Store.NO); Field field2 = NewTextField("term", termField.toString(), Field.Store.NO); doc.Add(field1); doc.Add(field2); writer.AddDocument(doc); writer.Dispose(); memory.AddField("foo", fooField.toString(), analyzer); memory.AddField("term", termField.toString(), analyzer); if (Verbose) { Console.WriteLine("Random MemoryIndex:\n" + memory.toString()); Console.WriteLine("Same index as RAMDirectory: " + RamUsageEstimator.HumanReadableUnits(RamUsageEstimator.SizeOf(ramdir))); Console.WriteLine(); } else { assertTrue(memory.GetMemorySize() > 0L); } AtomicReader reader = (AtomicReader)memory.CreateSearcher().IndexReader; DirectoryReader competitor = DirectoryReader.Open(ramdir); DuellReaders(competitor, reader); IOUtils.Dispose(reader, competitor); AssertAllQueries(memory, ramdir, analyzer); ramdir.Dispose(); }
public void TestDuellMemIndex() { LineFileDocs lineFileDocs = new LineFileDocs(Random); int numDocs = AtLeast(10); MemoryIndex memory = new MemoryIndex(Random.nextBoolean(), Random.nextInt(50) * 1024 * 1024); for (int i = 0; i < numDocs; i++) { Store.Directory dir = NewDirectory(); MockAnalyzer mockAnalyzer = new MockAnalyzer(Random); mockAnalyzer.MaxTokenLength = (TestUtil.NextInt32(Random, 1, IndexWriter.MAX_TERM_LENGTH)); IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(Random, TEST_VERSION_CURRENT, mockAnalyzer)); Document nextDoc = lineFileDocs.NextDoc(); Document doc = new Document(); foreach (IIndexableField field in nextDoc.Fields) { if (field.IndexableFieldType.IsIndexed) { doc.Add(field); if (Random.nextInt(3) == 0) { doc.Add(field); // randomly add the same field twice } } } writer.AddDocument(doc); writer.Dispose(); foreach (IIndexableField field in doc.Fields) { memory.AddField(field.Name, ((Field)field).GetStringValue(), mockAnalyzer); } DirectoryReader competitor = DirectoryReader.Open(dir); AtomicReader memIndexReader = (AtomicReader)memory.CreateSearcher().IndexReader; DuellReaders(competitor, memIndexReader); IOUtils.Dispose(competitor, memIndexReader); memory.Reset(); dir.Dispose(); } lineFileDocs.Dispose(); }
public void TestSameFieldAddedMultipleTimes() { MemoryIndex mindex = new MemoryIndex(Random.nextBoolean(), Random.nextInt(50) * 1024 * 1024); MockAnalyzer mockAnalyzer = new MockAnalyzer(Random); mindex.AddField("field", "the quick brown fox", mockAnalyzer); mindex.AddField("field", "jumps over the", mockAnalyzer); AtomicReader reader = (AtomicReader)mindex.CreateSearcher().IndexReader; assertEquals(7, reader.GetTerms("field").SumTotalTermFreq); PhraseQuery query = new PhraseQuery(); query.Add(new Term("field", "fox")); query.Add(new Term("field", "jumps")); assertTrue(mindex.Search(query) > 0.1); mindex.Reset(); mockAnalyzer.SetPositionIncrementGap(1 + Random.nextInt(10)); mindex.AddField("field", "the quick brown fox", mockAnalyzer); mindex.AddField("field", "jumps over the", mockAnalyzer); assertEquals(0, mindex.Search(query), 0.00001f); query.Slop = (10); assertTrue("posGap" + mockAnalyzer.GetPositionIncrementGap("field"), mindex.Search(query) > 0.0001); }