public override void SetUp() { base.SetUp(); dir = NewDirectory(); iw = new RandomIndexWriter( #if FEATURE_INSTANCE_TESTDATA_INITIALIZATION this, #endif Random, dir); Document doc = new Document(); Field idField = new StringField("id", "", Field.Store.NO); doc.Add(idField); // add 500 docs with id 0..499 for (int i = 0; i < 500; i++) { idField.SetStringValue(Convert.ToString(i)); iw.AddDocument(doc); } // delete 20 of them for (int i = 0; i < 20; i++) { iw.DeleteDocuments(new Term("id", Convert.ToString(Random.Next(iw.MaxDoc)))); } ir = iw.GetReader(); @is = NewSearcher(ir); }
public override void SetUp() { base.SetUp(); dir = NewDirectory(); IndexWriterConfig iwConfig = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)); iwConfig.SetMergePolicy(NewLogMergePolicy()); RandomIndexWriter iw = new RandomIndexWriter(Random, dir, iwConfig); Document document = new Document(); Field idField = new StringField("id", "", Field.Store.NO); document.Add(idField); Field byteField = new StringField("byte", "", Field.Store.NO); document.Add(byteField); Field doubleField = new StringField("double", "", Field.Store.NO); document.Add(doubleField); Field floatField = new StringField("float", "", Field.Store.NO); document.Add(floatField); Field intField = new StringField("int", "", Field.Store.NO); document.Add(intField); Field longField = new StringField("long", "", Field.Store.NO); document.Add(longField); Field shortField = new StringField("short", "", Field.Store.NO); document.Add(shortField); Field stringField = new StringField("string", "", Field.Store.NO); document.Add(stringField); Field textField = new TextField("text", "", Field.Store.NO); document.Add(textField); foreach (string[] doc in documents) { idField.SetStringValue(doc[0]); byteField.SetStringValue(doc[1]); doubleField.SetStringValue(doc[2]); floatField.SetStringValue(doc[3]); intField.SetStringValue(doc[4]); longField.SetStringValue(doc[5]); shortField.SetStringValue(doc[6]); stringField.SetStringValue(doc[7]); textField.SetStringValue(doc[8]); iw.AddDocument(document); } reader = iw.GetReader(); searcher = NewSearcher(reader); iw.Dispose(); }
public override void BeforeClass() { base.BeforeClass(); Random random = Random; m_directory = NewDirectory(); m_stopword = "" + GetRandomChar(); CharacterRunAutomaton stopset = new CharacterRunAutomaton(BasicAutomata.MakeString(m_stopword)); m_analyzer = new MockAnalyzer(random, MockTokenizer.WHITESPACE, false, stopset); RandomIndexWriter iw = new RandomIndexWriter(random, m_directory, m_analyzer); Document doc = new Document(); Field id = new StringField("id", "", Field.Store.NO); Field field = new TextField("field", "", Field.Store.NO); doc.Add(id); doc.Add(field); // index some docs int numDocs = AtLeast(1000); for (int i = 0; i < numDocs; i++) { id.SetStringValue(Convert.ToString(i, CultureInfo.InvariantCulture)); field.SetStringValue(RandomFieldContents()); iw.AddDocument(doc); } // delete some docs int numDeletes = numDocs / 20; for (int i = 0; i < numDeletes; i++) { Term toDelete = new Term("id", Convert.ToString(random.Next(numDocs), CultureInfo.InvariantCulture)); if (random.NextBoolean()) { iw.DeleteDocuments(toDelete); } else { iw.DeleteDocuments(new TermQuery(toDelete)); } } m_reader = iw.GetReader(); m_s1 = NewSearcher(m_reader); m_s2 = NewSearcher(m_reader); iw.Dispose(); }
public void TestRanking() { // number of documents: we will check each one int numDocs = AtLeast(100); // number of top-N snippets, we will check 1 .. N int maxTopN = 5; // maximum number of elements to put in a sentence. int maxSentenceLength = 10; // maximum number of sentences in a document int maxNumSentences = 20; Directory dir = NewDirectory(); RandomIndexWriter iw = new RandomIndexWriter( #if FEATURE_INSTANCE_TESTDATA_INITIALIZATION this, #endif Random, dir, new MockAnalyzer(Random, MockTokenizer.SIMPLE, true)); Document document = new Document(); Field id = new StringField("id", "", Field.Store.NO); FieldType offsetsType = new FieldType(TextField.TYPE_STORED); offsetsType.IndexOptions = (IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS); Field body = new Field("body", "", offsetsType); document.Add(id); document.Add(body); for (int i = 0; i < numDocs; i++) { StringBuilder bodyText = new StringBuilder(); int numSentences = TestUtil.NextInt32(Random, 1, maxNumSentences); for (int j = 0; j < numSentences; j++) { bodyText.Append(newSentence(Random, maxSentenceLength)); } body.SetStringValue(bodyText.ToString()); id.SetStringValue(i.ToString(CultureInfo.InvariantCulture)); iw.AddDocument(document); } IndexReader ir = iw.GetReader(); IndexSearcher searcher = NewSearcher(ir); for (int i = 0; i < numDocs; i++) { checkDocument(searcher, i, maxTopN); } iw.Dispose(); ir.Dispose(); dir.Dispose(); }
// [Test, Timeout(300000)] // LUCENENET NOTE: For now, we are overriding this test in every subclass to pull it into the right context for the subclass public virtual void TestConcurrentReads() { Directory dir = NewDirectory(); IndexWriterConfig iwConf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())); iwConf.SetMaxBufferedDocs(RandomInts.NextIntBetween(Random(), 2, 30)); RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwConf); // make sure the readers are properly cloned Document doc = new Document(); Field field = new StringField("fld", "", Field.Store.YES); doc.Add(field); int numDocs = AtLeast(1000); for (int i = 0; i < numDocs; ++i) { field.SetStringValue("" + i); iw.AddDocument(doc); } iw.Commit(); DirectoryReader rd = DirectoryReader.Open(dir); IndexSearcher searcher = new IndexSearcher(rd); int concurrentReads = AtLeast(5); int readsPerThread = AtLeast(50); IList <ThreadClass> readThreads = new List <ThreadClass>(); AtomicObject <Exception> ex = new AtomicObject <Exception>(); for (int i = 0; i < concurrentReads; ++i) { readThreads.Add(new ThreadAnonymousInnerClassHelper(numDocs, rd, searcher, readsPerThread, ex, i)); } foreach (ThreadClass thread in readThreads) { thread.Start(); } foreach (ThreadClass thread in readThreads) { thread.Join(); } rd.Dispose(); if (ex.Value != null) { throw ex.Value; } iw.Dispose(); dir.Dispose(); }
public virtual void TestConcurrentReads() { using Directory dir = NewDirectory(); IndexWriterConfig iwConf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)); iwConf.SetMaxBufferedDocs(RandomInts.RandomInt32Between(Random, 2, 30)); using RandomIndexWriter iw = new RandomIndexWriter(Random, dir, iwConf); // make sure the readers are properly cloned Document doc = new Document(); Field field = new StringField("fld", "", Field.Store.YES); doc.Add(field); int numDocs = AtLeast(1000); for (int i = 0; i < numDocs; ++i) { field.SetStringValue("" + i); iw.AddDocument(doc); } iw.Commit(); AtomicReference <Exception> ex = new AtomicReference <Exception>(); using (DirectoryReader rd = DirectoryReader.Open(dir)) { IndexSearcher searcher = new IndexSearcher(rd); int concurrentReads = AtLeast(5); int readsPerThread = AtLeast(50); IList <ThreadJob> readThreads = new JCG.List <ThreadJob>(); for (int i = 0; i < concurrentReads; ++i) { readThreads.Add(new ThreadAnonymousClass(numDocs, rd, searcher, readsPerThread, ex)); } foreach (ThreadJob thread in readThreads) { thread.Start(); } foreach (ThreadJob thread in readThreads) { thread.Join(); } } // rd.Dispose(); if (ex.Value != null) { ExceptionDispatchInfo.Capture(ex.Value).Throw(); // LUCENENET: Rethrow to preserve stack details from the other thread } }
public override void SetUp() { base.SetUp(); Dir = NewDirectory(); Iw = new RandomIndexWriter(Random(), Dir, Similarity, TimeZone); Document doc = new Document(); Field idField = new StringField("id", "", Field.Store.NO); doc.Add(idField); // add 500 docs with id 0..499 for (int i = 0; i < 500; i++) { idField.SetStringValue(Convert.ToString(i)); Iw.AddDocument(doc); } // delete 20 of them for (int i = 0; i < 20; i++) { Iw.DeleteDocuments(new Term("id", Convert.ToString(Random().Next(Iw.MaxDoc)))); } Ir = Iw.Reader; @is = NewSearcher(Ir); }
public void TestSearchAfterWhenSortingByFunctionValues() { Directory dir = NewDirectory(); IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, null); iwc.SetMergePolicy(NewLogMergePolicy()); // depends on docid order RandomIndexWriter writer = new RandomIndexWriter(Random, dir, iwc); Document doc = new Document(); Field field = new StringField("value", "", Field.Store.YES); doc.Add(field); // Save docs unsorted (decreasing value n, n-1, ...) const int NUM_VALS = 5; for (int val = NUM_VALS; val > 0; val--) { field.SetStringValue(Convert.ToString(val)); writer.AddDocument(doc); } // Open index IndexReader reader = writer.GetReader(); writer.Dispose(); IndexSearcher searcher = NewSearcher(reader); // Get ValueSource from FieldCache Int32FieldSource src = new Int32FieldSource("value"); // ...and make it a sort criterion SortField sf = src.GetSortField(false).Rewrite(searcher); Sort orderBy = new Sort(sf); // Get hits sorted by our FunctionValues (ascending values) Query q = new MatchAllDocsQuery(); TopDocs hits = searcher.Search(q, reader.MaxDoc, orderBy); assertEquals(NUM_VALS, hits.ScoreDocs.Length); // Verify that sorting works in general int i = 0; foreach (ScoreDoc hit in hits.ScoreDocs) { int valueFromDoc = Convert.ToInt32(reader.Document(hit.Doc).Get("value")); assertEquals(++i, valueFromDoc); } // Now get hits after hit #2 using IS.searchAfter() int afterIdx = 1; FieldDoc afterHit = (FieldDoc)hits.ScoreDocs[afterIdx]; hits = searcher.SearchAfter(afterHit, q, reader.MaxDoc, orderBy); // Expected # of hits: NUM_VALS - 2 assertEquals(NUM_VALS - (afterIdx + 1), hits.ScoreDocs.Length); // Verify that hits are actually "after" int afterValue = (int)((double?)afterHit.Fields[0]); foreach (ScoreDoc hit in hits.ScoreDocs) { int val = Convert.ToInt32(reader.Document(hit.Doc).Get("value")); assertTrue(afterValue <= val); assertFalse(hit.Doc == afterHit.Doc); } reader.Dispose(); dir.Dispose(); }
// [Test, LongRunningTest, Timeout(int.MaxValue)] // LUCENENET NOTE: For now, we are overriding this test in every subclass to pull it into the right context for the subclass public virtual void TestBigDocuments() { // "big" as "much bigger than the chunk size" // for this test we force a FS dir // we can't just use newFSDirectory, because this test doesn't really index anything. // so if we get NRTCachingDir+SimpleText, we make massive stored fields and OOM (LUCENE-4484) Directory dir = new MockDirectoryWrapper(Random(), new MMapDirectory(CreateTempDir("testBigDocuments"))); IndexWriterConfig iwConf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())); iwConf.SetMaxBufferedDocs(RandomInts.NextIntBetween(Random(), 2, 30)); RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwConf); if (dir is MockDirectoryWrapper) { ((MockDirectoryWrapper)dir).Throttling = MockDirectoryWrapper.Throttling_e.NEVER; } Document emptyDoc = new Document(); // emptyDoc Document bigDoc1 = new Document(); // lot of small fields Document bigDoc2 = new Document(); // 1 very big field Field idField = new StringField("id", "", Field.Store.NO); emptyDoc.Add(idField); bigDoc1.Add(idField); bigDoc2.Add(idField); FieldType onlyStored = new FieldType(StringField.TYPE_STORED); onlyStored.IsIndexed = false; Field smallField = new Field("fld", RandomByteArray(Random().Next(10), 256), onlyStored); int numFields = RandomInts.NextIntBetween(Random(), 500000, 1000000); for (int i = 0; i < numFields; ++i) { bigDoc1.Add(smallField); } Field bigField = new Field("fld", RandomByteArray(RandomInts.NextIntBetween(Random(), 1000000, 5000000), 2), onlyStored); bigDoc2.Add(bigField); int numDocs = AtLeast(5); Document[] docs = new Document[numDocs]; for (int i = 0; i < numDocs; ++i) { docs[i] = RandomInts.RandomFrom(Random(), Arrays.AsList(emptyDoc, bigDoc1, bigDoc2)); } for (int i = 0; i < numDocs; ++i) { idField.SetStringValue("" + i); iw.AddDocument(docs[i]); if (Random().Next(numDocs) == 0) { iw.Commit(); } } iw.Commit(); iw.ForceMerge(1); // look at what happens when big docs are merged DirectoryReader rd = DirectoryReader.Open(dir); IndexSearcher searcher = new IndexSearcher(rd); for (int i = 0; i < numDocs; ++i) { Query query = new TermQuery(new Term("id", "" + i)); TopDocs topDocs = searcher.Search(query, 1); Assert.AreEqual(1, topDocs.TotalHits, "" + i); Document doc = rd.Document(topDocs.ScoreDocs[0].Doc); Assert.IsNotNull(doc); IIndexableField[] fieldValues = doc.GetFields("fld"); Assert.AreEqual(docs[i].GetFields("fld").Length, fieldValues.Length); if (fieldValues.Length > 0) { Assert.AreEqual(docs[i].GetFields("fld")[0].GetBinaryValue(), fieldValues[0].GetBinaryValue()); } } rd.Dispose(); iw.Dispose(); dir.Dispose(); }