public virtual void TestMixedTypesDifferentThreads() { Directory dir = NewDirectory(); IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))); CountDownLatch startingGun = new CountDownLatch(1); AtomicBoolean hitExc = new AtomicBoolean(); ThreadClass[] threads = new ThreadClass[3]; for (int i = 0; i < 3; i++) { Field field; if (i == 0) { field = new SortedDocValuesField("foo", new BytesRef("hello")); } else if (i == 1) { field = new NumericDocValuesField("foo", 0); } else { field = new BinaryDocValuesField("foo", new BytesRef("bazz")); } Document doc = new Document(); doc.Add(field); threads[i] = new ThreadAnonymousInnerClassHelper(this, w, startingGun, hitExc, doc); threads[i].Start(); } startingGun.countDown(); foreach (ThreadClass t in threads) { t.Join(); } Assert.IsTrue(hitExc.Get()); w.Dispose(); dir.Dispose(); }
public virtual void TestDeleteAllNoDeadLock() { Directory dir = NewDirectory(); RandomIndexWriter modifier = new RandomIndexWriter(Random(), dir); int numThreads = AtLeast(2); ThreadClass[] threads = new ThreadClass[numThreads]; CountDownLatch latch = new CountDownLatch(1); CountDownLatch doneLatch = new CountDownLatch(numThreads); for (int i = 0; i < numThreads; i++) { int offset = i; threads[i] = new ThreadAnonymousInnerClassHelper(this, modifier, latch, doneLatch, offset); threads[i].Start(); } latch.countDown(); //Wait for 1 millisecond while (!doneLatch.@await(new TimeSpan(0, 0, 0, 0, 1))) { modifier.DeleteAll(); if (VERBOSE) { Console.WriteLine("del all"); } } modifier.DeleteAll(); foreach (ThreadClass thread in threads) { thread.Join(); } modifier.Dispose(); DirectoryReader reader = DirectoryReader.Open(dir); Assert.AreEqual(reader.MaxDoc, 0); Assert.AreEqual(reader.NumDocs, 0); Assert.AreEqual(reader.NumDeletedDocs, 0); reader.Dispose(); dir.Dispose(); }
public virtual void TestHashCodeWithThreads() { AutomatonQuery[] queries = new AutomatonQuery[1000]; for (int i = 0; i < queries.Length; i++) { queries[i] = new AutomatonQuery(new Term("bogus", "bogus"), AutomatonTestUtil.RandomAutomaton(Random())); } CountDownLatch startingGun = new CountDownLatch(1); int numThreads = TestUtil.NextInt(Random(), 2, 5); ThreadClass[] threads = new ThreadClass[numThreads]; for (int threadID = 0; threadID < numThreads; threadID++) { ThreadClass thread = new ThreadAnonymousInnerClassHelper(this, queries, startingGun); threads[threadID] = thread; thread.Start(); } startingGun.countDown(); foreach (ThreadClass thread in threads) { thread.Join(); } }
public virtual void Test() { Directory dir = NewDirectory(); MockAnalyzer analyzer = new MockAnalyzer(Random()); analyzer.MaxTokenLength = TestUtil.NextInt(Random(), 1, IndexWriter.MAX_TERM_LENGTH); RandomIndexWriter w = new RandomIndexWriter(Random(), dir, analyzer); LineFileDocs docs = new LineFileDocs(Random(), DefaultCodecSupportsDocValues()); int charsToIndex = AtLeast(100000); int charsIndexed = 0; //System.out.println("bytesToIndex=" + charsToIndex); while (charsIndexed < charsToIndex) { Document doc = docs.NextDoc(); charsIndexed += doc.Get("body").Length; w.AddDocument(doc); //System.out.println(" bytes=" + charsIndexed + " add: " + doc); } IndexReader r = w.Reader; //System.out.println("numDocs=" + r.NumDocs); w.Dispose(); IndexSearcher s = NewSearcher(r); Terms terms = MultiFields.GetFields(r).Terms("body"); int termCount = 0; TermsEnum termsEnum = terms.Iterator(null); while (termsEnum.Next() != null) { termCount++; } Assert.IsTrue(termCount > 0); // Target ~10 terms to search: double chance = 10.0 / termCount; termsEnum = terms.Iterator(termsEnum); IDictionary<BytesRef, TopDocs> answers = new Dictionary<BytesRef, TopDocs>(); while (termsEnum.Next() != null) { if (Random().NextDouble() <= chance) { BytesRef term = BytesRef.DeepCopyOf(termsEnum.Term()); answers[term] = s.Search(new TermQuery(new Term("body", term)), 100); } } if (answers.Count > 0) { CountDownLatch startingGun = new CountDownLatch(1); int numThreads = TestUtil.NextInt(Random(), 2, 5); ThreadClass[] threads = new ThreadClass[numThreads]; for (int threadID = 0; threadID < numThreads; threadID++) { ThreadClass thread = new ThreadAnonymousInnerClassHelper(this, s, answers, startingGun); threads[threadID] = thread; thread.Start(); } startingGun.countDown(); foreach (ThreadClass thread in threads) { thread.Join(); } } r.Dispose(); dir.Dispose(); }
public virtual void Test() { Directory dir = NewDirectory(); IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NewLogMergePolicy())); IList<long?> numbers = new List<long?>(); IList<BytesRef> binary = new List<BytesRef>(); IList<BytesRef> sorted = new List<BytesRef>(); int numDocs = AtLeast(100); for (int i = 0; i < numDocs; i++) { Document d = new Document(); long number = Random().NextLong(); d.Add(new NumericDocValuesField("number", number)); BytesRef bytes = new BytesRef(TestUtil.RandomRealisticUnicodeString(Random())); d.Add(new BinaryDocValuesField("bytes", bytes)); binary.Add(bytes); bytes = new BytesRef(TestUtil.RandomRealisticUnicodeString(Random())); d.Add(new SortedDocValuesField("sorted", bytes)); sorted.Add(bytes); w.AddDocument(d); numbers.Add(number); } w.ForceMerge(1); IndexReader r = w.Reader; w.Dispose(); Assert.AreEqual(1, r.Leaves.Count); AtomicReader ar = (AtomicReader)r.Leaves[0].Reader; int numThreads = TestUtil.NextInt(Random(), 2, 5); IList<ThreadClass> threads = new List<ThreadClass>(); CountDownLatch startingGun = new CountDownLatch(1); for (int t = 0; t < numThreads; t++) { Random threadRandom = new Random(Random().Next()); ThreadClass thread = new ThreadAnonymousInnerClassHelper(this, numbers, binary, sorted, numDocs, ar, startingGun, threadRandom); thread.Start(); threads.Add(thread); } startingGun.countDown(); foreach (ThreadClass thread in threads) { thread.Join(); } r.Dispose(); dir.Dispose(); }
public virtual void TestStressDeleteQueue() { DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue(); HashSet<Term> uniqueValues = new HashSet<Term>(); int size = 10000 + Random().Next(500) * RANDOM_MULTIPLIER; int?[] ids = new int?[size]; for (int i = 0; i < ids.Length; i++) { ids[i] = Random().Next(); uniqueValues.Add(new Term("id", ids[i].ToString())); } CountDownLatch latch = new CountDownLatch(1); AtomicInteger index = new AtomicInteger(0); int numThreads = 2 + Random().Next(5); UpdateThread[] threads = new UpdateThread[numThreads]; for (int i = 0; i < threads.Length; i++) { threads[i] = new UpdateThread(queue, index, ids, latch); threads[i].Start(); } latch.countDown(); for (int i = 0; i < threads.Length; i++) { threads[i].Join(); } foreach (UpdateThread updateThread in threads) { DeleteSlice slice = updateThread.Slice; queue.UpdateSlice(slice); BufferedUpdates deletes = updateThread.Deletes; slice.Apply(deletes, BufferedUpdates.MAX_INT); Assert.AreEqual(uniqueValues, deletes.Terms_Nunit().Keys); } queue.TryApplyGlobalSlice(); HashSet<Term> frozenSet = new HashSet<Term>(); foreach (Term t in queue.FreezeGlobalBuffer(null).TermsIterable()) { BytesRef bytesRef = new BytesRef(); bytesRef.CopyBytes(t.Bytes()); frozenSet.Add(new Term(t.Field(), bytesRef)); } Assert.AreEqual(0, queue.NumGlobalTermDeletes(), "num deletes must be 0 after freeze"); Assert.AreEqual(uniqueValues.Count, frozenSet.Count); Assert.AreEqual(uniqueValues, frozenSet); }
public virtual void Test() { IList<string> postingsList = new List<string>(); int numTerms = AtLeast(300); int maxTermsPerDoc = TestUtil.NextInt(Random(), 10, 20); bool isSimpleText = "SimpleText".Equals(TestUtil.GetPostingsFormat("field")); IndexWriterConfig iwc = NewIndexWriterConfig(Random(), TEST_VERSION_CURRENT, new MockAnalyzer(Random())); if ((isSimpleText || iwc.MergePolicy is MockRandomMergePolicy) && (TEST_NIGHTLY || RANDOM_MULTIPLIER > 1)) { // Otherwise test can take way too long (> 2 hours) numTerms /= 2; } if (VERBOSE) { Console.WriteLine("maxTermsPerDoc=" + maxTermsPerDoc); Console.WriteLine("numTerms=" + numTerms); } for (int i = 0; i < numTerms; i++) { string term = Convert.ToString(i); for (int j = 0; j < i; j++) { postingsList.Add(term); } } postingsList = CollectionsHelper.Shuffle(postingsList); ConcurrentQueue<string> postings = new ConcurrentQueue<string>(postingsList); Directory dir = NewFSDirectory(CreateTempDir("bagofpostings")); RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwc); int threadCount = TestUtil.NextInt(Random(), 1, 5); if (VERBOSE) { Console.WriteLine("config: " + iw.w.Config); Console.WriteLine("threadCount=" + threadCount); } ThreadClass[] threads = new ThreadClass[threadCount]; CountDownLatch startingGun = new CountDownLatch(1); for (int threadID = 0; threadID < threadCount; threadID++) { threads[threadID] = new ThreadAnonymousInnerClassHelper(this, maxTermsPerDoc, postings, iw, startingGun); threads[threadID].Start(); } startingGun.countDown(); foreach (ThreadClass t in threads) { t.Join(); } iw.ForceMerge(1); DirectoryReader ir = iw.Reader; Assert.AreEqual(1, ir.Leaves.Count); AtomicReader air = (AtomicReader)ir.Leaves[0].Reader; Terms terms = air.Terms("field"); // numTerms-1 because there cannot be a term 0 with 0 postings: Assert.AreEqual(numTerms - 1, air.Fields.UniqueTermCount); if (iwc.Codec is Lucene3xCodec == false) { Assert.AreEqual(numTerms - 1, terms.Size()); } TermsEnum termsEnum = terms.Iterator(null); BytesRef term_; while ((term_ = termsEnum.Next()) != null) { int value = Convert.ToInt32(term_.Utf8ToString()); Assert.AreEqual(value, termsEnum.DocFreq()); // don't really need to check more than this, as CheckIndex // will verify that docFreq == actual number of documents seen // from a docsAndPositionsEnum. } ir.Dispose(); iw.Dispose(); dir.Dispose(); }
public static void CheckRandomData(Random random, Analyzer a, int iterations, int maxWordLength, bool simple, bool offsetsAreCorrect) { CheckResetException(a, "best effort"); long seed = random.Next(); bool useCharFilter = random.NextBoolean(); Directory dir = null; RandomIndexWriter iw = null; string postingsFormat = TestUtil.GetPostingsFormat("dummy"); bool codecOk = iterations * maxWordLength < 100000 || !(postingsFormat.Equals("Memory") || postingsFormat.Equals("SimpleText")); if (Rarely(random) && codecOk) { dir = NewFSDirectory(CreateTempDir("bttc")); iw = new RandomIndexWriter(new Random((int)seed), dir, a); } bool success = false; try { CheckRandomData(new Random((int)seed), a, iterations, maxWordLength, useCharFilter, simple, offsetsAreCorrect, iw); // now test with multiple threads: note we do the EXACT same thing we did before in each thread, // so this should only really fail from another thread if its an actual thread problem int numThreads = TestUtil.NextInt(random, 2, 4); var startingGun = new CountDownLatch(1); var threads = new AnalysisThread[numThreads]; for (int i = 0; i < threads.Length; i++) { threads[i] = new AnalysisThread(seed, /*startingGun,*/ a, iterations, maxWordLength, useCharFilter, simple, offsetsAreCorrect, iw); } Array.ForEach(threads, thread => thread.Start()); startingGun.countDown(); foreach (var t in threads) { try { t.Join(); } catch (ThreadInterruptedException e) { Fail("Thread interrupted"); } } if (threads.Any(x => x.Failed)) Fail("Thread interrupted"); success = true; } finally { if (success) { IOUtils.Close(iw, dir); } else { IOUtils.CloseWhileHandlingException(iw, dir); // checkindex } } }
public virtual void Test() { IList<string> postingsList = new List<string>(); int numTerms = AtLeast(300); int maxTermsPerDoc = TestUtil.NextInt(Random(), 10, 20); bool isSimpleText = "SimpleText".Equals(TestUtil.GetPostingsFormat("field")); IndexWriterConfig iwc = NewIndexWriterConfig(Random(), TEST_VERSION_CURRENT, new MockAnalyzer(Random())); if ((isSimpleText || iwc.MergePolicy is MockRandomMergePolicy) && (TEST_NIGHTLY || RANDOM_MULTIPLIER > 1)) { // Otherwise test can take way too long (> 2 hours) numTerms /= 2; } if (VERBOSE) { Console.WriteLine("maxTermsPerDoc=" + maxTermsPerDoc); Console.WriteLine("numTerms=" + numTerms); } for (int i = 0; i < numTerms; i++) { string term = Convert.ToString(i); for (int j = 0; j < i; j++) { postingsList.Add(term); } } postingsList = CollectionsHelper.Shuffle(postingsList); ConcurrentQueue<string> postings = new ConcurrentQueue<string>(postingsList); Directory dir = NewFSDirectory(CreateTempDir(GetFullMethodName())); RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, iwc); int threadCount = TestUtil.NextInt(Random(), 1, 5); if (VERBOSE) { Console.WriteLine("config: " + iw.w.Config); Console.WriteLine("threadCount=" + threadCount); } Field prototype = NewTextField("field", "", Field.Store.NO); FieldType fieldType = new FieldType((FieldType)prototype.FieldType()); if (Random().NextBoolean()) { fieldType.OmitNorms = true; } int options = Random().Next(3); if (options == 0) { fieldType.IndexOptions = FieldInfo.IndexOptions.DOCS_AND_FREQS; // we dont actually need positions fieldType.StoreTermVectors = true; // but enforce term vectors when we do this so we check SOMETHING } else if (options == 1 && !DoesntSupportOffsets.Contains(TestUtil.GetPostingsFormat("field"))) { fieldType.IndexOptions = FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS; } // else just positions ThreadClass[] threads = new ThreadClass[threadCount]; CountDownLatch startingGun = new CountDownLatch(1); for (int threadID = 0; threadID < threadCount; threadID++) { Random threadRandom = new Random(Random().Next()); Document document = new Document(); Field field = new Field("field", "", fieldType); document.Add(field); threads[threadID] = new ThreadAnonymousInnerClassHelper(this, numTerms, maxTermsPerDoc, postings, iw, startingGun, threadRandom, document, field); threads[threadID].Start(); } startingGun.countDown(); foreach (ThreadClass t in threads) { t.Join(); } iw.ForceMerge(1); DirectoryReader ir = iw.Reader; Assert.AreEqual(1, ir.Leaves.Count); AtomicReader air = (AtomicReader)ir.Leaves[0].Reader; Terms terms = air.Terms("field"); // numTerms-1 because there cannot be a term 0 with 0 postings: Assert.AreEqual(numTerms - 1, terms.Size()); TermsEnum termsEnum = terms.Iterator(null); BytesRef termBR; while ((termBR = termsEnum.Next()) != null) { int value = Convert.ToInt32(termBR.Utf8ToString()); Assert.AreEqual(value, termsEnum.TotalTermFreq()); // don't really need to check more than this, as CheckIndex // will verify that totalTermFreq == total number of positions seen // from a docsAndPositionsEnum. } ir.Dispose(); iw.Dispose(); dir.Dispose(); }
public void TestThreads2() { AssumeTrue("Codec does not support GetDocsWithField", DefaultCodecSupportsDocsWithField()); AssumeTrue("Codec does not support SORTED_SET", DefaultCodecSupportsSortedSet()); Directory dir = NewDirectory(); IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())); RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, conf); Field idField = new StringField("id", "", Field.Store.NO); Field storedBinField = new StoredField("storedBin", new byte[0]); Field dvBinField = new BinaryDocValuesField("dvBin", new BytesRef()); Field dvSortedField = new SortedDocValuesField("dvSorted", new BytesRef()); Field storedNumericField = new StoredField("storedNum", ""); Field dvNumericField = new NumericDocValuesField("dvNum", 0); // index some docs int numDocs = AtLeast(300); for (int i = 0; i < numDocs; i++) { idField.StringValue = Convert.ToString(i); int length = TestUtil.NextInt(Random(), 0, 8); var buffer = new byte[length]; Random().NextBytes(buffer); storedBinField.BytesValue = new BytesRef(buffer); dvBinField.BytesValue = new BytesRef(buffer); dvSortedField.BytesValue = new BytesRef(buffer); long numericValue = Random().NextLong(); storedNumericField.StringValue = Convert.ToString(numericValue); dvNumericField.LongValue = numericValue; Document doc = new Document(); doc.Add(idField); if (Random().Next(4) > 0) { doc.Add(storedBinField); doc.Add(dvBinField); doc.Add(dvSortedField); } if (Random().Next(4) > 0) { doc.Add(storedNumericField); doc.Add(dvNumericField); } int numSortedSetFields = Random().Next(3); SortedSet<string> values = new SortedSet<string>(); for (int j = 0; j < numSortedSetFields; j++) { values.Add(TestUtil.RandomSimpleString(Random())); } foreach (string v in values) { doc.Add(new SortedSetDocValuesField("dvSortedSet", new BytesRef(v))); doc.Add(new StoredField("storedSortedSet", v)); } writer.AddDocument(doc); if (Random().Next(31) == 0) { writer.Commit(); } } // delete some docs int numDeletions = Random().Next(numDocs / 10); for (int i = 0; i < numDeletions; i++) { int id = Random().Next(numDocs); writer.DeleteDocuments(new Term("id", Convert.ToString(id))); } writer.Dispose(); // compare DirectoryReader ir = DirectoryReader.Open(dir); int numThreads = TestUtil.NextInt(Random(), 2, 7); ThreadClass[] threads = new ThreadClass[numThreads]; CountDownLatch startingGun = new CountDownLatch(1); for (int i = 0; i < threads.Length; i++) { threads[i] = new ThreadAnonymousInnerClassHelper2(this, ir, startingGun); threads[i].Start(); } startingGun.countDown(); foreach (ThreadClass t in threads) { t.Join(); } ir.Dispose(); dir.Dispose(); }