public ThreadAnonymousInnerClassHelper(TestAddTaxonomy outerInstance, int range, AtomicInteger numCats, DirectoryTaxonomyWriter tw)
{
this.outerInstance = outerInstance;
this.range = range;
this.numCats = numCats;
this.tw = tw;
}
private void Dotest(int ncats, int range)
{
AtomicInteger numCats = new AtomicInteger(ncats);
Directory[] dirs = new Directory[2];
for (int i = 0; i < dirs.Length; i++)
{
dirs[i] = NewDirectory();
var tw = new DirectoryTaxonomyWriter(dirs[i]);
ThreadClass[] addThreads = new ThreadClass[4];
for (int j = 0; j < addThreads.Length; j++)
{
addThreads[j] = new ThreadAnonymousInnerClassHelper(this, range, numCats, tw);
}
foreach (ThreadClass t in addThreads)
{
t.Start();
}
foreach (ThreadClass t in addThreads)
{
t.Join();
}
tw.Dispose();
}
var tw1 = new DirectoryTaxonomyWriter(dirs[0]);
IOrdinalMap map = randomOrdinalMap();
tw1.AddTaxonomy(dirs[1], map);
tw1.Dispose();
validate(dirs[0], dirs[1], map);
IOUtils.Close(dirs);
}
protected internal virtual void RunFlushByRam(int numThreads, double maxRamMB, bool ensureNotStalled)
{
int numDocumentsToIndex = 10 + AtLeast(30);
AtomicInteger numDocs = new AtomicInteger(numDocumentsToIndex);
Directory dir = NewDirectory();
MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy();
MockAnalyzer analyzer = new MockAnalyzer(Random());
analyzer.MaxTokenLength = TestUtil.NextInt(Random(), 1, IndexWriter.MAX_TERM_LENGTH);
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetFlushPolicy(flushPolicy);
int numDWPT = 1 + AtLeast(2);
DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numDWPT);
iwc.SetIndexerThreadPool(threadPool);
iwc.SetRAMBufferSizeMB(maxRamMB);
iwc.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
IndexWriter writer = new IndexWriter(dir, iwc);
flushPolicy = (MockDefaultFlushPolicy)writer.Config.FlushPolicy;
Assert.IsFalse(flushPolicy.FlushOnDocCount());
Assert.IsFalse(flushPolicy.FlushOnDeleteTerms());
Assert.IsTrue(flushPolicy.FlushOnRAM());
DocumentsWriter docsWriter = writer.DocsWriter;
Assert.IsNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.FlushControl;
Assert.AreEqual(0, flushControl.FlushBytes(), " bytes must be 0 after init");
IndexThread[] threads = new IndexThread[numThreads];
for (int x = 0; x < threads.Length; x++)
{
threads[x] = new IndexThread(this, numDocs, numThreads, writer, LineDocFile, false);
threads[x].Start();
}
for (int x = 0; x < threads.Length; x++)
{
threads[x].Join();
}
long maxRAMBytes = (long)(iwc.RAMBufferSizeMB * 1024.0 * 1024.0);
Assert.AreEqual(0, flushControl.FlushBytes(), " all flushes must be due numThreads=" + numThreads);
Assert.AreEqual(numDocumentsToIndex, writer.NumDocs());
Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc);
Assert.IsTrue(flushPolicy.PeakBytesWithoutFlush <= maxRAMBytes, "peak bytes without flush exceeded watermark");
AssertActiveBytesAfter(flushControl);
if (flushPolicy.HasMarkedPending)
{
Assert.IsTrue(maxRAMBytes < flushControl.PeakActiveBytes);
}
if (ensureNotStalled)
{
Assert.IsFalse(docsWriter.FlushControl.StallControl.WasStalled());
}
writer.Dispose();
Assert.AreEqual(0, flushControl.ActiveBytes());
dir.Dispose();
}
public virtual void TestApplyDeletesOnFlush()
{
Directory dir = NewDirectory();
// Cannot use RandomIndexWriter because we don't want to
// ever call commit() for this test:
AtomicInteger docsInSegment = new AtomicInteger();
AtomicBoolean closing = new AtomicBoolean();
AtomicBoolean sawAfterFlush = new AtomicBoolean();
IndexWriter w = new IndexWriterAnonymousInnerClassHelper(this, dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetRAMBufferSizeMB(0.5).SetMaxBufferedDocs(-1).SetMergePolicy(NoMergePolicy.NO_COMPOUND_FILES).SetReaderPooling(false), docsInSegment, closing, sawAfterFlush);
int id = 0;
while (true)
{
StringBuilder sb = new StringBuilder();
for (int termIDX = 0; termIDX < 100; termIDX++)
{
sb.Append(' ').Append(TestUtil.RandomRealisticUnicodeString(Random()));
}
if (id == 500)
{
w.DeleteDocuments(new Term("id", "0"));
}
Document doc = new Document();
doc.Add(NewStringField("id", "" + id, Field.Store.NO));
doc.Add(NewTextField("body", sb.ToString(), Field.Store.NO));
w.UpdateDocument(new Term("id", "" + id), doc);
docsInSegment.IncrementAndGet();
// TODO: fix this test
if (SlowFileExists(dir, "_0_1.del") || SlowFileExists(dir, "_0_1.liv"))
{
if (VERBOSE)
{
Console.WriteLine("TEST: deletes created @ id=" + id);
}
break;
}
id++;
}
closing.Set(true);
Assert.IsTrue(sawAfterFlush.Get());
w.Dispose();
dir.Dispose();
}
public virtual void TestBooleanScorerMax()
{
Directory dir = NewDirectory();
RandomIndexWriter riw = new RandomIndexWriter(Random(), dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
int docCount = AtLeast(10000);
for (int i = 0; i < docCount; i++)
{
Document doc = new Document();
doc.Add(NewField("field", "a", TextField.TYPE_NOT_STORED));
riw.AddDocument(doc);
}
riw.ForceMerge(1);
IndexReader r = riw.Reader;
riw.Dispose();
IndexSearcher s = NewSearcher(r);
BooleanQuery bq = new BooleanQuery();
bq.Add(new TermQuery(new Term("field", "a")), BooleanClause.Occur.SHOULD);
bq.Add(new TermQuery(new Term("field", "a")), BooleanClause.Occur.SHOULD);
Weight w = s.CreateNormalizedWeight(bq);
Assert.AreEqual(1, s.IndexReader.Leaves.Count);
BulkScorer scorer = w.BulkScorer(s.IndexReader.Leaves[0], false, null);
FixedBitSet hits = new FixedBitSet(docCount);
AtomicInteger end = new AtomicInteger();
Collector c = new CollectorAnonymousInnerClassHelper(this, scorer, hits, end);
while (end.Get() < docCount)
{
int inc = TestUtil.NextInt(Random(), 1, 1000);
end.AddAndGet(inc);
scorer.Score(c, end.Get());
}
Assert.AreEqual(docCount, hits.Cardinality());
r.Dispose();
dir.Dispose();
}
public virtual void TestConcurrency()
{
int ncats = AtLeast(100000); // add many categories
int range = ncats * 3; // affects the categories selection
AtomicInteger numCats = new AtomicInteger(ncats);
Directory dir = NewDirectory();
var values = new ConcurrentDictionary<string, string>();
double d = Random().NextDouble();
ITaxonomyWriterCache cache;
if (d < 0.7)
{
// this is the fastest, yet most memory consuming
cache = new Cl2oTaxonomyWriterCache(1024, 0.15f, 3);
}
else if (TEST_NIGHTLY && d > 0.98)
{
// this is the slowest, but tests the writer concurrency when no caching is done.
// only pick it during NIGHTLY tests, and even then, with very low chances.
cache = NO_OP_CACHE;
}
else
{
// this is slower than CL2O, but less memory consuming, and exercises finding categories on disk too.
cache = new LruTaxonomyWriterCache(ncats / 10);
}
if (VERBOSE)
{
Console.WriteLine("TEST: use cache=" + cache);
}
var tw = new DirectoryTaxonomyWriter(dir, OpenMode.CREATE, cache);
ThreadClass[] addThreads = new ThreadClass[AtLeast(4)];
for (int z = 0; z < addThreads.Length; z++)
{
addThreads[z] = new ThreadAnonymousInnerClassHelper(this, range, numCats, values, tw);
}
foreach (var t in addThreads)
{
t.Start();
}
foreach (var t in addThreads)
{
t.Join();
}
tw.Dispose();
DirectoryTaxonomyReader dtr = new DirectoryTaxonomyReader(dir);
// +1 for root category
if (values.Count + 1 != dtr.Count)
{
foreach (string value in values.Keys)
{
FacetLabel label = new FacetLabel(FacetsConfig.StringToPath(value));
if (dtr.GetOrdinal(label) == -1)
{
Console.WriteLine("FAIL: path=" + label + " not recognized");
}
}
Fail("mismatch number of categories");
}
int[] parents = dtr.ParallelTaxonomyArrays.Parents;
foreach (string cat in values.Keys)
{
FacetLabel cp = new FacetLabel(FacetsConfig.StringToPath(cat));
Assert.True(dtr.GetOrdinal(cp) > 0, "category not found " + cp);
int level = cp.Length;
int parentOrd = 0; // for root, parent is always virtual ROOT (ord=0)
FacetLabel path = new FacetLabel();
for (int i = 0; i < level; i++)
{
path = cp.Subpath(i + 1);
int ord = dtr.GetOrdinal(path);
Assert.AreEqual(parentOrd, parents[ord], "invalid parent for cp=" + path);
parentOrd = ord; // next level should have this parent
}
}
IOUtils.Close(dtr, dir);
}
public virtual void TestMaxMergeCount()
{
Directory dir = NewDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
int maxMergeCount = TestUtil.NextInt(Random(), 1, 5);
int maxMergeThreads = TestUtil.NextInt(Random(), 1, maxMergeCount);
CountdownEvent enoughMergesWaiting = new CountdownEvent(maxMergeCount);
AtomicInteger runningMergeCount = new AtomicInteger(0);
AtomicBoolean failed = new AtomicBoolean();
if (VERBOSE)
{
Console.WriteLine("TEST: maxMergeCount=" + maxMergeCount + " maxMergeThreads=" + maxMergeThreads);
}
ConcurrentMergeScheduler cms = new ConcurrentMergeSchedulerAnonymousInnerClassHelper(this, maxMergeCount, enoughMergesWaiting, runningMergeCount, failed);
cms.SetMaxMergesAndThreads(maxMergeCount, maxMergeThreads);
iwc.SetMergeScheduler(cms);
iwc.SetMaxBufferedDocs(2);
TieredMergePolicy tmp = new TieredMergePolicy();
iwc.SetMergePolicy(tmp);
tmp.MaxMergeAtOnce = 2;
tmp.SegmentsPerTier = 2;
IndexWriter w = new IndexWriter(dir, iwc);
Document doc = new Document();
doc.Add(NewField("field", "field", TextField.TYPE_NOT_STORED));
while (enoughMergesWaiting.CurrentCount != 0 && !failed.Get())
{
for (int i = 0; i < 10; i++)
{
w.AddDocument(doc);
}
}
w.Dispose(false);
dir.Dispose();
}
public CountListener(AtomicInteger count)
{
this.Count = count;
}
public ThreadAnonymousInnerClassHelper(TestNumericDocValuesUpdates outerInstance, string str, IndexWriter writer, int numDocs, CountDownLatch done, AtomicInteger numUpdates, string f, string cf)
: base(str)
{
this.OuterInstance = outerInstance;
this.Writer = writer;
this.NumDocs = numDocs;
this.Done = done;
this.NumUpdates = numUpdates;
this.f = f;
this.Cf = cf;
}
public ThreadAnonymousInnerClassHelper(TestConcurrentFacetedIndexing outerInstance, AtomicInteger numDocs, ConcurrentDictionary<string, string> values, IndexWriter iw, Lucene.Net.Facet.Taxonomy.Directory.DirectoryTaxonomyWriter tw, FacetsConfig config)
{
this.outerInstance = outerInstance;
this.numDocs = numDocs;
this.values = values;
this.iw = iw;
this.tw = tw;
this.config = config;
}
public IndexWriterAnonymousInnerClassHelper(TestIndexWriterDelete outerInstance, Directory dir, IndexWriterConfig setReaderPooling, AtomicInteger docsInSegment, AtomicBoolean closing, AtomicBoolean sawAfterFlush)
: base(dir, setReaderPooling)
{
this.OuterInstance = outerInstance;
this.DocsInSegment = docsInSegment;
this.Closing = closing;
this.SawAfterFlush = sawAfterFlush;
}
protected internal UpdateThread(DocumentsWriterDeleteQueue queue, AtomicInteger index, int?[] ids, CountDownLatch latch)
{
this.Queue = queue;
this.Index = index;
this.Ids = ids;
this.Slice = queue.NewSlice();
Deletes = new BufferedUpdates();
this.Latch = latch;
}
public virtual void TestConcurrency()
{
AtomicInteger numDocs = new AtomicInteger(AtLeast(10000));
Directory indexDir = NewDirectory();
Directory taxoDir = NewDirectory();
ConcurrentDictionary<string, string> values = new ConcurrentDictionary<string, string>();
IndexWriter iw = new IndexWriter(indexDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, null));
var tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE, NewTaxoWriterCache(numDocs.Get()));
ThreadClass[] indexThreads = new ThreadClass[AtLeast(4)];
FacetsConfig config = new FacetsConfig();
for (int i = 0; i < 10; i++)
{
config.SetHierarchical("l1." + i, true);
config.SetMultiValued("l1." + i, true);
}
for (int i = 0; i < indexThreads.Length; i++)
{
indexThreads[i] = new ThreadAnonymousInnerClassHelper(this, numDocs, values, iw, tw, config);
}
foreach (ThreadClass t in indexThreads)
{
t.Start();
}
foreach (ThreadClass t in indexThreads)
{
t.Join();
}
var tr = new DirectoryTaxonomyReader(tw);
// +1 for root category
if (values.Count + 1 != tr.Count)
{
foreach (string value in values.Keys)
{
FacetLabel label = new FacetLabel(FacetsConfig.StringToPath(value));
if (tr.GetOrdinal(label) == -1)
{
Console.WriteLine("FAIL: path=" + label + " not recognized");
}
}
Fail("mismatch number of categories");
}
int[] parents = tr.ParallelTaxonomyArrays.Parents;
foreach (string cat in values.Keys)
{
FacetLabel cp = new FacetLabel(FacetsConfig.StringToPath(cat));
Assert.True(tr.GetOrdinal(cp) > 0, "category not found " + cp);
int level = cp.Length;
int parentOrd = 0; // for root, parent is always virtual ROOT (ord=0)
FacetLabel path = null;
for (int i = 0; i < level; i++)
{
path = cp.Subpath(i + 1);
int ord = tr.GetOrdinal(path);
Assert.AreEqual(parentOrd, parents[ord], "invalid parent for cp=" + path);
parentOrd = ord; // next level should have this parent
}
}
IOUtils.Close(tw, iw, tr, taxoDir, indexDir);
}
public virtual void TestStressDeleteQueue()
{
DocumentsWriterDeleteQueue queue = new DocumentsWriterDeleteQueue();
HashSet<Term> uniqueValues = new HashSet<Term>();
int size = 10000 + Random().Next(500) * RANDOM_MULTIPLIER;
int?[] ids = new int?[size];
for (int i = 0; i < ids.Length; i++)
{
ids[i] = Random().Next();
uniqueValues.Add(new Term("id", ids[i].ToString()));
}
CountDownLatch latch = new CountDownLatch(1);
AtomicInteger index = new AtomicInteger(0);
int numThreads = 2 + Random().Next(5);
UpdateThread[] threads = new UpdateThread[numThreads];
for (int i = 0; i < threads.Length; i++)
{
threads[i] = new UpdateThread(queue, index, ids, latch);
threads[i].Start();
}
latch.countDown();
for (int i = 0; i < threads.Length; i++)
{
threads[i].Join();
}
foreach (UpdateThread updateThread in threads)
{
DeleteSlice slice = updateThread.Slice;
queue.UpdateSlice(slice);
BufferedUpdates deletes = updateThread.Deletes;
slice.Apply(deletes, BufferedUpdates.MAX_INT);
Assert.AreEqual(uniqueValues, deletes.Terms_Nunit().Keys);
}
queue.TryApplyGlobalSlice();
HashSet<Term> frozenSet = new HashSet<Term>();
foreach (Term t in queue.FreezeGlobalBuffer(null).TermsIterable())
{
BytesRef bytesRef = new BytesRef();
bytesRef.CopyBytes(t.Bytes());
frozenSet.Add(new Term(t.Field(), bytesRef));
}
Assert.AreEqual(0, queue.NumGlobalTermDeletes(), "num deletes must be 0 after freeze");
Assert.AreEqual(uniqueValues.Count, frozenSet.Count);
Assert.AreEqual(uniqueValues, frozenSet);
}
public virtual void Test()
{
// update variables
int commitPercent = Random().Next(20);
int softCommitPercent = Random().Next(100); // what percent of the commits are soft
int deletePercent = Random().Next(50);
int deleteByQueryPercent = Random().Next(25);
int ndocs = AtLeast(50);
int nWriteThreads = TestUtil.NextInt(Random(), 1, TEST_NIGHTLY ? 10 : 5);
int maxConcurrentCommits = TestUtil.NextInt(Random(), 1, TEST_NIGHTLY ? 10 : 5); // number of committers at a time... needed if we want to avoid commit errors due to exceeding the max
bool tombstones = Random().NextBoolean();
// query variables
AtomicLong operations = new AtomicLong(AtLeast(10000)); // number of query operations to perform in total
int nReadThreads = TestUtil.NextInt(Random(), 1, TEST_NIGHTLY ? 10 : 5);
InitModel(ndocs);
FieldType storedOnlyType = new FieldType();
storedOnlyType.Stored = true;
if (VERBOSE)
{
Console.WriteLine("\n");
Console.WriteLine("TEST: commitPercent=" + commitPercent);
Console.WriteLine("TEST: softCommitPercent=" + softCommitPercent);
Console.WriteLine("TEST: deletePercent=" + deletePercent);
Console.WriteLine("TEST: deleteByQueryPercent=" + deleteByQueryPercent);
Console.WriteLine("TEST: ndocs=" + ndocs);
Console.WriteLine("TEST: nWriteThreads=" + nWriteThreads);
Console.WriteLine("TEST: nReadThreads=" + nReadThreads);
Console.WriteLine("TEST: maxConcurrentCommits=" + maxConcurrentCommits);
Console.WriteLine("TEST: tombstones=" + tombstones);
Console.WriteLine("TEST: operations=" + operations);
Console.WriteLine("\n");
}
AtomicInteger numCommitting = new AtomicInteger();
IList<ThreadClass> threads = new List<ThreadClass>();
Directory dir = NewDirectory();
RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
writer.DoRandomForceMergeAssert = false;
writer.Commit();
Reader = DirectoryReader.Open(dir);
for (int i = 0; i < nWriteThreads; i++)
{
ThreadClass thread = new ThreadAnonymousInnerClassHelper(this, "WRITER" + i, commitPercent, softCommitPercent, deletePercent, deleteByQueryPercent, ndocs, maxConcurrentCommits, tombstones, operations, storedOnlyType, numCommitting, writer);
threads.Add(thread);
}
for (int i = 0; i < nReadThreads; i++)
{
ThreadClass thread = new ThreadAnonymousInnerClassHelper2(this, "READER" + i, ndocs, tombstones, operations);
threads.Add(thread);
}
foreach (ThreadClass thread in threads)
{
thread.Start();
}
foreach (ThreadClass thread in threads)
{
thread.Join();
}
writer.Dispose();
if (VERBOSE)
{
Console.WriteLine("TEST: close reader=" + Reader);
}
Reader.Dispose();
dir.Dispose();
}
public ThreadAnonymousInnerClassHelper(TestStressNRT outerInstance, string str, int commitPercent, int softCommitPercent, int deletePercent, int deleteByQueryPercent, int ndocs, int maxConcurrentCommits, bool tombstones, AtomicLong operations, FieldType storedOnlyType, AtomicInteger numCommitting, RandomIndexWriter writer)
: base(str)
{
this.OuterInstance = outerInstance;
this.CommitPercent = commitPercent;
this.SoftCommitPercent = softCommitPercent;
this.DeletePercent = deletePercent;
this.DeleteByQueryPercent = deleteByQueryPercent;
this.Ndocs = ndocs;
this.MaxConcurrentCommits = maxConcurrentCommits;
this.Tombstones = tombstones;
this.Operations = operations;
this.StoredOnlyType = storedOnlyType;
this.NumCommitting = numCommitting;
this.Writer = writer;
rand = new Random(Random().Next());
}
protected internal virtual void RunSearchThreads(DateTime stopTime)
{
int numThreads = TestUtil.NextInt(Random(), 1, 5);
ThreadClass[] searchThreads = new ThreadClass[numThreads];
AtomicInteger totHits = new AtomicInteger();
// silly starting guess:
AtomicInteger totTermCount = new AtomicInteger(100);
// TODO: we should enrich this to do more interesting searches
for (int thread = 0; thread < searchThreads.Length; thread++)
{
searchThreads[thread] = new ThreadAnonymousInnerClassHelper2(this, stopTime, totHits, totTermCount);
searchThreads[thread].SetDaemon(true);
searchThreads[thread].Start();
}
for (int thread = 0; thread < searchThreads.Length; thread++)
{
searchThreads[thread].Join();
}
if (VERBOSE)
{
Console.WriteLine("TEST: DONE search: totHits=" + totHits);
}
}
public virtual void TestFlushDocCount()
{
int[] numThreads = new int[] { 2 + AtLeast(1), 1 };
for (int i = 0; i < numThreads.Length; i++)
{
int numDocumentsToIndex = 50 + AtLeast(30);
AtomicInteger numDocs = new AtomicInteger(numDocumentsToIndex);
Directory dir = NewDirectory();
MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetFlushPolicy(flushPolicy);
int numDWPT = 1 + AtLeast(2);
DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numDWPT);
iwc.SetIndexerThreadPool(threadPool);
iwc.SetMaxBufferedDocs(2 + AtLeast(10));
iwc.SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
IndexWriter writer = new IndexWriter(dir, iwc);
flushPolicy = (MockDefaultFlushPolicy)writer.Config.FlushPolicy;
Assert.IsTrue(flushPolicy.FlushOnDocCount());
Assert.IsFalse(flushPolicy.FlushOnDeleteTerms());
Assert.IsFalse(flushPolicy.FlushOnRAM());
DocumentsWriter docsWriter = writer.DocsWriter;
Assert.IsNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.FlushControl;
Assert.AreEqual(0, flushControl.FlushBytes(), " bytes must be 0 after init");
IndexThread[] threads = new IndexThread[numThreads[i]];
for (int x = 0; x < threads.Length; x++)
{
threads[x] = new IndexThread(this, numDocs, numThreads[i], writer, LineDocFile, false);
threads[x].Start();
}
for (int x = 0; x < threads.Length; x++)
{
threads[x].Join();
}
Assert.AreEqual(0, flushControl.FlushBytes(), " all flushes must be due numThreads=" + numThreads[i]);
Assert.AreEqual(numDocumentsToIndex, writer.NumDocs());
Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc);
Assert.IsTrue(flushPolicy.PeakDocCountWithoutFlush <= iwc.MaxBufferedDocs, "peak bytes without flush exceeded watermark");
AssertActiveBytesAfter(flushControl);
writer.Dispose();
Assert.AreEqual(0, flushControl.ActiveBytes());
dir.Dispose();
}
}
public ThreadAnonymousInnerClassHelper2(ThreadedIndexingAndSearchingTestCase outerInstance, DateTime stopTime, AtomicInteger totHits, AtomicInteger totTermCount)
{
this.OuterInstance = outerInstance;
this.StopTime = stopTime;
this.TotHits = totHits;
this.TotTermCount = totTermCount;
}
public virtual void TestRandom()
{
int numThreads = 1 + Random().Next(8);
int numDocumentsToIndex = 50 + AtLeast(70);
AtomicInteger numDocs = new AtomicInteger(numDocumentsToIndex);
Directory dir = NewDirectory();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy();
iwc.SetFlushPolicy(flushPolicy);
int numDWPT = 1 + Random().Next(8);
DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numDWPT);
iwc.SetIndexerThreadPool(threadPool);
IndexWriter writer = new IndexWriter(dir, iwc);
flushPolicy = (MockDefaultFlushPolicy)writer.Config.FlushPolicy;
DocumentsWriter docsWriter = writer.DocsWriter;
Assert.IsNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.FlushControl;
Assert.AreEqual(0, flushControl.FlushBytes(), " bytes must be 0 after init");
IndexThread[] threads = new IndexThread[numThreads];
for (int x = 0; x < threads.Length; x++)
{
threads[x] = new IndexThread(this, numDocs, numThreads, writer, LineDocFile, true);
threads[x].Start();
}
for (int x = 0; x < threads.Length; x++)
{
threads[x].Join();
}
Assert.AreEqual(0, flushControl.FlushBytes(), " all flushes must be due");
Assert.AreEqual(numDocumentsToIndex, writer.NumDocs());
Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc);
if (flushPolicy.FlushOnRAM() && !flushPolicy.FlushOnDocCount() && !flushPolicy.FlushOnDeleteTerms())
{
long maxRAMBytes = (long)(iwc.RAMBufferSizeMB * 1024.0 * 1024.0);
Assert.IsTrue(flushPolicy.PeakBytesWithoutFlush <= maxRAMBytes, "peak bytes without flush exceeded watermark");
if (flushPolicy.HasMarkedPending)
{
assertTrue("max: " + maxRAMBytes + " " + flushControl.PeakActiveBytes, maxRAMBytes <= flushControl.PeakActiveBytes);
}
}
AssertActiveBytesAfter(flushControl);
writer.Commit();
Assert.AreEqual(0, flushControl.ActiveBytes());
IndexReader r = DirectoryReader.Open(dir);
Assert.AreEqual(numDocumentsToIndex, r.NumDocs);
Assert.AreEqual(numDocumentsToIndex, r.MaxDoc);
if (!flushPolicy.FlushOnRAM())
{
assertFalse("never stall if we don't flush on RAM", docsWriter.FlushControl.StallControl.WasStalled());
assertFalse("never block if we don't flush on RAM", docsWriter.FlushControl.StallControl.HasBlocked());
}
r.Dispose();
writer.Dispose();
dir.Dispose();
}
protected internal PayloadFilter(AtomicInteger payloadCount, TokenStream input)
: base(input)
{
this.PayloadCount = payloadCount;
PayloadAtt = AddAttribute<IPayloadAttribute>();
}
public virtual void TestStallControl()
{
int[] numThreads = new int[] { 4 + Random().Next(8), 1 };
int numDocumentsToIndex = 50 + Random().Next(50);
for (int i = 0; i < numThreads.Length; i++)
{
AtomicInteger numDocs = new AtomicInteger(numDocumentsToIndex);
MockDirectoryWrapper dir = NewMockDirectory();
// mock a very slow harddisk sometimes here so that flushing is very slow
dir.Throttling = MockDirectoryWrapper.Throttling_e.SOMETIMES;
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
iwc.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
FlushPolicy flushPolicy = new FlushByRamOrCountsPolicy();
iwc.SetFlushPolicy(flushPolicy);
DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numThreads[i] == 1 ? 1 : 2);
iwc.SetIndexerThreadPool(threadPool);
// with such a small ram buffer we should be stalled quiet quickly
iwc.SetRAMBufferSizeMB(0.25);
IndexWriter writer = new IndexWriter(dir, iwc);
IndexThread[] threads = new IndexThread[numThreads[i]];
for (int x = 0; x < threads.Length; x++)
{
threads[x] = new IndexThread(this, numDocs, numThreads[i], writer, LineDocFile, false);
threads[x].Start();
}
for (int x = 0; x < threads.Length; x++)
{
threads[x].Join();
}
DocumentsWriter docsWriter = writer.DocsWriter;
Assert.IsNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.FlushControl;
Assert.AreEqual(0, flushControl.FlushBytes(), " all flushes must be due");
Assert.AreEqual(numDocumentsToIndex, writer.NumDocs());
Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc);
if (numThreads[i] == 1)
{
assertFalse("single thread must not block numThreads: " + numThreads[i], docsWriter.FlushControl.StallControl.HasBlocked());
}
if (docsWriter.FlushControl.PeakNetBytes > (2d * iwc.RAMBufferSizeMB * 1024d * 1024d))
{
Assert.IsTrue(docsWriter.FlushControl.StallControl.WasStalled());
}
AssertActiveBytesAfter(flushControl);
writer.Dispose(true);
dir.Dispose();
}
}
public virtual void TestCloseUnderException()
{
Directory dir = NewDirectory();
IndexWriter writer = new IndexWriter(dir,
NewIndexWriterConfig(Random(), TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
writer.Commit();
writer.Dispose();
DirectoryReader open = DirectoryReader.Open(dir);
bool throwOnClose = !Rarely();
AtomicReader wrap = SlowCompositeReaderWrapper.Wrap(open);
FilterAtomicReader reader = new FilterAtomicReaderAnonymousInnerClassHelper(this, wrap, throwOnClose);
IList<IndexReader.ReaderClosedListener> listeners = new List<IndexReader.ReaderClosedListener>();
int listenerCount = Random().Next(20);
AtomicInteger count = new AtomicInteger();
bool faultySet = false;
for (int i = 0; i < listenerCount; i++)
{
if (Rarely())
{
faultySet = true;
reader.AddReaderClosedListener(new FaultyListener());
}
else
{
count.IncrementAndGet();
reader.AddReaderClosedListener(new CountListener(count));
}
}
if (!faultySet && !throwOnClose)
{
reader.AddReaderClosedListener(new FaultyListener());
}
try
{
reader.Dispose();
Assert.Fail("expected Exception");
}
catch (InvalidOperationException ex)
{
if (throwOnClose)
{
Assert.AreEqual("BOOM!", ex.Message);
}
else
{
Assert.AreEqual("GRRRRRRRRRRRR!", ex.Message);
}
}
try
{
var aaa = reader.Fields;
Assert.Fail("we are closed");
}
catch (AlreadyClosedException ex)
{
}
if (Random().NextBoolean())
{
reader.Dispose(); // call it again
}
Assert.AreEqual(0, count.Get());
wrap.Dispose();
dir.Dispose();
}
public IndexThread(TestFlushByRamOrCountsPolicy outerInstance, AtomicInteger pendingDocs, int numThreads, IndexWriter writer, LineFileDocs docs, bool doRandomCommit)
{
this.OuterInstance = outerInstance;
this.PendingDocs = pendingDocs;
this.Writer = writer;
Iwc = writer.Config;
this.Docs = docs;
this.DoRandomCommit = doRandomCommit;
}
public virtual void TestStressMultiThreading()
{
Directory dir = NewDirectory();
IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
IndexWriter writer = new IndexWriter(dir, conf);
// create index
int numThreads = TestUtil.NextInt(Random(), 3, 6);
int numDocs = AtLeast(2000);
for (int i = 0; i < numDocs; i++)
{
Document doc = new Document();
doc.Add(new StringField("id", "doc" + i, Store.NO));
double group = Random().NextDouble();
string g;
if (group < 0.1)
{
g = "g0";
}
else if (group < 0.5)
{
g = "g1";
}
else if (group < 0.8)
{
g = "g2";
}
else
{
g = "g3";
}
doc.Add(new StringField("updKey", g, Store.NO));
for (int j = 0; j < numThreads; j++)
{
long value = Random().Next();
doc.Add(new NumericDocValuesField("f" + j, value));
doc.Add(new NumericDocValuesField("cf" + j, value * 2)); // control, always updated to f * 2
}
writer.AddDocument(doc);
}
CountDownLatch done = new CountDownLatch(numThreads);
AtomicInteger numUpdates = new AtomicInteger(AtLeast(100));
// same thread updates a field as well as reopens
ThreadClass[] threads = new ThreadClass[numThreads];
for (int i = 0; i < threads.Length; i++)
{
string f = "f" + i;
string cf = "cf" + i;
threads[i] = new ThreadAnonymousInnerClassHelper(this, "UpdateThread-" + i, writer, numDocs, done, numUpdates, f, cf);
}
foreach (ThreadClass t in threads)
{
t.Start();
}
done.@await();
writer.Dispose();
DirectoryReader reader = DirectoryReader.Open(dir);
foreach (AtomicReaderContext context in reader.Leaves)
{
AtomicReader r = context.AtomicReader;
for (int i = 0; i < numThreads; i++)
{
NumericDocValues ndv = r.GetNumericDocValues("f" + i);
NumericDocValues control = r.GetNumericDocValues("cf" + i);
Bits docsWithNdv = r.GetDocsWithField("f" + i);
Bits docsWithControl = r.GetDocsWithField("cf" + i);
Bits liveDocs = r.LiveDocs;
for (int j = 0; j < r.MaxDoc; j++)
{
if (liveDocs == null || liveDocs.Get(j))
{
Assert.AreEqual(docsWithNdv.Get(j), docsWithControl.Get(j));
if (docsWithNdv.Get(j))
{
Assert.AreEqual(control.Get(j), ndv.Get(j) * 2);
}
}
}
}
}
reader.Dispose();
dir.Dispose();
}
public virtual void TestGetDocsWithFieldThreadSafety()
{
IFieldCache cache = FieldCache.DEFAULT;
cache.PurgeAllCaches();
int NUM_THREADS = 3;
ThreadClass[] threads = new ThreadClass[NUM_THREADS];
AtomicBoolean failed = new AtomicBoolean();
AtomicInteger iters = new AtomicInteger();
int NUM_ITER = 200 * RANDOM_MULTIPLIER;
Barrier restart = new Barrier(NUM_THREADS, (barrier) => new RunnableAnonymousInnerClassHelper(this, cache, iters).Run());
for (int threadIDX = 0; threadIDX < NUM_THREADS; threadIDX++)
{
threads[threadIDX] = new ThreadAnonymousInnerClassHelper(this, cache, failed, iters, NUM_ITER, restart);
threads[threadIDX].Start();
}
for (int threadIDX = 0; threadIDX < NUM_THREADS; threadIDX++)
{
threads[threadIDX].Join();
}
Assert.IsFalse(failed.Get());
}
public ThreadAnonymousInnerClassHelper(TestDirectoryTaxonomyWriter outerInstance, int range, AtomicInteger numCats, ConcurrentDictionary<string, string> values, Lucene.Net.Facet.Taxonomy.Directory.DirectoryTaxonomyWriter tw)
{
this.outerInstance = outerInstance;
this.range = range;
this.numCats = numCats;
this.values = values;
this.tw = tw;
}
public CollectorAnonymousInnerClassHelper(TestBooleanOr outerInstance, BulkScorer scorer, FixedBitSet hits, AtomicInteger end)
{
this.OuterInstance = outerInstance;
this.scorer = scorer;
this.Hits = hits;
this.End = end;
}
public ConcurrentMergeSchedulerAnonymousInnerClassHelper(TestConcurrentMergeScheduler outerInstance, int maxMergeCount, CountdownEvent enoughMergesWaiting, AtomicInteger runningMergeCount, AtomicBoolean failed)
{
this.OuterInstance = outerInstance;
this.MaxMergeCount = maxMergeCount;
this.EnoughMergesWaiting = enoughMergesWaiting;
this.RunningMergeCount = runningMergeCount;
this.Failed = failed;
}
public RunnableAnonymousInnerClassHelper(TestFieldCache outerInstance, IFieldCache cache, AtomicInteger iters)
{
this.OuterInstance = outerInstance;
this.Cache = cache;
this.Iters = iters;
}