public ThreadAnonymousInnerClassHelper(int range, AtomicInt32 numCats, ConcurrentDictionary <string, string> values, DirectoryTaxonomyWriter tw)
{
this.range = range;
this.numCats = numCats;
this.values = values;
this.tw = tw;
}
public CollectorAnonymousClass(TestBooleanOr outerInstance, BulkScorer scorer, FixedBitSet hits, AtomicInt32 end)
{
this.outerInstance = outerInstance;
this.scorer = scorer;
this.hits = hits;
this.end = end;
}
protected internal virtual void RunSearchThreads(long stopTime)
{
int numThreads = TestUtil.NextInt(Random(), 1, 5);
ThreadClass[] searchThreads = new ThreadClass[numThreads];
AtomicInt32 totHits = new AtomicInt32();
// silly starting guess:
AtomicInt32 totTermCount = new AtomicInt32(100);
// TODO: we should enrich this to do more interesting searches
for (int thread = 0; thread < searchThreads.Length; thread++)
{
searchThreads[thread] = new ThreadAnonymousInnerClassHelper2(this, stopTime, totHits, totTermCount);
searchThreads[thread].SetDaemon(true);
searchThreads[thread].Start();
}
for (int thread = 0; thread < searchThreads.Length; thread++)
{
searchThreads[thread].Join();
}
if (VERBOSE)
{
Console.WriteLine("TEST: DONE search: totHits=" + totHits);
}
}
public DoubleBarrelLRUCache(int maxCount)
{
this.maxSize = maxCount;
countdown = new AtomicInt32(maxCount);
cache1 = new ConcurrentDictionary <TKey, TValue>();
cache2 = new ConcurrentDictionary <TKey, TValue>();
}
public virtual void TestGetDocsWithFieldThreadSafety()
{
IFieldCache cache = FieldCache.DEFAULT;
cache.PurgeAllCaches();
int NUM_THREADS = 3;
ThreadJob[] threads = new ThreadJob[NUM_THREADS];
AtomicBoolean failed = new AtomicBoolean();
AtomicInt32 iters = new AtomicInt32();
int NUM_ITER = 200 * RANDOM_MULTIPLIER;
Barrier restart = new Barrier(NUM_THREADS, (barrier) => new RunnableAnonymousInnerClassHelper(this, cache, iters).Run());
for (int threadIDX = 0; threadIDX < NUM_THREADS; threadIDX++)
{
threads[threadIDX] = new ThreadAnonymousInnerClassHelper(this, cache, failed, iters, NUM_ITER, restart);
threads[threadIDX].Start();
}
for (int threadIDX = 0; threadIDX < NUM_THREADS; threadIDX++)
{
threads[threadIDX].Join();
}
Assert.IsFalse(failed);
}
public CollectorAnonymousInnerClassHelper(TestBooleanOr outerInstance, BulkScorer scorer, FixedBitSet hits, AtomicInt32 end)
{
this.OuterInstance = outerInstance;
this.scorer = scorer;
this.Hits = hits;
this.End = end;
}
protected virtual void RunSearchThreads(long stopTime)
{
int numThreads = TestUtil.NextInt32(Random, 1, 5);
ThreadJob[] searchThreads = new ThreadJob[numThreads];
AtomicInt32 totHits = new AtomicInt32();
// silly starting guess:
AtomicInt32 totTermCount = new AtomicInt32(100);
// TODO: we should enrich this to do more interesting searches
for (int thread = 0; thread < searchThreads.Length; thread++)
{
searchThreads[thread] = new ThreadAnonymousClass2(this, stopTime, totHits, totTermCount);
searchThreads[thread].IsBackground = (true);
searchThreads[thread].Start();
}
for (int thread = 0; thread < searchThreads.Length; thread++)
{
searchThreads[thread].Join();
}
if (Verbose)
{
Console.WriteLine("TEST: DONE search: totHits=" + totHits);
}
}
public ThreadAnonymousInnerClassHelper2(ThreadedIndexingAndSearchingTestCase outerInstance, long stopTimeMS, AtomicInt32 totHits, AtomicInt32 totTermCount)
{
this.outerInstance = outerInstance;
this.stopTimeMS = stopTimeMS;
this.totHits = totHits;
this.totTermCount = totTermCount;
}
public ThreadAnonymousInnerClassHelper(TestAddTaxonomy outerInstance, int range, AtomicInt32 numCats, DirectoryTaxonomyWriter tw)
{
this.outerInstance = outerInstance;
this.range = range;
this.numCats = numCats;
this.tw = tw;
}
public void AtomicInt32ToStringReturnsExpectedStringForDefault()
{
const int expected = 0;
var i = new AtomicInt32();
Assert.AreEqual(expected.ToString(), i.ToString());
}
public void AtomicInt32ToStringReturnsExpectedStringForSpecified()
{
const int expected = 47;
var i = new AtomicInt32(expected);
Assert.AreEqual(expected.ToString(), i.ToString());
}
public ThreadAnonymousInnerClassHelper(TestDirectoryTaxonomyWriter outerInstance, int range, AtomicInt32 numCats, ConcurrentDictionary <string, string> values, Lucene.Net.Facet.Taxonomy.Directory.DirectoryTaxonomyWriter tw)
{
this.outerInstance = outerInstance;
this.range = range;
this.numCats = numCats;
this.values = values;
this.tw = tw;
}
public IndexThread(AtomicInt32 pendingDocs, IndexWriter writer, LineFileDocs docs, bool doRandomCommit)
{
this.pendingDocs = pendingDocs;
this.writer = writer;
iwc = writer.Config;
this.docs = docs;
this.doRandomCommit = doRandomCommit;
}
public ConcurrentMergeSchedulerAnonymousInnerClassHelper(TestConcurrentMergeScheduler outerInstance, int maxMergeCount, CountdownEvent enoughMergesWaiting, AtomicInt32 runningMergeCount, AtomicBoolean failed)
{
this.outerInstance = outerInstance;
this.maxMergeCount = maxMergeCount;
this.enoughMergesWaiting = enoughMergesWaiting;
this.runningMergeCount = runningMergeCount;
this.failed = failed;
}
public ThreadAnonymousInnerClassHelper(TestConcurrentFacetedIndexing outerInstance, AtomicInt32 numDocs, ConcurrentDictionary <string, string> values, IndexWriter iw, DirectoryTaxonomyWriter tw, FacetsConfig config)
{
this.outerInstance = outerInstance;
this.numDocs = numDocs;
this.values = values;
this.iw = iw;
this.tw = tw;
this.config = config;
}
protected internal UpdateThread(DocumentsWriterDeleteQueue queue, AtomicInt32 index, int?[] ids, CountdownEvent latch)
{
this.Queue = queue;
this.Index = index;
this.Ids = ids;
this.Slice = queue.NewSlice();
Deletes = new BufferedUpdates();
this.Latch = latch;
}
public ThreadAnonymousInnerClassHelper(TestFieldCache outerInstance, IFieldCache cache, AtomicBoolean failed, AtomicInt32 iters, int NUM_ITER, Barrier restart)
{
this.OuterInstance = outerInstance;
this.Cache = cache;
this.Failed = failed;
this.Iters = iters;
this.NUM_ITER = NUM_ITER;
this.Restart = restart;
}
public IndexThread(TestFlushByRamOrCountsPolicy outerInstance, AtomicInt32 pendingDocs, int numThreads, IndexWriter writer, LineFileDocs docs, bool doRandomCommit)
{
this.OuterInstance = outerInstance;
this.PendingDocs = pendingDocs;
this.Writer = writer;
Iwc = writer.Config;
this.Docs = docs;
this.DoRandomCommit = doRandomCommit;
}
public void AtomicInt32InitialisesWithSpecifiedvalue()
{
const int expected = 47;
var i = new AtomicInt32(expected);
Assert.AreEqual(expected, (int)i);
Assert.AreEqual(expected, i.Value);
}
public void AtomicInt32InitialisesWithDefaultValue()
{
const int expected = 0;
var i = new AtomicInt32();
Assert.AreEqual(expected, (int)i);
Assert.AreEqual(expected, i.Value);
}
public ThreadAnonymousClass(TestFieldCache outerInstance, IFieldCache cache, AtomicBoolean failed, AtomicInt32 iters, int NUM_ITER, Barrier restart)
{
this.outerInstance = outerInstance;
this.cache = cache;
this.failed = failed;
this.iters = iters;
this.NUM_ITER = NUM_ITER;
this.restart = restart;
}
[Slow] // LUCENENET: occasionally
public virtual void TestStallControl()
{
// LUCENENET specific - disable the test if asserts are not enabled
AssumeTrue("This test requires asserts to be enabled.", Debugging.AssertsEnabled);
int[] numThreads = new int[] { 4 + Random.Next(8), 1 };
int numDocumentsToIndex = 50 + Random.Next(50);
for (int i = 0; i < numThreads.Length; i++)
{
AtomicInt32 numDocs = new AtomicInt32(numDocumentsToIndex);
MockDirectoryWrapper dir = NewMockDirectory();
// mock a very slow harddisk sometimes here so that flushing is very slow
dir.Throttling = Throttling.SOMETIMES;
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random));
iwc.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
FlushPolicy flushPolicy = new FlushByRamOrCountsPolicy();
iwc.SetFlushPolicy(flushPolicy);
DocumentsWriterPerThreadPool threadPool = new DocumentsWriterPerThreadPool(numThreads[i] == 1 ? 1 : 2);
iwc.SetIndexerThreadPool(threadPool);
// with such a small ram buffer we should be stalled quiet quickly
iwc.SetRAMBufferSizeMB(0.25);
IndexWriter writer = new IndexWriter(dir, iwc);
IndexThread[] threads = new IndexThread[numThreads[i]];
for (int x = 0; x < threads.Length; x++)
{
threads[x] = new IndexThread(numDocs, writer, lineDocFile, false);
threads[x].Start();
}
for (int x = 0; x < threads.Length; x++)
{
threads[x].Join();
}
DocumentsWriter docsWriter = writer.DocsWriter;
Assert.IsNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
Assert.AreEqual(0, flushControl.FlushBytes, " all flushes must be due");
Assert.AreEqual(numDocumentsToIndex, writer.NumDocs);
Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc);
if (numThreads[i] == 1)
{
assertFalse("single thread must not block numThreads: " + numThreads[i], docsWriter.flushControl.stallControl.HasBlocked);
}
if (docsWriter.flushControl.peakNetBytes > (2d * iwc.RAMBufferSizeMB * 1024d * 1024d))
{
Assert.IsTrue(docsWriter.flushControl.stallControl.WasStalled);
}
AssertActiveBytesAfter(flushControl);
writer.Dispose(true);
dir.Dispose();
}
}
public void TestConsistencyOnExceptions()
{
// so the handler's index isn't empty
replicator.Publish(CreateRevision(1));
client.UpdateNow();
client.Dispose();
callback.Dispose();
// Replicator violates write-once policy. It may be that the
// handler copies files to the index dir, then fails to copy a
// file and reverts the copy operation. On the next attempt, it
// will copy the same file again. There is nothing wrong with this
// in a real system, but it does violate write-once, and MDW
// doesn't like it. Disabling it means that we won't catch cases
// where the handler overwrites an existing index file, but
// there's nothing currently we can do about it, unless we don't
// use MDW.
handlerIndexDir.PreventDoubleWrite = (false);
handlerTaxoDir.PreventDoubleWrite = (false);
// wrap sourceDirFactory to return a MockDirWrapper so we can simulate errors
ISourceDirectoryFactory @in = sourceDirFactory;
AtomicInt32 failures = new AtomicInt32(AtLeast(10));
sourceDirFactory = new SourceDirectoryFactoryAnonymousInnerClass(this, @in, failures);
handler = new IndexAndTaxonomyReplicationHandler(handlerIndexDir, handlerTaxoDir, () =>
{
if (Random.NextDouble() < 0.2 && failures > 0)
{
throw new Exception("random exception from callback");
}
return(null);
});
client = new ReplicationClientAnonymousInnerClass(this, replicator, handler, @in, failures);
client.StartUpdateThread(10, "indexAndTaxo");
Directory baseHandlerIndexDir = handlerIndexDir.Delegate;
int numRevisions = AtLeast(20) + 2;
for (int i = 2; i < numRevisions; i++)
{
replicator.Publish(CreateRevision(i));
AssertHandlerRevision(i, baseHandlerIndexDir);
}
// disable errors -- maybe randomness didn't exhaust all allowed failures,
// and we don't want e.g. CheckIndex to hit false errors.
handlerIndexDir.MaxSizeInBytes = (0);
handlerIndexDir.RandomIOExceptionRate = (0.0);
handlerIndexDir.RandomIOExceptionRateOnOpen = (0.0);
handlerTaxoDir.MaxSizeInBytes = (0);
handlerTaxoDir.RandomIOExceptionRate = (0.0);
handlerTaxoDir.RandomIOExceptionRateOnOpen = (0.0);
}
public void AtomicInt32SupportsPreIncrementMultiThreaded()
{
const int expected = 1000;
var i = new AtomicInt32();
System.Threading.Tasks.Parallel.For(0, expected, j => i.PreIncrement());
Assert.AreEqual(expected, (int)i);
Assert.AreEqual(expected, i.Value);
}
public void AtomicInt32SupportsGetAndAddNegativesMultiThreaded()
{
const int expected = 1000;
var i = new AtomicInt32();
System.Threading.Tasks.Parallel.For(0, expected, j => i.GetAndAdd(1));
Assert.AreEqual(expected, (int)i);
Assert.AreEqual(expected, i.Value);
}
public ThreadAnonymousClass(TestMixedDocValuesUpdates outerInstance, string str, IndexWriter writer, int numDocs, CountdownEvent done, AtomicInt32 numUpdates, string f, string cf)
: base(str)
{
this.outerInstance = outerInstance;
this.writer = writer;
this.numDocs = numDocs;
this.done = done;
this.numUpdates = numUpdates;
this.f = f;
this.cf = cf;
}
public virtual void TestFlushDocCount()
{
// LUCENENET specific - disable the test if asserts are not enabled
AssumeTrue("This test requires asserts to be enabled.", Debugging.AssertsEnabled);
int[] numThreads = new int[] { 2 + AtLeast(1), 1 };
for (int i = 0; i < numThreads.Length; i++)
{
int numDocumentsToIndex = 50 + AtLeast(30);
AtomicInt32 numDocs = new AtomicInt32(numDocumentsToIndex);
Directory dir = NewDirectory();
MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy();
IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetFlushPolicy(flushPolicy);
int numDWPT = 1 + AtLeast(2);
DocumentsWriterPerThreadPool threadPool = new DocumentsWriterPerThreadPool(numDWPT);
iwc.SetIndexerThreadPool(threadPool);
iwc.SetMaxBufferedDocs(2 + AtLeast(10));
iwc.SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
IndexWriter writer = new IndexWriter(dir, iwc);
flushPolicy = (MockDefaultFlushPolicy)writer.Config.FlushPolicy;
Assert.IsTrue(flushPolicy.FlushOnDocCount);
Assert.IsFalse(flushPolicy.FlushOnDeleteTerms);
Assert.IsFalse(flushPolicy.FlushOnRAM);
DocumentsWriter docsWriter = writer.DocsWriter;
Assert.IsNotNull(docsWriter);
DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
Assert.AreEqual(0, flushControl.FlushBytes, " bytes must be 0 after init");
IndexThread[] threads = new IndexThread[numThreads[i]];
for (int x = 0; x < threads.Length; x++)
{
threads[x] = new IndexThread(numDocs, writer, lineDocFile, false);
threads[x].Start();
}
for (int x = 0; x < threads.Length; x++)
{
threads[x].Join();
}
Assert.AreEqual(0, flushControl.FlushBytes, " all flushes must be due numThreads=" + numThreads[i]);
Assert.AreEqual(numDocumentsToIndex, writer.NumDocs);
Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc);
Assert.IsTrue(flushPolicy.peakDocCountWithoutFlush <= iwc.MaxBufferedDocs, "peak bytes without flush exceeded watermark");
AssertActiveBytesAfter(flushControl);
writer.Dispose();
Assert.AreEqual(0, flushControl.ActiveBytes);
dir.Dispose();
}
}
public void TestAtomicInt32OperatorMinus()
{
for (var j = 0; j < 10; j++)
{
var x = AtomicInt32.From(1000);
var tasks = new Task[100];
for (var i = 0; i < 100; i++)
{
tasks[i] = Task.Factory.StartNew(() => x.Minus(5));
}
Task.WaitAll(tasks);
Assert.Equal(500, x.Value);
}
}
public void AtomicInt32SupportsGetAndAdd()
{
const int expected = 1000;
var i = new AtomicInt32();
for (int j = 0; j < expected; j++)
{
i.GetAndAdd(1);
}
Assert.AreEqual(expected, (int)i);
Assert.AreEqual(expected, i.Value);
}
public void AtomicInt32SupportsPreIncrement()
{
const int expected = 1000;
var i = new AtomicInt32();
for (int j = 0; j < expected; j++)
{
i.PreIncrement();
}
Assert.AreEqual(expected, (int)i);
Assert.AreEqual(expected, i.Value);
}
