public virtual void TestRandom()
{
DocumentsWriterStallControl ctrl = new DocumentsWriterStallControl();
ctrl.UpdateStalled(false);
ThreadClass[] stallThreads = new ThreadClass[AtLeast(3)];
for (int i = 0; i < stallThreads.Length; i++)
{
int stallProbability = 1 + Random().Next(10);
stallThreads[i] = new ThreadAnonymousInnerClassHelper(this, ctrl, stallProbability);
}
Start(stallThreads);
long time = DateTime.Now.Millisecond;
/*
* use a 100 sec timeout to make sure we not hang forever. join will fail in
* that case
*/
while ((DateTime.Now.Millisecond - time) < 100 * 1000 && !Terminated(stallThreads))
{
ctrl.UpdateStalled(false);
if (Random().NextBoolean())
{
Thread.@Yield();
}
else
{
Thread.Sleep(1);
}
}
Join(stallThreads);
}
internal DocumentsWriterFlushControl(DocumentsWriter documentsWriter, LiveIndexWriterConfig config, BufferedUpdatesStream bufferedUpdatesStream)
{
this.infoStream = config.InfoStream;
this.stallControl = new DocumentsWriterStallControl();
this.perThreadPool = documentsWriter.perThreadPool;
this.flushPolicy = documentsWriter.flushPolicy;
this.config = config;
this.hardMaxBytesPerDWPT = config.RAMPerThreadHardLimitMB * 1024 * 1024;
this.documentsWriter = documentsWriter;
this.bufferedUpdatesStream = bufferedUpdatesStream;
}
public virtual void TestSimpleStall()
{
DocumentsWriterStallControl ctrl = new DocumentsWriterStallControl();
ctrl.UpdateStalled(false);
ThreadClass[] waitThreads = WaitThreads(AtLeast(1), ctrl);
Start(waitThreads);
Assert.IsFalse(ctrl.HasBlocked());
Assert.IsFalse(ctrl.AnyStalledThreads());
Join(waitThreads);
// now stall threads and wake them up again
ctrl.UpdateStalled(true);
waitThreads = WaitThreads(AtLeast(1), ctrl);
Start(waitThreads);
AwaitState(ThreadState.WaitSleepJoin, waitThreads);
Assert.IsTrue(ctrl.HasBlocked());
Assert.IsTrue(ctrl.AnyStalledThreads());
ctrl.UpdateStalled(false);
Assert.IsFalse(ctrl.AnyStalledThreads());
Join(waitThreads);
}
public virtual void TestSimpleStall()
{
DocumentsWriterStallControl ctrl = new DocumentsWriterStallControl();
ctrl.UpdateStalled(false);
ThreadClass[] waitThreads = WaitThreads(AtLeast(1), ctrl);
Start(waitThreads);
Assert.IsFalse(ctrl.HasBlocked());
Assert.IsFalse(ctrl.AnyStalledThreads());
Join(waitThreads);
// now stall threads and wake them up again
ctrl.UpdateStalled(true);
waitThreads = WaitThreads(AtLeast(1), ctrl);
Start(waitThreads);
AwaitState(ThreadState.WaitSleepJoin, waitThreads);
Assert.IsTrue(ctrl.HasBlocked());
Assert.IsTrue(ctrl.AnyStalledThreads());
ctrl.UpdateStalled(false);
Assert.IsFalse(ctrl.AnyStalledThreads());
Join(waitThreads);
}
private void AssertState(int numReleasers, int numStallers, int numWaiters, ThreadClass[] threads, DocumentsWriterStallControl ctrl)
{
int millisToSleep = 100;
while (true)
{
if (ctrl.HasBlocked() && ctrl.Healthy)
{
for (int n = numReleasers + numStallers; n < numReleasers + numStallers + numWaiters; n++)
{
if (ctrl.IsThreadQueued(threads[n]))
{
if (millisToSleep < 60000)
{
Thread.Sleep(millisToSleep);
millisToSleep *= 2;
break;
}
else
{
Assert.Fail("control claims no stalled threads but waiter seems to be blocked ");
}
}
}
break;
}
else
{
break;
}
}
}
public virtual void TestAccquireReleaseRace()
{
DocumentsWriterStallControl ctrl = new DocumentsWriterStallControl();
ctrl.UpdateStalled(false);
AtomicBoolean stop = new AtomicBoolean(false);
AtomicBoolean checkPoint = new AtomicBoolean(true);
int numStallers = AtLeast(1);
int numReleasers = AtLeast(1);
int numWaiters = AtLeast(1);
var sync = new Synchronizer(numStallers + numReleasers, numStallers + numReleasers + numWaiters);
var threads = new ThreadClass[numReleasers + numStallers + numWaiters];
IList <Exception> exceptions = new SynchronizedCollection <Exception>();
for (int i = 0; i < numReleasers; i++)
{
threads[i] = new Updater(stop, checkPoint, ctrl, sync, true, exceptions);
}
for (int i = numReleasers; i < numReleasers + numStallers; i++)
{
threads[i] = new Updater(stop, checkPoint, ctrl, sync, false, exceptions);
}
for (int i = numReleasers + numStallers; i < numReleasers + numStallers + numWaiters; i++)
{
threads[i] = new Waiter(stop, checkPoint, ctrl, sync, exceptions);
}
Start(threads);
int iters = AtLeast(10000);
float checkPointProbability = TEST_NIGHTLY ? 0.5f : 0.1f;
for (int i = 0; i < iters; i++)
{
if (checkPoint.Get())
{
Assert.IsTrue(sync.UpdateJoin.@await(new TimeSpan(0, 0, 0, 10)), "timed out waiting for update threads - deadlock?");
if (exceptions.Count > 0)
{
foreach (Exception throwable in exceptions)
{
Console.WriteLine(throwable.ToString());
Console.Write(throwable.StackTrace);
}
Assert.Fail("got exceptions in threads");
}
if (ctrl.HasBlocked() && ctrl.Healthy)
{
AssertState(numReleasers, numStallers, numWaiters, threads, ctrl);
}
checkPoint.Set(false);
sync.Waiter.countDown();
sync.LeftCheckpoint.@await();
}
Assert.IsFalse(checkPoint.Get());
Assert.AreEqual(0, sync.Waiter.Remaining);
if (checkPointProbability >= (float)Random().NextDouble())
{
sync.Reset(numStallers + numReleasers, numStallers + numReleasers + numWaiters);
checkPoint.Set(true);
}
}
if (!checkPoint.Get())
{
sync.Reset(numStallers + numReleasers, numStallers + numReleasers + numWaiters);
checkPoint.Set(true);
}
Assert.IsTrue(sync.UpdateJoin.@await(new TimeSpan(0, 0, 0, 10)));
AssertState(numReleasers, numStallers, numWaiters, threads, ctrl);
checkPoint.Set(false);
stop.Set(true);
sync.Waiter.countDown();
sync.LeftCheckpoint.@await();
for (int i = 0; i < threads.Length; i++)
{
ctrl.UpdateStalled(false);
threads[i].Join(2000);
if (threads[i].IsAlive && threads[i] is Waiter)
{
if (threads[i].State == ThreadState.WaitSleepJoin)
{
Assert.Fail("waiter is not released - anyThreadsStalled: " + ctrl.AnyStalledThreads());
}
}
}
}
public ThreadAnonymousInnerClassHelper(TestDocumentsWriterStallControl outerInstance, DocumentsWriterStallControl ctrl, int stallProbability)
{
this.OuterInstance = outerInstance;
this.Ctrl = ctrl;
this.StallProbability = stallProbability;
}
public ThreadAnonymousInnerClassHelper(DocumentsWriterStallControl ctrl, int stallProbability)
{
this.ctrl = ctrl;
this.stallProbability = stallProbability;
}
public ThreadAnonymousInnerClassHelper2(DocumentsWriterStallControl ctrl)
{
this.Ctrl = ctrl;
}
public ThreadAnonymousInnerClassHelper(DocumentsWriterStallControl ctrl, int stallProbability)
{
this.Ctrl = ctrl;
this.StallProbability = stallProbability;
}
public virtual void TestAccquireReleaseRace()
{
DocumentsWriterStallControl ctrl = new DocumentsWriterStallControl();
ctrl.UpdateStalled(false);
AtomicBoolean stop = new AtomicBoolean(false);
AtomicBoolean checkPoint = new AtomicBoolean(true);
int numStallers = AtLeast(1);
int numReleasers = AtLeast(1);
int numWaiters = AtLeast(1);
var sync = new Synchronizer(numStallers + numReleasers, numStallers + numReleasers + numWaiters);
var threads = new ThreadJob[numReleasers + numStallers + numWaiters];
IList <Exception> exceptions = new SynchronizedList <Exception>();
for (int i = 0; i < numReleasers; i++)
{
threads[i] = new Updater(stop, checkPoint, ctrl, sync, true, exceptions);
}
for (int i = numReleasers; i < numReleasers + numStallers; i++)
{
threads[i] = new Updater(stop, checkPoint, ctrl, sync, false, exceptions);
}
for (int i = numReleasers + numStallers; i < numReleasers + numStallers + numWaiters; i++)
{
threads[i] = new Waiter(stop, checkPoint, ctrl, sync, exceptions);
}
Start(threads);
int iters = AtLeast(10000);
//float checkPointProbability = TestNightly ? 0.5f : 0.1f;
// LUCENENET specific - reduced probabliltiy on x86 to prevent it from timing out.
float checkPointProbability = TestNightly ? (Lucene.Net.Util.Constants.RUNTIME_IS_64BIT ? 0.5f : 0.25f) : 0.1f;
for (int i = 0; i < iters; i++)
{
if (checkPoint)
{
Assert.IsTrue(sync.updateJoin.Wait(TimeSpan.FromSeconds(10)), "timed out waiting for update threads - deadlock?");
if (exceptions.Count > 0)
{
foreach (Exception throwable in exceptions)
{
Console.WriteLine(throwable.ToString());
Console.Write(throwable.StackTrace);
}
Assert.Fail("got exceptions in threads");
}
if (ctrl.HasBlocked && ctrl.IsHealthy)
{
AssertState(numReleasers, numStallers, numWaiters, threads, ctrl);
}
checkPoint.Value = (false);
sync.waiter.Signal();
sync.leftCheckpoint.Wait();
}
Assert.IsFalse(checkPoint);
Assert.AreEqual(0, sync.waiter.CurrentCount);
if (checkPointProbability >= (float)Random.NextDouble())
{
sync.Reset(numStallers + numReleasers, numStallers + numReleasers + numWaiters);
checkPoint.Value = (true);
}
}
if (!checkPoint)
{
sync.Reset(numStallers + numReleasers, numStallers + numReleasers + numWaiters);
checkPoint.Value = (true);
}
Assert.IsTrue(sync.updateJoin.Wait(new TimeSpan(0, 0, 0, 10)));
AssertState(numReleasers, numStallers, numWaiters, threads, ctrl);
checkPoint.Value = (false);
stop.Value = (true);
sync.waiter.Signal();
sync.leftCheckpoint.Wait();
for (int i = 0; i < threads.Length; i++)
{
ctrl.UpdateStalled(false);
threads[i].Join(2000);
if (threads[i].IsAlive && threads[i] is Waiter)
{
if (threads[i].State == ThreadState.WaitSleepJoin)
{
Assert.Fail("waiter is not released - anyThreadsStalled: " + ctrl.AnyStalledThreads());
}
}
}
}
private void AssertState(int numReleasers, int numStallers, int numWaiters, ThreadClass[] threads, DocumentsWriterStallControl ctrl)
{
int millisToSleep = 100;
while (true)
{
if (ctrl.HasBlocked() && ctrl.Healthy)
{
for (int n = numReleasers + numStallers; n < numReleasers + numStallers + numWaiters; n++)
{
if (ctrl.IsThreadQueued(threads[n]))
{
if (millisToSleep < 60000)
{
Thread.Sleep(millisToSleep);
millisToSleep *= 2;
break;
}
else
{
Assert.Fail("control claims no stalled threads but waiter seems to be blocked ");
}
}
}
break;
}
else
{
break;
}
}
}
public virtual void TestAccquireReleaseRace()
{
DocumentsWriterStallControl ctrl = new DocumentsWriterStallControl();
ctrl.UpdateStalled(false);
AtomicBoolean stop = new AtomicBoolean(false);
AtomicBoolean checkPoint = new AtomicBoolean(true);
int numStallers = AtLeast(1);
int numReleasers = AtLeast(1);
int numWaiters = AtLeast(1);
var sync = new Synchronizer(numStallers + numReleasers, numStallers + numReleasers + numWaiters);
var threads = new ThreadClass[numReleasers + numStallers + numWaiters];
IList<Exception> exceptions = new SynchronizedCollection<Exception>();
for (int i = 0; i < numReleasers; i++)
{
threads[i] = new Updater(stop, checkPoint, ctrl, sync, true, exceptions);
}
for (int i = numReleasers; i < numReleasers + numStallers; i++)
{
threads[i] = new Updater(stop, checkPoint, ctrl, sync, false, exceptions);
}
for (int i = numReleasers + numStallers; i < numReleasers + numStallers + numWaiters; i++)
{
threads[i] = new Waiter(stop, checkPoint, ctrl, sync, exceptions);
}
Start(threads);
int iters = AtLeast(10000);
float checkPointProbability = TEST_NIGHTLY ? 0.5f : 0.1f;
for (int i = 0; i < iters; i++)
{
if (checkPoint.Get())
{
Assert.IsTrue(sync.UpdateJoin.@await(new TimeSpan(0, 0, 0, 10)), "timed out waiting for update threads - deadlock?");
if (exceptions.Count > 0)
{
foreach (Exception throwable in exceptions)
{
Console.WriteLine(throwable.ToString());
Console.Write(throwable.StackTrace);
}
Assert.Fail("got exceptions in threads");
}
if (ctrl.HasBlocked() && ctrl.Healthy)
{
AssertState(numReleasers, numStallers, numWaiters, threads, ctrl);
}
checkPoint.Set(false);
sync.Waiter.countDown();
sync.LeftCheckpoint.@await();
}
Assert.IsFalse(checkPoint.Get());
Assert.AreEqual(0, sync.Waiter.Remaining);
if (checkPointProbability >= (float)Random().NextDouble())
{
sync.Reset(numStallers + numReleasers, numStallers + numReleasers + numWaiters);
checkPoint.Set(true);
}
}
if (!checkPoint.Get())
{
sync.Reset(numStallers + numReleasers, numStallers + numReleasers + numWaiters);
checkPoint.Set(true);
}
Assert.IsTrue(sync.UpdateJoin.@await(new TimeSpan(0, 0, 0, 10)));
AssertState(numReleasers, numStallers, numWaiters, threads, ctrl);
checkPoint.Set(false);
stop.Set(true);
sync.Waiter.countDown();
sync.LeftCheckpoint.@await();
for (int i = 0; i < threads.Length; i++)
{
ctrl.UpdateStalled(false);
threads[i].Join(2000);
if (threads[i].IsAlive && threads[i] is Waiter)
{
if (threads[i].State == ThreadState.WaitSleepJoin)
{
Assert.Fail("waiter is not released - anyThreadsStalled: " + ctrl.AnyStalledThreads());
}
}
}
}
public static ThreadClass[] WaitThreads(int num, DocumentsWriterStallControl ctrl)
{
ThreadClass[] array = new ThreadClass[num];
for (int i = 0; i < array.Length; i++)
{
array[i] = new ThreadAnonymousInnerClassHelper2(ctrl);
}
return array;
}
public ThreadAnonymousInnerClassHelper(TestDocumentsWriterStallControl outerInstance, DocumentsWriterStallControl ctrl, int stallProbability)
{
this.OuterInstance = outerInstance;
this.Ctrl = ctrl;
this.StallProbability = stallProbability;
}
public ThreadAnonymousClass2(DocumentsWriterStallControl ctrl)
{
this.ctrl = ctrl;
}
public ThreadAnonymousInnerClassHelper2(DocumentsWriterStallControl ctrl)
{
this.Ctrl = ctrl;
}
public Waiter(AtomicBoolean stop, AtomicBoolean checkPoint, DocumentsWriterStallControl ctrl, Synchronizer sync, IList<Exception> exceptions)
: base("waiter")
{
this.Stop = stop;
this.CheckPoint = checkPoint;
this.Ctrl = ctrl;
this.Sync = sync;
this.Exceptions = exceptions;
}
