public async Task Nested_Action()
{
// Verify that action reentrancy actually works.
var inner1 = false;
var inner2 = false;
var inner3 = false;
using (var mutex = new AsyncReentrantMutex())
{
await mutex.ExecuteActionAsync(
async() =>
{
inner1 = true;
await mutex.ExecuteActionAsync(
async() =>
{
inner2 = true;
await mutex.ExecuteActionAsync(
async() =>
{
inner3 = true;
await Task.CompletedTask;
});
});
});
}
Assert.True(inner1);
Assert.True(inner2);
Assert.True(inner3);
}
public async Task Blocked_Action()
{
// Verify that non-nested action acquistions block.
using (var mutex = new AsyncReentrantMutex())
{
var task1Time = DateTime.MinValue;
var task2Time = DateTime.MinValue;
var task1 = mutex.ExecuteActionAsync(
async() =>
{
task1Time = DateTime.UtcNow;
await Task.Delay(TimeSpan.FromSeconds(2));
});
var task2 = mutex.ExecuteActionAsync(
async() =>
{
task2Time = DateTime.UtcNow;
await Task.Delay(TimeSpan.FromSeconds(2));
});
await task1;
await task2;
// So the two tasks above could execute in any order, but only
// one at a time. With the delay, this means that the recorded
// times should be at least 2 seconds apart.
//
// We'll verify at least a 1 second difference to mitigate any
// clock skew.
Assert.True(task1Time > DateTime.MinValue);
Assert.True(task2Time > DateTime.MinValue);
var delta = task1Time - task2Time;
if (delta < TimeSpan.Zero)
{
delta = -delta;
}
Assert.True(delta >= TimeSpan.FromSeconds(1));
}
}
//---------------------------------------------------------------------
// $todo(jefflill): At least support dependency injection when constructing the controller.
//
// https://github.com/nforgeio/neonKUBE/issues/1589
//
// For some reason, KubeOps does not seem to send RECONCILE events when no changes
// have been detected, even though we return a [ResourceControllerResult] with a
// delay. We're also not seeing any RECONCILE event when the operator starts and
// there are no resources. This used to work before we upgraded to KubeOps v7.0.0-preview2.
//
// NOTE: It's very possible that the old KubeOps behavior was invalid and the current
// behavior actually is correct.
//
// This completely breaks our logic where we expect to see an IDLE event after
// all of the existing resources have been discovered or when no resources were
// discovered.
//
// We're going to work around this with a pretty horrible hack for the time being:
//
// 1. We're going to use the [nextNoChangeReconcileUtc] field to track
// when the next IDLE event should be raised. This will default
// to the current time plus 1 minute when the resource manager is
// constructed. This gives KubeOps a chance to discover existing
// resources before we start raising IDLE events.
//
// 2. After RECONCILE events are handled by the operator's controller,
// we'll reset the [nextNoChangeReconcileUtc] property to be the current
// time plus the [reconciledNoChangeInterval].
//
// 3. The [NoChangeLoop()] method below loops watching for when [nextNoChangeReconcileUtc]
// indicates that an IDLE RECONCILE event should be raised. The loop
// will instantiate an instance of the controller, hardcoding the [IKubernetes]
// constructor parameter for now, rather than supporting real dependency
// injection. We'll then call [ReconcileAsync()] ourselves.
//
// The loop uses [mutex] to ensure that only controller event handler is
// called at a time, so this should be thread/task safe.
//
// 4. We're only going to do this for RECONCILE events right now: our
// operators aren't currently monitoring DELETED or STATUS-MODIFIED
// events and I suspect that KubeOps is probably doing the correct
// thing for these anyway.
//
// PROBLEM:
//
// This hack can result in a problem when KubeOps is not able to watch the resource
// for some reason. The problem is that if this continutes for the first 1 minute
// delay, then the loop below will tragger an IDLE RECONCILE event with no including
// no items, and then the operator could react by deleting any existing related physical
// resources, which would be REALLY BAD.
//
// To mitigate this, I'm going to special case the first IDLE reconcile to query the
// custom resources and only trigger the IDLE reconcile when the query succeeded and
// no items were returned. Otherwise KubeOps may be having trouble communicating with
// Kubernetes or when there are items, we should expect KubeOps to reconcile those for us.
/// <summary>
/// This loop handles raising of <see cref="IOperatorController{TEntity}.IdleAsync()"/>
/// events when there's been no changes to any of the monitored resources.
/// </summary>
/// <returns>The tracking <see cref="Task"/>.</returns>
private async Task IdleLoopAsync()
{
await SyncContext.Clear;
var loopDelay = TimeSpan.FromSeconds(1);
while (!isDisposed && !stopIdleLoop)
{
await Task.Delay(loopDelay);
if (DateTime.UtcNow >= nextIdleReconcileUtc)
{
// Don't send an IDLE RECONCILE while we're when we're not the leader.
if (IsLeader)
{
// We're going to log and otherwise ignore any exceptions thrown by the
// operator's controller or from any members above called by the controller.
await mutex.ExecuteActionAsync(
async() =>
{
try
{
// $todo(jefflill):
//
// We're currently assuming that operator controllers all have a constructor
// that accepts a single [IKubernetes] parameter. We should change this to
// doing actual dependency injection when we have the time.
//
// https://github.com/nforgeio/neonKUBE/issues/1589
var controller = CreateController();
await controller.IdleAsync();
}
catch (OperationCanceledException)
{
// Exit the loop when the [mutex] is disposed which happens
// when the resource manager is disposed.
return;
}
catch (Exception e)
{
options.IdleErrorCounter?.Inc();
log.LogError(e);
}
});
}
nextIdleReconcileUtc = DateTime.UtcNow + options.IdleInterval;
}
}
}
public async Task Dispose_Action()
{
// Verify that [ObjectDisposedException] is thrown for action tasks waiting
// to acquire the mutex.
var mutex = new AsyncReentrantMutex();
try
{
// Hold the mutex for 2 seconds so the tasks below will block.
var task1Acquired = false;
var task2Acquired = false;
var task3Acquired = false;
var task1 = mutex.ExecuteActionAsync(
async() =>
{
task1Acquired = true;
await Task.Delay(TimeSpan.FromSeconds(2));
});
// Wait for [task1] to actually acquire to mutex.
NeonHelper.WaitFor(() => task1Acquired, defaultTimeout);
// Start two new tasks that will block.
var task2 = mutex.ExecuteActionAsync(
async() =>
{
task2Acquired = true;
await Task.CompletedTask;
});
var task3 = mutex.ExecuteActionAsync(
async() =>
{
task3Acquired = true;
await Task.CompletedTask;
});
// Dispose the mutex. We're expecting [task1] to complete normally and
// [task2] and [task3] to fail with an [OperationCancelledException] with
// their actions never being invoked.
mutex.Dispose();
await Assert.ThrowsAsync <ObjectDisposedException>(async() => await task2);
Assert.False(task2Acquired);
await Assert.ThrowsAsync <ObjectDisposedException>(async() => await task3);
Assert.False(task3Acquired);
await task1;
Assert.True(task1Acquired);
}
finally
{
// Disposing this again shouldn't cause any trouble.
mutex.Dispose();
}
}