public async Task TaskRunnerBase_DoesNotShutDownFaultedThread_AfterRepeatedFaultsWithOverriddenRepeatedTaskFailureTime()
{
var counter = 0;
var exceptionHandlerCounter = 0;
var sut = new TaskRunnerActual((ct =>
{
return(new List <TaskRunnerInformationBase>()
{
GetTaskRunnerInformation <object>(
() => { counter++; throw (new Exception()); },
(ex, info) => { exceptionHandlerCounter++; return Task.FromResult(0); },
(ex, ts) => { return TimeSpan.FromDays(1); },
null,
ct
)
});
})
);
sut.Settings.MaxUnhandledExceptions = 1;
// This setting would allow the task be retried forever if respected
sut.Settings.DefaultRepeatedTaskFailureTime = TimeSpan.FromDays(-1);
await RunSUT(sut, () =>
{
Assert.True(counter > sut.Settings.MaxUnhandledExceptions + 1, $"Expected thread to be tried once and retried many times, was run {counter} times instead");
//We expect the exception handler NOT to run on counter runs up to MaxUnhandledExceptions.
//For example, with MUH = 2, we expect an initial run (counter = 0), and 1 more (counter = 1) which do not call the exception handler
var expectedExceptionHandlerCount = counter - (sut.Settings.MaxUnhandledExceptions);
Assert.True(exceptionHandlerCounter == expectedExceptionHandlerCount, $"Expected exception handler to be run {expectedExceptionHandlerCount} times, was ran {exceptionHandlerCounter} times instead");
}).ConfigureAwait(false);
}
public async Task TaskRunnerBase_DoesNotShutDownFaultedThread_AfterRepeatedSeparateFaultsWithOverriddenRepeatedTaskFailureTime()
{
var counter = 0;
var exceptionHandlerCounter = 0;
var sut = new TaskRunnerActual((ct =>
{
return(new List <TaskRunnerInformationBase>()
{
GetTaskRunnerInformation <object>(
() => { counter++; throw (new Exception()); },
(ex, info) => { exceptionHandlerCounter++; return Task.FromResult(0); },
(ex, ts) => { return TimeSpan.FromDays(-1); },
null,
ct
),
});
})
);
await RunSUT(sut, () =>
{
Assert.True(counter > sut.Settings.MaxUnhandledExceptions + 1, $"Expected thread to be tried once and retried many times, was run {counter} times instead");
Assert.True(exceptionHandlerCounter == 0, $"Expected exception handler to not be run, was ran {exceptionHandlerCounter} times instead");
}).ConfigureAwait(false);
}
public async Task TaskRunnerBase_DoesNotThrow_FailureInRepeatedTaskFailureTimeHandler()
{
var counter = 0;
var exceptionHandlerCounter = 0;
var sut = new TaskRunnerActual((ct =>
{
return(new List <TaskRunnerInformationBase>()
{
GetTaskRunnerInformation <object>(
() => { counter++; throw new Exception(); },
(ex, info) => { exceptionHandlerCounter++; return Task.FromResult(0); },
(ex, ts) => { throw new Exception(); },
null,
ct
)
});
})
);
await RunSUT(sut, () =>
{
Assert.True(counter > sut.Settings.MaxUnhandledExceptions + 1, $"Expected thread to be tried once and retried many times, was run {counter} times instead");
//We expect the exception handler NOT to run on counter runs up to MaxUnhandledExceptions.
//For example, with MUH = 2, we expect an initial run (counter = 0), and 1 more (counter = 1) which do not call the exception handler
var expectedExceptionHandlerCount = counter - (sut.Settings.MaxUnhandledExceptions);
Assert.True(exceptionHandlerCounter == expectedExceptionHandlerCount, $"Expected exception handler to be run {expectedExceptionHandlerCount} times, was ran {exceptionHandlerCounter} times instead");
}).ConfigureAwait(false);
}
public async Task TaskRunnerBase_Throws_PartitionKillerException()
{
var knownKillerExceptions = new List <Exception>()
{
new OutOfMemoryException()
};
foreach (var exception in knownKillerExceptions)
{
var sut = new TaskRunnerActual((ct =>
{
return(new List <TaskRunnerInformationBase>()
{
GetTaskRunnerInformation <object>(() => { throw(exception); }, null, null, null, ct)
});
}));
var task = Task.Run(() => sut.OuterRun(CancellationToken.None));
var e = await Assert.ThrowsAsync <AggregateException>(async() =>
{
await task.ConfigureAwait(false);
});
Assert.True(e.InnerException.GetType() == exception.GetType(),
$"Expected exception to be {exception.GetType()}, was {task.Exception.InnerException.GetType()} instead");
}
}
public async Task TaskRunnerBase_Cancels_AfterCancellationRequested()
{
var counter = 0;
var sut = new TaskRunnerActual((ct =>
{
return(new List <TaskRunnerInformationBase>()
{
GetTaskRunnerInformation <object>(async() => { counter++; await Task.Delay(Timeout.Infinite); }, null, null, null, ct)
});
}));
var source = new CancellationTokenSource();
var task = Task.Run(() => sut.OuterRun(source.Token));
await Task.Delay(TimeSpan.FromSeconds(1)).ConfigureAwait(false);
source.Cancel();
Assert.True(counter == 1, $"Expected thread to only run once, ran {counter} times instead");
Assert.True(sut.InitialCancellationToken.IsCancellationRequested == true, $"Expected cancellation token passed into SUT to be cancelled");
Assert.True(sut.LinkedCancellationToken.IsCancellationRequested == true, $"Expected linked cancellation token created in SUT to be cancelled");
await task.AwaitCompleteOrCancel();
}
public async Task TaskRunnerBase_DoesNotShutDownFaultedThread_AfterRepeatedSeparateFaults()
{
var counter = 0;
var exceptionHandlerCounter = 0;
var dateTimeProvider = new Mock <IDateTimeProvider>();
dateTimeProvider.Setup(x => x.UtcNow).Returns(() => { return(DateTime.UtcNow.AddDays(counter)); });
var sut = new TaskRunnerActual((ct =>
{
return(new List <TaskRunnerInformationBase>()
{
GetTaskRunnerInformation <object>(
() => { counter++; throw (new Exception()); },
(ex, info) => { exceptionHandlerCounter++; return Task.FromResult(0); },
null,
null,
ct
),
});
}),
dateTimeProvider.Object
);
await RunSUT(sut, () =>
{
Assert.True(counter > sut.Settings.MaxUnhandledExceptions + 1, $"Expected thread to be tried once and retried many times, was run {counter} times instead");
Assert.True(exceptionHandlerCounter == 0, $"Expected exception handler to not be run, was ran {exceptionHandlerCounter} times instead");
}).ConfigureAwait(false);
}
public async Task TaskRunnerBase_DoesNotThrow_ExceptionsFromExceptionHandler()
{
var counter = 0;
var exceptionHandlerCounter = 0;
var sut = new TaskRunnerActual((ct =>
{
return(new List <TaskRunnerInformationBase>()
{
GetTaskRunnerInformation <object>(
() => { counter++; throw new Exception(); },
(ex, info) => { exceptionHandlerCounter++; throw new Exception(); },
null,
null,
ct
)
});
})
);
await RunSUT(sut, () =>
{
VerifySubmittedLogWarning(sut.MockLog, "ClearTasks", "", Times.Never());
VerifySubmittedLogError(sut.MockLog, "RunTask", Times.Exactly(counter - sut.Settings.MaxUnhandledExceptions + 1));
VerifySubmittedLogWarning(sut.MockLog, "RunTask", "Unhandled exception in final exception handler", Times.Exactly(counter - sut.Settings.MaxUnhandledExceptions));
Assert.True(exceptionHandlerCounter == counter - sut.Settings.MaxUnhandledExceptions, $"Expected exception handler to run {counter - sut.Settings.MaxUnhandledExceptions} times, ran {exceptionHandlerCounter} times instead");
}).ConfigureAwait(false);
}
private async Task RunSUT(TaskRunnerActual sut, Action code)
{
var source = new CancellationTokenSource();
var task = Task.Run(() => sut.Run(source.Token));
await Task.Delay(TimeSpan.FromSeconds(5)).ConfigureAwait(false);
code();
source.Cancel();
await task.AwaitCompleteOrCancel();
}
public async Task TaskRunnerBase_Awaits_AfterAllComplete()
{
var counter = 0;
var sut = new TaskRunnerActual((ct =>
{
return(new List <TaskRunnerInformationBase>()
{
GetTaskRunnerInformation <object>(() => { counter++; return Task.FromResult(0); }, null, null, null, ct),
});
}));
await RunSUT(sut, () =>
{
Assert.True(counter == 1, $"Expected thread to only run once, ran {counter} times instead");
}).ConfigureAwait(false);
}
public async Task TaskRunnerBase_DoesNotShutDownFaultedThread_AfterRepeatedSeparateFaultsWithOccasionalSequentialFaults()
{
var counter = 0;
var exceptionHandlerCounter = 0;
var dateTimeProvider = new Mock <IDateTimeProvider>();
dateTimeProvider.Setup(x => x.UtcNow).Returns(() => {
// Half the time, return a value that makes it look like it's been a long time since the thread faulted
//Note that the "+1" is required here to prevent the spoofed date occuring on the same counter as the
//first exceptionCount that is not under the MaxUnhandledExceptions, as this would erroneously cause
//the test to fail (as only the UtcNow used for the taskStartTime would be increased by the mock,
//not the one within ShouldRespondToException).
if ((counter + 1) % 2 == 0)
{
return(DateTime.UtcNow.AddDays(counter));
}
// The rest of the time, return the current time, which will look like it's been no time since the task faulted
return(DateTime.UtcNow);
});
var sut = new TaskRunnerActual((ct =>
{
return(new List <TaskRunnerInformationBase>()
{
GetTaskRunnerInformation <object>(
() => { counter++; throw (new Exception()); },
(ex, info) => { exceptionHandlerCounter++; return Task.FromResult(0); },
null,
null,
ct
),
});
}),
dateTimeProvider.Object
);
await RunSUT(sut, () =>
{
Assert.True(counter > sut.Settings.MaxUnhandledExceptions + 1, $"Expected thread to be tried once and retried many times, was run {counter} times instead");
Assert.True(exceptionHandlerCounter == 0, $"Expected exception handler to not be run, was ran {exceptionHandlerCounter} times instead");
}).ConfigureAwait(false);
}
public async Task TaskRunnerBase_OnlyRestartsFaultedThread_AfterFault()
{
var counter = 0;
var exceptionCounter = 0;
var sut = new TaskRunnerActual((ct =>
{
return(new List <TaskRunnerInformationBase>()
{
GetTaskRunnerInformation <object>(() => { counter++; return Task.FromResult(0); }, null, null, null, ct),
GetTaskRunnerInformation <object>(async() => { exceptionCounter++; await Task.Delay(TimeSpan.FromSeconds(1)); throw(new Exception()); }, null, null, null, ct),
});
}));
await RunSUT(sut, () =>
{
Assert.True(counter == 1, $"Expected thread to only run once, ran {counter} times instead");
Assert.True(exceptionCounter == sut.Settings.MaxUnhandledExceptions + 1, $"Expected exception handler to run {sut.Settings.MaxUnhandledExceptions + 1} times, was run {exceptionCounter} times instead");
}).ConfigureAwait(false);
}
