public void Should_Return_Correct_Retry_Number_For_Grace_Period(int minimumBackOffTime, int maximumBackOffTime, int maximumRetries, int expectedRetries)
{
// GIVEN
var servicebusConfiguration = Options.Create(new ServiceBusConfiguration()
{
SbMaximumAllowedRetries = maximumRetries, SbMinimumAllowedBackoffTime = minimumBackOffTime, SbMaximumAllowedBackoffTime = maximumBackOffTime, EntityPath = "test", Namespace = "test"
});
var servicebusMonitor = new SubscriptionMonitor(servicebusConfiguration);
// WHEN
var calculatedRetries = servicebusMonitor.CalculateMaxNumberOfExceptionsByGracePeriod();
// THEN
Check.That(expectedRetries).IsEqualTo(calculatedRetries);
}
public void IsConnectionAlive_Should_Return_Correct_Value_For_Default_Configuration(int exceptionsNumber, bool healthStatus)
{
// GIVEN
var subscriptionMonitor = new SubscriptionMonitor(servicebusConfiguration);
var exceptionToReport = new Exception();
// WHEN
for (int i = 0; i < exceptionsNumber; i++)
{
subscriptionMonitor.ReportException(exceptionToReport);
}
// THEN
Check.That(subscriptionMonitor.IsConnectionAlive()).IsEqualTo(healthStatus);
}
public async Task Should_Restart_Pod_When_Transient_Errors_Exceed_Grace_Period()
{
const int podPollingInterval = 3_000;
var servicebusConfiguration = Options.Create(new ServiceBusConfiguration()
{
SbMaximumAllowedBackoffTime = 5,
SbMaximumAllowedRetries = 2,
SbMinimumAllowedBackoffTime = 0,
SbMonitorGracePeriod = 45,
EntityPath = "test",
Namespace = "test"
});
var subscriptionMonitor = new SubscriptionMonitor(servicebusConfiguration);
var timeoutForPodRestart = TimeSpan.FromMilliseconds(TimeoutBeforePodRestart);
using var cts = new CancellationTokenSource((int)timeoutForPodRestart.TotalMilliseconds);
var taskErrorReporter = Task.Run(
async() =>
{
var config = servicebusConfiguration.Value;
var retry = new RetryExponential(TimeSpan.FromSeconds(config.SbMinimumAllowedBackoffTime), TimeSpan.FromSeconds(config.SbMaximumAllowedBackoffTime), config.SbMaximumAllowedRetries);
while (!cts.IsCancellationRequested)
{
try
{
await retry.RunOperation(() => throw new ServiceBusCommunicationException("timeout"), TimeSpan.FromSeconds(10));
}
catch (ServiceBusCommunicationException ex)
{
subscriptionMonitor.ReportException(ex);
}
}
},
cts.Token);
var podRestarted = false;
var taskHealthProbe = Task.Run(
async() =>
{
var downCount = 0;
while (!cts.IsCancellationRequested && taskErrorReporter.Status <= TaskStatus.Running)
{
if (!cts.IsCancellationRequested)
{
await Task.Delay(podPollingInterval, cts.Token);
var healthState = subscriptionMonitor.IsConnectionAlive();
if (!healthState)
{
downCount++;
}
else
{
downCount = 0;
}
}
if (downCount == 3)
{
podRestarted = true;
cts.Cancel();
break;
}
}
},
cts.Token);
try
{
await Task.WhenAll(taskErrorReporter, taskHealthProbe);
}
catch (OperationCanceledException)
{
}
Check.That(podRestarted).IsEqualTo(true);
}
