public async Task RetryPolicy_FailsWhenWorkFailsWithNonTransientException()
{
var cancellationToken = new CancellationToken();
var logger = new Moq.Mock <ILogger <AzureSqlClient> >().Object;
var sqlCnFactoryLogger = new Moq.Mock <ILogger <AzureSqlDbConnectionFactory> >().Object;
var sqlDbConnectionFactory = new AzureSqlDbConnectionFactory("read-write", "read-only", sqlCnFactoryLogger);
var azureSqlClient = new AzureSqlClient(sqlDbConnectionFactory, logger);
var retryPolicy = azureSqlClient.GetAsyncRetryPolicy();
var retryCount = 0;
var policyResult = await retryPolicy.ExecuteAndCaptureAsync(
ct => ++ retryCount > 1?Task.CompletedTask : throw new ApplicationException(),
cancellationToken);
Assert.AreEqual(1, retryCount);
Assert.AreEqual(OutcomeType.Failure, policyResult.Outcome);
Assert.IsNotNull(policyResult.ExceptionType);
Assert.IsNotNull(policyResult.FinalException);
Assert.AreEqual(ExceptionType.Unhandled, policyResult.ExceptionType.Value);
Assert.IsInstanceOfType(policyResult.FinalException, typeof(ApplicationException));
}
public async Task RetryPolicy_RecoversSuccessfullyWhenWorkSucceedsAfterRetryUponTransientException()
{
var cancellationToken = new CancellationToken();
var logger = new Moq.Mock <ILogger <AzureSqlClient> >().Object;
var sqlCnFactoryLogger = new Moq.Mock <ILogger <AzureSqlDbConnectionFactory> >().Object;
var sqlDbConnectionFactory = new AzureSqlDbConnectionFactory("read-write", "read-only", sqlCnFactoryLogger);
var azureSqlClient = new AzureSqlClient(sqlDbConnectionFactory, logger);
var retryPolicy = azureSqlClient.GetAsyncRetryPolicy();
var retryCount = 0;
var policyResult = await retryPolicy.ExecuteAndCaptureAsync(
ct => ++ retryCount > 1?Task.CompletedTask : throw new TimeoutException(),
cancellationToken);
Assert.AreEqual(2, retryCount);
Assert.AreEqual(OutcomeType.Successful, policyResult.Outcome);
Assert.IsNull(policyResult.ExceptionType);
Assert.IsNull(policyResult.FinalException);
}
public async Task AsyncResiliencyPolicy_IsUnemployedWhenWorkSucceeds()
{
var cancellationToken = new CancellationToken();
var logger = new Moq.Mock <ILogger <AzureSqlClient> >().Object;
var sqlCnFactoryLogger = new Moq.Mock <ILogger <AzureSqlDbConnectionFactory> >().Object;
var sqlDbConnectionFactory = new AzureSqlDbConnectionFactory("read-write", "read-only", sqlCnFactoryLogger);
var azureSqlClient = new AzureSqlClient(sqlDbConnectionFactory, logger);
var retryPolicy = azureSqlClient.GetAsyncRetryPolicy();
var policyKey = Guid.NewGuid().ToString();
var globalResiliencyPolicy = AzureSqlClient.GetAsyncGlobalResiliencyPolicyFor(policyKey);
var policy = Policy.WrapAsync(retryPolicy, globalResiliencyPolicy);
var policyResult = await policy.ExecuteAndCaptureAsync(ct => Task.CompletedTask, cancellationToken);
Assert.AreEqual(OutcomeType.Successful, policyResult.Outcome);
Assert.IsNull(policyResult.ExceptionType);
Assert.IsNull(policyResult.FinalException);
}
public async Task RetryPolicy_FailsAfterMaxAttemptsAfterRetryOnTransientExceptions()
{
var cancellationToken = new CancellationToken();
var logger = new Moq.Mock <ILogger <AzureSqlClient> >().Object;
var sqlCnFactoryLogger = new Moq.Mock <ILogger <AzureSqlDbConnectionFactory> >().Object;
var sqlDbConnectionFactory = new AzureSqlDbConnectionFactory("read-write", "read-only", sqlCnFactoryLogger);
var azureSqlClient = new AzureSqlClient(sqlDbConnectionFactory, logger);
var retryPolicy = azureSqlClient.GetAsyncRetryPolicy();
var retryCount = 0;
var policyResult = await retryPolicy.ExecuteAndCaptureAsync(
ct => { retryCount++; throw new TimeoutException(); },
cancellationToken);
Assert.AreEqual(1 + 5, retryCount);
Assert.AreEqual(OutcomeType.Failure, policyResult.Outcome);
Assert.IsNotNull(policyResult.ExceptionType);
Assert.IsNotNull(policyResult.FinalException);
Assert.AreEqual(ExceptionType.HandledByThisPolicy, policyResult.ExceptionType.Value); // Exception type the policy handles but too many of them to keep going
Assert.IsInstanceOfType(policyResult.FinalException, typeof(TimeoutException));
}
public async Task RetryPolicy_EmploysExponentialBackoffUponTransientExceptions()
{
var cancellationToken = new CancellationToken();
var logger = new Moq.Mock <ILogger <AzureSqlClient> >().Object;
var sqlCnFactoryLogger = new Moq.Mock <ILogger <AzureSqlDbConnectionFactory> >().Object;
var sqlDbConnectionFactory = new AzureSqlDbConnectionFactory("read-write", "read-only", sqlCnFactoryLogger);
var azureSqlClient = new AzureSqlClient(sqlDbConnectionFactory, logger);
var retryPolicy = azureSqlClient.GetAsyncRetryPolicy();
var retryCount = 0;
var xs = new TimeSpan[6];
var sw = new Stopwatch();
sw.Start();
var policyResult = await retryPolicy.ExecuteAndCaptureAsync(
ct => { xs[retryCount++] = sw.Elapsed; throw new TimeoutException(); },
cancellationToken);
sw.Stop();
// First delay in array can be ignored because it's just the amount of time until the first invocation - ie not a retry
var firstDelay = xs[1] - xs[0];
var secondDelay = xs[2] - xs[1];
var thirdDelay = xs[3] - xs[2];
var fourthDelay = xs[4] - xs[3];
var fifthDelay = xs[5] - xs[4];
// Interestingly, it seems expoenential + jitter in Polly land doesn't actually mean exponential .. just pretty close.
// Could be a side effect of what it uses to time/sleep (ie best efforts), but in all practicality it doesn't actually make
// and difference to how we are using it; just makes it harder to test
const int TOLERANCE_IN_MS = 1000;
Assert.IsTrue(thirdDelay.TotalMilliseconds > (secondDelay * 2).TotalMilliseconds - TOLERANCE_IN_MS, $"Expected third delay '{thirdDelay}' to be at least twice as long as second '{secondDelay}'");
Assert.IsTrue(fourthDelay.TotalMilliseconds > (thirdDelay * 2).TotalMilliseconds - TOLERANCE_IN_MS, $"Expected fourth delay '{fourthDelay}' to be at least twice as long as third '{thirdDelay}'");
Assert.IsTrue(fifthDelay.TotalMilliseconds > (fourthDelay * 2).TotalMilliseconds - TOLERANCE_IN_MS, $"Expected fifth delay '{fifthDelay}' to be at least twice as long as fourth '{fourthDelay}'");
}
public async Task WriteToStreamAsync_SanityCheckForLocalDevOnly()
{
var cancellationToken = new CancellationToken();
var optionsAccessor = new Moq.Mock <IOptions <MemoryCacheOptions> >();
optionsAccessor.Setup(x => x.Value).Returns(new MemoryCacheOptions());
var memoryCache = new MemoryCache(optionsAccessor.Object);
var logger = new Moq.Mock <ILogger <FileRepository> >().Object;
var clock = new SystemClock();
var configurationBuilder = new ConfigurationBuilder();
// NB - Given the SUT is actually connecting to blob storage and a sql db, the connection strings etc are stored in a
// local secrets file that is not included in source control. If running these tests locally, ensure this file is
// present in your project and that it contains the entries we need
configurationBuilder.AddUserSecrets(Assembly.GetExecutingAssembly());
var configuration = configurationBuilder.Build();
var azurePlatformConfiguration = new AzurePlatformConfiguration()
{
AzureBlobStorage = new AzureBlobStorageConfiguration()
{
ContainerName = configuration.GetValue <string>("AzurePlatform:AzureBlobStorage:ContainerName")
}
};
var azurePlatformConfigurationOptionsSnapshot = new Moq.Mock <IOptionsSnapshot <AzurePlatformConfiguration> >();
azurePlatformConfigurationOptionsSnapshot.Setup(x => x.Value).Returns(azurePlatformConfiguration);
var primaryServiceUrl = new Uri(configuration.GetValue <string>("AzurePlatform:AzureBlobStorage:PrimaryServiceUrl"), UriKind.Absolute);
var geoRedundantServiceUrl = new Uri(configuration.GetValue <string>("AzurePlatform:AzureBlobStorage:GeoRedundantServiceUrl"), UriKind.Absolute);
var azureBlobStorageClient = new AzureBlobStoreClient(primaryServiceUrl, geoRedundantServiceUrl, memoryCache, clock, default);
var readWriteConnectionString = configuration.GetValue <string>("AzurePlatform:AzureSql:ReadWriteConnectionString");
var readOnlyConnectionString = configuration.GetValue <string>("AzurePlatform:AzureSql:ReadOnlyConnectionString");
var sqlLogger = new Moq.Mock <ILogger <AzureSqlClient> >().Object;
var sqlCnFactoryLogger = new Moq.Mock <ILogger <AzureSqlDbConnectionFactory> >().Object;
var sqlDbConnectionFactory = new AzureSqlDbConnectionFactory(readWriteConnectionString, readOnlyConnectionString, sqlCnFactoryLogger);
var azureSqlClient = new AzureSqlClient(sqlDbConnectionFactory, sqlLogger);
IFileRepository fileRepository = new FileRepository(azureBlobStorageClient, azureSqlClient, azurePlatformConfigurationOptionsSnapshot.Object, logger);
var blobName = "4d6fa0f8-34a7-4f34-922f-8b06416097e1.pdf";
var file = File.With("DF796179-DB2F-4A06-B4D5-AD7F012CC2CC", "2021-08-09T18:15:02.4214747Z");
var fileHash = "8n45KHxmXabrze7rq/s9Ww==";
using var destinationStream = new System.IO.MemoryStream();
var fileMetadata = new FileMetadata("title", "description", "group-name", file.Version, "owner", file.Name, ".extension", 396764, blobName, clock.UtcNow, fileHash, FileStatus.Verified);
var fileWriteDetails = await fileRepository.WriteToStreamAsync(fileMetadata, destinationStream, cancellationToken);
Assert.IsNotNull(fileWriteDetails);
var fileBytes = destinationStream.ToArray();
Assert.IsTrue(396764 == fileBytes.Length);
Assert.IsTrue(396764 == fileWriteDetails.ContentLength);
Assert.AreEqual(fileHash, fileWriteDetails.ContentHash);
}
public async Task GenerateEphemeralDownloadLink_SanityCheckForLocalDevOnly()
{
var cancellationToken = new CancellationToken();
var optionsAccessor = new Moq.Mock <IOptions <MemoryCacheOptions> >();
optionsAccessor.Setup(x => x.Value).Returns(new MemoryCacheOptions());
var memoryCache = new MemoryCache(optionsAccessor.Object);
var logger = new Moq.Mock <ILogger <FileRepository> >().Object;
var clock = new SystemClock();
var configurationBuilder = new ConfigurationBuilder();
// NB - Given the SUT is actually connecting to blob storage and a sql db, the connection strings etc are stored in a
// local secrets file that is not included in source control. If running these tests locally, ensure this file is
// present in your project and that it contains the entries we need
configurationBuilder.AddUserSecrets(Assembly.GetExecutingAssembly());
var configuration = configurationBuilder.Build();
var azurePlatformConfiguration = new AzurePlatformConfiguration()
{
AzureBlobStorage = new AzureBlobStorageConfiguration()
{
ContainerName = configuration.GetValue <string>("AzurePlatform:AzureBlobStorage:ContainerName")
}
};
var azurePlatformConfigurationOptionsSnapshot = new Moq.Mock <IOptionsSnapshot <AzurePlatformConfiguration> >();
azurePlatformConfigurationOptionsSnapshot.Setup(x => x.Value).Returns(azurePlatformConfiguration);
var primaryServiceUrl = new Uri(configuration.GetValue <string>("AzurePlatform:AzureBlobStorage:PrimaryServiceUrl"), UriKind.Absolute);
var geoRedundantServiceUrl = new Uri(configuration.GetValue <string>("AzurePlatform:AzureBlobStorage:GeoRedundantServiceUrl"), UriKind.Absolute);
var azureBlobStorageClient = new AzureBlobStoreClient(primaryServiceUrl, geoRedundantServiceUrl, memoryCache, clock, default);
var readWriteConnectionString = configuration.GetValue <string>("AzurePlatform:AzureSql:ReadWriteConnectionString");
var readOnlyConnectionString = configuration.GetValue <string>("AzurePlatform:AzureSql:ReadOnlyConnectionString");
var sqlLogger = new Moq.Mock <ILogger <AzureSqlClient> >().Object;
var sqlCnFactoryLogger = new Moq.Mock <ILogger <AzureSqlDbConnectionFactory> >().Object;
var sqlDbConnectionFactory = new AzureSqlDbConnectionFactory(readWriteConnectionString, readOnlyConnectionString, sqlCnFactoryLogger);
var azureSqlClient = new AzureSqlClient(sqlDbConnectionFactory, sqlLogger);
IFileRepository fileRepository = new FileRepository(azureBlobStorageClient, azureSqlClient, azurePlatformConfigurationOptionsSnapshot.Object, logger);
var file = File.With("DF796179-DB2F-4A06-B4D5-AD7F012CC2CC", "2021-08-09T18:15:02.4214747Z");
var fileMetadata = await fileRepository.GetMetadataAsync(file, cancellationToken);
var uri = await fileRepository.GeneratePublicEphemeralDownloadLink(fileMetadata, cancellationToken);
Assert.IsNotNull(uri);
Assert.IsTrue(uri.IsAbsoluteUri);
}
public void ResiliencyPolicy_BulkheadQueuesExcessLoadAndThrowsOutOldersWorkWhenTooMuchInQueue()
{
var policy = AzureSqlClient.GetAsyncBulkheadPolicy();
// Bulkhead is configured for a max concurrent rate of 3 with max queue size of 25
// In this test we will prevent all tasks from completing apart from the first, thus we should expect to see
// a max number of executing tasks of 3, 1 completion, 25 queuing and 4 rejections
const bool SIGNALED = true;
var gate = new AutoResetEvent(initialState: SIGNALED);
var invocations = 0;
var completed = 0;
var xs = new Task <PolicyResult> [50];
var cancellationToken = new CancellationToken();
for (var n = 0; n < 50; n++)
{
var root = Task.Run(
() =>
policy.ExecuteAndCaptureAsync(
ct => {
invocations++;
var signalled = gate.WaitOne(10);
if (signalled)
{
completed++;
return(Task.CompletedTask);
}
else
{
return(Task.Delay(-1));
}
}, cancellationToken),
cancellationToken);
xs[n] = root;
}
Task.WaitAll(xs, 1000, cancellationToken);
// 1 task should have completed as the gate was open to the first past the post
// 25 + 3 tasks should still be working (max queue size + max concurrency)
// 50 - (1 + 25 + 3) tasks should have been rejected by the policy
Assert.AreEqual(1, completed, "Expected just one task to complete");
var policyResults = xs.Where(_ => _.IsCompleted).Select(_ => _.Result).ToArray();
var failures = policyResults.Where(_ => _.Outcome == OutcomeType.Failure).ToArray();
Assert.AreEqual(21, failures.Length, "Expected 21 work items to have failed to complete");
var rejections = failures.Where(_ => _.FinalException.GetType() == typeof(Polly.Bulkhead.BulkheadRejectedException)).Count();
Assert.AreEqual(21, rejections, "Expected the bulkhead policy to reject 21 work items due to being full");
}
