private void AndGetFileSystemCache()
{
ProviderVersionSearchResult providerVersionSearchResult = NewProviderVersionSearchResult(_ => _.WithProviderId(_providerId));
_fileSystemCache
.Get(Arg.Is <FileSystemCacheKey>(_ => _.Path == $"{ProviderVersionFolderName}\\{_publishedProviderVersion}.json"))
.Returns(new MemoryStream(providerVersionSearchResult.AsJsonBytes()));
}
public async Task <IActionResult> GetPublishedProviderInformation(string publishedProviderVersion)
{
Guard.ArgumentNotNull(publishedProviderVersion, nameof(publishedProviderVersion));
string blobName = $"{publishedProviderVersion}.json";
ProviderVersionSystemCacheKey providerVersionFileSystemCacheKey = new ProviderVersionSystemCacheKey(blobName);
if (_cacheSettings.IsEnabled && _fileSystemCache.Exists(providerVersionFileSystemCacheKey))
{
await using Stream providerVersionDocumentStream = _fileSystemCache.Get(providerVersionFileSystemCacheKey);
ProviderVersionSearchResult cachedProviderVersionSearchResult = providerVersionDocumentStream.AsPoco <ProviderVersionSearchResult>();
return(new OkObjectResult(cachedProviderVersionSearchResult));
}
(string providerVersionId, string providerId)results =
await _publishedFundingRepositoryPolicy.ExecuteAsync(() => _publishedFundingRepository.GetPublishedProviderId(publishedProviderVersion));
if (string.IsNullOrEmpty(results.providerVersionId) || string.IsNullOrEmpty(results.providerId))
{
_logger.Error($"Failed to retrieve published provider with publishedProviderVersion: {publishedProviderVersion}");
return(new NotFoundResult());
}
ApiResponse <ProvidersApiClientModel.Search.ProviderVersionSearchResult> apiResponse =
await _providersApiClientPolicy.ExecuteAsync(() =>
_providersApiClient.GetProviderByIdFromProviderVersion(results.providerVersionId, results.providerId));
if (apiResponse?.Content == null || !apiResponse.StatusCode.IsSuccess())
{
string errorMessage = $"Failed to retrieve GetProviderByIdFromProviderVersion with " +
$"providerVersionId: {results.providerVersionId} and providerId: {results.providerId}";
_logger.Error(errorMessage);
return(new InternalServerErrorResult(errorMessage));
}
ProviderVersionSearchResult providerVersionSearchResult = _mapper.Map <ProviderVersionSearchResult>(apiResponse.Content);
if (_cacheSettings.IsEnabled)
{
if (!_fileSystemCache.Exists(providerVersionFileSystemCacheKey))
{
await using MemoryStream stream = new MemoryStream(providerVersionSearchResult.AsJsonBytes());
_fileSystemCache.Add(providerVersionFileSystemCacheKey, stream, ensureFolderExists: true);
}
}
return(new OkObjectResult(providerVersionSearchResult));
}
public async Task <string> GetFundingFeedDocument(string absoluteDocumentPathUrl,
bool isForPreLoad = false)
{
Guard.IsNullOrWhiteSpace(absoluteDocumentPathUrl, nameof(absoluteDocumentPathUrl));
string documentPath = ParseDocumentPathRelativeToBlobContainerFromFullUrl(absoluteDocumentPathUrl);
string fundingFeedDocumentName = Path.GetFileNameWithoutExtension(documentPath);
FundingFileSystemCacheKey fundingFileSystemCacheKey = new FundingFileSystemCacheKey(fundingFeedDocumentName);
if (_cacheSettings.IsEnabled && _fileSystemCache.Exists(fundingFileSystemCacheKey))
{
if (isForPreLoad)
{
return(null);
}
Stream fundingDocumentStream = _fileSystemCache.Get(fundingFileSystemCacheKey);
using BinaryReader binaryReader = new BinaryReader(fundingDocumentStream);
return(GetDocumentContentFromBytes(binaryReader.ReadBytes((int)fundingDocumentStream.Length)));
}
ICloudBlob blob = _blobClient.GetBlockBlobReference(documentPath);
if (!blob.Exists())
{
_logger.Error($"Failed to find blob with path: {documentPath}");
return(null);
}
using (MemoryStream fundingDocumentStream = (MemoryStream)await _publishedFundingRepositoryPolicy.ExecuteAsync(() => _blobClient.DownloadToStreamAsync(blob)))
{
if (fundingDocumentStream == null || fundingDocumentStream.Length == 0)
{
_logger.Error($"Invalid blob returned: {documentPath}");
return(null);
}
if (_cacheSettings.IsEnabled)
{
if (!_fileSystemCache.Exists(fundingFileSystemCacheKey))
{
_fileSystemCache.Add(fundingFileSystemCacheKey, fundingDocumentStream);
}
}
fundingDocumentStream.Position = 0;
return(isForPreLoad ? null : GetDocumentContentFromBytes(fundingDocumentStream.ToArray()));
}
}
public async Task <IActionResult> GetProviderFundingVersion(string providerFundingVersion)
{
if (string.IsNullOrWhiteSpace(providerFundingVersion))
{
return(new BadRequestObjectResult("Null or empty id provided."));
}
string blobName = $"{providerFundingVersion}.json";
try
{
ProviderFundingFileSystemCacheKey cacheKey = new ProviderFundingFileSystemCacheKey(providerFundingVersion);
if (_cacheSettings.IsEnabled && _fileSystemCache.Exists(cacheKey))
{
using Stream cachedStream = _fileSystemCache.Get(cacheKey);
return(await GetContentResultForStream(cachedStream));
}
bool exists = await _blobClientPolicy.ExecuteAsync(() => _blobClient.BlobExistsAsync(blobName));
if (!exists)
{
_logger.Error($"Blob '{blobName}' does not exist.");
return(new NotFoundResult());
}
ICloudBlob blob = await _blobClientPolicy.ExecuteAsync(() => _blobClient.GetBlobReferenceFromServerAsync(blobName));
using Stream blobStream = await _blobClientPolicy.ExecuteAsync(() => _blobClient.DownloadToStreamAsync(blob));
if (_cacheSettings.IsEnabled)
{
_fileSystemCache.Add(cacheKey, blobStream);
}
return(await GetContentResultForStream(blobStream));
}
catch (Exception ex)
{
string errorMessage = $"Failed to fetch blob '{blobName}' from azure storage";
_logger.Error(ex, errorMessage);
return(new InternalServerErrorResult(errorMessage));
}
}
public async Task <IActionResult> FetchCoreProviderData(string specificationId)
{
Guard.IsNullOrWhiteSpace(specificationId, nameof(specificationId));
bool fileSystemCacheEnabled = _scopedProvidersServiceSettings.IsFileSystemCacheEnabled;
if (fileSystemCacheEnabled)
{
_fileSystemCache.EnsureFoldersExist(ScopedProvidersFileSystemCacheKey.Folder);
}
string filesystemCacheKey = $"{CacheKeys.ScopedProviderSummariesFilesystemKeyPrefix}{specificationId}";
string cacheGuid = await _cachePolicy.ExecuteAsync(() => _cacheProvider.GetAsync <string>(filesystemCacheKey));
if (cacheGuid.IsNullOrEmpty())
{
cacheGuid = Guid.NewGuid().ToString();
await _cacheProvider.SetAsync(filesystemCacheKey, cacheGuid, TimeSpan.FromDays(7), true);
}
ScopedProvidersFileSystemCacheKey cacheKey = new ScopedProvidersFileSystemCacheKey(specificationId, cacheGuid);
if (fileSystemCacheEnabled && _fileSystemCache.Exists(cacheKey))
{
await using Stream cachedStream = _fileSystemCache.Get(cacheKey);
return(GetActionResultForStream(cachedStream, specificationId));
}
IEnumerable <ProviderSummary> providerSummaries = await GetScopedProvidersForSpecification(specificationId);
if (providerSummaries.IsNullOrEmpty())
{
return(new NoContentResult());
}
await using MemoryStream stream = new MemoryStream(providerSummaries.AsJsonBytes())
{
Position = 0
};
if (fileSystemCacheEnabled)
{
_fileSystemCache.Add(cacheKey, stream);
}
return(GetActionResultForStream(stream, specificationId));
}
public async Task <IActionResult> GetAllProviders(string providerVersionId)
{
Guard.IsNullOrWhiteSpace(providerVersionId, nameof(providerVersionId));
string blobName = $"{providerVersionId.ToLowerInvariant()}.json";
bool fileSystemCacheEnabled = _providerVersionServiceSettings.IsFileSystemCacheEnabled;
if (fileSystemCacheEnabled)
{
_fileSystemCache.EnsureFoldersExist(ProviderVersionFileSystemCacheKey.Folder);
}
ProviderVersionFileSystemCacheKey cacheKey = new ProviderVersionFileSystemCacheKey(providerVersionId);
if (fileSystemCacheEnabled && _fileSystemCache.Exists(cacheKey))
{
await using Stream cachedStream = _fileSystemCache.Get(cacheKey);
return(GetActionResultForStream(cachedStream, providerVersionId));
}
ICloudBlob blob = _blobClient.GetBlockBlobReference(blobName);
if (!blob.Exists())
{
_logger.Error($"Failed to find blob with path: {blobName}");
return(new NotFoundResult());
}
await using Stream blobClientStream = await _blobRepositoryPolicy.ExecuteAsync(()
=> _blobClient.DownloadToStreamAsync(blob));
if (fileSystemCacheEnabled)
{
_fileSystemCache.Add(cacheKey, blobClientStream);
}
return(GetActionResultForStream(blobClientStream, providerVersionId));
}
public async Task <Stream> GetAssembly(string specificationId,
string etag)
{
try
{
Guard.IsNullOrWhiteSpace(specificationId, nameof(specificationId));
Guard.IsNullOrWhiteSpace(etag, nameof(etag));
SpecificationAssemblyFileSystemCacheKey fileSystemCacheKey = GetFileSystemCacheKey(specificationId, etag);
if (_fileSystemCache.Exists(fileSystemCacheKey))
{
return(_fileSystemCache.Get(fileSystemCacheKey));
}
(Stream assembly, string etag)specificationAssemblyDetails = await GetAssemblyFromBlobStorage(specificationId);
if (specificationAssemblyDetails.etag.IsNotNullOrWhitespace())
{
if (!etag.Equals(specificationAssemblyDetails.etag, StringComparison.InvariantCultureIgnoreCase))
{
throw new NonRetriableException("Invalid specification assembly etag.");
}
}
await SetAssembly(specificationId, specificationAssemblyDetails.assembly);
return(specificationAssemblyDetails.assembly);
}
catch (Exception exception)
{
LogError(exception, $"Unable to fetch specification assembly for specification {specificationId} with etag {etag}");
throw;
}
}
public async Task GetProviderFundingVersionsBody_GivenReturnsFileSystemIsDisabledGetsFromBlobClientFetch_ReturnsOK()
{
//Arrange
string template = "just a string";
byte[] bytes = Encoding.UTF8.GetBytes(template);
Stream memoryStream = new MemoryStream(bytes);
ILogger logger = CreateLogger();
IFileSystemCache fileSystemCache = CreateFileSystemCache();
IBlobClient blobClient = CreateBlobClient();
fileSystemCache.Exists(Arg.Is <ProviderFundingFileSystemCacheKey>(
_ => _.Key == providerFundingVersion))
.Returns(true);
fileSystemCache.Get(Arg.Is <ProviderFundingFileSystemCacheKey>(
_ => _.Key == providerFundingVersion))
.Returns(memoryStream);
ICloudBlob cloudBlob = CreateBlob();
blobClient
.BlobExistsAsync(blobName)
.Returns(true);
blobClient
.GetBlobReferenceFromServerAsync(blobName)
.Returns(cloudBlob);
blobClient
.DownloadToStreamAsync(cloudBlob)
.Returns(memoryStream);
ProviderFundingVersionService service = CreateProviderFundingVersionService(logger, blobClient, fileSystemCache,
Substitute.For <IExternalApiFileSystemCacheSettings>());
//Act
IActionResult result = await service.GetProviderFundingVersion(providerFundingVersion);
//Assert
result
.Should()
.BeOfType <ContentResult>();
ContentResult contentResult = result as ContentResult;
contentResult
.ContentType
.Should()
.Be("application/json");
contentResult
.StatusCode
.Should()
.Be((int)HttpStatusCode.OK);
contentResult
.Content
.Should()
.Be(template);
fileSystemCache
.Received(0)
.Exists(Arg.Any <FileSystemCacheKey>());
fileSystemCache
.Received(0)
.Get(Arg.Any <FileSystemCacheKey>());
fileSystemCache
.Received(0)
.Add(Arg.Is <ProviderFundingFileSystemCacheKey>(_ => _.Key == providerFundingVersion),
memoryStream,
CancellationToken.None);
}
public async Task GetProviderFundingVersionsBody_GivenReturnsFromFileSystemCacheAndSkipBlobClientFetch_ReturnsOK()
{
//Arrange
string template = "just a string";
byte[] bytes = Encoding.UTF8.GetBytes(template);
Stream memoryStream = new MemoryStream(bytes);
ILogger logger = CreateLogger();
IFileSystemCache fileSystemCache = CreateFileSystemCache();
IBlobClient blobClient = CreateBlobClient();
fileSystemCache.Exists(Arg.Is <ProviderFundingFileSystemCacheKey>(
_ => _.Key == providerFundingVersion))
.Returns(true);
fileSystemCache.Get(Arg.Is <ProviderFundingFileSystemCacheKey>(
_ => _.Key == providerFundingVersion))
.Returns(memoryStream);
ProviderFundingVersionService service = CreateProviderFundingVersionService(logger, blobClient, fileSystemCache);
//Act
IActionResult result = await service.GetProviderFundingVersion(providerFundingVersion);
//Assert
result
.Should()
.BeOfType <ContentResult>();
ContentResult contentResult = result as ContentResult;
contentResult
.ContentType
.Should()
.Be("application/json");
contentResult
.StatusCode
.Should()
.Be((int)HttpStatusCode.OK);
contentResult
.Content
.Should()
.Be(template);
await blobClient
.Received(0)
.BlobExistsAsync(providerFundingVersion);
await blobClient
.Received(0)
.GetAsync(Arg.Any <string>());
fileSystemCache
.Received(0)
.Add(Arg.Is <ProviderFundingFileSystemCacheKey>(_ => _.Key == providerFundingVersion),
memoryStream,
CancellationToken.None);
}
public async Task GetFundingFeedDocument_WhenReturnsFromCache_ReturnsOK(bool fileSystemCacheEnabled,
int expectedCacheAccessCount,
int expectedBlobClientAccessCount)
{
//Arrange
string template = new RandomString();
string documentPath = "cromulent.json";
string uri = $"https://cfs/test/{documentPath}";
Stream memoryStream = new MemoryStream(Encoding.UTF8.GetBytes(template));
ILogger logger = CreateLogger();
ICloudBlob cloudBlob = CreateBlob();
IFileSystemCache fileSystemCache = CreateFileSystemCache();
IExternalApiFileSystemCacheSettings cacheSettings = Substitute.For <IExternalApiFileSystemCacheSettings>();
cacheSettings.IsEnabled.Returns(fileSystemCacheEnabled);
cloudBlob
.Exists()
.Returns(true);
cloudBlob
.Exists(Arg.Any <BlobRequestOptions>(), Arg.Any <OperationContext>())
.Returns(true);
string documentName = Path.GetFileNameWithoutExtension(documentPath);
fileSystemCache
.Exists(Arg.Is <FileSystemCacheKey>(_ => _.Key == documentName))
.Returns(true);
fileSystemCache
.Get(Arg.Is <FileSystemCacheKey>(_ => _.Key == documentName))
.Returns(memoryStream);
IBlobClient blobClient = CreateBlobClient();
blobClient
.GetBlockBlobReference(Arg.Is(documentPath))
.Returns(cloudBlob);
blobClient
.DownloadToStreamAsync(Arg.Is(cloudBlob))
.Returns(memoryStream);
PublishedFundingRetrievalService service = CreatePublishedFundingRetrievalService(
blobClient: blobClient,
logger: logger,
fileSystemCache: fileSystemCache,
cacheSettings: cacheSettings);
//Act
string result = await service.GetFundingFeedDocument(uri);
//Assert
result
.Should()
.Be(template);
blobClient
.Received(expectedBlobClientAccessCount)
.GetBlockBlobReference(documentPath);
cloudBlob
.Received(expectedBlobClientAccessCount)
.Exists();
logger
.Received(0)
.Error(Arg.Any <string>());
await blobClient
.Received(expectedBlobClientAccessCount)
.DownloadToStreamAsync(cloudBlob);
fileSystemCache
.Received(expectedCacheAccessCount)
.Get(Arg.Any <FundingFileSystemCacheKey>());
fileSystemCache
.Received(0)
.Add(Arg.Is <FundingFileSystemCacheKey>(
_ => _.Key == documentName),
memoryStream);
}
