public async Task <IActionResult> UploadBatch(BatchUploadRequest uploadRequest)
{
Guard.ArgumentNotNull(uploadRequest?.Stream, nameof(uploadRequest.Stream));
string batchId = _batchIdentifiers.CreateUniqueIdentifier();
await using MemoryStream stream = new MemoryStream(uploadRequest.Stream);
string blobName = new BatchUploadBlobName(batchId);
ICloudBlob blob = _blobClient.GetBlockBlobReference(blobName, ContainerName);
await _blobResilience.ExecuteAsync(() => _blobClient.UploadFileAsync(blob, stream));
_logger.Information($"Uploaded batch ids was allocated batch id {batchId}");
string saasUrl = _blobClient.GetBlobSasUrl(blobName,
_dateTime.UtcNow.AddHours(24),
SharedAccessBlobPermissions.Read,
ContainerName);
Guard.Ensure(saasUrl.IsNotNullOrWhitespace(), $"Could not create a temporary SAAS Url for batch upload file {blobName}");
return(new OkObjectResult(new BatchUploadResponse
{
BatchId = batchId,
Url = saasUrl
}));
}
public override async Task Process(Message message)
{
BatchUploadValidationProperties properties = message;
string batchId = properties.BatchId;
string fundingStreamId = properties.FundingStreamId;
string fundingPeriodId = properties.FundingPeriodId;
BatchUploadBlobName blobName = new BatchUploadBlobName(batchId);
IBatchUploadReader batchUploadReader = _batchUploadReaderFactory.CreateBatchUploadReader();
await batchUploadReader.LoadBatchUpload(blobName);
List <string> publishedProviderIds = new List <string>();
List <string> missingUkprns = new List <string>();
_logger.Information($"Starting validation for batch {batchId} with a total of {batchUploadReader.Count} to check");
while (batchUploadReader.HasPages)
{
string[] page = batchUploadReader.NextPage()?.ToArray() ?? new string[0];
IDictionary <string, string> publishedProviderIdPage = await _publishedFundingResilience.ExecuteAsync(() => _publishedFunding.GetPublishedProviderIdsForUkprns(fundingStreamId,
fundingPeriodId,
page)) ?? new Dictionary <string, string>();
if (page.Length != publishedProviderIdPage.Count)
{
missingUkprns.AddRange(page.Except(publishedProviderIdPage.Keys));
}
publishedProviderIds.AddRange(publishedProviderIdPage.Values);
ItemsProcessed = ItemsProcessed.GetValueOrDefault() + publishedProviderIdPage.Count;
}
ItemsProcessed = batchUploadReader.Count;
if (missingUkprns.Any())
{
ItemsFailed = missingUkprns.Count;
throw new NonRetriableException(
$"Did not locate the following ukprns for {fundingStreamId} and {fundingPeriodId}:\n{missingUkprns.JoinWith(',')}");
}
byte[] publishedProviderIdsJsonBytes = JsonExtensions.AsJsonBytes(publishedProviderIds.ToArray());
BatchUploadProviderIdsBlobName batchUploadProviderIdsBlobName = new BatchUploadProviderIdsBlobName(batchId);
ICloudBlob cloudBlob = _blobClient.GetBlockBlobReference(batchUploadProviderIdsBlobName, ContainerName);
await _blobClientResilience.ExecuteAsync(() => cloudBlob.UploadFromByteArrayAsync(publishedProviderIdsJsonBytes, 0, publishedProviderIdsJsonBytes.Length));
}