public async Task <IActionResult> UploadBatch(BatchUploadRequest uploadRequest) { Guard.ArgumentNotNull(uploadRequest?.Stream, nameof(uploadRequest.Stream)); string batchId = _batchIdentifiers.CreateUniqueIdentifier(); await using MemoryStream stream = new MemoryStream(uploadRequest.Stream); string blobName = new BatchUploadBlobName(batchId); ICloudBlob blob = _blobClient.GetBlockBlobReference(blobName, ContainerName); await _blobResilience.ExecuteAsync(() => _blobClient.UploadFileAsync(blob, stream)); _logger.Information($"Uploaded batch ids was allocated batch id {batchId}"); string saasUrl = _blobClient.GetBlobSasUrl(blobName, _dateTime.UtcNow.AddHours(24), SharedAccessBlobPermissions.Read, ContainerName); Guard.Ensure(saasUrl.IsNotNullOrWhitespace(), $"Could not create a temporary SAAS Url for batch upload file {blobName}"); return(new OkObjectResult(new BatchUploadResponse { BatchId = batchId, Url = saasUrl })); }
public override async Task Process(Message message) { BatchUploadValidationProperties properties = message; string batchId = properties.BatchId; string fundingStreamId = properties.FundingStreamId; string fundingPeriodId = properties.FundingPeriodId; BatchUploadBlobName blobName = new BatchUploadBlobName(batchId); IBatchUploadReader batchUploadReader = _batchUploadReaderFactory.CreateBatchUploadReader(); await batchUploadReader.LoadBatchUpload(blobName); List <string> publishedProviderIds = new List <string>(); List <string> missingUkprns = new List <string>(); _logger.Information($"Starting validation for batch {batchId} with a total of {batchUploadReader.Count} to check"); while (batchUploadReader.HasPages) { string[] page = batchUploadReader.NextPage()?.ToArray() ?? new string[0]; IDictionary <string, string> publishedProviderIdPage = await _publishedFundingResilience.ExecuteAsync(() => _publishedFunding.GetPublishedProviderIdsForUkprns(fundingStreamId, fundingPeriodId, page)) ?? new Dictionary <string, string>(); if (page.Length != publishedProviderIdPage.Count) { missingUkprns.AddRange(page.Except(publishedProviderIdPage.Keys)); } publishedProviderIds.AddRange(publishedProviderIdPage.Values); ItemsProcessed = ItemsProcessed.GetValueOrDefault() + publishedProviderIdPage.Count; } ItemsProcessed = batchUploadReader.Count; if (missingUkprns.Any()) { ItemsFailed = missingUkprns.Count; throw new NonRetriableException( $"Did not locate the following ukprns for {fundingStreamId} and {fundingPeriodId}:\n{missingUkprns.JoinWith(',')}"); } byte[] publishedProviderIdsJsonBytes = JsonExtensions.AsJsonBytes(publishedProviderIds.ToArray()); BatchUploadProviderIdsBlobName batchUploadProviderIdsBlobName = new BatchUploadProviderIdsBlobName(batchId); ICloudBlob cloudBlob = _blobClient.GetBlockBlobReference(batchUploadProviderIdsBlobName, ContainerName); await _blobClientResilience.ExecuteAsync(() => cloudBlob.UploadFromByteArrayAsync(publishedProviderIdsJsonBytes, 0, publishedProviderIdsJsonBytes.Length)); }