public async Task <ApprenticeshipVenueCorrection[]> AnalyseProviderApprenticeshipVenueReferences( [HttpTrigger(methods: "post")] List <int> ukprns, [Blob(ReportBlobContainer, Connection = BlobConnectionStringKey)] CloudBlobContainer cloudBlobContainer, CancellationToken shutdownCancellationToken) { await using var reportBlobStream = await GetReportBlobStream(cloudBlobContainer); var apprenticeships = await _cosmosDbQueryDispatcher.ExecuteQuery( new GetApprenticeships { Predicate = a => ukprns.Contains(a.ProviderUKPRN) }); var apprenticeshipVenueCorrections = new List <ApprenticeshipVenueCorrection>(); foreach (var apprenticeship in apprenticeships.Values) { apprenticeshipVenueCorrections.Add(await Analyse(apprenticeship)); } var counts = AnalysisCounts.GetCounts(apprenticeshipVenueCorrections); LogCounts(counts, "Specified providers' apprenticeships analysed."); await WriteReport(reportBlobStream, apprenticeshipVenueCorrections); return(apprenticeshipVenueCorrections.ToArray()); }
public async Task FixAllApprenticeshipVenueReferences( [HttpTrigger(methods: "post")] string unusedWorkaroundParam, // Unused param is a work around for https://github.com/Azure/azure-functions-vs-build-sdk/issues/168 [Blob(ReportBlobContainer, Connection = BlobConnectionStringKey)] CloudBlobContainer cloudBlobContainer, CancellationToken shutdownCancellationToken) { await using var reportBlobStream = await GetReportBlobStream(cloudBlobContainer, fixup : true); var totals = new AnalysisCounts(); int batchCounter = 0; int failureTotal = 0; await _cosmosDbQueryDispatcher.ExecuteQuery( new ProcessAllApprenticeships { Predicate = LiveApprenticeshipPredicate(), MaxBatchSize = BatchSize, ProcessChunk = async apprenticeshipChunk => { Interlocked.Increment(ref batchCounter); var apprenticeshipVenueCorrections = new List <ApprenticeshipVenueCorrection>(); try { foreach (var apprenticeship in apprenticeshipChunk) { apprenticeshipVenueCorrections.Add(await Analyse(apprenticeship)); } var counts = AnalysisCounts.GetCounts(apprenticeshipVenueCorrections); LogCounts(counts, $"Batch {batchCounter} analysed for fixup."); totals = totals.Add(counts); _logger.LogInformation($"{LogPrefix} Applying batch fixes..."); foreach (var apprenticeshipVenueCorrection in apprenticeshipVenueCorrections) { if (shutdownCancellationToken.IsCancellationRequested) { _logger.LogWarning($"{LogPrefix} Cancellation requested, flushing logs and exiting."); shutdownCancellationToken.ThrowIfCancellationRequested(); } await ApplyCorrection(apprenticeshipVenueCorrection, shutdownCancellationToken); } failureTotal += apprenticeshipVenueCorrections.Count(c => c.UpdateFailure != null); } finally { await WriteReport(reportBlobStream, apprenticeshipVenueCorrections); } }, }); LogCounts(totals, "Fixup grand total:"); if (failureTotal > 0) { _logger.LogError($"{LogPrefix} {failureTotal} FAILED UPDATES (see report in blob storage)."); } _logger.LogInformation($"{LogPrefix} Venue fixup for all data completed."); }
public async Task FixSpecificApprenticeshipVenueReferences([HttpTrigger(methods: "post")] List <Guid> apprenticeshipIds, [Blob(ReportBlobContainer, Connection = BlobConnectionStringKey)] CloudBlobContainer cloudBlobContainer, CancellationToken shutdownCancellationToken) { await using var reportBlobStream = await GetReportBlobStream(cloudBlobContainer, fixup : true); var apprenticeships = await _cosmosDbQueryDispatcher.ExecuteQuery( new GetApprenticeshipsByIds { ApprenticeshipIds = apprenticeshipIds }); var apprenticeshipVenueCorrections = new List <ApprenticeshipVenueCorrection>(); foreach (var apprenticeship in apprenticeships.Values) { apprenticeshipVenueCorrections.Add(await Analyse(apprenticeship)); } var counts = AnalysisCounts.GetCounts(apprenticeshipVenueCorrections); LogCounts(counts, "Batch analysed."); _logger.LogInformation($"{LogPrefix} Beginning fixup of specified apprenticeships..."); try { foreach (var apprenticeshipVenueCorrection in apprenticeshipVenueCorrections) { shutdownCancellationToken.ThrowIfCancellationRequested(); await ApplyCorrection(apprenticeshipVenueCorrection, shutdownCancellationToken); } } finally { await WriteReport(reportBlobStream, apprenticeshipVenueCorrections); } var failureTotal = apprenticeshipVenueCorrections.Count(c => c.UpdateFailure != null); if (failureTotal > 0) { _logger.LogError($"{LogPrefix} {failureTotal} FAILED UPDATES (see report in blob storage)."); } _logger.LogInformation($"{LogPrefix} Completed fixup of specific apprenticeships."); }