public async Task <ApprenticeshipVenueCorrection[]> AnalyseProviderApprenticeshipVenueReferences( [HttpTrigger(methods: "post")] List <int> ukprns, [Blob(ReportBlobContainer, Connection = BlobConnectionStringKey)] CloudBlobContainer cloudBlobContainer, CancellationToken shutdownCancellationToken) { await using var reportBlobStream = await GetReportBlobStream(cloudBlobContainer); var apprenticeships = await _cosmosDbQueryDispatcher.ExecuteQuery( new GetApprenticeships { Predicate = a => ukprns.Contains(a.ProviderUKPRN) }); var apprenticeshipVenueCorrections = new List <ApprenticeshipVenueCorrection>(); foreach (var apprenticeship in apprenticeships.Values) { apprenticeshipVenueCorrections.Add(await Analyse(apprenticeship)); } var counts = AnalysisCounts.GetCounts(apprenticeshipVenueCorrections); LogCounts(counts, "Specified providers' apprenticeships analysed."); await WriteReport(reportBlobStream, apprenticeshipVenueCorrections); return(apprenticeshipVenueCorrections.ToArray()); }
public async Task FixAllApprenticeshipVenueReferences( [HttpTrigger(methods: "post")] string unusedWorkaroundParam, // Unused param is a work around for https://github.com/Azure/azure-functions-vs-build-sdk/issues/168 [Blob(ReportBlobContainer, Connection = BlobConnectionStringKey)] CloudBlobContainer cloudBlobContainer, CancellationToken shutdownCancellationToken) { await using var reportBlobStream = await GetReportBlobStream(cloudBlobContainer, fixup : true); var totals = new AnalysisCounts(); int batchCounter = 0; int failureTotal = 0; await _cosmosDbQueryDispatcher.ExecuteQuery( new ProcessAllApprenticeships { Predicate = LiveApprenticeshipPredicate(), MaxBatchSize = BatchSize, ProcessChunk = async apprenticeshipChunk => { Interlocked.Increment(ref batchCounter); var apprenticeshipVenueCorrections = new List <ApprenticeshipVenueCorrection>(); try { foreach (var apprenticeship in apprenticeshipChunk) { apprenticeshipVenueCorrections.Add(await Analyse(apprenticeship)); } var counts = AnalysisCounts.GetCounts(apprenticeshipVenueCorrections); LogCounts(counts, $"Batch {batchCounter} analysed for fixup."); totals = totals.Add(counts); _logger.LogInformation($"{LogPrefix} Applying batch fixes..."); foreach (var apprenticeshipVenueCorrection in apprenticeshipVenueCorrections) { if (shutdownCancellationToken.IsCancellationRequested) { _logger.LogWarning($"{LogPrefix} Cancellation requested, flushing logs and exiting."); shutdownCancellationToken.ThrowIfCancellationRequested(); } await ApplyCorrection(apprenticeshipVenueCorrection, shutdownCancellationToken); } failureTotal += apprenticeshipVenueCorrections.Count(c => c.UpdateFailure != null); } finally { await WriteReport(reportBlobStream, apprenticeshipVenueCorrections); } }, }); LogCounts(totals, "Fixup grand total:"); if (failureTotal > 0) { _logger.LogError($"{LogPrefix} {failureTotal} FAILED UPDATES (see report in blob storage)."); } _logger.LogInformation($"{LogPrefix} Venue fixup for all data completed."); }
private void LogCounts(AnalysisCounts counts, string message) { var analysisStrings = counts.FixCounts.Select(c => c.UnfixableLocationVenueReason == null ? $"{c.CorruptionType}/Fixable {c.Count}" : $"{c.CorruptionType}/{c.UnfixableLocationVenueReason} {c.Count}"); _logger.LogInformation( $"{LogPrefix} {message} Batch size: {counts.BatchSize}. Corrupt locations analysed: {counts.CorruptLocationsAnalysed}. ({string.Join("; ", analysisStrings)})."); }
public AnalysisCounts Add(AnalysisCounts additionalCounts) { return(new AnalysisCounts { BatchSize = BatchSize + additionalCounts.BatchSize, CorruptLocationsAnalysed = CorruptLocationsAnalysed + additionalCounts.CorruptLocationsAnalysed, // merge FixCount lists https://stackoverflow.com/questions/720609/create-a-list-from-two-object-lists-with-linq/6772832#6772832 FixCounts = FixCounts.Concat(additionalCounts.FixCounts) .ToLookup(counts => new { counts.CorruptionType, counts.UnfixableLocationVenueReason }) .Select(group => group.Aggregate((a, b) => new FixCounts(a.CorruptionType, a.UnfixableLocationVenueReason ?? b.UnfixableLocationVenueReason, a.Count + b.Count))).ToList(), }); }
public async Task FixSpecificApprenticeshipVenueReferences([HttpTrigger(methods: "post")] List <Guid> apprenticeshipIds, [Blob(ReportBlobContainer, Connection = BlobConnectionStringKey)] CloudBlobContainer cloudBlobContainer, CancellationToken shutdownCancellationToken) { await using var reportBlobStream = await GetReportBlobStream(cloudBlobContainer, fixup : true); var apprenticeships = await _cosmosDbQueryDispatcher.ExecuteQuery( new GetApprenticeshipsByIds { ApprenticeshipIds = apprenticeshipIds }); var apprenticeshipVenueCorrections = new List <ApprenticeshipVenueCorrection>(); foreach (var apprenticeship in apprenticeships.Values) { apprenticeshipVenueCorrections.Add(await Analyse(apprenticeship)); } var counts = AnalysisCounts.GetCounts(apprenticeshipVenueCorrections); LogCounts(counts, "Batch analysed."); _logger.LogInformation($"{LogPrefix} Beginning fixup of specified apprenticeships..."); try { foreach (var apprenticeshipVenueCorrection in apprenticeshipVenueCorrections) { shutdownCancellationToken.ThrowIfCancellationRequested(); await ApplyCorrection(apprenticeshipVenueCorrection, shutdownCancellationToken); } } finally { await WriteReport(reportBlobStream, apprenticeshipVenueCorrections); } var failureTotal = apprenticeshipVenueCorrections.Count(c => c.UpdateFailure != null); if (failureTotal > 0) { _logger.LogError($"{LogPrefix} {failureTotal} FAILED UPDATES (see report in blob storage)."); } _logger.LogInformation($"{LogPrefix} Completed fixup of specific apprenticeships."); }