public async Task <ApprenticeshipVenueCorrection[]> AnalyseProviderApprenticeshipVenueReferences(
[HttpTrigger(methods: "post")] List <int> ukprns,
[Blob(ReportBlobContainer, Connection = BlobConnectionStringKey)] CloudBlobContainer cloudBlobContainer,
CancellationToken shutdownCancellationToken)
{
await using var reportBlobStream = await GetReportBlobStream(cloudBlobContainer);
var apprenticeships = await _cosmosDbQueryDispatcher.ExecuteQuery(
new GetApprenticeships { Predicate = a => ukprns.Contains(a.ProviderUKPRN) });
var apprenticeshipVenueCorrections = new List <ApprenticeshipVenueCorrection>();
foreach (var apprenticeship in apprenticeships.Values)
{
apprenticeshipVenueCorrections.Add(await Analyse(apprenticeship));
}
var counts = AnalysisCounts.GetCounts(apprenticeshipVenueCorrections);
LogCounts(counts, "Specified providers' apprenticeships analysed.");
await WriteReport(reportBlobStream, apprenticeshipVenueCorrections);
return(apprenticeshipVenueCorrections.ToArray());
}
public async Task FixAllApprenticeshipVenueReferences(
[HttpTrigger(methods: "post")]
string unusedWorkaroundParam, // Unused param is a work around for https://github.com/Azure/azure-functions-vs-build-sdk/issues/168
[Blob(ReportBlobContainer, Connection = BlobConnectionStringKey)]
CloudBlobContainer cloudBlobContainer,
CancellationToken shutdownCancellationToken)
{
await using var reportBlobStream = await GetReportBlobStream(cloudBlobContainer, fixup : true);
var totals = new AnalysisCounts();
int batchCounter = 0;
int failureTotal = 0;
await _cosmosDbQueryDispatcher.ExecuteQuery(
new ProcessAllApprenticeships
{
Predicate = LiveApprenticeshipPredicate(),
MaxBatchSize = BatchSize,
ProcessChunk = async apprenticeshipChunk =>
{
Interlocked.Increment(ref batchCounter);
var apprenticeshipVenueCorrections = new List <ApprenticeshipVenueCorrection>();
try
{
foreach (var apprenticeship in apprenticeshipChunk)
{
apprenticeshipVenueCorrections.Add(await Analyse(apprenticeship));
}
var counts = AnalysisCounts.GetCounts(apprenticeshipVenueCorrections);
LogCounts(counts, $"Batch {batchCounter} analysed for fixup.");
totals = totals.Add(counts);
_logger.LogInformation($"{LogPrefix} Applying batch fixes...");
foreach (var apprenticeshipVenueCorrection in apprenticeshipVenueCorrections)
{
if (shutdownCancellationToken.IsCancellationRequested)
{
_logger.LogWarning($"{LogPrefix} Cancellation requested, flushing logs and exiting.");
shutdownCancellationToken.ThrowIfCancellationRequested();
}
await ApplyCorrection(apprenticeshipVenueCorrection, shutdownCancellationToken);
}
failureTotal += apprenticeshipVenueCorrections.Count(c => c.UpdateFailure != null);
}
finally
{
await WriteReport(reportBlobStream, apprenticeshipVenueCorrections);
}
},
});
LogCounts(totals, "Fixup grand total:");
if (failureTotal > 0)
{
_logger.LogError($"{LogPrefix} {failureTotal} FAILED UPDATES (see report in blob storage).");
}
_logger.LogInformation($"{LogPrefix} Venue fixup for all data completed.");
}
private void LogCounts(AnalysisCounts counts, string message)
{
var analysisStrings = counts.FixCounts.Select(c => c.UnfixableLocationVenueReason == null
? $"{c.CorruptionType}/Fixable {c.Count}"
: $"{c.CorruptionType}/{c.UnfixableLocationVenueReason} {c.Count}");
_logger.LogInformation(
$"{LogPrefix} {message} Batch size: {counts.BatchSize}. Corrupt locations analysed: {counts.CorruptLocationsAnalysed}. ({string.Join("; ", analysisStrings)}).");
}
public AnalysisCounts Add(AnalysisCounts additionalCounts)
{
return(new AnalysisCounts
{
BatchSize = BatchSize + additionalCounts.BatchSize,
CorruptLocationsAnalysed = CorruptLocationsAnalysed + additionalCounts.CorruptLocationsAnalysed,
// merge FixCount lists https://stackoverflow.com/questions/720609/create-a-list-from-two-object-lists-with-linq/6772832#6772832
FixCounts = FixCounts.Concat(additionalCounts.FixCounts)
.ToLookup(counts => new { counts.CorruptionType, counts.UnfixableLocationVenueReason })
.Select(group => group.Aggregate((a, b) => new FixCounts(a.CorruptionType,
a.UnfixableLocationVenueReason ?? b.UnfixableLocationVenueReason, a.Count + b.Count))).ToList(),
});
}
public async Task FixSpecificApprenticeshipVenueReferences([HttpTrigger(methods: "post")] List <Guid> apprenticeshipIds,
[Blob(ReportBlobContainer, Connection = BlobConnectionStringKey)] CloudBlobContainer cloudBlobContainer,
CancellationToken shutdownCancellationToken)
{
await using var reportBlobStream = await GetReportBlobStream(cloudBlobContainer, fixup : true);
var apprenticeships = await _cosmosDbQueryDispatcher.ExecuteQuery(
new GetApprenticeshipsByIds { ApprenticeshipIds = apprenticeshipIds });
var apprenticeshipVenueCorrections = new List <ApprenticeshipVenueCorrection>();
foreach (var apprenticeship in apprenticeships.Values)
{
apprenticeshipVenueCorrections.Add(await Analyse(apprenticeship));
}
var counts = AnalysisCounts.GetCounts(apprenticeshipVenueCorrections);
LogCounts(counts, "Batch analysed.");
_logger.LogInformation($"{LogPrefix} Beginning fixup of specified apprenticeships...");
try
{
foreach (var apprenticeshipVenueCorrection in apprenticeshipVenueCorrections)
{
shutdownCancellationToken.ThrowIfCancellationRequested();
await ApplyCorrection(apprenticeshipVenueCorrection, shutdownCancellationToken);
}
}
finally
{
await WriteReport(reportBlobStream, apprenticeshipVenueCorrections);
}
var failureTotal = apprenticeshipVenueCorrections.Count(c => c.UpdateFailure != null);
if (failureTotal > 0)
{
_logger.LogError($"{LogPrefix} {failureTotal} FAILED UPDATES (see report in blob storage).");
}
_logger.LogInformation($"{LogPrefix} Completed fixup of specific apprenticeships.");
}