public static async Task Run(
[TimerTrigger("%MigrationReportSchedule%")] TimerInfo myTimer,
ILogger log,
[Inject] IConfigurationRoot configuration,
[Inject] ICosmosDbHelper cosmosDbHelper,
[Inject] IBlobStorageHelper blobHelper,
[Inject] IProviderCollectionService providerCollectionService,
[Inject] ICourseCollectionService courseCollectionService,
[Inject] IApprenticeshipCollectionService apprenticeshipCollectionService,
[Inject] IMigrationReportCollectionService migrationReportCollectionService)
{
await new MigrationReportGeneratorService().Run(
log, configuration, cosmosDbHelper, blobHelper, providerCollectionService, courseCollectionService,
apprenticeshipCollectionService, migrationReportCollectionService
);
}
public static async Task Run(
string input, // Work around https://github.com/Azure/azure-functions-vs-build-sdk/issues/168
[Inject] IConfigurationRoot configuration,
[Inject] ICosmosDbHelper cosmosDbHelper,
[Inject] ILoggerFactory loggerFactory,
[Inject] IApprenticeshipCollectionService apprenticeshipCollectionService)
{
var databaseId = configuration["CosmosDbSettings:DatabaseId"];
var apprenticeshipCollectionId = "apprenticeship";
var documentClient = cosmosDbHelper.GetClient();
var logger = loggerFactory.CreateLogger(typeof(ArchivePendingApprenticeships));
int count = 0;
var updatedBy = "ArchivePendingApprenticeships";
var queryResponse = await apprenticeshipCollectionService.GetArchivedApprenticeshipsAsync();
foreach (var doc in queryResponse)
{
//mark every location as arhived regardless of their status
foreach (var loc in doc.ApprenticeshipLocations)
{
loc.RecordStatus = CourseDirectory.Models.Enums.RecordStatus.Archived;
loc.UpdatedBy = updatedBy;
loc.UpdatedDate = DateTime.UtcNow;
}
doc.UpdatedBy = updatedBy;
doc.UpdatedDate = DateTime.UtcNow;
var documentLink = UriFactory.CreateDocumentUri(databaseId, apprenticeshipCollectionId, doc.id.ToString());
await documentClient.ReplaceDocumentAsync(documentLink, doc);
count++;
}
logger.LogInformation($"Archived {count} Apprenticeships");
Console.WriteLine($"Archived {count} Apprenticeships");
}
private static async Task GenerateApprenticeshipReport(
IConfiguration configuration,
IApprenticeshipCollectionService apprenticeshipCollectionService,
IMigrationReportCollectionService migrationReportCollectionService,
Provider provider,
IReadOnlyList <RecordStatus> migratedStatusList,
IDocumentClient cosmosClient)
{
var apprenticeships =
await apprenticeshipCollectionService.GetAllApprenticeshipsByUkprnAsync(provider
.UnitedKingdomProviderReferenceNumber);
var appReportEntry =
await migrationReportCollectionService.GetReportForApprenticeshipByUkprn(
int.Parse(provider.UnitedKingdomProviderReferenceNumber)) ?? new MigrationReportEntry();
appReportEntry.CreatedOn = DateTime.Now;
appReportEntry.CreatedBy = AppName;
appReportEntry.id = provider.UnitedKingdomProviderReferenceNumber;
appReportEntry.MigrationPendingCount = apprenticeships.Count(a => a.RecordStatus == RecordStatus.MigrationPending);
appReportEntry.MigrationReadyToGoLive = apprenticeships.Count(a => a.RecordStatus == RecordStatus.MigrationReadyToGoLive);
appReportEntry.BulkUploadPendingcount = apprenticeships.Count(a => a.RecordStatus == RecordStatus.BulkUloadPending);
appReportEntry.BulkUploadReadyToGoLiveCount = apprenticeships.Count(a => a.RecordStatus == RecordStatus.BulkUploadReadyToGoLive);
appReportEntry.LiveCount = apprenticeships.Count(cr => cr.RecordStatus == RecordStatus.Live);
appReportEntry.PendingCount = apprenticeships.Count(cr => cr.RecordStatus == RecordStatus.Pending);
appReportEntry.MigratedCount = apprenticeships.Count(cr => migratedStatusList.Contains(cr.RecordStatus));
appReportEntry.MigrationDate = provider.DateUpdated;
appReportEntry.ProviderType = (int)provider.ProviderType;
appReportEntry.ProviderName = provider.ProviderName;
appReportEntry.FailedMigrationCount = 0;
appReportEntry.MigrationRate = ApprenticeshipMigrationRate(apprenticeships, migratedStatusList);
var collectionUri = UriFactory.CreateDocumentCollectionUri(configuration["CosmosDbSettings:DatabaseId"],
configuration["CosmosDbCollectionSettings:MigrationReportApprenticeshipCollectionId"]);
await cosmosClient.UpsertDocumentAsync(collectionUri, appReportEntry);
}
public async Task Run(
ILogger log,
IConfigurationRoot configuration,
ICosmosDbHelper cosmosDbHelper,
IBlobStorageHelper blobHelper,
IProviderCollectionService providerCollectionService,
ICourseCollectionService courseCollectionService,
IApprenticeshipCollectionService apprenticeshipCollectionService,
IMigrationReportCollectionService migrationReportCollectionService)
{
log.LogInformation("Starting Migration Report generation");
var migrationLog = new StringBuilder();
migrationLog.AppendLine("-------------------------------------------------------");
migrationLog.AppendLine("Starting Migration Report generation");
migrationLog.AppendLine("-------------------------------------------------------");
var stopWatch = new Stopwatch();
stopWatch.Start();
IReadOnlyList <RecordStatus> migratedStatusList = new List <RecordStatus>
{
RecordStatus.Live,
RecordStatus.MigrationPending,
};
var blobContainer = blobHelper.GetBlobContainer(configuration["BlobStorageSettings:Container"]);
var whiteListedProviders = await GetProviderWhiteList(blobHelper, blobContainer);
var cosmosClient = cosmosDbHelper.GetClient();
log.LogDebug("Fetching migrated provider count...");
var migratedProvidersCount = (await providerCollectionService.GetAllMigratedProviders("Provider.Migrator")).Count;
log.LogDebug($"Migrated Provider count: {migratedProvidersCount}.");
log.LogDebug("Fetching providers...");
var providers = await providerCollectionService.GetDocumentsByUkprn(whiteListedProviders);
var providerTypeCounts = providers.GroupBy(t => t.ProviderType).Select(g => new { type = g.Key, qty = g.Count() });
log.LogDebug($"Provider counts: {string.Join("; ", providerTypeCounts.Select(c => $"{c.type}: {c.qty}"))}. Total: {providers.Count}");
int progress = 1;
int feCourseReportEntryCount = 0;
int apprenticeshipsReportEntryCount = 0;
foreach (var ukprn in whiteListedProviders)
{
try
{
var provider = providers.Single(p => int.Parse(p.UnitedKingdomProviderReferenceNumber) == ukprn);
var logStart = $"STARTED : Generating report for provider with UKPRN: {provider.UnitedKingdomProviderReferenceNumber}. Progress: {progress++}/{whiteListedProviders.Count}";
log.LogDebug(logStart);
migrationLog.AppendLine(logStart);
switch (provider.ProviderType)
{
case ProviderType.Both:
await GenerateApprenticeshipReport(
configuration, apprenticeshipCollectionService, migrationReportCollectionService, provider,
migratedStatusList, cosmosClient);
apprenticeshipsReportEntryCount++;
await GenerateFECourseReport(
configuration, courseCollectionService, migrationReportCollectionService, provider,
migratedStatusList, cosmosClient);
feCourseReportEntryCount++;
break;
case ProviderType.Apprenticeship:
await GenerateApprenticeshipReport(
configuration, apprenticeshipCollectionService, migrationReportCollectionService, provider,
migratedStatusList, cosmosClient);
apprenticeshipsReportEntryCount++;
break;
case ProviderType.FE:
await GenerateFECourseReport(
configuration, courseCollectionService, migrationReportCollectionService, provider,
migratedStatusList, cosmosClient);
feCourseReportEntryCount++;
break;
case ProviderType.Undefined:
break;
default:
throw new ArgumentOutOfRangeException();
}
migrationLog.AppendLine($"COMPLETED : Report for provider {provider.UnitedKingdomProviderReferenceNumber}");
}
catch (Exception ex)
{
migrationLog.AppendLine($"Error creating report for {ukprn}. {ex.GetBaseException().Message}");
log.LogError(ex, $"Error creating report for {ukprn}.");
}
}
stopWatch.Stop();
migrationLog.AppendLine("----------------------------------------------------------------");
migrationLog.AppendLine($"Completed Migration Report generation in {stopWatch.Elapsed.TotalMinutes} minutes.");
migrationLog.AppendLine($"Course Report Entries : {feCourseReportEntryCount} for {migratedProvidersCount} migrated providers.");
migrationLog.AppendLine($"Apps Report Entries : {apprenticeshipsReportEntryCount} for {migratedProvidersCount} migrated providers");
migrationLog.AppendLine("----------------------------------------------------------------");
log.LogDebug(migrationLog.ToString());
await blobHelper.UploadFile(
blobContainer,
$"MigrationReport_LogFile-{DateTime.Now:dd-MM-yy HHmm}.txt",
GetResultAsByteArray(migrationLog));
log.LogInformation($"Completed Migration Report generation. {feCourseReportEntryCount + apprenticeshipsReportEntryCount} records processed.");
}
public static async Task Run(
string input, // Work around https://github.com/Azure/azure-functions-vs-build-sdk/issues/168
ILogger log,
[Inject] IConfigurationRoot configuration,
[Inject] IApprenticeshipCollectionService apprenticeshipCollectionService,
[Inject] ICosmosDbHelper cosmosDbHelper,
[Inject] IBlobStorageHelper blobhelper,
[Inject] IUkrlpApiService ukrlpApiService
)
{
const string apprenticeshipAppName = "Validate.Apprenticeship";
var apprenticeshipValidationFileName = $"{apprenticeshipAppName}-{DateTime.Now.ToString("dd-MM-yy HHmm")}";
var blobContainer = blobhelper.GetBlobContainer(configuration["BlobStorageSettings:Container"]);
var validator = new ApprenticeshipValidator();
List <ApprenticeshipValidationResult> validationEResult = new List <ApprenticeshipValidationResult>();
var stopWatch = new Stopwatch();
stopWatch.Start();
var resultData = await apprenticeshipCollectionService.GetAllApprenticeshipsAsync();
foreach (var item in resultData)
{
//item.ContactEmail = "testing";
//item.Url = "testing";
//item.ContactWebsite = "testing";
//item.ContactTelephone = "101101abc";
//item.ApprenticeshipTitle = item.ApprenticeshipTitle + " @";
var validationResult = validator.Validate(item);
if (!validationResult.IsValid)
{
foreach (var error in validationResult.Errors)
{
validationEResult.Add(new ApprenticeshipValidationResult()
{
ApprenticeshipId = item.ApprenticeshipId,
ProviderUkprn = item.ProviderUKPRN,
FieldName = error.PropertyName,
ErrorMessage = error.ErrorMessage
});
}
}
}
var resultsObjBytes = GetResultAsByteArray(validationEResult);
await WriteResultsToBlobStorage(resultsObjBytes);
stopWatch.Stop();
byte[] GetResultAsByteArray(IList <ApprenticeshipValidationResult> message)
{
using (var memoryStream = new System.IO.MemoryStream())
{
using (var streamWriter = new System.IO.StreamWriter(memoryStream))
using (var csvWriter = new CsvWriter(streamWriter, CultureInfo.InvariantCulture))
{
csvWriter.WriteRecords <ApprenticeshipValidationResult>(message);
}
return(memoryStream.ToArray());
}
}
async Task WriteResultsToBlobStorage(byte[] data)
{
await blobhelper.UploadFile(blobContainer, apprenticeshipValidationFileName, data);
}
}
