private static async Task GenerateFECourseReport(
IConfiguration configuration,
ICourseCollectionService courseCollectionService,
IMigrationReportCollectionService migrationReportCollectionService,
Provider provider,
IReadOnlyList <RecordStatus> migratedStatusList,
IDocumentClient cosmosClient)
{
var ukprn = int.Parse(provider.UnitedKingdomProviderReferenceNumber);
var courseReportEntry = await migrationReportCollectionService.GetReportForCoursesByUkprn(ukprn) ??
new MigrationReportEntry();
var courses = await courseCollectionService.GetAllCoursesByUkprnAsync(ukprn);
courseReportEntry.CreatedOn = DateTime.Now;
courseReportEntry.CreatedBy = AppName;
courseReportEntry.id = provider.UnitedKingdomProviderReferenceNumber;
courseReportEntry.MigrationDate = provider.DateUpdated;
courseReportEntry.ProviderType = (int)provider.ProviderType;
courseReportEntry.ProviderName = provider.ProviderName;
courseReportEntry.FailedMigrationCount = 0;
courseReportEntry.MigrationRate = FeCourseMigrationRate(courses, migratedStatusList);
courseReportEntry.MigrationPendingCount = courses
.SelectMany(x => x.CourseRuns.Where(cr => cr.RecordStatus == RecordStatus.MigrationPending)).Count();
courseReportEntry.MigrationReadyToGoLive = courses
.SelectMany(x => x.CourseRuns.Where(cr => cr.RecordStatus == RecordStatus.MigrationReadyToGoLive))
.Count();
courseReportEntry.BulkUploadPendingcount = courses
.SelectMany(x => x.CourseRuns.Where(cr => cr.RecordStatus == RecordStatus.BulkUloadPending)).Count();
courseReportEntry.BulkUploadReadyToGoLiveCount = courses
.SelectMany(x => x.CourseRuns.Where(cr => cr.RecordStatus == RecordStatus.BulkUploadReadyToGoLive))
.Count();
courseReportEntry.LiveCount =
courses.SelectMany(c => c.CourseRuns.Where(cr => cr.RecordStatus == RecordStatus.Live))
.Count();
courseReportEntry.PendingCount =
courses.SelectMany(c => c.CourseRuns.Where(cr => cr.RecordStatus == RecordStatus.Pending))
.Count();
courseReportEntry.MigratedCount = courses
.SelectMany(c => c.CourseRuns.Where(cr => migratedStatusList.Contains(cr.RecordStatus)))
.Count();
var collectionUri = UriFactory.CreateDocumentCollectionUri(
configuration["CosmosDbSettings:DatabaseId"],
configuration["CosmosDbCollectionSettings:MigrationReportCoursesCollectionId"]);
await cosmosClient.UpsertDocumentAsync(collectionUri, courseReportEntry);
}
public static async Task Run(
[TimerTrigger("%MigrationReportSchedule%")] TimerInfo myTimer,
ILogger log,
[Inject] IConfigurationRoot configuration,
[Inject] ICosmosDbHelper cosmosDbHelper,
[Inject] IBlobStorageHelper blobHelper,
[Inject] IProviderCollectionService providerCollectionService,
[Inject] ICourseCollectionService courseCollectionService,
[Inject] IApprenticeshipCollectionService apprenticeshipCollectionService,
[Inject] IMigrationReportCollectionService migrationReportCollectionService)
{
await new MigrationReportGeneratorService().Run(
log, configuration, cosmosDbHelper, blobHelper, providerCollectionService, courseCollectionService,
apprenticeshipCollectionService, migrationReportCollectionService
);
}
private static async Task <string> GenerateCoursesExportForProvider(
ILogger log,
ICourseCollectionService courseCollectionService,
StringBuilder logFile,
IMiragtionProviderItem mpItem,
CloudBlobContainer containerExporter,
string containerNameExporter)
{
logFile.AppendLine($"Conditional logic for {mpItem} is True.");
logFile.AppendLine($"\tAttempting to get courses' data for: {mpItem}");
var courses = await courseCollectionService.GetAllLiveCoursesAsJsonForUkprnAsync(mpItem.Ukprn);
logFile.AppendLine($"\tGot courses' data for: {mpItem}");
if (courses != "[]")
{
logFile.AppendLine($"\t\tHas courses' data for: {mpItem}");
var coursesFileName =
$"{DateTime.Today.ToString("yyyyMMdd")}\\Generated\\Courses_for_Provider_{mpItem.Ukprn}_{DateTime.Now.ToString("yyyy-MM-ddTHH-mm-ss")}.json";
logFile.AppendLine($"\t\tGot reference to block blob containers for file: {coursesFileName}");
var coursesBlob = containerExporter.GetBlockBlobReference(coursesFileName);
logFile.AppendLine(
$"\t\tAttempting to upload file {coursesFileName} to blob container {containerNameExporter}");
await coursesBlob.UploadTextAsync(courses);
log.LogInformation($"uploaded {coursesFileName}");
logFile.AppendLine($"\t\tUploaded file {coursesFileName} to blob container {containerNameExporter}");
return(coursesFileName);
}
else
{
logFile.AppendLine($"\t\tHas no courses' data for: {mpItem}");
return(null);
}
}
public static async Task Run(
[TimerTrigger("%schedule%")] TimerInfo myTimer,
ILogger log,
[Inject] IOptions <ExporterSettings> exporterSettings,
[Inject] IBlobStorageHelper blobStorageHelper,
[Inject] IProviderCollectionService providerCollectionService,
[Inject] ICourseCollectionService courseCollectionService,
[Inject] IVenueCollectionService venueCollectionService)
{
var configuration = exporterSettings.Value;
log.LogInformation("[Export] waiting for trigger...");
//TODO: add more logging after you get this working ...
var logFile = new StringBuilder();
logFile.AppendLine($"Starting {nameof(Export)} at {DateTime.Now}");
var fileNames = new List <string>();
var startDate = configuration.ExporterStartDate;
var providersFileName = $"{DateTime.Today.ToString("yyyyMMdd")}\\Generated\\Providers_{DateTime.Now.ToString("yyyy-MM-ddTHH-mm-ss")}.json";
logFile.AppendLine($"Start date: {startDate:dd/MM/yyyy hh:mm}");
logFile.AppendLine($"Provider filename: {providersFileName}");
var containerNameExporter = configuration.ContainerNameExporter;
var containerNameProviderFiles = configuration.ContainerNameProviderFiles;
var migrationProviderCsv = configuration.MigrationProviderCsv;
logFile.AppendLine($"Attempting to get reference to blob containers: {containerNameExporter}, {containerNameProviderFiles}");
var containerExporter = blobStorageHelper.GetBlobContainer(containerNameExporter);
var containerProviderFiles = blobStorageHelper.GetBlobContainer(containerNameProviderFiles);
logFile.AppendLine($"Got references to blob containers: {containerNameExporter}, {containerNameProviderFiles}");
try
{
log.LogInformation("[Export] grabbing providers");
var providersForExport = await GetProvidersFromCsv(migrationProviderCsv, blobStorageHelper, logFile, containerProviderFiles);
var providerFileName = await GenerateProvidersExport(providerCollectionService, providersForExport, logFile, providersFileName, containerExporter, containerNameExporter);
fileNames.Add(providerFileName);
var count = 0;
var total = providersForExport.Count();
// N.B. Deliberately not doing these in parallel to avoid creating too many DocumentClients...
foreach (var provider in providersForExport)
{
count++;
log.LogInformation($"[Export] checking {provider.Ukprn} [{count} of {total}]");
var export = await CheckForProviderUpdates(log, courseCollectionService,
venueCollectionService, logFile, provider, startDate, containerExporter,
containerNameExporter)
.ConfigureAwait(false);
fileNames.AddRange(export);
}
var fileNamesFileName = $"{DateTime.Today.ToString("yyyyMMdd")}\\Generated\\FileNames.json";
var fileNamesBlob = containerExporter.GetBlockBlobReference(fileNamesFileName);
await fileNamesBlob.UploadTextAsync(JsonConvert.SerializeObject(fileNames, Formatting.Indented));
}
catch (Exception e)
{
logFile.AppendLine(e.Message);
logFile.AppendLine(e.ToString());
throw;
}
finally
{
logFile.AppendLine($"Ending {nameof(Export)} at {DateTime.Now}");
var logFileName = $"{DateTime.Today.ToString("yyyyMMdd")}\\Generated\\Log_{DateTime.Now.ToString("yyyy-MM-ddTHH-mm-ss")}.txt";
var logFileNameBlob = containerExporter.GetBlockBlobReference(logFileName);
await logFileNameBlob.UploadTextAsync(logFile.ToString());
}
}
private static async Task <IEnumerable <string> > CheckForProviderUpdates(
ILogger log,
ICourseCollectionService courseCollectionService,
IVenueCollectionService venueCollectionService,
StringBuilder logFile,
IMiragtionProviderItem mpItem,
DateTime fromDate,
CloudBlobContainer containerExporter,
string containerNameExporter)
{
var fileNames = new List <string>();
logFile.AppendLine($"Attempting to get conditional data for: {mpItem}");
var hasTodaysDate = mpItem.DateMigrated.Date == DateTime.Today;
var dateMigratedIsInThePast = mpItem.DateMigrated.Date < DateTime.Today;
var hasCreatedCourses = await courseCollectionService.HasCoursesBeenCreatedSinceAsync(mpItem.Ukprn, fromDate);
var hasCreatedCourseRuns =
await courseCollectionService.HasCourseRunsBeenCreatedSinceAsync(mpItem.Ukprn, fromDate);
var hasUpdatedCourses = await courseCollectionService.HasCoursesBeenUpdatedSinceAsync(mpItem.Ukprn, fromDate);
var hasUpdatedCourseRuns =
await courseCollectionService.HasCourseRunsBeenUpdatedSinceAsync(mpItem.Ukprn, fromDate);
var hasDeletedCourses = await courseCollectionService.HasCoursesBeenDeletedSinceAsync(mpItem.Ukprn, fromDate);
var hasDeletedCourseRuns =
await courseCollectionService.HasCourseRunsBeenDeletedSinceAsync(mpItem.Ukprn, fromDate);
var hasUpdatedVenues = await venueCollectionService.HasBeenAnUpdatedSinceAsync(mpItem.Ukprn, fromDate);
logFile.AppendLine($"Got conditional data for: {mpItem}");
logFile.AppendLine($"\tHas today's date: {hasTodaysDate}");
logFile.AppendLine($"\tDate migrated is in the past: {dateMigratedIsInThePast}");
logFile.AppendLine($"\tHas created Courses: {hasCreatedCourses}");
logFile.AppendLine($"\tHas created CourseRuns: {hasCreatedCourseRuns}");
logFile.AppendLine($"\tHas updated Courses: {hasUpdatedCourses}");
logFile.AppendLine($"\tHas updated CourseRuns: {hasUpdatedCourseRuns}");
logFile.AppendLine($"\tHas deleted Courses: {hasDeletedCourses}");
logFile.AppendLine($"\tHas deleted CourseRuns: {hasDeletedCourseRuns}");
logFile.AppendLine($"\tHas updated Venues: {hasUpdatedVenues}");
logFile.AppendLine($"End of conditional data for: {mpItem}");
if (hasTodaysDate || (dateMigratedIsInThePast))
{
if (hasCreatedCourses || hasCreatedCourseRuns || hasUpdatedCourses || hasUpdatedCourseRuns ||
hasDeletedCourses || hasDeletedCourseRuns)
{
log.LogInformation($"updating courses for {mpItem.Ukprn}");
var courseFilename = await GenerateCoursesExportForProvider(log, courseCollectionService, logFile, mpItem, containerExporter,
containerNameExporter).ConfigureAwait(false);
fileNames.Add(courseFilename);
}
if (hasUpdatedVenues)
{
log.LogInformation($"updating venues for {mpItem.Ukprn}");
var venueFilename = await GenerateVenuesExportForProvider(log, venueCollectionService, logFile, mpItem, containerExporter,
containerNameExporter).ConfigureAwait(false);
fileNames.Add(venueFilename);
}
}
else
{
logFile.AppendLine($"Conditional logic for {mpItem} is False.");
}
return(fileNames);
}
public async Task Run(
ILogger log,
IConfigurationRoot configuration,
ICosmosDbHelper cosmosDbHelper,
IBlobStorageHelper blobHelper,
IProviderCollectionService providerCollectionService,
ICourseCollectionService courseCollectionService,
IApprenticeshipCollectionService apprenticeshipCollectionService,
IMigrationReportCollectionService migrationReportCollectionService)
{
log.LogInformation("Starting Migration Report generation");
var migrationLog = new StringBuilder();
migrationLog.AppendLine("-------------------------------------------------------");
migrationLog.AppendLine("Starting Migration Report generation");
migrationLog.AppendLine("-------------------------------------------------------");
var stopWatch = new Stopwatch();
stopWatch.Start();
IReadOnlyList <RecordStatus> migratedStatusList = new List <RecordStatus>
{
RecordStatus.Live,
RecordStatus.MigrationPending,
};
var blobContainer = blobHelper.GetBlobContainer(configuration["BlobStorageSettings:Container"]);
var whiteListedProviders = await GetProviderWhiteList(blobHelper, blobContainer);
var cosmosClient = cosmosDbHelper.GetClient();
log.LogDebug("Fetching migrated provider count...");
var migratedProvidersCount = (await providerCollectionService.GetAllMigratedProviders("Provider.Migrator")).Count;
log.LogDebug($"Migrated Provider count: {migratedProvidersCount}.");
log.LogDebug("Fetching providers...");
var providers = await providerCollectionService.GetDocumentsByUkprn(whiteListedProviders);
var providerTypeCounts = providers.GroupBy(t => t.ProviderType).Select(g => new { type = g.Key, qty = g.Count() });
log.LogDebug($"Provider counts: {string.Join("; ", providerTypeCounts.Select(c => $"{c.type}: {c.qty}"))}. Total: {providers.Count}");
int progress = 1;
int feCourseReportEntryCount = 0;
int apprenticeshipsReportEntryCount = 0;
foreach (var ukprn in whiteListedProviders)
{
try
{
var provider = providers.Single(p => int.Parse(p.UnitedKingdomProviderReferenceNumber) == ukprn);
var logStart = $"STARTED : Generating report for provider with UKPRN: {provider.UnitedKingdomProviderReferenceNumber}. Progress: {progress++}/{whiteListedProviders.Count}";
log.LogDebug(logStart);
migrationLog.AppendLine(logStart);
switch (provider.ProviderType)
{
case ProviderType.Both:
await GenerateApprenticeshipReport(
configuration, apprenticeshipCollectionService, migrationReportCollectionService, provider,
migratedStatusList, cosmosClient);
apprenticeshipsReportEntryCount++;
await GenerateFECourseReport(
configuration, courseCollectionService, migrationReportCollectionService, provider,
migratedStatusList, cosmosClient);
feCourseReportEntryCount++;
break;
case ProviderType.Apprenticeship:
await GenerateApprenticeshipReport(
configuration, apprenticeshipCollectionService, migrationReportCollectionService, provider,
migratedStatusList, cosmosClient);
apprenticeshipsReportEntryCount++;
break;
case ProviderType.FE:
await GenerateFECourseReport(
configuration, courseCollectionService, migrationReportCollectionService, provider,
migratedStatusList, cosmosClient);
feCourseReportEntryCount++;
break;
case ProviderType.Undefined:
break;
default:
throw new ArgumentOutOfRangeException();
}
migrationLog.AppendLine($"COMPLETED : Report for provider {provider.UnitedKingdomProviderReferenceNumber}");
}
catch (Exception ex)
{
migrationLog.AppendLine($"Error creating report for {ukprn}. {ex.GetBaseException().Message}");
log.LogError(ex, $"Error creating report for {ukprn}.");
}
}
stopWatch.Stop();
migrationLog.AppendLine("----------------------------------------------------------------");
migrationLog.AppendLine($"Completed Migration Report generation in {stopWatch.Elapsed.TotalMinutes} minutes.");
migrationLog.AppendLine($"Course Report Entries : {feCourseReportEntryCount} for {migratedProvidersCount} migrated providers.");
migrationLog.AppendLine($"Apps Report Entries : {apprenticeshipsReportEntryCount} for {migratedProvidersCount} migrated providers");
migrationLog.AppendLine("----------------------------------------------------------------");
log.LogDebug(migrationLog.ToString());
await blobHelper.UploadFile(
blobContainer,
$"MigrationReport_LogFile-{DateTime.Now:dd-MM-yy HHmm}.txt",
GetResultAsByteArray(migrationLog));
log.LogInformation($"Completed Migration Report generation. {feCourseReportEntryCount + apprenticeshipsReportEntryCount} records processed.");
}
public static async Task Run(
string input, // Work around https://github.com/Azure/azure-functions-vs-build-sdk/issues/168
ILogger log,
[Inject] IConfigurationRoot configuration,
[Inject] ICourseCollectionService courseCollectionService,
[Inject] ICosmosDbHelper cosmosDbHelper,
[Inject] IBlobStorageHelper blobhelper,
[Inject] IUkrlpApiService ukrlpApiService
)
{
const string apprenticeshipAppName = "Validate.Course";
var apprenticeshipValidationFileName = $"{apprenticeshipAppName}-{DateTime.Now.ToString("dd-MM-yy HHmm")}";
var blobContainer = blobhelper.GetBlobContainer(configuration["BlobStorageSettings:Container"]);
var validator = new CourseValidator();
List <CourseValidationResult> validationEResult = new List <CourseValidationResult>();
var stopWatch = new Stopwatch();
stopWatch.Start();
var resultData = await courseCollectionService.GetAllCoursesAsync();
foreach (var item in resultData)
{
//item.ContactEmail = "testing";
//item.Url = "testing";
//item.ContactWebsite = "testing";
//item.ContactTelephone = "101101abc";
//item.ApprenticeshipTitle = item.ApprenticeshipTitle + " @";
var validationResult = validator.Validate(item);
if (!validationResult.IsValid)
{
foreach (var error in validationResult.Errors)
{
validationEResult.Add(new CourseValidationResult()
{
CourseId = item.CourseId,
ProviderUkprn = item.ProviderUKPRN,
FieldName = error.PropertyName,
ErrorMessage = error.ErrorMessage
});
}
}
}
var resultsObjBytes = GetResultAsByteArray(validationEResult);
await WriteResultsToBlobStorage(resultsObjBytes);
stopWatch.Stop();
byte[] GetResultAsByteArray(IList <CourseValidationResult> message)
{
using (var memoryStream = new System.IO.MemoryStream())
{
using (var streamWriter = new System.IO.StreamWriter(memoryStream))
using (var csvWriter = new CsvWriter(streamWriter, CultureInfo.InvariantCulture))
{
csvWriter.WriteRecords <CourseValidationResult>(message);
}
return(memoryStream.ToArray());
}
}
async Task WriteResultsToBlobStorage(byte[] data)
{
await blobhelper.UploadFile(blobContainer, apprenticeshipValidationFileName, data);
}
}
