public static async Task Run(
string input, // Work around https://github.com/Azure/azure-functions-vs-build-sdk/issues/168
ILogger logger,
[Inject] IConfigurationRoot configuration,
[Inject] IVenueCollectionService venueCollectionService,
[Inject] IProviderCollectionService providerCollectionService,
[Inject] ICosmosDbHelper cosmosDbHelper,
[Inject] IBlobStorageHelper blobhelper
)
{
logger.LogDebug("VenueRestorer: Starting...");
var connectionString = configuration.GetConnectionString("TribalRestore");
var venuesCollectionId = configuration["CosmosDbCollectionSettings:VenuesCollectionId"];
var blobContainer = configuration["BlobStorageSettings:Container"];
var container = configuration["BlobStorageSettings:Container"];
var whiteListProviders = await GetProviderWhiteList();
logger.LogDebug($"VenueRestorer: {whiteListProviders.Count} white-listed providers to process");
var coursesCollectionId = "courses";
var apprenticeshipCollectionId = "apprenticeship";
var databaseId = configuration["CosmosDbSettings:DatabaseId"];
var documentClient = cosmosDbHelper.GetClient();
var venueCollectionUri = UriFactory.CreateDocumentCollectionUri(databaseId, venuesCollectionId);
var venueCollection_OldUri = UriFactory.CreateDocumentCollectionUri(databaseId, "venues_old");
var coursesCollectionUri = UriFactory.CreateDocumentCollectionUri(databaseId, coursesCollectionId);
var apprenticeshipCollectionUri = UriFactory.CreateDocumentCollectionUri(databaseId, apprenticeshipCollectionId);
var logFileName = $"VenueRestorer--{DateTime.Now.ToString("dd-MM-yy HHmm")}";
var allVenues = GetAllOldVenues();
var result = new List <ResultMessage>();
const string WHITE_LIST_FILE = "ProviderWhiteList.txt";
var ukprnCache = new List <int>();
var updatedBy = "VenueRestorer";
//grand totals
var invalidCourseRunReferences = 0;
var totalInvalidCourseRunReferences = 0;
var uniqueInvalidVenues = new HashSet <string>();
var replacedInvalidVenues = new HashSet <Venue>();
var references = new List <VenueRestorerReference>();
var rereferencedApprenticeshipLocations = 0;
//provider scoped totals
var totalInvalidApprenticeshipLocationReferences = 0;
var invalidApprenticeshipLocationReferences = 0;
int processedProviderCount = 0;
using (var logStream = new MemoryStream())
using (var logStreamWriter = new StreamWriter(logStream))
using (var logCsvWriter = new CsvWriter(logStreamWriter, CultureInfo.InvariantCulture))
{
foreach (var ukprn in whiteListProviders)
{
//reference for old venue so that courseruns & apprenticeship locations can be
//re-referenced
var venuesReplacedForProvider = new List <Tuple <Venue, Venue, Guid> >();
try
{
//reset counters
invalidCourseRunReferences = 0;
invalidApprenticeshipLocationReferences = 0;
//fetch data for ukprn
var allCoursesForProvider = await GetCourses(ukprn);
var allApprenticeshipsForProvider = await GetApprenticeships(ukprn);
var venues = await GetVenues(ukprn);
var old_venues = GetOldVenues(ukprn);
//courses
foreach (var course in allCoursesForProvider)
{
//course runs
foreach (var courserun in course.CourseRuns)
{
//only courses that references a venue (classroom based or both)
if (courserun.VenueId != null && courserun.VenueId != Guid.Empty)
{
//current venue & old venue (pre migration)
var invalidReferencedVenue = await GetVenueById(courserun.VenueId?.ToString());
var restoredVenue = old_venues.FirstOrDefault(x => new Guid(x.ID) == courserun.VenueId);
//if current venues provider is different to course then attempt to replace it with
//old venue from cosmos backup.
if (invalidReferencedVenue != null && invalidReferencedVenue.UKPRN != course.ProviderUKPRN)
{
//replace existing venue with old venue if a match is found
if (restoredVenue != null && restoredVenue.UKPRN == course.ProviderUKPRN)
{
await ReplaceVenue(invalidReferencedVenue.ID, restoredVenue, updatedBy);
//the venue that was referenced needs to be inserted again but with a new id.
var newId = Guid.NewGuid();
await ReplaceVenue(newId.ToString(), invalidReferencedVenue, updatedBy);
//reload venues as we have just replaced a venue
venues = await GetVenues(ukprn);
replacedInvalidVenues.Add(invalidReferencedVenue);
//store old venue so that apprenticeship locations can be re-referenced
venuesReplacedForProvider.Add(Tuple.Create(restoredVenue, invalidReferencedVenue, newId));
//log changes
references.Add(new VenueRestorerReference()
{
UKPRN = course.ProviderUKPRN,
VenueId = courserun.VenueId.ToString(),
CurrentVenueUKPRN = invalidReferencedVenue.UKPRN,
CurrentAddress1 = invalidReferencedVenue.Address1,
CurrentPostcode = invalidReferencedVenue.PostCode,
CurrentVenueName = invalidReferencedVenue.VenueName,
RestoredAddress1 = restoredVenue.Address1,
RestoredVenueName = restoredVenue.VenueName,
RestoredPostcode = restoredVenue.PostCode,
RestoredVenueUKPRN = restoredVenue.UKPRN,
UKPRNMatched = (course.ProviderUKPRN == invalidReferencedVenue.UKPRN),
Message = "Replaced Venue",
Type = "Course",
CourseId = course.id,
CourseRunId = courserun.id,
});
}
else
{
references.Add(new VenueRestorerReference()
{
UKPRN = course.ProviderUKPRN,
VenueId = courserun.VenueId.ToString(),
CurrentVenueUKPRN = invalidReferencedVenue.UKPRN,
CurrentAddress1 = invalidReferencedVenue.Address1,
CurrentPostcode = invalidReferencedVenue.PostCode,
CurrentVenueName = invalidReferencedVenue.VenueName,
UKPRNMatched = (course.ProviderUKPRN == invalidReferencedVenue.UKPRN),
Message = "Unable to replace Venue, as old venue was not found in backup",
Type = "Course",
CourseId = course.id,
CourseRunId = courserun.id,
});
}
//invalid references
uniqueInvalidVenues.Add(invalidReferencedVenue.ID);
invalidCourseRunReferences++;
}
}
}
//total for all providers
totalInvalidCourseRunReferences += invalidCourseRunReferences;
}
//apprenticeships
foreach (var apprenticeship in allApprenticeshipsForProvider)
{
//apprenticeship locations
foreach (var location in apprenticeship.ApprenticeshipLocations)
{
//only apprenticeshiplocations that references a venue (classroom based or both)
if (location.VenueId.HasValue && location.LocationGuidId.HasValue && location.LocationGuidId != Guid.Empty)
{
var invalidReferencedVenue = await GetVenueById(location.LocationGuidId?.ToString());
if (invalidReferencedVenue != null && invalidReferencedVenue.UKPRN != apprenticeship.ProviderUKPRN)
{
var restoredVenue = old_venues.FirstOrDefault(x => new Guid(x.ID) == location.LocationGuidId);
//replace existing venue with old venue if a match is found
if (restoredVenue != null && restoredVenue.UKPRN == apprenticeship.ProviderUKPRN)
{
logger.LogDebug($"VenueRestorer: Invalid Apprenticeship location, apprenticeship should reference {restoredVenue.VenueName}");
//old venue from json backup is the correct venue that should be referenced
//swap invalid venue, with restoredVenue
await ReplaceVenue(invalidReferencedVenue.ID, restoredVenue, updatedBy);
//the venue that was referenced needs to be inserted again but with a new id.
var newId = Guid.NewGuid();
await ReplaceVenue(newId.ToString(), invalidReferencedVenue, updatedBy);
//store old venue so that apprenticeship locations can be re-referenced
venuesReplacedForProvider.Add(Tuple.Create(restoredVenue, invalidReferencedVenue, newId));
//reload venues as we have just replaced a venue
venues = await GetVenues(ukprn);
//keep a track of the incorrect venue, these will be inserted with a new id.
replacedInvalidVenues.Add(invalidReferencedVenue);
references.Add(new VenueRestorerReference()
{
UKPRN = apprenticeship.ProviderUKPRN,
ApprenticeshipLocationUKPRN = location.ProviderUKPRN,
VenueId = location.LocationGuidId.ToString(),
CurrentVenueUKPRN = invalidReferencedVenue.UKPRN,
CurrentAddress1 = invalidReferencedVenue.Address1,
CurrentPostcode = invalidReferencedVenue.PostCode,
CurrentVenueName = invalidReferencedVenue.VenueName,
RestoredVenueUKPRN = restoredVenue.UKPRN,
RestoredAddress1 = restoredVenue?.Address1,
RestoredPostcode = restoredVenue?.PostCode,
RestoredVenueName = restoredVenue.VenueName,
UKPRNMatched = (apprenticeship.ProviderUKPRN == invalidReferencedVenue.UKPRN),
Message = "Replaced Venue",
Type = "Apprenticeship",
ApprenticeshipId = apprenticeship.id
});
}
else
{
references.Add(new VenueRestorerReference()
{
UKPRN = apprenticeship.ProviderUKPRN,
ApprenticeshipLocationUKPRN = location.ProviderUKPRN,
UKPRNMatched = false,
CurrentVenueUKPRN = -1,
VenueId = location.LocationGuidId.ToString(),
Type = "Apprenticeship",
Message = "Unable to replace Venue, as old venue was not found in backup",
ApprenticeshipId = apprenticeship.id
});
}
}
}
}
totalInvalidApprenticeshipLocationReferences += invalidApprenticeshipLocationReferences;
}
//rereference apprenticeship locations
//if there is a venue that has been replaced but is referenced by an apprenticeshiplocation then the apprenticeship
//record needs to be updated to point the the new venue record to save data loss.
foreach (var apprenticeship in allApprenticeshipsForProvider)
{
var updated = false;
foreach (var location in apprenticeship.ApprenticeshipLocations)
{
var replacedVenue = venuesReplacedForProvider.FirstOrDefault(x => new Guid(x.Item2.ID) == location.LocationGuidId);
if (replacedVenue != null)
{
updated = true;
location.LocationGuidId = replacedVenue.Item3;
}
}
if (updated)
{
var documentLink = UriFactory.CreateDocumentUri(databaseId, apprenticeshipCollectionId, apprenticeship.id.ToString());
await documentClient.ReplaceDocumentAsync(documentLink, apprenticeship, new RequestOptions()
{
PartitionKey = new Microsoft.Azure.Documents.PartitionKey(ukprn)
});
}
}
}
catch (Exception e)
{
logger.LogError($"VenueRestorer: error for ukprn {ukprn}: {e.Message}", e);
}
processedProviderCount++;
if (processedProviderCount % 100 == 0)
{
logger.LogInformation($"VenueRestorer: {processedProviderCount}/{whiteListProviders.Count} providers processed.");
}
logger.LogDebug(
$"VenueRestorer: completed for UKPRN {ukprn}, {processedProviderCount}/{whiteListProviders.Count}. " +
$"{invalidCourseRunReferences} invalid venue references, " +
$"{invalidApprenticeshipLocationReferences} invalid apprenticeship references, " +
$"{rereferencedApprenticeshipLocations} apprenticeship locations were re-referenced.");
}
//write csv file
logCsvWriter.WriteHeader(typeof(VenueRestorerReference));
logCsvWriter.NextRecord();
foreach (var reference in references)
{
logCsvWriter.WriteRecord(reference);
logCsvWriter.NextRecord();
}
// Upload log CSV to blob storage
logStreamWriter.Flush();
logStream.Seek(0L, SeekOrigin.Begin);
var blob = blobhelper.GetBlobContainer(blobContainer).GetBlockBlobReference(logFileName);
await blob.UploadFromStreamAsync(logStream);
logger.LogInformation($"VenueRestorer: log uploaded as {logFileName}");
}
logger.LogInformation(
$"VenueRestorer: completed. " +
$"{totalInvalidCourseRunReferences} invalid CourseRun references in total, " +
$"{totalInvalidApprenticeshipLocationReferences} Apprenticeship location invalid references in total, " +
$"{replacedInvalidVenues.Count()} Venues have been reverted back to old venues, " +
$"{uniqueInvalidVenues.Count()} Venues were invalid");
async Task <List <Course> > GetCourses(int ukprn)
{
var courses = new List <Course>();
//Get all courses
string continuation = null;
do
{
var feedOptions = new FeedOptions()
{
RequestContinuation = continuation,
PartitionKey = new Microsoft.Azure.Documents.PartitionKey(ukprn)
};
var queryResponse = await documentClient.CreateDocumentQuery <Course>(coursesCollectionUri, feedOptions)
.Where(p => p.ProviderUKPRN == ukprn && p.CourseStatus != CourseDirectory.Models.Enums.RecordStatus.Archived)
.AsDocumentQuery()
.ExecuteNextAsync <Course>();
courses.AddRange(queryResponse.ToList());
continuation = queryResponse.ResponseContinuation;
}while (continuation != null);
return(courses);
}
async Task <List <Venue> > GetVenues(int ukprn)
{
var venues = new List <Venue>();
string continuation = null;
do
{
var feedOptions = new FeedOptions()
{
RequestContinuation = continuation
};
var queryResponse = await documentClient.CreateDocumentQuery <Venue>(venueCollectionUri, feedOptions)
.Where(p => p.UKPRN == ukprn)
.AsDocumentQuery()
.ExecuteNextAsync <Venue>();
venues.AddRange(queryResponse.ToList());
continuation = queryResponse.ResponseContinuation;
}while (continuation != null);
return(venues);
}
List <Venue> GetOldVenues(int ukprn)
{
return(allVenues.Where(x => x.UKPRN == ukprn).ToList());
}
async Task <Venue> GetVenueById(string id)
{
var collectionLink = UriFactory.CreateDocumentCollectionUri(databaseId, venuesCollectionId);
var query = documentClient
.CreateDocumentQuery <Venue>(collectionLink, new FeedOptions()
{
EnableCrossPartitionQuery = true
})
.Where(d => d.ID == id)
.AsDocumentQuery();
return((await query.ExecuteNextAsync()).FirstOrDefault());
}
async Task <List <Apprenticeship> > GetApprenticeships(int ukprn)
{
var apprenticeships = new List <Apprenticeship>();
string continuation = null;
do
{
var feedOptions = new FeedOptions()
{
RequestContinuation = continuation,
PartitionKey = new Microsoft.Azure.Documents.PartitionKey(ukprn)
};
try
{
var queryResponse = await documentClient.CreateDocumentQuery <Apprenticeship>(apprenticeshipCollectionUri, feedOptions)
.Where(p => p.ProviderUKPRN == ukprn && p.RecordStatus != CourseDirectory.Models.Enums.RecordStatus.Archived)
.AsDocumentQuery()
.ExecuteNextAsync <Apprenticeship>();
apprenticeships.AddRange(queryResponse);
continuation = queryResponse.ResponseContinuation;
}
catch (Exception)
{
continuation = null;
}
}while (continuation != null);
return(apprenticeships);
}
async Task <ISet <int> > GetProviderWhiteList()
{
var blob = blobhelper.GetBlobContainer(blobContainer).GetBlockBlobReference(WHITE_LIST_FILE);
var ms = new MemoryStream();
await blob.DownloadToStreamAsync(ms);
ms.Seek(0L, SeekOrigin.Begin);
var results = new HashSet <int>();
string line;
using (var reader = new StreamReader(ms))
{
while ((line = reader.ReadLine()) != null)
{
if (string.IsNullOrEmpty(line))
{
continue;
}
var ukprn = int.Parse(line);
results.Add(ukprn);
}
}
return(results);
}
async Task ReplaceVenue(string id, Venue matchedVenue, string updatedby)
{
Uri collectionUri = UriFactory.CreateDocumentCollectionUri(databaseId, venuesCollectionId);
var editedVenue = new Dfc.CourseDirectory.Models.Models.Venues.Venue()
{
ID = id,
UKPRN = matchedVenue.UKPRN,
VenueName = matchedVenue.VenueName,
Address1 = matchedVenue.Address1,
Address2 = matchedVenue.Address2,
Town = matchedVenue.Town,
County = matchedVenue.County,
PostCode = matchedVenue.PostCode,
Latitude = matchedVenue.Latitude,
Longitude = matchedVenue.Longitude,
Status = matchedVenue.Status,
UpdatedBy = updatedBy,
DateUpdated = matchedVenue.DateUpdated,
VenueID = matchedVenue.VenueID,
ProviderID = matchedVenue.ProviderID,
ProvVenueID = matchedVenue.ProvVenueID,
Email = matchedVenue.Email,
Website = matchedVenue.Website,
Telephone = matchedVenue.Telephone,
CreatedBy = matchedVenue.CreatedBy,
CreatedDate = DateTime.Now,
LocationId = matchedVenue.LocationId,
TribalLocationId = matchedVenue.TribalLocationId
};
await documentClient.UpsertDocumentAsync(collectionUri, editedVenue);
}
IList <Venue> GetAllOldVenues()
{
var list = new List <Venue>();
var lookupFileResourceName = "Dfc.ProviderPortal.TribalExporter.2020-01-24_0325-venues-backup.json";
using (var stream = typeof(VenueReferenceChecker).Assembly.GetManifestResourceStream(lookupFileResourceName))
using (StreamReader file = new StreamReader(stream))
{
using (JsonTextReader reader = new JsonTextReader(file))
{
while (reader.Read())
{
JArray o2 = (JArray)JToken.ReadFrom(reader);
foreach (var item in o2)
{
Venue v = item.ToObject <Venue>();
list.Add(v);
}
}
}
}
return(list);
}
}
public static async Task Run(
string input, // Work around https://github.com/Azure/azure-functions-vs-build-sdk/issues/168
ILogger log,
[Inject] IConfigurationRoot configuration,
[Inject] IVenueCollectionService venueCollectionService,
[Inject] IProviderCollectionService providerCollectionService,
[Inject] ICosmosDbHelper cosmosDbHelper,
[Inject] IBlobStorageHelper blobhelper
)
{
var venuesCollectionId = configuration["CosmosDbCollectionSettings:VenuesCollectionId"];
var connectionString = configuration.GetConnectionString("TribalRestore");
var blobContainer = configuration["BlobStorageSettings:Container"];
var container = configuration["BlobStorageSettings:Container"];
var whiteListProviders = await GetProviderWhiteList();
var coursesCollectionId = "courses";
var apprenticeshipCollectionId = "apprenticeship";
var databaseId = configuration["CosmosDbSettings:DatabaseId"];
var documentClient = cosmosDbHelper.GetClient();
var venueCollectionUri = UriFactory.CreateDocumentCollectionUri(databaseId, venuesCollectionId);
var coursesCollectionUri = UriFactory.CreateDocumentCollectionUri(databaseId, coursesCollectionId);
var apprenticeshipCollectionUri = UriFactory.CreateDocumentCollectionUri(databaseId, apprenticeshipCollectionId);
var logFileName = $"ProvidersWithInvalidVenueReferences--{DateTime.Now.ToString("dd-MM-yy HHmm")}";
var result = new List <VenueReference>();
const string WHITE_LIST_FILE = "ProviderWhiteList.txt";
//counters
var ukprnsThatFailedToFetchVenues = 0;
var uniqueInvalidVenues = new HashSet <string>();
var replacedInvalidVenues = new HashSet <string>();
var allVenues = GetAllOldVenues();
//grand totals
var totalInvalidApprenticeshipLocationReferences = 0;
var totalInvalidCourseRunReferences = 0;
//provider scoped totals
var invalidCourseRunReferences = 0;
var invalidApprenticeshipLocationReferences = 0;
using (var logStream = new MemoryStream())
using (var logStreamWriter = new StreamWriter(logStream))
using (var logCsvWriter = new CsvWriter(logStreamWriter, CultureInfo.InvariantCulture))
{
//every provider
foreach (var ukprn in whiteListProviders)
{
try
{
//reset counters
invalidCourseRunReferences = 0;
invalidApprenticeshipLocationReferences = 0;
//fetch data for ukprn
var allCoursesForProvider = await GetCourses(ukprn);
var allApprenticeshipsForProvider = await GetApprenticeships(ukprn);
//courses
foreach (var course in allCoursesForProvider)
{
//course runs
foreach (var courserun in course.CourseRuns)
{
//only courses that references a venue (classroom based or both)
if (courserun.VenueId != null && courserun.VenueId != Guid.Empty)
{
//current venue & old venue (pre migration)
var currentVenue = await GetVenueById(courserun.VenueId?.ToString());
if (currentVenue != null)
{
if (course.ProviderUKPRN != currentVenue.UKPRN)
{
uniqueInvalidVenues.Add(courserun.VenueId.ToString());
invalidCourseRunReferences++;
}
result.Add(new VenueReference()
{
UKPRN = course.ProviderUKPRN,
VenueId = courserun.VenueId.ToString(),
VenueUKPRN = currentVenue.UKPRN,
Address1 = currentVenue.Address1,
Postcode = currentVenue.PostCode,
VenueName = currentVenue.VenueName,
UKPRNMatched = (course.ProviderUKPRN == currentVenue.UKPRN),
Message = (course.ProviderUKPRN == currentVenue.UKPRN) ? "Venue UKPRN Matches Course UKPRN" : "Venue UKPRN Does not match Course UKPRN",
Type = "Course",
CourseId = course.id,
CourseRunId = courserun.id
});
}
else
{
result.Add(new VenueReference()
{
UKPRN = course.ProviderUKPRN,
UKPRNMatched = false,
VenueUKPRN = -1,
VenueId = courserun.VenueId.ToString(),
Message = "VenueId does not exist in venues",
Type = "Course",
CourseId = course.id,
CourseRunId = courserun.id
});
}
}
}
//total for all providers
totalInvalidCourseRunReferences += invalidCourseRunReferences;
}
//apprenticeships
foreach (var apprenticeship in allApprenticeshipsForProvider)
{
//apprenticeship locations
foreach (var location in apprenticeship.ApprenticeshipLocations)
{
//only apprenticeshiplocations that references a venue (classroom based or both)
if (location.VenueId.HasValue && location.LocationGuidId.HasValue && location.LocationGuidId != Guid.Empty)
{
var currentVenue = await GetVenueById(location.LocationGuidId.ToString());
//venue exists in cosmos
if (currentVenue != null)
{
if (location.ProviderUKPRN != currentVenue.UKPRN)
{
uniqueInvalidVenues.Add(location.VenueId.ToString());
invalidApprenticeshipLocationReferences++;
}
//apprenticeshipId
result.Add(new VenueReference()
{
UKPRN = apprenticeship.ProviderUKPRN,
ApprenticeshipLocationUKPRN = location.ProviderUKPRN,
VenueId = location.LocationGuidId.ToString(),
VenueUKPRN = currentVenue.UKPRN,
Address1 = currentVenue.Address1,
Postcode = currentVenue.PostCode,
VenueName = currentVenue.VenueName,
UKPRNMatched = (apprenticeship.ProviderUKPRN == currentVenue.UKPRN),
Message = (apprenticeship.ProviderUKPRN == currentVenue.UKPRN) ? "Venue UKPRN Matches Apprenticeship UKPRN" : "Venue UKPRN Does not match Apprenticeship UKPRN",
Type = "Apprenticeship",
ApprenticeshipId = apprenticeship.id
});
}
else
{
result.Add(new VenueReference()
{
UKPRN = apprenticeship.ProviderUKPRN,
ApprenticeshipLocationUKPRN = location.ProviderUKPRN,
UKPRNMatched = false,
VenueUKPRN = -1,
VenueId = location.LocationGuidId.ToString(),
Type = "Apprenticeship",
Message = "VenueId does not exist in venues",
ApprenticeshipId = apprenticeship.id
});
}
}
}
totalInvalidApprenticeshipLocationReferences += invalidApprenticeshipLocationReferences;
}
//total for provider
Console.WriteLine($"{invalidCourseRunReferences} invalid venue references for {ukprn}");
Console.WriteLine($"{invalidApprenticeshipLocationReferences} invalid apprenticeship references for {ukprn}");
Console.WriteLine($"{uniqueInvalidVenues.Count()} unique venues");
}
catch (Exception e)
{
log.LogError(e.Message, e);
}
}
//block to try and fetch all venues for every provider
//to make sure that venues can be fetched without error.
foreach (var ukprn in whiteListProviders)
{
try
{
var venues = await GetVenues(ukprn);
}
catch (Exception e)
{
log.LogError(e.Message, e);
Console.WriteLine($"{ukprn} - failed to fetch venues");
ukprnsThatFailedToFetchVenues++;
}
}
//write venue reference documents
logCsvWriter.WriteHeader(typeof(VenueReference));
logCsvWriter.NextRecord();
foreach (var id in result)
{
logCsvWriter.WriteRecord(id);
logCsvWriter.NextRecord();
}
// Upload log CSV to blob storage
{
logStreamWriter.Flush();
logStream.Seek(0L, SeekOrigin.Begin);
var blob = blobhelper.GetBlobContainer(blobContainer).GetBlockBlobReference(logFileName);
await blob.UploadFromStreamAsync(logStream);
}
}
//
Console.WriteLine($"{totalInvalidCourseRunReferences} courserun invalid references in total");
Console.WriteLine($"{totalInvalidApprenticeshipLocationReferences} apprenticeship location invalid references in total");
Console.WriteLine($"{replacedInvalidVenues.Count()} venues have been reverted back to old venues");
Console.WriteLine($"{uniqueInvalidVenues.Count()} Venues were invalid");
Console.WriteLine($"{ukprnsThatFailedToFetchVenues} ukprns failed to fetch venues");
async Task <List <Course> > GetCourses(int ukprn)
{
var courses = new List <Course>();
string continuation = null;
do
{
var feedOptions = new FeedOptions()
{
RequestContinuation = continuation,
PartitionKey = new Microsoft.Azure.Documents.PartitionKey(ukprn)
};
var queryResponse = await documentClient.CreateDocumentQuery <Course>(coursesCollectionUri, feedOptions)
.Where(p => p.ProviderUKPRN == ukprn && p.CourseStatus != CourseDirectory.Models.Enums.RecordStatus.Archived)
.AsDocumentQuery()
.ExecuteNextAsync <Course>();
courses.AddRange(queryResponse.ToList());
continuation = queryResponse.ResponseContinuation;
}while (continuation != null);
return(courses);
}
async Task <List <Venue> > GetVenues(int ukprn)
{
var venues = new List <Venue>();
string continuation = null;
do
{
var feedOptions = new FeedOptions()
{
RequestContinuation = continuation
};
var queryResponse = await documentClient.CreateDocumentQuery <Venue>(venueCollectionUri, feedOptions)
.Where(p => p.UKPRN == ukprn)
.AsDocumentQuery()
.ExecuteNextAsync <Venue>();
venues.AddRange(queryResponse.ToList());
continuation = queryResponse.ResponseContinuation;
}while (continuation != null);
return(venues);
}
async Task <Venue> GetVenueById(string id)
{
var collectionLink = UriFactory.CreateDocumentCollectionUri(databaseId, venuesCollectionId);
var query = documentClient
.CreateDocumentQuery <Venue>(collectionLink, new FeedOptions()
{
EnableCrossPartitionQuery = true
})
.Where(d => d.ID == id)
.AsDocumentQuery();
return((await query.ExecuteNextAsync()).FirstOrDefault());
}
async Task <List <Apprenticeship> > GetApprenticeships(int ukprn)
{
var apprenticeships = new List <Apprenticeship>();
string continuation = null;
do
{
var feedOptions = new FeedOptions()
{
RequestContinuation = continuation,
PartitionKey = new Microsoft.Azure.Documents.PartitionKey(ukprn)
};
try
{
var queryResponse = await documentClient.CreateDocumentQuery <Apprenticeship>(apprenticeshipCollectionUri, feedOptions)
.Where(p => p.ProviderUKPRN == ukprn && p.RecordStatus != CourseDirectory.Models.Enums.RecordStatus.Archived)
.AsDocumentQuery()
.ExecuteNextAsync <Apprenticeship>();
apprenticeships.AddRange(queryResponse);
continuation = queryResponse.ResponseContinuation;
}
catch (Exception)
{
continuation = null;
}
}while (continuation != null);
return(apprenticeships);
}
async Task <ISet <int> > GetProviderWhiteList()
{
var blob = blobhelper.GetBlobContainer(blobContainer).GetBlockBlobReference(WHITE_LIST_FILE);
var ms = new MemoryStream();
await blob.DownloadToStreamAsync(ms);
ms.Seek(0L, SeekOrigin.Begin);
var results = new HashSet <int>();
string line;
using (var reader = new StreamReader(ms))
{
while ((line = reader.ReadLine()) != null)
{
if (string.IsNullOrEmpty(line))
{
continue;
}
var ukprn = int.Parse(line);
results.Add(ukprn);
}
}
return(results);
}
IList <Venue> GetAllOldVenues()
{
var list = new List <Venue>();
var lookupFileResourceName = "Dfc.ProviderPortal.TribalExporter.2020-01-24_0325-venues-backup.json";
using (var stream = typeof(VenueReferenceChecker).Assembly.GetManifestResourceStream(lookupFileResourceName))
using (StreamReader file = new StreamReader(stream))
{
using (JsonTextReader reader = new JsonTextReader(file))
{
while (reader.Read())
{
JArray o2 = (JArray)JToken.ReadFrom(reader);
foreach (var item in o2)
{
Venue v = item.ToObject <Venue>();
list.Add(v);
}
}
}
}
return(list);
}
}
public static async Task Run(
string input, // Work around https://github.com/Azure/azure-functions-vs-build-sdk/issues/168
ILogger log,
[Inject] IConfigurationRoot configuration,
[Inject] IProviderCollectionService providerCollectionService,
[Inject] ICosmosDbHelper cosmosDbHelper,
[Inject] IBlobStorageHelper blobhelper,
[Inject] IApprenticeReferenceDataService apprenticeReferenceDataService,
[Inject] IApprenticeshipServiceWrapper apprenticeshipService,
[Inject] IVenueCollectionService venueCollectionService,
[Inject] IOnspdService onspdService
)
{
var apprenticeshipCollectionId = configuration["CosmosDbCollectionSettings:ApprenticeshipCollectionId"];
var connectionString = configuration.GetConnectionString("TribalRestore");
var blobContainer = blobhelper.GetBlobContainer(configuration["BlobStorageSettings:Container"]);
var whiteListProviders = await GetProviderWhiteList();
var result = new List <ApprenticeshipResultMessage>();
var venueExportFileName = $"ApprenticeshipExport-{DateTime.Now.ToString("dd-MM-yy HHmm")}";
const string WHITE_LIST_FILE = "ProviderWhiteList-Apprenticeships.txt";
var ukprnCache = new List <int>();
var databaseId = configuration["CosmosDbSettings:DatabaseId"];
var apprenticeshipList = new List <ApprenticeshipResult>();
var createdBy = "ApprenticeshipMigrator";
var createdDate = DateTime.Now;
SemaphoreSlim semaphore = new SemaphoreSlim(5);
var client = cosmosDbHelper.GetClient();
var apprenticeshipSQL = @"SELECT a.ApprenticeshipId,
p.ProviderId,
a.FrameworkCode,
a.ProgType,
a.PathwayCode,
a.StandardCode,
a.[Version],
a.MarketingInformation,
a.[Url],
a.ContactEmail,
a.ContactTelephone,
a.ContactWebsite,
a.RecordStatusId,
a.CreatedByUserId,
a.CreatedDateTimeUtc,
p.Ukprn,
coalesce(s.StandardName,f.NasTitle) as [ApprenticeshipTitle],
s.NotionalEndLevel
FROM Apprenticeship a
INNER JOIN Provider p on p.ProviderId = a.ProviderId
LEFT JOIN [Standard] s on (s.StandardCode = a.StandardCode and s.Version = a.Version)
LEFT jOIN [Framework] f on (f.FrameworkCode = a.FrameworkCode AND f.PathwayCode = a.PathwayCode AND f.ProgType = a.ProgType)
WHERE a.recordStatusId=2
ORDER BY ProviderId
";
var apprenticeshipLocationsSQL = @"SELECT al.ApprenticeshipId,
al.ApprenticeshipLocationId,
l.LocationId,
a.AddressId,
a.AddressLine1,
a.AddressLine2,
a.County,
a.Postcode,
a.Town,
a.Longitude,
a.Latitude,
l.Website,
l.Email,
als.CSV as DeliveryModeStr,
l.Telephone,
l.LocationName,
p.ProviderId,
p.Ukprn,
al.Radius
FROM ApprenticeshipLocation al
INNER JOIN Location l on l.LocationId = al.LocationId
INNER JOIN Provider p on p.ProviderId = l.ProviderId
INNER JOIN Address a ON a.AddressId = l.AddressId
CROSS APPLY (SELECT STRING_AGG(DeliveryModeId,',') as CSV,
aldm.ApprenticeshipLocationId
FROM ApprenticeshipLocationDeliveryMode aldm
WHERE ApprenticeshipLocationId = al.ApprenticeshipLocationId
GROUP BY aldm.ApprenticeshipLocationId
) als
WHERE al.RecordStatusId = 2 and
al.ApprenticeshipId = @ApprenticeshipId
ORDER BY ApprenticeshipId,ApprenticeshipLocationId";
try
{
using (var conn1 = new SqlConnection(connectionString))
using (var apprenticeshipscmd = conn1.CreateCommand())
{
await conn1.OpenAsync();
apprenticeshipscmd.CommandText = apprenticeshipSQL;
using (var apprenticeshipReader = apprenticeshipscmd.ExecuteReader())
{
while (await apprenticeshipReader.ReadAsync())
{
apprenticeshipList.Add(ApprenticeshipResult.FromDataReader(apprenticeshipReader));
}
}
}
}
catch (Exception e)
{
AddResultMessage(0, 0, "Failed", null, e.Message);
log.LogError("Error occured Migrating Apprenticeships", e.Message);
}
await Task.WhenAll(apprenticeshipList.Select(async apprenticeship =>
{
var apprenticeshipErrors = new List <string>(); //Errors Here cause apprenticeships to go into pending
var apprenticeshipWarning = new List <string>(); //informational messages
await semaphore.WaitAsync();
try
{
if (IsOnWhiteList(apprenticeship.UKPRN))
{
apprenticeshipErrors = new List <string>();
//get relevant info
var exisitingApprenticeship = await GetExistingApprenticeship(apprenticeship);
var referenceDataFramework = await GetReferenceDataFramework(apprenticeship);
var referenceDataStandard = await GetReferenceDataStandard(apprenticeship);
var locations = await GetLocations(apprenticeship);
var cosmosVenues = await GetCosmosVenues(locations);
var cosmosProvider = await GetProvider(apprenticeship);
//map objects for creating cosmos record
var locs = MapLocations(locations, cosmosVenues);
var id = exisitingApprenticeship?.id.ToString() ?? Guid.NewGuid().ToString();
var apprenticeType = MapApprenticeshipType(apprenticeship);
//check to see if framework code/standard code is valid
VerifiyIfStandardOrFramework(apprenticeship, referenceDataFramework, referenceDataStandard);
var mappedStatus = MapApprenticeshipRecordStatus(locs);
var mappedApprenticeship = MapApprenticeship(locs,
id,
apprenticeship,
apprenticeType,
mappedStatus,
referenceDataFramework?.Id.ToString(),
referenceDataStandard?.id.ToString(),
cosmosProvider?.id.ToString());
//insert record into cosmos
await CreateOrUpdateApprenticeshipRecord(mappedApprenticeship);
//log message to output
AddResultMessage(apprenticeship.ApprenticeshipID,
apprenticeship.UKPRN,
Enum.GetName(typeof(RecordStatus),
mappedApprenticeship.RecordStatus),
mappedApprenticeship.ApprenticeshipTitle,
string.Join("\n", apprenticeshipErrors),
string.Join("\n", apprenticeshipWarning));
}
else
{
AddResultMessage(apprenticeship.ApprenticeshipID, apprenticeship.UKPRN, "Skipped", null, $"PRN {apprenticeship.UKPRN} not whitelisted");
}
}
catch (Exception e)
{
AddResultMessage(apprenticeship.ApprenticeshipID, apprenticeship.UKPRN, "Failed", null, $"Exception occured creating record - {e.Message}");
log.LogError("Error occurred creating or updating apprenticeship record!", e);
}
finally
{
semaphore.Release();
}
void VerifiyIfStandardOrFramework(ApprenticeshipResult tribalRecord, ReferenceDataFramework refDataFramework, ReferenceDateStandard refDataStandard)
{
if (refDataFramework == null && refDataStandard == null)
{
apprenticeshipWarning.Add($"Standard/Framework code does not exist - framework code {tribalRecord.FrameworkCode}, pathway code: {tribalRecord.PathWayCode}, standard code: {tribalRecord.StandardCode}, version: {tribalRecord.Version}");
}
}
ApprenticeshipDTO MapApprenticeship(IList <ApprenticeshipLocationDTO> locs, string id, ApprenticeshipResult tribalRecord, ApprenticeshipType apprenticeshipTye,
RecordStatus recordStatus, string frameworkId, string standardId, string providerId)
{
var cosmosApprenticeship = new ApprenticeshipDTO()
{
id = id,
ApprenticeshipId = tribalRecord.ApprenticeshipID,
ApprenticeshipTitle = tribalRecord.ApprenticeshipTitle,
ProviderId = providerId,
PathWayCode = tribalRecord.PathWayCode,
ProgType = tribalRecord.ProgType,
ProviderUKPRN = tribalRecord.UKPRN,
FrameworkId = frameworkId,
StandardId = standardId,
FrameworkCode = tribalRecord.FrameworkCode,
StandardCode = tribalRecord.StandardCode,
Version = tribalRecord.Version,
MarketingInformation = tribalRecord.MarketingInformation,
Url = tribalRecord.Url,
ContactTelephone = tribalRecord.ContactTelephone,
ContactEmail = tribalRecord.ContactEmail,
ContactWebsite = tribalRecord.ContactWebsite,
CreatedBy = createdBy,
CreatedDate = createdDate,
NotionalNVQLevelv2 = tribalRecord.NotionalEndLevel,
ApprenticeshipLocations = locs,
ApprenticeshipType = apprenticeshipTye,
RecordStatus = recordStatus
};
return(cosmosApprenticeship);
}
async Task <IList <Dfc.CourseDirectory.Models.Models.Venues.Venue> > GetCosmosVenues(IList <ApprenticeshipLocationResult> locations)
{
IList <Dfc.CourseDirectory.Models.Models.Venues.Venue> lst = new List <Dfc.CourseDirectory.Models.Models.Venues.Venue>();
foreach (var s in locations)
{
var venue = await venueCollectionService.GetDocumentByLocationId(s.LocationId, s.UKPRN);
if (venue != null)
{
lst.Add(venue);
}
}
return(lst);
}
async Task <Provider> GetProvider(ApprenticeshipResult item)
{
return(await providerCollectionService.GetDocumentByUkprn(item.UKPRN));
}
async Task <List <ApprenticeshipLocationResult> > GetLocations(ApprenticeshipResult item)
{
using (var sqlConnection = new SqlConnection(connectionString))
{
var lst = await sqlConnection.QueryAsync <ApprenticeshipLocationResult>(apprenticeshipLocationsSQL, new { apprenticeshipId = item.ApprenticeshipID }, commandType: CommandType.Text);
return(lst.ToList());
}
}
async Task <ReferenceDateStandard> GetReferenceDataStandard(ApprenticeshipResult item)
{
var app = await apprenticeReferenceDataService.GetStandardById(item.StandardCode ?? 0, item.Version ?? 0);
return(app?.Value?.Value);
}
async Task <ReferenceDataFramework> GetReferenceDataFramework(ApprenticeshipResult item)
{
//checks for framework apprenticeship
var app = await apprenticeReferenceDataService.GetFrameworkByCode(item.FrameworkCode ?? 0, item.ProgType ?? 0, item.PathWayCode ?? 0);
return(app?.Value?.Value);
}
async Task <Apprenticeship> GetExistingApprenticeship(ApprenticeshipResult item)
{
//fetch existing apprenticeship row.
return(await apprenticeshipService.GetApprenticeshipByApprenticeshipID(item.ApprenticeshipID));
}
async Task CreateOrUpdateApprenticeshipRecord(ApprenticeshipDTO app)
{
var s = UriFactory.CreateDocumentUri(databaseId, apprenticeshipCollectionId, app.id);
Uri collectionUri = UriFactory.CreateDocumentCollectionUri(databaseId, apprenticeshipCollectionId);
var res = await client.UpsertDocumentAsync(collectionUri, app);
}
RecordStatus MapApprenticeshipRecordStatus(IList <ApprenticeshipLocationDTO> mappedLocation)
{
//if there are any errors with apprenticeshipREcord, set record to migration pending.
if (apprenticeshipErrors.Any())
{
return(RecordStatus.MigrationPending);
}
else
{
return(RecordStatus.Live);
}
}
//Taken entirely from previous migration logic.
ApprenticeshipLocationType GetApprenticeshipLocationType(ApprenticeshipLocationResult lo)
{
var deliveryModes = lo.DeliveryModes;
if ((deliveryModes.Contains(1) && !deliveryModes.Contains(2) && deliveryModes.Contains(3)) ||
(deliveryModes.Contains(1) && deliveryModes.Contains(2) && !deliveryModes.Contains(3)) ||
(deliveryModes.Contains(1) && deliveryModes.Contains(2) && deliveryModes.Contains(3)))
{
return(ApprenticeshipLocationType.ClassroomBasedAndEmployerBased);
}
else if ((!deliveryModes.Contains(1) && !deliveryModes.Contains(2) && deliveryModes.Contains(3)) ||
(!deliveryModes.Contains(1) && deliveryModes.Contains(2) && !deliveryModes.Contains(3)) ||
(!deliveryModes.Contains(1) && deliveryModes.Contains(2) && deliveryModes.Contains(3)))
{
return(ApprenticeshipLocationType.ClassroomBased);
}
else if (deliveryModes.Contains(1) && !deliveryModes.Contains(2) && !deliveryModes.Contains(3))
{
return(ApprenticeshipLocationType.EmployerBased);
}
else
{
return(ApprenticeshipLocationType.Undefined);
}
}
ApprenticeshipType MapApprenticeshipType(ApprenticeshipResult tribalRecord)
{
if (tribalRecord.StandardCode.HasValue)
{
return(ApprenticeshipType.StandardCode);
}
else if (tribalRecord.FrameworkCode.HasValue)
{
return(ApprenticeshipType.FrameworkCode);
}
else
{
apprenticeshipWarning.Add($"ApprenticeshipId: {tribalRecord.ApprenticeshipID} has undefined apprenticeshipType");
return(ApprenticeshipType.Undefined);
}
}
IList <ApprenticeshipLocationDTO> MapLocations(IList <ApprenticeshipLocationResult> locations, IList <Dfc.CourseDirectory.Models.Models.Venues.Venue> venues)
{
var locationBasedApprenticeshipLocation = new List <ApprenticeshipLocationDTO>();
var regionBasedApprenticeshipLocation = new List <ApprenticeshipLocationDTO>();
//no need to proceed
if (locations == null)
{
return(null);
}
//employer based apprenticeships - group all locations into regions/subregions
foreach (var location in locations)
{
var type = GetApprenticeshipLocationType(location);
if (type == ApprenticeshipLocationType.EmployerBased)
{
var allRegionsWithSubRegions = new SelectRegionModel();
var onspdRegionSubregion = onspdService.GetOnspdData(new OnspdSearchCriteria(location.Postcode));
if (onspdRegionSubregion.IsFailure)
{
apprenticeshipWarning.Add($"LocationId: {location.LocationId} - Querying onspd failed - {onspdRegionSubregion.Error}");
continue;
}
else if (!onspdRegionSubregion.HasValue)
{
apprenticeshipWarning.Add($"Location:{location.LocationId} - Did not find a record for postcode: {location.Postcode}");
continue;
}
var selectedSubRegion = allRegionsWithSubRegions.RegionItems.SelectMany(sr => sr.SubRegion.Where(sb =>
sb.SubRegionName == onspdRegionSubregion.Value.Value.LocalAuthority ||
sb.SubRegionName == onspdRegionSubregion.Value.Value.County ||
onspdRegionSubregion.Value.Value.LocalAuthority.Contains(sb.SubRegionName)
)).FirstOrDefault();
if (selectedSubRegion == null)
{
apprenticeshipWarning.Add($"Location:{location.LocationId} Unable to match region with ons data api, location skipped");
continue;
}
else
{
var appLocation = new ApprenticeshipLocationDTO()
{
Id = Guid.NewGuid().ToString(),
VenueId = Guid.Empty.ToString(),
TribalId = location.ApprenticeshipLocationId,
DeliveryModes = location.DeliveryModes,
LocationId = selectedSubRegion.ApiLocationId,
Name = location.LocationName,
ProviderId = location.ProviderId,
ProviderUKPRN = location.UKPRN,
Radius = location.Radius,
ApprenticeshipLocationType = type,
LocationType = LocationType.SubRegion,
LocationGuidId = null,
Regions = new List <string> {
selectedSubRegion.Id
},
RecordStatus = VenueStatus.Live,
CreatedBy = createdBy,
CreatedDate = createdDate,
UpdatedBy = createdBy,
UpdatedDate = createdDate
};
//region based apprenticeships
regionBasedApprenticeshipLocation.Add(appLocation);
}
}
else if (type == ApprenticeshipLocationType.ClassroomBased || type == ApprenticeshipLocationType.ClassroomBasedAndEmployerBased)
{
//venue based (location based apprenticeships)
var cosmosVenueItem = venues.FirstOrDefault(x => x.LocationId == location.LocationId);
var status = default(VenueStatus);
//set status be that of what the venue status is, otherwise if venue is not found
//set status to pending.
if (cosmosVenueItem != null)
{
status = cosmosVenueItem.Status;
}
else
{
apprenticeshipWarning.Add($"LocationId: {location.LocationId} did not find a venue in cosmos, record marked as pending");
continue;
}
var appLocation = new ApprenticeshipLocationDTO()
{
Id = Guid.NewGuid().ToString(),
VenueId = Guid.Empty.ToString(),
TribalId = location.ApprenticeshipLocationId,
Address = new Dfc.CourseDirectory.Models.Models.Apprenticeships.Address()
{
Address1 = cosmosVenueItem?.Address1,
Address2 = cosmosVenueItem?.Address2,
County = cosmosVenueItem?.County,
Email = cosmosVenueItem?.Email,
Website = cosmosVenueItem?.Website,
Longitude = cosmosVenueItem?.Longitude,
Latitude = cosmosVenueItem?.Latitude,
Postcode = cosmosVenueItem?.PostCode,
Town = cosmosVenueItem?.Town,
Phone = cosmosVenueItem?.Telephone
},
DeliveryModes = location.DeliveryModes,
LocationId = location.LocationId,
Name = location.LocationName,
ProviderId = location.ProviderId,
ProviderUKPRN = location.UKPRN,
Radius = location.Radius,
ApprenticeshipLocationType = type,
LocationType = LocationType.Venue,
LocationGuidId = cosmosVenueItem?.ID,
Regions = null,
RecordStatus = status,
CreatedBy = createdBy,
CreatedDate = createdDate,
UpdatedBy = createdBy,
UpdatedDate = createdDate
};
locationBasedApprenticeshipLocation.Add(appLocation);
}
else
{
apprenticeshipWarning.Add($"LocationId: {location.LocationId} skipped as type was unknown {type}");
continue;
}
}
//add a new location with all distinct regions.
if (regionBasedApprenticeshipLocation.Any(x => x.RecordStatus == VenueStatus.Live))
{
var regionLocation = regionBasedApprenticeshipLocation.FirstOrDefault(x => x.RecordStatus == VenueStatus.Live);
regionLocation.Regions = regionBasedApprenticeshipLocation.Where(x => x.Regions != null).SelectMany(x => x.Regions).Distinct().ToList();
locationBasedApprenticeshipLocation.Add(regionLocation);
}
return(locationBasedApprenticeshipLocation);
}
}));
//Log Results to blob storage
var resultsObjBytes = GetResultAsByteArray(result);
await WriteResultsToBlobStorage(resultsObjBytes);
//log completion
log.LogInformation("Migrating Apprenticeships Complete");
async Task <IList <int> > GetProviderWhiteList()
{
var list = new List <int>();
var whiteList = await blobhelper.ReadFileAsync(blobContainer, WHITE_LIST_FILE);
if (!string.IsNullOrEmpty(whiteList))
{
var lines = whiteList.Split(new[] { Environment.NewLine }, StringSplitOptions.None);
foreach (string line in lines)
{
if (int.TryParse(line, out int id))
{
list.Add(id);
}
}
}
return(list);
}
async Task WriteResultsToBlobStorage(byte[] data)
{
await blobhelper.UploadFile(blobContainer, venueExportFileName, data);
}
void AddResultMessage(int apprenticeshipId, int ukprn, string status, string apprenticeshipTitle, string message = "", string warnings = "")
{
lock (result)
{
var validateResult = new ApprenticeshipResultMessage()
{
ApprenticeshipID = apprenticeshipId, Status = status, Message = message, UKPRN = ukprn, ApprenticeshipTitle = apprenticeshipTitle, Warnings = warnings
};
result.Add(validateResult);
}
}
byte[] GetResultAsByteArray(IList <ApprenticeshipResultMessage> ob)
{
using (var memoryStream = new System.IO.MemoryStream())
{
using (var streamWriter = new System.IO.StreamWriter(memoryStream))
using (var csvWriter = new CsvWriter(streamWriter, CultureInfo.InvariantCulture))
{
csvWriter.WriteRecords <ApprenticeshipResultMessage>(ob);
csvWriter.Flush();
}
return(memoryStream.ToArray());
}
}
bool IsOnWhiteList(int ukprn)
{
if (!whiteListProviders.Any(x => x == ukprn))
{
return(false);
}
else
{
return(true);
}
}
}
