public async Task <ServiceStatus> GetApprenticeshipFeedHealthStatusAsync()
{
var checkFrameWork = "512"; //Plumbing and Heating
var checkStandard = "225"; //Plumbing and Domestic Heating Technician(Level 3)
var serviceStatus = new ServiceStatus {
ApplicationName = "Apprenticeship Feed", Status = ServiceState.Red, Notes = string.Empty
};
var checkSocMapping = new SocMapping()
{
SocCode = "5314", Frameworks = new string[] { checkFrameWork }, Standards = new string[] { checkStandard }
};
serviceStatus.CheckParametersUsed = $"SocCode = {checkSocMapping.SocCode} - FrameWork = {checkFrameWork} - Standard = {checkStandard}";
try
{
var apprenticeshipVacancySummaryResponse = await AvService.GetAVSumaryPageAsync(checkSocMapping, 1);
serviceStatus.Status = ServiceState.Amber;
if (apprenticeshipVacancySummaryResponse.TotalReturned > 0)
{
var apprenticeshipVacancyDetailsResponse = await AvService.GetApprenticeshipVacancyDetailsAsync(apprenticeshipVacancySummaryResponse.Results.Take(1).FirstOrDefault().VacancyReference.ToString());
serviceStatus.Status = ServiceState.Green;
}
}
catch (Exception ex)
{
LogFailedMessage(serviceStatus, ex);
}
return(serviceStatus);
}
public async Task GetAVSumaryPageAsync(string socCode, string standard, string framework)
{
var fakeAPIService = A.Fake <IApprenticeshipVacancyApi>();
var fakeLogger = A.Fake <IApplicationLogger>();
var pageNumber = 1;
A.CallTo(() => fakeAPIService.GetAsync(A <string> ._, RequestType.search));
var aVapiService = new AVAPIService(fakeAPIService, fakeLogger);
var mapping = new SocMapping
{
SocCode = socCode,
SocMappingId = Guid.NewGuid(),
Standards = new[] { standard },
Frameworks = new[] { framework }
};
var result = await aVapiService.GetAVSumaryPageAsync(mapping, pageNumber);
if (!mapping.Standards.Where(s => !string.IsNullOrEmpty(s)).Distinct().Any() && !mapping.Frameworks.Where(f => !string.IsNullOrEmpty(f)).Distinct().Any())
{
result.Should().BeNull();
}
else
{
A.CallTo(() => fakeAPIService.GetAsync(A <string> ._, RequestType.search)).MustHaveHappened();
}
}
public async System.Threading.Tasks.Task GetAVsForMultipleProvidersAsync(string socCode, Guid socMappingId, string[] standard, string[] framework)
{
var fakeLogger = A.Fake <IApplicationLogger>();
var fakeAPIService = A.Fake <IApprenticeshipVacancyApi>();
var pageNumber = 1;
var pageSize = 5;
var returnDiffrentProvidersOnPage = 2;
A.CallTo(() => fakeAPIService.GetAsync(A <string> ._, RequestType.search)).Returns(FAAAPIDummyResposnes.GetDummyApprenticeshipVacancySummaryResponse(pageNumber, 50, pageSize, pageSize, returnDiffrentProvidersOnPage)).Once().
Then.Returns(FAAAPIDummyResposnes.GetDummyApprenticeshipVacancySummaryResponse((pageNumber + 1), 50, pageSize, pageSize, returnDiffrentProvidersOnPage));
var aVapiService = new AVAPIService(fakeAPIService, fakeLogger);
var mapping = new SocMapping
{
SocCode = socCode,
SocMappingId = socMappingId,
Standards = standard,
Frameworks = framework
};
var aVSumaryList = await aVapiService.GetAVsForMultipleProvidersAsync(mapping);
if (!mapping.Standards.Where(s => !string.IsNullOrEmpty(s)).Distinct().Any() && !mapping.Frameworks.Where(f => !string.IsNullOrEmpty(f)).Distinct().Any())
{
aVSumaryList.Count().Should().BeLessOrEqualTo(0);
}
else
{
aVSumaryList.Count().Should().BeGreaterThan(pageSize);
var numberProviders = aVSumaryList.Select(v => v.TrainingProviderName).Distinct().Count();
numberProviders.Should().BeGreaterThan(1);
A.CallTo(() => fakeAPIService.GetAsync(A <string> ._, RequestType.search)).MustHaveHappened(Repeated.Exactly.Twice);
}
}
public async System.Threading.Tasks.Task GetAVSumaryPageTestAsync()
{
var fakeLogger = A.Fake <IApplicationLogger>();
var fakeAPIService = A.Fake <IApprenticeshipVacancyApi>();
var fakeAuditService = A.Fake <IAuditService>();
var pageNumber = 1;
var pageSize = 5;
var returnDiffrentProvidersOnPage = 1;
A.CallTo(() => fakeAPIService.GetAsync(A <string> ._, RequestType.search)).Returns(FAAAPIDummyResposnes.GetDummyApprenticeshipVacancySummaryResponse(pageNumber, 50, pageSize, pageSize, returnDiffrentProvidersOnPage));
var client = new ClientProxy(fakeLogger, fakeAuditService);
var aVAPIService = new AVAPIService(fakeAPIService, fakeLogger);
var mapping = new SocMapping
{
SocCode = "1234",
SocMappingId = Guid.NewGuid(),
Standards = new string[] { "225" },
Frameworks = new string[] { "512" }
};
var pageSumary = await aVAPIService.GetAVSumaryPageAsync(mapping, 1);
pageSumary.CurrentPage.Should().Be(pageNumber);
pageSumary.Results.Count().Should().Be(pageSize);
A.CallTo(() => fakeAPIService.GetAsync(A <string> ._, RequestType.search)).MustHaveHappened();
//check null exception
Func <Task> f = async() => { await aVAPIService.GetAVSumaryPageAsync(null, 1); };
f.Should().Throw <ArgumentNullException>();
}
public static async Task <MappedVacancySummary> RunAsync(SocMapping myQueueItem, RunMode mode, IAsyncCollector <AuditRecord <object, object> > asyncCollector, AuditRecord <object, object> masterRecord)
{
var container = ConfigureContainer(mode, asyncCollector, masterRecord);
var getAvFunc = container.Resolve <IGetAvForSocFunc>();
await getAvFunc.Execute(myQueueItem);
return(getAvFunc.GetOutput());
}
private static async Task GetAVForSocMappingFunc(SocMapping myQueueItem, IAsyncCollector <ProjectedVacancySummary> projectedVacancySummary, IAsyncCollector <AuditRecord <object, object> > auditRecord, IAsyncCollector <SocMapping> invalidSocMappings, TraceWriter log, int attempt = 1)
{
try
{
var startTime = DateTime.UtcNow;
ConfigureLog.ConfigureNLogWithAppInsightsTarget();
var mappedResult = await Startup.RunAsync(myQueueItem, RunMode.Azure, auditRecord, new AuditRecord <object, object>
{
CorrelationId = myQueueItem.CorrelationId,
StartedAt = startTime,
EndedAt = DateTime.Now,
Function = nameof(GetAVForSocMappingAzFunction),
Input = "",
Output = ""
}).ConfigureAwait(false);
await AuditMapping(myQueueItem, auditRecord, startTime, mappedResult);
var projectedResult = Function.ProjectVacanciesForSoc.Startup.Run(RunMode.Azure, mappedResult);
projectedResult.CorrelationId = myQueueItem.CorrelationId;
await projectedVacancySummary.AddAsync(projectedResult).ConfigureAwait(false);
await AuditProjections(myQueueItem, auditRecord, startTime, mappedResult, projectedResult);
}
catch (AvApiResponseException responseException)
{
if (responseException.StatusCode == HttpStatusCode.BadRequest)
{
await invalidSocMappings.AddAsync(myQueueItem);
log.Warning($"Exception raised while fetching AV for SOC {myQueueItem.SocCode} API -Url:{responseException.Message} with ErrorCode :{responseException.StatusCode}");
}
else if (responseException.StatusCode == ((HttpStatusCode)429) && attempt < 3)
{
log.Info($"Got API request limit error for SOC {myQueueItem.SocCode} - will wait for 60 seconds");
var retryInSeconds = 60;
int.TryParse(Regex.Match(responseException.Message, "\\d+")?.Value, out retryInSeconds);
await Task.Delay(retryInSeconds * 1000);
await GetAVForSocMappingFunc(myQueueItem, projectedVacancySummary, auditRecord, invalidSocMappings, log, attempt ++);
}
else
{
throw;
}
}
finally
{
log.Info($"C# Queue trigger function processed: {myQueueItem}");
}
}
private static async Task AuditProjections(SocMapping myQueueItem, IAsyncCollector <AuditRecord <object, object> > auditRecord, DateTime startTime, MappedVacancySummary mappedResult, ProjectedVacancySummary projectedResult)
{
myQueueItem.AccessToken = null;
await auditRecord.AddAsync(new AuditRecord <object, object>
{
CorrelationId = myQueueItem.CorrelationId,
StartedAt = startTime,
EndedAt = DateTime.Now,
Function = "ProjectVacanciesAzFunction",
Input = new
{
CorrelationId = myQueueItem.CorrelationId,
SocCode = mappedResult.SocCode,
SocMappingId = mappedResult.SocMappingId,
VacanciesCount = mappedResult.Vacancies.Count()
},
Output = projectedResult,
});
}
public static async Task Run(
[QueueTrigger("socmapping")]
SocMapping myQueueItem,
[Queue("socmapping-invalid")]
IAsyncCollector <SocMapping> invalidSocMappings,
TraceWriter log,
[Queue("projectedavfeedforsocmapping")]
IAsyncCollector <ProjectedVacancySummary> projectedVacancySummary,
[DocumentDB("AVFeedAudit", "AuditRecords", ConnectionStringSetting = "AVAuditCosmosDB")]
IAsyncCollector <AuditRecord <object, object> > auditRecord)
{
try
{
await GetAVForSocMappingFunc(myQueueItem, projectedVacancySummary, auditRecord, invalidSocMappings, log);
}
finally
{
log.Info($"C# Queue trigger function for SOC {myQueueItem.SocCode} processed: {myQueueItem}");
}
}
private static async Task AuditMapping(SocMapping myQueueItem, IAsyncCollector <AuditRecord <object, object> > auditRecord, DateTime startTime, MappedVacancySummary mappedResult)
{
myQueueItem.AccessToken = null;
await auditRecord.AddAsync(new AuditRecord <object, object>
{
CorrelationId = myQueueItem.CorrelationId,
StartedAt = startTime,
EndedAt = DateTime.Now,
Function = nameof(GetAVForSocMappingAzFunction),
Input = myQueueItem,
Output = new
{
CorrelationId = myQueueItem.CorrelationId,
SocCode = mappedResult.SocCode,
SocMappingId = mappedResult.SocMappingId,
VacanciesCount = mappedResult.Vacancies.Count()
},
});
int index = 1;
foreach (var item in mappedResult.Vacancies)
{
await auditRecord.AddAsync(new AuditRecord <object, object>
{
CorrelationId = myQueueItem.CorrelationId,
StartedAt = startTime,
EndedAt = DateTime.Now,
Function = nameof(GetAVForSocMappingAzFunction),
Input = myQueueItem,
Output = new
{
CorrelationId = myQueueItem.CorrelationId,
SocCode = mappedResult.SocCode,
SocMappingId = mappedResult.SocMappingId,
Vacancy = item,
Index = index++
},
});
}
}
public async System.Threading.Tasks.Task GetAVsForMultipleProvidersTestAsync()
{
var fakeLogger = A.Fake <IApplicationLogger>();
var fakeAPIService = A.Fake <IApprenticeshipVacancyApi>();
var fakeAuditService = A.Fake <IAuditService>();
var pageNumber = 1;
var pageSize = 5;
var returnDiffrentProvidersOnPage = 2;
A.CallTo(() => fakeAPIService.GetAsync(A <string> ._, RequestType.search)).Returns(FAAAPIDummyResposnes.GetDummyApprenticeshipVacancySummaryResponse(pageNumber, 50, pageSize, pageSize, returnDiffrentProvidersOnPage)).Once().
Then.Returns(FAAAPIDummyResposnes.GetDummyApprenticeshipVacancySummaryResponse((pageNumber + 1), 50, pageSize, pageSize, returnDiffrentProvidersOnPage));
var client = new ClientProxy(fakeLogger, fakeAuditService);
var aVAPIService = new AVAPIService(fakeAPIService, fakeLogger);
var mapping = new SocMapping
{
SocCode = "1234",
SocMappingId = Guid.NewGuid(),
Standards = new string[] { "225" },
Frameworks = new string[] { "512" }
};
var aVSumaryList = await aVAPIService.GetAVsForMultipleProvidersAsync(mapping);
//must have got more then 1 page to get multipe supplier
aVSumaryList.Count().Should().BeGreaterThan(pageSize);
var numberProviders = aVSumaryList.Select(v => v.TrainingProviderName).Distinct().Count();
numberProviders.Should().BeGreaterThan(1);
A.CallTo(() => fakeAPIService.GetAsync(A <string> ._, RequestType.search)).MustHaveHappened(Repeated.Exactly.Twice);
//check null exception
Func <Task> f = async() => { await aVAPIService.GetAVsForMultipleProvidersAsync(null); };
f.Should().Throw <ArgumentNullException>();
}
public async System.Threading.Tasks.Task GetAVsForMultipleProvidersAsyncPageCallTest()
{
var fakeLogger = A.Fake <IApplicationLogger>();
var fakeAPIService = A.Fake <IApprenticeshipVacancyApi>();
var pageNumber = 1;
var totalPages = 2;
A.CallTo(() => fakeAPIService.GetAsync(A <string> ._, RequestType.search)).Returns(FAAAPIDummyResposnes.GetDummyApprenticeshipVacancySummaryResponseSameProvider(pageNumber, totalPages)).Once().
Then.Returns(FAAAPIDummyResposnes.GetDummyApprenticeshipVacancySummaryResponseSameProvider((pageNumber + 1), totalPages));
var aVapiService = new AVAPIService(fakeAPIService, fakeLogger);
var mapping = new SocMapping
{
SocCode = "DummySOc",
SocMappingId = new Guid(),
Standards = new string[] { "dummy standard" },
Frameworks = new string[] { "dummy framework" }
};
var aVSumaryList = await aVapiService.GetAVsForMultipleProvidersAsync(mapping);
A.CallTo(() => fakeAPIService.GetAsync(A <string> ._, RequestType.search)).MustHaveHappened(totalPages, Times.Exactly);
}
public static async Task <MappedVacancySummary> RunAsync(SocMapping myQueueItem, RunMode mode)
{
return(await RunAsync(myQueueItem, mode, null, null));
}
public async Task AddAvDetailsForSocAsync(SocMapping socMapping, IEnumerable <ApprenticeshipVacancyDetails> apprenticeshipVacancyDetails) => await Task.FromResult(0);
public async Task Execute(SocMapping mapping)
{
socMapping = mapping ?? throw new ArgumentNullException(nameof(mapping));
vacancySummaries = await avService.GetAVsForMultipleProvidersAsync(socMapping);
}
public async Task <ApprenticeshipVacancySummaryResponse> GetAVSumaryPageAsync(SocMapping mapping, int pageNumber)
{
if (mapping == null)
{
throw new ArgumentNullException(nameof(SocMapping));
}
logger.Trace($"Extracting AV summaries for SOC {mapping.SocCode} page : {pageNumber}");
var queryString = HttpUtility.ParseQueryString(string.Empty);
if (!mapping.Standards.Where(s => !string.IsNullOrEmpty(s)).Distinct().Any() && !mapping.Frameworks.Where(f => !string.IsNullOrEmpty(f)).Distinct().Any())
{
return(null);
}
queryString["standardLarsCodes"] = string.Join(",", mapping.Standards.Where(std => !string.IsNullOrEmpty(std)));
queryString["frameworkLarsCodes"] = string.Join(",", mapping.Frameworks.Where(fwrk => !string.IsNullOrEmpty(fwrk)));
queryString["pageSize"] = ConfigurationManager.AppSettings.Get("FAA.PageSize");
queryString["pageNumber"] = pageNumber.ToString();
queryString["sortBy"] = _sortBy;
var responseResult = await apprenticeshipVacancyApi.GetAsync(queryString.ToString(), RequestType.search);
return(JsonConvert.DeserializeObject <ApprenticeshipVacancySummaryResponse>(responseResult));
}
/// <summary>
/// Get Apprenticeship Vacancy Summaries untill we have at leat two providers or all of them for the mapping
/// </summary>
/// <param name="mapping"></param>
/// <returns></returns>
///
public async Task <IEnumerable <ApprenticeshipVacancySummary> > GetAVsForMultipleProvidersAsync(SocMapping mapping)
{
if (mapping == null)
{
throw new ArgumentNullException(nameof(SocMapping));
}
var avSummary = new List <ApprenticeshipVacancySummary>();
var pageNumber = 0;
var maxPagesToTry = int.Parse(ConfigurationManager.AppSettings.Get("FAA.MaxPagesToTryPerMapping"));
logger.Trace($"Getting vacancies for mapping {JsonConvert.SerializeObject(mapping)}");
//Allways break after a given number off loops
while (maxPagesToTry > pageNumber)
{
var apprenticeshipVacancySummaryResponse = await GetAVSumaryPageAsync(mapping, ++pageNumber);
logger.Trace(
$"Got {apprenticeshipVacancySummaryResponse?.TotalReturned} vacancies of {apprenticeshipVacancySummaryResponse?.TotalMatched} on page: {pageNumber} of {apprenticeshipVacancySummaryResponse?.TotalPages} for SOC {mapping.SocCode}");
if (apprenticeshipVacancySummaryResponse?.Results != null)
{
avSummary.AddRange(apprenticeshipVacancySummaryResponse.Results);
}
//stop when there are no more pages or we have more then multiple supplier
if (apprenticeshipVacancySummaryResponse?.TotalPages <= pageNumber ||
avSummary.Select(v => v.TrainingProviderName).Distinct().Count() > 1)
{
break;
}
}
return(avSummary);
}
