public async Task AuditServiceFailAsync(IJobContextMessage jobContextMessage, string message)
{
await AuditAsync(
jobContextMessage,
AuditEventType.ServiceFailed,
message);
}
public async Task AuditServiceFailAsync(IJobContextMessage jobContextMessage, Exception ex)
{
await AuditAsync(
jobContextMessage,
AuditEventType.ServiceFailed,
ex.ToString());
}
public SourceFileModel GetSourceFileData(IJobContextMessage jobContextMessage)
{
if (!jobContextMessage.KeyValuePairs.ContainsKey(JobContextMessageKey.Filename))
{
throw new ArgumentException($"{nameof(JobContextMessageKey.Filename)} is required");
}
var fileName = jobContextMessage.KeyValuePairs[JobContextMessageKey.Filename].ToString();
string[] fileNameParts = FileNameHelper.SplitFileName(fileName);
if (fileNameParts.Length != 5)
{
throw new ArgumentException($"{nameof(JobContextMessageKey.Filename)} is invalid");
}
var fileNameDatePart = FileNameHelper.GetPreparedDateFromFileName(fileName);
if (!DateTime.TryParse(fileNameDatePart, out var preparationDateTime))
{
throw new ArgumentException($"{nameof(JobContextMessageKey.Filename)} is invalid");
}
var jobId = jobContextMessage.JobId;
return(new SourceFileModel
{
ConRefNumber = fileNameParts[2],
UKPRN = fileNameParts[1],
FileName = fileName,
PreparationDate = preparationDateTime,
SuppliedDate = jobContextMessage.SubmissionDateTimeUtc,
JobId = jobId
});
}
public override void AddExtraKeys(IJobContextMessage message, FileUploadJob metaData)
{
message.KeyValuePairs.Add(JobContextMessageKey.FileSizeInBytes, metaData.FileSize);
if (metaData.IsFirstStage)
{
message.KeyValuePairs.Add(JobContextMessageKey.PauseWhenFinished, "1");
}
if (metaData.Ukprn == 0)
{
return;
}
message.KeyValuePairs.Add(JobContextMessageKey.InvalidLearnRefNumbers, GenerateKey(metaData.Ukprn, metaData.JobId, "ValidationInvalidLearners", "json"));
message.KeyValuePairs.Add(JobContextMessageKey.ValidLearnRefNumbers, GenerateKey(metaData.Ukprn, metaData.JobId, "ValidationValidLearners", "json"));
message.KeyValuePairs.Add(JobContextMessageKey.ValidationErrors, GenerateKey(metaData.Ukprn, metaData.JobId, "ValidationErrors", "json"));
message.KeyValuePairs.Add(JobContextMessageKey.ValidationErrorLookups, GenerateKey(metaData.Ukprn, metaData.JobId, "ValidationErrorsLookup", "json"));
message.KeyValuePairs.Add(JobContextMessageKey.FundingAlbOutput, GenerateKey(metaData.Ukprn, metaData.JobId, "FundingAlbOutput", "json"));
message.KeyValuePairs.Add(JobContextMessageKey.FundingFm35Output, GenerateKey(metaData.Ukprn, metaData.JobId, "FundingFm35Output", "json"));
message.KeyValuePairs.Add(JobContextMessageKey.FundingFm25Output, GenerateKey(metaData.Ukprn, metaData.JobId, "FundingFm25Output", "json"));
message.KeyValuePairs.Add(JobContextMessageKey.FundingFm36Output, GenerateKey(metaData.Ukprn, metaData.JobId, JobContextMessageKey.FundingFm36Output, "json"));
message.KeyValuePairs.Add("FundingFm70Output", GenerateKey(metaData.Ukprn, metaData.JobId, "FundingFm70Output", "json"));
message.KeyValuePairs.Add("FundingFm81Output", GenerateKey(metaData.Ukprn, metaData.JobId, "FundingFm81Output", "json"));
message.KeyValuePairs.Add("OriginalFilename", metaData.FileName);
message.KeyValuePairs.Add("CollectionYear", metaData.CollectionYear);
}
public async Task RunTasks(
IJobContextMessage jobContextMessage,
IReadOnlyList <ITaskItem> tasks,
CancellationToken cancellationToken)
{
var wrapper = new SupplementaryDataWrapper();
var sourceFileModel = new SourceFileModel();
if (tasks.SelectMany(t => t.Tasks).Contains(Constants.ValidationTask))
{
sourceFileModel = _fileHelper.GetSourceFileData(jobContextMessage);
wrapper = await _fileValidationService.GetFile(sourceFileModel, cancellationToken);
if (!wrapper.ValidErrorModels.Any())
{
wrapper = _fileValidationService.RunFileValidators(sourceFileModel, wrapper);
}
if (wrapper.ValidErrorModels.Any())
{
await _storageController.StoreValidationOnly(sourceFileModel, wrapper, cancellationToken);
await _reportingController.FileLevelErrorReport(wrapper, sourceFileModel, cancellationToken);
return;
}
}
await _taskHelper.ExecuteTasks(tasks, sourceFileModel, wrapper, cancellationToken);
}
private static SubmissionEvent GetSubmissionEvent <T>(IJobContextMessage message) where T : SubmissionEvent, new()
{
return(new T
{
AcademicYear = short.Parse(message.KeyValuePairs[JobContextMessageKey.CollectionYear].ToString()),
CollectionPeriod = byte.Parse(message.KeyValuePairs[JobContextMessageKey.ReturnPeriod].ToString()),
IlrSubmissionDateTime = message.SubmissionDateTimeUtc,
JobId = message.JobId,
Ukprn = long.Parse(message.KeyValuePairs[JobContextMessageKey.UkPrn].ToString()),
EventTime = DateTimeOffset.UtcNow
});
}
public async Task AuditEndAsync(IJobContextMessage jobContextMessage)
{
await AuditAsync(
jobContextMessage,
AuditEventType.ServiceFinished);
if (jobContextMessage.TopicPointer == jobContextMessage.Topics.Count - 1)
{
await AuditAsync(
jobContextMessage,
AuditEventType.JobFinished);
}
}
public async Task AuditStartAsync(IJobContextMessage jobContextMessage)
{
if (jobContextMessage.TopicPointer == 0)
{
await AuditAsync(
jobContextMessage,
AuditEventType.JobStarted);
}
await AuditAsync(
jobContextMessage,
AuditEventType.ServiceStarted);
}
public async Task AuditAsync(
IJobContextMessage jobContextMessage,
AuditEventType eventType,
string extraInfo = null)
{
await AuditAsync(
jobContextMessage.Topics[jobContextMessage.TopicPointer].SubscriptionName,
eventType,
(string)jobContextMessage.KeyValuePairs[JobContextMessageKey.Username],
jobContextMessage.JobId,
(string)jobContextMessage.KeyValuePairs[JobContextMessageKey.Filename],
(string)jobContextMessage.KeyValuePairs[JobContextMessageKey.UkPrn],
extraInfo);
}
public async Task <bool> Callback(IJobContextMessage jobContextMessage, CancellationToken cancellationToken)
{
_logger.LogInfo("ESF callback invoked");
var tasks = jobContextMessage.Topics[jobContextMessage.TopicPointer].Tasks;
if (!tasks.Any())
{
_logger.LogInfo("ESF. No tasks to run.");
return(true);
}
if (cancellationToken.IsCancellationRequested)
{
return(true);
}
await _controller.RunTasks(jobContextMessage, tasks, cancellationToken);
return(true);
}
public static IEsfJobContext MapJobContextToModel(IJobContextMessage message)
{
var collectionYear = message.KeyValuePairs[JobContextMessageKey.CollectionYear].ToString();
return(new EsfJobContext
{
JobId = message.JobId,
UkPrn = Convert.ToInt32(message.KeyValuePairs[JobContextMessageKey.UkPrn]),
BlobContainerName = message.KeyValuePairs[JobContextMessageKey.Container].ToString(),
SubmissionDateTimeUtc = message.SubmissionDateTimeUtc,
FileName = message.KeyValuePairs[JobContextMessageKey.Filename].ToString(),
CurrentPeriod = Convert.ToInt32(message.KeyValuePairs[JobContextMessageKey.ReturnPeriod]),
CollectionYear = Convert.ToInt32(message.KeyValuePairs[JobContextMessageKey.CollectionYear]),
Tasks = message.Topics[message.TopicPointer].Tasks?.SelectMany(t => t.Tasks).ToList(),
IlrReferenceDataKey = message.KeyValuePairs.ContainsKey(JobContextMessageKey.IlrReferenceData)
? message.KeyValuePairs[JobContextMessageKey.IlrReferenceData].ToString()
: null,
CollectionName = message.KeyValuePairs[JobContextMessageKey.CollectionName].ToString(),
StartCollectionYearAbbreviation = collectionYear.Substring(0, 2),
EndCollectionYearAbbreviation = collectionYear.Substring(2)
});
}
public DataStoreJobContextMessageContext(IJobContextMessage jobContextMessage, PersistDataConfiguration persistDataConfiguration)
{
_jobContextMessage = jobContextMessage;
_persistDataConfiguration = persistDataConfiguration;
}
public DataStoreJobContextMessageContext(IJobContextMessage jobContextMessage)
{
_jobContextMessage = jobContextMessage;
}
public abstract void AddExtraKeys(IJobContextMessage message, FileUploadJob metaData);
public override void AddExtraKeys(IJobContextMessage message, FileUploadJob metaData)
{
}
public ReportServiceContext(IJobContextMessage jobContextMessage)
{
_jobContextMessage = jobContextMessage;
}
public JobContextMessageValidationContext(IJobContextMessage jobContextMessage)
{
_jobContextMessage = jobContextMessage;
}
public ReportServiceContext(IJobContextMessage jobContextMessage, ISerializationService serializationService, DataPersistConfiguration dataPersistConfiguration)
{
_jobContextMessage = jobContextMessage;
_serializationService = serializationService;
_dataPersistConfiguration = dataPersistConfiguration;
}
public async Task FinishSuccessfully(IJobContextMessage jobContextMessage)
{
await _auditor.AuditEndAsync(jobContextMessage);
}
public async Task ExecuteAsync(IJobContextMessage jobContextMessage, CancellationToken cancellationToken)
{
var stopWatch = new Stopwatch();
stopWatch.Start();
var stopWatchSteps = new Stopwatch();
stopWatchSteps.Start();
FundingServiceDto fundingServiceDto = (FundingServiceDto)_fundingServiceDto;
fundingServiceDto.Message = await _ilrFileProviderService.Provide(jobContextMessage.KeyValuePairs[JobContextMessageKey.Filename].ToString()).ConfigureAwait(false);
_logger.LogDebug($"Funding Service got file in: {stopWatchSteps.ElapsedMilliseconds}");
stopWatchSteps.Restart();
cancellationToken.ThrowIfCancellationRequested();
// get valid and invalid learners from intermediate storage and store it in the dto for rulebases
string validLearnerString = await _keyValuePersistenceService.GetAsync(
jobContextMessage.KeyValuePairs[JobContextMessageKey.ValidLearnRefNumbers].ToString(), cancellationToken);
fundingServiceDto.ValidLearners = _jsonSerializationService.Deserialize <string[]>(validLearnerString);
fundingServiceDto.InvalidLearners = _jsonSerializationService.Deserialize <string[]>(
await _keyValuePersistenceService.GetAsync(
jobContextMessage.KeyValuePairs[JobContextMessageKey.InvalidLearnRefNumbers].ToString(), cancellationToken));
_logger.LogDebug($"Funding Service got valid learners in: {stopWatchSteps.ElapsedMilliseconds} validlearner(length): {validLearnerString.Length}");
cancellationToken.ThrowIfCancellationRequested();
stopWatchSteps.Restart();
await _populationService.PopulateAsync(cancellationToken).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
string externalDataCache = _jsonSerializationService.Serialize(_externalDataCache);
string internalDataCache = _jsonSerializationService.Serialize(_internalDataCache);
string fileDataCache = _jsonSerializationService.Serialize(_fileDataCache);
_logger.LogDebug($"Funding Service got external data: {stopWatchSteps.ElapsedMilliseconds}");
//List<FundingActorDto> fundingActorDtos = _learnerPagingService
// .BuildPages()
// .Select(p =>
// new FundingActorDto
// {
// JobId = jobContextMessage.JobId,
// ExternalDataCache = externalDataCache,
// InternalDataCache = internalDataCache,
// FileDataCache = fileDataCache,
// ValidLearners = _jsonSerializationService.Serialize(p)
// }).ToList();
List <string> taskNames = jobContextMessage.Topics[jobContextMessage.TopicPointer].Tasks.SelectMany(t => t.Tasks).ToList();
List <Task> fundingTasks = new List <Task>();
if (taskNames.Contains(_topicAndTaskSectionConfig.TopicFunding_TaskPerformFM81Calculation))
{
List <FundingActorDto> fundingActorDtos = GetFundingModelPages(new List <int> {
81
}, jobContextMessage, externalDataCache, internalDataCache, fileDataCache);
_logger.LogDebug($"Funding Service FM81 {fundingActorDtos.Count} pages");
fundingTasks.Add(_fm81ActorTask.Execute(fundingActorDtos, jobContextMessage.KeyValuePairs[JobContextMessageKey.FundingFm81Output].ToString(), cancellationToken));
}
if (taskNames.Contains(_topicAndTaskSectionConfig.TopicFunding_TaskPerformFM70Calculation))
{
List <FundingActorDto> fundingActorDtos = GetFundingModelPages(new List <int> {
70
}, jobContextMessage, externalDataCache, internalDataCache, fileDataCache);
_logger.LogDebug($"Funding Service FM70 {fundingActorDtos.Count} pages");
fundingTasks.Add(_fm70ActorTask.Execute(fundingActorDtos, jobContextMessage.KeyValuePairs[JobContextMessageKey.FundingFm70Output].ToString(), cancellationToken));
}
if (taskNames.Contains(_topicAndTaskSectionConfig.TopicFunding_TaskPerformFM35Calculation))
{
List <FundingActorDto> fundingActorDtos = GetFundingModelPages(new List <int> {
35
}, jobContextMessage, externalDataCache, internalDataCache, fileDataCache);
_logger.LogDebug($"Funding Service FM35 {fundingActorDtos.Count} pages");
fundingTasks.Add(_fm35ActorTask.Execute(fundingActorDtos, jobContextMessage.KeyValuePairs[JobContextMessageKey.FundingFm35Output].ToString(), cancellationToken));
}
if (taskNames.Contains(_topicAndTaskSectionConfig.TopicFunding_TaskPerformFM36Calculation))
{
List <FundingActorDto> fundingActorDtos = GetFundingModelPages(new List <int> {
36
}, jobContextMessage, externalDataCache, internalDataCache, fileDataCache);
_logger.LogDebug($"Funding Service FM36 {fundingActorDtos.Count} pages");
fundingTasks.Add(_fm36ActorTask.Execute(fundingActorDtos, jobContextMessage.KeyValuePairs[JobContextMessageKey.FundingFm36Output].ToString(), cancellationToken));
}
if (taskNames.Contains(_topicAndTaskSectionConfig.TopicFunding_TaskPerformFM25Calculation))
{
List <FundingActorDto> fundingActorDtos = GetFundingModelPages(new List <int> {
25
}, jobContextMessage, externalDataCache, internalDataCache, fileDataCache);
_logger.LogDebug($"Funding Service FM25 {fundingActorDtos.Count} pages");
fundingTasks.Add(_fm25ActorTask.Execute(fundingActorDtos, jobContextMessage.KeyValuePairs[JobContextMessageKey.FundingFm25Output].ToString(), cancellationToken));
}
if (taskNames.Contains(_topicAndTaskSectionConfig.TopicFunding_TaskPerformALBCalculation))
{
List <FundingActorDto> fundingActorDtos = GetFundingModelPages(new List <int> {
99, 81
}, jobContextMessage, externalDataCache, internalDataCache, fileDataCache);
_logger.LogDebug($"Funding Service FM99/81 {fundingActorDtos.Count} pages");
fundingTasks.Add(_albActorTask.Execute(fundingActorDtos, jobContextMessage.KeyValuePairs[JobContextMessageKey.FundingAlbOutput].ToString(), cancellationToken));
}
await Task.WhenAll(fundingTasks).ConfigureAwait(false);
_logger.LogDebug($"Completed Funding Service for given rule bases in: {stopWatch.ElapsedMilliseconds}");
}
private List <FundingActorDto> GetFundingModelPages(IEnumerable <int> filter, IJobContextMessage jobContextMessage, string externalDataCache, string internalDataCache, string fileDataCache)
{
return(_learnerPagingService
.BuildPages(filter)
.Select(p =>
new FundingActorDto
{
JobId = jobContextMessage.JobId,
ExternalDataCache = externalDataCache,
InternalDataCache = internalDataCache,
FileDataCache = fileDataCache,
ValidLearners = _jsonSerializationService.Serialize(p)
}).ToList());
}
public async Task AuditJobFailAsync(IJobContextMessage jobContextMessage)
{
await AuditAsync(
jobContextMessage,
AuditEventType.JobFailed);
}
public async Task FinishError(IJobContextMessage jobContextMessage)
{
await _auditor.AuditJobFailAsync(jobContextMessage);
}
public NcsJobContextMessage(IJobContextMessage jobContextMessage)
{
_jobContextMessage = jobContextMessage;
}
