public async Task ExecuteAsync(IPreValidationContext validationContext, CancellationToken cancellationToken)
{
var stopWatch = new Stopwatch();
stopWatch.Start();
try
{
// get ILR data from file
await _errorLookupPopulationService.PopulateAsync(cancellationToken).ConfigureAwait(false);
_logger.LogDebug($"Error lookup service completed in: {stopWatch.ElapsedMilliseconds}");
if (_validationErrorCache.ValidationErrors.Any())
{
return;
}
cancellationToken.ThrowIfCancellationRequested();
await _preValidationPopulationService.PopulateAsync(cancellationToken).ConfigureAwait(false);
_logger.LogDebug($"Population service completed in: {stopWatch.ElapsedMilliseconds}");
// Set the filename
_fileDataCache.FileName = validationContext.Input;
// File Validation
await _ruleSetOrchestrationService.ExecuteAsync(validationContext.Tasks, cancellationToken).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
if (_validationErrorCache.ValidationErrors.Any(IsFail))
{
_logger.LogDebug(
$"File schema catastrophic error, so will not execute learner validation actors, error count: {_validationErrorCache.ValidationErrors.Count}");
return;
}
await ExecuteValidationActors(validationContext, cancellationToken).ConfigureAwait(false);
_logger.LogDebug(
$"Actors results collated {_validationErrorCache.ValidationErrors.Count} validation errors");
}
catch (Exception ex)
{
_logger.LogError("Validation Critical Error", ex);
throw;
}
finally
{
cancellationToken.ThrowIfCancellationRequested();
await _validationOutputService.ProcessAsync(cancellationToken).ConfigureAwait(false);
_logger.LogDebug($"Validation final results persisted in {stopWatch.ElapsedMilliseconds}");
}
}
public async Task ExecuteAsync(IValidationContext validationContext, CancellationToken cancellationToken)
{
// get the file name
_fileDataCache.FileName = validationContext.Filename;
// get ILR data from file
await _preValidationPopulationService.PopulateAsync(validationContext, cancellationToken);
await _messageRuleSetOrchestrationService.ExecuteAsync(validationContext, cancellationToken);
await _learnerRuleSetOrchestrationService.ExecuteAsync(validationContext, cancellationToken);
await _validationOutputService.ProcessAsync(validationContext, CancellationToken.None);
}
public void ExecuteTasks(IMessage message)
{
// Build Persistance Dictionary
BuildKeyValueDictionary(message);
_populationService.PopulateAsync(CancellationToken.None).Wait();
// pre funding
var learnersToProcess = _learnerPerActorService.BuildPages(new List <int> {
35
});
// process funding
var fundingOutputs = ProcessFunding(learnersToProcess);
// persist
var serializedOutputs = _jsonSerializationService.Serialize(fundingOutputs);
System.IO.File.WriteAllText(@"C:\Code\temp\FM35FundingService\Json_Output.json", serializedOutputs);
}
public async Task ExecuteAsync(
IPreValidationContext preValidationContext,
CancellationToken cancellationToken)
{
// get the file name
_fileDataCache.FileName = preValidationContext.Input;
// get ILR data from file
await _preValidationPopulationService.PopulateAsync(cancellationToken);
// xsd schema validations first; if failed then erturn.
// TODO: Load only what is required in _preValidationPopulationService.Populate()
if (_validateXMLSchemaService.Validate())
{
await _messageRuleSetOrchestrationService.Execute(cancellationToken);
await _learnerRuleSetOrchestrationService.Execute(cancellationToken);
}
await _validationOutputService.ProcessAsync(CancellationToken.None);
}
public async Task <IEnumerable <U> > ExecuteAsync(IPreValidationContext validationContext, CancellationToken cancellationToken)
{
var stopWatch = new Stopwatch();
stopWatch.Start();
// get ILR data from file
await _preValidationPopulationService.PopulateAsync(cancellationToken);
_logger.LogDebug($"Population service completed in: {stopWatch.ElapsedMilliseconds}");
cancellationToken.ThrowIfCancellationRequested();
// get the learners
var ilrMessage = _messageCache.Item;
// Possible the zip file was corrupt so we dont have message at this point
if (ilrMessage == null)
{
_logger.LogWarning($"ILR Message is null, will not execute any Learner validation Job Id: {validationContext.Input}");
}
else
{
// Call XSD validation
_validateXmlSchemaService.Validate();
if (!_validationErrorCache.ValidationErrors.Any(x => (((IValidationError)x).Severity ?? Interface.Enum.Severity.Error) == Interface.Enum.Severity.Error))
{
// get the filename
_fileDataCache.FileName = validationContext.Input;
// Message Validation
await _ruleSetOrchestrationService.Execute(cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
if (!_validationErrorCache.ValidationErrors.Any(x => (((IValidationError)x).Severity ?? Interface.Enum.Severity.Error) == Interface.Enum.Severity.Error))
{
await ExecuteValidationActors(validationContext, cancellationToken);
}
else
{
_logger.LogDebug(
$"Header validation failed, so will not execute learner validation actors , error count : {_validationErrorCache.ValidationErrors.Count}");
}
}
else
{
_logger.LogDebug($"possible xsd validation failure : {_validationErrorCache.ValidationErrors.Count}");
}
cancellationToken.ThrowIfCancellationRequested();
_logger.LogDebug(
$"Actors results collated {_validationErrorCache.ValidationErrors.Count} validation errors");
await _validationOutputService.ProcessAsync(cancellationToken);
_logger.LogDebug($"Validation Final results persisted {stopWatch.ElapsedMilliseconds}");
}
return(null);
}
public async Task ExecuteAsync(IJobContextMessage jobContextMessage, CancellationToken cancellationToken)
{
var stopWatch = new Stopwatch();
stopWatch.Start();
var stopWatchSteps = new Stopwatch();
stopWatchSteps.Start();
FundingServiceDto fundingServiceDto = (FundingServiceDto)_fundingServiceDto;
fundingServiceDto.Message = await _ilrFileProviderService.Provide(jobContextMessage.KeyValuePairs[JobContextMessageKey.Filename].ToString()).ConfigureAwait(false);
_logger.LogDebug($"Funding Service got file in: {stopWatchSteps.ElapsedMilliseconds}");
stopWatchSteps.Restart();
cancellationToken.ThrowIfCancellationRequested();
// get valid and invalid learners from intermediate storage and store it in the dto for rulebases
string validLearnerString = await _keyValuePersistenceService.GetAsync(
jobContextMessage.KeyValuePairs[JobContextMessageKey.ValidLearnRefNumbers].ToString(), cancellationToken);
fundingServiceDto.ValidLearners = _jsonSerializationService.Deserialize <string[]>(validLearnerString);
fundingServiceDto.InvalidLearners = _jsonSerializationService.Deserialize <string[]>(
await _keyValuePersistenceService.GetAsync(
jobContextMessage.KeyValuePairs[JobContextMessageKey.InvalidLearnRefNumbers].ToString(), cancellationToken));
_logger.LogDebug($"Funding Service got valid learners in: {stopWatchSteps.ElapsedMilliseconds} validlearner(length): {validLearnerString.Length}");
cancellationToken.ThrowIfCancellationRequested();
stopWatchSteps.Restart();
await _populationService.PopulateAsync(cancellationToken).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
string externalDataCache = _jsonSerializationService.Serialize(_externalDataCache);
string internalDataCache = _jsonSerializationService.Serialize(_internalDataCache);
string fileDataCache = _jsonSerializationService.Serialize(_fileDataCache);
_logger.LogDebug($"Funding Service got external data: {stopWatchSteps.ElapsedMilliseconds}");
//List<FundingActorDto> fundingActorDtos = _learnerPagingService
// .BuildPages()
// .Select(p =>
// new FundingActorDto
// {
// JobId = jobContextMessage.JobId,
// ExternalDataCache = externalDataCache,
// InternalDataCache = internalDataCache,
// FileDataCache = fileDataCache,
// ValidLearners = _jsonSerializationService.Serialize(p)
// }).ToList();
List <string> taskNames = jobContextMessage.Topics[jobContextMessage.TopicPointer].Tasks.SelectMany(t => t.Tasks).ToList();
List <Task> fundingTasks = new List <Task>();
if (taskNames.Contains(_topicAndTaskSectionConfig.TopicFunding_TaskPerformFM81Calculation))
{
List <FundingActorDto> fundingActorDtos = GetFundingModelPages(new List <int> {
81
}, jobContextMessage, externalDataCache, internalDataCache, fileDataCache);
_logger.LogDebug($"Funding Service FM81 {fundingActorDtos.Count} pages");
fundingTasks.Add(_fm81ActorTask.Execute(fundingActorDtos, jobContextMessage.KeyValuePairs[JobContextMessageKey.FundingFm81Output].ToString(), cancellationToken));
}
if (taskNames.Contains(_topicAndTaskSectionConfig.TopicFunding_TaskPerformFM70Calculation))
{
List <FundingActorDto> fundingActorDtos = GetFundingModelPages(new List <int> {
70
}, jobContextMessage, externalDataCache, internalDataCache, fileDataCache);
_logger.LogDebug($"Funding Service FM70 {fundingActorDtos.Count} pages");
fundingTasks.Add(_fm70ActorTask.Execute(fundingActorDtos, jobContextMessage.KeyValuePairs[JobContextMessageKey.FundingFm70Output].ToString(), cancellationToken));
}
if (taskNames.Contains(_topicAndTaskSectionConfig.TopicFunding_TaskPerformFM35Calculation))
{
List <FundingActorDto> fundingActorDtos = GetFundingModelPages(new List <int> {
35
}, jobContextMessage, externalDataCache, internalDataCache, fileDataCache);
_logger.LogDebug($"Funding Service FM35 {fundingActorDtos.Count} pages");
fundingTasks.Add(_fm35ActorTask.Execute(fundingActorDtos, jobContextMessage.KeyValuePairs[JobContextMessageKey.FundingFm35Output].ToString(), cancellationToken));
}
if (taskNames.Contains(_topicAndTaskSectionConfig.TopicFunding_TaskPerformFM36Calculation))
{
List <FundingActorDto> fundingActorDtos = GetFundingModelPages(new List <int> {
36
}, jobContextMessage, externalDataCache, internalDataCache, fileDataCache);
_logger.LogDebug($"Funding Service FM36 {fundingActorDtos.Count} pages");
fundingTasks.Add(_fm36ActorTask.Execute(fundingActorDtos, jobContextMessage.KeyValuePairs[JobContextMessageKey.FundingFm36Output].ToString(), cancellationToken));
}
if (taskNames.Contains(_topicAndTaskSectionConfig.TopicFunding_TaskPerformFM25Calculation))
{
List <FundingActorDto> fundingActorDtos = GetFundingModelPages(new List <int> {
25
}, jobContextMessage, externalDataCache, internalDataCache, fileDataCache);
_logger.LogDebug($"Funding Service FM25 {fundingActorDtos.Count} pages");
fundingTasks.Add(_fm25ActorTask.Execute(fundingActorDtos, jobContextMessage.KeyValuePairs[JobContextMessageKey.FundingFm25Output].ToString(), cancellationToken));
}
if (taskNames.Contains(_topicAndTaskSectionConfig.TopicFunding_TaskPerformALBCalculation))
{
List <FundingActorDto> fundingActorDtos = GetFundingModelPages(new List <int> {
99, 81
}, jobContextMessage, externalDataCache, internalDataCache, fileDataCache);
_logger.LogDebug($"Funding Service FM99/81 {fundingActorDtos.Count} pages");
fundingTasks.Add(_albActorTask.Execute(fundingActorDtos, jobContextMessage.KeyValuePairs[JobContextMessageKey.FundingAlbOutput].ToString(), cancellationToken));
}
await Task.WhenAll(fundingTasks).ConfigureAwait(false);
_logger.LogDebug($"Completed Funding Service for given rule bases in: {stopWatch.ElapsedMilliseconds}");
}
public async Task ExecuteAsync(IPreValidationContext validationContext, CancellationToken cancellationToken)
{
var stopWatch = new Stopwatch();
stopWatch.Start();
try
{
// get ILR data from file
await _errorLookupPopulationService.PopulateAsync(cancellationToken).ConfigureAwait(false);
_logger.LogDebug($"Error lookup service completed in: {stopWatch.ElapsedMilliseconds}");
// Todo: Remove this when XML is known to be schema valid
Stream fileStream = await _streamProvider.Provide(cancellationToken);
if (fileStream != null)
{
fileStream.Seek(0, SeekOrigin.Begin);
UTF8Encoding utF8Encoding = new UTF8Encoding(false, true);
using (StreamReader reader = new StreamReader(fileStream, utF8Encoding, true, 1024, true))
{
Cache <string> fileContentCache = (Cache <string>)_cache;
fileContentCache.Item = reader.ReadToEnd();
}
}
if (_validationErrorCache.ValidationErrors.Any())
{
return;
}
cancellationToken.ThrowIfCancellationRequested();
// Call XSD validation
_validateXmlSchemaService.Validate();
_logger.LogDebug($"XML validation schema service completed in: {stopWatch.ElapsedMilliseconds}");
if (_validationErrorCache.ValidationErrors.Any(IsErrorOrFail))
{
_logger.LogDebug(
$"Possible xsd validation failure: {_validationErrorCache.ValidationErrors.Count}");
return;
}
await _preValidationPopulationService.PopulateAsync(cancellationToken).ConfigureAwait(false);
_logger.LogDebug($"Population service completed in: {stopWatch.ElapsedMilliseconds}");
// Set the filename
_fileDataCache.FileName = validationContext.Input;
// File Validation
await _ruleSetOrchestrationService.Execute(cancellationToken).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
if (_validationErrorCache.ValidationErrors.Any(IsErrorOrFail))
{
_logger.LogDebug(
$"Header validation failed, so will not execute learner validation actors, error count: {_validationErrorCache.ValidationErrors.Count}");
return;
}
await ExecuteValidationActors(validationContext, cancellationToken).ConfigureAwait(false);
_logger.LogDebug(
$"Actors results collated {_validationErrorCache.ValidationErrors.Count} validation errors");
}
finally
{
cancellationToken.ThrowIfCancellationRequested();
await _validationOutputService.ProcessAsync(cancellationToken).ConfigureAwait(false);
_logger.LogDebug($"Validation final results persisted in {stopWatch.ElapsedMilliseconds}");
}
}
