public async Task ExecuteAsync(IPreValidationContext validationContext, CancellationToken cancellationToken) { var stopWatch = new Stopwatch(); stopWatch.Start(); try { // get ILR data from file await _errorLookupPopulationService.PopulateAsync(cancellationToken).ConfigureAwait(false); _logger.LogDebug($"Error lookup service completed in: {stopWatch.ElapsedMilliseconds}"); if (_validationErrorCache.ValidationErrors.Any()) { return; } cancellationToken.ThrowIfCancellationRequested(); await _preValidationPopulationService.PopulateAsync(cancellationToken).ConfigureAwait(false); _logger.LogDebug($"Population service completed in: {stopWatch.ElapsedMilliseconds}"); // Set the filename _fileDataCache.FileName = validationContext.Input; // File Validation await _ruleSetOrchestrationService.ExecuteAsync(validationContext.Tasks, cancellationToken).ConfigureAwait(false); cancellationToken.ThrowIfCancellationRequested(); if (_validationErrorCache.ValidationErrors.Any(IsFail)) { _logger.LogDebug( $"File schema catastrophic error, so will not execute learner validation actors, error count: {_validationErrorCache.ValidationErrors.Count}"); return; } await ExecuteValidationActors(validationContext, cancellationToken).ConfigureAwait(false); _logger.LogDebug( $"Actors results collated {_validationErrorCache.ValidationErrors.Count} validation errors"); } catch (Exception ex) { _logger.LogError("Validation Critical Error", ex); throw; } finally { cancellationToken.ThrowIfCancellationRequested(); await _validationOutputService.ProcessAsync(cancellationToken).ConfigureAwait(false); _logger.LogDebug($"Validation final results persisted in {stopWatch.ElapsedMilliseconds}"); } }
public async Task ExecuteAsync(IValidationContext validationContext, CancellationToken cancellationToken) { // get the file name _fileDataCache.FileName = validationContext.Filename; // get ILR data from file await _preValidationPopulationService.PopulateAsync(validationContext, cancellationToken); await _messageRuleSetOrchestrationService.ExecuteAsync(validationContext, cancellationToken); await _learnerRuleSetOrchestrationService.ExecuteAsync(validationContext, cancellationToken); await _validationOutputService.ProcessAsync(validationContext, CancellationToken.None); }
public void ExecuteTasks(IMessage message) { // Build Persistance Dictionary BuildKeyValueDictionary(message); _populationService.PopulateAsync(CancellationToken.None).Wait(); // pre funding var learnersToProcess = _learnerPerActorService.BuildPages(new List <int> { 35 }); // process funding var fundingOutputs = ProcessFunding(learnersToProcess); // persist var serializedOutputs = _jsonSerializationService.Serialize(fundingOutputs); System.IO.File.WriteAllText(@"C:\Code\temp\FM35FundingService\Json_Output.json", serializedOutputs); }
public async Task ExecuteAsync( IPreValidationContext preValidationContext, CancellationToken cancellationToken) { // get the file name _fileDataCache.FileName = preValidationContext.Input; // get ILR data from file await _preValidationPopulationService.PopulateAsync(cancellationToken); // xsd schema validations first; if failed then erturn. // TODO: Load only what is required in _preValidationPopulationService.Populate() if (_validateXMLSchemaService.Validate()) { await _messageRuleSetOrchestrationService.Execute(cancellationToken); await _learnerRuleSetOrchestrationService.Execute(cancellationToken); } await _validationOutputService.ProcessAsync(CancellationToken.None); }
public async Task <IEnumerable <U> > ExecuteAsync(IPreValidationContext validationContext, CancellationToken cancellationToken) { var stopWatch = new Stopwatch(); stopWatch.Start(); // get ILR data from file await _preValidationPopulationService.PopulateAsync(cancellationToken); _logger.LogDebug($"Population service completed in: {stopWatch.ElapsedMilliseconds}"); cancellationToken.ThrowIfCancellationRequested(); // get the learners var ilrMessage = _messageCache.Item; // Possible the zip file was corrupt so we dont have message at this point if (ilrMessage == null) { _logger.LogWarning($"ILR Message is null, will not execute any Learner validation Job Id: {validationContext.Input}"); } else { // Call XSD validation _validateXmlSchemaService.Validate(); if (!_validationErrorCache.ValidationErrors.Any(x => (((IValidationError)x).Severity ?? Interface.Enum.Severity.Error) == Interface.Enum.Severity.Error)) { // get the filename _fileDataCache.FileName = validationContext.Input; // Message Validation await _ruleSetOrchestrationService.Execute(cancellationToken); cancellationToken.ThrowIfCancellationRequested(); if (!_validationErrorCache.ValidationErrors.Any(x => (((IValidationError)x).Severity ?? Interface.Enum.Severity.Error) == Interface.Enum.Severity.Error)) { await ExecuteValidationActors(validationContext, cancellationToken); } else { _logger.LogDebug( $"Header validation failed, so will not execute learner validation actors , error count : {_validationErrorCache.ValidationErrors.Count}"); } } else { _logger.LogDebug($"possible xsd validation failure : {_validationErrorCache.ValidationErrors.Count}"); } cancellationToken.ThrowIfCancellationRequested(); _logger.LogDebug( $"Actors results collated {_validationErrorCache.ValidationErrors.Count} validation errors"); await _validationOutputService.ProcessAsync(cancellationToken); _logger.LogDebug($"Validation Final results persisted {stopWatch.ElapsedMilliseconds}"); } return(null); }
public async Task ExecuteAsync(IJobContextMessage jobContextMessage, CancellationToken cancellationToken) { var stopWatch = new Stopwatch(); stopWatch.Start(); var stopWatchSteps = new Stopwatch(); stopWatchSteps.Start(); FundingServiceDto fundingServiceDto = (FundingServiceDto)_fundingServiceDto; fundingServiceDto.Message = await _ilrFileProviderService.Provide(jobContextMessage.KeyValuePairs[JobContextMessageKey.Filename].ToString()).ConfigureAwait(false); _logger.LogDebug($"Funding Service got file in: {stopWatchSteps.ElapsedMilliseconds}"); stopWatchSteps.Restart(); cancellationToken.ThrowIfCancellationRequested(); // get valid and invalid learners from intermediate storage and store it in the dto for rulebases string validLearnerString = await _keyValuePersistenceService.GetAsync( jobContextMessage.KeyValuePairs[JobContextMessageKey.ValidLearnRefNumbers].ToString(), cancellationToken); fundingServiceDto.ValidLearners = _jsonSerializationService.Deserialize <string[]>(validLearnerString); fundingServiceDto.InvalidLearners = _jsonSerializationService.Deserialize <string[]>( await _keyValuePersistenceService.GetAsync( jobContextMessage.KeyValuePairs[JobContextMessageKey.InvalidLearnRefNumbers].ToString(), cancellationToken)); _logger.LogDebug($"Funding Service got valid learners in: {stopWatchSteps.ElapsedMilliseconds} validlearner(length): {validLearnerString.Length}"); cancellationToken.ThrowIfCancellationRequested(); stopWatchSteps.Restart(); await _populationService.PopulateAsync(cancellationToken).ConfigureAwait(false); cancellationToken.ThrowIfCancellationRequested(); string externalDataCache = _jsonSerializationService.Serialize(_externalDataCache); string internalDataCache = _jsonSerializationService.Serialize(_internalDataCache); string fileDataCache = _jsonSerializationService.Serialize(_fileDataCache); _logger.LogDebug($"Funding Service got external data: {stopWatchSteps.ElapsedMilliseconds}"); //List<FundingActorDto> fundingActorDtos = _learnerPagingService // .BuildPages() // .Select(p => // new FundingActorDto // { // JobId = jobContextMessage.JobId, // ExternalDataCache = externalDataCache, // InternalDataCache = internalDataCache, // FileDataCache = fileDataCache, // ValidLearners = _jsonSerializationService.Serialize(p) // }).ToList(); List <string> taskNames = jobContextMessage.Topics[jobContextMessage.TopicPointer].Tasks.SelectMany(t => t.Tasks).ToList(); List <Task> fundingTasks = new List <Task>(); if (taskNames.Contains(_topicAndTaskSectionConfig.TopicFunding_TaskPerformFM81Calculation)) { List <FundingActorDto> fundingActorDtos = GetFundingModelPages(new List <int> { 81 }, jobContextMessage, externalDataCache, internalDataCache, fileDataCache); _logger.LogDebug($"Funding Service FM81 {fundingActorDtos.Count} pages"); fundingTasks.Add(_fm81ActorTask.Execute(fundingActorDtos, jobContextMessage.KeyValuePairs[JobContextMessageKey.FundingFm81Output].ToString(), cancellationToken)); } if (taskNames.Contains(_topicAndTaskSectionConfig.TopicFunding_TaskPerformFM70Calculation)) { List <FundingActorDto> fundingActorDtos = GetFundingModelPages(new List <int> { 70 }, jobContextMessage, externalDataCache, internalDataCache, fileDataCache); _logger.LogDebug($"Funding Service FM70 {fundingActorDtos.Count} pages"); fundingTasks.Add(_fm70ActorTask.Execute(fundingActorDtos, jobContextMessage.KeyValuePairs[JobContextMessageKey.FundingFm70Output].ToString(), cancellationToken)); } if (taskNames.Contains(_topicAndTaskSectionConfig.TopicFunding_TaskPerformFM35Calculation)) { List <FundingActorDto> fundingActorDtos = GetFundingModelPages(new List <int> { 35 }, jobContextMessage, externalDataCache, internalDataCache, fileDataCache); _logger.LogDebug($"Funding Service FM35 {fundingActorDtos.Count} pages"); fundingTasks.Add(_fm35ActorTask.Execute(fundingActorDtos, jobContextMessage.KeyValuePairs[JobContextMessageKey.FundingFm35Output].ToString(), cancellationToken)); } if (taskNames.Contains(_topicAndTaskSectionConfig.TopicFunding_TaskPerformFM36Calculation)) { List <FundingActorDto> fundingActorDtos = GetFundingModelPages(new List <int> { 36 }, jobContextMessage, externalDataCache, internalDataCache, fileDataCache); _logger.LogDebug($"Funding Service FM36 {fundingActorDtos.Count} pages"); fundingTasks.Add(_fm36ActorTask.Execute(fundingActorDtos, jobContextMessage.KeyValuePairs[JobContextMessageKey.FundingFm36Output].ToString(), cancellationToken)); } if (taskNames.Contains(_topicAndTaskSectionConfig.TopicFunding_TaskPerformFM25Calculation)) { List <FundingActorDto> fundingActorDtos = GetFundingModelPages(new List <int> { 25 }, jobContextMessage, externalDataCache, internalDataCache, fileDataCache); _logger.LogDebug($"Funding Service FM25 {fundingActorDtos.Count} pages"); fundingTasks.Add(_fm25ActorTask.Execute(fundingActorDtos, jobContextMessage.KeyValuePairs[JobContextMessageKey.FundingFm25Output].ToString(), cancellationToken)); } if (taskNames.Contains(_topicAndTaskSectionConfig.TopicFunding_TaskPerformALBCalculation)) { List <FundingActorDto> fundingActorDtos = GetFundingModelPages(new List <int> { 99, 81 }, jobContextMessage, externalDataCache, internalDataCache, fileDataCache); _logger.LogDebug($"Funding Service FM99/81 {fundingActorDtos.Count} pages"); fundingTasks.Add(_albActorTask.Execute(fundingActorDtos, jobContextMessage.KeyValuePairs[JobContextMessageKey.FundingAlbOutput].ToString(), cancellationToken)); } await Task.WhenAll(fundingTasks).ConfigureAwait(false); _logger.LogDebug($"Completed Funding Service for given rule bases in: {stopWatch.ElapsedMilliseconds}"); }
public async Task ExecuteAsync(IPreValidationContext validationContext, CancellationToken cancellationToken) { var stopWatch = new Stopwatch(); stopWatch.Start(); try { // get ILR data from file await _errorLookupPopulationService.PopulateAsync(cancellationToken).ConfigureAwait(false); _logger.LogDebug($"Error lookup service completed in: {stopWatch.ElapsedMilliseconds}"); // Todo: Remove this when XML is known to be schema valid Stream fileStream = await _streamProvider.Provide(cancellationToken); if (fileStream != null) { fileStream.Seek(0, SeekOrigin.Begin); UTF8Encoding utF8Encoding = new UTF8Encoding(false, true); using (StreamReader reader = new StreamReader(fileStream, utF8Encoding, true, 1024, true)) { Cache <string> fileContentCache = (Cache <string>)_cache; fileContentCache.Item = reader.ReadToEnd(); } } if (_validationErrorCache.ValidationErrors.Any()) { return; } cancellationToken.ThrowIfCancellationRequested(); // Call XSD validation _validateXmlSchemaService.Validate(); _logger.LogDebug($"XML validation schema service completed in: {stopWatch.ElapsedMilliseconds}"); if (_validationErrorCache.ValidationErrors.Any(IsErrorOrFail)) { _logger.LogDebug( $"Possible xsd validation failure: {_validationErrorCache.ValidationErrors.Count}"); return; } await _preValidationPopulationService.PopulateAsync(cancellationToken).ConfigureAwait(false); _logger.LogDebug($"Population service completed in: {stopWatch.ElapsedMilliseconds}"); // Set the filename _fileDataCache.FileName = validationContext.Input; // File Validation await _ruleSetOrchestrationService.Execute(cancellationToken).ConfigureAwait(false); cancellationToken.ThrowIfCancellationRequested(); if (_validationErrorCache.ValidationErrors.Any(IsErrorOrFail)) { _logger.LogDebug( $"Header validation failed, so will not execute learner validation actors, error count: {_validationErrorCache.ValidationErrors.Count}"); return; } await ExecuteValidationActors(validationContext, cancellationToken).ConfigureAwait(false); _logger.LogDebug( $"Actors results collated {_validationErrorCache.ValidationErrors.Count} validation errors"); } finally { cancellationToken.ThrowIfCancellationRequested(); await _validationOutputService.ProcessAsync(cancellationToken).ConfigureAwait(false); _logger.LogDebug($"Validation final results persisted in {stopWatch.ElapsedMilliseconds}"); } }