public async Task ExecuteAsync(IPreValidationContext validationContext, CancellationToken cancellationToken) { var stopWatch = new Stopwatch(); stopWatch.Start(); try { // get ILR data from file await _errorLookupPopulationService.PopulateAsync(cancellationToken).ConfigureAwait(false); _logger.LogDebug($"Error lookup service completed in: {stopWatch.ElapsedMilliseconds}"); if (_validationErrorCache.ValidationErrors.Any()) { return; } cancellationToken.ThrowIfCancellationRequested(); await _preValidationPopulationService.PopulateAsync(cancellationToken).ConfigureAwait(false); _logger.LogDebug($"Population service completed in: {stopWatch.ElapsedMilliseconds}"); // Set the filename _fileDataCache.FileName = validationContext.Input; // File Validation await _ruleSetOrchestrationService.ExecuteAsync(validationContext.Tasks, cancellationToken).ConfigureAwait(false); cancellationToken.ThrowIfCancellationRequested(); if (_validationErrorCache.ValidationErrors.Any(IsFail)) { _logger.LogDebug( $"File schema catastrophic error, so will not execute learner validation actors, error count: {_validationErrorCache.ValidationErrors.Count}"); return; } await ExecuteValidationActors(validationContext, cancellationToken).ConfigureAwait(false); _logger.LogDebug( $"Actors results collated {_validationErrorCache.ValidationErrors.Count} validation errors"); } catch (Exception ex) { _logger.LogError("Validation Critical Error", ex); throw; } finally { cancellationToken.ThrowIfCancellationRequested(); await _validationOutputService.ProcessAsync(cancellationToken).ConfigureAwait(false); _logger.LogDebug($"Validation final results persisted in {stopWatch.ElapsedMilliseconds}"); } }
public AzureStorageFileContentStringProviderService( IPreValidationContext preValidationContext, IStreamableKeyValuePersistenceService keyValuePersistenceService, ILogger logger) { _preValidationContext = preValidationContext; _logger = logger; _keyValuePersistenceService = keyValuePersistenceService; }
private async Task ExecuteValidationActors(IPreValidationContext validationContext, CancellationToken cancellationToken) { // Get L/A and split the learners into separate lists var messageShards = _learnerPerActorService.Process(); var actorTasks = new List <Task <string> >(); foreach (var messageShard in messageShards) { _logger.LogDebug($"validation Shard has {messageShard.Learners.Count} learners"); // create actors for each Shard. var actor = GetValidationActor(); // TODO:get reference data per each shard and send it to Actors var ilrMessageAsBytes = Encoding.UTF8.GetBytes(_jsonSerializationService.Serialize(messageShard)); var internalDataCacheAsBytes = Encoding.UTF8.GetBytes(_jsonSerializationService.Serialize(_internalDataCache)); var externalDataCacheAsBytes = Encoding.UTF8.GetBytes(_jsonSerializationService.Serialize(_externalDataCache)); var fileDataCacheAsBytes = Encoding.UTF8.GetBytes(_jsonSerializationService.Serialize(_fileDataCache)); var validationActorModel = new ValidationActorModel { JobId = validationContext.JobId, Message = ilrMessageAsBytes, InternalDataCache = internalDataCacheAsBytes, ExternalDataCache = externalDataCacheAsBytes, FileDataCache = fileDataCacheAsBytes, }; actorTasks.Add(actor.Validate(validationActorModel, cancellationToken)); } _logger.LogDebug($"Starting {actorTasks.Count} validation actors"); await Task.WhenAll(actorTasks.ToArray()); _logger.LogDebug("all Actors completed"); cancellationToken.ThrowIfCancellationRequested(); foreach (Task <string> actorTask in actorTasks) { var errors = _jsonSerializationService.Deserialize <IEnumerable <U> >(actorTask.Result); foreach (var error in errors) { _validationErrorCache.Add(error); } } }
public AzureStorageCompressedFileContentStringProviderService( IPreValidationContext preValidationContext, ILogger logger, IStreamableKeyValuePersistenceService streamableKeyValuePersistenceService, IDateTimeProvider dateTimeProvider) { _preValidationContext = preValidationContext; _logger = logger; _streamableKeyValuePersistenceService = streamableKeyValuePersistenceService; _dateTimeProvider = dateTimeProvider; }
public ValidationOutputService( IValidationErrorCache <IValidationError> validationErrorCache, ICache <IMessage> messageCache, [KeyFilter(PersistenceStorageKeys.Redis)] IKeyValuePersistenceService keyValuePersistenceService, IPreValidationContext validationContext, IJsonSerializationService serializationService, IValidationErrorsDataService validationErrorsDataService) { _validationErrorCache = validationErrorCache; _messageCache = messageCache; _keyValuePersistenceService = keyValuePersistenceService; _validationContext = validationContext; _serializationService = serializationService; _validationErrorsDataService = validationErrorsDataService; }
private ValidationOutputService NewService( IValidationErrorCache <IValidationError> validationErrorCache = null, ICache <IMessage> messageCache = null, IKeyValuePersistenceService keyValuePersistenceService = null, IPreValidationContext preValidationContext = null, IJsonSerializationService jsonSerializationService = null, IValidationErrorsDataService validationErrorsDataService = null) { return(new ValidationOutputService( validationErrorCache, messageCache, keyValuePersistenceService, preValidationContext, jsonSerializationService, validationErrorsDataService)); }
public async Task ExecuteAsync( IPreValidationContext preValidationContext, CancellationToken cancellationToken) { // get the file name _fileDataCache.FileName = preValidationContext.Input; // get ILR data from file await _preValidationPopulationService.PopulateAsync(cancellationToken); await _messageRuleSetOrchestrationService.ExecuteAsync(new List <string>(), cancellationToken); await _learnerRuleSetOrchestrationService.ExecuteAsync(new List <string>(), cancellationToken); await _validationOutputService.ProcessAsync(CancellationToken.None); }
public async Task ExecuteAsync( IPreValidationContext preValidationContext, CancellationToken cancellationToken) { // get the file name _fileDataCache.FileName = preValidationContext.Input; // get ILR data from file await _preValidationPopulationService.PopulateAsync(cancellationToken); // xsd schema validations first; if failed then erturn. // TODO: Load only what is required in _preValidationPopulationService.Populate() if (_validateXMLSchemaService.Validate()) { await _messageRuleSetOrchestrationService.Execute(cancellationToken); await _learnerRuleSetOrchestrationService.Execute(cancellationToken); } await _validationOutputService.ProcessAsync(CancellationToken.None); }
public FileSystemFileContentStringProviderService(IPreValidationContext preValidationContext) { _preValidationContext = preValidationContext; }
public async Task <IEnumerable <U> > ExecuteAsync(IPreValidationContext validationContext, CancellationToken cancellationToken) { var stopWatch = new Stopwatch(); stopWatch.Start(); // get ILR data from file await _preValidationPopulationService.PopulateAsync(cancellationToken); _logger.LogDebug($"Population service completed in: {stopWatch.ElapsedMilliseconds}"); cancellationToken.ThrowIfCancellationRequested(); // get the learners var ilrMessage = _messageCache.Item; // Possible the zip file was corrupt so we dont have message at this point if (ilrMessage == null) { _logger.LogWarning($"ILR Message is null, will not execute any Learner validation Job Id: {validationContext.Input}"); } else { // Call XSD validation _validateXmlSchemaService.Validate(); if (!_validationErrorCache.ValidationErrors.Any(x => (((IValidationError)x).Severity ?? Interface.Enum.Severity.Error) == Interface.Enum.Severity.Error)) { // get the filename _fileDataCache.FileName = validationContext.Input; // Message Validation await _ruleSetOrchestrationService.Execute(cancellationToken); cancellationToken.ThrowIfCancellationRequested(); if (!_validationErrorCache.ValidationErrors.Any(x => (((IValidationError)x).Severity ?? Interface.Enum.Severity.Error) == Interface.Enum.Severity.Error)) { await ExecuteValidationActors(validationContext, cancellationToken); } else { _logger.LogDebug( $"Header validation failed, so will not execute learner validation actors , error count : {_validationErrorCache.ValidationErrors.Count}"); } } else { _logger.LogDebug($"possible xsd validation failure : {_validationErrorCache.ValidationErrors.Count}"); } cancellationToken.ThrowIfCancellationRequested(); _logger.LogDebug( $"Actors results collated {_validationErrorCache.ValidationErrors.Count} validation errors"); await _validationOutputService.ProcessAsync(cancellationToken); _logger.LogDebug($"Validation Final results persisted {stopWatch.ElapsedMilliseconds}"); } return(null); }
private static void RegisterContext(ContainerBuilder containerBuilder, IPreValidationContext preValidationContext) { containerBuilder.RegisterInstance(preValidationContext).As <IPreValidationContext>(); }
private async Task ExecuteValidationActors(IPreValidationContext validationContext, CancellationToken cancellationToken) { // Get L/A and split the learners into separate lists IEnumerable <IMessage> learnerMessageShards = await _learnerPerActorProviderService.ProvideAsync(); IEnumerable <IMessage> learnerDPMessageShards = await _learnerDPPerActorProviderService.ProvideAsync(); List <IValidationActor> learnerValidationActors = new List <IValidationActor>(); List <IValidationDPActor> learnerDPValidationActors = new List <IValidationDPActor>(); List <Task <string> > actorTasks = new List <Task <string> >(); List <Task> actorDestroys = new List <Task>(); Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); _logger.LogDebug($"Validation will create {learnerMessageShards?.Count() ?? 0} actors"); _logger.LogDebug($"DP Validation will create {learnerDPMessageShards?.Count() ?? 0} actors"); string internalDataCacheAsString = _jsonSerializationService.Serialize(_internalDataCache); _logger.LogDebug($"_internalDataCache {internalDataCacheAsString.Length}"); string fileDataCacheAsString = _jsonSerializationService.Serialize(_fileDataCache); _logger.LogDebug($"fileDataCacheAsString {fileDataCacheAsString.Length}"); string externalDataCacheAsString = _jsonSerializationService.Serialize(_externalDataCache); _logger.LogDebug($"ExternalDataCache: {externalDataCacheAsString.Length}"); string taskListAsString = _jsonSerializationService.Serialize(validationContext.Tasks); _logger.LogDebug($"taskListAsString {taskListAsString.Length}"); if (learnerMessageShards != null) { foreach (IMessage messageShard in learnerMessageShards) { _logger.LogDebug($"Validation Shard has {messageShard.Learners.Count} learners"); // create actors for each Shard. IValidationActor actor = GetValidationActor(); learnerValidationActors.Add(actor); // TODO:get reference data per each shard and send it to Actors string ilrMessageAsString = _jsonSerializationService.Serialize(messageShard); ValidationActorModel validationActorModel = new ValidationActorModel { JobId = validationContext.JobId, Message = ilrMessageAsString, InternalDataCache = internalDataCacheAsString, ExternalDataCache = externalDataCacheAsString, FileDataCache = fileDataCacheAsString, TaskList = taskListAsString }; actorTasks.Add(actor.Validate(validationActorModel, cancellationToken)); } } if (learnerDPMessageShards != null) { foreach (IMessage messageShard in learnerDPMessageShards) { _logger.LogDebug($"DP Validation Shard has {messageShard.LearnerDestinationAndProgressions.Count} learner DP records"); // create actors for each Shard. IValidationDPActor actor = GetValidationDPActor(); learnerDPValidationActors.Add(actor); // TODO:get reference data per each shard and send it to Actors string ilrMessageAsString = _jsonSerializationService.Serialize(messageShard); ValidationDPActorModel validationActorModel = new ValidationDPActorModel { JobId = validationContext.JobId, Message = ilrMessageAsString, InternalDataCache = internalDataCacheAsString, ExternalDataCache = externalDataCacheAsString, FileDataCache = fileDataCacheAsString, TaskList = taskListAsString }; actorTasks.Add(actor.Validate(validationActorModel, cancellationToken)); } } _logger.LogDebug($"Starting {actorTasks.Count} validation actors after {stopWatch.ElapsedMilliseconds}ms prep time"); stopWatch.Restart(); await Task.WhenAll(actorTasks.ToArray()).ConfigureAwait(false); cancellationToken.ThrowIfCancellationRequested(); _logger.LogDebug($"Collating {actorTasks.Count} validation actors after {stopWatch.ElapsedMilliseconds}ms execution time"); stopWatch.Restart(); foreach (Task <string> actorTask in actorTasks) { IEnumerable <U> errors = _jsonSerializationService.Deserialize <IEnumerable <U> >(actorTask.Result); foreach (U error in errors) { _validationErrorCache.Add(error); } } _logger.LogDebug($"Destroying {actorTasks.Count} validation actors after {stopWatch.ElapsedMilliseconds}ms collation time"); foreach (IValidationActor validationActor in learnerValidationActors) { actorDestroys.Add(DestroyValidationActorAsync(validationActor, cancellationToken)); } foreach (IValidationDPActor validationDPActor in learnerDPValidationActors) { actorDestroys.Add(DestroyValidationDPActorAsync(validationDPActor, cancellationToken)); } await Task.WhenAll(actorDestroys.ToArray()).ConfigureAwait(false); }
public async Task ExecuteAsync(IPreValidationContext validationContext, CancellationToken cancellationToken) { var stopWatch = new Stopwatch(); stopWatch.Start(); try { // get ILR data from file await _errorLookupPopulationService.PopulateAsync(cancellationToken).ConfigureAwait(false); _logger.LogDebug($"Error lookup service completed in: {stopWatch.ElapsedMilliseconds}"); // Todo: Remove this when XML is known to be schema valid Stream fileStream = await _streamProvider.Provide(cancellationToken); if (fileStream != null) { fileStream.Seek(0, SeekOrigin.Begin); UTF8Encoding utF8Encoding = new UTF8Encoding(false, true); using (StreamReader reader = new StreamReader(fileStream, utF8Encoding, true, 1024, true)) { Cache <string> fileContentCache = (Cache <string>)_cache; fileContentCache.Item = reader.ReadToEnd(); } } if (_validationErrorCache.ValidationErrors.Any()) { return; } cancellationToken.ThrowIfCancellationRequested(); // Call XSD validation _validateXmlSchemaService.Validate(); _logger.LogDebug($"XML validation schema service completed in: {stopWatch.ElapsedMilliseconds}"); if (_validationErrorCache.ValidationErrors.Any(IsErrorOrFail)) { _logger.LogDebug( $"Possible xsd validation failure: {_validationErrorCache.ValidationErrors.Count}"); return; } await _preValidationPopulationService.PopulateAsync(cancellationToken).ConfigureAwait(false); _logger.LogDebug($"Population service completed in: {stopWatch.ElapsedMilliseconds}"); // Set the filename _fileDataCache.FileName = validationContext.Input; // File Validation await _ruleSetOrchestrationService.Execute(cancellationToken).ConfigureAwait(false); cancellationToken.ThrowIfCancellationRequested(); if (_validationErrorCache.ValidationErrors.Any(IsErrorOrFail)) { _logger.LogDebug( $"Header validation failed, so will not execute learner validation actors, error count: {_validationErrorCache.ValidationErrors.Count}"); return; } await ExecuteValidationActors(validationContext, cancellationToken).ConfigureAwait(false); _logger.LogDebug( $"Actors results collated {_validationErrorCache.ValidationErrors.Count} validation errors"); } finally { cancellationToken.ThrowIfCancellationRequested(); await _validationOutputService.ProcessAsync(cancellationToken).ConfigureAwait(false); _logger.LogDebug($"Validation final results persisted in {stopWatch.ElapsedMilliseconds}"); } }
public ContextMessageStringProviderService(IPreValidationContext validationContext) { _validationContext = validationContext; }