public async Task ExecuteAsync(IPreValidationContext validationContext, CancellationToken cancellationToken)
{
var stopWatch = new Stopwatch();
stopWatch.Start();
try
{
// get ILR data from file
await _errorLookupPopulationService.PopulateAsync(cancellationToken).ConfigureAwait(false);
_logger.LogDebug($"Error lookup service completed in: {stopWatch.ElapsedMilliseconds}");
if (_validationErrorCache.ValidationErrors.Any())
{
return;
}
cancellationToken.ThrowIfCancellationRequested();
await _preValidationPopulationService.PopulateAsync(cancellationToken).ConfigureAwait(false);
_logger.LogDebug($"Population service completed in: {stopWatch.ElapsedMilliseconds}");
// Set the filename
_fileDataCache.FileName = validationContext.Input;
// File Validation
await _ruleSetOrchestrationService.ExecuteAsync(validationContext.Tasks, cancellationToken).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
if (_validationErrorCache.ValidationErrors.Any(IsFail))
{
_logger.LogDebug(
$"File schema catastrophic error, so will not execute learner validation actors, error count: {_validationErrorCache.ValidationErrors.Count}");
return;
}
await ExecuteValidationActors(validationContext, cancellationToken).ConfigureAwait(false);
_logger.LogDebug(
$"Actors results collated {_validationErrorCache.ValidationErrors.Count} validation errors");
}
catch (Exception ex)
{
_logger.LogError("Validation Critical Error", ex);
throw;
}
finally
{
cancellationToken.ThrowIfCancellationRequested();
await _validationOutputService.ProcessAsync(cancellationToken).ConfigureAwait(false);
_logger.LogDebug($"Validation final results persisted in {stopWatch.ElapsedMilliseconds}");
}
}
public AzureStorageFileContentStringProviderService(
IPreValidationContext preValidationContext,
IStreamableKeyValuePersistenceService keyValuePersistenceService,
ILogger logger)
{
_preValidationContext = preValidationContext;
_logger = logger;
_keyValuePersistenceService = keyValuePersistenceService;
}
private async Task ExecuteValidationActors(IPreValidationContext validationContext, CancellationToken cancellationToken)
{
// Get L/A and split the learners into separate lists
var messageShards = _learnerPerActorService.Process();
var actorTasks = new List <Task <string> >();
foreach (var messageShard in messageShards)
{
_logger.LogDebug($"validation Shard has {messageShard.Learners.Count} learners");
// create actors for each Shard.
var actor = GetValidationActor();
// TODO:get reference data per each shard and send it to Actors
var ilrMessageAsBytes = Encoding.UTF8.GetBytes(_jsonSerializationService.Serialize(messageShard));
var internalDataCacheAsBytes =
Encoding.UTF8.GetBytes(_jsonSerializationService.Serialize(_internalDataCache));
var externalDataCacheAsBytes =
Encoding.UTF8.GetBytes(_jsonSerializationService.Serialize(_externalDataCache));
var fileDataCacheAsBytes =
Encoding.UTF8.GetBytes(_jsonSerializationService.Serialize(_fileDataCache));
var validationActorModel = new ValidationActorModel
{
JobId = validationContext.JobId,
Message = ilrMessageAsBytes,
InternalDataCache = internalDataCacheAsBytes,
ExternalDataCache = externalDataCacheAsBytes,
FileDataCache = fileDataCacheAsBytes,
};
actorTasks.Add(actor.Validate(validationActorModel, cancellationToken));
}
_logger.LogDebug($"Starting {actorTasks.Count} validation actors");
await Task.WhenAll(actorTasks.ToArray());
_logger.LogDebug("all Actors completed");
cancellationToken.ThrowIfCancellationRequested();
foreach (Task <string> actorTask in actorTasks)
{
var errors = _jsonSerializationService.Deserialize <IEnumerable <U> >(actorTask.Result);
foreach (var error in errors)
{
_validationErrorCache.Add(error);
}
}
}
public AzureStorageCompressedFileContentStringProviderService(
IPreValidationContext preValidationContext,
ILogger logger,
IStreamableKeyValuePersistenceService streamableKeyValuePersistenceService,
IDateTimeProvider dateTimeProvider)
{
_preValidationContext = preValidationContext;
_logger = logger;
_streamableKeyValuePersistenceService = streamableKeyValuePersistenceService;
_dateTimeProvider = dateTimeProvider;
}
public ValidationOutputService(
IValidationErrorCache <IValidationError> validationErrorCache,
ICache <IMessage> messageCache,
[KeyFilter(PersistenceStorageKeys.Redis)] IKeyValuePersistenceService keyValuePersistenceService,
IPreValidationContext validationContext,
IJsonSerializationService serializationService,
IValidationErrorsDataService validationErrorsDataService)
{
_validationErrorCache = validationErrorCache;
_messageCache = messageCache;
_keyValuePersistenceService = keyValuePersistenceService;
_validationContext = validationContext;
_serializationService = serializationService;
_validationErrorsDataService = validationErrorsDataService;
}
private ValidationOutputService NewService(
IValidationErrorCache <IValidationError> validationErrorCache = null,
ICache <IMessage> messageCache = null,
IKeyValuePersistenceService keyValuePersistenceService = null,
IPreValidationContext preValidationContext = null,
IJsonSerializationService jsonSerializationService = null,
IValidationErrorsDataService validationErrorsDataService = null)
{
return(new ValidationOutputService(
validationErrorCache,
messageCache,
keyValuePersistenceService,
preValidationContext,
jsonSerializationService,
validationErrorsDataService));
}
public async Task ExecuteAsync(
IPreValidationContext preValidationContext,
CancellationToken cancellationToken)
{
// get the file name
_fileDataCache.FileName = preValidationContext.Input;
// get ILR data from file
await _preValidationPopulationService.PopulateAsync(cancellationToken);
await _messageRuleSetOrchestrationService.ExecuteAsync(new List <string>(), cancellationToken);
await _learnerRuleSetOrchestrationService.ExecuteAsync(new List <string>(), cancellationToken);
await _validationOutputService.ProcessAsync(CancellationToken.None);
}
public async Task ExecuteAsync(
IPreValidationContext preValidationContext,
CancellationToken cancellationToken)
{
// get the file name
_fileDataCache.FileName = preValidationContext.Input;
// get ILR data from file
await _preValidationPopulationService.PopulateAsync(cancellationToken);
// xsd schema validations first; if failed then erturn.
// TODO: Load only what is required in _preValidationPopulationService.Populate()
if (_validateXMLSchemaService.Validate())
{
await _messageRuleSetOrchestrationService.Execute(cancellationToken);
await _learnerRuleSetOrchestrationService.Execute(cancellationToken);
}
await _validationOutputService.ProcessAsync(CancellationToken.None);
}
public FileSystemFileContentStringProviderService(IPreValidationContext preValidationContext)
{
_preValidationContext = preValidationContext;
}
public async Task <IEnumerable <U> > ExecuteAsync(IPreValidationContext validationContext, CancellationToken cancellationToken)
{
var stopWatch = new Stopwatch();
stopWatch.Start();
// get ILR data from file
await _preValidationPopulationService.PopulateAsync(cancellationToken);
_logger.LogDebug($"Population service completed in: {stopWatch.ElapsedMilliseconds}");
cancellationToken.ThrowIfCancellationRequested();
// get the learners
var ilrMessage = _messageCache.Item;
// Possible the zip file was corrupt so we dont have message at this point
if (ilrMessage == null)
{
_logger.LogWarning($"ILR Message is null, will not execute any Learner validation Job Id: {validationContext.Input}");
}
else
{
// Call XSD validation
_validateXmlSchemaService.Validate();
if (!_validationErrorCache.ValidationErrors.Any(x => (((IValidationError)x).Severity ?? Interface.Enum.Severity.Error) == Interface.Enum.Severity.Error))
{
// get the filename
_fileDataCache.FileName = validationContext.Input;
// Message Validation
await _ruleSetOrchestrationService.Execute(cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
if (!_validationErrorCache.ValidationErrors.Any(x => (((IValidationError)x).Severity ?? Interface.Enum.Severity.Error) == Interface.Enum.Severity.Error))
{
await ExecuteValidationActors(validationContext, cancellationToken);
}
else
{
_logger.LogDebug(
$"Header validation failed, so will not execute learner validation actors , error count : {_validationErrorCache.ValidationErrors.Count}");
}
}
else
{
_logger.LogDebug($"possible xsd validation failure : {_validationErrorCache.ValidationErrors.Count}");
}
cancellationToken.ThrowIfCancellationRequested();
_logger.LogDebug(
$"Actors results collated {_validationErrorCache.ValidationErrors.Count} validation errors");
await _validationOutputService.ProcessAsync(cancellationToken);
_logger.LogDebug($"Validation Final results persisted {stopWatch.ElapsedMilliseconds}");
}
return(null);
}
private static void RegisterContext(ContainerBuilder containerBuilder, IPreValidationContext preValidationContext)
{
containerBuilder.RegisterInstance(preValidationContext).As <IPreValidationContext>();
}
private async Task ExecuteValidationActors(IPreValidationContext validationContext, CancellationToken cancellationToken)
{
// Get L/A and split the learners into separate lists
IEnumerable <IMessage> learnerMessageShards = await _learnerPerActorProviderService.ProvideAsync();
IEnumerable <IMessage> learnerDPMessageShards = await _learnerDPPerActorProviderService.ProvideAsync();
List <IValidationActor> learnerValidationActors = new List <IValidationActor>();
List <IValidationDPActor> learnerDPValidationActors = new List <IValidationDPActor>();
List <Task <string> > actorTasks = new List <Task <string> >();
List <Task> actorDestroys = new List <Task>();
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
_logger.LogDebug($"Validation will create {learnerMessageShards?.Count() ?? 0} actors");
_logger.LogDebug($"DP Validation will create {learnerDPMessageShards?.Count() ?? 0} actors");
string internalDataCacheAsString =
_jsonSerializationService.Serialize(_internalDataCache);
_logger.LogDebug($"_internalDataCache {internalDataCacheAsString.Length}");
string fileDataCacheAsString =
_jsonSerializationService.Serialize(_fileDataCache);
_logger.LogDebug($"fileDataCacheAsString {fileDataCacheAsString.Length}");
string externalDataCacheAsString =
_jsonSerializationService.Serialize(_externalDataCache);
_logger.LogDebug($"ExternalDataCache: {externalDataCacheAsString.Length}");
string taskListAsString = _jsonSerializationService.Serialize(validationContext.Tasks);
_logger.LogDebug($"taskListAsString {taskListAsString.Length}");
if (learnerMessageShards != null)
{
foreach (IMessage messageShard in learnerMessageShards)
{
_logger.LogDebug($"Validation Shard has {messageShard.Learners.Count} learners");
// create actors for each Shard.
IValidationActor actor = GetValidationActor();
learnerValidationActors.Add(actor);
// TODO:get reference data per each shard and send it to Actors
string ilrMessageAsString = _jsonSerializationService.Serialize(messageShard);
ValidationActorModel validationActorModel = new ValidationActorModel
{
JobId = validationContext.JobId,
Message = ilrMessageAsString,
InternalDataCache = internalDataCacheAsString,
ExternalDataCache = externalDataCacheAsString,
FileDataCache = fileDataCacheAsString,
TaskList = taskListAsString
};
actorTasks.Add(actor.Validate(validationActorModel, cancellationToken));
}
}
if (learnerDPMessageShards != null)
{
foreach (IMessage messageShard in learnerDPMessageShards)
{
_logger.LogDebug($"DP Validation Shard has {messageShard.LearnerDestinationAndProgressions.Count} learner DP records");
// create actors for each Shard.
IValidationDPActor actor = GetValidationDPActor();
learnerDPValidationActors.Add(actor);
// TODO:get reference data per each shard and send it to Actors
string ilrMessageAsString = _jsonSerializationService.Serialize(messageShard);
ValidationDPActorModel validationActorModel = new ValidationDPActorModel
{
JobId = validationContext.JobId,
Message = ilrMessageAsString,
InternalDataCache = internalDataCacheAsString,
ExternalDataCache = externalDataCacheAsString,
FileDataCache = fileDataCacheAsString,
TaskList = taskListAsString
};
actorTasks.Add(actor.Validate(validationActorModel, cancellationToken));
}
}
_logger.LogDebug($"Starting {actorTasks.Count} validation actors after {stopWatch.ElapsedMilliseconds}ms prep time");
stopWatch.Restart();
await Task.WhenAll(actorTasks.ToArray()).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
_logger.LogDebug($"Collating {actorTasks.Count} validation actors after {stopWatch.ElapsedMilliseconds}ms execution time");
stopWatch.Restart();
foreach (Task <string> actorTask in actorTasks)
{
IEnumerable <U> errors = _jsonSerializationService.Deserialize <IEnumerable <U> >(actorTask.Result);
foreach (U error in errors)
{
_validationErrorCache.Add(error);
}
}
_logger.LogDebug($"Destroying {actorTasks.Count} validation actors after {stopWatch.ElapsedMilliseconds}ms collation time");
foreach (IValidationActor validationActor in learnerValidationActors)
{
actorDestroys.Add(DestroyValidationActorAsync(validationActor, cancellationToken));
}
foreach (IValidationDPActor validationDPActor in learnerDPValidationActors)
{
actorDestroys.Add(DestroyValidationDPActorAsync(validationDPActor, cancellationToken));
}
await Task.WhenAll(actorDestroys.ToArray()).ConfigureAwait(false);
}
public async Task ExecuteAsync(IPreValidationContext validationContext, CancellationToken cancellationToken)
{
var stopWatch = new Stopwatch();
stopWatch.Start();
try
{
// get ILR data from file
await _errorLookupPopulationService.PopulateAsync(cancellationToken).ConfigureAwait(false);
_logger.LogDebug($"Error lookup service completed in: {stopWatch.ElapsedMilliseconds}");
// Todo: Remove this when XML is known to be schema valid
Stream fileStream = await _streamProvider.Provide(cancellationToken);
if (fileStream != null)
{
fileStream.Seek(0, SeekOrigin.Begin);
UTF8Encoding utF8Encoding = new UTF8Encoding(false, true);
using (StreamReader reader = new StreamReader(fileStream, utF8Encoding, true, 1024, true))
{
Cache <string> fileContentCache = (Cache <string>)_cache;
fileContentCache.Item = reader.ReadToEnd();
}
}
if (_validationErrorCache.ValidationErrors.Any())
{
return;
}
cancellationToken.ThrowIfCancellationRequested();
// Call XSD validation
_validateXmlSchemaService.Validate();
_logger.LogDebug($"XML validation schema service completed in: {stopWatch.ElapsedMilliseconds}");
if (_validationErrorCache.ValidationErrors.Any(IsErrorOrFail))
{
_logger.LogDebug(
$"Possible xsd validation failure: {_validationErrorCache.ValidationErrors.Count}");
return;
}
await _preValidationPopulationService.PopulateAsync(cancellationToken).ConfigureAwait(false);
_logger.LogDebug($"Population service completed in: {stopWatch.ElapsedMilliseconds}");
// Set the filename
_fileDataCache.FileName = validationContext.Input;
// File Validation
await _ruleSetOrchestrationService.Execute(cancellationToken).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
if (_validationErrorCache.ValidationErrors.Any(IsErrorOrFail))
{
_logger.LogDebug(
$"Header validation failed, so will not execute learner validation actors, error count: {_validationErrorCache.ValidationErrors.Count}");
return;
}
await ExecuteValidationActors(validationContext, cancellationToken).ConfigureAwait(false);
_logger.LogDebug(
$"Actors results collated {_validationErrorCache.ValidationErrors.Count} validation errors");
}
finally
{
cancellationToken.ThrowIfCancellationRequested();
await _validationOutputService.ProcessAsync(cancellationToken).ConfigureAwait(false);
_logger.LogDebug($"Validation final results persisted in {stopWatch.ElapsedMilliseconds}");
}
}
public ContextMessageStringProviderService(IPreValidationContext validationContext)
{
_validationContext = validationContext;
}
