public async Task <string> Validate(ValidationActorModel actorModel, CancellationToken cancellationToken)
{
IEnumerable <IValidationError> results = await RunValidation(actorModel, cancellationToken);
actorModel = null;
GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced, true, true);
return(_jsonSerializationService.Serialize(results));
}
private async Task ExecuteValidationActors(IPreValidationContext validationContext, CancellationToken cancellationToken)
{
// Get L/A and split the learners into separate lists
var messageShards = _learnerPerActorService.Process();
var actorTasks = new List <Task <string> >();
foreach (var messageShard in messageShards)
{
_logger.LogDebug($"validation Shard has {messageShard.Learners.Count} learners");
// create actors for each Shard.
var actor = GetValidationActor();
// TODO:get reference data per each shard and send it to Actors
var ilrMessageAsBytes = Encoding.UTF8.GetBytes(_jsonSerializationService.Serialize(messageShard));
var internalDataCacheAsBytes =
Encoding.UTF8.GetBytes(_jsonSerializationService.Serialize(_internalDataCache));
var externalDataCacheAsBytes =
Encoding.UTF8.GetBytes(_jsonSerializationService.Serialize(_externalDataCache));
var fileDataCacheAsBytes =
Encoding.UTF8.GetBytes(_jsonSerializationService.Serialize(_fileDataCache));
var validationActorModel = new ValidationActorModel
{
JobId = validationContext.JobId,
Message = ilrMessageAsBytes,
InternalDataCache = internalDataCacheAsBytes,
ExternalDataCache = externalDataCacheAsBytes,
FileDataCache = fileDataCacheAsBytes,
};
actorTasks.Add(actor.Validate(validationActorModel, cancellationToken));
}
_logger.LogDebug($"Starting {actorTasks.Count} validation actors");
await Task.WhenAll(actorTasks.ToArray());
_logger.LogDebug("all Actors completed");
cancellationToken.ThrowIfCancellationRequested();
foreach (Task <string> actorTask in actorTasks)
{
var errors = _jsonSerializationService.Deserialize <IEnumerable <U> >(actorTask.Result);
foreach (var error in errors)
{
_validationErrorCache.Add(error);
}
}
}
public async Task <string> Validate(ValidationActorModel validationActorModel, CancellationToken cancellationToken)
{
var jsonSerializationService = _parentLifeTimeScope.Resolve <IJsonSerializationService>();
var internalDataCache = jsonSerializationService.Deserialize <InternalDataCache>(Encoding.UTF8.GetString(validationActorModel.InternalDataCache));
var externalDataCache = jsonSerializationService.Deserialize <ExternalDataCache>(Encoding.UTF8.GetString(validationActorModel.ExternalDataCache));
var fileDataCache = jsonSerializationService.Deserialize <FileDataCache>(Encoding.UTF8.GetString(validationActorModel.FileDataCache));
var message = jsonSerializationService.Deserialize <Message>(new MemoryStream(validationActorModel.Message));
cancellationToken.ThrowIfCancellationRequested();
var validationContext = new ValidationContext
{
Input = message
};
using (var childLifeTimeScope = _parentLifeTimeScope.BeginLifetimeScope(c =>
{
c.RegisterInstance(validationContext).As <IValidationContext>();
c.RegisterInstance(new Cache <IMessage> {
Item = message
}).As <ICache <IMessage> >();
c.RegisterInstance(internalDataCache).As <IInternalDataCache>();
c.RegisterInstance(externalDataCache).As <IExternalDataCache>();
c.RegisterInstance(fileDataCache).As <IFileDataCache>();
}))
{
var executionContext = (ExecutionContext)childLifeTimeScope.Resolve <IExecutionContext>();
executionContext.JobId = validationActorModel.JobId;
executionContext.TaskKey = _actorId.ToString();
var logger = childLifeTimeScope.Resolve <ILogger>();
try
{
logger.LogDebug($"Validation Actor {executionContext.TaskKey} started learners: {validationContext.Input.Learners.Count}");
var preValidationOrchestrationService = childLifeTimeScope
.Resolve <IRuleSetOrchestrationService <ILearner, IValidationError> >();
var errors = await preValidationOrchestrationService.Execute(cancellationToken);
logger.LogDebug($"Validation Actor {executionContext.TaskKey} validation done");
var errorString = jsonSerializationService.Serialize(errors);
logger.LogDebug($"Validation Actor {executionContext.TaskKey} completed job");
return(errorString);
}
catch (Exception ex)
{
ActorEventSource.Current.ActorMessage(this, "Exception-{0}", ex.ToString());
logger.LogError("Error while processing Actor job", ex);
throw;
}
}
}
private async Task <IEnumerable <IValidationError> > RunValidation(ValidationActorModel actorModel, CancellationToken cancellationToken)
{
if (_executionContext is ExecutionContext executionContextObj)
{
executionContextObj.JobId = "-1";
executionContextObj.TaskKey = _actorId.ToString();
}
ILogger logger = _parentLifeTimeScope.Resolve <ILogger>();
InternalDataCache internalDataCache;
ExternalDataCache externalDataCache;
FileDataCache fileDataCache;
Message message;
ValidationContext validationContext;
IEnumerable <IValidationError> errors;
try
{
logger.LogDebug($"{nameof(ValidationActor)} {_actorId} {GC.GetGeneration(actorModel)} starting");
internalDataCache = _jsonSerializationService.Deserialize <InternalDataCache>(Encoding.UTF8.GetString(actorModel.InternalDataCache));
externalDataCache = _jsonSerializationService.Deserialize <ExternalDataCache>(Encoding.UTF8.GetString(actorModel.ExternalDataCache));
fileDataCache = _jsonSerializationService.Deserialize <FileDataCache>(Encoding.UTF8.GetString(actorModel.FileDataCache));
message = _jsonSerializationService.Deserialize <Message>(new MemoryStream(actorModel.Message));
validationContext = new ValidationContext
{
Input = message
};
logger.LogDebug($"{nameof(ValidationActor)} {_actorId} {GC.GetGeneration(actorModel)} finished getting input data");
cancellationToken.ThrowIfCancellationRequested();
}
catch (Exception ex)
{
ActorEventSource.Current.ActorMessage(this, "Exception-{0}", ex.ToString());
logger.LogError($"Error while processing {nameof(ValidationActor)}", ex);
throw;
}
using (var childLifeTimeScope = _parentLifeTimeScope.BeginLifetimeScope(c =>
{
c.RegisterInstance(validationContext).As <IValidationContext>();
c.RegisterInstance(new Cache <IMessage> {
Item = message
}).As <ICache <IMessage> >();
c.RegisterInstance(internalDataCache).As <IInternalDataCache>();
c.RegisterInstance(externalDataCache).As <IExternalDataCache>();
c.RegisterInstance(fileDataCache).As <IFileDataCache>();
}))
{
ExecutionContext executionContext = (ExecutionContext)childLifeTimeScope.Resolve <IExecutionContext>();
executionContext.JobId = actorModel.JobId;
executionContext.TaskKey = _actorId.ToString();
ILogger jobLogger = childLifeTimeScope.Resolve <ILogger>();
try
{
jobLogger.LogDebug($"{nameof(ValidationActor)} {_actorId} {GC.GetGeneration(actorModel)} {executionContext.TaskKey} started learners: {validationContext.Input.Learners.Count}");
IRuleSetOrchestrationService <ILearner, IValidationError> preValidationOrchestrationService = childLifeTimeScope
.Resolve <IRuleSetOrchestrationService <ILearner, IValidationError> >();
errors = await preValidationOrchestrationService.Execute(cancellationToken);
jobLogger.LogDebug($"{nameof(ValidationActor)} {_actorId} {GC.GetGeneration(actorModel)} {executionContext.TaskKey} validation done");
}
catch (Exception ex)
{
ActorEventSource.Current.ActorMessage(this, "Exception-{0}", ex.ToString());
jobLogger.LogError($"Error while processing {nameof(ValidationActor)}", ex);
throw;
}
}
internalDataCache = null;
externalDataCache = null;
fileDataCache = null;
message = null;
validationContext = null;
return(errors);
}
private async Task <IEnumerable <IValidationError> > RunValidation(ValidationActorModel actorModel, CancellationToken cancellationToken)
{
if (_executionContext is ExecutionContext executionContextObj)
{
executionContextObj.JobId = "-1";
executionContextObj.TaskKey = _actorId.ToString();
}
ILogger logger = _parentLifeTimeScope.Resolve <ILogger>();
InternalDataCache internalDataCache;
ExternalDataCache externalDataCacheGet;
ExternalDataCache externalDataCache;
FileDataCache fileDataCache;
Message message;
IEnumerable <string> tasks;
ValidationContext validationContext;
IEnumerable <IValidationError> errors;
try
{
logger.LogDebug($"{nameof(ValidationActor)} {_actorId} {GC.GetGeneration(actorModel)} starting");
internalDataCache = _jsonSerializationService.Deserialize <InternalDataCache>(actorModel.InternalDataCache);
externalDataCacheGet = _jsonSerializationService.Deserialize <ExternalDataCache>(actorModel.ExternalDataCache);
fileDataCache = _jsonSerializationService.Deserialize <FileDataCache>(actorModel.FileDataCache);
message = _jsonSerializationService.Deserialize <Message>(actorModel.Message);
tasks = _jsonSerializationService.Deserialize <IEnumerable <string> >(actorModel.TaskList);
externalDataCache = new ExternalDataCache
{
LearningDeliveries = externalDataCacheGet.LearningDeliveries.ToCaseInsensitiveDictionary(),
EPAOrganisations = externalDataCacheGet.EPAOrganisations.ToCaseInsensitiveDictionary(),
ERNs = externalDataCacheGet.ERNs,
FCSContractAllocations = externalDataCacheGet.FCSContractAllocations,
Frameworks = externalDataCacheGet.Frameworks,
Organisations = externalDataCacheGet.Organisations,
Postcodes = externalDataCacheGet.Postcodes.ToCaseInsensitiveHashSet(),
ONSPostcodes = externalDataCacheGet.ONSPostcodes,
Standards = externalDataCacheGet.Standards,
StandardValidities = externalDataCacheGet.StandardValidities,
ULNs = externalDataCacheGet.ULNs,
ValidationErrors = externalDataCacheGet.ValidationErrors.ToCaseInsensitiveDictionary(),
CampusIdentifiers = externalDataCacheGet.CampusIdentifiers
};
validationContext = new ValidationContext
{
Input = message
};
logger.LogDebug($"{nameof(ValidationActor)} {_actorId} {GC.GetGeneration(actorModel)} finished getting input data");
cancellationToken.ThrowIfCancellationRequested();
}
catch (Exception ex)
{
ActorEventSource.Current.ActorMessage(this, "Exception-{0}", ex.ToString());
logger.LogError($"Error while processing {nameof(ValidationActor)}", ex);
throw;
}
using (var childLifeTimeScope = _parentLifeTimeScope.BeginLifetimeScope(c =>
{
c.RegisterInstance(validationContext).As <IValidationContext>();
c.RegisterInstance(new Cache <IMessage> {
Item = message
}).As <ICache <IMessage> >();
c.RegisterInstance(internalDataCache).As <IInternalDataCache>();
c.RegisterInstance(externalDataCache).As <IExternalDataCache>();
c.RegisterInstance(fileDataCache).As <IFileDataCache>();
}))
{
ExecutionContext executionContext = (ExecutionContext)childLifeTimeScope.Resolve <IExecutionContext>();
executionContext.JobId = actorModel.JobId;
executionContext.TaskKey = _actorId.ToString();
ILogger jobLogger = childLifeTimeScope.Resolve <ILogger>();
try
{
jobLogger.LogDebug($"{nameof(ValidationActor)} {_actorId} {GC.GetGeneration(actorModel)} {executionContext.TaskKey} started learners: {validationContext.Input.Learners.Count}");
IRuleSetOrchestrationService <ILearner, IValidationError> preValidationOrchestrationService = childLifeTimeScope
.Resolve <IRuleSetOrchestrationService <ILearner, IValidationError> >();
errors = await preValidationOrchestrationService.ExecuteAsync(tasks, cancellationToken);
jobLogger.LogDebug($"{nameof(ValidationActor)} {_actorId} {GC.GetGeneration(actorModel)} {executionContext.TaskKey} validation done");
}
catch (Exception ex)
{
ActorEventSource.Current.ActorMessage(this, "Exception-{0}", ex.ToString());
jobLogger.LogError($"Error while processing {nameof(ValidationActor)}", ex);
throw;
}
}
internalDataCache = null;
externalDataCache = null;
fileDataCache = null;
message = null;
validationContext = null;
return(errors);
}
private async Task ExecuteValidationActors(IPreValidationContext validationContext, CancellationToken cancellationToken)
{
// Get L/A and split the learners into separate lists
IEnumerable <IMessage> learnerMessageShards = await _learnerPerActorProviderService.ProvideAsync();
IEnumerable <IMessage> learnerDPMessageShards = await _learnerDPPerActorProviderService.ProvideAsync();
List <IValidationActor> learnerValidationActors = new List <IValidationActor>();
List <IValidationDPActor> learnerDPValidationActors = new List <IValidationDPActor>();
List <Task <string> > actorTasks = new List <Task <string> >();
List <Task> actorDestroys = new List <Task>();
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
_logger.LogDebug($"Validation will create {learnerMessageShards?.Count() ?? 0} actors");
_logger.LogDebug($"DP Validation will create {learnerDPMessageShards?.Count() ?? 0} actors");
string internalDataCacheAsString =
_jsonSerializationService.Serialize(_internalDataCache);
_logger.LogDebug($"_internalDataCache {internalDataCacheAsString.Length}");
string fileDataCacheAsString =
_jsonSerializationService.Serialize(_fileDataCache);
_logger.LogDebug($"fileDataCacheAsString {fileDataCacheAsString.Length}");
string externalDataCacheAsString =
_jsonSerializationService.Serialize(_externalDataCache);
_logger.LogDebug($"ExternalDataCache: {externalDataCacheAsString.Length}");
string taskListAsString = _jsonSerializationService.Serialize(validationContext.Tasks);
_logger.LogDebug($"taskListAsString {taskListAsString.Length}");
if (learnerMessageShards != null)
{
foreach (IMessage messageShard in learnerMessageShards)
{
_logger.LogDebug($"Validation Shard has {messageShard.Learners.Count} learners");
// create actors for each Shard.
IValidationActor actor = GetValidationActor();
learnerValidationActors.Add(actor);
// TODO:get reference data per each shard and send it to Actors
string ilrMessageAsString = _jsonSerializationService.Serialize(messageShard);
ValidationActorModel validationActorModel = new ValidationActorModel
{
JobId = validationContext.JobId,
Message = ilrMessageAsString,
InternalDataCache = internalDataCacheAsString,
ExternalDataCache = externalDataCacheAsString,
FileDataCache = fileDataCacheAsString,
TaskList = taskListAsString
};
actorTasks.Add(actor.Validate(validationActorModel, cancellationToken));
}
}
if (learnerDPMessageShards != null)
{
foreach (IMessage messageShard in learnerDPMessageShards)
{
_logger.LogDebug($"DP Validation Shard has {messageShard.LearnerDestinationAndProgressions.Count} learner DP records");
// create actors for each Shard.
IValidationDPActor actor = GetValidationDPActor();
learnerDPValidationActors.Add(actor);
// TODO:get reference data per each shard and send it to Actors
string ilrMessageAsString = _jsonSerializationService.Serialize(messageShard);
ValidationDPActorModel validationActorModel = new ValidationDPActorModel
{
JobId = validationContext.JobId,
Message = ilrMessageAsString,
InternalDataCache = internalDataCacheAsString,
ExternalDataCache = externalDataCacheAsString,
FileDataCache = fileDataCacheAsString,
TaskList = taskListAsString
};
actorTasks.Add(actor.Validate(validationActorModel, cancellationToken));
}
}
_logger.LogDebug($"Starting {actorTasks.Count} validation actors after {stopWatch.ElapsedMilliseconds}ms prep time");
stopWatch.Restart();
await Task.WhenAll(actorTasks.ToArray()).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
_logger.LogDebug($"Collating {actorTasks.Count} validation actors after {stopWatch.ElapsedMilliseconds}ms execution time");
stopWatch.Restart();
foreach (Task <string> actorTask in actorTasks)
{
IEnumerable <U> errors = _jsonSerializationService.Deserialize <IEnumerable <U> >(actorTask.Result);
foreach (U error in errors)
{
_validationErrorCache.Add(error);
}
}
_logger.LogDebug($"Destroying {actorTasks.Count} validation actors after {stopWatch.ElapsedMilliseconds}ms collation time");
foreach (IValidationActor validationActor in learnerValidationActors)
{
actorDestroys.Add(DestroyValidationActorAsync(validationActor, cancellationToken));
}
foreach (IValidationDPActor validationDPActor in learnerDPValidationActors)
{
actorDestroys.Add(DestroyValidationDPActorAsync(validationDPActor, cancellationToken));
}
await Task.WhenAll(actorDestroys.ToArray()).ConfigureAwait(false);
}