public async Task <string> Validate(ValidationDPActorModel actorModel, CancellationToken cancellationToken)
{
IEnumerable <IValidationError> results = await RunValidation(actorModel, cancellationToken);
actorModel = null;
GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced, true, true);
return(_jsonSerializationService.Serialize(results));
}
private async Task ExecuteValidationActors(IPreValidationContext validationContext, CancellationToken cancellationToken)
{
// Get L/A and split the learners into separate lists
IEnumerable <IMessage> learnerMessageShards = await _learnerPerActorProviderService.ProvideAsync();
IEnumerable <IMessage> learnerDPMessageShards = await _learnerDPPerActorProviderService.ProvideAsync();
List <IValidationActor> learnerValidationActors = new List <IValidationActor>();
List <IValidationDPActor> learnerDPValidationActors = new List <IValidationDPActor>();
List <Task <string> > actorTasks = new List <Task <string> >();
List <Task> actorDestroys = new List <Task>();
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
_logger.LogDebug($"Validation will create {learnerMessageShards?.Count() ?? 0} actors");
_logger.LogDebug($"DP Validation will create {learnerDPMessageShards?.Count() ?? 0} actors");
string internalDataCacheAsString =
_jsonSerializationService.Serialize(_internalDataCache);
_logger.LogDebug($"_internalDataCache {internalDataCacheAsString.Length}");
string fileDataCacheAsString =
_jsonSerializationService.Serialize(_fileDataCache);
_logger.LogDebug($"fileDataCacheAsString {fileDataCacheAsString.Length}");
string externalDataCacheAsString =
_jsonSerializationService.Serialize(_externalDataCache);
_logger.LogDebug($"ExternalDataCache: {externalDataCacheAsString.Length}");
string taskListAsString = _jsonSerializationService.Serialize(validationContext.Tasks);
_logger.LogDebug($"taskListAsString {taskListAsString.Length}");
if (learnerMessageShards != null)
{
foreach (IMessage messageShard in learnerMessageShards)
{
_logger.LogDebug($"Validation Shard has {messageShard.Learners.Count} learners");
// create actors for each Shard.
IValidationActor actor = GetValidationActor();
learnerValidationActors.Add(actor);
// TODO:get reference data per each shard and send it to Actors
string ilrMessageAsString = _jsonSerializationService.Serialize(messageShard);
ValidationActorModel validationActorModel = new ValidationActorModel
{
JobId = validationContext.JobId,
Message = ilrMessageAsString,
InternalDataCache = internalDataCacheAsString,
ExternalDataCache = externalDataCacheAsString,
FileDataCache = fileDataCacheAsString,
TaskList = taskListAsString
};
actorTasks.Add(actor.Validate(validationActorModel, cancellationToken));
}
}
if (learnerDPMessageShards != null)
{
foreach (IMessage messageShard in learnerDPMessageShards)
{
_logger.LogDebug($"DP Validation Shard has {messageShard.LearnerDestinationAndProgressions.Count} learner DP records");
// create actors for each Shard.
IValidationDPActor actor = GetValidationDPActor();
learnerDPValidationActors.Add(actor);
// TODO:get reference data per each shard and send it to Actors
string ilrMessageAsString = _jsonSerializationService.Serialize(messageShard);
ValidationDPActorModel validationActorModel = new ValidationDPActorModel
{
JobId = validationContext.JobId,
Message = ilrMessageAsString,
InternalDataCache = internalDataCacheAsString,
ExternalDataCache = externalDataCacheAsString,
FileDataCache = fileDataCacheAsString,
TaskList = taskListAsString
};
actorTasks.Add(actor.Validate(validationActorModel, cancellationToken));
}
}
_logger.LogDebug($"Starting {actorTasks.Count} validation actors after {stopWatch.ElapsedMilliseconds}ms prep time");
stopWatch.Restart();
await Task.WhenAll(actorTasks.ToArray()).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
_logger.LogDebug($"Collating {actorTasks.Count} validation actors after {stopWatch.ElapsedMilliseconds}ms execution time");
stopWatch.Restart();
foreach (Task <string> actorTask in actorTasks)
{
IEnumerable <U> errors = _jsonSerializationService.Deserialize <IEnumerable <U> >(actorTask.Result);
foreach (U error in errors)
{
_validationErrorCache.Add(error);
}
}
_logger.LogDebug($"Destroying {actorTasks.Count} validation actors after {stopWatch.ElapsedMilliseconds}ms collation time");
foreach (IValidationActor validationActor in learnerValidationActors)
{
actorDestroys.Add(DestroyValidationActorAsync(validationActor, cancellationToken));
}
foreach (IValidationDPActor validationDPActor in learnerDPValidationActors)
{
actorDestroys.Add(DestroyValidationDPActorAsync(validationDPActor, cancellationToken));
}
await Task.WhenAll(actorDestroys.ToArray()).ConfigureAwait(false);
}
private async Task <IEnumerable <IValidationError> > RunValidation(ValidationDPActorModel actorModel, CancellationToken cancellationToken)
{
if (_executionContext is ExecutionContext executionContextObj)
{
executionContextObj.JobId = "-1";
executionContextObj.TaskKey = _actorId.ToString();
}
ILogger logger = _parentLifeTimeScope.Resolve <ILogger>();
InternalDataCache internalDataCache;
ExternalDataCache externalDataCacheGet;
ExternalDataCache externalDataCache;
FileDataCache fileDataCache;
Message message;
IEnumerable <string> tasks;
ValidationContext validationContext;
IEnumerable <IValidationError> errors;
try
{
logger.LogDebug($"{nameof(ValidationDPActor)} {_actorId} {GC.GetGeneration(actorModel)} starting");
internalDataCache = _jsonSerializationService.Deserialize <InternalDataCache>(actorModel.InternalDataCache);
externalDataCacheGet = _jsonSerializationService.Deserialize <ExternalDataCache>(actorModel.ExternalDataCache);
fileDataCache = _jsonSerializationService.Deserialize <FileDataCache>(actorModel.FileDataCache);
message = _jsonSerializationService.Deserialize <Message>(actorModel.Message);
tasks = _jsonSerializationService.Deserialize <IEnumerable <string> >(actorModel.TaskList);
externalDataCache = new ExternalDataCache
{
ULNs = externalDataCacheGet.ULNs,
ValidationErrors = externalDataCacheGet.ValidationErrors.ToCaseInsensitiveDictionary()
};
validationContext = new ValidationContext
{
Input = message
};
logger.LogDebug($"{nameof(ValidationDPActor)} {_actorId} {GC.GetGeneration(actorModel)} finished getting input data");
cancellationToken.ThrowIfCancellationRequested();
}
catch (Exception ex)
{
ActorEventSource.Current.ActorMessage(this, "Exception-{0}", ex.ToString());
logger.LogError($"Error while processing {nameof(ValidationDPActor)}", ex);
throw;
}
using (var childLifeTimeScope = _parentLifeTimeScope.BeginLifetimeScope(c =>
{
c.RegisterInstance(validationContext).As <IValidationContext>();
c.RegisterInstance(new Cache <IMessage> {
Item = message
}).As <ICache <IMessage> >();
c.RegisterInstance(internalDataCache).As <IInternalDataCache>();
c.RegisterInstance(externalDataCache).As <IExternalDataCache>();
c.RegisterInstance(fileDataCache).As <IFileDataCache>();
}))
{
ExecutionContext executionContext = (ExecutionContext)childLifeTimeScope.Resolve <IExecutionContext>();
executionContext.JobId = actorModel.JobId;
executionContext.TaskKey = _actorId.ToString();
ILogger jobLogger = childLifeTimeScope.Resolve <ILogger>();
try
{
jobLogger.LogDebug($"{nameof(ValidationDPActor)} {_actorId} {GC.GetGeneration(actorModel)} {executionContext.TaskKey} started Destination and Progressions: {validationContext.Input.LearnerDestinationAndProgressions.Count}");
IRuleSetOrchestrationService <ILearnerDestinationAndProgression, IValidationError> preValidationOrchestrationService = childLifeTimeScope
.Resolve <IRuleSetOrchestrationService <ILearnerDestinationAndProgression, IValidationError> >();
errors = await preValidationOrchestrationService.ExecuteAsync(tasks, cancellationToken);
jobLogger.LogDebug($"{nameof(ValidationDPActor)} {_actorId} {GC.GetGeneration(actorModel)} {executionContext.TaskKey} Destination and Progression validation done");
}
catch (Exception ex)
{
ActorEventSource.Current.ActorMessage(this, "Exception-{0}", ex.ToString());
jobLogger.LogError($"Error while processing {nameof(ValidationDPActor)}", ex);
throw;
}
}
internalDataCache = null;
externalDataCache = null;
fileDataCache = null;
message = null;
validationContext = null;
return(errors);
}
