public async Task <string> Validate(ValidationActorModel actorModel, CancellationToken cancellationToken)
        {
            IEnumerable <IValidationError> results = await RunValidation(actorModel, cancellationToken);

            actorModel = null;

            GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
            GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced, true, true);

            return(_jsonSerializationService.Serialize(results));
        }
示例#2
0
        private async Task ExecuteValidationActors(IPreValidationContext validationContext, CancellationToken cancellationToken)
        {
            // Get L/A and split the learners into separate lists
            var messageShards = _learnerPerActorService.Process();

            var actorTasks = new List <Task <string> >();

            foreach (var messageShard in messageShards)
            {
                _logger.LogDebug($"validation Shard has {messageShard.Learners.Count} learners");

                // create actors for each Shard.
                var actor = GetValidationActor();

                // TODO:get reference data per each shard and send it to Actors
                var ilrMessageAsBytes = Encoding.UTF8.GetBytes(_jsonSerializationService.Serialize(messageShard));

                var internalDataCacheAsBytes =
                    Encoding.UTF8.GetBytes(_jsonSerializationService.Serialize(_internalDataCache));
                var externalDataCacheAsBytes =
                    Encoding.UTF8.GetBytes(_jsonSerializationService.Serialize(_externalDataCache));
                var fileDataCacheAsBytes =
                    Encoding.UTF8.GetBytes(_jsonSerializationService.Serialize(_fileDataCache));

                var validationActorModel = new ValidationActorModel
                {
                    JobId             = validationContext.JobId,
                    Message           = ilrMessageAsBytes,
                    InternalDataCache = internalDataCacheAsBytes,
                    ExternalDataCache = externalDataCacheAsBytes,
                    FileDataCache     = fileDataCacheAsBytes,
                };

                actorTasks.Add(actor.Validate(validationActorModel, cancellationToken));
            }

            _logger.LogDebug($"Starting {actorTasks.Count} validation actors");

            await Task.WhenAll(actorTasks.ToArray());

            _logger.LogDebug("all Actors completed");

            cancellationToken.ThrowIfCancellationRequested();

            foreach (Task <string> actorTask in actorTasks)
            {
                var errors = _jsonSerializationService.Deserialize <IEnumerable <U> >(actorTask.Result);

                foreach (var error in errors)
                {
                    _validationErrorCache.Add(error);
                }
            }
        }
        public async Task <string> Validate(ValidationActorModel validationActorModel, CancellationToken cancellationToken)
        {
            var jsonSerializationService = _parentLifeTimeScope.Resolve <IJsonSerializationService>();

            var internalDataCache = jsonSerializationService.Deserialize <InternalDataCache>(Encoding.UTF8.GetString(validationActorModel.InternalDataCache));
            var externalDataCache = jsonSerializationService.Deserialize <ExternalDataCache>(Encoding.UTF8.GetString(validationActorModel.ExternalDataCache));
            var fileDataCache     = jsonSerializationService.Deserialize <FileDataCache>(Encoding.UTF8.GetString(validationActorModel.FileDataCache));
            var message           = jsonSerializationService.Deserialize <Message>(new MemoryStream(validationActorModel.Message));

            cancellationToken.ThrowIfCancellationRequested();

            var validationContext = new ValidationContext
            {
                Input = message
            };

            using (var childLifeTimeScope = _parentLifeTimeScope.BeginLifetimeScope(c =>
            {
                c.RegisterInstance(validationContext).As <IValidationContext>();
                c.RegisterInstance(new Cache <IMessage> {
                    Item = message
                }).As <ICache <IMessage> >();
                c.RegisterInstance(internalDataCache).As <IInternalDataCache>();
                c.RegisterInstance(externalDataCache).As <IExternalDataCache>();
                c.RegisterInstance(fileDataCache).As <IFileDataCache>();
            }))
            {
                var executionContext = (ExecutionContext)childLifeTimeScope.Resolve <IExecutionContext>();
                executionContext.JobId   = validationActorModel.JobId;
                executionContext.TaskKey = _actorId.ToString();
                var logger = childLifeTimeScope.Resolve <ILogger>();
                try
                {
                    logger.LogDebug($"Validation Actor {executionContext.TaskKey} started learners: {validationContext.Input.Learners.Count}");
                    var preValidationOrchestrationService = childLifeTimeScope
                                                            .Resolve <IRuleSetOrchestrationService <ILearner, IValidationError> >();

                    var errors = await preValidationOrchestrationService.Execute(cancellationToken);

                    logger.LogDebug($"Validation Actor {executionContext.TaskKey} validation done");

                    var errorString = jsonSerializationService.Serialize(errors);
                    logger.LogDebug($"Validation Actor {executionContext.TaskKey} completed job");
                    return(errorString);
                }
                catch (Exception ex)
                {
                    ActorEventSource.Current.ActorMessage(this, "Exception-{0}", ex.ToString());
                    logger.LogError("Error while processing Actor job", ex);
                    throw;
                }
            }
        }
        private async Task <IEnumerable <IValidationError> > RunValidation(ValidationActorModel actorModel, CancellationToken cancellationToken)
        {
            if (_executionContext is ExecutionContext executionContextObj)
            {
                executionContextObj.JobId   = "-1";
                executionContextObj.TaskKey = _actorId.ToString();
            }

            ILogger logger = _parentLifeTimeScope.Resolve <ILogger>();

            InternalDataCache internalDataCache;
            ExternalDataCache externalDataCache;
            FileDataCache     fileDataCache;
            Message           message;
            ValidationContext validationContext;
            IEnumerable <IValidationError> errors;

            try
            {
                logger.LogDebug($"{nameof(ValidationActor)} {_actorId} {GC.GetGeneration(actorModel)} starting");

                internalDataCache = _jsonSerializationService.Deserialize <InternalDataCache>(Encoding.UTF8.GetString(actorModel.InternalDataCache));
                externalDataCache = _jsonSerializationService.Deserialize <ExternalDataCache>(Encoding.UTF8.GetString(actorModel.ExternalDataCache));
                fileDataCache     = _jsonSerializationService.Deserialize <FileDataCache>(Encoding.UTF8.GetString(actorModel.FileDataCache));
                message           = _jsonSerializationService.Deserialize <Message>(new MemoryStream(actorModel.Message));

                validationContext = new ValidationContext
                {
                    Input = message
                };

                logger.LogDebug($"{nameof(ValidationActor)} {_actorId} {GC.GetGeneration(actorModel)} finished getting input data");

                cancellationToken.ThrowIfCancellationRequested();
            }
            catch (Exception ex)
            {
                ActorEventSource.Current.ActorMessage(this, "Exception-{0}", ex.ToString());
                logger.LogError($"Error while processing {nameof(ValidationActor)}", ex);
                throw;
            }

            using (var childLifeTimeScope = _parentLifeTimeScope.BeginLifetimeScope(c =>
            {
                c.RegisterInstance(validationContext).As <IValidationContext>();
                c.RegisterInstance(new Cache <IMessage> {
                    Item = message
                }).As <ICache <IMessage> >();
                c.RegisterInstance(internalDataCache).As <IInternalDataCache>();
                c.RegisterInstance(externalDataCache).As <IExternalDataCache>();
                c.RegisterInstance(fileDataCache).As <IFileDataCache>();
            }))
            {
                ExecutionContext executionContext = (ExecutionContext)childLifeTimeScope.Resolve <IExecutionContext>();
                executionContext.JobId   = actorModel.JobId;
                executionContext.TaskKey = _actorId.ToString();
                ILogger jobLogger = childLifeTimeScope.Resolve <ILogger>();
                try
                {
                    jobLogger.LogDebug($"{nameof(ValidationActor)} {_actorId} {GC.GetGeneration(actorModel)} {executionContext.TaskKey} started learners: {validationContext.Input.Learners.Count}");
                    IRuleSetOrchestrationService <ILearner, IValidationError> preValidationOrchestrationService = childLifeTimeScope
                                                                                                                  .Resolve <IRuleSetOrchestrationService <ILearner, IValidationError> >();

                    errors = await preValidationOrchestrationService.Execute(cancellationToken);

                    jobLogger.LogDebug($"{nameof(ValidationActor)} {_actorId} {GC.GetGeneration(actorModel)} {executionContext.TaskKey} validation done");
                }
                catch (Exception ex)
                {
                    ActorEventSource.Current.ActorMessage(this, "Exception-{0}", ex.ToString());
                    jobLogger.LogError($"Error while processing {nameof(ValidationActor)}", ex);
                    throw;
                }
            }

            internalDataCache = null;
            externalDataCache = null;
            fileDataCache     = null;
            message           = null;
            validationContext = null;

            return(errors);
        }
        private async Task <IEnumerable <IValidationError> > RunValidation(ValidationActorModel actorModel, CancellationToken cancellationToken)
        {
            if (_executionContext is ExecutionContext executionContextObj)
            {
                executionContextObj.JobId   = "-1";
                executionContextObj.TaskKey = _actorId.ToString();
            }

            ILogger logger = _parentLifeTimeScope.Resolve <ILogger>();

            InternalDataCache              internalDataCache;
            ExternalDataCache              externalDataCacheGet;
            ExternalDataCache              externalDataCache;
            FileDataCache                  fileDataCache;
            Message                        message;
            IEnumerable <string>           tasks;
            ValidationContext              validationContext;
            IEnumerable <IValidationError> errors;

            try
            {
                logger.LogDebug($"{nameof(ValidationActor)} {_actorId} {GC.GetGeneration(actorModel)} starting");

                internalDataCache    = _jsonSerializationService.Deserialize <InternalDataCache>(actorModel.InternalDataCache);
                externalDataCacheGet = _jsonSerializationService.Deserialize <ExternalDataCache>(actorModel.ExternalDataCache);
                fileDataCache        = _jsonSerializationService.Deserialize <FileDataCache>(actorModel.FileDataCache);
                message = _jsonSerializationService.Deserialize <Message>(actorModel.Message);
                tasks   = _jsonSerializationService.Deserialize <IEnumerable <string> >(actorModel.TaskList);

                externalDataCache = new ExternalDataCache
                {
                    LearningDeliveries = externalDataCacheGet.LearningDeliveries.ToCaseInsensitiveDictionary(),
                    EPAOrganisations   = externalDataCacheGet.EPAOrganisations.ToCaseInsensitiveDictionary(),
                    ERNs = externalDataCacheGet.ERNs,
                    FCSContractAllocations = externalDataCacheGet.FCSContractAllocations,
                    Frameworks             = externalDataCacheGet.Frameworks,
                    Organisations          = externalDataCacheGet.Organisations,
                    Postcodes          = externalDataCacheGet.Postcodes.ToCaseInsensitiveHashSet(),
                    ONSPostcodes       = externalDataCacheGet.ONSPostcodes,
                    Standards          = externalDataCacheGet.Standards,
                    StandardValidities = externalDataCacheGet.StandardValidities,
                    ULNs              = externalDataCacheGet.ULNs,
                    ValidationErrors  = externalDataCacheGet.ValidationErrors.ToCaseInsensitiveDictionary(),
                    CampusIdentifiers = externalDataCacheGet.CampusIdentifiers
                };

                validationContext = new ValidationContext
                {
                    Input = message
                };

                logger.LogDebug($"{nameof(ValidationActor)} {_actorId} {GC.GetGeneration(actorModel)} finished getting input data");

                cancellationToken.ThrowIfCancellationRequested();
            }
            catch (Exception ex)
            {
                ActorEventSource.Current.ActorMessage(this, "Exception-{0}", ex.ToString());
                logger.LogError($"Error while processing {nameof(ValidationActor)}", ex);
                throw;
            }

            using (var childLifeTimeScope = _parentLifeTimeScope.BeginLifetimeScope(c =>
            {
                c.RegisterInstance(validationContext).As <IValidationContext>();
                c.RegisterInstance(new Cache <IMessage> {
                    Item = message
                }).As <ICache <IMessage> >();
                c.RegisterInstance(internalDataCache).As <IInternalDataCache>();
                c.RegisterInstance(externalDataCache).As <IExternalDataCache>();
                c.RegisterInstance(fileDataCache).As <IFileDataCache>();
            }))
            {
                ExecutionContext executionContext = (ExecutionContext)childLifeTimeScope.Resolve <IExecutionContext>();
                executionContext.JobId   = actorModel.JobId;
                executionContext.TaskKey = _actorId.ToString();
                ILogger jobLogger = childLifeTimeScope.Resolve <ILogger>();
                try
                {
                    jobLogger.LogDebug($"{nameof(ValidationActor)} {_actorId} {GC.GetGeneration(actorModel)} {executionContext.TaskKey} started learners: {validationContext.Input.Learners.Count}");
                    IRuleSetOrchestrationService <ILearner, IValidationError> preValidationOrchestrationService = childLifeTimeScope
                                                                                                                  .Resolve <IRuleSetOrchestrationService <ILearner, IValidationError> >();

                    errors = await preValidationOrchestrationService.ExecuteAsync(tasks, cancellationToken);

                    jobLogger.LogDebug($"{nameof(ValidationActor)} {_actorId} {GC.GetGeneration(actorModel)} {executionContext.TaskKey} validation done");
                }
                catch (Exception ex)
                {
                    ActorEventSource.Current.ActorMessage(this, "Exception-{0}", ex.ToString());
                    jobLogger.LogError($"Error while processing {nameof(ValidationActor)}", ex);
                    throw;
                }
            }

            internalDataCache = null;
            externalDataCache = null;
            fileDataCache     = null;
            message           = null;
            validationContext = null;

            return(errors);
        }
        private async Task ExecuteValidationActors(IPreValidationContext validationContext, CancellationToken cancellationToken)
        {
            // Get L/A and split the learners into separate lists
            IEnumerable <IMessage> learnerMessageShards = await _learnerPerActorProviderService.ProvideAsync();

            IEnumerable <IMessage> learnerDPMessageShards = await _learnerDPPerActorProviderService.ProvideAsync();

            List <IValidationActor>   learnerValidationActors   = new List <IValidationActor>();
            List <IValidationDPActor> learnerDPValidationActors = new List <IValidationDPActor>();
            List <Task <string> >     actorTasks = new List <Task <string> >();
            List <Task> actorDestroys            = new List <Task>();

            Stopwatch stopWatch = new Stopwatch();

            stopWatch.Start();

            _logger.LogDebug($"Validation will create {learnerMessageShards?.Count() ?? 0} actors");
            _logger.LogDebug($"DP Validation will create {learnerDPMessageShards?.Count() ?? 0} actors");

            string internalDataCacheAsString =
                _jsonSerializationService.Serialize(_internalDataCache);

            _logger.LogDebug($"_internalDataCache {internalDataCacheAsString.Length}");
            string fileDataCacheAsString =
                _jsonSerializationService.Serialize(_fileDataCache);

            _logger.LogDebug($"fileDataCacheAsString {fileDataCacheAsString.Length}");
            string externalDataCacheAsString =
                _jsonSerializationService.Serialize(_externalDataCache);

            _logger.LogDebug($"ExternalDataCache: {externalDataCacheAsString.Length}");
            string taskListAsString = _jsonSerializationService.Serialize(validationContext.Tasks);

            _logger.LogDebug($"taskListAsString {taskListAsString.Length}");

            if (learnerMessageShards != null)
            {
                foreach (IMessage messageShard in learnerMessageShards)
                {
                    _logger.LogDebug($"Validation Shard has {messageShard.Learners.Count} learners");

                    // create actors for each Shard.
                    IValidationActor actor = GetValidationActor();
                    learnerValidationActors.Add(actor);

                    // TODO:get reference data per each shard and send it to Actors
                    string ilrMessageAsString = _jsonSerializationService.Serialize(messageShard);

                    ValidationActorModel validationActorModel = new ValidationActorModel
                    {
                        JobId             = validationContext.JobId,
                        Message           = ilrMessageAsString,
                        InternalDataCache = internalDataCacheAsString,
                        ExternalDataCache = externalDataCacheAsString,
                        FileDataCache     = fileDataCacheAsString,
                        TaskList          = taskListAsString
                    };

                    actorTasks.Add(actor.Validate(validationActorModel, cancellationToken));
                }
            }

            if (learnerDPMessageShards != null)
            {
                foreach (IMessage messageShard in learnerDPMessageShards)
                {
                    _logger.LogDebug($"DP Validation Shard has {messageShard.LearnerDestinationAndProgressions.Count} learner DP records");

                    // create actors for each Shard.
                    IValidationDPActor actor = GetValidationDPActor();
                    learnerDPValidationActors.Add(actor);

                    // TODO:get reference data per each shard and send it to Actors
                    string ilrMessageAsString = _jsonSerializationService.Serialize(messageShard);

                    ValidationDPActorModel validationActorModel = new ValidationDPActorModel
                    {
                        JobId             = validationContext.JobId,
                        Message           = ilrMessageAsString,
                        InternalDataCache = internalDataCacheAsString,
                        ExternalDataCache = externalDataCacheAsString,
                        FileDataCache     = fileDataCacheAsString,
                        TaskList          = taskListAsString
                    };

                    actorTasks.Add(actor.Validate(validationActorModel, cancellationToken));
                }
            }

            _logger.LogDebug($"Starting {actorTasks.Count} validation actors after {stopWatch.ElapsedMilliseconds}ms prep time");
            stopWatch.Restart();

            await Task.WhenAll(actorTasks.ToArray()).ConfigureAwait(false);

            cancellationToken.ThrowIfCancellationRequested();

            _logger.LogDebug($"Collating {actorTasks.Count} validation actors after {stopWatch.ElapsedMilliseconds}ms execution time");
            stopWatch.Restart();

            foreach (Task <string> actorTask in actorTasks)
            {
                IEnumerable <U> errors = _jsonSerializationService.Deserialize <IEnumerable <U> >(actorTask.Result);

                foreach (U error in errors)
                {
                    _validationErrorCache.Add(error);
                }
            }

            _logger.LogDebug($"Destroying {actorTasks.Count} validation actors after {stopWatch.ElapsedMilliseconds}ms collation time");

            foreach (IValidationActor validationActor in learnerValidationActors)
            {
                actorDestroys.Add(DestroyValidationActorAsync(validationActor, cancellationToken));
            }

            foreach (IValidationDPActor validationDPActor in learnerDPValidationActors)
            {
                actorDestroys.Add(DestroyValidationDPActorAsync(validationDPActor, cancellationToken));
            }

            await Task.WhenAll(actorDestroys.ToArray()).ConfigureAwait(false);
        }