Exemple #1
0
        public async Task <string> Validate(ValidationDPActorModel actorModel, CancellationToken cancellationToken)
        {
            IEnumerable <IValidationError> results = await RunValidation(actorModel, cancellationToken);

            actorModel = null;

            GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
            GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced, true, true);

            return(_jsonSerializationService.Serialize(results));
        }
        private async Task ExecuteValidationActors(IPreValidationContext validationContext, CancellationToken cancellationToken)
        {
            // Get L/A and split the learners into separate lists
            IEnumerable <IMessage> learnerMessageShards = await _learnerPerActorProviderService.ProvideAsync();

            IEnumerable <IMessage> learnerDPMessageShards = await _learnerDPPerActorProviderService.ProvideAsync();

            List <IValidationActor>   learnerValidationActors   = new List <IValidationActor>();
            List <IValidationDPActor> learnerDPValidationActors = new List <IValidationDPActor>();
            List <Task <string> >     actorTasks = new List <Task <string> >();
            List <Task> actorDestroys            = new List <Task>();

            Stopwatch stopWatch = new Stopwatch();

            stopWatch.Start();

            _logger.LogDebug($"Validation will create {learnerMessageShards?.Count() ?? 0} actors");
            _logger.LogDebug($"DP Validation will create {learnerDPMessageShards?.Count() ?? 0} actors");

            string internalDataCacheAsString =
                _jsonSerializationService.Serialize(_internalDataCache);

            _logger.LogDebug($"_internalDataCache {internalDataCacheAsString.Length}");
            string fileDataCacheAsString =
                _jsonSerializationService.Serialize(_fileDataCache);

            _logger.LogDebug($"fileDataCacheAsString {fileDataCacheAsString.Length}");
            string externalDataCacheAsString =
                _jsonSerializationService.Serialize(_externalDataCache);

            _logger.LogDebug($"ExternalDataCache: {externalDataCacheAsString.Length}");
            string taskListAsString = _jsonSerializationService.Serialize(validationContext.Tasks);

            _logger.LogDebug($"taskListAsString {taskListAsString.Length}");

            if (learnerMessageShards != null)
            {
                foreach (IMessage messageShard in learnerMessageShards)
                {
                    _logger.LogDebug($"Validation Shard has {messageShard.Learners.Count} learners");

                    // create actors for each Shard.
                    IValidationActor actor = GetValidationActor();
                    learnerValidationActors.Add(actor);

                    // TODO:get reference data per each shard and send it to Actors
                    string ilrMessageAsString = _jsonSerializationService.Serialize(messageShard);

                    ValidationActorModel validationActorModel = new ValidationActorModel
                    {
                        JobId             = validationContext.JobId,
                        Message           = ilrMessageAsString,
                        InternalDataCache = internalDataCacheAsString,
                        ExternalDataCache = externalDataCacheAsString,
                        FileDataCache     = fileDataCacheAsString,
                        TaskList          = taskListAsString
                    };

                    actorTasks.Add(actor.Validate(validationActorModel, cancellationToken));
                }
            }

            if (learnerDPMessageShards != null)
            {
                foreach (IMessage messageShard in learnerDPMessageShards)
                {
                    _logger.LogDebug($"DP Validation Shard has {messageShard.LearnerDestinationAndProgressions.Count} learner DP records");

                    // create actors for each Shard.
                    IValidationDPActor actor = GetValidationDPActor();
                    learnerDPValidationActors.Add(actor);

                    // TODO:get reference data per each shard and send it to Actors
                    string ilrMessageAsString = _jsonSerializationService.Serialize(messageShard);

                    ValidationDPActorModel validationActorModel = new ValidationDPActorModel
                    {
                        JobId             = validationContext.JobId,
                        Message           = ilrMessageAsString,
                        InternalDataCache = internalDataCacheAsString,
                        ExternalDataCache = externalDataCacheAsString,
                        FileDataCache     = fileDataCacheAsString,
                        TaskList          = taskListAsString
                    };

                    actorTasks.Add(actor.Validate(validationActorModel, cancellationToken));
                }
            }

            _logger.LogDebug($"Starting {actorTasks.Count} validation actors after {stopWatch.ElapsedMilliseconds}ms prep time");
            stopWatch.Restart();

            await Task.WhenAll(actorTasks.ToArray()).ConfigureAwait(false);

            cancellationToken.ThrowIfCancellationRequested();

            _logger.LogDebug($"Collating {actorTasks.Count} validation actors after {stopWatch.ElapsedMilliseconds}ms execution time");
            stopWatch.Restart();

            foreach (Task <string> actorTask in actorTasks)
            {
                IEnumerable <U> errors = _jsonSerializationService.Deserialize <IEnumerable <U> >(actorTask.Result);

                foreach (U error in errors)
                {
                    _validationErrorCache.Add(error);
                }
            }

            _logger.LogDebug($"Destroying {actorTasks.Count} validation actors after {stopWatch.ElapsedMilliseconds}ms collation time");

            foreach (IValidationActor validationActor in learnerValidationActors)
            {
                actorDestroys.Add(DestroyValidationActorAsync(validationActor, cancellationToken));
            }

            foreach (IValidationDPActor validationDPActor in learnerDPValidationActors)
            {
                actorDestroys.Add(DestroyValidationDPActorAsync(validationDPActor, cancellationToken));
            }

            await Task.WhenAll(actorDestroys.ToArray()).ConfigureAwait(false);
        }
Exemple #3
0
        private async Task <IEnumerable <IValidationError> > RunValidation(ValidationDPActorModel actorModel, CancellationToken cancellationToken)
        {
            if (_executionContext is ExecutionContext executionContextObj)
            {
                executionContextObj.JobId   = "-1";
                executionContextObj.TaskKey = _actorId.ToString();
            }

            ILogger logger = _parentLifeTimeScope.Resolve <ILogger>();

            InternalDataCache              internalDataCache;
            ExternalDataCache              externalDataCacheGet;
            ExternalDataCache              externalDataCache;
            FileDataCache                  fileDataCache;
            Message                        message;
            IEnumerable <string>           tasks;
            ValidationContext              validationContext;
            IEnumerable <IValidationError> errors;

            try
            {
                logger.LogDebug($"{nameof(ValidationDPActor)} {_actorId} {GC.GetGeneration(actorModel)} starting");

                internalDataCache    = _jsonSerializationService.Deserialize <InternalDataCache>(actorModel.InternalDataCache);
                externalDataCacheGet = _jsonSerializationService.Deserialize <ExternalDataCache>(actorModel.ExternalDataCache);
                fileDataCache        = _jsonSerializationService.Deserialize <FileDataCache>(actorModel.FileDataCache);
                message = _jsonSerializationService.Deserialize <Message>(actorModel.Message);
                tasks   = _jsonSerializationService.Deserialize <IEnumerable <string> >(actorModel.TaskList);

                externalDataCache = new ExternalDataCache
                {
                    ULNs             = externalDataCacheGet.ULNs,
                    ValidationErrors = externalDataCacheGet.ValidationErrors.ToCaseInsensitiveDictionary()
                };

                validationContext = new ValidationContext
                {
                    Input = message
                };

                logger.LogDebug($"{nameof(ValidationDPActor)} {_actorId} {GC.GetGeneration(actorModel)} finished getting input data");

                cancellationToken.ThrowIfCancellationRequested();
            }
            catch (Exception ex)
            {
                ActorEventSource.Current.ActorMessage(this, "Exception-{0}", ex.ToString());
                logger.LogError($"Error while processing {nameof(ValidationDPActor)}", ex);
                throw;
            }

            using (var childLifeTimeScope = _parentLifeTimeScope.BeginLifetimeScope(c =>
            {
                c.RegisterInstance(validationContext).As <IValidationContext>();
                c.RegisterInstance(new Cache <IMessage> {
                    Item = message
                }).As <ICache <IMessage> >();
                c.RegisterInstance(internalDataCache).As <IInternalDataCache>();
                c.RegisterInstance(externalDataCache).As <IExternalDataCache>();
                c.RegisterInstance(fileDataCache).As <IFileDataCache>();
            }))
            {
                ExecutionContext executionContext = (ExecutionContext)childLifeTimeScope.Resolve <IExecutionContext>();
                executionContext.JobId   = actorModel.JobId;
                executionContext.TaskKey = _actorId.ToString();
                ILogger jobLogger = childLifeTimeScope.Resolve <ILogger>();
                try
                {
                    jobLogger.LogDebug($"{nameof(ValidationDPActor)} {_actorId} {GC.GetGeneration(actorModel)} {executionContext.TaskKey} started Destination and Progressions: {validationContext.Input.LearnerDestinationAndProgressions.Count}");
                    IRuleSetOrchestrationService <ILearnerDestinationAndProgression, IValidationError> preValidationOrchestrationService = childLifeTimeScope
                                                                                                                                           .Resolve <IRuleSetOrchestrationService <ILearnerDestinationAndProgression, IValidationError> >();

                    errors = await preValidationOrchestrationService.ExecuteAsync(tasks, cancellationToken);

                    jobLogger.LogDebug($"{nameof(ValidationDPActor)} {_actorId} {GC.GetGeneration(actorModel)} {executionContext.TaskKey} Destination and Progression validation done");
                }
                catch (Exception ex)
                {
                    ActorEventSource.Current.ActorMessage(this, "Exception-{0}", ex.ToString());
                    jobLogger.LogError($"Error while processing {nameof(ValidationDPActor)}", ex);
                    throw;
                }
            }

            internalDataCache = null;
            externalDataCache = null;
            fileDataCache     = null;
            message           = null;
            validationContext = null;

            return(errors);
        }