public async Task <List <ValidationErrorModel> > ValidateFile(IEasJobContext easJobContext, IReadOnlyDictionary <string, ValidationErrorRule> validationErrorReferenceData, CancellationToken cancellationToken) { _logger.LogInfo("Starting File Level validation"); var validationErrorModels = new List <ValidationErrorModel>(); await ValidateHeader(easJobContext, validationErrorModels, cancellationToken); if (validationErrorModels.Any()) { await GenerateErrorOutputsAsync(easJobContext, validationErrorModels, validationErrorReferenceData, cancellationToken); return(validationErrorModels); } await ValidateContent(easJobContext, validationErrorModels, cancellationToken); if (validationErrorModels.Any()) { await GenerateErrorOutputsAsync(easJobContext, validationErrorModels, validationErrorReferenceData, cancellationToken); return(validationErrorModels); } _logger.LogInfo("Finished File Level validation"); return(validationErrorModels); }
public async Task ExecuteAsync(IEasJobContext easJobContext, CancellationToken cancellationToken) { _logger.LogInfo("Validate Task is called."); try { var validationErrorReferenceData = await _validationErrorRuleService.GetAllValidationErrorRules(cancellationToken); var fileValidationErrors = await _fileValidationService.ValidateFile(easJobContext, validationErrorReferenceData, cancellationToken); if (fileValidationErrors.Any()) { return; } cancellationToken.ThrowIfCancellationRequested(); await _validationService.ValidateDataAsync(easJobContext, validationErrorReferenceData, cancellationToken); } catch (Exception ex) { _logger.LogError("Failed to validate data", ex); throw; } }
public async Task ExecuteAsync(IEasJobContext easJobContext, CancellationToken cancellationToken) { _logger.LogInfo("Storage Task is called."); try { var fileDataCache = await _fileDataCacheService.GetFileDataCacheAsync(easJobContext.Ukprn, cancellationToken); if (fileDataCache != null && !fileDataCache.FailedFileValidation) { List <PaymentType> paymentTypes = await _easPaymentService.GetAllPaymentTypes(cancellationToken); Guid submissionId = Guid.NewGuid(); List <EasSubmission> submissionList = BuildSubmissionList(easJobContext.Ukprn, fileDataCache.ValidEasCsvRecords, submissionId); List <EasSubmissionValue> submissionValuesList = BuildEasSubmissionValues(fileDataCache.ValidEasCsvRecords, paymentTypes, submissionId); await _easSubmissionService.PersistEasSubmissionAsync(submissionList, submissionValuesList, easJobContext.Ukprn, cancellationToken); await _validationErrorLoggerService.LogValidationErrorsAsync(easJobContext, fileDataCache.ValidationErrors, cancellationToken); } } catch (Exception ex) { _logger.LogError("Storage Task failed", ex); throw; } }
public async Task FileLevelErrorReportAsync( IEasJobContext easContext, IEnumerable <EasCsvRecord> models, IEnumerable <ValidationErrorModel> errors, CancellationToken cancellationToken) { if (cancellationToken.IsCancellationRequested) { return; } var reportOutputFilenames = new List <string>(); if (!string.IsNullOrWhiteSpace(easContext.ReportOutputFileNames)) { reportOutputFilenames.AddRange(easContext.ReportOutputFileNames.Split('|')); } foreach (var validationReport in _validationReports) { var reportsGenerated = await validationReport.GenerateReportAsync(easContext, models, errors, cancellationToken); reportOutputFilenames.AddRange(reportsGenerated); } await _resultReport.GenerateReportAsync(easContext, models, errors, cancellationToken); var zipName = _fileNameService.GetZipName(easContext.Ukprn, easContext.JobId, _zipName); await _zipService.CreateZipAsync(zipName, reportOutputFilenames, easContext.Container.ToString(), cancellationToken); easContext.ReportOutputFileNames = string.Join("|", reportOutputFilenames); }
public async Task LogValidationErrorsAsync(IEasJobContext easJobContext, IEnumerable <ValidationErrorModel> validationErrors, CancellationToken cancellationToken) { var sourceFile = new SourceFile() { Ukprn = easJobContext.Ukprn.ToString(), DateTime = easJobContext.SubmissionDateTimeUtc, FileName = easJobContext.FileReference, FilePreparationDate = BuildFilePrepDate(easJobContext.FileReference) }; var successfullyCommitted = false; _logger.LogInfo("Starting Validation Error persist to DEDs"); using (var connection = new SqlConnection(_easServiceConfiguration.EasdbConnectionString)) { SqlTransaction transaction = null; try { await connection.OpenAsync(cancellationToken); cancellationToken.ThrowIfCancellationRequested(); transaction = connection.BeginTransaction(); int sourceFileId = await LogErrorSourceFileAsync(transaction, connection, sourceFile); var validationErrorList = BuildErrors(validationErrors, sourceFileId); await LogValidationErrorsAsync(connection, transaction, validationErrorList, cancellationToken); transaction.Commit(); successfullyCommitted = true; _logger.LogInfo("Finished Validation Error persist to DEDs"); } catch (Exception ex) { _logger.LogError("Failed - Validation Error persist to DEDs", ex); throw; } finally { if (!successfullyCommitted) { try { transaction?.Rollback(); } catch (Exception ex) { _logger.LogError("Failed to rollback DEDs persist transaction", ex); throw; } } } } }
public async Task <List <ValidationErrorModel> > ValidateDataAsync(IEasJobContext easJobContext, IReadOnlyDictionary <string, ValidationErrorRule> validationErrorReferenceData, CancellationToken cancellationToken) { var easCsvRecords = await GetCsvRecords(easJobContext, cancellationToken); List <ValidationErrorModel> validationErrorModels = await ValidateAsync(easJobContext, validationErrorReferenceData, easCsvRecords.ToList(), cancellationToken); List <EasCsvRecord> validRecords = GetValidRows(easCsvRecords, validationErrorModels); var fileDataCache = _fileDataCacheService.BuildFileDataCache(easJobContext.Ukprn, easJobContext.FileReference, easCsvRecords, validRecords, validationErrorModels, false); await _fileDataCacheService.PopulateFileDataCacheAsync(fileDataCache, cancellationToken); return(validationErrorModels); }
public async Task <IEnumerable <string> > GenerateReportAsync( IEasJobContext easContext, IEnumerable <EasCsvRecord> data, IEnumerable <ValidationErrorModel> validationErrors, CancellationToken cancellationToken) { var models = GetReportData(data, validationErrors); var fileName = _fileNameService.GetFilename(easContext.Ukprn, easContext.JobId, ReportNameConstants.FundingReport, easContext.SubmissionDateTimeUtc, OutputTypes.Csv); await _csvService.WriteAsync <EasCsvRecord, EasCsvRecordMapper>(models, fileName, easContext.Container, cancellationToken); return(new[] { fileName }); }
private async Task GenerateErrorOutputsAsync(IEasJobContext easJobContext, ICollection <ValidationErrorModel> errorModels, IReadOnlyDictionary <string, ValidationErrorRule> validationErrorReferenceData, CancellationToken cancellationToken) { _logger.LogInfo("File Level validation - Error(s) found."); var easCsvRecords = Enumerable.Empty <EasCsvRecord>(); var errorsToReport = _validationErrorBuilder.BuildFileLevelValidationErrors(errorModels, validationErrorReferenceData); await _validationErrorLoggerService.LogValidationErrorsAsync(easJobContext, errorsToReport, cancellationToken); await _reportingController.FileLevelErrorReportAsync(easJobContext, easCsvRecords, errorsToReport, cancellationToken); var fileDataCache = _fileDataCacheService.BuildFileDataCache(easJobContext.Ukprn, easJobContext.FileReference, easCsvRecords, null, null, true); await _fileDataCacheService.PopulateFileDataCacheAsync(fileDataCache, cancellationToken); }
public async Task<IEnumerable<string>> GenerateReportAsync( IEasJobContext easContext, IEnumerable<EasCsvRecord> data, IEnumerable<ValidationErrorModel> validationErrors, CancellationToken cancellationToken) { var report = GetValidationReport(data, validationErrors); var fileName = _fileNameService.GetFilename(easContext.Ukprn, easContext.JobId, ReportNameConstants.ValidationResultReport, easContext.SubmissionDateTimeUtc, OutputTypes.Json); using (var fileStream = await _fileService.OpenWriteStreamAsync(fileName, easContext.Container, cancellationToken)) { _jsonSerializationService.Serialize(report, fileStream); } return new[] { fileName }; }
public async Task <ICollection <ValidationErrorModel> > ValidateHeader(IEasJobContext easJobContext, ICollection <ValidationErrorModel> errorModels, CancellationToken cancellationToken) { try { string[] header; using (var stream = await _fileService.OpenReadStreamAsync(easJobContext.FileReference, easJobContext.Container, cancellationToken)) { using (var reader = new StreamReader(stream)) { using (var csvReader = new CsvReader(reader)) { csvReader.Read(); csvReader.ReadHeader(); header = csvReader.Context.HeaderRecord; } } } if (_easCSVHeaders.Count() != header.Count()) { throw new InvalidCsvHeaderException(); } foreach (var column in _easCSVHeaders) { if (!header.Contains(column)) { throw new InvalidCsvHeaderException(); } } } catch (InvalidCsvHeaderException ex) { _logger.LogInfo("Invalid csv header found."); errorModels.Add(BuildValidationError(ErrorNameConstants.FileFormat_01)); } catch (Exception ex) { _logger.LogInfo("Invalid csv header found."); errorModels.Add(BuildValidationError(ErrorNameConstants.FileFormat_01)); } return(errorModels); }
public async Task ExecuteAsync(IEasJobContext easJobContext, CancellationToken cancellationToken) { _logger.LogInfo("Reporting Task is called."); try { IEnumerable <ValidationErrorModel> validationErrorModels; IEnumerable <EasCsvRecord> easCsvRecords; IFileDataCache fileDataCache = await _fileDataCacheService.GetFileDataCacheAsync(easJobContext.Ukprn, cancellationToken); if (fileDataCache == null) { List <PaymentType> allPaymentTypes = await _easPaymentService.GetAllPaymentTypes(cancellationToken); List <EasSubmissionValue> easSubmissionValues = await _easSubmissionService.GetEasSubmissionValuesAsync(easJobContext.Ukprn, cancellationToken); var validationErrors = await _validationErrorService.GetValidationErrorsAsync(easJobContext.Ukprn, cancellationToken); easCsvRecords = BuildEasCsvRecords(allPaymentTypes, easSubmissionValues); validationErrorModels = BuildValidationErrorModels(validationErrors); if (easCsvRecords.Any() || validationErrorModels.Any()) { await _reportingController.ProduceReportsAsync(easJobContext, easCsvRecords, validationErrorModels, cancellationToken); } } if (fileDataCache != null && !fileDataCache.FailedFileValidation) { easCsvRecords = fileDataCache.AllEasCsvRecords; validationErrorModels = fileDataCache.ValidationErrors; await _reportingController.ProduceReportsAsync(easJobContext, easCsvRecords, validationErrorModels, cancellationToken); } if (fileDataCache != null && fileDataCache.FailedFileValidation) { _logger.LogError($"Reports are not generated as File- {easJobContext.FileReference} failed file Validation"); } } catch (System.Exception ex) { _logger.LogError("Reporting Task failed", ex); throw; } }
private async Task <List <ValidationErrorModel> > ValidateAsync(IEasJobContext easJobContext, IReadOnlyDictionary <string, ValidationErrorRule> validationErrorReferenceData, IEnumerable <EasCsvRecord> easCsvRecords, CancellationToken cancellationToken) { List <ValidationResult> validationResults = new List <ValidationResult>(); List <ValidationResult> businessRulesValidationResults = new List <ValidationResult>(); cancellationToken.ThrowIfCancellationRequested(); List <PaymentType> paymentTypes = await _easPaymentService.GetAllPaymentTypes(cancellationToken); List <ContractAllocation> contractsForProvider = await _fcsDataService.GetContractsForProviderAsync(easJobContext.Ukprn, cancellationToken); List <FundingLineContractTypeMapping> fundingLineContractTypeMappings = await _fundingLineContractTypeMappingDataService.GetAllFundingLineContractTypeMappings(cancellationToken); var devolvedContracts = await _fcsDataService.GetDevolvedContractsForProviderAsync(easJobContext.Ukprn, cancellationToken); var sofCodeMcaShortCodeDictionary = await _postcodesDataService.GetMcaShortCodesForSofCodesAsync(DataServiceConstants.ValidDevolvedSourceOfFundingCodes, cancellationToken); BusinessRulesValidator validator = new BusinessRulesValidator(contractsForProvider, fundingLineContractTypeMappings, paymentTypes, devolvedContracts, sofCodeMcaShortCodeDictionary, _dateTimeProvider, easJobContext.ReturnPeriod); // Business Rule validators foreach (EasCsvRecord easRecord in easCsvRecords) { ValidationResult validate = validator.Validate(easRecord); if (!validate.IsValid) { businessRulesValidationResults.Add(validate); } } // Cross Record Validation var crossRecordValidationResult = new CrossRecordValidator().Validate(easCsvRecords); validationResults.AddRange(businessRulesValidationResults); if (!crossRecordValidationResult.IsValid) { validationResults.Add(crossRecordValidationResult); } var validationErrorList = _validationErrorBuilder.BuildValidationErrors(validationResults, validationErrorReferenceData).ToList(); return(validationErrorList); }
public async Task <bool> CallbackAsync(IEasJobContext easJobContext, CancellationToken cancellationToken, IList <IEasServiceTask> easServiceTasks) { _logger.LogInfo("EAS callback invoked"); if (!easJobContext.Tasks.Any()) { _logger.LogInfo("EAS. No tasks to run."); return(true); } foreach (var task in easServiceTasks) { _logger.LogInfo($"EAS Service Task : {task.TaskName} Starting"); await task.ExecuteAsync(easJobContext, cancellationToken); _logger.LogInfo($"EAS Service Task : {task.TaskName} Finished"); } return(true); }
public async Task <ICollection <ValidationErrorModel> > ValidateContent(IEasJobContext easJobContext, ICollection <ValidationErrorModel> errorModels, CancellationToken cancellationToken) { try { using (var stream = await _fileService.OpenReadStreamAsync(easJobContext.FileReference, easJobContext.Container, cancellationToken)) { using (var reader = new StreamReader(stream)) { using (var csv = new CsvReader(reader)) { var columns = new List <IEnumerable <dynamic> >(); csv.Configuration.HasHeaderRecord = false; csv.Configuration.HeaderValidated = null; csv.Configuration.IgnoreBlankLines = true; csv.Configuration.TrimOptions = TrimOptions.Trim; while (csv.Read()) { if (csv.Context.Record.Length != _expectedDataColumnColumnCount) { throw new InvalidCsvException(); } } } } } } catch (InvalidCsvException ex) { _logger.LogInfo("Invalid csv content found."); errorModels.Add(BuildValidationError(ErrorNameConstants.FileFormat_02)); } catch (Exception ex) { _logger.LogInfo("Invalid csv content found."); errorModels.Add(BuildValidationError(ErrorNameConstants.FileFormat_02)); } return(errorModels); }
private async Task <List <EasCsvRecord> > GetCsvRecords(IEasJobContext easJobContext, CancellationToken cancellationToken) { var records = await _easFileDataProviderService.ProvideData(easJobContext.FileReference, easJobContext.Container, cancellationToken); return(records); }