public async Task <List <ValidationErrorModel> > ValidateFile(IEasJobContext easJobContext, IReadOnlyDictionary <string, ValidationErrorRule> validationErrorReferenceData, CancellationToken cancellationToken)
{
_logger.LogInfo("Starting File Level validation");
var validationErrorModels = new List <ValidationErrorModel>();
await ValidateHeader(easJobContext, validationErrorModels, cancellationToken);
if (validationErrorModels.Any())
{
await GenerateErrorOutputsAsync(easJobContext, validationErrorModels, validationErrorReferenceData, cancellationToken);
return(validationErrorModels);
}
await ValidateContent(easJobContext, validationErrorModels, cancellationToken);
if (validationErrorModels.Any())
{
await GenerateErrorOutputsAsync(easJobContext, validationErrorModels, validationErrorReferenceData, cancellationToken);
return(validationErrorModels);
}
_logger.LogInfo("Finished File Level validation");
return(validationErrorModels);
}
public async Task ExecuteAsync(IEasJobContext easJobContext, CancellationToken cancellationToken)
{
_logger.LogInfo("Validate Task is called.");
try
{
var validationErrorReferenceData = await _validationErrorRuleService.GetAllValidationErrorRules(cancellationToken);
var fileValidationErrors = await _fileValidationService.ValidateFile(easJobContext, validationErrorReferenceData, cancellationToken);
if (fileValidationErrors.Any())
{
return;
}
cancellationToken.ThrowIfCancellationRequested();
await _validationService.ValidateDataAsync(easJobContext, validationErrorReferenceData, cancellationToken);
}
catch (Exception ex)
{
_logger.LogError("Failed to validate data", ex);
throw;
}
}
public async Task ExecuteAsync(IEasJobContext easJobContext, CancellationToken cancellationToken)
{
_logger.LogInfo("Storage Task is called.");
try
{
var fileDataCache = await _fileDataCacheService.GetFileDataCacheAsync(easJobContext.Ukprn, cancellationToken);
if (fileDataCache != null && !fileDataCache.FailedFileValidation)
{
List <PaymentType> paymentTypes = await _easPaymentService.GetAllPaymentTypes(cancellationToken);
Guid submissionId = Guid.NewGuid();
List <EasSubmission> submissionList = BuildSubmissionList(easJobContext.Ukprn, fileDataCache.ValidEasCsvRecords, submissionId);
List <EasSubmissionValue> submissionValuesList = BuildEasSubmissionValues(fileDataCache.ValidEasCsvRecords, paymentTypes, submissionId);
await _easSubmissionService.PersistEasSubmissionAsync(submissionList, submissionValuesList, easJobContext.Ukprn, cancellationToken);
await _validationErrorLoggerService.LogValidationErrorsAsync(easJobContext, fileDataCache.ValidationErrors, cancellationToken);
}
}
catch (Exception ex)
{
_logger.LogError("Storage Task failed", ex);
throw;
}
}
public async Task FileLevelErrorReportAsync(
IEasJobContext easContext,
IEnumerable <EasCsvRecord> models,
IEnumerable <ValidationErrorModel> errors,
CancellationToken cancellationToken)
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
var reportOutputFilenames = new List <string>();
if (!string.IsNullOrWhiteSpace(easContext.ReportOutputFileNames))
{
reportOutputFilenames.AddRange(easContext.ReportOutputFileNames.Split('|'));
}
foreach (var validationReport in _validationReports)
{
var reportsGenerated = await validationReport.GenerateReportAsync(easContext, models, errors, cancellationToken);
reportOutputFilenames.AddRange(reportsGenerated);
}
await _resultReport.GenerateReportAsync(easContext, models, errors, cancellationToken);
var zipName = _fileNameService.GetZipName(easContext.Ukprn, easContext.JobId, _zipName);
await _zipService.CreateZipAsync(zipName, reportOutputFilenames, easContext.Container.ToString(), cancellationToken);
easContext.ReportOutputFileNames = string.Join("|", reportOutputFilenames);
}
public async Task LogValidationErrorsAsync(IEasJobContext easJobContext, IEnumerable <ValidationErrorModel> validationErrors, CancellationToken cancellationToken)
{
var sourceFile = new SourceFile()
{
Ukprn = easJobContext.Ukprn.ToString(),
DateTime = easJobContext.SubmissionDateTimeUtc,
FileName = easJobContext.FileReference,
FilePreparationDate = BuildFilePrepDate(easJobContext.FileReference)
};
var successfullyCommitted = false;
_logger.LogInfo("Starting Validation Error persist to DEDs");
using (var connection = new SqlConnection(_easServiceConfiguration.EasdbConnectionString))
{
SqlTransaction transaction = null;
try
{
await connection.OpenAsync(cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
transaction = connection.BeginTransaction();
int sourceFileId = await LogErrorSourceFileAsync(transaction, connection, sourceFile);
var validationErrorList = BuildErrors(validationErrors, sourceFileId);
await LogValidationErrorsAsync(connection, transaction, validationErrorList, cancellationToken);
transaction.Commit();
successfullyCommitted = true;
_logger.LogInfo("Finished Validation Error persist to DEDs");
}
catch (Exception ex)
{
_logger.LogError("Failed - Validation Error persist to DEDs", ex);
throw;
}
finally
{
if (!successfullyCommitted)
{
try
{
transaction?.Rollback();
}
catch (Exception ex)
{
_logger.LogError("Failed to rollback DEDs persist transaction", ex);
throw;
}
}
}
}
}
public async Task <List <ValidationErrorModel> > ValidateDataAsync(IEasJobContext easJobContext, IReadOnlyDictionary <string, ValidationErrorRule> validationErrorReferenceData, CancellationToken cancellationToken)
{
var easCsvRecords = await GetCsvRecords(easJobContext, cancellationToken);
List <ValidationErrorModel> validationErrorModels = await ValidateAsync(easJobContext, validationErrorReferenceData, easCsvRecords.ToList(), cancellationToken);
List <EasCsvRecord> validRecords = GetValidRows(easCsvRecords, validationErrorModels);
var fileDataCache = _fileDataCacheService.BuildFileDataCache(easJobContext.Ukprn, easJobContext.FileReference, easCsvRecords, validRecords, validationErrorModels, false);
await _fileDataCacheService.PopulateFileDataCacheAsync(fileDataCache, cancellationToken);
return(validationErrorModels);
}
public async Task <IEnumerable <string> > GenerateReportAsync(
IEasJobContext easContext,
IEnumerable <EasCsvRecord> data,
IEnumerable <ValidationErrorModel> validationErrors,
CancellationToken cancellationToken)
{
var models = GetReportData(data, validationErrors);
var fileName = _fileNameService.GetFilename(easContext.Ukprn, easContext.JobId, ReportNameConstants.FundingReport, easContext.SubmissionDateTimeUtc, OutputTypes.Csv);
await _csvService.WriteAsync <EasCsvRecord, EasCsvRecordMapper>(models, fileName, easContext.Container, cancellationToken);
return(new[] { fileName });
}
private async Task GenerateErrorOutputsAsync(IEasJobContext easJobContext, ICollection <ValidationErrorModel> errorModels, IReadOnlyDictionary <string, ValidationErrorRule> validationErrorReferenceData, CancellationToken cancellationToken)
{
_logger.LogInfo("File Level validation - Error(s) found.");
var easCsvRecords = Enumerable.Empty <EasCsvRecord>();
var errorsToReport = _validationErrorBuilder.BuildFileLevelValidationErrors(errorModels, validationErrorReferenceData);
await _validationErrorLoggerService.LogValidationErrorsAsync(easJobContext, errorsToReport, cancellationToken);
await _reportingController.FileLevelErrorReportAsync(easJobContext, easCsvRecords, errorsToReport, cancellationToken);
var fileDataCache = _fileDataCacheService.BuildFileDataCache(easJobContext.Ukprn, easJobContext.FileReference, easCsvRecords, null, null, true);
await _fileDataCacheService.PopulateFileDataCacheAsync(fileDataCache, cancellationToken);
}
public async Task<IEnumerable<string>> GenerateReportAsync(
IEasJobContext easContext,
IEnumerable<EasCsvRecord> data,
IEnumerable<ValidationErrorModel> validationErrors,
CancellationToken cancellationToken)
{
var report = GetValidationReport(data, validationErrors);
var fileName = _fileNameService.GetFilename(easContext.Ukprn, easContext.JobId, ReportNameConstants.ValidationResultReport, easContext.SubmissionDateTimeUtc, OutputTypes.Json);
using (var fileStream = await _fileService.OpenWriteStreamAsync(fileName, easContext.Container, cancellationToken))
{
_jsonSerializationService.Serialize(report, fileStream);
}
return new[] { fileName };
}
public async Task <ICollection <ValidationErrorModel> > ValidateHeader(IEasJobContext easJobContext, ICollection <ValidationErrorModel> errorModels, CancellationToken cancellationToken)
{
try
{
string[] header;
using (var stream = await _fileService.OpenReadStreamAsync(easJobContext.FileReference,
easJobContext.Container, cancellationToken))
{
using (var reader = new StreamReader(stream))
{
using (var csvReader = new CsvReader(reader))
{
csvReader.Read();
csvReader.ReadHeader();
header = csvReader.Context.HeaderRecord;
}
}
}
if (_easCSVHeaders.Count() != header.Count())
{
throw new InvalidCsvHeaderException();
}
foreach (var column in _easCSVHeaders)
{
if (!header.Contains(column))
{
throw new InvalidCsvHeaderException();
}
}
}
catch (InvalidCsvHeaderException ex)
{
_logger.LogInfo("Invalid csv header found.");
errorModels.Add(BuildValidationError(ErrorNameConstants.FileFormat_01));
}
catch (Exception ex)
{
_logger.LogInfo("Invalid csv header found.");
errorModels.Add(BuildValidationError(ErrorNameConstants.FileFormat_01));
}
return(errorModels);
}
public async Task ExecuteAsync(IEasJobContext easJobContext, CancellationToken cancellationToken)
{
_logger.LogInfo("Reporting Task is called.");
try
{
IEnumerable <ValidationErrorModel> validationErrorModels;
IEnumerable <EasCsvRecord> easCsvRecords;
IFileDataCache fileDataCache = await _fileDataCacheService.GetFileDataCacheAsync(easJobContext.Ukprn, cancellationToken);
if (fileDataCache == null)
{
List <PaymentType> allPaymentTypes = await _easPaymentService.GetAllPaymentTypes(cancellationToken);
List <EasSubmissionValue> easSubmissionValues = await _easSubmissionService.GetEasSubmissionValuesAsync(easJobContext.Ukprn, cancellationToken);
var validationErrors = await _validationErrorService.GetValidationErrorsAsync(easJobContext.Ukprn, cancellationToken);
easCsvRecords = BuildEasCsvRecords(allPaymentTypes, easSubmissionValues);
validationErrorModels = BuildValidationErrorModels(validationErrors);
if (easCsvRecords.Any() || validationErrorModels.Any())
{
await _reportingController.ProduceReportsAsync(easJobContext, easCsvRecords, validationErrorModels, cancellationToken);
}
}
if (fileDataCache != null && !fileDataCache.FailedFileValidation)
{
easCsvRecords = fileDataCache.AllEasCsvRecords;
validationErrorModels = fileDataCache.ValidationErrors;
await _reportingController.ProduceReportsAsync(easJobContext, easCsvRecords, validationErrorModels, cancellationToken);
}
if (fileDataCache != null && fileDataCache.FailedFileValidation)
{
_logger.LogError($"Reports are not generated as File- {easJobContext.FileReference} failed file Validation");
}
}
catch (System.Exception ex)
{
_logger.LogError("Reporting Task failed", ex);
throw;
}
}
private async Task <List <ValidationErrorModel> > ValidateAsync(IEasJobContext easJobContext, IReadOnlyDictionary <string, ValidationErrorRule> validationErrorReferenceData, IEnumerable <EasCsvRecord> easCsvRecords, CancellationToken cancellationToken)
{
List <ValidationResult> validationResults = new List <ValidationResult>();
List <ValidationResult> businessRulesValidationResults = new List <ValidationResult>();
cancellationToken.ThrowIfCancellationRequested();
List <PaymentType> paymentTypes = await _easPaymentService.GetAllPaymentTypes(cancellationToken);
List <ContractAllocation> contractsForProvider = await _fcsDataService.GetContractsForProviderAsync(easJobContext.Ukprn, cancellationToken);
List <FundingLineContractTypeMapping> fundingLineContractTypeMappings = await _fundingLineContractTypeMappingDataService.GetAllFundingLineContractTypeMappings(cancellationToken);
var devolvedContracts = await _fcsDataService.GetDevolvedContractsForProviderAsync(easJobContext.Ukprn, cancellationToken);
var sofCodeMcaShortCodeDictionary = await _postcodesDataService.GetMcaShortCodesForSofCodesAsync(DataServiceConstants.ValidDevolvedSourceOfFundingCodes, cancellationToken);
BusinessRulesValidator validator = new BusinessRulesValidator(contractsForProvider, fundingLineContractTypeMappings, paymentTypes, devolvedContracts, sofCodeMcaShortCodeDictionary, _dateTimeProvider, easJobContext.ReturnPeriod);
// Business Rule validators
foreach (EasCsvRecord easRecord in easCsvRecords)
{
ValidationResult validate = validator.Validate(easRecord);
if (!validate.IsValid)
{
businessRulesValidationResults.Add(validate);
}
}
// Cross Record Validation
var crossRecordValidationResult = new CrossRecordValidator().Validate(easCsvRecords);
validationResults.AddRange(businessRulesValidationResults);
if (!crossRecordValidationResult.IsValid)
{
validationResults.Add(crossRecordValidationResult);
}
var validationErrorList = _validationErrorBuilder.BuildValidationErrors(validationResults, validationErrorReferenceData).ToList();
return(validationErrorList);
}
public async Task <bool> CallbackAsync(IEasJobContext easJobContext, CancellationToken cancellationToken, IList <IEasServiceTask> easServiceTasks)
{
_logger.LogInfo("EAS callback invoked");
if (!easJobContext.Tasks.Any())
{
_logger.LogInfo("EAS. No tasks to run.");
return(true);
}
foreach (var task in easServiceTasks)
{
_logger.LogInfo($"EAS Service Task : {task.TaskName} Starting");
await task.ExecuteAsync(easJobContext, cancellationToken);
_logger.LogInfo($"EAS Service Task : {task.TaskName} Finished");
}
return(true);
}
public async Task <ICollection <ValidationErrorModel> > ValidateContent(IEasJobContext easJobContext, ICollection <ValidationErrorModel> errorModels, CancellationToken cancellationToken)
{
try
{
using (var stream = await _fileService.OpenReadStreamAsync(easJobContext.FileReference, easJobContext.Container, cancellationToken))
{
using (var reader = new StreamReader(stream))
{
using (var csv = new CsvReader(reader))
{
var columns = new List <IEnumerable <dynamic> >();
csv.Configuration.HasHeaderRecord = false;
csv.Configuration.HeaderValidated = null;
csv.Configuration.IgnoreBlankLines = true;
csv.Configuration.TrimOptions = TrimOptions.Trim;
while (csv.Read())
{
if (csv.Context.Record.Length != _expectedDataColumnColumnCount)
{
throw new InvalidCsvException();
}
}
}
}
}
}
catch (InvalidCsvException ex)
{
_logger.LogInfo("Invalid csv content found.");
errorModels.Add(BuildValidationError(ErrorNameConstants.FileFormat_02));
}
catch (Exception ex)
{
_logger.LogInfo("Invalid csv content found.");
errorModels.Add(BuildValidationError(ErrorNameConstants.FileFormat_02));
}
return(errorModels);
}
private async Task <List <EasCsvRecord> > GetCsvRecords(IEasJobContext easJobContext, CancellationToken cancellationToken)
{
var records = await _easFileDataProviderService.ProvideData(easJobContext.FileReference, easJobContext.Container, cancellationToken);
return(records);
}
