public void SupportedJobTypes(FundingLineCsvGeneratorJobType jobType,
bool expectedIsSupportedFlag)
{
BatchProcessor.IsForJobType(jobType)
.Should()
.Be(expectedIsSupportedFlag);
}
public async Task <bool> GenerateCsv(FundingLineCsvGeneratorJobType jobType,
string specificationId,
string fundingPeriodId,
string temporaryFilePath,
IFundingLineCsvTransform fundingLineCsvTransform,
string fundingLineCode,
string fundingStreamId)
{
bool outputHeaders = true;
bool processedResults = false;
string predicate = _predicateBuilder.BuildPredicate(jobType);
string joinPredicate = _predicateBuilder.BuildJoinPredicate(jobType);
await _publishedFundingRepository.ExecuteAsync(() => _publishedFunding.PublishedProviderBatchProcessing(predicate,
specificationId,
publishedProviders =>
{
IEnumerable <ExpandoObject> csvRows = fundingLineCsvTransform.Transform(publishedProviders);
AppendCsvFragment(temporaryFilePath, csvRows, outputHeaders);
outputHeaders = false;
processedResults = true;
return(Task.CompletedTask);
}, BatchSize, joinPredicate, fundingLineCode)
);
return(processedResults);
}
public void ReturnsJoinPredicatesAppropriateToJobTypes(FundingLineCsvGeneratorJobType jobType,
string expectedPredicate)
{
_predicateBuilder.BuildJoinPredicate(jobType)
.Should()
.Be(expectedPredicate);
}
public void SupportsCurrentStateJobType(FundingLineCsvGeneratorJobType jobType,
bool expectedSupportsFlag)
{
_transformation.IsForJobType(jobType)
.Should()
.Be(expectedSupportsFlag);
}
public async Task <bool> GenerateCsv(
FundingLineCsvGeneratorJobType jobType,
string specificationId,
string fundingPeriodId,
string temporaryFilePath,
IFundingLineCsvTransform fundingLineCsvTransform,
string fundingLineCode,
string fundingStreamId)
{
bool outputHeaders = true;
bool processedResults = false;
await _publishedFundingPolicy.ExecuteAsync(() => _publishedFunding.PublishedFundingVersionBatchProcessing(specificationId,
fundingStreamId,
fundingPeriodId,
publishedFundingVersions =>
{
IEnumerable <ExpandoObject> csvRows = fundingLineCsvTransform.Transform(publishedFundingVersions);
AppendCsvFragment(temporaryFilePath, csvRows, outputHeaders);
outputHeaders = false;
processedResults = true;
return(Task.CompletedTask);
},
BatchSize));
return(processedResults);
}
protected async Task <bool> WhenTheCsvIsGenerated(FundingLineCsvGeneratorJobType jobType,
string specificationId,
string fundingPeriodId,
string path,
string fundingLineCode,
string fundingStreamId)
{
return(await BatchProcessor.GenerateCsv(jobType, specificationId, fundingPeriodId, path, _transformation.Object, fundingLineCode, fundingStreamId));
}
protected void AndTheJobWasCreated(string specificationId,
string correlationId,
Reference user,
FundingLineCsvGeneratorJobType jobType,
string fundingLineCode,
string fundingStreamId)
{
ThenTheJobWasCreated(specificationId, correlationId, user, jobType, fundingLineCode, fundingStreamId);
}
private void AndTheBatchProcessorProcessedResults(FundingLineCsvGeneratorJobType jobType,
string specificationId,
string fundingPeriodId,
string filePath,
string fundingLineCode,
string fundingStreamId)
{
_batchProcessor.Setup(_ => _.GenerateCsv(jobType, specificationId, fundingPeriodId, filePath, _transformation.Object, fundingLineCode, fundingStreamId))
.ReturnsAsync(true)
.Verifiable();
}
private Task <Job> CreatePublishedFundingCsvJob(
string specificationId,
string correlationId,
Reference user,
FundingLineCsvGeneratorJobType jobType,
string fundingLineCode = null,
string fundingStreamId = null,
string fundingPeriodId = null)
{
return(_createGeneratePublishedFundingCsvJobs.CreateJob(specificationId, user, correlationId, JobProperties(jobType, fundingLineCode, fundingStreamId, fundingPeriodId)));
}
public override async Task Process(Message message)
{
JobParameters parameters = message;
string specificationId = parameters.SpecificationId;
string fundingLineCode = parameters.FundingLineCode;
string fundingStreamId = parameters.FundingStreamId;
string fundingPeriodId = parameters.FundingPeriodId;
FundingLineCsvGeneratorJobType jobType = parameters.JobType;
CsvFileInfo fileInfo = new CsvFileInfo(_fileSystemCacheSettings.Path,
jobType,
specificationId,
fundingLineCode,
fundingStreamId,
fundingPeriodId);
string temporaryPath = fileInfo.TemporaryPath;
EnsureFileIsNew(temporaryPath);
IFundingLineCsvTransform fundingLineCsvTransform = _transformServiceLocator.GetService(jobType);
IFundingLineCsvBatchProcessor fundingLineCsvBatchProcessor = _batchProcessorServiceLocator.GetService(jobType);
fundingLineCsvTransform.FundingLineCode = fundingLineCode;
bool processedResults = await fundingLineCsvBatchProcessor.GenerateCsv(jobType,
specificationId,
fundingPeriodId,
temporaryPath,
fundingLineCsvTransform,
fundingLineCode,
fundingStreamId);
if (!processedResults)
{
_logger.Information(
$"Did not create a new csv report as no providers matched for the job type {jobType} in the specification {specificationId}" +
$" and funding line code {fundingLineCode} and funding stream id {fundingStreamId}");
return;
}
ICloudBlob blob = _blobClient.GetBlockBlobReference(fileInfo.FileName, PublishedFundingReportContainerName);
blob.Properties.ContentDisposition = fileInfo.ContentDisposition;
await using (Stream csvFileStream = _fileSystemAccess.OpenRead(temporaryPath))
{
await _blobClientPolicy.ExecuteAsync(() => UploadBlob(blob, csvFileStream, parameters.ToDictionary()));
}
}
private Dictionary <string, string> JobProperties(FundingLineCsvGeneratorJobType jobType,
string fundingLineCode,
string fundingStreamId,
string fundingPeriodId)
{
return(new Dictionary <string, string>
{
{ "job-type", jobType.ToString() },
{ "funding-line-code", fundingLineCode },
{ "funding-stream-id", fundingStreamId },
{ "funding-period-id", fundingPeriodId },
});
}
public CsvFileInfo(string root,
FundingLineCsvGeneratorJobType jobType,
string specificationId,
string fundingLineCode,
string fundingStreamId,
string fundingPeriodId)
{
ContentDisposition = $"attachment; filename={GetPrettyFileName(jobType, fundingLineCode, fundingStreamId, fundingPeriodId)}";
fundingLineCode = WithPrefixDelimiterOrEmpty(fundingLineCode);
fundingStreamId = WithPrefixDelimiterOrEmpty(fundingStreamId);
FileName = $"funding-lines-{specificationId}-{jobType}{fundingLineCode}{fundingStreamId}.csv";
TemporaryPath = Path.Combine(root, FileName);
}
private static string GetPrettyFileName(FundingLineCsvGeneratorJobType jobType,
string fundingLineCode,
string fundingStreamId,
string fundingPeriodId)
{
string utcNow = DateTimeOffset.UtcNow.ToString("s").Replace(":", null);
return(jobType switch
{
FundingLineCsvGeneratorJobType.CurrentState => $"{fundingStreamId} {fundingPeriodId} Provider Funding Lines Current State {utcNow}.csv",
FundingLineCsvGeneratorJobType.Released => $"{fundingStreamId} {fundingPeriodId} Provider Funding Lines Released Only {utcNow}.csv",
FundingLineCsvGeneratorJobType.History => $"{fundingStreamId} {fundingPeriodId} Provider Funding Lines All Versions {utcNow}.csv",
FundingLineCsvGeneratorJobType.HistoryProfileValues => $"{fundingStreamId} {fundingPeriodId} {fundingLineCode} Profile All Versions {utcNow}.csv",
FundingLineCsvGeneratorJobType.CurrentProfileValues => $"{fundingStreamId} {fundingPeriodId} {fundingLineCode} Profile Current State {utcNow}.csv",
FundingLineCsvGeneratorJobType.CurrentOrganisationGroupValues => $"{fundingStreamId} {fundingPeriodId} Funding Lines Current State {utcNow}.csv",
FundingLineCsvGeneratorJobType.HistoryOrganisationGroupValues => $"{fundingStreamId} {fundingPeriodId} Funding Lines All Versions {utcNow}.csv",
FundingLineCsvGeneratorJobType.PublishedGroups => $"{fundingStreamId} {fundingPeriodId} Published Groups {utcNow}.csv",
_ => throw new ArgumentOutOfRangeException()
});
private void ThenTheJobWasCreated(string specificationId,
string correlationId,
Reference user,
FundingLineCsvGeneratorJobType jobType,
string fundingLineCode,
string fundingStreamId)
{
CreateGeneratePublishedFundingCsvJobs.Verify(_ => _.CreateJob(specificationId, user, correlationId,
It.Is <Dictionary <string, string> >(props
=> props.ContainsKey(JobType) &&
props[JobType] == jobType.ToString() &&
props.ContainsKey(FundingLineCode) &&
props[FundingLineCode] == fundingLineCode &&
props.ContainsKey(FundingStreamId) &&
props[FundingStreamId] == fundingStreamId),
null,
null,
false),
Times.Once);
}
public async Task <bool> GenerateCsv(FundingLineCsvGeneratorJobType jobType,
string specificationId,
string fundingPeriodId,
string temporaryFilePath,
IFundingLineCsvTransform fundingLineCsvTransform,
string fundingLineCode,
string fundingStreamId)
{
bool outputHeaders = true;
bool processedResults = false;
await _publishedFundingPolicy.ExecuteAsync(() => _publishedFundingRepository.PublishedGroupBatchProcessing(
specificationId,
async publishedFundings =>
{
List <PublishedFundingWithProvider> publishedfundingsWithProviders = new List <PublishedFundingWithProvider>();
foreach (var publishedFunding in publishedFundings)
{
IEnumerable <PublishedProvider> providers = Enumerable.Empty <PublishedProvider>();
if (publishedFunding.Current.ProviderFundings.Any())
{
providers = await _publishedFundingRepository.QueryPublishedProvider(specificationId, publishedFunding.Current.ProviderFundings);
}
publishedfundingsWithProviders.Add(new PublishedFundingWithProvider {
PublishedFunding = publishedFunding, PublishedProviders = providers
});
}
IEnumerable <ExpandoObject> csvRows = fundingLineCsvTransform.Transform(publishedfundingsWithProviders);
AppendCsvFragment(temporaryFilePath, csvRows, outputHeaders);
outputHeaders = false;
processedResults = true;
}, BatchSize)
);
return(processedResults);
}
public async Task ExitsEarlyIfNoProvidersMatchForTheJobTypePredicate()
{
string specificationId = NewRandomString();
string fundingLineCode = NewRandomString();
string jobId = NewRandomString();
string expectedInterimFilePath = Path.Combine(_rootPath, $"funding-lines-{specificationId}-Released-{fundingLineCode}.csv");
FundingLineCsvGeneratorJobType jobType = FundingLineCsvGeneratorJobType.Released;
GivenTheMessageProperties(("specification-id", specificationId), ("job-type", jobType.ToString()), ("jobId", jobId), ("funding-line-code", fundingLineCode));
AndTheFileExists(expectedInterimFilePath);
AndTheJobExists(jobId);
AndTheTransformForJobType(jobType);
AndTheBatchProcessorForJobType(jobType);
await WhenTheCsvIsGenerated();
_jobManagement.Verify(_ => _.UpdateJobStatus(jobId, 0, 0, null, null),
Times.Once);
_fileSystemAccess
.Verify(_ => _.Delete(expectedInterimFilePath),
Times.Once);
_fileSystemAccess
.Verify(_ => _.Append(expectedInterimFilePath,
It.IsAny <string>(),
default),
Times.Never);
_blobClient
.Verify(_ => _.UploadFileAsync(_cloudBlob.Object, It.IsAny <Stream>()),
Times.Never);
_jobManagement.Verify(_ => _.UpdateJobStatus(jobId, 0, 0, true, null),
Times.Once);
}
public IFundingLineCsvTransform GetService(FundingLineCsvGeneratorJobType jobType)
{
return(_transforms.SingleOrDefault(_ => _.IsForJobType(jobType)) ?? throw new ArgumentOutOfRangeException());
}
private void AndTheBatchProcessorForJobType(FundingLineCsvGeneratorJobType jobType)
{
_batchProcessorServiceLocator.Setup(_ => _.GetService(jobType))
.Returns(_batchProcessor.Object);
}
private void AndTheTransformForJobType(FundingLineCsvGeneratorJobType jobType)
{
_transformServiceLocator.Setup(_ => _.GetService(jobType))
.Returns(_transformation.Object);
}
public async Task TransformsPublishedProvidersForSpecificationInBatchesAndCreatesCsvWithResults(
FundingLineCsvGeneratorJobType jobType)
{
string specificationId = NewRandomString();
string fundingPeriodId = NewRandomString();
string fundingLineCode = NewRandomString();
string fundingStreamId = NewRandomString();
string expectedInterimFilePath = NewRandomString();
IEnumerable <PublishedProvider> publishProvidersOne = new []
{
NewPublishedProvider(),
};
IEnumerable <PublishedProvider> publishedProvidersTwo = new []
{
NewPublishedProvider(),
NewPublishedProvider(),
NewPublishedProvider()
};
ExpandoObject[] transformedRowsOne =
{
new ExpandoObject(),
new ExpandoObject(),
new ExpandoObject(),
new ExpandoObject(),
};
ExpandoObject[] transformedRowsTwo =
{
new ExpandoObject(),
new ExpandoObject(),
new ExpandoObject(),
new ExpandoObject(),
};
string expectedCsvOne = NewRandomString();
string expectedCsvTwo = NewRandomString();
string predicate = NewRandomString();
string joinPredicate = NewRandomString();
GivenTheCsvRowTransformation(publishProvidersOne, transformedRowsOne, expectedCsvOne, true);
AndTheCsvRowTransformation(publishedProvidersTwo, transformedRowsTwo, expectedCsvTwo, false);
AndThePredicate(jobType, predicate);
AndTheJoinPredicate(jobType, joinPredicate);
PublishedFunding.Setup(_ => _.PublishedProviderBatchProcessing(predicate,
specificationId,
It.IsAny <Func <List <PublishedProvider>, Task> >(),
100,
joinPredicate,
fundingLineCode))
.Callback <string, string, Func <List <PublishedProvider>, Task>, int, string, string>((pred, spec,
batchProcessor, batchSize, joinPred, flc) =>
{
batchProcessor(publishProvidersOne.ToList())
.GetAwaiter()
.GetResult();
batchProcessor(publishedProvidersTwo.ToList())
.GetAwaiter()
.GetResult();
})
.Returns(Task.CompletedTask);
bool processedResults = await WhenTheCsvIsGenerated(jobType, specificationId, fundingPeriodId, expectedInterimFilePath, fundingLineCode, fundingStreamId);
processedResults
.Should()
.BeTrue();
FileSystemAccess
.Verify(_ => _.Append(expectedInterimFilePath,
expectedCsvOne,
default),
Times.Once);
FileSystemAccess
.Verify(_ => _.Append(expectedInterimFilePath,
expectedCsvTwo,
default),
Times.Once);
}
public bool IsForJobType(FundingLineCsvGeneratorJobType jobType)
{
return(jobType == FundingLineCsvGeneratorJobType.Released ||
jobType == FundingLineCsvGeneratorJobType.CurrentState ||
jobType == FundingLineCsvGeneratorJobType.CurrentProfileValues);
}
private void GivenTheTransformSupportsTheJobType(FundingLineCsvGeneratorJobType jobType,
int transformIndex)
{
_batchProcessors[transformIndex].Setup(_ => _.IsForJobType(jobType))
.Returns(true);
}
public string BuildJoinPredicate(FundingLineCsvGeneratorJobType jobType)
{
return(Joins.TryGetValue(jobType, out string @join) ? @join : string.Empty);
}
public string BuildPredicate(FundingLineCsvGeneratorJobType jobType)
{
return(Predicates.TryGetValue(jobType, out string predicate) ? predicate : throw new ArgumentOutOfRangeException());
}
public async Task TransformsPublishedProvidersForSpecificationInBatchesAndCreatesCsvWithResultsOrFailJob(
FundingLineCsvGeneratorJobType jobType,
string specificationId,
string fundingLineCode,
string fundingStreamId,
string fundingPeriodId,
string expectedFileName,
string expectedContentDisposition,
bool throwException)
{
string jobId = NewRandomString();
string expectedInterimFilePath = Path.Combine(_rootPath, expectedFileName);
MemoryStream incrementalFileStream = new MemoryStream();
string predicate = NewRandomString();
GivenTheMessageProperties(("specification-id", specificationId),
("job-type", jobType.ToString()),
("jobId", jobId),
("funding-line-code", fundingLineCode),
("funding-period-id", fundingPeriodId),
("funding-stream-id", fundingStreamId));
AndTheCloudBlobForFileName(expectedFileName);
AndTheFileStream(expectedInterimFilePath, incrementalFileStream);
AndTheFileExists(expectedInterimFilePath);
AndTheTransformForJobType(jobType);
AndThePredicate(jobType, predicate);
AndTheJobExists(jobId);
AndTheBatchProcessorForJobType(jobType);
AndTheBatchProcessorProcessedResults(jobType, specificationId, fundingPeriodId, expectedInterimFilePath, fundingLineCode, fundingStreamId);
string errorMessage = "Unable to complete funding line csv generation job.";
if (throwException)
{
AndTheBlobThrowsError(incrementalFileStream, new NonRetriableException(errorMessage));
Func <Task> test = async() => await WhenTheCsvIsGenerated();
test
.Should()
.ThrowExactly <NonRetriableException>()
.Which
.Message
.Should()
.Be(errorMessage);
}
else
{
await WhenTheCsvIsGenerated();
}
_jobManagement.Verify(_ => _.UpdateJobStatus(jobId, 0, 0, null, null),
Times.Once);
_fileSystemAccess
.Verify(_ => _.Delete(expectedInterimFilePath),
Times.Once);
_blobProperties?.ContentDisposition
.Should()
.StartWith($"attachment; filename={expectedContentDisposition} {DateTimeOffset.UtcNow:yyyy-MM-dd}")
.And
.NotContain(":");
_blobClient
.Verify(_ => _.UploadFileAsync(_cloudBlob.Object, incrementalFileStream),
Times.Once);
if (throwException)
{
_jobManagement.Verify(_ => _.UpdateJobStatus(jobId, 0, 0, false, errorMessage),
Times.Once);
}
else
{
_jobManagement.Verify(_ => _.UpdateJobStatus(jobId, 0, 0, true, null),
Times.Once);
}
}
public override bool IsForJobType(FundingLineCsvGeneratorJobType jobType)
{
return(jobType == FundingLineCsvGeneratorJobType.HistoryProfileValues);
}
public override bool IsForJobType(FundingLineCsvGeneratorJobType jobType) => jobType == FundingLineCsvGeneratorJobType.HistoryOrganisationGroupValues;
private void AndThePredicate(FundingLineCsvGeneratorJobType jobType, string predicate)
{
_predicateBuilder.Setup(_ => _.BuildPredicate(jobType))
.Returns(predicate);
}
public async Task TransformsPublishedGroupsForSpecificationInBatchesAndCreatesCsvWithResults(
FundingLineCsvGeneratorJobType jobType)
{
string specificationId = NewRandomString();
string fundingPeriodId = NewRandomString();
string fundingLineCode = NewRandomString();
string fundingStreamId = NewRandomString();
string expectedInterimFilePath = NewRandomString();
IEnumerable <PublishedFunding> publishedFundingOne = new List <PublishedFunding>
{
NewPublishedFunding(_ => _.WithCurrent(
NewPublishedFundingVersion(ppv => ppv.WithFundingId("f-id")
.WithProviderFundings(new List <string>())))),
};
IEnumerable <PublishedFunding> publishedFundingWithProviderTwo = new[]
{
NewPublishedFunding(_ => _.WithCurrent(
NewPublishedFundingVersion(ppv => ppv.WithFundingId("f-id1")))),
NewPublishedFunding(_ => _.WithCurrent(
NewPublishedFundingVersion(ppv => ppv.WithFundingId("f-id2")))),
NewPublishedFunding(_ => _.WithCurrent(
NewPublishedFundingVersion(ppv => ppv.WithFundingId("f-id3"))))
};
ExpandoObject[] transformedRowsOne =
{
new ExpandoObject(),
new ExpandoObject(),
new ExpandoObject(),
new ExpandoObject(),
};
ExpandoObject[] transformedRowsTwo =
{
new ExpandoObject(),
new ExpandoObject(),
new ExpandoObject(),
new ExpandoObject(),
};
string expectedCsvOne = NewRandomString();
string expectedCsvTwo = NewRandomString();
string predicate = NewRandomString();
string joinPredicate = NewRandomString();
GivenTheCsvRowTransformation(publishedFundingOne, transformedRowsOne, expectedCsvOne, true);
AndTheCsvRowTransformation(publishedFundingWithProviderTwo, transformedRowsTwo, expectedCsvTwo, false);
AndThePredicate(jobType, predicate);
AndTheJoinPredicate(jobType, joinPredicate);
PublishedFunding.Setup(_ => _.PublishedGroupBatchProcessing(specificationId,
It.IsAny <Func <List <PublishedFunding>, Task> >(),
100))
.Callback <string, Func <List <PublishedFunding>, Task>, int>((spec,
batchProcessor, batchSize) =>
{
batchProcessor(publishedFundingOne.ToList())
.GetAwaiter()
.GetResult();
})
.Returns(Task.CompletedTask);
bool processedResults = await WhenTheCsvIsGenerated(jobType, specificationId, fundingPeriodId, expectedInterimFilePath, fundingLineCode, fundingStreamId);
processedResults
.Should()
.BeTrue();
}
protected void AndTheJoinPredicate(FundingLineCsvGeneratorJobType jobType, string predicate)
{
PredicateBuilder.Setup(_ => _.BuildJoinPredicate(jobType))
.Returns(predicate);
}
