public async Task PreviewCalculationResult_GivenCachedAggregateValuesExist_CalculateProviderResultsCallReceived()
{
IAllocationModel allocationModel = Substitute.For <IAllocationModel>();
_calculationEngine
.GenerateAllocationModel(Arg.Any <Assembly>())
.Returns(allocationModel);
ProviderVersionSearchResult providerVersionSearchResult = new ProviderVersionSearchResult
{
UKPRN = providerId
};
IEnumerable <string> dataDefinitionRelationshipIds = new List <string>();
SpecificationSummary specificationSummary = new SpecificationSummary
{
DataDefinitionRelationshipIds = dataDefinitionRelationshipIds,
ProviderVersionId = providerVersionId
};
_specificationsApiClient
.GetSpecificationSummaryById(Arg.Is(specificationId))
.Returns(new ApiResponse <SpecificationSummary>(HttpStatusCode.OK, specificationSummary));
_providersApiClient
.GetProviderByIdFromProviderVersion(Arg.Is(providerVersionId), Arg.Is(providerId))
.Returns(new ApiResponse <ProviderVersionSearchResult>(HttpStatusCode.OK, providerVersionSearchResult));
CalculationSummaryModel previewCalculationSummaryModel = new CalculationSummaryModel();
IEnumerable <CalculationSummaryModel> calculationSummaryModels = new List <CalculationSummaryModel>
{
new CalculationSummaryModel(),
new CalculationSummaryModel()
};
List <CalculationSummaryModel> expectedCalculationSummaryModels = calculationSummaryModels.ToList();
expectedCalculationSummaryModels.Add(previewCalculationSummaryModel);
_calculationsRepository
.GetCalculationSummariesForSpecification(Arg.Is(specificationId))
.Returns(calculationSummaryModels);
Dictionary <string, ProviderSourceDataset> sourceDatasets = new Dictionary <string, ProviderSourceDataset>();
Dictionary <string, Dictionary <string, ProviderSourceDataset> > providerSourceDatasets = new Dictionary <string, Dictionary <string, ProviderSourceDataset> >
{
{ providerId, sourceDatasets }
};
_providerSourceDatasetsRepository
.GetProviderSourceDatasetsByProviderIdsAndRelationshipIds(
Arg.Is(specificationId),
Arg.Is <IEnumerable <string> >(_ => _ != null && _.Count() == 1 && _.FirstOrDefault() == providerId),
Arg.Is <IEnumerable <string> >(_ => _ != null && _.SequenceEqual(dataDefinitionRelationshipIds)))
.Returns(providerSourceDatasets);
IEnumerable <CalculationAggregation> calculationAggregations = new List <CalculationAggregation>();
_calculationAggregationService
.BuildAggregations(Arg.Is <BuildAggregationRequest>(_ => _ != null && _.SpecificationId == specificationId))
.Returns(calculationAggregations);
PreviewCalculationRequest previewCalculationRequest = new PreviewCalculationRequest
{
AssemblyContent = MockData.GetMockAssembly(),
PreviewCalculationSummaryModel = previewCalculationSummaryModel
};
IActionResult actionResult =
await _calculationEnginePreviewService.PreviewCalculationResult(specificationId, providerId, previewCalculationRequest);
_calculationEngine
.Received(1)
.CalculateProviderResults(
Arg.Is(allocationModel),
specificationId,
Arg.Is <IEnumerable <CalculationSummaryModel> >(_ => _.SequenceEqual(expectedCalculationSummaryModels)),
Arg.Is <ProviderSummary>(_ => _.UKPRN == providerId),
Arg.Is <Dictionary <string, ProviderSourceDataset> >(_ => _.SequenceEqual(sourceDatasets)),
Arg.Is <IEnumerable <CalculationAggregation> >(_ => _.SequenceEqual(calculationAggregations)));
}
public async Task <IActionResult> PreviewCalculationResult(
string specificationId,
string providerId,
PreviewCalculationRequest previewCalculationRequest)
{
Guard.IsNullOrWhiteSpace(specificationId, nameof(specificationId));
Guard.IsNullOrWhiteSpace(providerId, nameof(providerId));
Guard.ArgumentNotNull(previewCalculationRequest, nameof(previewCalculationRequest));
Assembly assembly = Assembly.Load(previewCalculationRequest.AssemblyContent);
IAllocationModel allocationModel = _calculationEngine.GenerateAllocationModel(assembly);
SpecificationSummary specificationSummary = await GetSpecificationSummary(specificationId);
ApiResponse <ProviderVersionSearchResult> providerVersionSearchResultApiResponse =
await _providersApiClientPolicy.ExecuteAsync(() => _providersApiClient.GetProviderByIdFromProviderVersion(
specificationSummary.ProviderVersionId,
providerId));
ProviderVersionSearchResult providerVersionSearchResult = providerVersionSearchResultApiResponse.Content;
if (providerVersionSearchResult == null)
{
return(new NotFoundResult());
}
ProviderSummary providerSummary = _mapper.Map <ProviderSummary>(providerVersionSearchResult);
List <CalculationSummaryModel> calculationSummaries = new List <CalculationSummaryModel>();
IEnumerable <CalculationSummaryModel> specCalculationSummaries = await GetCalculationSummaries(specificationId);
calculationSummaries.AddRange(specCalculationSummaries);
calculationSummaries.Add(previewCalculationRequest.PreviewCalculationSummaryModel);
Dictionary <string, Dictionary <string, ProviderSourceDataset> > providerSourceDatasets =
await _providerSourceDatasetsRepository.GetProviderSourceDatasetsByProviderIdsAndRelationshipIds(
specificationId,
new[] { providerId },
specificationSummary.DataDefinitionRelationshipIds);
Dictionary <string, ProviderSourceDataset> providerSourceDataset = providerSourceDatasets[providerId];
BuildAggregationRequest buildAggregationRequest = new BuildAggregationRequest
{
SpecificationId = specificationId,
GenerateCalculationAggregationsOnly = true,
BatchCount = 100
};
IEnumerable <CalculationAggregation> calculationAggregations =
await _calculationAggregationService.BuildAggregations(buildAggregationRequest);
ProviderResult providerResult = _calculationEngine.CalculateProviderResults(
allocationModel,
specificationId,
calculationSummaries,
providerSummary,
providerSourceDataset,
calculationAggregations
);
return(new OkObjectResult(providerResult));
}
private async Task <CalculationResultsModel> CalculateResults(string specificationId, IEnumerable <ProviderSummary> summaries,
IEnumerable <CalculationSummaryModel> calculations,
IEnumerable <CalculationAggregation> aggregations,
IEnumerable <string> dataRelationshipIds,
byte[] assemblyForSpecification,
GenerateAllocationMessageProperties messageProperties,
int providerBatchSize,
int index)
{
ConcurrentBag <ProviderResult> providerResults = new ConcurrentBag <ProviderResult>();
Guard.ArgumentNotNull(summaries, nameof(summaries));
IEnumerable <ProviderSummary> partitionedSummaries = summaries.Skip(index).Take(providerBatchSize);
IList <string> providerIdList = partitionedSummaries.Select(m => m.Id).ToList();
Stopwatch providerSourceDatasetsStopwatch = Stopwatch.StartNew();
_logger.Information($"Fetching provider sources for specification id {messageProperties.SpecificationId}");
Dictionary <string, Dictionary <string, ProviderSourceDataset> > providerSourceDatasetResult = await _providerSourceDatasetsRepositoryPolicy.ExecuteAsync(
() => _providerSourceDatasetsRepository.GetProviderSourceDatasetsByProviderIdsAndRelationshipIds(specificationId, providerIdList, dataRelationshipIds));
providerSourceDatasetsStopwatch.Stop();
_logger.Information($"Fetched provider sources found for specification id {messageProperties.SpecificationId}");
_logger.Information($"Calculating results for specification id {messageProperties.SpecificationId}");
Stopwatch assemblyLoadStopwatch = Stopwatch.StartNew();
Assembly assembly = Assembly.Load(assemblyForSpecification);
assemblyLoadStopwatch.Stop();
Stopwatch calculationStopwatch = Stopwatch.StartNew();
List <Task> allTasks = new List <Task>();
SemaphoreSlim throttler = new SemaphoreSlim(_engineSettings.CalculateProviderResultsDegreeOfParallelism);
IAllocationModel allocationModel = _calculationEngine.GenerateAllocationModel(assembly);
foreach (ProviderSummary provider in partitionedSummaries)
{
await throttler.WaitAsync();
allTasks.Add(
Task.Run(() =>
{
try
{
if (provider == null)
{
throw new Exception("Provider summary is null");
}
if (providerSourceDatasetResult.AnyWithNullCheck())
{
if (!providerSourceDatasetResult.TryGetValue(provider.Id, out Dictionary <string, ProviderSourceDataset> providerDatasets))
{
throw new Exception($"Provider source dataset not found for {provider.Id}.");
}
ProviderResult result = _calculationEngine.CalculateProviderResults(allocationModel, specificationId, calculations, provider, providerDatasets, aggregations);
if (result == null)
{
throw new InvalidOperationException("Null result from Calc Engine CalculateProviderResults");
}
providerResults.Add(result);
}
}
finally
{
throttler.Release();
}
return(Task.CompletedTask);
}));
}
await TaskHelper.WhenAllAndThrow(allTasks.ToArray());
calculationStopwatch.Stop();
_logger.Information($"Calculating results complete for specification id {messageProperties.SpecificationId} in {calculationStopwatch.ElapsedMilliseconds}ms");
return(new CalculationResultsModel
{
ProviderResults = providerResults,
PartitionedSummaries = partitionedSummaries,
CalculationRunMs = calculationStopwatch.ElapsedMilliseconds,
AssemblyLoadMs = assemblyLoadStopwatch.ElapsedMilliseconds,
ProviderSourceDatasetsLookupMs = providerSourceDatasetsStopwatch.ElapsedMilliseconds,
});
}
private async Task <CalculationResultsModel> CalculateResults(IEnumerable <ProviderSummary> summaries, IEnumerable <CalculationSummaryModel> calculations, IEnumerable <CalculationAggregation> aggregations, BuildProject buildProject,
GenerateAllocationMessageProperties messageProperties, int providerBatchSize, int index, Stopwatch providerSourceDatasetsStopwatch, Stopwatch calculationStopwatch)
{
ConcurrentBag <ProviderResult> providerResults = new ConcurrentBag <ProviderResult>();
IEnumerable <ProviderSummary> partitionedSummaries = summaries.Skip(index).Take(providerBatchSize);
IList <string> providerIdList = partitionedSummaries.Select(m => m.Id).ToList();
providerSourceDatasetsStopwatch.Start();
_logger.Information($"Fetching provider sources for specification id {messageProperties.SpecificationId}");
List <ProviderSourceDataset> providerSourceDatasets = new List <ProviderSourceDataset>(await _providerSourceDatasetsRepositoryPolicy.ExecuteAsync(() => _providerSourceDatasetsRepository.GetProviderSourceDatasetsByProviderIdsAndSpecificationId(providerIdList, messageProperties.SpecificationId)));
providerSourceDatasetsStopwatch.Stop();
if (providerSourceDatasets == null)
{
_logger.Information($"No provider sources found for specification id {messageProperties.SpecificationId}");
providerSourceDatasets = new List <ProviderSourceDataset>();
}
_logger.Information($"fetched provider sources found for specification id {messageProperties.SpecificationId}");
calculationStopwatch.Start();
_logger.Information($"calculating results for specification id {messageProperties.SpecificationId}");
Assembly assembly = Assembly.Load(buildProject.Build.Assembly);
Parallel.ForEach(partitionedSummaries, new ParallelOptions {
MaxDegreeOfParallelism = _engineSettings.CalculateProviderResultsDegreeOfParallelism
}, provider =>
{
IAllocationModel allocationModel = _calculationEngine.GenerateAllocationModel(assembly);
IEnumerable <ProviderSourceDataset> providerDatasets = providerSourceDatasets.Where(m => m.ProviderId == provider.Id);
ProviderResult result = _calculationEngine.CalculateProviderResults(allocationModel, buildProject, calculations, provider, providerDatasets, aggregations);
if (result != null)
{
providerResults.Add(result);
}
else
{
throw new InvalidOperationException("Null result from Calc Engine CalculateProviderResults");
}
});
_logger.Information($"calculating results complete for specification id {messageProperties.SpecificationId}");
calculationStopwatch.Stop();
return(new CalculationResultsModel
{
ProviderResults = providerResults,
PartitionedSummaries = partitionedSummaries
});
}