async public Task <IEnumerable <ProviderResult> > GenerateAllocations(BuildProject buildProject, IEnumerable <ProviderSummary> providers, Func <string, string, Task <IEnumerable <ProviderSourceDataset> > > getProviderSourceDatasets)
{
var assembly = Assembly.Load(buildProject.Build.Assembly);
var allocationModel = _allocationFactory.CreateAllocationModel(assembly);
ConcurrentBag <ProviderResult> providerResults = new ConcurrentBag <ProviderResult>();
IEnumerable <CalculationSummaryModel> calculations = await _calculationsRepository.GetCalculationSummariesForSpecification(buildProject.SpecificationId);
Parallel.ForEach(providers, new ParallelOptions {
MaxDegreeOfParallelism = 5
}, provider =>
{
var stopwatch = new Stopwatch();
stopwatch.Start();
IEnumerable <ProviderSourceDataset> providerSourceDatasets = getProviderSourceDatasets(provider.Id, buildProject.SpecificationId).Result;
if (providerSourceDatasets == null)
{
providerSourceDatasets = Enumerable.Empty <ProviderSourceDataset>();
}
var result = CalculateProviderResults(allocationModel, buildProject, calculations, provider, providerSourceDatasets.ToList());
providerResults.Add(result);
stopwatch.Stop();
_logger.Debug($"Generated result for {provider.Name} in {stopwatch.ElapsedMilliseconds}ms");
});
return(providerResults);
}
async public Task GenerateAllocations_GivenBuildProject_Runs()
{
//Arrange
BuildProject buildProject = CreateBuildProject();
buildProject.Build.Assembly = MockData.GetMockAssembly();
IEnumerable <ProviderSummary> providers = new[]
{
new ProviderSummary {
Id = ProviderId, Name = ProviderName
}
};
IEnumerable <ProviderSourceDataset> datasets = new[]
{
new ProviderSourceDataset()
};
Func <string, string, Task <IEnumerable <ProviderSourceDataset> > > func = (s, p) =>
{
return(Task.FromResult(datasets));
};
IEnumerable <CalculationResult> calculationResults = new[]
{
new CalculationResult
{
Calculation = new Reference {
Id = CalculationId
},
}
};
IAllocationModel allocationModel = Substitute.For <IAllocationModel>();
allocationModel
.Execute(Arg.Any <List <ProviderSourceDataset> >(), Arg.Any <ProviderSummary>())
.Returns(calculationResults);
IAllocationFactory allocationFactory = Substitute.For <IAllocationFactory>();
allocationFactory
.CreateAllocationModel(Arg.Any <Assembly>())
.Returns(allocationModel);
ILogger logger = CreateLogger();
ICalculationsRepository calculationsRepository = CreateCalculationsRepository();
List <CalculationSummaryModel> calculations = new List <CalculationSummaryModel>()
{
new CalculationSummaryModel()
{
Id = CalculationId,
},
new CalculationSummaryModel()
{
Id = "calc2",
},
new CalculationSummaryModel()
{
Id = "calc3",
}
};
calculationsRepository
.GetCalculationSummariesForSpecification(Arg.Any <string>())
.Returns(calculations);
CalculationEngine calculationEngine = CreateCalculationEngine(allocationFactory, calculationsRepository, logger: logger);
//Act
IEnumerable <ProviderResult> results = await calculationEngine.GenerateAllocations(buildProject, providers, func);
//Assert
results
.Count()
.Should()
.Be(1);
results
.First()
.CalculationResults
.Count
.Should()
.Be(3);
}
public async Task PreviewCalculationResult_GivenCachedAggregateValuesExist_CalculateProviderResultsCallReceived()
{
IAllocationModel allocationModel = Substitute.For <IAllocationModel>();
_calculationEngine
.GenerateAllocationModel(Arg.Any <Assembly>())
.Returns(allocationModel);
ProviderVersionSearchResult providerVersionSearchResult = new ProviderVersionSearchResult
{
UKPRN = providerId
};
IEnumerable <string> dataDefinitionRelationshipIds = new List <string>();
SpecificationSummary specificationSummary = new SpecificationSummary
{
DataDefinitionRelationshipIds = dataDefinitionRelationshipIds,
ProviderVersionId = providerVersionId
};
_specificationsApiClient
.GetSpecificationSummaryById(Arg.Is(specificationId))
.Returns(new ApiResponse <SpecificationSummary>(HttpStatusCode.OK, specificationSummary));
_providersApiClient
.GetProviderByIdFromProviderVersion(Arg.Is(providerVersionId), Arg.Is(providerId))
.Returns(new ApiResponse <ProviderVersionSearchResult>(HttpStatusCode.OK, providerVersionSearchResult));
CalculationSummaryModel previewCalculationSummaryModel = new CalculationSummaryModel();
IEnumerable <CalculationSummaryModel> calculationSummaryModels = new List <CalculationSummaryModel>
{
new CalculationSummaryModel(),
new CalculationSummaryModel()
};
List <CalculationSummaryModel> expectedCalculationSummaryModels = calculationSummaryModels.ToList();
expectedCalculationSummaryModels.Add(previewCalculationSummaryModel);
_calculationsRepository
.GetCalculationSummariesForSpecification(Arg.Is(specificationId))
.Returns(calculationSummaryModels);
Dictionary <string, ProviderSourceDataset> sourceDatasets = new Dictionary <string, ProviderSourceDataset>();
Dictionary <string, Dictionary <string, ProviderSourceDataset> > providerSourceDatasets = new Dictionary <string, Dictionary <string, ProviderSourceDataset> >
{
{ providerId, sourceDatasets }
};
_providerSourceDatasetsRepository
.GetProviderSourceDatasetsByProviderIdsAndRelationshipIds(
Arg.Is(specificationId),
Arg.Is <IEnumerable <string> >(_ => _ != null && _.Count() == 1 && _.FirstOrDefault() == providerId),
Arg.Is <IEnumerable <string> >(_ => _ != null && _.SequenceEqual(dataDefinitionRelationshipIds)))
.Returns(providerSourceDatasets);
IEnumerable <CalculationAggregation> calculationAggregations = new List <CalculationAggregation>();
_calculationAggregationService
.BuildAggregations(Arg.Is <BuildAggregationRequest>(_ => _ != null && _.SpecificationId == specificationId))
.Returns(calculationAggregations);
PreviewCalculationRequest previewCalculationRequest = new PreviewCalculationRequest
{
AssemblyContent = MockData.GetMockAssembly(),
PreviewCalculationSummaryModel = previewCalculationSummaryModel
};
IActionResult actionResult =
await _calculationEnginePreviewService.PreviewCalculationResult(specificationId, providerId, previewCalculationRequest);
_calculationEngine
.Received(1)
.CalculateProviderResults(
Arg.Is(allocationModel),
specificationId,
Arg.Is <IEnumerable <CalculationSummaryModel> >(_ => _.SequenceEqual(expectedCalculationSummaryModels)),
Arg.Is <ProviderSummary>(_ => _.UKPRN == providerId),
Arg.Is <Dictionary <string, ProviderSourceDataset> >(_ => _.SequenceEqual(sourceDatasets)),
Arg.Is <IEnumerable <CalculationAggregation> >(_ => _.SequenceEqual(calculationAggregations)));
}
private async Task <(IEnumerable <CalculationSummaryModel>, long)> GetCalculationSummaries(string specificationId)
{
Stopwatch calculationsLookupStopwatch = Stopwatch.StartNew();
IEnumerable <CalculationSummaryModel> calculations = await _calculationsApiClientPolicy.ExecuteAsync(() =>
_calculationsRepository.GetCalculationSummariesForSpecification(specificationId));
if (calculations == null)
{
_logger.Error($"Calculations lookup API returned null for specification id {specificationId}");
throw new InvalidOperationException("Calculations lookup API returned null");
}
calculationsLookupStopwatch.Stop();
return(calculations, calculationsLookupStopwatch.ElapsedMilliseconds);
}
private async Task <IEnumerable <CalculationSummaryModel> > GetCalculationSummaries(string specificationId)
{
IEnumerable <CalculationSummaryModel> calculations = await _calculationsApiClientPolicy.ExecuteAsync(() =>
_calculationsRepository.GetCalculationSummariesForSpecification(specificationId));
if (calculations == null)
{
_logger.Error($"Calculations lookup API returned null for specification id {specificationId}");
throw new InvalidOperationException("Calculations lookup API returned null");
}
return(calculations);
}
public async Task GenerateAllocations(Message message)
{
Guard.ArgumentNotNull(message, nameof(message));
_logger.Information($"Validating new allocations message");
CalculationEngineServiceValidator.ValidateMessage(_logger, message);
GenerateAllocationMessageProperties messageProperties = GetMessageProperties(message);
JobViewModel job = await AddStartingProcessJobLog(messageProperties.JobId);
if (job == null)
{
return;
}
messageProperties.GenerateCalculationAggregationsOnly = (job.JobDefinitionId == JobConstants.DefinitionNames.GenerateCalculationAggregationsJob);
IEnumerable <ProviderSummary> summaries = null;
_logger.Information($"Generating allocations for specification id {messageProperties.SpecificationId}");
BuildProject buildProject = await GetBuildProject(messageProperties.SpecificationId);
byte[] assembly = await _calculationsRepositoryPolicy.ExecuteAsync(() => _calculationsRepository.GetAssemblyBySpecificationId(messageProperties.SpecificationId));
if (assembly == null)
{
string error = $"Failed to get assembly for specification Id '{messageProperties.SpecificationId}'";
_logger.Error(error);
throw new RetriableException(error);
}
buildProject.Build.Assembly = assembly;
Dictionary <string, List <decimal> > cachedCalculationAggregationsBatch = CreateCalculationAggregateBatchDictionary(messageProperties);
_logger.Information($"processing partition index {messageProperties.PartitionIndex} for batch size {messageProperties.PartitionSize}");
int start = messageProperties.PartitionIndex;
int stop = start + messageProperties.PartitionSize - 1;
summaries = await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.ListRangeAsync <ProviderSummary>(messageProperties.ProviderCacheKey, start, stop));
int providerBatchSize = _engineSettings.ProviderBatchSize;
Stopwatch calculationsLookupStopwatch = Stopwatch.StartNew();
IEnumerable <CalculationSummaryModel> calculations = await _calculationsRepositoryPolicy.ExecuteAsync(() => _calculationsRepository.GetCalculationSummariesForSpecification(messageProperties.SpecificationId));
if (calculations == null)
{
_logger.Error($"Calculations lookup API returned null for specification id {messageProperties.SpecificationId}");
throw new InvalidOperationException("Calculations lookup API returned null");
}
calculationsLookupStopwatch.Stop();
IEnumerable <CalculationAggregation> aggregations = await BuildAggregations(messageProperties);
int totalProviderResults = 0;
bool calculationResultsHaveExceptions = false;
for (int i = 0; i < summaries.Count(); i += providerBatchSize)
{
Stopwatch calculationStopwatch = new Stopwatch();
Stopwatch providerSourceDatasetsStopwatch = new Stopwatch();
Stopwatch calcTiming = Stopwatch.StartNew();
CalculationResultsModel calculationResults = await CalculateResults(summaries, calculations, aggregations, buildProject, messageProperties, providerBatchSize, i, providerSourceDatasetsStopwatch, calculationStopwatch);
_logger.Information($"calculating results complete for specification id {messageProperties.SpecificationId}");
long saveCosmosElapsedMs = -1;
long saveSearchElapsedMs = -1;
long saveRedisElapsedMs = 0;
long saveQueueElapsedMs = 0;
if (calculationResults.ProviderResults.Any())
{
if (messageProperties.GenerateCalculationAggregationsOnly)
{
PopulateCachedCalculationAggregationsBatch(calculationResults.ProviderResults, cachedCalculationAggregationsBatch, messageProperties);
}
else
{
(long saveCosmosElapsedMs, long saveSearchElapsedMs, long saveRedisElapsedMs, long saveQueueElapsedMs)timingMetrics = await ProcessProviderResults(calculationResults.ProviderResults, messageProperties, message);
saveCosmosElapsedMs = timingMetrics.saveCosmosElapsedMs;
saveSearchElapsedMs = timingMetrics.saveSearchElapsedMs;
saveRedisElapsedMs = timingMetrics.saveRedisElapsedMs;
saveQueueElapsedMs = timingMetrics.saveQueueElapsedMs;
totalProviderResults += calculationResults.ProviderResults.Count();
if (calculationResults.ResultsContainExceptions)
{
if (!calculationResultsHaveExceptions)
{
calculationResultsHaveExceptions = true;
}
}
}
}
calcTiming.Stop();
IDictionary <string, double> metrics = new Dictionary <string, double>()
{
{ "calculation-run-providersProcessed", calculationResults.PartitionedSummaries.Count() },
{ "calculation-run-lookupCalculationDefinitionsMs", calculationsLookupStopwatch.ElapsedMilliseconds },
{ "calculation-run-providersResultsFromCache", summaries.Count() },
{ "calculation-run-partitionSize", messageProperties.PartitionSize },
{ "calculation-run-providerSourceDatasetQueryMs", providerSourceDatasetsStopwatch.ElapsedMilliseconds },
{ "calculation-run-saveProviderResultsRedisMs", saveRedisElapsedMs },
{ "calculation-run-saveProviderResultsServiceBusMs", saveQueueElapsedMs },
{ "calculation-run-runningCalculationMs", calculationStopwatch.ElapsedMilliseconds },
};
if (saveCosmosElapsedMs > -1)
{
metrics.Add("calculation-run-elapsedMilliseconds", calcTiming.ElapsedMilliseconds);
metrics.Add("calculation-run-saveProviderResultsCosmosMs", saveCosmosElapsedMs);
metrics.Add("calculation-run-saveProviderResultsSearchMs", saveSearchElapsedMs);
}
else
{
metrics.Add("calculation-run-for-tests-ms", calcTiming.ElapsedMilliseconds);
}
_telemetry.TrackEvent("CalculationRunProvidersProcessed",
new Dictionary <string, string>()
{
{ "specificationId", messageProperties.SpecificationId },
{ "buildProjectId", buildProject.Id },
},
metrics
);
}
if (calculationResultsHaveExceptions)
{
await FailJob(messageProperties.JobId, totalProviderResults, "Exceptions were thrown during generation of calculation results");
}
else
{
await CompleteBatch(messageProperties, cachedCalculationAggregationsBatch, summaries.Count(), totalProviderResults);
}
}