public async Task PreviewCalculationResult_GivenCachedAggregateValuesExist_CalculateProviderResultsCallReceived() { IAllocationModel allocationModel = Substitute.For <IAllocationModel>(); _calculationEngine .GenerateAllocationModel(Arg.Any <Assembly>()) .Returns(allocationModel); ProviderVersionSearchResult providerVersionSearchResult = new ProviderVersionSearchResult { UKPRN = providerId }; IEnumerable <string> dataDefinitionRelationshipIds = new List <string>(); SpecificationSummary specificationSummary = new SpecificationSummary { DataDefinitionRelationshipIds = dataDefinitionRelationshipIds, ProviderVersionId = providerVersionId }; _specificationsApiClient .GetSpecificationSummaryById(Arg.Is(specificationId)) .Returns(new ApiResponse <SpecificationSummary>(HttpStatusCode.OK, specificationSummary)); _providersApiClient .GetProviderByIdFromProviderVersion(Arg.Is(providerVersionId), Arg.Is(providerId)) .Returns(new ApiResponse <ProviderVersionSearchResult>(HttpStatusCode.OK, providerVersionSearchResult)); CalculationSummaryModel previewCalculationSummaryModel = new CalculationSummaryModel(); IEnumerable <CalculationSummaryModel> calculationSummaryModels = new List <CalculationSummaryModel> { new CalculationSummaryModel(), new CalculationSummaryModel() }; List <CalculationSummaryModel> expectedCalculationSummaryModels = calculationSummaryModels.ToList(); expectedCalculationSummaryModels.Add(previewCalculationSummaryModel); _calculationsRepository .GetCalculationSummariesForSpecification(Arg.Is(specificationId)) .Returns(calculationSummaryModels); Dictionary <string, ProviderSourceDataset> sourceDatasets = new Dictionary <string, ProviderSourceDataset>(); Dictionary <string, Dictionary <string, ProviderSourceDataset> > providerSourceDatasets = new Dictionary <string, Dictionary <string, ProviderSourceDataset> > { { providerId, sourceDatasets } }; _providerSourceDatasetsRepository .GetProviderSourceDatasetsByProviderIdsAndRelationshipIds( Arg.Is(specificationId), Arg.Is <IEnumerable <string> >(_ => _ != null && _.Count() == 1 && _.FirstOrDefault() == providerId), Arg.Is <IEnumerable <string> >(_ => _ != null && _.SequenceEqual(dataDefinitionRelationshipIds))) .Returns(providerSourceDatasets); IEnumerable <CalculationAggregation> calculationAggregations = new List <CalculationAggregation>(); _calculationAggregationService .BuildAggregations(Arg.Is <BuildAggregationRequest>(_ => _ != null && _.SpecificationId == specificationId)) .Returns(calculationAggregations); PreviewCalculationRequest previewCalculationRequest = new PreviewCalculationRequest { AssemblyContent = MockData.GetMockAssembly(), PreviewCalculationSummaryModel = previewCalculationSummaryModel }; IActionResult actionResult = await _calculationEnginePreviewService.PreviewCalculationResult(specificationId, providerId, previewCalculationRequest); _calculationEngine .Received(1) .CalculateProviderResults( Arg.Is(allocationModel), specificationId, Arg.Is <IEnumerable <CalculationSummaryModel> >(_ => _.SequenceEqual(expectedCalculationSummaryModels)), Arg.Is <ProviderSummary>(_ => _.UKPRN == providerId), Arg.Is <Dictionary <string, ProviderSourceDataset> >(_ => _.SequenceEqual(sourceDatasets)), Arg.Is <IEnumerable <CalculationAggregation> >(_ => _.SequenceEqual(calculationAggregations))); }
public async Task <IActionResult> PreviewCalculationResult( string specificationId, string providerId, PreviewCalculationRequest previewCalculationRequest) { Guard.IsNullOrWhiteSpace(specificationId, nameof(specificationId)); Guard.IsNullOrWhiteSpace(providerId, nameof(providerId)); Guard.ArgumentNotNull(previewCalculationRequest, nameof(previewCalculationRequest)); Assembly assembly = Assembly.Load(previewCalculationRequest.AssemblyContent); IAllocationModel allocationModel = _calculationEngine.GenerateAllocationModel(assembly); SpecificationSummary specificationSummary = await GetSpecificationSummary(specificationId); ApiResponse <ProviderVersionSearchResult> providerVersionSearchResultApiResponse = await _providersApiClientPolicy.ExecuteAsync(() => _providersApiClient.GetProviderByIdFromProviderVersion( specificationSummary.ProviderVersionId, providerId)); ProviderVersionSearchResult providerVersionSearchResult = providerVersionSearchResultApiResponse.Content; if (providerVersionSearchResult == null) { return(new NotFoundResult()); } ProviderSummary providerSummary = _mapper.Map <ProviderSummary>(providerVersionSearchResult); List <CalculationSummaryModel> calculationSummaries = new List <CalculationSummaryModel>(); IEnumerable <CalculationSummaryModel> specCalculationSummaries = await GetCalculationSummaries(specificationId); calculationSummaries.AddRange(specCalculationSummaries); calculationSummaries.Add(previewCalculationRequest.PreviewCalculationSummaryModel); Dictionary <string, Dictionary <string, ProviderSourceDataset> > providerSourceDatasets = await _providerSourceDatasetsRepository.GetProviderSourceDatasetsByProviderIdsAndRelationshipIds( specificationId, new[] { providerId }, specificationSummary.DataDefinitionRelationshipIds); Dictionary <string, ProviderSourceDataset> providerSourceDataset = providerSourceDatasets[providerId]; BuildAggregationRequest buildAggregationRequest = new BuildAggregationRequest { SpecificationId = specificationId, GenerateCalculationAggregationsOnly = true, BatchCount = 100 }; IEnumerable <CalculationAggregation> calculationAggregations = await _calculationAggregationService.BuildAggregations(buildAggregationRequest); ProviderResult providerResult = _calculationEngine.CalculateProviderResults( allocationModel, specificationId, calculationSummaries, providerSummary, providerSourceDataset, calculationAggregations ); return(new OkObjectResult(providerResult)); }
private async Task <CalculationResultsModel> CalculateResults(string specificationId, IEnumerable <ProviderSummary> summaries, IEnumerable <CalculationSummaryModel> calculations, IEnumerable <CalculationAggregation> aggregations, IEnumerable <string> dataRelationshipIds, byte[] assemblyForSpecification, GenerateAllocationMessageProperties messageProperties, int providerBatchSize, int index) { ConcurrentBag <ProviderResult> providerResults = new ConcurrentBag <ProviderResult>(); Guard.ArgumentNotNull(summaries, nameof(summaries)); IEnumerable <ProviderSummary> partitionedSummaries = summaries.Skip(index).Take(providerBatchSize); IList <string> providerIdList = partitionedSummaries.Select(m => m.Id).ToList(); Stopwatch providerSourceDatasetsStopwatch = Stopwatch.StartNew(); _logger.Information($"Fetching provider sources for specification id {messageProperties.SpecificationId}"); Dictionary <string, Dictionary <string, ProviderSourceDataset> > providerSourceDatasetResult = await _providerSourceDatasetsRepositoryPolicy.ExecuteAsync( () => _providerSourceDatasetsRepository.GetProviderSourceDatasetsByProviderIdsAndRelationshipIds(specificationId, providerIdList, dataRelationshipIds)); providerSourceDatasetsStopwatch.Stop(); _logger.Information($"Fetched provider sources found for specification id {messageProperties.SpecificationId}"); _logger.Information($"Calculating results for specification id {messageProperties.SpecificationId}"); Stopwatch assemblyLoadStopwatch = Stopwatch.StartNew(); Assembly assembly = Assembly.Load(assemblyForSpecification); assemblyLoadStopwatch.Stop(); Stopwatch calculationStopwatch = Stopwatch.StartNew(); List <Task> allTasks = new List <Task>(); SemaphoreSlim throttler = new SemaphoreSlim(_engineSettings.CalculateProviderResultsDegreeOfParallelism); IAllocationModel allocationModel = _calculationEngine.GenerateAllocationModel(assembly); foreach (ProviderSummary provider in partitionedSummaries) { await throttler.WaitAsync(); allTasks.Add( Task.Run(() => { try { if (provider == null) { throw new Exception("Provider summary is null"); } if (providerSourceDatasetResult.AnyWithNullCheck()) { if (!providerSourceDatasetResult.TryGetValue(provider.Id, out Dictionary <string, ProviderSourceDataset> providerDatasets)) { throw new Exception($"Provider source dataset not found for {provider.Id}."); } ProviderResult result = _calculationEngine.CalculateProviderResults(allocationModel, specificationId, calculations, provider, providerDatasets, aggregations); if (result == null) { throw new InvalidOperationException("Null result from Calc Engine CalculateProviderResults"); } providerResults.Add(result); } } finally { throttler.Release(); } return(Task.CompletedTask); })); } await TaskHelper.WhenAllAndThrow(allTasks.ToArray()); calculationStopwatch.Stop(); _logger.Information($"Calculating results complete for specification id {messageProperties.SpecificationId} in {calculationStopwatch.ElapsedMilliseconds}ms"); return(new CalculationResultsModel { ProviderResults = providerResults, PartitionedSummaries = partitionedSummaries, CalculationRunMs = calculationStopwatch.ElapsedMilliseconds, AssemblyLoadMs = assemblyLoadStopwatch.ElapsedMilliseconds, ProviderSourceDatasetsLookupMs = providerSourceDatasetsStopwatch.ElapsedMilliseconds, }); }
private async Task <CalculationResultsModel> CalculateResults(IEnumerable <ProviderSummary> summaries, IEnumerable <CalculationSummaryModel> calculations, IEnumerable <CalculationAggregation> aggregations, BuildProject buildProject, GenerateAllocationMessageProperties messageProperties, int providerBatchSize, int index, Stopwatch providerSourceDatasetsStopwatch, Stopwatch calculationStopwatch) { ConcurrentBag <ProviderResult> providerResults = new ConcurrentBag <ProviderResult>(); IEnumerable <ProviderSummary> partitionedSummaries = summaries.Skip(index).Take(providerBatchSize); IList <string> providerIdList = partitionedSummaries.Select(m => m.Id).ToList(); providerSourceDatasetsStopwatch.Start(); _logger.Information($"Fetching provider sources for specification id {messageProperties.SpecificationId}"); List <ProviderSourceDataset> providerSourceDatasets = new List <ProviderSourceDataset>(await _providerSourceDatasetsRepositoryPolicy.ExecuteAsync(() => _providerSourceDatasetsRepository.GetProviderSourceDatasetsByProviderIdsAndSpecificationId(providerIdList, messageProperties.SpecificationId))); providerSourceDatasetsStopwatch.Stop(); if (providerSourceDatasets == null) { _logger.Information($"No provider sources found for specification id {messageProperties.SpecificationId}"); providerSourceDatasets = new List <ProviderSourceDataset>(); } _logger.Information($"fetched provider sources found for specification id {messageProperties.SpecificationId}"); calculationStopwatch.Start(); _logger.Information($"calculating results for specification id {messageProperties.SpecificationId}"); Assembly assembly = Assembly.Load(buildProject.Build.Assembly); Parallel.ForEach(partitionedSummaries, new ParallelOptions { MaxDegreeOfParallelism = _engineSettings.CalculateProviderResultsDegreeOfParallelism }, provider => { IAllocationModel allocationModel = _calculationEngine.GenerateAllocationModel(assembly); IEnumerable <ProviderSourceDataset> providerDatasets = providerSourceDatasets.Where(m => m.ProviderId == provider.Id); ProviderResult result = _calculationEngine.CalculateProviderResults(allocationModel, buildProject, calculations, provider, providerDatasets, aggregations); if (result != null) { providerResults.Add(result); } else { throw new InvalidOperationException("Null result from Calc Engine CalculateProviderResults"); } }); _logger.Information($"calculating results complete for specification id {messageProperties.SpecificationId}"); calculationStopwatch.Stop(); return(new CalculationResultsModel { ProviderResults = providerResults, PartitionedSummaries = partitionedSummaries }); }