public async Task PreviewCalculationResult_GivenCachedAggregateValuesExist_CalculateProviderResultsCallReceived()
        {
            IAllocationModel allocationModel = Substitute.For <IAllocationModel>();

            _calculationEngine
            .GenerateAllocationModel(Arg.Any <Assembly>())
            .Returns(allocationModel);

            ProviderVersionSearchResult providerVersionSearchResult = new ProviderVersionSearchResult
            {
                UKPRN = providerId
            };

            IEnumerable <string> dataDefinitionRelationshipIds = new List <string>();

            SpecificationSummary specificationSummary = new SpecificationSummary
            {
                DataDefinitionRelationshipIds = dataDefinitionRelationshipIds,
                ProviderVersionId             = providerVersionId
            };

            _specificationsApiClient
            .GetSpecificationSummaryById(Arg.Is(specificationId))
            .Returns(new ApiResponse <SpecificationSummary>(HttpStatusCode.OK, specificationSummary));

            _providersApiClient
            .GetProviderByIdFromProviderVersion(Arg.Is(providerVersionId), Arg.Is(providerId))
            .Returns(new ApiResponse <ProviderVersionSearchResult>(HttpStatusCode.OK, providerVersionSearchResult));

            CalculationSummaryModel previewCalculationSummaryModel = new CalculationSummaryModel();

            IEnumerable <CalculationSummaryModel> calculationSummaryModels = new List <CalculationSummaryModel>
            {
                new CalculationSummaryModel(),
                new CalculationSummaryModel()
            };

            List <CalculationSummaryModel> expectedCalculationSummaryModels = calculationSummaryModels.ToList();

            expectedCalculationSummaryModels.Add(previewCalculationSummaryModel);

            _calculationsRepository
            .GetCalculationSummariesForSpecification(Arg.Is(specificationId))
            .Returns(calculationSummaryModels);

            Dictionary <string, ProviderSourceDataset> sourceDatasets = new Dictionary <string, ProviderSourceDataset>();

            Dictionary <string, Dictionary <string, ProviderSourceDataset> > providerSourceDatasets = new Dictionary <string, Dictionary <string, ProviderSourceDataset> >
            {
                { providerId, sourceDatasets }
            };

            _providerSourceDatasetsRepository
            .GetProviderSourceDatasetsByProviderIdsAndRelationshipIds(
                Arg.Is(specificationId),
                Arg.Is <IEnumerable <string> >(_ => _ != null && _.Count() == 1 && _.FirstOrDefault() == providerId),
                Arg.Is <IEnumerable <string> >(_ => _ != null && _.SequenceEqual(dataDefinitionRelationshipIds)))
            .Returns(providerSourceDatasets);

            IEnumerable <CalculationAggregation> calculationAggregations = new List <CalculationAggregation>();

            _calculationAggregationService
            .BuildAggregations(Arg.Is <BuildAggregationRequest>(_ => _ != null && _.SpecificationId == specificationId))
            .Returns(calculationAggregations);

            PreviewCalculationRequest previewCalculationRequest = new PreviewCalculationRequest
            {
                AssemblyContent = MockData.GetMockAssembly(),
                PreviewCalculationSummaryModel = previewCalculationSummaryModel
            };

            IActionResult actionResult =
                await _calculationEnginePreviewService.PreviewCalculationResult(specificationId, providerId, previewCalculationRequest);

            _calculationEngine
            .Received(1)
            .CalculateProviderResults(
                Arg.Is(allocationModel),
                specificationId,
                Arg.Is <IEnumerable <CalculationSummaryModel> >(_ => _.SequenceEqual(expectedCalculationSummaryModels)),
                Arg.Is <ProviderSummary>(_ => _.UKPRN == providerId),
                Arg.Is <Dictionary <string, ProviderSourceDataset> >(_ => _.SequenceEqual(sourceDatasets)),
                Arg.Is <IEnumerable <CalculationAggregation> >(_ => _.SequenceEqual(calculationAggregations)));
        }
示例#2
0
        public async Task <IActionResult> PreviewCalculationResult(
            string specificationId,
            string providerId,
            PreviewCalculationRequest previewCalculationRequest)
        {
            Guard.IsNullOrWhiteSpace(specificationId, nameof(specificationId));
            Guard.IsNullOrWhiteSpace(providerId, nameof(providerId));
            Guard.ArgumentNotNull(previewCalculationRequest, nameof(previewCalculationRequest));

            Assembly         assembly        = Assembly.Load(previewCalculationRequest.AssemblyContent);
            IAllocationModel allocationModel = _calculationEngine.GenerateAllocationModel(assembly);

            SpecificationSummary specificationSummary = await GetSpecificationSummary(specificationId);

            ApiResponse <ProviderVersionSearchResult> providerVersionSearchResultApiResponse =
                await _providersApiClientPolicy.ExecuteAsync(() => _providersApiClient.GetProviderByIdFromProviderVersion(
                                                                 specificationSummary.ProviderVersionId,
                                                                 providerId));

            ProviderVersionSearchResult providerVersionSearchResult = providerVersionSearchResultApiResponse.Content;

            if (providerVersionSearchResult == null)
            {
                return(new NotFoundResult());
            }

            ProviderSummary providerSummary = _mapper.Map <ProviderSummary>(providerVersionSearchResult);

            List <CalculationSummaryModel>        calculationSummaries     = new List <CalculationSummaryModel>();
            IEnumerable <CalculationSummaryModel> specCalculationSummaries = await GetCalculationSummaries(specificationId);

            calculationSummaries.AddRange(specCalculationSummaries);
            calculationSummaries.Add(previewCalculationRequest.PreviewCalculationSummaryModel);

            Dictionary <string, Dictionary <string, ProviderSourceDataset> > providerSourceDatasets =
                await _providerSourceDatasetsRepository.GetProviderSourceDatasetsByProviderIdsAndRelationshipIds(
                    specificationId,
                    new[] { providerId },
                    specificationSummary.DataDefinitionRelationshipIds);

            Dictionary <string, ProviderSourceDataset> providerSourceDataset = providerSourceDatasets[providerId];

            BuildAggregationRequest buildAggregationRequest = new BuildAggregationRequest
            {
                SpecificationId = specificationId,
                GenerateCalculationAggregationsOnly = true,
                BatchCount = 100
            };
            IEnumerable <CalculationAggregation> calculationAggregations =
                await _calculationAggregationService.BuildAggregations(buildAggregationRequest);

            ProviderResult providerResult = _calculationEngine.CalculateProviderResults(
                allocationModel,
                specificationId,
                calculationSummaries,
                providerSummary,
                providerSourceDataset,
                calculationAggregations
                );

            return(new OkObjectResult(providerResult));
        }
        private async Task <CalculationResultsModel> CalculateResults(string specificationId, IEnumerable <ProviderSummary> summaries,
                                                                      IEnumerable <CalculationSummaryModel> calculations,
                                                                      IEnumerable <CalculationAggregation> aggregations,
                                                                      IEnumerable <string> dataRelationshipIds,
                                                                      byte[] assemblyForSpecification,
                                                                      GenerateAllocationMessageProperties messageProperties,
                                                                      int providerBatchSize,
                                                                      int index)
        {
            ConcurrentBag <ProviderResult> providerResults = new ConcurrentBag <ProviderResult>();

            Guard.ArgumentNotNull(summaries, nameof(summaries));

            IEnumerable <ProviderSummary> partitionedSummaries = summaries.Skip(index).Take(providerBatchSize);

            IList <string> providerIdList = partitionedSummaries.Select(m => m.Id).ToList();

            Stopwatch providerSourceDatasetsStopwatch = Stopwatch.StartNew();

            _logger.Information($"Fetching provider sources for specification id {messageProperties.SpecificationId}");

            Dictionary <string, Dictionary <string, ProviderSourceDataset> > providerSourceDatasetResult = await _providerSourceDatasetsRepositoryPolicy.ExecuteAsync(
                () => _providerSourceDatasetsRepository.GetProviderSourceDatasetsByProviderIdsAndRelationshipIds(specificationId, providerIdList, dataRelationshipIds));

            providerSourceDatasetsStopwatch.Stop();

            _logger.Information($"Fetched provider sources found for specification id {messageProperties.SpecificationId}");


            _logger.Information($"Calculating results for specification id {messageProperties.SpecificationId}");
            Stopwatch assemblyLoadStopwatch = Stopwatch.StartNew();
            Assembly  assembly = Assembly.Load(assemblyForSpecification);

            assemblyLoadStopwatch.Stop();

            Stopwatch calculationStopwatch = Stopwatch.StartNew();

            List <Task>   allTasks  = new List <Task>();
            SemaphoreSlim throttler = new SemaphoreSlim(_engineSettings.CalculateProviderResultsDegreeOfParallelism);

            IAllocationModel allocationModel = _calculationEngine.GenerateAllocationModel(assembly);

            foreach (ProviderSummary provider in partitionedSummaries)
            {
                await throttler.WaitAsync();

                allTasks.Add(
                    Task.Run(() =>
                {
                    try
                    {
                        if (provider == null)
                        {
                            throw new Exception("Provider summary is null");
                        }

                        if (providerSourceDatasetResult.AnyWithNullCheck())
                        {
                            if (!providerSourceDatasetResult.TryGetValue(provider.Id, out Dictionary <string, ProviderSourceDataset> providerDatasets))
                            {
                                throw new Exception($"Provider source dataset not found for {provider.Id}.");
                            }

                            ProviderResult result = _calculationEngine.CalculateProviderResults(allocationModel, specificationId, calculations, provider, providerDatasets, aggregations);

                            if (result == null)
                            {
                                throw new InvalidOperationException("Null result from Calc Engine CalculateProviderResults");
                            }

                            providerResults.Add(result);
                        }
                    }
                    finally
                    {
                        throttler.Release();
                    }

                    return(Task.CompletedTask);
                }));
            }

            await TaskHelper.WhenAllAndThrow(allTasks.ToArray());

            calculationStopwatch.Stop();

            _logger.Information($"Calculating results complete for specification id {messageProperties.SpecificationId} in {calculationStopwatch.ElapsedMilliseconds}ms");

            return(new CalculationResultsModel
            {
                ProviderResults = providerResults,
                PartitionedSummaries = partitionedSummaries,
                CalculationRunMs = calculationStopwatch.ElapsedMilliseconds,
                AssemblyLoadMs = assemblyLoadStopwatch.ElapsedMilliseconds,
                ProviderSourceDatasetsLookupMs = providerSourceDatasetsStopwatch.ElapsedMilliseconds,
            });
        }
        private async Task <CalculationResultsModel> CalculateResults(IEnumerable <ProviderSummary> summaries, IEnumerable <CalculationSummaryModel> calculations, IEnumerable <CalculationAggregation> aggregations, BuildProject buildProject,
                                                                      GenerateAllocationMessageProperties messageProperties, int providerBatchSize, int index, Stopwatch providerSourceDatasetsStopwatch, Stopwatch calculationStopwatch)
        {
            ConcurrentBag <ProviderResult> providerResults = new ConcurrentBag <ProviderResult>();

            IEnumerable <ProviderSummary> partitionedSummaries = summaries.Skip(index).Take(providerBatchSize);

            IList <string> providerIdList = partitionedSummaries.Select(m => m.Id).ToList();

            providerSourceDatasetsStopwatch.Start();

            _logger.Information($"Fetching provider sources for specification id {messageProperties.SpecificationId}");

            List <ProviderSourceDataset> providerSourceDatasets = new List <ProviderSourceDataset>(await _providerSourceDatasetsRepositoryPolicy.ExecuteAsync(() => _providerSourceDatasetsRepository.GetProviderSourceDatasetsByProviderIdsAndSpecificationId(providerIdList, messageProperties.SpecificationId)));

            providerSourceDatasetsStopwatch.Stop();

            if (providerSourceDatasets == null)
            {
                _logger.Information($"No provider sources found for specification id {messageProperties.SpecificationId}");

                providerSourceDatasets = new List <ProviderSourceDataset>();
            }

            _logger.Information($"fetched provider sources found for specification id {messageProperties.SpecificationId}");

            calculationStopwatch.Start();

            _logger.Information($"calculating results for specification id {messageProperties.SpecificationId}");

            Assembly assembly = Assembly.Load(buildProject.Build.Assembly);

            Parallel.ForEach(partitionedSummaries, new ParallelOptions {
                MaxDegreeOfParallelism = _engineSettings.CalculateProviderResultsDegreeOfParallelism
            }, provider =>
            {
                IAllocationModel allocationModel = _calculationEngine.GenerateAllocationModel(assembly);

                IEnumerable <ProviderSourceDataset> providerDatasets = providerSourceDatasets.Where(m => m.ProviderId == provider.Id);

                ProviderResult result = _calculationEngine.CalculateProviderResults(allocationModel, buildProject, calculations, provider, providerDatasets, aggregations);

                if (result != null)
                {
                    providerResults.Add(result);
                }
                else
                {
                    throw new InvalidOperationException("Null result from Calc Engine CalculateProviderResults");
                }
            });


            _logger.Information($"calculating results complete for specification id {messageProperties.SpecificationId}");

            calculationStopwatch.Stop();

            return(new CalculationResultsModel
            {
                ProviderResults = providerResults,
                PartitionedSummaries = partitionedSummaries
            });
        }