public async Task <IActionResult> PreviewCalculationResult( string specificationId, string providerId, PreviewCalculationRequest previewCalculationRequest) { Guard.IsNullOrWhiteSpace(specificationId, nameof(specificationId)); Guard.IsNullOrWhiteSpace(providerId, nameof(providerId)); Guard.ArgumentNotNull(previewCalculationRequest, nameof(previewCalculationRequest)); Assembly assembly = Assembly.Load(previewCalculationRequest.AssemblyContent); IAllocationModel allocationModel = _calculationEngine.GenerateAllocationModel(assembly); SpecificationSummary specificationSummary = await GetSpecificationSummary(specificationId); ApiResponse <ProviderVersionSearchResult> providerVersionSearchResultApiResponse = await _providersApiClientPolicy.ExecuteAsync(() => _providersApiClient.GetProviderByIdFromProviderVersion( specificationSummary.ProviderVersionId, providerId)); ProviderVersionSearchResult providerVersionSearchResult = providerVersionSearchResultApiResponse.Content; if (providerVersionSearchResult == null) { return(new NotFoundResult()); } ProviderSummary providerSummary = _mapper.Map <ProviderSummary>(providerVersionSearchResult); List <CalculationSummaryModel> calculationSummaries = new List <CalculationSummaryModel>(); IEnumerable <CalculationSummaryModel> specCalculationSummaries = await GetCalculationSummaries(specificationId); calculationSummaries.AddRange(specCalculationSummaries); calculationSummaries.Add(previewCalculationRequest.PreviewCalculationSummaryModel); Dictionary <string, Dictionary <string, ProviderSourceDataset> > providerSourceDatasets = await _providerSourceDatasetsRepository.GetProviderSourceDatasetsByProviderIdsAndRelationshipIds( specificationId, new[] { providerId }, specificationSummary.DataDefinitionRelationshipIds); Dictionary <string, ProviderSourceDataset> providerSourceDataset = providerSourceDatasets[providerId]; BuildAggregationRequest buildAggregationRequest = new BuildAggregationRequest { SpecificationId = specificationId, GenerateCalculationAggregationsOnly = true, BatchCount = 100 }; IEnumerable <CalculationAggregation> calculationAggregations = await _calculationAggregationService.BuildAggregations(buildAggregationRequest); ProviderResult providerResult = _calculationEngine.CalculateProviderResults( allocationModel, specificationId, calculationSummaries, providerSummary, providerSourceDataset, calculationAggregations ); return(new OkObjectResult(providerResult)); }
private async Task <CalculationResultsModel> CalculateResults(string specificationId, IEnumerable <ProviderSummary> summaries, IEnumerable <CalculationSummaryModel> calculations, IEnumerable <CalculationAggregation> aggregations, IEnumerable <string> dataRelationshipIds, byte[] assemblyForSpecification, GenerateAllocationMessageProperties messageProperties, int providerBatchSize, int index) { ConcurrentBag <ProviderResult> providerResults = new ConcurrentBag <ProviderResult>(); Guard.ArgumentNotNull(summaries, nameof(summaries)); IEnumerable <ProviderSummary> partitionedSummaries = summaries.Skip(index).Take(providerBatchSize); IList <string> providerIdList = partitionedSummaries.Select(m => m.Id).ToList(); Stopwatch providerSourceDatasetsStopwatch = Stopwatch.StartNew(); _logger.Information($"Fetching provider sources for specification id {messageProperties.SpecificationId}"); Dictionary <string, Dictionary <string, ProviderSourceDataset> > providerSourceDatasetResult = await _providerSourceDatasetsRepositoryPolicy.ExecuteAsync( () => _providerSourceDatasetsRepository.GetProviderSourceDatasetsByProviderIdsAndRelationshipIds(specificationId, providerIdList, dataRelationshipIds)); providerSourceDatasetsStopwatch.Stop(); _logger.Information($"Fetched provider sources found for specification id {messageProperties.SpecificationId}"); _logger.Information($"Calculating results for specification id {messageProperties.SpecificationId}"); Stopwatch assemblyLoadStopwatch = Stopwatch.StartNew(); Assembly assembly = Assembly.Load(assemblyForSpecification); assemblyLoadStopwatch.Stop(); Stopwatch calculationStopwatch = Stopwatch.StartNew(); List <Task> allTasks = new List <Task>(); SemaphoreSlim throttler = new SemaphoreSlim(_engineSettings.CalculateProviderResultsDegreeOfParallelism); IAllocationModel allocationModel = _calculationEngine.GenerateAllocationModel(assembly); foreach (ProviderSummary provider in partitionedSummaries) { await throttler.WaitAsync(); allTasks.Add( Task.Run(() => { try { if (provider == null) { throw new Exception("Provider summary is null"); } if (providerSourceDatasetResult.AnyWithNullCheck()) { if (!providerSourceDatasetResult.TryGetValue(provider.Id, out Dictionary <string, ProviderSourceDataset> providerDatasets)) { throw new Exception($"Provider source dataset not found for {provider.Id}."); } ProviderResult result = _calculationEngine.CalculateProviderResults(allocationModel, specificationId, calculations, provider, providerDatasets, aggregations); if (result == null) { throw new InvalidOperationException("Null result from Calc Engine CalculateProviderResults"); } providerResults.Add(result); } } finally { throttler.Release(); } return(Task.CompletedTask); })); } await TaskHelper.WhenAllAndThrow(allTasks.ToArray()); calculationStopwatch.Stop(); _logger.Information($"Calculating results complete for specification id {messageProperties.SpecificationId} in {calculationStopwatch.ElapsedMilliseconds}ms"); return(new CalculationResultsModel { ProviderResults = providerResults, PartitionedSummaries = partitionedSummaries, CalculationRunMs = calculationStopwatch.ElapsedMilliseconds, AssemblyLoadMs = assemblyLoadStopwatch.ElapsedMilliseconds, ProviderSourceDatasetsLookupMs = providerSourceDatasetsStopwatch.ElapsedMilliseconds, }); }
private async Task <CalculationResultsModel> CalculateResults(IEnumerable <ProviderSummary> summaries, IEnumerable <CalculationSummaryModel> calculations, IEnumerable <CalculationAggregation> aggregations, BuildProject buildProject, GenerateAllocationMessageProperties messageProperties, int providerBatchSize, int index, Stopwatch providerSourceDatasetsStopwatch, Stopwatch calculationStopwatch) { ConcurrentBag <ProviderResult> providerResults = new ConcurrentBag <ProviderResult>(); IEnumerable <ProviderSummary> partitionedSummaries = summaries.Skip(index).Take(providerBatchSize); IList <string> providerIdList = partitionedSummaries.Select(m => m.Id).ToList(); providerSourceDatasetsStopwatch.Start(); _logger.Information($"Fetching provider sources for specification id {messageProperties.SpecificationId}"); List <ProviderSourceDataset> providerSourceDatasets = new List <ProviderSourceDataset>(await _providerSourceDatasetsRepositoryPolicy.ExecuteAsync(() => _providerSourceDatasetsRepository.GetProviderSourceDatasetsByProviderIdsAndSpecificationId(providerIdList, messageProperties.SpecificationId))); providerSourceDatasetsStopwatch.Stop(); if (providerSourceDatasets == null) { _logger.Information($"No provider sources found for specification id {messageProperties.SpecificationId}"); providerSourceDatasets = new List <ProviderSourceDataset>(); } _logger.Information($"fetched provider sources found for specification id {messageProperties.SpecificationId}"); calculationStopwatch.Start(); _logger.Information($"calculating results for specification id {messageProperties.SpecificationId}"); Assembly assembly = Assembly.Load(buildProject.Build.Assembly); Parallel.ForEach(partitionedSummaries, new ParallelOptions { MaxDegreeOfParallelism = _engineSettings.CalculateProviderResultsDegreeOfParallelism }, provider => { IAllocationModel allocationModel = _calculationEngine.GenerateAllocationModel(assembly); IEnumerable <ProviderSourceDataset> providerDatasets = providerSourceDatasets.Where(m => m.ProviderId == provider.Id); ProviderResult result = _calculationEngine.CalculateProviderResults(allocationModel, buildProject, calculations, provider, providerDatasets, aggregations); if (result != null) { providerResults.Add(result); } else { throw new InvalidOperationException("Null result from Calc Engine CalculateProviderResults"); } }); _logger.Information($"calculating results complete for specification id {messageProperties.SpecificationId}"); calculationStopwatch.Stop(); return(new CalculationResultsModel { ProviderResults = providerResults, PartitionedSummaries = partitionedSummaries }); }