public async Task <IEnumerable <string> > VerifyCalculationPrerequisites(SpecificationSummary specification)
        {
            List <string> validationErrors = new List <string>();

            string specificationId = specification.Id;

            ApiResponse <IEnumerable <CalculationMetadata> > calculationsResponse = await _policy.ExecuteAsync(() => _calcsApiClient.GetCalculationMetadataForSpecification(specificationId));

            if (calculationsResponse?.Content == null)
            {
                string errorMessage = $"Did locate any calculation metadata for specification {specificationId}. Unable to complete prerequisite checks";

                _logger.Error(errorMessage);
                validationErrors.Add(errorMessage);

                return(validationErrors);
            }

            validationErrors.AddRange(calculationsResponse?.Content.Where(_ => _.PublishStatus != PublishStatus.Approved && _.CalculationType == CalculationType.Template)
                                      .Select(_ => $"Calculation {_.Name} must be approved but is {_.PublishStatus}"));

            foreach (var fundingStream in specification.FundingStreams)
            {
                ApiResponse <TemplateMapping> templateMappingResponse = await _calcsApiClient.GetTemplateMapping(specificationId, fundingStream.Id);

                foreach (TemplateMappingItem calcInError in templateMappingResponse.Content.TemplateMappingItems.Where(c => string.IsNullOrWhiteSpace(c.CalculationId)))
                {
                    validationErrors.Add($"{calcInError.EntityType} {calcInError.Name} is not mapped to a calculation in CFS");
                }
            }

            return(validationErrors);
        }
Пример #2
0
        public async Task GetTemplateMapping_GivenFailedResponseFromCalculationsApi_ThrowsRetriableException()
        {
            //Arrange
            ApiResponse <TemplateMapping> apiResponse = new ApiResponse <TemplateMapping>(HttpStatusCode.NotFound);

            ICalculationsApiClient calculationsApiClient = CreateCalculationsApiClient();

            calculationsApiClient
            .GetTemplateMapping(Arg.Is(specificationId), Arg.Is(fundingStreamId))
            .Returns(apiResponse);

            ILogger logger = CreateLogger();

            string errorMessage = $"Failed to retrieve template mapping for specification id '{specificationId}' and  funding stream id '{fundingStreamId}'" +
                                  $" with status code '{apiResponse.StatusCode}'";

            CalculationsService calculationsService = CreateCalculationsService(calculationsApiClient, logger);

            //Act
            Func <Task> test = async() => await calculationsService.GetTemplateMapping(specificationId, fundingStreamId);

            //Assert
            test
            .Should()
            .ThrowExactly <RetriableException>()
            .Which
            .Message
            .Should()
            .Be(errorMessage);

            logger
            .Received(1)
            .Error(Arg.Is(errorMessage));
        }
Пример #3
0
        public async Task GetTemplateMapping_GivenResponseIsSuccess_ReturnsTempalteMappingValue()
        {
            //Arrange
            TemplateMapping expectedTemplateMapping = new TemplateMapping()
            {
                SpecificationId = specificationId, FundingStreamId = fundingStreamId
            };
            ApiResponse <TemplateMapping> apiResponse = new ApiResponse <TemplateMapping>(HttpStatusCode.OK, expectedTemplateMapping);

            ICalculationsApiClient calculationsApiClient = CreateCalculationsApiClient();

            calculationsApiClient
            .GetTemplateMapping(Arg.Is(specificationId), Arg.Is(fundingStreamId))
            .Returns(apiResponse);

            ILogger logger = CreateLogger();

            CalculationsService calculationsService = CreateCalculationsService(calculationsApiClient, logger);

            //Act
            TemplateMapping responseValue = await calculationsService.GetTemplateMapping(specificationId, fundingStreamId);

            //Assert
            responseValue
            .Should()
            .Be(expectedTemplateMapping);
        }
        private async Task <TemplateMapping> GetTemplateMapping(Reference fundingStream, string specificationId)
        {
            ApiResponse <TemplateMapping> calculationMappingResult =
                await _calculationsApiClientPolicy.ExecuteAsync(() => _calculationsApiClient.GetTemplateMapping(specificationId, fundingStream.Id));

            if (calculationMappingResult == null)
            {
                throw new Exception($"calculationMappingResult returned null for funding stream {fundingStream.Id}");
            }

            return(calculationMappingResult.Content);
        }
        private void AndTemplateMapping()
        {
            TemplateMappingItem[] templateMappingItems = new[] {
                NewTemplateMappingItem(_ => _.WithTemplateId(_calculationTemplateIds[0].TemplateCalculationId)
                                       .WithCalculationId(_calculationResults[0].Id)),
                NewTemplateMappingItem(_ => _.WithTemplateId(_calculationTemplateIds[1].TemplateCalculationId)
                                       .WithCalculationId(_calculationResults[1].Id)),
                NewTemplateMappingItem(_ => _.WithTemplateId(_calculationTemplateIds[2].TemplateCalculationId)
                                       .WithCalculationId(_calculationResults[2].Id))
            };

            _templateMapping = NewTemplateMapping(_ => _.WithItems(templateMappingItems));

            _calculationsApiClient
            .GetTemplateMapping(_specificationSummary.Id, FundingStreamId)
            .Returns(new ApiResponse <TemplateMapping>(HttpStatusCode.OK, _templateMapping));
        }
        public async Task <TemplateMapping> GetTemplateMapping(string specificationId, string fundingStreamId)
        {
            Guard.IsNullOrWhiteSpace(specificationId, nameof(specificationId));
            Guard.IsNullOrWhiteSpace(fundingStreamId, nameof(fundingStreamId));

            ApiResponse <TemplateMapping> apiResponse = await _calcsApiClientPolicy.ExecuteAsync(
                () => _calculationsApiClient.GetTemplateMapping(specificationId, fundingStreamId));

            if (!apiResponse.StatusCode.IsSuccess())
            {
                string errorMessage = $"Failed to retrieve template mapping for specification id '{specificationId}' and  funding stream id '{fundingStreamId}'" +
                                      $" with status code '{apiResponse.StatusCode}'";

                _logger.Error(errorMessage);

                throw new RetriableException(errorMessage);
            }

            return(apiResponse.Content);
        }
 private void GivenTheTemplateMappingForTheSpecificationIdAndFundingStreamId(string specificationId, string fundingStreamId, TemplateMapping templateMapping)
 {
     _calculationsApiClient.GetTemplateMapping(specificationId, fundingStreamId)
     .Returns(new ApiResponse <TemplateMapping>(HttpStatusCode.OK, templateMapping));
 }
Пример #8
0
        private void ValidScenarioSetup(string fundingStreamId)
        {
            SpecificationSummary specificationSummary = new SpecificationSummary
            {
                Id          = SpecificationId,
                TemplateIds = new Dictionary <string, string>
                {
                    [fundingStreamId] = TemplateVersion
                }
            };

            TemplateMetadataContents templateMetadataContents = new TemplateMetadataContents
            {
                RootFundingLines = new List <FundingLine>
                {
                    new FundingLine
                    {
                        TemplateLineId = 123,
                        Name           = "FundingLine-1"
                    },
                    new FundingLine
                    {
                        TemplateLineId = 234,
                        Name           = "FundingLine-2-withFundingLines",
                        FundingLines   = new List <FundingLine>
                        {
                            new FundingLine
                            {
                                TemplateLineId = 345,
                                Name           = "FundingLine-2-fl-1"
                            },
                            new FundingLine
                            {
                                TemplateLineId = 456,
                                Name           = "FundingLine-2-fl-2",
                                FundingLines   = new List <FundingLine>
                                {
                                    new FundingLine
                                    {
                                        TemplateLineId = 890,
                                        Name           = "FundingLine-2-fl-2-fl-1"
                                    }
                                }
                            }
                        }
                    },
                    new FundingLine
                    {
                        TemplateLineId = 567,
                        Name           = "FundingLine-3-withCalculationsAndFundingLines",
                        FundingLines   = new List <FundingLine>
                        {
                            new FundingLine
                            {
                                TemplateLineId = 678,
                                Name           = "FundingLine-3-fl-1"
                            }
                        },
                        Calculations = new List <Calculation>
                        {
                            new Calculation
                            {
                                Name = "FundingLine-3-calc-1",
                                TemplateCalculationId = 7,
                                Type = Common.TemplateMetadata.Enums.CalculationType.Cash
                            },
                            new Calculation
                            {
                                Name = "FundingLine-3-calc-2",
                                TemplateCalculationId = 11,
                                Calculations          = new List <Calculation>
                                {
                                    new Calculation
                                    {
                                        Name = "FundingLine-3-calc-2-calc-1",
                                        TemplateCalculationId = 8,
                                        Type = Common.TemplateMetadata.Enums.CalculationType.Cash
                                    }
                                }
                            },
                            new Calculation
                            {
                                Name = "FundingLine-3-calc-3",
                                TemplateCalculationId = 10,
                                Type = Common.TemplateMetadata.Enums.CalculationType.Cash
                            }
                        }
                    },
                    new FundingLine
                    {
                        TemplateLineId = 789,
                        Name           = "FundingLine-4"
                    },
                }
            };

            _specificationsService.GetSpecificationSummaryById(SpecificationId)
            .Returns(new OkObjectResult(specificationSummary));

            _policiesApiClient.GetFundingTemplateContents(FundingStreamId, FundingPeriodId, TemplateVersion)
            .Returns(new ApiResponse <TemplateMetadataContents>(HttpStatusCode.OK, templateMetadataContents));

            _calculationsApiClient.GetTemplateMapping(SpecificationId, FundingStreamId)
            .Returns(new ApiResponse <TemplateMapping>(HttpStatusCode.OK,
                                                       new TemplateMapping
            {
                FundingStreamId      = FundingStreamId,
                SpecificationId      = SpecificationId,
                TemplateMappingItems = new List <TemplateMappingItem>
                {
                    new TemplateMappingItem
                    {
                        TemplateId    = 7,
                        CalculationId = aValidCalculationId1
                    },
                    new TemplateMappingItem
                    {
                        TemplateId    = 11,
                        CalculationId = aValidCalculationId2
                    },
                    new TemplateMappingItem
                    {
                        TemplateId    = 8,
                        CalculationId = "CalculationIdForTemplateCalculationId2"
                    },
                    new TemplateMappingItem
                    {
                        TemplateId    = 10,
                        CalculationId = aValidCalculationId3
                    }
                }
            }));

            _calculationsApiClient.GetCalculationMetadataForSpecification(SpecificationId)
            .Returns(new ApiResponse <IEnumerable <CalculationMetadata> >(HttpStatusCode.OK,
                                                                          new List <CalculationMetadata>
            {
                new CalculationMetadata
                {
                    SpecificationId = SpecificationId,
                    CalculationId   = aValidCalculationId1,
                    PublishStatus   = CalculationExpectedPublishStatus
                },
                new CalculationMetadata
                {
                    SpecificationId = SpecificationId,
                    CalculationId   = aValidCalculationId2,
                    PublishStatus   = CalculationExpectedPublishStatus
                },
                new CalculationMetadata
                {
                    SpecificationId = SpecificationId,
                    CalculationId   = aValidCalculationId3,
                    PublishStatus   = CalculationExpectedPublishStatus
                }
            }));

            _graphApiClient.GetCircularDependencies(SpecificationId)
            .Returns(new ApiResponse <IEnumerable <Entity <Common.ApiClient.Graph.Models.Calculation> > >(
                         HttpStatusCode.OK, new List <Entity <Common.ApiClient.Graph.Models.Calculation> >()
            {
                new Entity <Common.ApiClient.Graph.Models.Calculation>()
                {
                    Node = new Common.ApiClient.Graph.Models.Calculation()
                    {
                        SpecificationId = SpecificationId,
                        CalculationId   = "CalculationIdForTemplateCalculationId2"
                    }
                }
            }));
        }
        public async Task <IActionResult> GetFundingStructureResultsForProviderAndSpecification(
            [FromRoute] string providerId,
            [FromRoute] string specificationId)
        {
            Guard.ArgumentNotNull(providerId, nameof(providerId));
            Guard.ArgumentNotNull(specificationId, nameof(specificationId));

            ApiResponse <SpecificationSummary> specificationResult = await _specsClient.GetSpecificationSummaryById(specificationId);

            IActionResult specificationErrorResult =
                specificationResult.IsSuccessOrReturnFailureResult("GetFundingStructureResultsForProviderAndSpecification");

            if (specificationErrorResult != null)
            {
                return(specificationErrorResult);
            }

            SpecificationSummary specification = specificationResult.Content;

            // NOTE: This API is designed with the assumption that there is only a single funding stream per specification.
            // The return type will need to change to be funding stream specific if more than one is ever used
            Reference fundingStream = specification.FundingStreams.First();

            string fundingStreamTemplateVersion = specification.TemplateIds[fundingStream.Id];

            Task <ApiResponse <TemplateMetadataDistinctContents> > distictTemplateContentsRequest = _policiesClient.GetDistinctTemplateMetadataContents(fundingStream.Id, specification.FundingPeriod.Id, fundingStreamTemplateVersion);
            Task <ApiResponse <TemplateMapping> > templateMappingRequest = _calculationsClient.GetTemplateMapping(specificationId, fundingStream.Id);
            Task <ApiResponse <Common.ApiClient.Results.Models.ProviderResultResponse> > calculationEngineResultsRequest = _resultsClient.GetProviderResults(providerId, specificationId);
            Task <ApiResponse <IEnumerable <CalculationMetadata> > > calculationMetadataRequest = _calculationsClient.GetCalculationMetadataForSpecification(specificationId);

            await TaskHelper.WhenAllAndThrow(distictTemplateContentsRequest, templateMappingRequest, calculationEngineResultsRequest, calculationMetadataRequest);

            IActionResult distictTemplateContentsErrorResult  = distictTemplateContentsRequest.Result.IsSuccessOrReturnFailureResult(nameof(TemplateMetadataDistinctContents));
            IActionResult templateMappingErrorResult          = templateMappingRequest.Result.IsSuccessOrReturnFailureResult(nameof(TemplateMapping));
            IActionResult calculationEngineResultsErrorResult = calculationEngineResultsRequest.Result.IsSuccessOrReturnFailureResult(nameof(Common.ApiClient.Results.Models.ProviderResultResponse));
            IActionResult calculationMetadataErrorResult      = calculationMetadataRequest.Result.IsSuccessOrReturnFailureResult(nameof(CalculationMetadata));

            if (distictTemplateContentsErrorResult != null)
            {
                return(distictTemplateContentsErrorResult);
            }

            if (templateMappingErrorResult != null)
            {
                return(templateMappingErrorResult);
            }

            if (calculationEngineResultsErrorResult != null)
            {
                return(calculationEngineResultsErrorResult);
            }

            if (calculationMetadataErrorResult != null)
            {
                return(calculationMetadataErrorResult);
            }

            IEnumerable <TemplateMetadataCalculation> templateCalculations = distictTemplateContentsRequest.Result.Content.Calculations;
            IEnumerable <TemplateMetadataFundingLine> templateFundingLines = distictTemplateContentsRequest.Result.Content.FundingLines;

            IEnumerable <TemplateMappingItem> templateMapping = templateMappingRequest.Result.Content.TemplateMappingItems;

            Dictionary <uint, TemplateCalculationResult> calculationResults = GenerateCalculationResults(
                templateMapping,
                templateCalculations,
                calculationEngineResultsRequest.Result.Content.CalculationResults,
                calculationMetadataRequest.Result.Content
                );

            Dictionary <uint, FundingLineResult> fundingLineResults = GenerateFundingLineResults(templateFundingLines,
                                                                                                 calculationEngineResultsRequest.Result.Content.FundingLineResults);

            return(Ok(new ProviderResultForSpecification()
            {
                SpecificationId = specificationId,
                SpecificationName = specification.Name,
                FundingStreamName = fundingStream.Name,
                FundingStreamId = fundingStream.Id,
                CalculationResults = calculationResults,
                FundingLineResults = fundingLineResults,
            }));
        }
        public async Task <TemplateMapping> GetTemplateMapping(string specificationId, string fundingStreamId)
        {
            if (string.IsNullOrWhiteSpace(specificationId))
            {
                throw new ArgumentNullException(nameof(specificationId));
            }

            ApiResponse <Common.ApiClient.Calcs.Models.TemplateMapping> apiResponse = await _apiClient.GetTemplateMapping(specificationId, fundingStreamId);

            return(_mapper.Map <TemplateMapping>(apiResponse?.Content));
        }
Пример #11
0
        public async Task <IActionResult> GetFundingStructure(string fundingStreamId, string fundingPeriodId, string specificationId)
        {
            IActionResult specificationSummaryResult = await _specificationsService.GetSpecificationSummaryById(specificationId);

            if (!(specificationSummaryResult is OkObjectResult))
            {
                return(specificationSummaryResult);
            }

            SpecificationSummary specificationSummary = (specificationSummaryResult as OkObjectResult).Value
                                                        as SpecificationSummary;

            string templateVersion = specificationSummary.TemplateIds.ContainsKey(fundingStreamId)
                ? specificationSummary.TemplateIds[fundingStreamId]
                : null;

            if (templateVersion == null)
            {
                return(new InternalServerErrorResult(
                           $"Specification contains no matching template version for funding stream '{fundingStreamId}'"));
            }

            ApiResponse <TemplateMetadataContents> templateMetadataContentsApiResponse =
                await _policiesResilience.ExecuteAsync(() => _policiesApiClient.GetFundingTemplateContents(fundingStreamId, fundingPeriodId, templateVersion));

            IActionResult templateMetadataContentsApiResponseErrorResult =
                templateMetadataContentsApiResponse.IsSuccessOrReturnFailureResult("GetFundingTemplateContents");

            if (templateMetadataContentsApiResponseErrorResult != null)
            {
                return(templateMetadataContentsApiResponseErrorResult);
            }

            ApiResponse <TemplateMapping> templateMappingResponse =
                await _calculationsResilience.ExecuteAsync(() => _calculationsApiClient.GetTemplateMapping(specificationId, fundingStreamId));

            IActionResult templateMappingResponseErrorResult =
                templateMappingResponse.IsSuccessOrReturnFailureResult("GetTemplateMapping");

            if (templateMappingResponseErrorResult != null)
            {
                return(templateMappingResponseErrorResult);
            }

            ApiResponse <IEnumerable <CalculationMetadata> > calculationMetadata =
                await _calculationsResilience.ExecuteAsync(() => _calculationsApiClient.GetCalculationMetadataForSpecification(specificationId));

            IActionResult calculationMetadataErrorResult =
                calculationMetadata.IsSuccessOrReturnFailureResult("calculationMetadata");

            if (calculationMetadataErrorResult != null)
            {
                return(calculationMetadataErrorResult);
            }

            List <string> calculationIdsWithError = new List <string>();

            ApiResponse <IEnumerable <Entity <Calculation> > > getCircularDependenciesApiResponse =
                await _graphApiClient.GetCircularDependencies(specificationId);

            IActionResult circularDependenciesApiErrorResult =
                getCircularDependenciesApiResponse.IsSuccessOrReturnFailureResult("GetCircularDependencies");

            if (circularDependenciesApiErrorResult == null)
            {
                calculationIdsWithError =
                    getCircularDependenciesApiResponse.Content.Select(calcs => calcs.Node.CalculationId).ToList();
            }

            List <FundingStructureItem> fundingStructures = new List <FundingStructureItem>();

            RecursivelyAddFundingLineToFundingStructure(
                fundingStructures,
                templateMetadataContentsApiResponse.Content.RootFundingLines,
                templateMappingResponse.Content.TemplateMappingItems.ToList(),
                calculationMetadata.Content.ToList(),
                calculationIdsWithError);

            FundingStructure fundingStructure = new FundingStructure
            {
                Items        = fundingStructures,
                LastModified = await GetFundingStructureTimeStamp(fundingStreamId,
                                                                  fundingPeriodId,
                                                                  specificationId)
            };

            return(new OkObjectResult(fundingStructure));
        }
Пример #12
0
        private async Task RefreshFundingStream(Reference fundingStream,
                                                SpecificationSummary specification,
                                                IDictionary <string, Provider> scopedProviders,
                                                IDictionary <string, ProviderCalculationResult> allCalculationResults,
                                                string jobId, Reference author,
                                                string correlationId,
                                                IEnumerable <PublishedProvider> existingPublishedProviders,
                                                string fundingPeriodId)
        {
            TemplateMetadataContents templateMetadataContents = await _policiesService.GetTemplateMetadataContents(fundingStream.Id, specification.FundingPeriod.Id, specification.TemplateIds[fundingStream.Id]);

            if (templateMetadataContents == null)
            {
                _logger.Information($"Unable to locate template meta data contents for funding stream:'{fundingStream.Id}' and template id:'{specification.TemplateIds[fundingStream.Id]}'");
                return;
            }

            IEnumerable <ProfileVariationPointer> variationPointers = await _specificationService.GetProfileVariationPointers(specification.Id) ?? ArraySegment <ProfileVariationPointer> .Empty;

            Dictionary <string, PublishedProvider> publishedProviders = new Dictionary <string, PublishedProvider>();

            foreach (PublishedProvider publishedProvider in existingPublishedProviders)
            {
                if (publishedProvider.Current.FundingStreamId == fundingStream.Id)
                {
                    publishedProviders.Add(publishedProvider.Current.ProviderId, publishedProvider);
                }
            }

            // Create PublishedProvider for providers which don't already have a record (eg ProviderID-FundingStreamId-FundingPeriodId)
            IDictionary <string, PublishedProvider> newProviders = _providerService.GenerateMissingPublishedProviders(scopedProviders.Values, specification, fundingStream, publishedProviders);

            publishedProviders.AddRange(newProviders);

            // Get TemplateMapping for calcs from Calcs API client nuget
            ApiResponse <Common.ApiClient.Calcs.Models.TemplateMapping> calculationMappingResult = await _calculationsApiClientPolicy.ExecuteAsync(() => _calculationsApiClient.GetTemplateMapping(specification.Id, fundingStream.Id));

            if (calculationMappingResult == null)
            {
                throw new Exception($"calculationMappingResult returned null for funding stream {fundingStream.Id}");
            }

            Common.ApiClient.Calcs.Models.TemplateMapping templateMapping = calculationMappingResult.Content;

            _logger.Information("Generating PublishedProviders for refresh");

            // Generate populated data for each provider in this funding line
            IDictionary <string, GeneratedProviderResult> generatedPublishedProviderData;

            try
            {
                generatedPublishedProviderData = _publishedProviderDataGenerator.Generate(templateMetadataContents, templateMapping, scopedProviders.Values, allCalculationResults);
            }
            catch (Exception ex)
            {
                _logger.Error(ex, "Exception during generating provider data");
                throw;
            }
            _logger.Information("Populated PublishedProviders for refresh");

            Dictionary <string, PublishedProvider> publishedProvidersToUpdate = new Dictionary <string, PublishedProvider>();

            Dictionary <string, PublishedProvider> existingPublishedProvidersToUpdate = new Dictionary <string, PublishedProvider>();

            FundingLine[] flattenedTemplateFundingLines = templateMetadataContents.RootFundingLines.Flatten(_ => _.FundingLines).ToArray();

            _logger.Information("Profiling providers for refresh");

            try
            {
                await ProfileProviders(publishedProviders, newProviders, generatedPublishedProviderData);
            }
            catch (Exception ex)
            {
                _logger.Error(ex, "Exception during generating provider profiling");

                if (ex is NonRetriableException)
                {
                    await _jobManagement.UpdateJobStatus(jobId, 0, 0, false, "Refresh job failed during generating provider profiling.");
                }

                throw;
            }

            _logger.Information("Finished profiling providers for refresh");

            _logger.Information("Start snapshots for published provider variations");
            // snapshot the current published providers so any changes aren't reflected when we detect variations later
            _variationService.SnapShot(publishedProviders, fundingStream.Id);
            _logger.Information("Finished snapshots for published provider variations");

            //we need enumerate a readonly cut of this as we add to it in some variations now (for missing providers not in scope)
            Dictionary <string, PublishedProvider> publishedProvidersReadonlyDictionary = publishedProviders.ToDictionary(_ => _.Key, _ => _.Value);

            _logger.Information($"Start getting funding configuration for funding stream '{fundingStream.Id}'");
            // set up the published providers context for error detection laterawait
            FundingConfiguration fundingConfiguration = await _policiesService.GetFundingConfiguration(fundingStream.Id, specification.FundingPeriod.Id);

            _logger.Information($"Retrieved funding stream configuration for '{fundingStream.Id}'");

            PublishedProvidersContext publishedProvidersContext = new PublishedProvidersContext
            {
                ScopedProviders         = scopedProviders.Values,
                SpecificationId         = specification.Id,
                ProviderVersionId       = specification.ProviderVersionId,
                CurrentPublishedFunding = (await _publishingResiliencePolicy.ExecuteAsync(() => _publishedFundingDataService.GetCurrentPublishedFunding(specification.Id, GroupingReason.Payment)))
                                          .Where(x => x.Current.GroupingReason == CalculateFunding.Models.Publishing.GroupingReason.Payment),
                OrganisationGroupResultsData = new Dictionary <string, HashSet <string> >(),
                FundingConfiguration         = fundingConfiguration
            };

            _logger.Information("Starting to process providers for variations and exclusions");

            foreach (KeyValuePair <string, PublishedProvider> publishedProvider in publishedProvidersReadonlyDictionary)
            {
                PublishedProviderVersion publishedProviderVersion = publishedProvider.Value.Current;

                string providerId = publishedProviderVersion.ProviderId;

                // this could be a retry and the key may not exist as the provider has been created as a successor so we need to skip
                if (!generatedPublishedProviderData.ContainsKey(publishedProvider.Key))
                {
                    continue;
                }

                GeneratedProviderResult generatedProviderResult = generatedPublishedProviderData[publishedProvider.Key];

                PublishedProviderExclusionCheckResult exclusionCheckResult =
                    _providerExclusionCheck.ShouldBeExcluded(generatedProviderResult, flattenedTemplateFundingLines);

                if (exclusionCheckResult.ShouldBeExcluded)
                {
                    if (newProviders.ContainsKey(publishedProvider.Key))
                    {
                        newProviders.Remove(publishedProvider.Key);
                        continue;
                    }

                    if (!_fundingLineValueOverride.TryOverridePreviousFundingLineValues(publishedProviderVersion, generatedProviderResult))
                    {
                        //there are no none null payment funding line values and we didn't have to override any previous
                        //version funding lines with a zero amount now they are all null so skip this published provider
                        //the updates check

                        continue;
                    }
                }

                bool publishedProviderUpdated = _publishedProviderDataPopulator.UpdatePublishedProvider(publishedProviderVersion,
                                                                                                        generatedProviderResult,
                                                                                                        scopedProviders[providerId],
                                                                                                        specification.TemplateIds[fundingStream.Id],
                                                                                                        newProviders.ContainsKey(publishedProvider.Key));

                _logger.Verbose($"Published provider '{publishedProvider.Key}' updated: '{publishedProviderUpdated}'");

                //reapply any custom profiles this provider has and internally check for errors
                _reApplyCustomProfiles.ProcessPublishedProvider(publishedProviderVersion);

                // process published provider and detect errors
                await _detection.ProcessPublishedProvider(publishedProvider.Value, publishedProvidersContext);

                if (publishedProviderUpdated && existingPublishedProviders.AnyWithNullCheck())
                {
                    IDictionary <string, PublishedProvider> newPublishedProviders = await _variationService.PrepareVariedProviders(generatedProviderResult.TotalFunding,
                                                                                                                                   publishedProviders,
                                                                                                                                   publishedProvider.Value,
                                                                                                                                   scopedProviders[providerId],
                                                                                                                                   fundingConfiguration?.Variations,
                                                                                                                                   variationPointers,
                                                                                                                                   fundingStream.Id,
                                                                                                                                   specification.ProviderVersionId);

                    if (!newPublishedProviders.IsNullOrEmpty())
                    {
                        newProviders.AddRange(newPublishedProviders);
                    }
                }

                if (!publishedProviderUpdated)
                {
                    continue;
                }

                if (!newProviders.ContainsKey(publishedProvider.Key))
                {
                    existingPublishedProvidersToUpdate.Add(publishedProvider.Key, publishedProvider.Value);
                }

                publishedProvidersToUpdate.Add(publishedProvider.Key, publishedProvider.Value);
            }

            _logger.Information("Finished processing providers for variations and exclusions");

            _logger.Information("Adding additional variation reasons");
            AddInitialPublishVariationReasons(newProviders.Values);
            _logger.Information("Finished adding additional variation reasons");

            _logger.Information("Starting to apply variations");

            if (!(await _variationService.ApplyVariations(publishedProvidersToUpdate, newProviders, specification.Id, jobId)))
            {
                await _jobManagement.UpdateJobStatus(jobId, 0, 0, false, "Refresh job failed with variations errors.");

                throw new NonRetriableException($"Unable to refresh funding. Variations generated {_variationService.ErrorCount} errors. Check log for details");
            }

            _logger.Information("Finished applying variations");

            _logger.Information($"Updating a total of {publishedProvidersToUpdate.Count} published providers");

            if (publishedProvidersToUpdate.Count > 0)
            {
                if (existingPublishedProvidersToUpdate.Count > 0 || newProviders.Count > 0)
                {
                    using (Transaction transaction = _transactionFactory.NewTransaction <RefreshService>())
                    {
                        try
                        {
                            // if any error occurs while updating or indexing then we need to re-index all published providers for consistency
                            transaction.Enroll(async() =>
                            {
                                await _publishedProviderVersionService.CreateReIndexJob(author, correlationId, specification.Id, jobId);
                            });

                            // Save updated PublishedProviders to cosmos and increment version status
                            if (existingPublishedProvidersToUpdate.Count > 0)
                            {
                                _logger.Information($"Saving updates to existing published providers. Total={existingPublishedProvidersToUpdate.Count}");
                                await _publishedProviderStatusUpdateService.UpdatePublishedProviderStatus(existingPublishedProvidersToUpdate.Values, author, PublishedProviderStatus.Updated, jobId, correlationId);

                                _logger.Information("Indexing existing PublishedProviders");
                                await _publishedProviderIndexerService.IndexPublishedProviders(existingPublishedProvidersToUpdate.Values.Select(_ => _.Current));
                            }

                            if (newProviders.Count > 0)
                            {
                                _logger.Information($"Saving new published providers. Total={newProviders.Count}");
                                await _publishedProviderStatusUpdateService.UpdatePublishedProviderStatus(newProviders.Values, author, PublishedProviderStatus.Draft, jobId, correlationId);

                                _logger.Information("Indexing newly added PublishedProviders");
                                await _publishedProviderIndexerService.IndexPublishedProviders(newProviders.Values.Select(_ => _.Current));
                            }

                            transaction.Complete();
                        }
                        catch (Exception ex)
                        {
                            await transaction.Compensate();

                            throw;
                        }
                    }

                    _logger.Information("Creating generate Csv jobs");

                    IGeneratePublishedFundingCsvJobsCreation generateCsvJobs = _generateCsvJobsLocator
                                                                               .GetService(GeneratePublishingCsvJobsCreationAction.Refresh);
                    IEnumerable <string> fundingLineCodes = await _publishedFundingDataService.GetPublishedProviderFundingLines(specification.Id);

                    IEnumerable <string>           fundingStreamIds = Array.Empty <string>();
                    PublishedFundingCsvJobsRequest publishedFundingCsvJobsRequest = new PublishedFundingCsvJobsRequest
                    {
                        SpecificationId  = specification.Id,
                        CorrelationId    = correlationId,
                        User             = author,
                        FundingLineCodes = fundingLineCodes,
                        FundingStreamIds = fundingStreamIds,
                        FundingPeriodId  = fundingPeriodId
                    };
                    await generateCsvJobs.CreateJobs(publishedFundingCsvJobsRequest);
                }
            }
        }