public void ShouldNotExclueProviderIfAtleastOnePaymentFundingLineHaveValue() { string providerId = NewRandomString(); uint templateLineId1 = NewRandomUint(); uint templateLineId2 = NewRandomUint(); GeneratedProviderResult generatedProviderResult = NewGeneratedProviderResult(_ => _.WithFundlines(NewFundingLine(f => f.WithFundingLineType(FundingLineType.Payment).WithTemplateLineId(templateLineId1)), NewFundingLine(f => f.WithFundingLineType(FundingLineType.Payment).WithTemplateLineId(templateLineId2).WithValue(1m))) .WithProvider(NewProvider(p => p.WithProviderId(providerId)))); TemplateFundingLine[] templateFundingLines = new[] { NewTemplateFundingLine(_ => _.WithTemplateLineId(templateLineId1)), NewTemplateFundingLine(_ => _.WithTemplateLineId(templateLineId2)) }; PublishedProviderExclusionCheck exclusionCheck = new PublishedProviderExclusionCheck(); PublishedProviderExclusionCheckResult result = exclusionCheck.ShouldBeExcluded(generatedProviderResult, templateFundingLines); result.ProviderId .Should() .Be(providerId); result.ShouldBeExcluded .Should() .BeFalse(); }
public void UpdatesGeneratedResultIfPreviousVersionFundingLineIsPresent() { uint templateLineId = NewRandomUint(); FundingLine generatedResultFundingLine = NewFundingLine(fl => fl .WithTemplateLineId(templateLineId) .WithFundingLineType(FundingLineType.Payment)); GeneratedProviderResult generatedProviderResult = NewGeneratedProviderResult(_ => _.WithFundlines(NewFundingLine(fl => fl .WithTemplateLineId(templateLineId) .WithFundingLineType(FundingLineType.Payment)))); PublishedProviderVersion publishedProviderVersion = NewPublishedProviderVersion(_ => _.WithFundingLines(NewFundingLine(fl => fl.WithTemplateLineId(templateLineId) .WithValue(98990M)))); bool hasZeroedFundingLine = _fundingLineValueOverride.TryOverridePreviousFundingLineValues(publishedProviderVersion, generatedProviderResult); hasZeroedFundingLine .Should() .BeTrue(); generatedResultFundingLine.Value .Should() .BeNull(); }
public void ThrowsExceptionWhenContentGeneratorReturnsNull() { // Arrange TemplateMetadataContents templateMetadataContents = Substitute.For <TemplateMetadataContents>(); TemplateMapping templateMapping = Substitute.For <TemplateMapping>(); IPublishedProviderContentsGenerator publishedProviderContentsGenerator = Substitute.For <IPublishedProviderContentsGenerator>(); Dictionary <string, GeneratedProviderResult> generatedPublishedProviderData = new Dictionary <string, GeneratedProviderResult>(); List <PublishedProvider> publishedProvidersToUpdate = new List <PublishedProvider>(); GeneratedProviderResult generatedProviderResult = new GeneratedProviderResult(); PublishedProviderVersion publishedProviderVersion = NewPublishedProviderVersion(providerVersion => providerVersion .WithProviderId(ProviderVersionProviderId) .WithFundingPeriodId(ProviderVersionFundingPeriodId) .WithFundingStreamId(ProviderVersionFundingStreamId) .WithVersion(ProviderVersionVersion)); PublishedProvider publishedProvider = NewPublishedProvider(provider => provider.WithCurrent(publishedProviderVersion)); generatedPublishedProviderData.Add(key, generatedProviderResult); publishedProvidersToUpdate.Add(publishedProvider); // Act Func <Task> invocation = async() => await _publishedProviderContentPersistanceService.SavePublishedProviderContents( templateMetadataContents, templateMapping, publishedProvidersToUpdate, publishedProviderContentsGenerator); // Assert ThenExceptionShouldBeThrown($"Generator failed to generate content for published provider version with id: '{publishedProviderVersion.Id}'", invocation); }
public async Task <int> Generate(FeedOptions options) { IEnumerable <Provider> records = GetRecords(options.InputFilePath); foreach (Provider provider in records) { PublishedProviderVersion publishedProviderVersion = GetPublishedProviderVersion(provider); Common.TemplateMetadata.Models.TemplateMetadataContents templateMetadataContents = GetProviderTemplateMetadataContents(); TemplateMapping templateMapping = GetTemplateMapping(); GeneratedProviderResult generatedProviderResult = GetGeneratedProviderResult(provider); string generatedProviderDocumentContent = _publishedProviderContentsGenerator.GenerateContents(publishedProviderVersion, templateMetadataContents, templateMapping); PublishProviderDocument(options, publishedProviderVersion, generatedProviderDocumentContent); } int fundingIndex = 0; foreach (IGrouping <(string, string, string), Provider> groupingKey in records.GroupBy(provider => (provider.LACode, provider.MajorVersionNo, provider.AllocationID))) { OrganisationGroupLookupParameters organisationGroupLookupParameters = new OrganisationGroupLookupParameters { OrganisationGroupTypeCode = GetOrganisationGroupTypeCode(groupingKey.Key.Item3), IdentifierValue = groupingKey.Key.Item1, GroupTypeIdentifier = CalculateFunding.Common.ApiClient.Policies.Models.OrganisationGroupTypeIdentifier.LACode, ProviderVersionId = options.ProviderVersion }; IEnumerable <ProviderApiClient> apiClientProviders = GetApiClientProviders(groupingKey); TargetOrganisationGroup targetOrganisationGroup = null; try { targetOrganisationGroup = await organisationGroupTargetProviderLookup.GetTargetProviderDetails(organisationGroupLookupParameters, Common.ApiClient.Policies.Models.GroupingReason.Payment, apiClientProviders); } catch (Exception ex) { string message = $"Could not find provider with ID:{organisationGroupLookupParameters.IdentifierValue} with error message {ex}"; _logger.Error(message); Console.WriteLine(message); continue; } PublishedFundingVersion publishedFundingVersion = GetPublishedFundingVersion(groupingKey, targetOrganisationGroup, fundingIndex); Common.TemplateMetadata.Models.TemplateMetadataContents templateMetadataContents = GetFundingTemplateMetadataContents(); string generatedFundingDocumentContent = _publishedFundingContentsGenerator.GenerateContents(publishedFundingVersion, templateMetadataContents); PublishFundingDocument(options, publishedFundingVersion, generatedFundingDocumentContent); fundingIndex++; } _logger.Information("NAV Data generation completed."); return(1); }
public PublishedProviderExclusionCheckResult ShouldBeExcluded( GeneratedProviderResult generatedProviderResult, Common.TemplateMetadata.Models.FundingLine[] flattenedTemplateFundingLines) { IEnumerable <FundingLine> paymentFundingLines = generatedProviderResult.FundingLines.Where( x => x.Type == FundingLineType.Payment && flattenedTemplateFundingLines.Any(y => y.TemplateLineId == x.TemplateLineId && y.Type == Common.TemplateMetadata.Enums.FundingLineType.Payment)); bool shouldBeExcluded = !paymentFundingLines.Any() || paymentFundingLines.All(c => c.Value == null); return(new PublishedProviderExclusionCheckResult(generatedProviderResult.Provider.ProviderId, shouldBeExcluded)); }
/// <summary> /// Updates the given data on the Published Provider. /// This method is responsible for applying the data passed into on to the PublishedProviderVersion and returning if the PublishedProviderVersion has been updated /// </summary> /// <param name="publishedProviderVersion">Published Provider Version</param> /// <param name="generatedProviderResult">Funding lines and profiling information, calculations, reference data</param> /// <param name="provider">Core provider information</param> /// <param name="templateVersion">The template version used for the specification and provider</param> /// <param name="variationForProvider"></param> /// <returns>True when the PublishedProviderVersion has been updated, false if not</returns> public bool UpdatePublishedProvider(PublishedProviderVersion publishedProviderVersion, GeneratedProviderResult generatedProviderResult, Provider provider, string templateVersion, bool isNewProvider) { Guard.ArgumentNotNull(publishedProviderVersion, nameof(publishedProviderVersion)); Guard.ArgumentNotNull(generatedProviderResult, nameof(generatedProviderResult)); Guard.ArgumentNotNull(provider, nameof(provider)); Guard.IsNullOrWhiteSpace(templateVersion, nameof(templateVersion)); Provider mappedProvider = _providerMapper.Map <Provider>(provider); PublishedProviderVersionComparer publishedProviderVersionComparer = new PublishedProviderVersionComparer(); // if this is a new provider then it will always need to be updated bool equal = !isNewProvider; if (equal) { equal = publishedProviderVersionComparer.Equals(publishedProviderVersion, new PublishedProviderVersion { FundingLines = generatedProviderResult.FundingLines, Calculations = generatedProviderResult.Calculations, ReferenceData = generatedProviderResult.ReferenceData, TemplateVersion = templateVersion, Provider = mappedProvider }); if (!equal) { _logger.Information($"changes for new published provider version : {publishedProviderVersion.Id} : {publishedProviderVersionComparer.Variances.AsJson()}"); } } publishedProviderVersion.FundingLines = generatedProviderResult.FundingLines; publishedProviderVersion.Calculations = generatedProviderResult.Calculations; publishedProviderVersion.ReferenceData = generatedProviderResult.ReferenceData; publishedProviderVersion.TemplateVersion = templateVersion; publishedProviderVersion.TotalFunding = generatedProviderResult.TotalFunding; publishedProviderVersion.Provider = mappedProvider; return(!equal); }
public void SetUp() { _templateVersion = NewRandomString(); _publishedProviderVersion = CreateProviderVersion(_templateVersion); _generatedProviderResult = CreateGeneratedProviderResult(); _provider = CreateProvider(); _publishedProviderVersionForMapping = (PublishedProviderVersion)_publishedProviderVersion.Clone(); _mapper = CreateMapper(); _mapper .Map <Provider>(_provider) .Returns(_publishedProviderVersionForMapping.Provider); _logger = CreateLogger(); _publishedProviderDataPopulator = new PublishedProviderDataPopulator(_mapper, _logger); }
public void DoesNotUpdateGeneratedResultIfPreviousVersionHasNoMatchingFundingLine() { uint templateLineId = NewRandomUint(); GeneratedProviderResult generatedProviderResult = NewGeneratedProviderResult(_ => _.WithFundlines(NewFundingLine(fl => fl .WithTemplateLineId(templateLineId) .WithFundingLineType(FundingLineType.Payment)))); PublishedProviderVersion publishedProviderVersion = NewPublishedProviderVersion(_ => _.WithFundingLines(NewFundingLine())); bool hasZeroedFundingLine = _fundingLineValueOverride.TryOverridePreviousFundingLineValues(publishedProviderVersion, generatedProviderResult); hasZeroedFundingLine .Should() .BeFalse(); }
public async Task <int> Generate(FeedOptions options) { IEnumerable <Provider> records = GetRecords(options.InputFilePath); foreach (Provider provider in records) { PublishedProviderVersion publishedProviderVersion = GetPublishedProviderVersion(provider); Common.TemplateMetadata.Models.TemplateMetadataContents templateMetadataContents = GetProviderTemplateMetadataContents(); TemplateMapping templateMapping = GetTemplateMapping(); GeneratedProviderResult generatedProviderResult = GetGeneratedProviderResult(provider); string generatedProviderDocumentContent = _publishedProviderContentsGenerator.GenerateContents(publishedProviderVersion, templateMetadataContents, templateMapping); PublishProviderDocument(options, publishedProviderVersion, generatedProviderDocumentContent); } int fundingIndex = 0; foreach (IGrouping <string, Provider> laGroup in records.GroupBy(x => x.ProviderLaCode)) { OrganisationGroupLookupParameters organisationGroupLookupParameters = new OrganisationGroupLookupParameters { OrganisationGroupTypeCode = Common.ApiClient.Policies.Models.OrganisationGroupTypeCode.LocalAuthority, IdentifierValue = laGroup.Key, GroupTypeIdentifier = Common.ApiClient.Policies.Models.OrganisationGroupTypeIdentifier.LACode, ProviderVersionId = options.ProviderVersion }; IEnumerable <ProviderApiClient> apiClientProviders = GetApiClientProviders(laGroup); TargetOrganisationGroup targetOrganisationGroup = await organisationGroupTargetProviderLookup.GetTargetProviderDetails(organisationGroupLookupParameters, Common.ApiClient.Policies.Models.GroupingReason.Payment, apiClientProviders); PublishedFundingVersion publishedFundingVersion = GetPublishedFundingVersion(laGroup, targetOrganisationGroup, fundingIndex); Common.TemplateMetadata.Models.TemplateMetadataContents templateMetadataContents = GetFundingTemplateMetadataContents(); string generatedFundingDocumentContent = _publishedFundingContentsGenerator.GenerateContents(publishedFundingVersion, templateMetadataContents); PublishFundingDocument(options, publishedFundingVersion, generatedFundingDocumentContent); fundingIndex++; } _logger.Information("NAV Data generation completed."); return(1); }
public bool TryOverridePreviousFundingLineValues(PublishedProviderVersion publishedProviderVersion, GeneratedProviderResult generatedProviderResult) { var haveZeroedNullPaymentLine = false; foreach (FundingLine fundingLine in generatedProviderResult.FundingLines?.Where(_ => _.Type == FundingLineType.Payment) ?? new FundingLine[0]) { FundingLine previousFundingLineVersion = publishedProviderVersion.FundingLines?.SingleOrDefault(_ => _.TemplateLineId == fundingLine.TemplateLineId); if (previousFundingLineVersion == null) { continue; } fundingLine.Value = 0M; haveZeroedNullPaymentLine = true; } return(haveZeroedNullPaymentLine); }
/// <summary> /// Generate Funding Lines, Calculations and Reference Data for a funding stream for all in scope providers /// </summary> /// <param name="templateMetadata">Template Metadata</param> /// <param name="templateMapping">Template Mapping</param> /// <param name="scopedProviders">Scoped providers for a specification</param> /// <param name="calculationResults">Calculation Results</param> /// <returns>Dictionary of Generated Provider Results, keyed on ProviderId</returns> public IDictionary <string, GeneratedProviderResult> Generate(TemplateMetadataContents templateMetadata, Common.ApiClient.Calcs.Models.TemplateMapping templateMapping, IEnumerable <Provider> scopedProviders, IDictionary <string, ProviderCalculationResult> calculationResults) { Guard.ArgumentNotNull(templateMetadata, nameof(templateMetadata)); Guard.ArgumentNotNull(templateMapping, nameof(templateMapping)); Guard.ArgumentNotNull(calculationResults, nameof(calculationResults)); ConcurrentDictionary <string, GeneratedProviderResult> results = new ConcurrentDictionary <string, GeneratedProviderResult>(); TimeSpan loggingPeriod = TimeSpan.FromMinutes(5); using (new Timer( _ => _logger.Information($"{results.Count}: Published Providers processed."), null, loggingPeriod, loggingPeriod)) Parallel.ForEach(scopedProviders, new ParallelOptions { MaxDegreeOfParallelism = 15 }, (provider) => { GeneratedProviderResult generatedProviderResult = new GeneratedProviderResult(); ProviderCalculationResult calculationResultsForProvider = null; calculationResults.TryGetValue(provider.ProviderId, out calculationResultsForProvider); if (calculationResultsForProvider != null) { GeneratorModels.FundingValue fundingValue = _fundingLineTotalAggregator.GenerateTotals(templateMetadata, templateMapping.TemplateMappingItems.ToDictionary(_ => _.TemplateId), calculationResultsForProvider.Results.ToDictionary(_ => _.Id)); // Get funding lines IEnumerable <GeneratorModels.FundingLine> fundingLines = fundingValue.FundingLines?.Flatten(_ => _.FundingLines) ?? new GeneratorModels.FundingLine[0]; Dictionary <uint, GeneratorModels.FundingLine> uniqueFundingLine = new Dictionary <uint, GeneratorModels.FundingLine>(); generatedProviderResult.FundingLines = fundingLines.Where(_ => uniqueFundingLine.TryAdd(_.TemplateLineId, _)).Select(_ => { return(_mapper.Map <FundingLine>(_)); }).ToList(); // Set total funding generatedProviderResult.TotalFunding = generatedProviderResult.FundingLines .Sum(p => { return(p.Type == FundingLineType.Payment ? p.Value : 0); }); Dictionary <uint, GeneratorModels.Calculation> uniqueCalculations = new Dictionary <uint, GeneratorModels.Calculation>(); // Get calculations IEnumerable <GeneratorModels.Calculation> fundingCalculations = uniqueFundingLine.Values?.SelectMany(_ => _.Calculations.Flatten(calc => calc.Calculations)) ?? new GeneratorModels.Calculation[0]; generatedProviderResult.Calculations = fundingCalculations.Where(_ => uniqueCalculations.TryAdd(_.TemplateCalculationId, _)).Select(_ => { return(_mapper.Map <FundingCalculation>(_)); }).ToList(); // Set Provider information generatedProviderResult.Provider = _mapper.Map <Provider>(provider); results.TryAdd(provider.ProviderId, generatedProviderResult); } }); return(results); }
private async Task RefreshFundingStream(Reference fundingStream, SpecificationSummary specification, IDictionary <string, Provider> scopedProviders, IDictionary <string, ProviderCalculationResult> allCalculationResults, string jobId, Reference author, string correlationId, IEnumerable <PublishedProvider> existingPublishedProviders, string fundingPeriodId) { TemplateMetadataContents templateMetadataContents = await _policiesService.GetTemplateMetadataContents(fundingStream.Id, specification.FundingPeriod.Id, specification.TemplateIds[fundingStream.Id]); if (templateMetadataContents == null) { _logger.Information($"Unable to locate template meta data contents for funding stream:'{fundingStream.Id}' and template id:'{specification.TemplateIds[fundingStream.Id]}'"); return; } IEnumerable <ProfileVariationPointer> variationPointers = await _specificationService.GetProfileVariationPointers(specification.Id) ?? ArraySegment <ProfileVariationPointer> .Empty; Dictionary <string, PublishedProvider> publishedProviders = new Dictionary <string, PublishedProvider>(); foreach (PublishedProvider publishedProvider in existingPublishedProviders) { if (publishedProvider.Current.FundingStreamId == fundingStream.Id) { publishedProviders.Add(publishedProvider.Current.ProviderId, publishedProvider); } } // Create PublishedProvider for providers which don't already have a record (eg ProviderID-FundingStreamId-FundingPeriodId) IDictionary <string, PublishedProvider> newProviders = _providerService.GenerateMissingPublishedProviders(scopedProviders.Values, specification, fundingStream, publishedProviders); publishedProviders.AddRange(newProviders); // Get TemplateMapping for calcs from Calcs API client nuget ApiResponse <Common.ApiClient.Calcs.Models.TemplateMapping> calculationMappingResult = await _calculationsApiClientPolicy.ExecuteAsync(() => _calculationsApiClient.GetTemplateMapping(specification.Id, fundingStream.Id)); if (calculationMappingResult == null) { throw new Exception($"calculationMappingResult returned null for funding stream {fundingStream.Id}"); } Common.ApiClient.Calcs.Models.TemplateMapping templateMapping = calculationMappingResult.Content; _logger.Information("Generating PublishedProviders for refresh"); // Generate populated data for each provider in this funding line IDictionary <string, GeneratedProviderResult> generatedPublishedProviderData; try { generatedPublishedProviderData = _publishedProviderDataGenerator.Generate(templateMetadataContents, templateMapping, scopedProviders.Values, allCalculationResults); } catch (Exception ex) { _logger.Error(ex, "Exception during generating provider data"); throw; } _logger.Information("Populated PublishedProviders for refresh"); Dictionary <string, PublishedProvider> publishedProvidersToUpdate = new Dictionary <string, PublishedProvider>(); Dictionary <string, PublishedProvider> existingPublishedProvidersToUpdate = new Dictionary <string, PublishedProvider>(); FundingLine[] flattenedTemplateFundingLines = templateMetadataContents.RootFundingLines.Flatten(_ => _.FundingLines).ToArray(); _logger.Information("Profiling providers for refresh"); try { await ProfileProviders(publishedProviders, newProviders, generatedPublishedProviderData); } catch (Exception ex) { _logger.Error(ex, "Exception during generating provider profiling"); if (ex is NonRetriableException) { await _jobManagement.UpdateJobStatus(jobId, 0, 0, false, "Refresh job failed during generating provider profiling."); } throw; } _logger.Information("Finished profiling providers for refresh"); _logger.Information("Start snapshots for published provider variations"); // snapshot the current published providers so any changes aren't reflected when we detect variations later _variationService.SnapShot(publishedProviders, fundingStream.Id); _logger.Information("Finished snapshots for published provider variations"); //we need enumerate a readonly cut of this as we add to it in some variations now (for missing providers not in scope) Dictionary <string, PublishedProvider> publishedProvidersReadonlyDictionary = publishedProviders.ToDictionary(_ => _.Key, _ => _.Value); _logger.Information($"Start getting funding configuration for funding stream '{fundingStream.Id}'"); // set up the published providers context for error detection laterawait FundingConfiguration fundingConfiguration = await _policiesService.GetFundingConfiguration(fundingStream.Id, specification.FundingPeriod.Id); _logger.Information($"Retrieved funding stream configuration for '{fundingStream.Id}'"); PublishedProvidersContext publishedProvidersContext = new PublishedProvidersContext { ScopedProviders = scopedProviders.Values, SpecificationId = specification.Id, ProviderVersionId = specification.ProviderVersionId, CurrentPublishedFunding = (await _publishingResiliencePolicy.ExecuteAsync(() => _publishedFundingDataService.GetCurrentPublishedFunding(specification.Id, GroupingReason.Payment))) .Where(x => x.Current.GroupingReason == CalculateFunding.Models.Publishing.GroupingReason.Payment), OrganisationGroupResultsData = new Dictionary <string, HashSet <string> >(), FundingConfiguration = fundingConfiguration }; _logger.Information("Starting to process providers for variations and exclusions"); foreach (KeyValuePair <string, PublishedProvider> publishedProvider in publishedProvidersReadonlyDictionary) { PublishedProviderVersion publishedProviderVersion = publishedProvider.Value.Current; string providerId = publishedProviderVersion.ProviderId; // this could be a retry and the key may not exist as the provider has been created as a successor so we need to skip if (!generatedPublishedProviderData.ContainsKey(publishedProvider.Key)) { continue; } GeneratedProviderResult generatedProviderResult = generatedPublishedProviderData[publishedProvider.Key]; PublishedProviderExclusionCheckResult exclusionCheckResult = _providerExclusionCheck.ShouldBeExcluded(generatedProviderResult, flattenedTemplateFundingLines); if (exclusionCheckResult.ShouldBeExcluded) { if (newProviders.ContainsKey(publishedProvider.Key)) { newProviders.Remove(publishedProvider.Key); continue; } if (!_fundingLineValueOverride.TryOverridePreviousFundingLineValues(publishedProviderVersion, generatedProviderResult)) { //there are no none null payment funding line values and we didn't have to override any previous //version funding lines with a zero amount now they are all null so skip this published provider //the updates check continue; } } bool publishedProviderUpdated = _publishedProviderDataPopulator.UpdatePublishedProvider(publishedProviderVersion, generatedProviderResult, scopedProviders[providerId], specification.TemplateIds[fundingStream.Id], newProviders.ContainsKey(publishedProvider.Key)); _logger.Verbose($"Published provider '{publishedProvider.Key}' updated: '{publishedProviderUpdated}'"); //reapply any custom profiles this provider has and internally check for errors _reApplyCustomProfiles.ProcessPublishedProvider(publishedProviderVersion); // process published provider and detect errors await _detection.ProcessPublishedProvider(publishedProvider.Value, publishedProvidersContext); if (publishedProviderUpdated && existingPublishedProviders.AnyWithNullCheck()) { IDictionary <string, PublishedProvider> newPublishedProviders = await _variationService.PrepareVariedProviders(generatedProviderResult.TotalFunding, publishedProviders, publishedProvider.Value, scopedProviders[providerId], fundingConfiguration?.Variations, variationPointers, fundingStream.Id, specification.ProviderVersionId); if (!newPublishedProviders.IsNullOrEmpty()) { newProviders.AddRange(newPublishedProviders); } } if (!publishedProviderUpdated) { continue; } if (!newProviders.ContainsKey(publishedProvider.Key)) { existingPublishedProvidersToUpdate.Add(publishedProvider.Key, publishedProvider.Value); } publishedProvidersToUpdate.Add(publishedProvider.Key, publishedProvider.Value); } _logger.Information("Finished processing providers for variations and exclusions"); _logger.Information("Adding additional variation reasons"); AddInitialPublishVariationReasons(newProviders.Values); _logger.Information("Finished adding additional variation reasons"); _logger.Information("Starting to apply variations"); if (!(await _variationService.ApplyVariations(publishedProvidersToUpdate, newProviders, specification.Id, jobId))) { await _jobManagement.UpdateJobStatus(jobId, 0, 0, false, "Refresh job failed with variations errors."); throw new NonRetriableException($"Unable to refresh funding. Variations generated {_variationService.ErrorCount} errors. Check log for details"); } _logger.Information("Finished applying variations"); _logger.Information($"Updating a total of {publishedProvidersToUpdate.Count} published providers"); if (publishedProvidersToUpdate.Count > 0) { if (existingPublishedProvidersToUpdate.Count > 0 || newProviders.Count > 0) { using (Transaction transaction = _transactionFactory.NewTransaction <RefreshService>()) { try { // if any error occurs while updating or indexing then we need to re-index all published providers for consistency transaction.Enroll(async() => { await _publishedProviderVersionService.CreateReIndexJob(author, correlationId, specification.Id, jobId); }); // Save updated PublishedProviders to cosmos and increment version status if (existingPublishedProvidersToUpdate.Count > 0) { _logger.Information($"Saving updates to existing published providers. Total={existingPublishedProvidersToUpdate.Count}"); await _publishedProviderStatusUpdateService.UpdatePublishedProviderStatus(existingPublishedProvidersToUpdate.Values, author, PublishedProviderStatus.Updated, jobId, correlationId); _logger.Information("Indexing existing PublishedProviders"); await _publishedProviderIndexerService.IndexPublishedProviders(existingPublishedProvidersToUpdate.Values.Select(_ => _.Current)); } if (newProviders.Count > 0) { _logger.Information($"Saving new published providers. Total={newProviders.Count}"); await _publishedProviderStatusUpdateService.UpdatePublishedProviderStatus(newProviders.Values, author, PublishedProviderStatus.Draft, jobId, correlationId); _logger.Information("Indexing newly added PublishedProviders"); await _publishedProviderIndexerService.IndexPublishedProviders(newProviders.Values.Select(_ => _.Current)); } transaction.Complete(); } catch (Exception ex) { await transaction.Compensate(); throw; } } _logger.Information("Creating generate Csv jobs"); IGeneratePublishedFundingCsvJobsCreation generateCsvJobs = _generateCsvJobsLocator .GetService(GeneratePublishingCsvJobsCreationAction.Refresh); IEnumerable <string> fundingLineCodes = await _publishedFundingDataService.GetPublishedProviderFundingLines(specification.Id); IEnumerable <string> fundingStreamIds = Array.Empty <string>(); PublishedFundingCsvJobsRequest publishedFundingCsvJobsRequest = new PublishedFundingCsvJobsRequest { SpecificationId = specification.Id, CorrelationId = correlationId, User = author, FundingLineCodes = fundingLineCodes, FundingStreamIds = fundingStreamIds, FundingPeriodId = fundingPeriodId }; await generateCsvJobs.CreateJobs(publishedFundingCsvJobsRequest); } } }