public void ShouldNotExclueProviderIfAtleastOnePaymentFundingLineHaveValue()
{
string providerId = NewRandomString();
uint templateLineId1 = NewRandomUint();
uint templateLineId2 = NewRandomUint();
GeneratedProviderResult generatedProviderResult = NewGeneratedProviderResult(_ =>
_.WithFundlines(NewFundingLine(f => f.WithFundingLineType(FundingLineType.Payment).WithTemplateLineId(templateLineId1)),
NewFundingLine(f => f.WithFundingLineType(FundingLineType.Payment).WithTemplateLineId(templateLineId2).WithValue(1m)))
.WithProvider(NewProvider(p => p.WithProviderId(providerId))));
TemplateFundingLine[] templateFundingLines = new[]
{
NewTemplateFundingLine(_ => _.WithTemplateLineId(templateLineId1)),
NewTemplateFundingLine(_ => _.WithTemplateLineId(templateLineId2))
};
PublishedProviderExclusionCheck exclusionCheck = new PublishedProviderExclusionCheck();
PublishedProviderExclusionCheckResult result = exclusionCheck.ShouldBeExcluded(generatedProviderResult, templateFundingLines);
result.ProviderId
.Should()
.Be(providerId);
result.ShouldBeExcluded
.Should()
.BeFalse();
}
public void UpdatesGeneratedResultIfPreviousVersionFundingLineIsPresent()
{
uint templateLineId = NewRandomUint();
FundingLine generatedResultFundingLine = NewFundingLine(fl => fl
.WithTemplateLineId(templateLineId)
.WithFundingLineType(FundingLineType.Payment));
GeneratedProviderResult generatedProviderResult = NewGeneratedProviderResult(_ =>
_.WithFundlines(NewFundingLine(fl => fl
.WithTemplateLineId(templateLineId)
.WithFundingLineType(FundingLineType.Payment))));
PublishedProviderVersion publishedProviderVersion = NewPublishedProviderVersion(_ =>
_.WithFundingLines(NewFundingLine(fl => fl.WithTemplateLineId(templateLineId)
.WithValue(98990M))));
bool hasZeroedFundingLine = _fundingLineValueOverride.TryOverridePreviousFundingLineValues(publishedProviderVersion,
generatedProviderResult);
hasZeroedFundingLine
.Should()
.BeTrue();
generatedResultFundingLine.Value
.Should()
.BeNull();
}
public void ThrowsExceptionWhenContentGeneratorReturnsNull()
{
// Arrange
TemplateMetadataContents templateMetadataContents = Substitute.For <TemplateMetadataContents>();
TemplateMapping templateMapping = Substitute.For <TemplateMapping>();
IPublishedProviderContentsGenerator publishedProviderContentsGenerator = Substitute.For <IPublishedProviderContentsGenerator>();
Dictionary <string, GeneratedProviderResult> generatedPublishedProviderData = new Dictionary <string, GeneratedProviderResult>();
List <PublishedProvider> publishedProvidersToUpdate = new List <PublishedProvider>();
GeneratedProviderResult generatedProviderResult = new GeneratedProviderResult();
PublishedProviderVersion publishedProviderVersion = NewPublishedProviderVersion(providerVersion => providerVersion
.WithProviderId(ProviderVersionProviderId)
.WithFundingPeriodId(ProviderVersionFundingPeriodId)
.WithFundingStreamId(ProviderVersionFundingStreamId)
.WithVersion(ProviderVersionVersion));
PublishedProvider publishedProvider = NewPublishedProvider(provider => provider.WithCurrent(publishedProviderVersion));
generatedPublishedProviderData.Add(key, generatedProviderResult);
publishedProvidersToUpdate.Add(publishedProvider);
// Act
Func <Task> invocation = async() => await _publishedProviderContentPersistanceService.SavePublishedProviderContents(
templateMetadataContents, templateMapping, publishedProvidersToUpdate, publishedProviderContentsGenerator);
// Assert
ThenExceptionShouldBeThrown($"Generator failed to generate content for published provider version with id: '{publishedProviderVersion.Id}'", invocation);
}
public async Task <int> Generate(FeedOptions options)
{
IEnumerable <Provider> records = GetRecords(options.InputFilePath);
foreach (Provider provider in records)
{
PublishedProviderVersion publishedProviderVersion = GetPublishedProviderVersion(provider);
Common.TemplateMetadata.Models.TemplateMetadataContents templateMetadataContents = GetProviderTemplateMetadataContents();
TemplateMapping templateMapping = GetTemplateMapping();
GeneratedProviderResult generatedProviderResult = GetGeneratedProviderResult(provider);
string generatedProviderDocumentContent = _publishedProviderContentsGenerator.GenerateContents(publishedProviderVersion, templateMetadataContents, templateMapping);
PublishProviderDocument(options, publishedProviderVersion, generatedProviderDocumentContent);
}
int fundingIndex = 0;
foreach (IGrouping <(string, string, string), Provider> groupingKey in records.GroupBy(provider => (provider.LACode, provider.MajorVersionNo, provider.AllocationID)))
{
OrganisationGroupLookupParameters organisationGroupLookupParameters = new OrganisationGroupLookupParameters
{
OrganisationGroupTypeCode = GetOrganisationGroupTypeCode(groupingKey.Key.Item3),
IdentifierValue = groupingKey.Key.Item1,
GroupTypeIdentifier = CalculateFunding.Common.ApiClient.Policies.Models.OrganisationGroupTypeIdentifier.LACode,
ProviderVersionId = options.ProviderVersion
};
IEnumerable <ProviderApiClient> apiClientProviders = GetApiClientProviders(groupingKey);
TargetOrganisationGroup targetOrganisationGroup = null;
try
{
targetOrganisationGroup = await organisationGroupTargetProviderLookup.GetTargetProviderDetails(organisationGroupLookupParameters, Common.ApiClient.Policies.Models.GroupingReason.Payment, apiClientProviders);
}
catch (Exception ex)
{
string message = $"Could not find provider with ID:{organisationGroupLookupParameters.IdentifierValue} with error message {ex}";
_logger.Error(message);
Console.WriteLine(message);
continue;
}
PublishedFundingVersion publishedFundingVersion = GetPublishedFundingVersion(groupingKey, targetOrganisationGroup, fundingIndex);
Common.TemplateMetadata.Models.TemplateMetadataContents templateMetadataContents = GetFundingTemplateMetadataContents();
string generatedFundingDocumentContent = _publishedFundingContentsGenerator.GenerateContents(publishedFundingVersion, templateMetadataContents);
PublishFundingDocument(options, publishedFundingVersion, generatedFundingDocumentContent);
fundingIndex++;
}
_logger.Information("NAV Data generation completed.");
return(1);
}
public PublishedProviderExclusionCheckResult ShouldBeExcluded(
GeneratedProviderResult generatedProviderResult,
Common.TemplateMetadata.Models.FundingLine[] flattenedTemplateFundingLines)
{
IEnumerable <FundingLine> paymentFundingLines = generatedProviderResult.FundingLines.Where(
x => x.Type == FundingLineType.Payment &&
flattenedTemplateFundingLines.Any(y => y.TemplateLineId == x.TemplateLineId && y.Type == Common.TemplateMetadata.Enums.FundingLineType.Payment));
bool shouldBeExcluded = !paymentFundingLines.Any() || paymentFundingLines.All(c => c.Value == null);
return(new PublishedProviderExclusionCheckResult(generatedProviderResult.Provider.ProviderId, shouldBeExcluded));
}
/// <summary>
/// Updates the given data on the Published Provider.
/// This method is responsible for applying the data passed into on to the PublishedProviderVersion and returning if the PublishedProviderVersion has been updated
/// </summary>
/// <param name="publishedProviderVersion">Published Provider Version</param>
/// <param name="generatedProviderResult">Funding lines and profiling information, calculations, reference data</param>
/// <param name="provider">Core provider information</param>
/// <param name="templateVersion">The template version used for the specification and provider</param>
/// <param name="variationForProvider"></param>
/// <returns>True when the PublishedProviderVersion has been updated, false if not</returns>
public bool UpdatePublishedProvider(PublishedProviderVersion publishedProviderVersion,
GeneratedProviderResult generatedProviderResult,
Provider provider,
string templateVersion,
bool isNewProvider)
{
Guard.ArgumentNotNull(publishedProviderVersion, nameof(publishedProviderVersion));
Guard.ArgumentNotNull(generatedProviderResult, nameof(generatedProviderResult));
Guard.ArgumentNotNull(provider, nameof(provider));
Guard.IsNullOrWhiteSpace(templateVersion, nameof(templateVersion));
Provider mappedProvider = _providerMapper.Map <Provider>(provider);
PublishedProviderVersionComparer publishedProviderVersionComparer = new PublishedProviderVersionComparer();
// if this is a new provider then it will always need to be updated
bool equal = !isNewProvider;
if (equal)
{
equal = publishedProviderVersionComparer.Equals(publishedProviderVersion, new PublishedProviderVersion
{
FundingLines = generatedProviderResult.FundingLines,
Calculations = generatedProviderResult.Calculations,
ReferenceData = generatedProviderResult.ReferenceData,
TemplateVersion = templateVersion,
Provider = mappedProvider
});
if (!equal)
{
_logger.Information($"changes for new published provider version : {publishedProviderVersion.Id} : {publishedProviderVersionComparer.Variances.AsJson()}");
}
}
publishedProviderVersion.FundingLines = generatedProviderResult.FundingLines;
publishedProviderVersion.Calculations = generatedProviderResult.Calculations;
publishedProviderVersion.ReferenceData = generatedProviderResult.ReferenceData;
publishedProviderVersion.TemplateVersion = templateVersion;
publishedProviderVersion.TotalFunding = generatedProviderResult.TotalFunding;
publishedProviderVersion.Provider = mappedProvider;
return(!equal);
}
public void SetUp()
{
_templateVersion = NewRandomString();
_publishedProviderVersion = CreateProviderVersion(_templateVersion);
_generatedProviderResult = CreateGeneratedProviderResult();
_provider = CreateProvider();
_publishedProviderVersionForMapping = (PublishedProviderVersion)_publishedProviderVersion.Clone();
_mapper = CreateMapper();
_mapper
.Map <Provider>(_provider)
.Returns(_publishedProviderVersionForMapping.Provider);
_logger = CreateLogger();
_publishedProviderDataPopulator = new PublishedProviderDataPopulator(_mapper, _logger);
}
public void DoesNotUpdateGeneratedResultIfPreviousVersionHasNoMatchingFundingLine()
{
uint templateLineId = NewRandomUint();
GeneratedProviderResult generatedProviderResult = NewGeneratedProviderResult(_ =>
_.WithFundlines(NewFundingLine(fl => fl
.WithTemplateLineId(templateLineId)
.WithFundingLineType(FundingLineType.Payment))));
PublishedProviderVersion publishedProviderVersion = NewPublishedProviderVersion(_ =>
_.WithFundingLines(NewFundingLine()));
bool hasZeroedFundingLine = _fundingLineValueOverride.TryOverridePreviousFundingLineValues(publishedProviderVersion,
generatedProviderResult);
hasZeroedFundingLine
.Should()
.BeFalse();
}
public async Task <int> Generate(FeedOptions options)
{
IEnumerable <Provider> records = GetRecords(options.InputFilePath);
foreach (Provider provider in records)
{
PublishedProviderVersion publishedProviderVersion = GetPublishedProviderVersion(provider);
Common.TemplateMetadata.Models.TemplateMetadataContents templateMetadataContents = GetProviderTemplateMetadataContents();
TemplateMapping templateMapping = GetTemplateMapping();
GeneratedProviderResult generatedProviderResult = GetGeneratedProviderResult(provider);
string generatedProviderDocumentContent = _publishedProviderContentsGenerator.GenerateContents(publishedProviderVersion, templateMetadataContents, templateMapping);
PublishProviderDocument(options, publishedProviderVersion, generatedProviderDocumentContent);
}
int fundingIndex = 0;
foreach (IGrouping <string, Provider> laGroup in records.GroupBy(x => x.ProviderLaCode))
{
OrganisationGroupLookupParameters organisationGroupLookupParameters = new OrganisationGroupLookupParameters
{
OrganisationGroupTypeCode = Common.ApiClient.Policies.Models.OrganisationGroupTypeCode.LocalAuthority,
IdentifierValue = laGroup.Key,
GroupTypeIdentifier = Common.ApiClient.Policies.Models.OrganisationGroupTypeIdentifier.LACode,
ProviderVersionId = options.ProviderVersion
};
IEnumerable <ProviderApiClient> apiClientProviders = GetApiClientProviders(laGroup);
TargetOrganisationGroup targetOrganisationGroup = await organisationGroupTargetProviderLookup.GetTargetProviderDetails(organisationGroupLookupParameters, Common.ApiClient.Policies.Models.GroupingReason.Payment, apiClientProviders);
PublishedFundingVersion publishedFundingVersion = GetPublishedFundingVersion(laGroup, targetOrganisationGroup, fundingIndex);
Common.TemplateMetadata.Models.TemplateMetadataContents templateMetadataContents = GetFundingTemplateMetadataContents();
string generatedFundingDocumentContent = _publishedFundingContentsGenerator.GenerateContents(publishedFundingVersion, templateMetadataContents);
PublishFundingDocument(options, publishedFundingVersion, generatedFundingDocumentContent);
fundingIndex++;
}
_logger.Information("NAV Data generation completed.");
return(1);
}
public bool TryOverridePreviousFundingLineValues(PublishedProviderVersion publishedProviderVersion,
GeneratedProviderResult generatedProviderResult)
{
var haveZeroedNullPaymentLine = false;
foreach (FundingLine fundingLine in generatedProviderResult.FundingLines?.Where(_ => _.Type == FundingLineType.Payment)
?? new FundingLine[0])
{
FundingLine previousFundingLineVersion =
publishedProviderVersion.FundingLines?.SingleOrDefault(_ => _.TemplateLineId == fundingLine.TemplateLineId);
if (previousFundingLineVersion == null)
{
continue;
}
fundingLine.Value = 0M;
haveZeroedNullPaymentLine = true;
}
return(haveZeroedNullPaymentLine);
}
/// <summary>
/// Generate Funding Lines, Calculations and Reference Data for a funding stream for all in scope providers
/// </summary>
/// <param name="templateMetadata">Template Metadata</param>
/// <param name="templateMapping">Template Mapping</param>
/// <param name="scopedProviders">Scoped providers for a specification</param>
/// <param name="calculationResults">Calculation Results</param>
/// <returns>Dictionary of Generated Provider Results, keyed on ProviderId</returns>
public IDictionary <string, GeneratedProviderResult> Generate(TemplateMetadataContents templateMetadata, Common.ApiClient.Calcs.Models.TemplateMapping templateMapping, IEnumerable <Provider> scopedProviders, IDictionary <string, ProviderCalculationResult> calculationResults)
{
Guard.ArgumentNotNull(templateMetadata, nameof(templateMetadata));
Guard.ArgumentNotNull(templateMapping, nameof(templateMapping));
Guard.ArgumentNotNull(calculationResults, nameof(calculationResults));
ConcurrentDictionary <string, GeneratedProviderResult> results = new ConcurrentDictionary <string, GeneratedProviderResult>();
TimeSpan loggingPeriod = TimeSpan.FromMinutes(5);
using (new Timer(
_ => _logger.Information($"{results.Count}: Published Providers processed."),
null, loggingPeriod, loggingPeriod))
Parallel.ForEach(scopedProviders, new ParallelOptions {
MaxDegreeOfParallelism = 15
}, (provider) =>
{
GeneratedProviderResult generatedProviderResult = new GeneratedProviderResult();
ProviderCalculationResult calculationResultsForProvider = null;
calculationResults.TryGetValue(provider.ProviderId, out calculationResultsForProvider);
if (calculationResultsForProvider != null)
{
GeneratorModels.FundingValue fundingValue = _fundingLineTotalAggregator.GenerateTotals(templateMetadata, templateMapping.TemplateMappingItems.ToDictionary(_ => _.TemplateId), calculationResultsForProvider.Results.ToDictionary(_ => _.Id));
// Get funding lines
IEnumerable <GeneratorModels.FundingLine> fundingLines = fundingValue.FundingLines?.Flatten(_ => _.FundingLines) ?? new GeneratorModels.FundingLine[0];
Dictionary <uint, GeneratorModels.FundingLine> uniqueFundingLine = new Dictionary <uint, GeneratorModels.FundingLine>();
generatedProviderResult.FundingLines = fundingLines.Where(_ => uniqueFundingLine.TryAdd(_.TemplateLineId, _)).Select(_ =>
{
return(_mapper.Map <FundingLine>(_));
}).ToList();
// Set total funding
generatedProviderResult.TotalFunding = generatedProviderResult.FundingLines
.Sum(p =>
{
return(p.Type == FundingLineType.Payment ? p.Value : 0);
});
Dictionary <uint, GeneratorModels.Calculation> uniqueCalculations = new Dictionary <uint, GeneratorModels.Calculation>();
// Get calculations
IEnumerable <GeneratorModels.Calculation> fundingCalculations = uniqueFundingLine.Values?.SelectMany(_ => _.Calculations.Flatten(calc => calc.Calculations)) ?? new GeneratorModels.Calculation[0];
generatedProviderResult.Calculations = fundingCalculations.Where(_ => uniqueCalculations.TryAdd(_.TemplateCalculationId, _)).Select(_ =>
{
return(_mapper.Map <FundingCalculation>(_));
}).ToList();
// Set Provider information
generatedProviderResult.Provider = _mapper.Map <Provider>(provider);
results.TryAdd(provider.ProviderId, generatedProviderResult);
}
});
return(results);
}
private async Task RefreshFundingStream(Reference fundingStream,
SpecificationSummary specification,
IDictionary <string, Provider> scopedProviders,
IDictionary <string, ProviderCalculationResult> allCalculationResults,
string jobId, Reference author,
string correlationId,
IEnumerable <PublishedProvider> existingPublishedProviders,
string fundingPeriodId)
{
TemplateMetadataContents templateMetadataContents = await _policiesService.GetTemplateMetadataContents(fundingStream.Id, specification.FundingPeriod.Id, specification.TemplateIds[fundingStream.Id]);
if (templateMetadataContents == null)
{
_logger.Information($"Unable to locate template meta data contents for funding stream:'{fundingStream.Id}' and template id:'{specification.TemplateIds[fundingStream.Id]}'");
return;
}
IEnumerable <ProfileVariationPointer> variationPointers = await _specificationService.GetProfileVariationPointers(specification.Id) ?? ArraySegment <ProfileVariationPointer> .Empty;
Dictionary <string, PublishedProvider> publishedProviders = new Dictionary <string, PublishedProvider>();
foreach (PublishedProvider publishedProvider in existingPublishedProviders)
{
if (publishedProvider.Current.FundingStreamId == fundingStream.Id)
{
publishedProviders.Add(publishedProvider.Current.ProviderId, publishedProvider);
}
}
// Create PublishedProvider for providers which don't already have a record (eg ProviderID-FundingStreamId-FundingPeriodId)
IDictionary <string, PublishedProvider> newProviders = _providerService.GenerateMissingPublishedProviders(scopedProviders.Values, specification, fundingStream, publishedProviders);
publishedProviders.AddRange(newProviders);
// Get TemplateMapping for calcs from Calcs API client nuget
ApiResponse <Common.ApiClient.Calcs.Models.TemplateMapping> calculationMappingResult = await _calculationsApiClientPolicy.ExecuteAsync(() => _calculationsApiClient.GetTemplateMapping(specification.Id, fundingStream.Id));
if (calculationMappingResult == null)
{
throw new Exception($"calculationMappingResult returned null for funding stream {fundingStream.Id}");
}
Common.ApiClient.Calcs.Models.TemplateMapping templateMapping = calculationMappingResult.Content;
_logger.Information("Generating PublishedProviders for refresh");
// Generate populated data for each provider in this funding line
IDictionary <string, GeneratedProviderResult> generatedPublishedProviderData;
try
{
generatedPublishedProviderData = _publishedProviderDataGenerator.Generate(templateMetadataContents, templateMapping, scopedProviders.Values, allCalculationResults);
}
catch (Exception ex)
{
_logger.Error(ex, "Exception during generating provider data");
throw;
}
_logger.Information("Populated PublishedProviders for refresh");
Dictionary <string, PublishedProvider> publishedProvidersToUpdate = new Dictionary <string, PublishedProvider>();
Dictionary <string, PublishedProvider> existingPublishedProvidersToUpdate = new Dictionary <string, PublishedProvider>();
FundingLine[] flattenedTemplateFundingLines = templateMetadataContents.RootFundingLines.Flatten(_ => _.FundingLines).ToArray();
_logger.Information("Profiling providers for refresh");
try
{
await ProfileProviders(publishedProviders, newProviders, generatedPublishedProviderData);
}
catch (Exception ex)
{
_logger.Error(ex, "Exception during generating provider profiling");
if (ex is NonRetriableException)
{
await _jobManagement.UpdateJobStatus(jobId, 0, 0, false, "Refresh job failed during generating provider profiling.");
}
throw;
}
_logger.Information("Finished profiling providers for refresh");
_logger.Information("Start snapshots for published provider variations");
// snapshot the current published providers so any changes aren't reflected when we detect variations later
_variationService.SnapShot(publishedProviders, fundingStream.Id);
_logger.Information("Finished snapshots for published provider variations");
//we need enumerate a readonly cut of this as we add to it in some variations now (for missing providers not in scope)
Dictionary <string, PublishedProvider> publishedProvidersReadonlyDictionary = publishedProviders.ToDictionary(_ => _.Key, _ => _.Value);
_logger.Information($"Start getting funding configuration for funding stream '{fundingStream.Id}'");
// set up the published providers context for error detection laterawait
FundingConfiguration fundingConfiguration = await _policiesService.GetFundingConfiguration(fundingStream.Id, specification.FundingPeriod.Id);
_logger.Information($"Retrieved funding stream configuration for '{fundingStream.Id}'");
PublishedProvidersContext publishedProvidersContext = new PublishedProvidersContext
{
ScopedProviders = scopedProviders.Values,
SpecificationId = specification.Id,
ProviderVersionId = specification.ProviderVersionId,
CurrentPublishedFunding = (await _publishingResiliencePolicy.ExecuteAsync(() => _publishedFundingDataService.GetCurrentPublishedFunding(specification.Id, GroupingReason.Payment)))
.Where(x => x.Current.GroupingReason == CalculateFunding.Models.Publishing.GroupingReason.Payment),
OrganisationGroupResultsData = new Dictionary <string, HashSet <string> >(),
FundingConfiguration = fundingConfiguration
};
_logger.Information("Starting to process providers for variations and exclusions");
foreach (KeyValuePair <string, PublishedProvider> publishedProvider in publishedProvidersReadonlyDictionary)
{
PublishedProviderVersion publishedProviderVersion = publishedProvider.Value.Current;
string providerId = publishedProviderVersion.ProviderId;
// this could be a retry and the key may not exist as the provider has been created as a successor so we need to skip
if (!generatedPublishedProviderData.ContainsKey(publishedProvider.Key))
{
continue;
}
GeneratedProviderResult generatedProviderResult = generatedPublishedProviderData[publishedProvider.Key];
PublishedProviderExclusionCheckResult exclusionCheckResult =
_providerExclusionCheck.ShouldBeExcluded(generatedProviderResult, flattenedTemplateFundingLines);
if (exclusionCheckResult.ShouldBeExcluded)
{
if (newProviders.ContainsKey(publishedProvider.Key))
{
newProviders.Remove(publishedProvider.Key);
continue;
}
if (!_fundingLineValueOverride.TryOverridePreviousFundingLineValues(publishedProviderVersion, generatedProviderResult))
{
//there are no none null payment funding line values and we didn't have to override any previous
//version funding lines with a zero amount now they are all null so skip this published provider
//the updates check
continue;
}
}
bool publishedProviderUpdated = _publishedProviderDataPopulator.UpdatePublishedProvider(publishedProviderVersion,
generatedProviderResult,
scopedProviders[providerId],
specification.TemplateIds[fundingStream.Id],
newProviders.ContainsKey(publishedProvider.Key));
_logger.Verbose($"Published provider '{publishedProvider.Key}' updated: '{publishedProviderUpdated}'");
//reapply any custom profiles this provider has and internally check for errors
_reApplyCustomProfiles.ProcessPublishedProvider(publishedProviderVersion);
// process published provider and detect errors
await _detection.ProcessPublishedProvider(publishedProvider.Value, publishedProvidersContext);
if (publishedProviderUpdated && existingPublishedProviders.AnyWithNullCheck())
{
IDictionary <string, PublishedProvider> newPublishedProviders = await _variationService.PrepareVariedProviders(generatedProviderResult.TotalFunding,
publishedProviders,
publishedProvider.Value,
scopedProviders[providerId],
fundingConfiguration?.Variations,
variationPointers,
fundingStream.Id,
specification.ProviderVersionId);
if (!newPublishedProviders.IsNullOrEmpty())
{
newProviders.AddRange(newPublishedProviders);
}
}
if (!publishedProviderUpdated)
{
continue;
}
if (!newProviders.ContainsKey(publishedProvider.Key))
{
existingPublishedProvidersToUpdate.Add(publishedProvider.Key, publishedProvider.Value);
}
publishedProvidersToUpdate.Add(publishedProvider.Key, publishedProvider.Value);
}
_logger.Information("Finished processing providers for variations and exclusions");
_logger.Information("Adding additional variation reasons");
AddInitialPublishVariationReasons(newProviders.Values);
_logger.Information("Finished adding additional variation reasons");
_logger.Information("Starting to apply variations");
if (!(await _variationService.ApplyVariations(publishedProvidersToUpdate, newProviders, specification.Id, jobId)))
{
await _jobManagement.UpdateJobStatus(jobId, 0, 0, false, "Refresh job failed with variations errors.");
throw new NonRetriableException($"Unable to refresh funding. Variations generated {_variationService.ErrorCount} errors. Check log for details");
}
_logger.Information("Finished applying variations");
_logger.Information($"Updating a total of {publishedProvidersToUpdate.Count} published providers");
if (publishedProvidersToUpdate.Count > 0)
{
if (existingPublishedProvidersToUpdate.Count > 0 || newProviders.Count > 0)
{
using (Transaction transaction = _transactionFactory.NewTransaction <RefreshService>())
{
try
{
// if any error occurs while updating or indexing then we need to re-index all published providers for consistency
transaction.Enroll(async() =>
{
await _publishedProviderVersionService.CreateReIndexJob(author, correlationId, specification.Id, jobId);
});
// Save updated PublishedProviders to cosmos and increment version status
if (existingPublishedProvidersToUpdate.Count > 0)
{
_logger.Information($"Saving updates to existing published providers. Total={existingPublishedProvidersToUpdate.Count}");
await _publishedProviderStatusUpdateService.UpdatePublishedProviderStatus(existingPublishedProvidersToUpdate.Values, author, PublishedProviderStatus.Updated, jobId, correlationId);
_logger.Information("Indexing existing PublishedProviders");
await _publishedProviderIndexerService.IndexPublishedProviders(existingPublishedProvidersToUpdate.Values.Select(_ => _.Current));
}
if (newProviders.Count > 0)
{
_logger.Information($"Saving new published providers. Total={newProviders.Count}");
await _publishedProviderStatusUpdateService.UpdatePublishedProviderStatus(newProviders.Values, author, PublishedProviderStatus.Draft, jobId, correlationId);
_logger.Information("Indexing newly added PublishedProviders");
await _publishedProviderIndexerService.IndexPublishedProviders(newProviders.Values.Select(_ => _.Current));
}
transaction.Complete();
}
catch (Exception ex)
{
await transaction.Compensate();
throw;
}
}
_logger.Information("Creating generate Csv jobs");
IGeneratePublishedFundingCsvJobsCreation generateCsvJobs = _generateCsvJobsLocator
.GetService(GeneratePublishingCsvJobsCreationAction.Refresh);
IEnumerable <string> fundingLineCodes = await _publishedFundingDataService.GetPublishedProviderFundingLines(specification.Id);
IEnumerable <string> fundingStreamIds = Array.Empty <string>();
PublishedFundingCsvJobsRequest publishedFundingCsvJobsRequest = new PublishedFundingCsvJobsRequest
{
SpecificationId = specification.Id,
CorrelationId = correlationId,
User = author,
FundingLineCodes = fundingLineCodes,
FundingStreamIds = fundingStreamIds,
FundingPeriodId = fundingPeriodId
};
await generateCsvJobs.CreateJobs(publishedFundingCsvJobsRequest);
}
}
}
