public async Task <PublishedFundingInput> GeneratePublishedFundingInput(IDictionary <string, PublishedProvider> publishedProvidersForFundingStream,
IEnumerable <Provider> scopedProviders,
Reference fundingStream,
SpecificationSummary specification,
IEnumerable <PublishedProvider> publishedProvidersInScope)
{
Guard.ArgumentNotNull(publishedProvidersForFundingStream, nameof(publishedProvidersForFundingStream));
Guard.ArgumentNotNull(scopedProviders, nameof(scopedProviders));
Guard.ArgumentNotNull(fundingStream, nameof(fundingStream));
Guard.ArgumentNotNull(specification, nameof(specification));
_logger.Information($"Fetching existing published funding");
// Get latest version of existing published funding
IEnumerable <PublishedFunding> publishedFunding = await _publishingResiliencePolicy.ExecuteAsync(() =>
_publishedFundingDataService.GetCurrentPublishedFunding(fundingStream.Id, specification.FundingPeriod.Id));
_logger.Information($"Fetched {publishedFunding.Count()} existing published funding items");
_logger.Information($"Generating organisation groups");
FundingConfiguration fundingConfiguration = await _policiesService.GetFundingConfiguration(fundingStream.Id, specification.FundingPeriod.Id);
TemplateMetadataContents templateMetadataContents = await ReadTemplateMetadataContents(fundingStream, specification);
// Foreach group, determine the provider versions required to be latest
IEnumerable <OrganisationGroupResult> organisationGroups =
await _organisationGroupGenerator.GenerateOrganisationGroup(fundingConfiguration, _mapper.Map <IEnumerable <ApiProvider> >(scopedProviders), specification.ProviderVersionId, specification.ProviderSnapshotId);
// filter out organisation groups which don't contain a provider which is in scope
if (!publishedProvidersInScope.IsNullOrEmpty())
{
HashSet <string> publishedProviderIdsInScope = new HashSet <string>(publishedProvidersInScope.DistinctBy(_ => _.Current.ProviderId).Select(_ => _.Current.ProviderId));
organisationGroups = organisationGroups.Where(_ => _.Providers.Any(provider => publishedProviderIdsInScope.Contains(provider.ProviderId)));
}
_logger.Information($"A total of {organisationGroups.Count()} were generated");
_logger.Information($"Generating organisation groups to save");
// Compare existing published provider versions with existing current PublishedFundingVersion
IEnumerable <(PublishedFunding PublishedFunding, OrganisationGroupResult OrganisationGroupResult)> organisationGroupsToSave =
_publishedFundingChangeDetectorService.GenerateOrganisationGroupsToSave(organisationGroups, publishedFunding, publishedProvidersForFundingStream);
_logger.Information($"A total of {organisationGroupsToSave.Count()} organisation groups returned to save");
// Generate PublishedFundingVersion for new and updated PublishedFundings
return(new PublishedFundingInput()
{
OrganisationGroupsToSave = organisationGroupsToSave,
TemplateMetadataContents = templateMetadataContents,
TemplateVersion = specification.TemplateIds[fundingStream.Id],
FundingStream = fundingStream,
FundingPeriod = await _policiesService.GetFundingPeriodByConfigurationId(specification.FundingPeriod.Id),
PublishingDates = await _publishedFundingDateService.GetDatesForSpecification(specification.Id),
SpecificationId = specification.Id,
});
}
public async Task <IActionResult> AssignProfilePatternKey(
string fundingStreamId,
string fundingPeriodId,
string providerId,
ProfilePatternKey profilePatternKey,
Reference author)
{
Guard.IsNullOrWhiteSpace(fundingStreamId, nameof(fundingStreamId));
Guard.IsNullOrWhiteSpace(fundingPeriodId, nameof(fundingPeriodId));
Guard.IsNullOrWhiteSpace(providerId, nameof(providerId));
Guard.ArgumentNotNull(profilePatternKey, nameof(profilePatternKey));
PublishedProvider publishedProvider = await _publishingResiliencePolicy.ExecuteAsync(async() =>
await _publishedFundingRepository.GetPublishedProvider(fundingStreamId, fundingPeriodId, providerId));
if (publishedProvider == null)
{
return(new StatusCodeResult((int)HttpStatusCode.NotFound));
}
FundingConfiguration fundingConfiguration = await _policiesService.GetFundingConfiguration(fundingStreamId, fundingPeriodId);
if (fundingConfiguration == null || !fundingConfiguration.EnableUserEditableRuleBasedProfiles)
{
return(new BadRequestObjectResult($"User not allowed to edit rule based profiles for funding stream - '{fundingStreamId}' and funding period - '{fundingPeriodId}'"));
}
if (MatchingProfilePatternKeyExists(publishedProvider.Current, profilePatternKey))
{
return(new StatusCodeResult((int)HttpStatusCode.NotModified));
}
PublishedProvider modifiedPublishedProvider = await CreateVersion(publishedProvider, author);
if (modifiedPublishedProvider == null)
{
return(new StatusCodeResult((int)HttpStatusCode.BadRequest));
}
PublishedProviderVersion newPublishedProviderVersion = publishedProvider.Current;
newPublishedProviderVersion.SetProfilePatternKey(profilePatternKey, author);
await ProfileFundingLineValues(newPublishedProviderVersion, profilePatternKey);
await _publishedProviderErrorDetection.ProcessPublishedProvider(publishedProvider, _ => _ is FundingLineValueProfileMismatchErrorDetector);
await SavePublishedProvider(publishedProvider, newPublishedProviderVersion);
return(new StatusCodeResult((int)HttpStatusCode.OK));
}
private async Task RefreshFundingStream(Reference fundingStream,
SpecificationSummary specification,
IDictionary <string, Provider> scopedProviders,
IDictionary <string, ProviderCalculationResult> allCalculationResults,
string jobId, Reference author,
string correlationId,
IEnumerable <PublishedProvider> existingPublishedProviders,
string fundingPeriodId)
{
TemplateMetadataContents templateMetadataContents = await _policiesService.GetTemplateMetadataContents(fundingStream.Id, specification.FundingPeriod.Id, specification.TemplateIds[fundingStream.Id]);
if (templateMetadataContents == null)
{
_logger.Information($"Unable to locate template meta data contents for funding stream:'{fundingStream.Id}' and template id:'{specification.TemplateIds[fundingStream.Id]}'");
return;
}
IEnumerable <ProfileVariationPointer> variationPointers = await _specificationService.GetProfileVariationPointers(specification.Id) ?? ArraySegment <ProfileVariationPointer> .Empty;
Dictionary <string, PublishedProvider> publishedProviders = new Dictionary <string, PublishedProvider>();
foreach (PublishedProvider publishedProvider in existingPublishedProviders)
{
if (publishedProvider.Current.FundingStreamId == fundingStream.Id)
{
publishedProviders.Add(publishedProvider.Current.ProviderId, publishedProvider);
}
}
// Create PublishedProvider for providers which don't already have a record (eg ProviderID-FundingStreamId-FundingPeriodId)
IDictionary <string, PublishedProvider> newProviders = _providerService.GenerateMissingPublishedProviders(scopedProviders.Values, specification, fundingStream, publishedProviders);
publishedProviders.AddRange(newProviders);
// Get TemplateMapping for calcs from Calcs API client nuget
ApiResponse <Common.ApiClient.Calcs.Models.TemplateMapping> calculationMappingResult = await _calculationsApiClientPolicy.ExecuteAsync(() => _calculationsApiClient.GetTemplateMapping(specification.Id, fundingStream.Id));
if (calculationMappingResult == null)
{
throw new Exception($"calculationMappingResult returned null for funding stream {fundingStream.Id}");
}
Common.ApiClient.Calcs.Models.TemplateMapping templateMapping = calculationMappingResult.Content;
_logger.Information("Generating PublishedProviders for refresh");
// Generate populated data for each provider in this funding line
IDictionary <string, GeneratedProviderResult> generatedPublishedProviderData;
try
{
generatedPublishedProviderData = _publishedProviderDataGenerator.Generate(templateMetadataContents, templateMapping, scopedProviders.Values, allCalculationResults);
}
catch (Exception ex)
{
_logger.Error(ex, "Exception during generating provider data");
throw;
}
_logger.Information("Populated PublishedProviders for refresh");
Dictionary <string, PublishedProvider> publishedProvidersToUpdate = new Dictionary <string, PublishedProvider>();
Dictionary <string, PublishedProvider> existingPublishedProvidersToUpdate = new Dictionary <string, PublishedProvider>();
FundingLine[] flattenedTemplateFundingLines = templateMetadataContents.RootFundingLines.Flatten(_ => _.FundingLines).ToArray();
_logger.Information("Profiling providers for refresh");
try
{
await ProfileProviders(publishedProviders, newProviders, generatedPublishedProviderData);
}
catch (Exception ex)
{
_logger.Error(ex, "Exception during generating provider profiling");
if (ex is NonRetriableException)
{
await _jobManagement.UpdateJobStatus(jobId, 0, 0, false, "Refresh job failed during generating provider profiling.");
}
throw;
}
_logger.Information("Finished profiling providers for refresh");
_logger.Information("Start snapshots for published provider variations");
// snapshot the current published providers so any changes aren't reflected when we detect variations later
_variationService.SnapShot(publishedProviders, fundingStream.Id);
_logger.Information("Finished snapshots for published provider variations");
//we need enumerate a readonly cut of this as we add to it in some variations now (for missing providers not in scope)
Dictionary <string, PublishedProvider> publishedProvidersReadonlyDictionary = publishedProviders.ToDictionary(_ => _.Key, _ => _.Value);
_logger.Information($"Start getting funding configuration for funding stream '{fundingStream.Id}'");
// set up the published providers context for error detection laterawait
FundingConfiguration fundingConfiguration = await _policiesService.GetFundingConfiguration(fundingStream.Id, specification.FundingPeriod.Id);
_logger.Information($"Retrieved funding stream configuration for '{fundingStream.Id}'");
PublishedProvidersContext publishedProvidersContext = new PublishedProvidersContext
{
ScopedProviders = scopedProviders.Values,
SpecificationId = specification.Id,
ProviderVersionId = specification.ProviderVersionId,
CurrentPublishedFunding = (await _publishingResiliencePolicy.ExecuteAsync(() => _publishedFundingDataService.GetCurrentPublishedFunding(specification.Id, GroupingReason.Payment)))
.Where(x => x.Current.GroupingReason == CalculateFunding.Models.Publishing.GroupingReason.Payment),
OrganisationGroupResultsData = new Dictionary <string, HashSet <string> >(),
FundingConfiguration = fundingConfiguration
};
_logger.Information("Starting to process providers for variations and exclusions");
foreach (KeyValuePair <string, PublishedProvider> publishedProvider in publishedProvidersReadonlyDictionary)
{
PublishedProviderVersion publishedProviderVersion = publishedProvider.Value.Current;
string providerId = publishedProviderVersion.ProviderId;
// this could be a retry and the key may not exist as the provider has been created as a successor so we need to skip
if (!generatedPublishedProviderData.ContainsKey(publishedProvider.Key))
{
continue;
}
GeneratedProviderResult generatedProviderResult = generatedPublishedProviderData[publishedProvider.Key];
PublishedProviderExclusionCheckResult exclusionCheckResult =
_providerExclusionCheck.ShouldBeExcluded(generatedProviderResult, flattenedTemplateFundingLines);
if (exclusionCheckResult.ShouldBeExcluded)
{
if (newProviders.ContainsKey(publishedProvider.Key))
{
newProviders.Remove(publishedProvider.Key);
continue;
}
if (!_fundingLineValueOverride.TryOverridePreviousFundingLineValues(publishedProviderVersion, generatedProviderResult))
{
//there are no none null payment funding line values and we didn't have to override any previous
//version funding lines with a zero amount now they are all null so skip this published provider
//the updates check
continue;
}
}
bool publishedProviderUpdated = _publishedProviderDataPopulator.UpdatePublishedProvider(publishedProviderVersion,
generatedProviderResult,
scopedProviders[providerId],
specification.TemplateIds[fundingStream.Id],
newProviders.ContainsKey(publishedProvider.Key));
_logger.Verbose($"Published provider '{publishedProvider.Key}' updated: '{publishedProviderUpdated}'");
//reapply any custom profiles this provider has and internally check for errors
_reApplyCustomProfiles.ProcessPublishedProvider(publishedProviderVersion);
// process published provider and detect errors
await _detection.ProcessPublishedProvider(publishedProvider.Value, publishedProvidersContext);
if (publishedProviderUpdated && existingPublishedProviders.AnyWithNullCheck())
{
IDictionary <string, PublishedProvider> newPublishedProviders = await _variationService.PrepareVariedProviders(generatedProviderResult.TotalFunding,
publishedProviders,
publishedProvider.Value,
scopedProviders[providerId],
fundingConfiguration?.Variations,
variationPointers,
fundingStream.Id,
specification.ProviderVersionId);
if (!newPublishedProviders.IsNullOrEmpty())
{
newProviders.AddRange(newPublishedProviders);
}
}
if (!publishedProviderUpdated)
{
continue;
}
if (!newProviders.ContainsKey(publishedProvider.Key))
{
existingPublishedProvidersToUpdate.Add(publishedProvider.Key, publishedProvider.Value);
}
publishedProvidersToUpdate.Add(publishedProvider.Key, publishedProvider.Value);
}
_logger.Information("Finished processing providers for variations and exclusions");
_logger.Information("Adding additional variation reasons");
AddInitialPublishVariationReasons(newProviders.Values);
_logger.Information("Finished adding additional variation reasons");
_logger.Information("Starting to apply variations");
if (!(await _variationService.ApplyVariations(publishedProvidersToUpdate, newProviders, specification.Id, jobId)))
{
await _jobManagement.UpdateJobStatus(jobId, 0, 0, false, "Refresh job failed with variations errors.");
throw new NonRetriableException($"Unable to refresh funding. Variations generated {_variationService.ErrorCount} errors. Check log for details");
}
_logger.Information("Finished applying variations");
_logger.Information($"Updating a total of {publishedProvidersToUpdate.Count} published providers");
if (publishedProvidersToUpdate.Count > 0)
{
if (existingPublishedProvidersToUpdate.Count > 0 || newProviders.Count > 0)
{
using (Transaction transaction = _transactionFactory.NewTransaction <RefreshService>())
{
try
{
// if any error occurs while updating or indexing then we need to re-index all published providers for consistency
transaction.Enroll(async() =>
{
await _publishedProviderVersionService.CreateReIndexJob(author, correlationId, specification.Id, jobId);
});
// Save updated PublishedProviders to cosmos and increment version status
if (existingPublishedProvidersToUpdate.Count > 0)
{
_logger.Information($"Saving updates to existing published providers. Total={existingPublishedProvidersToUpdate.Count}");
await _publishedProviderStatusUpdateService.UpdatePublishedProviderStatus(existingPublishedProvidersToUpdate.Values, author, PublishedProviderStatus.Updated, jobId, correlationId);
_logger.Information("Indexing existing PublishedProviders");
await _publishedProviderIndexerService.IndexPublishedProviders(existingPublishedProvidersToUpdate.Values.Select(_ => _.Current));
}
if (newProviders.Count > 0)
{
_logger.Information($"Saving new published providers. Total={newProviders.Count}");
await _publishedProviderStatusUpdateService.UpdatePublishedProviderStatus(newProviders.Values, author, PublishedProviderStatus.Draft, jobId, correlationId);
_logger.Information("Indexing newly added PublishedProviders");
await _publishedProviderIndexerService.IndexPublishedProviders(newProviders.Values.Select(_ => _.Current));
}
transaction.Complete();
}
catch (Exception ex)
{
await transaction.Compensate();
throw;
}
}
_logger.Information("Creating generate Csv jobs");
IGeneratePublishedFundingCsvJobsCreation generateCsvJobs = _generateCsvJobsLocator
.GetService(GeneratePublishingCsvJobsCreationAction.Refresh);
IEnumerable <string> fundingLineCodes = await _publishedFundingDataService.GetPublishedProviderFundingLines(specification.Id);
IEnumerable <string> fundingStreamIds = Array.Empty <string>();
PublishedFundingCsvJobsRequest publishedFundingCsvJobsRequest = new PublishedFundingCsvJobsRequest
{
SpecificationId = specification.Id,
CorrelationId = correlationId,
User = author,
FundingLineCodes = fundingLineCodes,
FundingStreamIds = fundingStreamIds,
FundingPeriodId = fundingPeriodId
};
await generateCsvJobs.CreateJobs(publishedFundingCsvJobsRequest);
}
}
}
public ApplyCustomProfileRequestValidator(
IPublishedFundingRepository publishedFunding,
IPublishingResiliencePolicies resiliencePolicies,
IPoliciesService policiesService)
{
Guard.ArgumentNotNull(publishedFunding, nameof(publishedFunding));
Guard.ArgumentNotNull(resiliencePolicies?.PublishedFundingRepository, nameof(resiliencePolicies.PublishedFundingRepository));
Guard.ArgumentNotNull(policiesService, nameof(policiesService));
RuleFor(_ => _.FundingStreamId)
.NotEmpty()
.WithMessage("You must supply a funding stream id");
RuleFor(_ => _.FundingPeriodId)
.NotEmpty()
.WithMessage("You must supply a funding period id");
RuleFor(_ => _.FundingLineCode)
.NotEmpty()
.WithMessage("You must supply a funding line code");
RuleFor(_ => _.ProviderId)
.NotEmpty()
.WithMessage("You must supply a provider id");
RuleFor(_ => _.CustomProfileName)
.NotEmpty()
.WithMessage("You must supply a custom profile name");
RuleFor(_ => _.ProfilePeriods)
.NotEmpty()
.WithMessage("You must supply at least one profile period");
RuleFor(_ => _)
.CustomAsync(async(request, ctx, ct) =>
{
string providerId = request.ProviderId;
string fundingPeriodId = request.FundingPeriodId;
string fundingStreamId = request.FundingStreamId;
string fundingLineCode = request.FundingLineCode;
if (providerId.IsNotNullOrWhitespace() &&
fundingStreamId.IsNotNullOrWhitespace() &&
fundingPeriodId.IsNotNullOrWhitespace() &&
fundingLineCode.IsNotNullOrWhitespace())
{
string id = $"publishedprovider-{providerId}-{fundingPeriodId}-{fundingStreamId}";
PublishedProvider publishedProvider = await resiliencePolicies.PublishedFundingRepository.ExecuteAsync(() =>
publishedFunding.GetPublishedProviderById(id, id));
if (publishedProvider == null)
{
ctx.AddFailure("Request", "No matching published provider located");
}
else if (publishedProvider.Current.FundingLines.All(_ => _.FundingLineCode != fundingLineCode))
{
ctx.AddFailure(nameof(request.FundingLineCode),
$"Did not locate a funding line with code {fundingLineCode}");
}
}
//TODO: check whether the custom name is already in use on the provider??
ProfilePeriod[] profilePeriods = (request.ProfilePeriods ?? Array.Empty <ProfilePeriod>()).ToArray();
if (profilePeriods.GroupBy(_ => new
{
_.Year,
_.Type,
_.TypeValue,
_.Occurrence,
}).Any(_ => _.Count() > 1))
{
ctx.AddFailure(nameof(DistributionPeriod.ProfilePeriods),
"The profile periods must be for unique occurrences in a funding line");
}
if (profilePeriods.Any(_ => _.DistributionPeriodId == null || _.DistributionPeriodId.Trim().Length == 0))
{
ctx.AddFailure(nameof(DistributionPeriod.ProfilePeriods),
"The distribution id must be supplied for all profile periods");
}
});
RuleFor(_ => _)
.CustomAsync(async(request, ctx, ct) =>
{
string fundingPeriodId = request.FundingPeriodId;
string fundingStreamId = request.FundingStreamId;
string fundingLineCode = request.FundingLineCode;
if (fundingStreamId.IsNotNullOrWhitespace() &&
fundingPeriodId.IsNotNullOrWhitespace())
{
var fundingConfiguration = await policiesService.GetFundingConfiguration(fundingStreamId, fundingPeriodId);
if (fundingConfiguration == null || !fundingConfiguration.EnableUserEditableCustomProfiles)
{
ctx.AddFailure("Request", $"User not allowed to edit custom profiles for funding stream - '{fundingStreamId}' and funding period - '{fundingPeriodId}'");
}
}
});
}
