private void GivenTheOrganisationGroupsForTheFundingConfiguration(FundingConfiguration fundingConfiguration, IEnumerable <Provider> scopedProviders, string providerVersionId, IEnumerable <OrganisationGroupResult> organisationGroupResults)
{
_organisationGroupGenerator.GenerateOrganisationGroup(
Arg.Is <FundingConfiguration>(f => f.Id == fundingConfiguration.Id),
Arg.Is <IEnumerable <Common.ApiClient.Providers.Models.Provider> >(p => p.All(i => scopedProviders.Any(sp => sp.ProviderId == i.ProviderId))),
providerVersionId)
.Returns(organisationGroupResults);
}
private void GivenTheFundingConfiguration(Action <FundingConfigurationBuilder> setUp = null)
{
FundingConfigurationBuilder fundingConfigurationBuilder = new FundingConfigurationBuilder();
setUp?.Invoke(fundingConfigurationBuilder);
_fundingConfiguration = fundingConfigurationBuilder.Build();
}
public async Task <PublishedFundingInput> GeneratePublishedFundingInput(IDictionary <string, PublishedProvider> publishedProvidersForFundingStream,
IEnumerable <Provider> scopedProviders,
Reference fundingStream,
SpecificationSummary specification,
IEnumerable <PublishedProvider> publishedProvidersInScope)
{
Guard.ArgumentNotNull(publishedProvidersForFundingStream, nameof(publishedProvidersForFundingStream));
Guard.ArgumentNotNull(scopedProviders, nameof(scopedProviders));
Guard.ArgumentNotNull(fundingStream, nameof(fundingStream));
Guard.ArgumentNotNull(specification, nameof(specification));
_logger.Information($"Fetching existing published funding");
// Get latest version of existing published funding
IEnumerable <PublishedFunding> publishedFunding = await _publishingResiliencePolicy.ExecuteAsync(() =>
_publishedFundingDataService.GetCurrentPublishedFunding(fundingStream.Id, specification.FundingPeriod.Id));
_logger.Information($"Fetched {publishedFunding.Count()} existing published funding items");
_logger.Information($"Generating organisation groups");
FundingConfiguration fundingConfiguration = await _policiesService.GetFundingConfiguration(fundingStream.Id, specification.FundingPeriod.Id);
TemplateMetadataContents templateMetadataContents = await ReadTemplateMetadataContents(fundingStream, specification);
// Foreach group, determine the provider versions required to be latest
IEnumerable <OrganisationGroupResult> organisationGroups =
await _organisationGroupGenerator.GenerateOrganisationGroup(fundingConfiguration, _mapper.Map <IEnumerable <ApiProvider> >(scopedProviders), specification.ProviderVersionId, specification.ProviderSnapshotId);
// filter out organisation groups which don't contain a provider which is in scope
if (!publishedProvidersInScope.IsNullOrEmpty())
{
HashSet <string> publishedProviderIdsInScope = new HashSet <string>(publishedProvidersInScope.DistinctBy(_ => _.Current.ProviderId).Select(_ => _.Current.ProviderId));
organisationGroups = organisationGroups.Where(_ => _.Providers.Any(provider => publishedProviderIdsInScope.Contains(provider.ProviderId)));
}
_logger.Information($"A total of {organisationGroups.Count()} were generated");
_logger.Information($"Generating organisation groups to save");
// Compare existing published provider versions with existing current PublishedFundingVersion
IEnumerable <(PublishedFunding PublishedFunding, OrganisationGroupResult OrganisationGroupResult)> organisationGroupsToSave =
_publishedFundingChangeDetectorService.GenerateOrganisationGroupsToSave(organisationGroups, publishedFunding, publishedProvidersForFundingStream);
_logger.Information($"A total of {organisationGroupsToSave.Count()} organisation groups returned to save");
// Generate PublishedFundingVersion for new and updated PublishedFundings
return(new PublishedFundingInput()
{
OrganisationGroupsToSave = organisationGroupsToSave,
TemplateMetadataContents = templateMetadataContents,
TemplateVersion = specification.TemplateIds[fundingStream.Id],
FundingStream = fundingStream,
FundingPeriod = await _policiesService.GetFundingPeriodByConfigurationId(specification.FundingPeriod.Id),
PublishingDates = await _publishedFundingDateService.GetDatesForSpecification(specification.Id),
SpecificationId = specification.Id,
});
}
public async Task <IEnumerable <OrganisationGroupResult> > GenerateOrganisationGroup(
FundingConfiguration fundingConfiguration,
IEnumerable <Provider> scopedProviders,
string providerVersionId,
int?providerSnapshotId = null)
{
return(await GenerateOrganisationGroup(fundingConfiguration.OrganisationGroupings, fundingConfiguration.ProviderSource,
fundingConfiguration.PaymentOrganisationSource, scopedProviders, providerVersionId, providerSnapshotId));
}
protected void AndTheFundingConfigurationsForSpecificationSummary(FundingConfiguration fundingConfiguration)
{
FundingConfigurationService
.GetFundingConfigurations(Arg.Is <SpecificationSummary>(_ => _.Id == SpecificationId))
.Returns(new Dictionary <string, FundingConfiguration>
{
{ NewRandomString(), fundingConfiguration }
});
}
public void GivenFundingConfiguration(
HttpStatusCode httpStatusCode,
FundingConfiguration fundingConfiguration = null)
{
_policiesApiClient
.Setup(_ => _.GetFundingConfiguration(
_fundingStreamId,
_fundingPeriodId))
.ReturnsAsync(new ApiResponse <FundingConfiguration>(httpStatusCode, fundingConfiguration));
}
public async Task GetFundingConfiguration__GivenFundingConfigurationWasFound_ReturnsSuccess(string fundingStreamId, string fundingPeriodId)
{
// Arrange
FundingStream fundingStream = new FundingStream
{
Id = fundingStreamId
};
FundingPeriod fundingPeriod = new FundingPeriod
{
Id = fundingPeriodId
};
string configId = $"config-{fundingStreamId}-{fundingPeriodId}";
FundingConfiguration fundingConfiguration = new FundingConfiguration
{
Id = configId
};
IPolicyRepository policyRepository = CreatePolicyRepository();
policyRepository
.GetFundingStreamById(Arg.Is(fundingStreamId))
.Returns(fundingStream);
policyRepository
.GetFundingPeriodById(Arg.Is(fundingPeriodId))
.Returns(fundingPeriod);
policyRepository
.GetFundingConfiguration(Arg.Is(configId))
.Returns(fundingConfiguration);
FundingConfigurationService fundingConfigurationsService = CreateFundingConfigurationService(policyRepository: policyRepository);
// Act
IActionResult result = await fundingConfigurationsService.GetFundingConfiguration(fundingStreamId, fundingPeriodId);
// Assert
result
.Should()
.BeOfType <OkObjectResult>()
.Which
.Value
.Should()
.Be(fundingConfiguration);
FundingConfiguration fundingConfigurationResult = ((OkObjectResult)result).Value.As <FundingConfiguration>();
fundingConfigurationResult.ProviderSource.Should().Be(CalculateFunding.Models.Providers.ProviderSource.CFS);
fundingConfigurationResult.PaymentOrganisationSource.Should().Be(PaymentOrganisationSource.PaymentOrganisationAsProvider);
}
public Task <ApiResponse <FundingConfiguration> > SaveFundingConfiguration(string fundingStreamId, string fundingPeriodId, FundingConfiguration configuration)
{
Guard.IsNullOrWhiteSpace(fundingStreamId, nameof(fundingStreamId));
Guard.IsNullOrWhiteSpace(fundingPeriodId, nameof(fundingPeriodId));
Guard.ArgumentNotNull(configuration, nameof(configuration));
string fundingConfigurationKey = $"{fundingStreamId}-{fundingPeriodId}";
_fundingConfigurations[fundingConfigurationKey] = configuration;
return(Task.FromResult(new ApiResponse <FundingConfiguration>(System.Net.HttpStatusCode.OK, configuration)));
}
public async Task GetFundingConfiguration__GivenFundingConfigurationAlreadyInCache_ReturnsSuccessWithConfigurationFromCache(string fundingStreamId, string fundingPeriodId)
{
// Arrange
FundingStream fundingStream = new FundingStream
{
Id = fundingStreamId
};
FundingPeriod fundingPeriod = new FundingPeriod
{
Id = fundingPeriodId
};
string configId = $"config-{fundingStreamId}-{fundingPeriodId}";
FundingConfiguration fundingConfiguration = new FundingConfiguration
{
Id = configId
};
IPolicyRepository policyRepository = CreatePolicyRepository();
policyRepository
.GetFundingStreamById(Arg.Is(fundingStreamId))
.Returns(fundingStream);
policyRepository
.GetFundingPeriodById(Arg.Is(fundingPeriodId))
.Returns(fundingPeriod);
string cacheKey = $"{CacheKeys.FundingConfig}{fundingStreamId}-{fundingPeriodId}";
ICacheProvider cacheProvider = CreateCacheProvider();
cacheProvider
.GetAsync <FundingConfiguration>(Arg.Is(cacheKey))
.Returns(fundingConfiguration);
FundingConfigurationService fundingConfigurationsService = CreateFundingConfigurationService(policyRepository: policyRepository, cacheProvider: cacheProvider);
// Act
IActionResult result = await fundingConfigurationsService.GetFundingConfiguration(fundingStreamId, fundingPeriodId);
// Assert
result
.Should()
.BeOfType <OkObjectResult>()
.Which
.Value
.Should()
.Be(fundingConfiguration);
}
public void GivenTheFundingConfigurationHasTheFollowingFundingVariations(IEnumerable <FundingVariation> fundingVariations)
{
//TODO; this is all a bit frankenstein - remove the variable and access it via the in memory repo methods ideally
FundingConfiguration fundingConfiguration = _policiesStepContext
.CreateFundingConfiguration;
fundingConfiguration
.Should()
.NotBeNull();
fundingConfiguration
.Variations = fundingVariations ?? new FundingVariation[0];
}
public async Task <IActionResult> AssignProfilePatternKey(
string fundingStreamId,
string fundingPeriodId,
string providerId,
ProfilePatternKey profilePatternKey,
Reference author)
{
Guard.IsNullOrWhiteSpace(fundingStreamId, nameof(fundingStreamId));
Guard.IsNullOrWhiteSpace(fundingPeriodId, nameof(fundingPeriodId));
Guard.IsNullOrWhiteSpace(providerId, nameof(providerId));
Guard.ArgumentNotNull(profilePatternKey, nameof(profilePatternKey));
PublishedProvider publishedProvider = await _publishingResiliencePolicy.ExecuteAsync(async() =>
await _publishedFundingRepository.GetPublishedProvider(fundingStreamId, fundingPeriodId, providerId));
if (publishedProvider == null)
{
return(new StatusCodeResult((int)HttpStatusCode.NotFound));
}
FundingConfiguration fundingConfiguration = await _policiesService.GetFundingConfiguration(fundingStreamId, fundingPeriodId);
if (fundingConfiguration == null || !fundingConfiguration.EnableUserEditableRuleBasedProfiles)
{
return(new BadRequestObjectResult($"User not allowed to edit rule based profiles for funding stream - '{fundingStreamId}' and funding period - '{fundingPeriodId}'"));
}
if (MatchingProfilePatternKeyExists(publishedProvider.Current, profilePatternKey))
{
return(new StatusCodeResult((int)HttpStatusCode.NotModified));
}
PublishedProvider modifiedPublishedProvider = await CreateVersion(publishedProvider, author);
if (modifiedPublishedProvider == null)
{
return(new StatusCodeResult((int)HttpStatusCode.BadRequest));
}
PublishedProviderVersion newPublishedProviderVersion = publishedProvider.Current;
newPublishedProviderVersion.SetProfilePatternKey(profilePatternKey, author);
await ProfileFundingLineValues(newPublishedProviderVersion, profilePatternKey);
await _publishedProviderErrorDetection.ProcessPublishedProvider(publishedProvider, _ => _ is FundingLineValueProfileMismatchErrorDetector);
await SavePublishedProvider(publishedProvider, newPublishedProviderVersion);
return(new StatusCodeResult((int)HttpStatusCode.OK));
}
public async Task PerformSearch_GivenResultsWithoutErrors_ReturnsExpected()
{
int numberOfItems = 25;
SearchResults <PublishedProviderSearchItem> searchResults = GenerateSearchResults(numberOfItems);
_publishingClient
.Setup(x => x.SearchPublishedProvider(It.IsAny <SearchModel>()))
.ReturnsAsync(new ApiResponse <SearchResults <PublishedProviderSearchItem> >(HttpStatusCode.OK, searchResults));
var providerStats = new ProviderFundingStreamStatusResponse {
ProviderApprovedCount = 0, TotalFunding = 1234, ProviderDraftCount = numberOfItems
};
_publishingClient
.Setup(x => x.GetProviderStatusCounts(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>()))
.ReturnsAsync(new ApiResponse <IEnumerable <ProviderFundingStreamStatusResponse> >(HttpStatusCode.OK, new []
{
providerStats
}));
FundingConfiguration config = new FundingConfiguration
{
ApprovalMode = ApprovalMode.All,
FundingStreamId = "stream",
FundingPeriodId = "period",
ProviderSource = ProviderSource.CFS
};
_policiesApiClient
.Setup(x => x.GetFundingConfiguration(It.IsAny <string>(), It.IsAny <string>()))
.ReturnsAsync(new ApiResponse <FundingConfiguration>(HttpStatusCode.OK, config));
_service = new PublishedProviderSearchService(_publishingClient.Object, _policiesApiClient.Object, _logger, _mapper);
SearchRequestViewModel request = new SearchRequestViewModel {
PageSize = 12
};
PublishProviderSearchResultViewModel result = await _service.PerformSearch(request);
result.CanApprove.Should().BeTrue();
result.Providers.Should().HaveCount(numberOfItems);
result.TotalErrorResults.Should().Be(0);
result.TotalProvidersToApprove.Should().Be(numberOfItems);
}
private void AndGetFundingConfiguration(
string fundingStreamId,
string fundingPeriodId,
ProviderSource providerSource = ProviderSource.CFS,
bool withRunCalculationEngineAfterCoreProviderUpdate = false)
{
FundingConfiguration fundingConfiguration = NewFundingConfiguration(_ => _
.WithDefaultTemplateVersion(NewRandomString())
.WithProviderSource(providerSource)
.WithRunCalculationEngineAfterCoreProviderUpdate(withRunCalculationEngineAfterCoreProviderUpdate));
ApiResponse <FundingConfiguration> fundingConfigResponse =
new ApiResponse <FundingConfiguration>(HttpStatusCode.OK, fundingConfiguration);
_policiesApiClient
.GetFundingConfiguration(
Arg.Is(fundingStreamId),
Arg.Is(fundingPeriodId))
.Returns(fundingConfigResponse);
}
public void GivenAFundingConfigurationExistsForFundingStreamInFundingPeriod(string fundingStreamId, string fundingPeriodId, Table table)
{
fundingStreamId
.Should()
.NotBeNullOrWhiteSpace();
fundingPeriodId
.Should()
.NotBeNullOrWhiteSpace();
_policiesStepContext.CreateFundingStreamId = fundingStreamId;
_policiesStepContext.CreateFundingPeriodId = fundingPeriodId;
FundingConfiguration fundingConfiguration = table.CreateInstance <FundingConfiguration>();
fundingConfiguration.FundingPeriodId = fundingPeriodId;
fundingConfiguration.FundingStreamId = fundingStreamId;
_policiesStepContext.CreateFundingConfiguration = fundingConfiguration;
_policiesStepContext.Repo.SetFundingConfiguration(fundingPeriodId, fundingStreamId, fundingConfiguration);
}
public async Task <IActionResult> GetFundingConfiguration(string fundingStreamId, string fundingPeriodId)
{
if (string.IsNullOrWhiteSpace(fundingStreamId))
{
_logger.Error("No funding stream Id was provided to GetFundingConfiguration");
return(new BadRequestObjectResult("Null or empty funding stream Id provided"));
}
if (string.IsNullOrWhiteSpace(fundingPeriodId))
{
_logger.Error("No funding period Id was provided to GetFundingConfiguration");
return(new BadRequestObjectResult("Null or empty funding period Id provided"));
}
string cacheKey = $"{CacheKeys.FundingConfig}{fundingStreamId}-{fundingPeriodId}";
string configId = $"config-{fundingStreamId}-{fundingPeriodId}";
FundingConfiguration fundingConfiguration = await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.GetAsync <FundingConfiguration>(cacheKey));
if (fundingConfiguration == null)
{
fundingConfiguration = await _policyRepositoryPolicy.ExecuteAsync(() => _policyRepository.GetFundingConfiguration(configId));
await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.SetAsync(cacheKey, fundingConfiguration));
}
if (fundingConfiguration == null)
{
_logger.Error($"No funding Configuration was found for funding stream id : {fundingStreamId} and funding period id : {fundingPeriodId}");
return(new NotFoundResult());
}
return(new OkObjectResult(fundingConfiguration));
}
public async Task <IEnumerable <OrganisationGroupResult> > GenerateOrganisationGroup(FundingConfiguration fundingConfiguration, IEnumerable <Provider> scopedProviders, string providerVersionId)
{
Guard.ArgumentNotNull(fundingConfiguration, nameof(fundingConfiguration));
Guard.ArgumentNotNull(scopedProviders, nameof(scopedProviders));
Guard.IsNullOrWhiteSpace(providerVersionId, nameof(providerVersionId));
List <OrganisationGroupResult> results = new List <OrganisationGroupResult>();
foreach (OrganisationGroupingConfiguration grouping in fundingConfiguration.OrganisationGroupings)
{
// Get the provider attribute required to group
Func <Provider, string> providerFilterAttribute = GetProviderFieldForGrouping(grouping.GroupTypeIdentifier, grouping.OrganisationGroupTypeCode, grouping.GroupingReason);
// Filter providers based on provider type and subtypes
IEnumerable <Provider> providersForGroup = grouping.ProviderTypeMatch.IsNullOrEmpty() ? scopedProviders : scopedProviders.Where(_ => ShouldIncludeProvider(_, grouping.ProviderTypeMatch));
// Group providers by the fields and discard any providers with null values for that field
IEnumerable <IGrouping <string, Provider> > groupedProviders = providersForGroup.GroupBy(providerFilterAttribute);
// Common values for all groups
Enums.OrganisationGroupTypeClassification organisationGroupTypeClassification = grouping.GroupingReason == GroupingReason.Payment ? Enums.OrganisationGroupTypeClassification.LegalEntity : Enums.OrganisationGroupTypeClassification.GeographicalBoundary;
Enums.OrganisationGroupTypeCode organisationGroupTypeCode = grouping.OrganisationGroupTypeCode.AsMatchingEnum <Enums.OrganisationGroupTypeCode>();
// Generate Organisation Group results based on the grouped providers
foreach (IGrouping <string, Provider> providerGrouping in groupedProviders)
{
// Ignore providers without the matching data in the key
if (string.IsNullOrWhiteSpace(providerGrouping.Key))
{
continue;
}
TargetOrganisationGroup targetOrganisationGroup = null;
OrganisationGroupLookupParameters organisationGroupLookupParameters = new OrganisationGroupLookupParameters
{
IdentifierValue = providerGrouping.Key,
OrganisationGroupTypeCode = grouping.OrganisationGroupTypeCode,
ProviderVersionId = providerVersionId,
GroupTypeIdentifier = grouping.GroupTypeIdentifier
};
targetOrganisationGroup = await _organisationGroupTargetProviderLookup.GetTargetProviderDetails(organisationGroupLookupParameters, grouping.GroupingReason, providerGrouping);
if (targetOrganisationGroup == null)
{
// TODO: improve logging
throw new Exception($"Target Organisation Group could not be found for identifier '{providerGrouping.Key}'");
}
OrganisationGroupResult organisationGroupResult = new OrganisationGroupResult()
{
GroupTypeClassification = organisationGroupTypeClassification,
GroupTypeCode = organisationGroupTypeCode,
GroupTypeIdentifier = grouping.GroupTypeIdentifier.AsMatchingEnum <Enums.OrganisationGroupTypeIdentifier>(),
GroupReason = grouping.GroupingReason.AsMatchingEnum <Enums.OrganisationGroupingReason>(),
IdentifierValue = targetOrganisationGroup.Identifier,
Name = targetOrganisationGroup.Name,
Identifiers = targetOrganisationGroup.Identifiers,
SearchableName = GenerateSearchableName(targetOrganisationGroup.Name),
Providers = providerGrouping,
};
results.Add(organisationGroupResult);
}
}
return(results);
}
public async Task <IEnumerable <OrganisationGroupResult> > GenerateOrganisationGroup(
FundingConfiguration fundingConfiguration,
IEnumerable <Provider> scopedProviders,
string providerVersionId,
int?providerSnapshotId = null)
{
Guard.ArgumentNotNull(fundingConfiguration, nameof(fundingConfiguration));
Guard.ArgumentNotNull(scopedProviders, nameof(scopedProviders));
Guard.IsNullOrWhiteSpace(providerVersionId, nameof(providerVersionId));
List <OrganisationGroupResult> results = new List <OrganisationGroupResult>();
Dictionary <string, FdzPaymentOrganisation> paymentOrganisations = new Dictionary <string, FdzPaymentOrganisation>();
if (fundingConfiguration.ProviderSource == ProviderSource.FDZ &&
fundingConfiguration.OrganisationGroupings.Any(g => g.GroupingReason == GroupingReason.Payment ||
g.GroupingReason == GroupingReason.Contracting ||
g.GroupingReason == GroupingReason.Indicative))
{
if (!providerSnapshotId.HasValue)
{
throw new InvalidOperationException("No provider snapshot ID provided, but it is required fto lookup Payment Organisations from FDZ");
}
ApiResponse <IEnumerable <FdzPaymentOrganisation> > paymentOrganisationsResponse = await _fundingDataZoneApiClient.GetAllOrganisations(providerSnapshotId.Value);
if (paymentOrganisationsResponse.StatusCode == System.Net.HttpStatusCode.OK && paymentOrganisationsResponse.Content != null)
{
foreach (FdzPaymentOrganisation fdzPaymentOrganisation in paymentOrganisationsResponse.Content)
{
if (paymentOrganisations.ContainsKey(fdzPaymentOrganisation.Ukprn))
{
throw new Exception($"The payment organisation group: '{fdzPaymentOrganisation.Ukprn}' needs to be unique for provider snapshot ID '{providerSnapshotId}'.");
}
else
{
paymentOrganisations.Add(fdzPaymentOrganisation.Ukprn, fdzPaymentOrganisation);
}
}
}
else
{
throw new InvalidOperationException($"Unable to retreive payment organisations from provider snapshot ID of {providerSnapshotId}");
}
}
foreach (OrganisationGroupingConfiguration grouping in fundingConfiguration.OrganisationGroupings)
{
// Get the provider attribute required to group
Func <Provider, string> providerFilterAttribute = GetProviderFieldForGrouping(grouping.GroupTypeIdentifier, grouping.OrganisationGroupTypeCode, grouping.GroupingReason, fundingConfiguration.PaymentOrganisationSource);
// Filter providers based on provider type and subtypes
IEnumerable <Provider> providersForGroup = grouping.ProviderTypeMatch.IsNullOrEmpty() ? scopedProviders : scopedProviders.Where(_ => ShouldIncludeProvider(_, grouping.ProviderTypeMatch));
// Filter providers based on provider status
providersForGroup = grouping.ProviderStatus.IsNullOrEmpty() ? providersForGroup : providersForGroup.Where(_ => ShouldIncludeProvider(_, grouping.ProviderStatus));
// Group providers by the fields and discard any providers with null values for that field
IEnumerable <IGrouping <string, Provider> > groupedProviders = providersForGroup.GroupBy(providerFilterAttribute);
// Common values for all groups
Enums.OrganisationGroupTypeClassification organisationGroupTypeClassification = grouping.GroupingReason.IsForProviderPayment() ? Enums.OrganisationGroupTypeClassification.LegalEntity : Enums.OrganisationGroupTypeClassification.GeographicalBoundary;
Enums.OrganisationGroupTypeCode organisationGroupTypeCode = grouping.OrganisationGroupTypeCode.AsMatchingEnum <Enums.OrganisationGroupTypeCode>();
// Generate Organisation Group results based on the grouped providers
foreach (IGrouping <string, Provider> providerGrouping in groupedProviders)
{
// Ignore providers without the matching data in the key
if (string.IsNullOrWhiteSpace(providerGrouping.Key))
{
continue;
}
TargetOrganisationGroup targetOrganisationGroup = null;
if (fundingConfiguration.PaymentOrganisationSource == PaymentOrganisationSource.PaymentOrganisationFields &&
grouping.GroupingReason.IsForProviderPayment())
{
IEnumerable <OrganisationIdentifier> identifiers;
// lookup alternative identifier and name from FDZ's PaymentOrganisation table via FDZ service
if (fundingConfiguration.ProviderSource == ProviderSource.FDZ &&
paymentOrganisations.TryGetValue(providerGrouping.Key, out FdzPaymentOrganisation fdzPaymentOrganisation))
{
identifiers = GetIdentifiers(fdzPaymentOrganisation);
}
else
{
identifiers = new OrganisationIdentifier[0];
}
// Will use providerGrouping.Key as the identifier of the PaymentOrganisation
targetOrganisationGroup = new TargetOrganisationGroup()
{
Identifier = providerGrouping.First().PaymentOrganisationIdentifier,
Name = providerGrouping.First().PaymentOrganisationName,
Identifiers = identifiers,
};
}
else if (fundingConfiguration.PaymentOrganisationSource == PaymentOrganisationSource.PaymentOrganisationAsProvider ||
(fundingConfiguration.PaymentOrganisationSource == PaymentOrganisationSource.PaymentOrganisationFields &&
!grouping.GroupingReason.IsForProviderPayment())
)
{
targetOrganisationGroup = await ObtainTargetOrganisationGroupFromProviderData(fundingConfiguration, providerVersionId, grouping, providerGrouping, targetOrganisationGroup);
}
if (targetOrganisationGroup == null)
{
// TODO: improve logging
throw new Exception($"Target Organisation Group could not be found for identifier '{providerGrouping.Key}'");
}
OrganisationGroupResult organisationGroupResult = new OrganisationGroupResult()
{
GroupTypeClassification = organisationGroupTypeClassification,
GroupTypeCode = organisationGroupTypeCode,
GroupTypeIdentifier = grouping.GroupTypeIdentifier.AsMatchingEnum <Enums.OrganisationGroupTypeIdentifier>(),
GroupReason = grouping.GroupingReason.AsMatchingEnum <Enums.OrganisationGroupingReason>(),
IdentifierValue = targetOrganisationGroup.Identifier,
Name = targetOrganisationGroup.Name,
Identifiers = targetOrganisationGroup.Identifiers,
SearchableName = Sanitiser.SanitiseName(targetOrganisationGroup.Name),
Providers = providerGrouping,
};
results.Add(organisationGroupResult);
}
}
return(results);
}
private async Task <TargetOrganisationGroup> ObtainTargetOrganisationGroupFromProviderData(FundingConfiguration fundingConfiguration, string providerVersionId, OrganisationGroupingConfiguration grouping, IGrouping <string, Common.ApiClient.Providers.Models.Provider> providerGrouping, TargetOrganisationGroup targetOrganisationGroup)
{
OrganisationGroupLookupParameters organisationGroupLookupParameters = new OrganisationGroupLookupParameters
{
IdentifierValue = providerGrouping.Key,
OrganisationGroupTypeCode = grouping.OrganisationGroupTypeCode,
ProviderVersionId = providerVersionId,
GroupTypeIdentifier = grouping.GroupTypeIdentifier
};
targetOrganisationGroup = await _organisationGroupTargetProviderLookup.GetTargetProviderDetails(organisationGroupLookupParameters, grouping.GroupingReason, providerGrouping);
return(targetOrganisationGroup);
}
public async Task ReturnsErrorMessageWhenTrustIdMisatch()
{
// Arrange
string specificationId = NewRandomString();
string providerVersionId = NewRandomString();
string fundingStreamId = NewRandomString();
string fundingPeriodId = NewRandomString();
string providerId1 = NewRandomString();
int majorVersion1 = NewRandomInt();
int minorVersion1 = NewRandomInt();
string providerId2 = NewRandomString();
int majorVersion2 = NewRandomInt();
int minorVersion2 = NewRandomInt();
string identifierValue1 = NewRandomString();
string identifierValue2 = NewRandomString();
string fundingConfigurationId = NewRandomString();
Generators.OrganisationGroup.Enums.OrganisationGroupTypeIdentifier groupTypeIdentifier = Generators.OrganisationGroup.Enums.OrganisationGroupTypeIdentifier.UKPRN;
string summaryErrorMessage = "TrustId not matched";
string detailedErrorMessage = $"TrustId {groupTypeIdentifier}-{identifierValue2} not matched.";
SpecificationSummary specificationSummary = NewSpecificationSummary(_ => _.WithId(specificationId).WithProviderVersionId(providerVersionId));
PublishedProvider publishedProvider = NewPublishedProvider(_ => _
.WithCurrent(NewPublishedProviderVersion(pv => pv.WithFundingStreamId(fundingStreamId)
.WithFundingPeriodId(fundingPeriodId)))
.WithReleased(NewPublishedProviderVersion(pv => pv.WithFundingStreamId(fundingStreamId)
.WithFundingPeriodId(fundingPeriodId)
.WithProviderId(providerId2)
.WithMajorVersion(majorVersion2)
.WithMinorVersion(minorVersion2))));
IEnumerable <Provider> providers = new[]
{
NewProvider(_ => _.WithProviderId(providerId1)),
NewProvider(_ => _.WithProviderId(providerId2))
};
IEnumerable <OrganisationGroupResult> organisationGroupResults = new[]
{
NewOrganisationGroupResult(_ => _
.WithIdentifiers(new []
{
NewOrganisationIdentifier(i => i.WithType(groupTypeIdentifier).WithValue(identifierValue1)),
NewOrganisationIdentifier(i => i.WithType(groupTypeIdentifier).WithValue(NewRandomString()))
})),
NewOrganisationGroupResult(_ => _
.WithIdentifiers(new []
{
NewOrganisationIdentifier(i => i.WithType(groupTypeIdentifier).WithValue(identifierValue2)),
NewOrganisationIdentifier(i => i.WithType(groupTypeIdentifier).WithValue(NewRandomString()))
}))
};
IEnumerable <PublishedFunding> publishedFundings = new[]
{
NewPublishedFunding(_ => _
.WithCurrent(NewPublishedFundingVersion(fv => fv.WithProviderFundings(new []
{
$"{fundingStreamId}-{fundingPeriodId}-{providerId1}-{majorVersion1}_{minorVersion1}",
$"{fundingStreamId}-{fundingPeriodId}-{providerId2}-{majorVersion2}_{minorVersion2}"
})
.WithGroupReason(CalculateFunding.Models.Publishing.GroupingReason.Payment)
.WithOrganisationGroupIdentifierValue(identifierValue1)
.WithOrganisationGroupTypeIdentifier(groupTypeIdentifier)))),
NewPublishedFunding(_ => _
.WithCurrent(NewPublishedFundingVersion(fv => fv.WithProviderFundings(new [] { $"{fundingStreamId}-{fundingPeriodId}-{providerId1}-{majorVersion1}_{minorVersion1}", NewRandomString() })
.WithGroupReason(CalculateFunding.Models.Publishing.GroupingReason.Payment)
.WithOrganisationGroupIdentifierValue(identifierValue2)
.WithOrganisationGroupTypeIdentifier(groupTypeIdentifier))))
};
FundingConfiguration fundingConfiguration = NewFundingConfiguration(_ => _.WithId(fundingConfigurationId));
GivenTheCurrentPublishedFundingForTheSpecification(specificationId, publishedFundings);
GivenTheOrganisationGroupsForTheFundingConfiguration(fundingConfiguration, providers, providerVersionId, organisationGroupResults);
// Act
await WhenErrorsAreDetectedOnThePublishedProvider(publishedProvider, providers, specificationId, providerVersionId, fundingConfiguration, publishedFundings);
publishedProvider.Current
.Errors
.Should()
.NotBeNullOrEmpty();
publishedProvider
.Current
.Errors
.First()
.DetailedErrorMessage
.Should()
.Be(detailedErrorMessage);
publishedProvider
.Current
.Errors
.First()
.SummaryErrorMessage
.Should()
.Be(summaryErrorMessage);
}
private async Task RefreshFundingStream(Reference fundingStream,
SpecificationSummary specification,
IDictionary <string, Provider> scopedProviders,
IDictionary <string, ProviderCalculationResult> allCalculationResults,
string jobId, Reference author,
string correlationId,
IEnumerable <PublishedProvider> existingPublishedProviders,
string fundingPeriodId)
{
TemplateMetadataContents templateMetadataContents = await _policiesService.GetTemplateMetadataContents(fundingStream.Id, specification.FundingPeriod.Id, specification.TemplateIds[fundingStream.Id]);
if (templateMetadataContents == null)
{
_logger.Information($"Unable to locate template meta data contents for funding stream:'{fundingStream.Id}' and template id:'{specification.TemplateIds[fundingStream.Id]}'");
return;
}
IEnumerable <ProfileVariationPointer> variationPointers = await _specificationService.GetProfileVariationPointers(specification.Id) ?? ArraySegment <ProfileVariationPointer> .Empty;
Dictionary <string, PublishedProvider> publishedProviders = new Dictionary <string, PublishedProvider>();
foreach (PublishedProvider publishedProvider in existingPublishedProviders)
{
if (publishedProvider.Current.FundingStreamId == fundingStream.Id)
{
publishedProviders.Add(publishedProvider.Current.ProviderId, publishedProvider);
}
}
// Create PublishedProvider for providers which don't already have a record (eg ProviderID-FundingStreamId-FundingPeriodId)
IDictionary <string, PublishedProvider> newProviders = _providerService.GenerateMissingPublishedProviders(scopedProviders.Values, specification, fundingStream, publishedProviders);
publishedProviders.AddRange(newProviders);
// Get TemplateMapping for calcs from Calcs API client nuget
ApiResponse <Common.ApiClient.Calcs.Models.TemplateMapping> calculationMappingResult = await _calculationsApiClientPolicy.ExecuteAsync(() => _calculationsApiClient.GetTemplateMapping(specification.Id, fundingStream.Id));
if (calculationMappingResult == null)
{
throw new Exception($"calculationMappingResult returned null for funding stream {fundingStream.Id}");
}
Common.ApiClient.Calcs.Models.TemplateMapping templateMapping = calculationMappingResult.Content;
_logger.Information("Generating PublishedProviders for refresh");
// Generate populated data for each provider in this funding line
IDictionary <string, GeneratedProviderResult> generatedPublishedProviderData;
try
{
generatedPublishedProviderData = _publishedProviderDataGenerator.Generate(templateMetadataContents, templateMapping, scopedProviders.Values, allCalculationResults);
}
catch (Exception ex)
{
_logger.Error(ex, "Exception during generating provider data");
throw;
}
_logger.Information("Populated PublishedProviders for refresh");
Dictionary <string, PublishedProvider> publishedProvidersToUpdate = new Dictionary <string, PublishedProvider>();
Dictionary <string, PublishedProvider> existingPublishedProvidersToUpdate = new Dictionary <string, PublishedProvider>();
FundingLine[] flattenedTemplateFundingLines = templateMetadataContents.RootFundingLines.Flatten(_ => _.FundingLines).ToArray();
_logger.Information("Profiling providers for refresh");
try
{
await ProfileProviders(publishedProviders, newProviders, generatedPublishedProviderData);
}
catch (Exception ex)
{
_logger.Error(ex, "Exception during generating provider profiling");
if (ex is NonRetriableException)
{
await _jobManagement.UpdateJobStatus(jobId, 0, 0, false, "Refresh job failed during generating provider profiling.");
}
throw;
}
_logger.Information("Finished profiling providers for refresh");
_logger.Information("Start snapshots for published provider variations");
// snapshot the current published providers so any changes aren't reflected when we detect variations later
_variationService.SnapShot(publishedProviders, fundingStream.Id);
_logger.Information("Finished snapshots for published provider variations");
//we need enumerate a readonly cut of this as we add to it in some variations now (for missing providers not in scope)
Dictionary <string, PublishedProvider> publishedProvidersReadonlyDictionary = publishedProviders.ToDictionary(_ => _.Key, _ => _.Value);
_logger.Information($"Start getting funding configuration for funding stream '{fundingStream.Id}'");
// set up the published providers context for error detection laterawait
FundingConfiguration fundingConfiguration = await _policiesService.GetFundingConfiguration(fundingStream.Id, specification.FundingPeriod.Id);
_logger.Information($"Retrieved funding stream configuration for '{fundingStream.Id}'");
PublishedProvidersContext publishedProvidersContext = new PublishedProvidersContext
{
ScopedProviders = scopedProviders.Values,
SpecificationId = specification.Id,
ProviderVersionId = specification.ProviderVersionId,
CurrentPublishedFunding = (await _publishingResiliencePolicy.ExecuteAsync(() => _publishedFundingDataService.GetCurrentPublishedFunding(specification.Id, GroupingReason.Payment)))
.Where(x => x.Current.GroupingReason == CalculateFunding.Models.Publishing.GroupingReason.Payment),
OrganisationGroupResultsData = new Dictionary <string, HashSet <string> >(),
FundingConfiguration = fundingConfiguration
};
_logger.Information("Starting to process providers for variations and exclusions");
foreach (KeyValuePair <string, PublishedProvider> publishedProvider in publishedProvidersReadonlyDictionary)
{
PublishedProviderVersion publishedProviderVersion = publishedProvider.Value.Current;
string providerId = publishedProviderVersion.ProviderId;
// this could be a retry and the key may not exist as the provider has been created as a successor so we need to skip
if (!generatedPublishedProviderData.ContainsKey(publishedProvider.Key))
{
continue;
}
GeneratedProviderResult generatedProviderResult = generatedPublishedProviderData[publishedProvider.Key];
PublishedProviderExclusionCheckResult exclusionCheckResult =
_providerExclusionCheck.ShouldBeExcluded(generatedProviderResult, flattenedTemplateFundingLines);
if (exclusionCheckResult.ShouldBeExcluded)
{
if (newProviders.ContainsKey(publishedProvider.Key))
{
newProviders.Remove(publishedProvider.Key);
continue;
}
if (!_fundingLineValueOverride.TryOverridePreviousFundingLineValues(publishedProviderVersion, generatedProviderResult))
{
//there are no none null payment funding line values and we didn't have to override any previous
//version funding lines with a zero amount now they are all null so skip this published provider
//the updates check
continue;
}
}
bool publishedProviderUpdated = _publishedProviderDataPopulator.UpdatePublishedProvider(publishedProviderVersion,
generatedProviderResult,
scopedProviders[providerId],
specification.TemplateIds[fundingStream.Id],
newProviders.ContainsKey(publishedProvider.Key));
_logger.Verbose($"Published provider '{publishedProvider.Key}' updated: '{publishedProviderUpdated}'");
//reapply any custom profiles this provider has and internally check for errors
_reApplyCustomProfiles.ProcessPublishedProvider(publishedProviderVersion);
// process published provider and detect errors
await _detection.ProcessPublishedProvider(publishedProvider.Value, publishedProvidersContext);
if (publishedProviderUpdated && existingPublishedProviders.AnyWithNullCheck())
{
IDictionary <string, PublishedProvider> newPublishedProviders = await _variationService.PrepareVariedProviders(generatedProviderResult.TotalFunding,
publishedProviders,
publishedProvider.Value,
scopedProviders[providerId],
fundingConfiguration?.Variations,
variationPointers,
fundingStream.Id,
specification.ProviderVersionId);
if (!newPublishedProviders.IsNullOrEmpty())
{
newProviders.AddRange(newPublishedProviders);
}
}
if (!publishedProviderUpdated)
{
continue;
}
if (!newProviders.ContainsKey(publishedProvider.Key))
{
existingPublishedProvidersToUpdate.Add(publishedProvider.Key, publishedProvider.Value);
}
publishedProvidersToUpdate.Add(publishedProvider.Key, publishedProvider.Value);
}
_logger.Information("Finished processing providers for variations and exclusions");
_logger.Information("Adding additional variation reasons");
AddInitialPublishVariationReasons(newProviders.Values);
_logger.Information("Finished adding additional variation reasons");
_logger.Information("Starting to apply variations");
if (!(await _variationService.ApplyVariations(publishedProvidersToUpdate, newProviders, specification.Id, jobId)))
{
await _jobManagement.UpdateJobStatus(jobId, 0, 0, false, "Refresh job failed with variations errors.");
throw new NonRetriableException($"Unable to refresh funding. Variations generated {_variationService.ErrorCount} errors. Check log for details");
}
_logger.Information("Finished applying variations");
_logger.Information($"Updating a total of {publishedProvidersToUpdate.Count} published providers");
if (publishedProvidersToUpdate.Count > 0)
{
if (existingPublishedProvidersToUpdate.Count > 0 || newProviders.Count > 0)
{
using (Transaction transaction = _transactionFactory.NewTransaction <RefreshService>())
{
try
{
// if any error occurs while updating or indexing then we need to re-index all published providers for consistency
transaction.Enroll(async() =>
{
await _publishedProviderVersionService.CreateReIndexJob(author, correlationId, specification.Id, jobId);
});
// Save updated PublishedProviders to cosmos and increment version status
if (existingPublishedProvidersToUpdate.Count > 0)
{
_logger.Information($"Saving updates to existing published providers. Total={existingPublishedProvidersToUpdate.Count}");
await _publishedProviderStatusUpdateService.UpdatePublishedProviderStatus(existingPublishedProvidersToUpdate.Values, author, PublishedProviderStatus.Updated, jobId, correlationId);
_logger.Information("Indexing existing PublishedProviders");
await _publishedProviderIndexerService.IndexPublishedProviders(existingPublishedProvidersToUpdate.Values.Select(_ => _.Current));
}
if (newProviders.Count > 0)
{
_logger.Information($"Saving new published providers. Total={newProviders.Count}");
await _publishedProviderStatusUpdateService.UpdatePublishedProviderStatus(newProviders.Values, author, PublishedProviderStatus.Draft, jobId, correlationId);
_logger.Information("Indexing newly added PublishedProviders");
await _publishedProviderIndexerService.IndexPublishedProviders(newProviders.Values.Select(_ => _.Current));
}
transaction.Complete();
}
catch (Exception ex)
{
await transaction.Compensate();
throw;
}
}
_logger.Information("Creating generate Csv jobs");
IGeneratePublishedFundingCsvJobsCreation generateCsvJobs = _generateCsvJobsLocator
.GetService(GeneratePublishingCsvJobsCreationAction.Refresh);
IEnumerable <string> fundingLineCodes = await _publishedFundingDataService.GetPublishedProviderFundingLines(specification.Id);
IEnumerable <string> fundingStreamIds = Array.Empty <string>();
PublishedFundingCsvJobsRequest publishedFundingCsvJobsRequest = new PublishedFundingCsvJobsRequest
{
SpecificationId = specification.Id,
CorrelationId = correlationId,
User = author,
FundingLineCodes = fundingLineCodes,
FundingStreamIds = fundingStreamIds,
FundingPeriodId = fundingPeriodId
};
await generateCsvJobs.CreateJobs(publishedFundingCsvJobsRequest);
}
}
}
public async Task <HttpStatusCode> SaveFundingConfiguration(FundingConfiguration fundingConfiguration)
{
Guard.ArgumentNotNull(fundingConfiguration, nameof(fundingConfiguration));
return(await _cosmosRepository.UpsertAsync <FundingConfiguration>(fundingConfiguration, fundingConfiguration.Id, true));
}
public async Task <IActionResult> SaveFundingConfiguration(string actionName,
string controllerName,
FundingConfigurationViewModel configurationViewModel,
string fundingStreamId,
string fundingPeriodId)
{
Guard.IsNullOrWhiteSpace(actionName, nameof(actionName));
Guard.IsNullOrWhiteSpace(controllerName, nameof(controllerName));
Guard.IsNullOrWhiteSpace(fundingStreamId, nameof(fundingStreamId));
Guard.IsNullOrWhiteSpace(fundingPeriodId, nameof(fundingPeriodId));
Guard.ArgumentNotNull(configurationViewModel, nameof(configurationViewModel));
FundingConfiguration fundingConfiguration = _mapper.Map <FundingConfiguration>(configurationViewModel, opt =>
{
opt.Items[nameof(FundingConfiguration.FundingStreamId)] = fundingStreamId;
opt.Items[nameof(FundingConfiguration.FundingPeriodId)] = fundingPeriodId;
});
BadRequestObjectResult validationResult = (await _fundingConfigurationValidator.ValidateAsync(fundingConfiguration)).PopulateModelState();
if (validationResult != null)
{
return(validationResult);
}
//TODO; add validation on existence of template (what is a template exactly) when supplied as the default template id (funding template I'm guessing)
try
{
HttpStatusCode result = await _policyRepositoryPolicy.ExecuteAsync(() => _policyRepository.SaveFundingConfiguration(fundingConfiguration));
if (!result.IsSuccess())
{
int statusCode = (int)result;
string errorMessage = $"Failed to save configuration file for funding stream id: {fundingStreamId} and period id: {fundingPeriodId} to cosmos db with status {statusCode}";
_logger.Error(errorMessage);
return(new InternalServerErrorResult(errorMessage));
}
}
catch (Exception exception)
{
string errorMessage = $"Exception occurred writing to configuration file for funding stream id: {fundingStreamId} and period id: {fundingPeriodId} to cosmos db";
_logger.Error(exception, errorMessage);
return(new InternalServerErrorResult(errorMessage));
}
string fundingPeriodFundingConfigurationCacheKey = $"{CacheKeys.FundingConfig}{fundingStreamId}-{fundingPeriodId}";
await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.SetAsync(fundingPeriodFundingConfigurationCacheKey, fundingConfiguration));
string fundingStreamFundingConfigurationCacheKey = $"{CacheKeys.FundingConfig}{fundingStreamId}";
await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.RemoveAsync <List <FundingConfiguration> >(fundingStreamFundingConfigurationCacheKey));
_logger.Information($"Successfully saved configuration file for funding stream id: {fundingStreamId} and period id: {fundingPeriodId} to cosmos db");
return(new CreatedAtActionResult(actionName, controllerName, new { fundingStreamId, fundingPeriodId }, string.Empty));
}
public void SetFundingConfiguration(string fundingStreamId, string fundingPeriodId,
FundingConfiguration fundingConfiguration)
{
_fundingConfigurations[$"{fundingStreamId}-{fundingPeriodId}"] = fundingConfiguration;
}
private async Task WhenErrorsAreDetectedOnThePublishedProvider(PublishedProvider publishedProvider, IEnumerable <Provider> providers, string specificationId, string providerVersionId, FundingConfiguration fundingConfiguration, IEnumerable <PublishedFunding> publishedFundings)
{
PublishedProvidersContext publishedProvidersContext = new PublishedProvidersContext
{
ScopedProviders = providers,
SpecificationId = specificationId,
ProviderVersionId = providerVersionId,
CurrentPublishedFunding = publishedFundings,
OrganisationGroupResultsData = new Dictionary <string, HashSet <string> >(),
FundingConfiguration = fundingConfiguration
};
await _errorDetector.DetectErrors(publishedProvider, publishedProvidersContext);
}