public void Register(string schemaVersion, IPublishedProviderContentsGenerator publishedProviderContentsGenerator)
{
Guard.IsNullOrWhiteSpace(schemaVersion, nameof(schemaVersion));
Guard.ArgumentNotNull(publishedProviderContentsGenerator, nameof(publishedProviderContentsGenerator));
_supportedVersions.TryAdd(schemaVersion, publishedProviderContentsGenerator);
}
public void ThrowsExceptionWhenContentGeneratorReturnsNull()
{
// Arrange
TemplateMetadataContents templateMetadataContents = Substitute.For <TemplateMetadataContents>();
TemplateMapping templateMapping = Substitute.For <TemplateMapping>();
IPublishedProviderContentsGenerator publishedProviderContentsGenerator = Substitute.For <IPublishedProviderContentsGenerator>();
Dictionary <string, GeneratedProviderResult> generatedPublishedProviderData = new Dictionary <string, GeneratedProviderResult>();
List <PublishedProvider> publishedProvidersToUpdate = new List <PublishedProvider>();
GeneratedProviderResult generatedProviderResult = new GeneratedProviderResult();
PublishedProviderVersion publishedProviderVersion = NewPublishedProviderVersion(providerVersion => providerVersion
.WithProviderId(ProviderVersionProviderId)
.WithFundingPeriodId(ProviderVersionFundingPeriodId)
.WithFundingStreamId(ProviderVersionFundingStreamId)
.WithVersion(ProviderVersionVersion));
PublishedProvider publishedProvider = NewPublishedProvider(provider => provider.WithCurrent(publishedProviderVersion));
generatedPublishedProviderData.Add(key, generatedProviderResult);
publishedProvidersToUpdate.Add(publishedProvider);
// Act
Func <Task> invocation = async() => await _publishedProviderContentPersistanceService.SavePublishedProviderContents(
templateMetadataContents, templateMapping, publishedProvidersToUpdate, publishedProviderContentsGenerator);
// Assert
ThenExceptionShouldBeThrown($"Generator failed to generate content for published provider version with id: '{publishedProviderVersion.Id}'", invocation);
}
public ProviderDocumentGenerator(IPublishedProviderContentsGenerator publishedProviderContentsGenerator, IPublishedFundingContentsGenerator publishedFundingContentsGenerator,
IProvidersApiClient providersApiClient, IOrganisationGroupResiliencePolicies organisationGroupResiliencePolicies, ILogger logger)
{
_publishedProviderContentsGenerator = publishedProviderContentsGenerator;
_publishedFundingContentsGenerator = publishedFundingContentsGenerator;
_logger = logger;
organisationGroupTargetProviderLookup = new OrganisationGroupTargetProviderLookup(providersApiClient, organisationGroupResiliencePolicies);
}
public void ThrowsExceptionWhenPublishedProviderVersionServiceThrowsException()
{
// Arrange
TemplateMetadataContents templateMetadataContents = Substitute.For <TemplateMetadataContents>();
TemplateMapping templateMapping = Substitute.For <TemplateMapping>();
IPublishedProviderContentsGenerator publishedProviderContentsGenerator = Substitute.For <IPublishedProviderContentsGenerator>();
List <PublishedProvider> publishedProvidersToUpdate = new List <PublishedProvider>();
PublishedProviderVersion publishedProviderVersion = NewPublishedProviderVersion(providerVersion => providerVersion
.WithProviderId(ProviderVersionProviderId)
.WithFundingPeriodId(ProviderVersionFundingPeriodId)
.WithFundingStreamId(ProviderVersionFundingStreamId)
.WithVersion(ProviderVersionVersion)
.WithSpecificationId(ProviderVersionSpecificationId));
PublishedProvider publishedProvider = NewPublishedProvider(provider => provider.WithCurrent(publishedProviderVersion));
publishedProvidersToUpdate.Add(publishedProvider);
publishedProviderContentsGenerator
.GenerateContents(publishedProviderVersion, templateMetadataContents, templateMapping)
.Returns("contents");
string exceptionMessage = "Exception Message";
_publishedProviderVersionService
.SavePublishedProviderVersionBody(publishedProviderVersion.FundingId, Arg.Any <string>(), publishedProviderVersion.SpecificationId)
.Throws(new Exception(exceptionMessage));
// Act
Func <Task> invocation = async() => await _publishedProviderContentPersistanceService.SavePublishedProviderContents(
templateMetadataContents, templateMapping, publishedProvidersToUpdate, publishedProviderContentsGenerator);
// Assert
ThenExceptionShouldBeThrown(exceptionMessage, invocation);
}
public async Task SavesAndIndexesPublishedProviderVersionWhenCorrectInputGiven()
{
// Arrange
TemplateMetadataContents templateMetadataContents = Substitute.For <TemplateMetadataContents>();
TemplateMapping templateMapping = Substitute.For <TemplateMapping>();
IPublishedProviderContentsGenerator publishedProviderContentsGenerator = Substitute.For <IPublishedProviderContentsGenerator>();
List <PublishedProvider> publishedProvidersToUpdate = new List <PublishedProvider>();
PublishedProviderVersion publishedProviderVersion = NewPublishedProviderVersion(providerVersion => providerVersion
.WithProviderId(ProviderVersionProviderId)
.WithFundingPeriodId(ProviderVersionFundingPeriodId)
.WithFundingStreamId(ProviderVersionFundingStreamId)
.WithVersion(ProviderVersionVersion)
.WithSpecificationId(ProviderVersionSpecificationId));
PublishedProvider publishedProvider = NewPublishedProvider(provider => provider.WithCurrent(publishedProviderVersion));
publishedProvidersToUpdate.Add(publishedProvider);
publishedProviderContentsGenerator
.GenerateContents(publishedProviderVersion, templateMetadataContents, templateMapping)
.Returns("contents");
// Act
await _publishedProviderContentPersistanceService.SavePublishedProviderContents(
templateMetadataContents, templateMapping, publishedProvidersToUpdate, publishedProviderContentsGenerator);
// Assert
await _publishedProviderVersionService
.Received()
.SavePublishedProviderVersionBody(publishedProviderVersion.FundingId, Arg.Any <string>(), publishedProviderVersion.SpecificationId);
await _publishedProviderIndexerService
.Received()
.IndexPublishedProvider(publishedProviderVersion);
}
public override async Task Process(Message message)
{
Guard.ArgumentNotNull(message, nameof(message));
Reference author = message.GetUserDetails();
string specificationId = message.UserProperties["specification-id"] as string;
bool publishAll = false;
if (message.UserProperties.ContainsKey("publish-all"))
{
publishAll = bool.Parse(message.UserProperties["publish-all"].ToString());
}
IEnumerable <string> batchProviders = null;
if (message.UserProperties.ContainsKey("providers-batch"))
{
batchProviders = JsonExtensions.AsPoco <IEnumerable <string> >(message.UserProperties["providers-batch"].ToString());
}
SpecificationSummary specification = await _specificationService.GetSpecificationSummaryById(specificationId);
if (specification == null)
{
throw new NonRetriableException($"Could not find specification with id '{specificationId}'");
}
foreach (Reference fundingStream in specification.FundingStreams)
{
(IDictionary <string, PublishedProvider> publishedProvidersForFundingStream,
IDictionary <string, PublishedProvider> scopedPublishedProviders) = await _providerService.GetPublishedProviders(fundingStream,
specification);
IDictionary <string, PublishedProvider> publishedProvidersByPublishedProviderId = publishedProvidersForFundingStream.Values.ToDictionary(_ => _.PublishedProviderId);
IEnumerable <PublishedProvider> selectedPublishedProviders =
batchProviders.IsNullOrEmpty() ?
publishedProvidersForFundingStream.Values :
batchProviders.Where(_ => publishedProvidersByPublishedProviderId.ContainsKey(_)).Select(_ => publishedProvidersByPublishedProviderId[_]);
TemplateMapping templateMapping = await GetTemplateMapping(fundingStream, specification.Id);
IEnumerable <PublishedFundingVersion> publishedFundingVersions = publishAll ?
await _publishedFundingVersionDataService.GetPublishedFundingVersion(fundingStream.Id, specification.FundingPeriod.Id) :
(await _publishingResiliencePolicy.ExecuteAsync(() =>
_publishedFundingDataService.GetCurrentPublishedFunding(fundingStream.Id, specification.FundingPeriod.Id))).Select(_ => _.Current);
TemplateMetadataContents templateMetadataContents = await _policiesService.GetTemplateMetadataContents(fundingStream.Id, specification.FundingPeriod.Id, specification.TemplateIds[fundingStream.Id]);
if (templateMetadataContents == null)
{
throw new NonRetriableException($"Unable to get template metadata contents for funding stream. '{fundingStream.Id}'");
}
// Save contents to blob storage and search for the feed
_logger.Information($"Saving published funding contents");
await _publishedFundingContentsPersistanceService.SavePublishedFundingContents(publishedFundingVersions,
templateMetadataContents);
_logger.Information($"Finished saving published funding contents");
// Generate contents JSON for provider and save to blob storage
IPublishedProviderContentsGenerator generator = _publishedProviderContentsGeneratorResolver.GetService(templateMetadataContents.SchemaVersion);
await _publishedProviderContentsPersistanceService.SavePublishedProviderContents(templateMetadataContents, templateMapping,
selectedPublishedProviders, generator, publishAll);
}
}
public async Task SavePublishedProviderContents(TemplateMetadataContents templateMetadataContents, TemplateMapping templateMapping, IEnumerable <PublishedProvider> publishedProvidersToUpdate, IPublishedProviderContentsGenerator generator, bool publishAll = false)
{
_logger.Information("Saving published provider contents");
List <Task> allTasks = new List <Task>();
SemaphoreSlim throttler = new SemaphoreSlim(initialCount: _publishingEngineOptions.SavePublishedProviderContentsConcurrencyCount);
foreach (PublishedProvider provider in publishedProvidersToUpdate)
{
await throttler.WaitAsync();
allTasks.Add(
Task.Run(async() =>
{
try
{
IEnumerable <PublishedProviderVersion> publishedProviderVersions = publishAll ?
await _publishedProviderVersioningService.GetVersions(provider) :
new[] { provider.Current };
foreach (PublishedProviderVersion publishedProviderVersion in publishedProviderVersions)
{
string contents = generator.GenerateContents(publishedProviderVersion, templateMetadataContents, templateMapping);
if (string.IsNullOrWhiteSpace(contents))
{
throw new RetriableException($"Generator failed to generate content for published provider version with id: '{publishedProviderVersion.Id}'");
}
try
{
await _publishedProviderVersionService.SavePublishedProviderVersionBody(
publishedProviderVersion.FundingId, contents, publishedProviderVersion.SpecificationId);
}
catch (Exception ex)
{
throw new RetriableException(ex.Message);
}
try
{
await _publishedProviderIndexerService.IndexPublishedProvider(publishedProviderVersion);
}
catch (Exception ex)
{
throw new RetriableException(ex.Message);
}
}
}
finally
{
throttler.Release();
}
}));
}
await TaskHelper.WhenAllAndThrow(allTasks.ToArray());
}
public bool TryGetService(string schemaVersion, out IPublishedProviderContentsGenerator publishedProviderContentsGenerator)
{
return(_supportedVersions.TryGetValue(schemaVersion, out publishedProviderContentsGenerator));
}
private async Task PublishFundingStream(Reference fundingStream,
SpecificationSummary specification,
string jobId,
Reference author,
string correlationId,
PrerequisiteCheckerType prerequisiteCheckerType,
string[] batchPublishedProviderIds = null)
{
_logger.Information($"Processing Publish Funding for {fundingStream.Id} in specification {specification.Id}");
if (!specification.TemplateIds.ContainsKey(fundingStream.Id) || string.IsNullOrWhiteSpace(specification.TemplateIds[fundingStream.Id]))
{
_logger.Information($"Skipped publishing {fundingStream.Id} as no template exists");
return;
}
// we always need to get every provider in scope whether it is released or otherwise so that we always genarate the contents
// this is just in case an error has occurred during a release so we never get a case where we don't get blobs generated for the published providers
(IDictionary <string, PublishedProvider> publishedProvidersForFundingStream,
IDictionary <string, PublishedProvider> scopedPublishedProviders) = await _providerService.GetPublishedProviders(fundingStream,
specification);
IDictionary <string, PublishedProvider> publishedProvidersByPublishedProviderId = publishedProvidersForFundingStream.Values.ToDictionary(_ => _.PublishedProviderId);
IEnumerable <PublishedProvider> selectedPublishedProviders =
batchPublishedProviderIds.IsNullOrEmpty() ?
publishedProvidersForFundingStream.Values :
batchPublishedProviderIds.Where(_ => publishedProvidersByPublishedProviderId.ContainsKey(_)).Select(_ => publishedProvidersByPublishedProviderId[_]);
AddInitialPublishVariationReasons(selectedPublishedProviders);
_logger.Information($"Verifying prerequisites for funding publish");
IPrerequisiteChecker prerequisiteChecker = _prerequisiteCheckerLocator.GetPreReqChecker(prerequisiteCheckerType);
try
{
await prerequisiteChecker.PerformChecks(specification, jobId, selectedPublishedProviders?.ToList());
}
catch (JobPrereqFailedException ex)
{
throw new NonRetriableException(ex.Message, ex);
}
_logger.Information("Prerequisites for publish passed");
TemplateMapping templateMapping = await GetTemplateMapping(fundingStream, specification.Id);
PublishedFundingInput publishedFundingInput = await _publishedFundingService.GeneratePublishedFundingInput(publishedProvidersForFundingStream,
scopedPublishedProviders?.Values.Select(_ => _.Current.Provider),
fundingStream,
specification,
batchPublishedProviderIds.IsNullOrEmpty()?null : selectedPublishedProviders);
using Transaction transaction = _transactionFactory.NewTransaction <PublishService>();
try
{
// if any error occurs while updating or indexing then we need to re-index all published providers and persist published funding for consistency
transaction.Enroll(async() =>
{
await _publishedProviderVersionService.CreateReIndexJob(author, correlationId, specification.Id, jobId);
await _createPublishIntegrityJob.CreateJob(specification.Id,
author,
correlationId,
batchPublishedProviderIds.IsNullOrEmpty() ? null : new Dictionary <string, string>
{
{ "providers-batch", JsonExtensions.AsJson(selectedPublishedProviders.Select(_ => _.PublishedProviderId)) }
},
parentJobId: jobId);
});
await SavePublishedProvidersAsReleased(jobId, author, selectedPublishedProviders, correlationId);
ICollection <PublishedProvider> publishedProviders = publishedProvidersForFundingStream?.Values;
_logger.Information($"Generating published funding");
IEnumerable <(PublishedFunding PublishedFunding, PublishedFundingVersion PublishedFundingVersion)> publishedFundingToSave =
_publishedFundingGenerator.GeneratePublishedFunding(publishedFundingInput, publishedProviders).ToList();
_logger.Information($"A total of {publishedFundingToSave.Count()} published funding versions created to save.");
foreach ((PublishedFunding PublishedFunding, PublishedFundingVersion PublishedFundingVersion)publishedFundingItems in publishedFundingToSave)
{
PropagateProviderVariationReasons(publishedFundingItems.PublishedFundingVersion, publishedProviders);
}
// if any error occurs while updating then we still need to run the indexer to be consistent
transaction.Enroll(async() =>
{
await _publishedIndexSearchResiliencePolicy.ExecuteAsync(() => _publishedFundingSearchRepository.RunIndexer());
});
// Save a version of published funding and set this version to current
_logger.Information($"Saving published funding");
await _publishedFundingStatusUpdateService.UpdatePublishedFundingStatus(publishedFundingToSave, author, PublishedFundingStatus.Released, jobId, correlationId);
_logger.Information($"Finished saving published funding");
// Save contents to blob storage and search for the feed
_logger.Information($"Saving published funding contents");
await _publishedFundingContentsPersistanceService.SavePublishedFundingContents(publishedFundingToSave.Select(_ => _.PublishedFundingVersion),
publishedFundingInput.TemplateMetadataContents);
_logger.Information($"Finished saving published funding contents");
if (!selectedPublishedProviders.IsNullOrEmpty())
{
// Generate contents JSON for provider and save to blob storage
IPublishedProviderContentsGenerator generator = _publishedProviderContentsGeneratorResolver.GetService(publishedFundingInput.TemplateMetadataContents.SchemaVersion);
await _publishedProviderContentsPersistanceService.SavePublishedProviderContents(publishedFundingInput.TemplateMetadataContents, templateMapping,
selectedPublishedProviders, generator);
}
transaction.Complete();
}
catch (Exception ex)
{
await transaction.Compensate();
throw;
}
}
