public void Register(string schemaVersion, IPublishedProviderContentsGenerator publishedProviderContentsGenerator) { Guard.IsNullOrWhiteSpace(schemaVersion, nameof(schemaVersion)); Guard.ArgumentNotNull(publishedProviderContentsGenerator, nameof(publishedProviderContentsGenerator)); _supportedVersions.TryAdd(schemaVersion, publishedProviderContentsGenerator); }
public void ThrowsExceptionWhenContentGeneratorReturnsNull() { // Arrange TemplateMetadataContents templateMetadataContents = Substitute.For <TemplateMetadataContents>(); TemplateMapping templateMapping = Substitute.For <TemplateMapping>(); IPublishedProviderContentsGenerator publishedProviderContentsGenerator = Substitute.For <IPublishedProviderContentsGenerator>(); Dictionary <string, GeneratedProviderResult> generatedPublishedProviderData = new Dictionary <string, GeneratedProviderResult>(); List <PublishedProvider> publishedProvidersToUpdate = new List <PublishedProvider>(); GeneratedProviderResult generatedProviderResult = new GeneratedProviderResult(); PublishedProviderVersion publishedProviderVersion = NewPublishedProviderVersion(providerVersion => providerVersion .WithProviderId(ProviderVersionProviderId) .WithFundingPeriodId(ProviderVersionFundingPeriodId) .WithFundingStreamId(ProviderVersionFundingStreamId) .WithVersion(ProviderVersionVersion)); PublishedProvider publishedProvider = NewPublishedProvider(provider => provider.WithCurrent(publishedProviderVersion)); generatedPublishedProviderData.Add(key, generatedProviderResult); publishedProvidersToUpdate.Add(publishedProvider); // Act Func <Task> invocation = async() => await _publishedProviderContentPersistanceService.SavePublishedProviderContents( templateMetadataContents, templateMapping, publishedProvidersToUpdate, publishedProviderContentsGenerator); // Assert ThenExceptionShouldBeThrown($"Generator failed to generate content for published provider version with id: '{publishedProviderVersion.Id}'", invocation); }
public ProviderDocumentGenerator(IPublishedProviderContentsGenerator publishedProviderContentsGenerator, IPublishedFundingContentsGenerator publishedFundingContentsGenerator, IProvidersApiClient providersApiClient, IOrganisationGroupResiliencePolicies organisationGroupResiliencePolicies, ILogger logger) { _publishedProviderContentsGenerator = publishedProviderContentsGenerator; _publishedFundingContentsGenerator = publishedFundingContentsGenerator; _logger = logger; organisationGroupTargetProviderLookup = new OrganisationGroupTargetProviderLookup(providersApiClient, organisationGroupResiliencePolicies); }
public void ThrowsExceptionWhenPublishedProviderVersionServiceThrowsException() { // Arrange TemplateMetadataContents templateMetadataContents = Substitute.For <TemplateMetadataContents>(); TemplateMapping templateMapping = Substitute.For <TemplateMapping>(); IPublishedProviderContentsGenerator publishedProviderContentsGenerator = Substitute.For <IPublishedProviderContentsGenerator>(); List <PublishedProvider> publishedProvidersToUpdate = new List <PublishedProvider>(); PublishedProviderVersion publishedProviderVersion = NewPublishedProviderVersion(providerVersion => providerVersion .WithProviderId(ProviderVersionProviderId) .WithFundingPeriodId(ProviderVersionFundingPeriodId) .WithFundingStreamId(ProviderVersionFundingStreamId) .WithVersion(ProviderVersionVersion) .WithSpecificationId(ProviderVersionSpecificationId)); PublishedProvider publishedProvider = NewPublishedProvider(provider => provider.WithCurrent(publishedProviderVersion)); publishedProvidersToUpdate.Add(publishedProvider); publishedProviderContentsGenerator .GenerateContents(publishedProviderVersion, templateMetadataContents, templateMapping) .Returns("contents"); string exceptionMessage = "Exception Message"; _publishedProviderVersionService .SavePublishedProviderVersionBody(publishedProviderVersion.FundingId, Arg.Any <string>(), publishedProviderVersion.SpecificationId) .Throws(new Exception(exceptionMessage)); // Act Func <Task> invocation = async() => await _publishedProviderContentPersistanceService.SavePublishedProviderContents( templateMetadataContents, templateMapping, publishedProvidersToUpdate, publishedProviderContentsGenerator); // Assert ThenExceptionShouldBeThrown(exceptionMessage, invocation); }
public async Task SavesAndIndexesPublishedProviderVersionWhenCorrectInputGiven() { // Arrange TemplateMetadataContents templateMetadataContents = Substitute.For <TemplateMetadataContents>(); TemplateMapping templateMapping = Substitute.For <TemplateMapping>(); IPublishedProviderContentsGenerator publishedProviderContentsGenerator = Substitute.For <IPublishedProviderContentsGenerator>(); List <PublishedProvider> publishedProvidersToUpdate = new List <PublishedProvider>(); PublishedProviderVersion publishedProviderVersion = NewPublishedProviderVersion(providerVersion => providerVersion .WithProviderId(ProviderVersionProviderId) .WithFundingPeriodId(ProviderVersionFundingPeriodId) .WithFundingStreamId(ProviderVersionFundingStreamId) .WithVersion(ProviderVersionVersion) .WithSpecificationId(ProviderVersionSpecificationId)); PublishedProvider publishedProvider = NewPublishedProvider(provider => provider.WithCurrent(publishedProviderVersion)); publishedProvidersToUpdate.Add(publishedProvider); publishedProviderContentsGenerator .GenerateContents(publishedProviderVersion, templateMetadataContents, templateMapping) .Returns("contents"); // Act await _publishedProviderContentPersistanceService.SavePublishedProviderContents( templateMetadataContents, templateMapping, publishedProvidersToUpdate, publishedProviderContentsGenerator); // Assert await _publishedProviderVersionService .Received() .SavePublishedProviderVersionBody(publishedProviderVersion.FundingId, Arg.Any <string>(), publishedProviderVersion.SpecificationId); await _publishedProviderIndexerService .Received() .IndexPublishedProvider(publishedProviderVersion); }
public override async Task Process(Message message) { Guard.ArgumentNotNull(message, nameof(message)); Reference author = message.GetUserDetails(); string specificationId = message.UserProperties["specification-id"] as string; bool publishAll = false; if (message.UserProperties.ContainsKey("publish-all")) { publishAll = bool.Parse(message.UserProperties["publish-all"].ToString()); } IEnumerable <string> batchProviders = null; if (message.UserProperties.ContainsKey("providers-batch")) { batchProviders = JsonExtensions.AsPoco <IEnumerable <string> >(message.UserProperties["providers-batch"].ToString()); } SpecificationSummary specification = await _specificationService.GetSpecificationSummaryById(specificationId); if (specification == null) { throw new NonRetriableException($"Could not find specification with id '{specificationId}'"); } foreach (Reference fundingStream in specification.FundingStreams) { (IDictionary <string, PublishedProvider> publishedProvidersForFundingStream, IDictionary <string, PublishedProvider> scopedPublishedProviders) = await _providerService.GetPublishedProviders(fundingStream, specification); IDictionary <string, PublishedProvider> publishedProvidersByPublishedProviderId = publishedProvidersForFundingStream.Values.ToDictionary(_ => _.PublishedProviderId); IEnumerable <PublishedProvider> selectedPublishedProviders = batchProviders.IsNullOrEmpty() ? publishedProvidersForFundingStream.Values : batchProviders.Where(_ => publishedProvidersByPublishedProviderId.ContainsKey(_)).Select(_ => publishedProvidersByPublishedProviderId[_]); TemplateMapping templateMapping = await GetTemplateMapping(fundingStream, specification.Id); IEnumerable <PublishedFundingVersion> publishedFundingVersions = publishAll ? await _publishedFundingVersionDataService.GetPublishedFundingVersion(fundingStream.Id, specification.FundingPeriod.Id) : (await _publishingResiliencePolicy.ExecuteAsync(() => _publishedFundingDataService.GetCurrentPublishedFunding(fundingStream.Id, specification.FundingPeriod.Id))).Select(_ => _.Current); TemplateMetadataContents templateMetadataContents = await _policiesService.GetTemplateMetadataContents(fundingStream.Id, specification.FundingPeriod.Id, specification.TemplateIds[fundingStream.Id]); if (templateMetadataContents == null) { throw new NonRetriableException($"Unable to get template metadata contents for funding stream. '{fundingStream.Id}'"); } // Save contents to blob storage and search for the feed _logger.Information($"Saving published funding contents"); await _publishedFundingContentsPersistanceService.SavePublishedFundingContents(publishedFundingVersions, templateMetadataContents); _logger.Information($"Finished saving published funding contents"); // Generate contents JSON for provider and save to blob storage IPublishedProviderContentsGenerator generator = _publishedProviderContentsGeneratorResolver.GetService(templateMetadataContents.SchemaVersion); await _publishedProviderContentsPersistanceService.SavePublishedProviderContents(templateMetadataContents, templateMapping, selectedPublishedProviders, generator, publishAll); } }
public async Task SavePublishedProviderContents(TemplateMetadataContents templateMetadataContents, TemplateMapping templateMapping, IEnumerable <PublishedProvider> publishedProvidersToUpdate, IPublishedProviderContentsGenerator generator, bool publishAll = false) { _logger.Information("Saving published provider contents"); List <Task> allTasks = new List <Task>(); SemaphoreSlim throttler = new SemaphoreSlim(initialCount: _publishingEngineOptions.SavePublishedProviderContentsConcurrencyCount); foreach (PublishedProvider provider in publishedProvidersToUpdate) { await throttler.WaitAsync(); allTasks.Add( Task.Run(async() => { try { IEnumerable <PublishedProviderVersion> publishedProviderVersions = publishAll ? await _publishedProviderVersioningService.GetVersions(provider) : new[] { provider.Current }; foreach (PublishedProviderVersion publishedProviderVersion in publishedProviderVersions) { string contents = generator.GenerateContents(publishedProviderVersion, templateMetadataContents, templateMapping); if (string.IsNullOrWhiteSpace(contents)) { throw new RetriableException($"Generator failed to generate content for published provider version with id: '{publishedProviderVersion.Id}'"); } try { await _publishedProviderVersionService.SavePublishedProviderVersionBody( publishedProviderVersion.FundingId, contents, publishedProviderVersion.SpecificationId); } catch (Exception ex) { throw new RetriableException(ex.Message); } try { await _publishedProviderIndexerService.IndexPublishedProvider(publishedProviderVersion); } catch (Exception ex) { throw new RetriableException(ex.Message); } } } finally { throttler.Release(); } })); } await TaskHelper.WhenAllAndThrow(allTasks.ToArray()); }
public bool TryGetService(string schemaVersion, out IPublishedProviderContentsGenerator publishedProviderContentsGenerator) { return(_supportedVersions.TryGetValue(schemaVersion, out publishedProviderContentsGenerator)); }
private async Task PublishFundingStream(Reference fundingStream, SpecificationSummary specification, string jobId, Reference author, string correlationId, PrerequisiteCheckerType prerequisiteCheckerType, string[] batchPublishedProviderIds = null) { _logger.Information($"Processing Publish Funding for {fundingStream.Id} in specification {specification.Id}"); if (!specification.TemplateIds.ContainsKey(fundingStream.Id) || string.IsNullOrWhiteSpace(specification.TemplateIds[fundingStream.Id])) { _logger.Information($"Skipped publishing {fundingStream.Id} as no template exists"); return; } // we always need to get every provider in scope whether it is released or otherwise so that we always genarate the contents // this is just in case an error has occurred during a release so we never get a case where we don't get blobs generated for the published providers (IDictionary <string, PublishedProvider> publishedProvidersForFundingStream, IDictionary <string, PublishedProvider> scopedPublishedProviders) = await _providerService.GetPublishedProviders(fundingStream, specification); IDictionary <string, PublishedProvider> publishedProvidersByPublishedProviderId = publishedProvidersForFundingStream.Values.ToDictionary(_ => _.PublishedProviderId); IEnumerable <PublishedProvider> selectedPublishedProviders = batchPublishedProviderIds.IsNullOrEmpty() ? publishedProvidersForFundingStream.Values : batchPublishedProviderIds.Where(_ => publishedProvidersByPublishedProviderId.ContainsKey(_)).Select(_ => publishedProvidersByPublishedProviderId[_]); AddInitialPublishVariationReasons(selectedPublishedProviders); _logger.Information($"Verifying prerequisites for funding publish"); IPrerequisiteChecker prerequisiteChecker = _prerequisiteCheckerLocator.GetPreReqChecker(prerequisiteCheckerType); try { await prerequisiteChecker.PerformChecks(specification, jobId, selectedPublishedProviders?.ToList()); } catch (JobPrereqFailedException ex) { throw new NonRetriableException(ex.Message, ex); } _logger.Information("Prerequisites for publish passed"); TemplateMapping templateMapping = await GetTemplateMapping(fundingStream, specification.Id); PublishedFundingInput publishedFundingInput = await _publishedFundingService.GeneratePublishedFundingInput(publishedProvidersForFundingStream, scopedPublishedProviders?.Values.Select(_ => _.Current.Provider), fundingStream, specification, batchPublishedProviderIds.IsNullOrEmpty()?null : selectedPublishedProviders); using Transaction transaction = _transactionFactory.NewTransaction <PublishService>(); try { // if any error occurs while updating or indexing then we need to re-index all published providers and persist published funding for consistency transaction.Enroll(async() => { await _publishedProviderVersionService.CreateReIndexJob(author, correlationId, specification.Id, jobId); await _createPublishIntegrityJob.CreateJob(specification.Id, author, correlationId, batchPublishedProviderIds.IsNullOrEmpty() ? null : new Dictionary <string, string> { { "providers-batch", JsonExtensions.AsJson(selectedPublishedProviders.Select(_ => _.PublishedProviderId)) } }, parentJobId: jobId); }); await SavePublishedProvidersAsReleased(jobId, author, selectedPublishedProviders, correlationId); ICollection <PublishedProvider> publishedProviders = publishedProvidersForFundingStream?.Values; _logger.Information($"Generating published funding"); IEnumerable <(PublishedFunding PublishedFunding, PublishedFundingVersion PublishedFundingVersion)> publishedFundingToSave = _publishedFundingGenerator.GeneratePublishedFunding(publishedFundingInput, publishedProviders).ToList(); _logger.Information($"A total of {publishedFundingToSave.Count()} published funding versions created to save."); foreach ((PublishedFunding PublishedFunding, PublishedFundingVersion PublishedFundingVersion)publishedFundingItems in publishedFundingToSave) { PropagateProviderVariationReasons(publishedFundingItems.PublishedFundingVersion, publishedProviders); } // if any error occurs while updating then we still need to run the indexer to be consistent transaction.Enroll(async() => { await _publishedIndexSearchResiliencePolicy.ExecuteAsync(() => _publishedFundingSearchRepository.RunIndexer()); }); // Save a version of published funding and set this version to current _logger.Information($"Saving published funding"); await _publishedFundingStatusUpdateService.UpdatePublishedFundingStatus(publishedFundingToSave, author, PublishedFundingStatus.Released, jobId, correlationId); _logger.Information($"Finished saving published funding"); // Save contents to blob storage and search for the feed _logger.Information($"Saving published funding contents"); await _publishedFundingContentsPersistanceService.SavePublishedFundingContents(publishedFundingToSave.Select(_ => _.PublishedFundingVersion), publishedFundingInput.TemplateMetadataContents); _logger.Information($"Finished saving published funding contents"); if (!selectedPublishedProviders.IsNullOrEmpty()) { // Generate contents JSON for provider and save to blob storage IPublishedProviderContentsGenerator generator = _publishedProviderContentsGeneratorResolver.GetService(publishedFundingInput.TemplateMetadataContents.SchemaVersion); await _publishedProviderContentsPersistanceService.SavePublishedProviderContents(publishedFundingInput.TemplateMetadataContents, templateMapping, selectedPublishedProviders, generator); } transaction.Complete(); } catch (Exception ex) { await transaction.Compensate(); throw; } }