/// <summary> /// Generate instances of the PublishedFundingVersion to save into cosmos for the Organisation Group Results /// </summary> /// <param name="publishedFundingInput"></param> /// <param name="publishedProviders"></param> /// <returns></returns> public IEnumerable <(PublishedFunding, PublishedFundingVersion)> GeneratePublishedFunding(PublishedFundingInput publishedFundingInput, IEnumerable <PublishedProvider> publishedProviders) { Guard.ArgumentNotNull(publishedFundingInput, nameof(publishedFundingInput)); Guard.ArgumentNotNull(publishedFundingInput.FundingPeriod, nameof(publishedFundingInput.FundingPeriod)); Guard.ArgumentNotNull(publishedFundingInput.FundingStream, nameof(publishedFundingInput.FundingStream)); Guard.ArgumentNotNull(publishedFundingInput.OrganisationGroupsToSave, nameof(publishedFundingInput.OrganisationGroupsToSave)); Guard.ArgumentNotNull(publishedProviders, nameof(publishedProviders)); Guard.ArgumentNotNull(publishedFundingInput.PublishingDates, nameof(publishedFundingInput.PublishingDates)); Guard.ArgumentNotNull(publishedFundingInput.TemplateMetadataContents, nameof(publishedFundingInput.TemplateMetadataContents)); Guard.IsNullOrWhiteSpace(publishedFundingInput.TemplateVersion, nameof(publishedFundingInput.TemplateVersion)); Guard.IsNullOrWhiteSpace(publishedFundingInput.SpecificationId, nameof(publishedFundingInput.SpecificationId)); IEnumerable <(PublishedFunding PublishedFunding, OrganisationGroupResult OrganisationGroupResult)> organisationGroupsToSave = publishedFundingInput.OrganisationGroupsToSave; TemplateMetadataContents templateMetadataContents = publishedFundingInput.TemplateMetadataContents; string templateVersion = publishedFundingInput.TemplateVersion; FundingPeriod fundingPeriod = publishedFundingInput.FundingPeriod; FundingValueAggregator fundingValueAggregator = new FundingValueAggregator(); foreach ((PublishedFunding PublishedFunding, OrganisationGroupResult OrganisationGroupResult)organisationGroup in organisationGroupsToSave) { // TODO: extract interface IEnumerable <string> providerIds = organisationGroup.OrganisationGroupResult.Providers.Select(p => p.ProviderId); IEnumerable <string> publishedProvidersIds = publishedProviders.Select(p => p.Current.ProviderId); List <PublishedProvider> publishedProvidersForOrganisationGroup = new List <PublishedProvider>(publishedProviders.Where(p => providerIds.Contains(p.Current.ProviderId))); List <PublishedProviderVersion> publishedProviderVersionsForOrganisationGroup = new List <PublishedProviderVersion>( publishedProvidersForOrganisationGroup.Select(p => p.Current)); IEnumerable <string> missingProviders = providerIds.Except(publishedProvidersIds); if (missingProviders.AnyWithNullCheck()) { string providerIdsString = string.Join(", ", missingProviders); throw new Exception($"Missing PublishedProvider result for organisation group '{organisationGroup.OrganisationGroupResult.GroupReason}' '{organisationGroup.OrganisationGroupResult.GroupTypeCode}' '{organisationGroup.OrganisationGroupResult.GroupTypeIdentifier}' '{organisationGroup.OrganisationGroupResult.IdentifierValue}'. Provider IDs={providerIdsString}"); } List <AggregateFundingLine> fundingLineAggregates = new List <AggregateFundingLine>( fundingValueAggregator.GetTotals(templateMetadataContents, publishedProviderVersionsForOrganisationGroup)); IEnumerable <Common.TemplateMetadata.Models.FundingLine> fundingLineDefinitions = templateMetadataContents.RootFundingLines.Flatten(_ => _.FundingLines) ?? Enumerable.Empty <Common.TemplateMetadata.Models.FundingLine>(); // Add in calculations in numerator/demoninator and percentagechange targets List <PublishingModels.FundingLine> fundingLines = GenerateFundingLines(fundingLineAggregates, fundingLineDefinitions); List <FundingCalculation> calculations = GenerateCalculations(fundingLineAggregates.Flatten(_ => _.FundingLines) .SelectMany(c => c.Calculations ?? Enumerable.Empty <AggregateFundingCalculation>())); decimal?totalFunding = publishedProviderVersionsForOrganisationGroup.Sum(_ => _.TotalFunding); PublishedFundingVersion publishedFundingVersion = new PublishedFundingVersion { FundingStreamId = publishedFundingInput.FundingStream.Id, FundingStreamName = publishedFundingInput.FundingStream.Name, TotalFunding = totalFunding, FundingPeriod = new PublishedFundingPeriod { Type = Enum.Parse <PublishedFundingPeriodType>(fundingPeriod.Type.GetValueOrDefault().ToString()), Period = fundingPeriod.Period, EndDate = fundingPeriod.EndDate, StartDate = fundingPeriod.StartDate, Name = fundingPeriod.Name, }, SpecificationId = publishedFundingInput.SpecificationId, OrganisationGroupTypeCode = organisationGroup.OrganisationGroupResult.GroupTypeCode.ToString(), OrganisationGroupTypeIdentifier = organisationGroup.OrganisationGroupResult.GroupTypeIdentifier.ToString(), OrganisationGroupIdentifierValue = organisationGroup.OrganisationGroupResult.IdentifierValue, OrganisationGroupTypeClassification = organisationGroup.OrganisationGroupResult.GroupTypeClassification.ToString(), OrganisationGroupName = organisationGroup.OrganisationGroupResult.Name, OrganisationGroupSearchableName = organisationGroup.OrganisationGroupResult.SearchableName, OrganisationGroupIdentifiers = _mapper.Map <IEnumerable <PublishedOrganisationGroupTypeIdentifier> >(organisationGroup.OrganisationGroupResult.Identifiers), FundingLines = fundingLines, Calculations = calculations, SchemaVersion = templateMetadataContents.SchemaVersion, Status = PublishedFundingStatus.Approved, GroupingReason = organisationGroup.OrganisationGroupResult.GroupReason.AsMatchingEnum <PublishingModels.GroupingReason>(), ProviderFundings = publishedProviderVersionsForOrganisationGroup.Select(_ => _.FundingId), TemplateVersion = templateVersion, StatusChangedDate = publishedFundingInput.PublishingDates.StatusChangedDate.TrimToTheSecond(), EarliestPaymentAvailableDate = publishedFundingInput.PublishingDates.EarliestPaymentAvailableDate.TrimToTheMinute(), ExternalPublicationDate = publishedFundingInput.PublishingDates.ExternalPublicationDate.TrimToTheMinute(), }; publishedFundingVersion.FundingId = _publishedFundingIdGeneratorResolver.GetService(templateMetadataContents.SchemaVersion).GetFundingId(publishedFundingVersion); PublishedFunding publishedFundingResult = organisationGroup.PublishedFunding; if (publishedFundingResult == null) { publishedFundingResult = new PublishedFunding() { Current = publishedFundingVersion, }; } yield return(publishedFundingResult, publishedFundingVersion); } }
private async Task PublishFundingStream(Reference fundingStream, SpecificationSummary specification, string jobId, Reference author, string correlationId, PrerequisiteCheckerType prerequisiteCheckerType, string[] batchPublishedProviderIds = null) { _logger.Information($"Processing Publish Funding for {fundingStream.Id} in specification {specification.Id}"); if (!specification.TemplateIds.ContainsKey(fundingStream.Id) || string.IsNullOrWhiteSpace(specification.TemplateIds[fundingStream.Id])) { _logger.Information($"Skipped publishing {fundingStream.Id} as no template exists"); return; } // we always need to get every provider in scope whether it is released or otherwise so that we always genarate the contents // this is just in case an error has occurred during a release so we never get a case where we don't get blobs generated for the published providers (IDictionary <string, PublishedProvider> publishedProvidersForFundingStream, IDictionary <string, PublishedProvider> scopedPublishedProviders) = await _providerService.GetPublishedProviders(fundingStream, specification); IDictionary <string, PublishedProvider> publishedProvidersByPublishedProviderId = publishedProvidersForFundingStream.Values.ToDictionary(_ => _.PublishedProviderId); IEnumerable <PublishedProvider> selectedPublishedProviders = batchPublishedProviderIds.IsNullOrEmpty() ? publishedProvidersForFundingStream.Values : batchPublishedProviderIds.Where(_ => publishedProvidersByPublishedProviderId.ContainsKey(_)).Select(_ => publishedProvidersByPublishedProviderId[_]); AddInitialPublishVariationReasons(selectedPublishedProviders); _logger.Information($"Verifying prerequisites for funding publish"); IPrerequisiteChecker prerequisiteChecker = _prerequisiteCheckerLocator.GetPreReqChecker(prerequisiteCheckerType); try { await prerequisiteChecker.PerformChecks(specification, jobId, selectedPublishedProviders?.ToList()); } catch (JobPrereqFailedException ex) { throw new NonRetriableException(ex.Message, ex); } _logger.Information("Prerequisites for publish passed"); TemplateMapping templateMapping = await GetTemplateMapping(fundingStream, specification.Id); PublishedFundingInput publishedFundingInput = await _publishedFundingService.GeneratePublishedFundingInput(publishedProvidersForFundingStream, scopedPublishedProviders?.Values.Select(_ => _.Current.Provider), fundingStream, specification, batchPublishedProviderIds.IsNullOrEmpty()?null : selectedPublishedProviders); using Transaction transaction = _transactionFactory.NewTransaction <PublishService>(); try { // if any error occurs while updating or indexing then we need to re-index all published providers and persist published funding for consistency transaction.Enroll(async() => { await _publishedProviderVersionService.CreateReIndexJob(author, correlationId, specification.Id, jobId); await _createPublishIntegrityJob.CreateJob(specification.Id, author, correlationId, batchPublishedProviderIds.IsNullOrEmpty() ? null : new Dictionary <string, string> { { "providers-batch", JsonExtensions.AsJson(selectedPublishedProviders.Select(_ => _.PublishedProviderId)) } }, parentJobId: jobId); }); await SavePublishedProvidersAsReleased(jobId, author, selectedPublishedProviders, correlationId); ICollection <PublishedProvider> publishedProviders = publishedProvidersForFundingStream?.Values; _logger.Information($"Generating published funding"); IEnumerable <(PublishedFunding PublishedFunding, PublishedFundingVersion PublishedFundingVersion)> publishedFundingToSave = _publishedFundingGenerator.GeneratePublishedFunding(publishedFundingInput, publishedProviders).ToList(); _logger.Information($"A total of {publishedFundingToSave.Count()} published funding versions created to save."); foreach ((PublishedFunding PublishedFunding, PublishedFundingVersion PublishedFundingVersion)publishedFundingItems in publishedFundingToSave) { PropagateProviderVariationReasons(publishedFundingItems.PublishedFundingVersion, publishedProviders); } // if any error occurs while updating then we still need to run the indexer to be consistent transaction.Enroll(async() => { await _publishedIndexSearchResiliencePolicy.ExecuteAsync(() => _publishedFundingSearchRepository.RunIndexer()); }); // Save a version of published funding and set this version to current _logger.Information($"Saving published funding"); await _publishedFundingStatusUpdateService.UpdatePublishedFundingStatus(publishedFundingToSave, author, PublishedFundingStatus.Released, jobId, correlationId); _logger.Information($"Finished saving published funding"); // Save contents to blob storage and search for the feed _logger.Information($"Saving published funding contents"); await _publishedFundingContentsPersistanceService.SavePublishedFundingContents(publishedFundingToSave.Select(_ => _.PublishedFundingVersion), publishedFundingInput.TemplateMetadataContents); _logger.Information($"Finished saving published funding contents"); if (!selectedPublishedProviders.IsNullOrEmpty()) { // Generate contents JSON for provider and save to blob storage IPublishedProviderContentsGenerator generator = _publishedProviderContentsGeneratorResolver.GetService(publishedFundingInput.TemplateMetadataContents.SchemaVersion); await _publishedProviderContentsPersistanceService.SavePublishedProviderContents(publishedFundingInput.TemplateMetadataContents, templateMapping, selectedPublishedProviders, generator); } transaction.Complete(); } catch (Exception ex) { await transaction.Compensate(); throw; } }