private async Task RefreshFundingStream(Reference fundingStream, SpecificationSummary specification, IDictionary <string, Provider> scopedProviders, IDictionary <string, ProviderCalculationResult> allCalculationResults, string jobId, Reference author, string correlationId, IEnumerable <PublishedProvider> existingPublishedProviders, string fundingPeriodId) { TemplateMetadataContents templateMetadataContents = await _policiesService.GetTemplateMetadataContents(fundingStream.Id, specification.FundingPeriod.Id, specification.TemplateIds[fundingStream.Id]); if (templateMetadataContents == null) { _logger.Information($"Unable to locate template meta data contents for funding stream:'{fundingStream.Id}' and template id:'{specification.TemplateIds[fundingStream.Id]}'"); return; } IEnumerable <ProfileVariationPointer> variationPointers = await _specificationService.GetProfileVariationPointers(specification.Id) ?? ArraySegment <ProfileVariationPointer> .Empty; Dictionary <string, PublishedProvider> publishedProviders = new Dictionary <string, PublishedProvider>(); foreach (PublishedProvider publishedProvider in existingPublishedProviders) { if (publishedProvider.Current.FundingStreamId == fundingStream.Id) { publishedProviders.Add(publishedProvider.Current.ProviderId, publishedProvider); } } // Create PublishedProvider for providers which don't already have a record (eg ProviderID-FundingStreamId-FundingPeriodId) IDictionary <string, PublishedProvider> newProviders = _providerService.GenerateMissingPublishedProviders(scopedProviders.Values, specification, fundingStream, publishedProviders); publishedProviders.AddRange(newProviders); // Get TemplateMapping for calcs from Calcs API client nuget ApiResponse <Common.ApiClient.Calcs.Models.TemplateMapping> calculationMappingResult = await _calculationsApiClientPolicy.ExecuteAsync(() => _calculationsApiClient.GetTemplateMapping(specification.Id, fundingStream.Id)); if (calculationMappingResult == null) { throw new Exception($"calculationMappingResult returned null for funding stream {fundingStream.Id}"); } Common.ApiClient.Calcs.Models.TemplateMapping templateMapping = calculationMappingResult.Content; _logger.Information("Generating PublishedProviders for refresh"); // Generate populated data for each provider in this funding line IDictionary <string, GeneratedProviderResult> generatedPublishedProviderData; try { generatedPublishedProviderData = _publishedProviderDataGenerator.Generate(templateMetadataContents, templateMapping, scopedProviders.Values, allCalculationResults); } catch (Exception ex) { _logger.Error(ex, "Exception during generating provider data"); throw; } _logger.Information("Populated PublishedProviders for refresh"); Dictionary <string, PublishedProvider> publishedProvidersToUpdate = new Dictionary <string, PublishedProvider>(); Dictionary <string, PublishedProvider> existingPublishedProvidersToUpdate = new Dictionary <string, PublishedProvider>(); FundingLine[] flattenedTemplateFundingLines = templateMetadataContents.RootFundingLines.Flatten(_ => _.FundingLines).ToArray(); _logger.Information("Profiling providers for refresh"); try { await ProfileProviders(publishedProviders, newProviders, generatedPublishedProviderData); } catch (Exception ex) { _logger.Error(ex, "Exception during generating provider profiling"); if (ex is NonRetriableException) { await _jobManagement.UpdateJobStatus(jobId, 0, 0, false, "Refresh job failed during generating provider profiling."); } throw; } _logger.Information("Finished profiling providers for refresh"); _logger.Information("Start snapshots for published provider variations"); // snapshot the current published providers so any changes aren't reflected when we detect variations later _variationService.SnapShot(publishedProviders, fundingStream.Id); _logger.Information("Finished snapshots for published provider variations"); //we need enumerate a readonly cut of this as we add to it in some variations now (for missing providers not in scope) Dictionary <string, PublishedProvider> publishedProvidersReadonlyDictionary = publishedProviders.ToDictionary(_ => _.Key, _ => _.Value); _logger.Information($"Start getting funding configuration for funding stream '{fundingStream.Id}'"); // set up the published providers context for error detection laterawait FundingConfiguration fundingConfiguration = await _policiesService.GetFundingConfiguration(fundingStream.Id, specification.FundingPeriod.Id); _logger.Information($"Retrieved funding stream configuration for '{fundingStream.Id}'"); PublishedProvidersContext publishedProvidersContext = new PublishedProvidersContext { ScopedProviders = scopedProviders.Values, SpecificationId = specification.Id, ProviderVersionId = specification.ProviderVersionId, CurrentPublishedFunding = (await _publishingResiliencePolicy.ExecuteAsync(() => _publishedFundingDataService.GetCurrentPublishedFunding(specification.Id, GroupingReason.Payment))) .Where(x => x.Current.GroupingReason == CalculateFunding.Models.Publishing.GroupingReason.Payment), OrganisationGroupResultsData = new Dictionary <string, HashSet <string> >(), FundingConfiguration = fundingConfiguration }; _logger.Information("Starting to process providers for variations and exclusions"); foreach (KeyValuePair <string, PublishedProvider> publishedProvider in publishedProvidersReadonlyDictionary) { PublishedProviderVersion publishedProviderVersion = publishedProvider.Value.Current; string providerId = publishedProviderVersion.ProviderId; // this could be a retry and the key may not exist as the provider has been created as a successor so we need to skip if (!generatedPublishedProviderData.ContainsKey(publishedProvider.Key)) { continue; } GeneratedProviderResult generatedProviderResult = generatedPublishedProviderData[publishedProvider.Key]; PublishedProviderExclusionCheckResult exclusionCheckResult = _providerExclusionCheck.ShouldBeExcluded(generatedProviderResult, flattenedTemplateFundingLines); if (exclusionCheckResult.ShouldBeExcluded) { if (newProviders.ContainsKey(publishedProvider.Key)) { newProviders.Remove(publishedProvider.Key); continue; } if (!_fundingLineValueOverride.TryOverridePreviousFundingLineValues(publishedProviderVersion, generatedProviderResult)) { //there are no none null payment funding line values and we didn't have to override any previous //version funding lines with a zero amount now they are all null so skip this published provider //the updates check continue; } } bool publishedProviderUpdated = _publishedProviderDataPopulator.UpdatePublishedProvider(publishedProviderVersion, generatedProviderResult, scopedProviders[providerId], specification.TemplateIds[fundingStream.Id], newProviders.ContainsKey(publishedProvider.Key)); _logger.Verbose($"Published provider '{publishedProvider.Key}' updated: '{publishedProviderUpdated}'"); //reapply any custom profiles this provider has and internally check for errors _reApplyCustomProfiles.ProcessPublishedProvider(publishedProviderVersion); // process published provider and detect errors await _detection.ProcessPublishedProvider(publishedProvider.Value, publishedProvidersContext); if (publishedProviderUpdated && existingPublishedProviders.AnyWithNullCheck()) { IDictionary <string, PublishedProvider> newPublishedProviders = await _variationService.PrepareVariedProviders(generatedProviderResult.TotalFunding, publishedProviders, publishedProvider.Value, scopedProviders[providerId], fundingConfiguration?.Variations, variationPointers, fundingStream.Id, specification.ProviderVersionId); if (!newPublishedProviders.IsNullOrEmpty()) { newProviders.AddRange(newPublishedProviders); } } if (!publishedProviderUpdated) { continue; } if (!newProviders.ContainsKey(publishedProvider.Key)) { existingPublishedProvidersToUpdate.Add(publishedProvider.Key, publishedProvider.Value); } publishedProvidersToUpdate.Add(publishedProvider.Key, publishedProvider.Value); } _logger.Information("Finished processing providers for variations and exclusions"); _logger.Information("Adding additional variation reasons"); AddInitialPublishVariationReasons(newProviders.Values); _logger.Information("Finished adding additional variation reasons"); _logger.Information("Starting to apply variations"); if (!(await _variationService.ApplyVariations(publishedProvidersToUpdate, newProviders, specification.Id, jobId))) { await _jobManagement.UpdateJobStatus(jobId, 0, 0, false, "Refresh job failed with variations errors."); throw new NonRetriableException($"Unable to refresh funding. Variations generated {_variationService.ErrorCount} errors. Check log for details"); } _logger.Information("Finished applying variations"); _logger.Information($"Updating a total of {publishedProvidersToUpdate.Count} published providers"); if (publishedProvidersToUpdate.Count > 0) { if (existingPublishedProvidersToUpdate.Count > 0 || newProviders.Count > 0) { using (Transaction transaction = _transactionFactory.NewTransaction <RefreshService>()) { try { // if any error occurs while updating or indexing then we need to re-index all published providers for consistency transaction.Enroll(async() => { await _publishedProviderVersionService.CreateReIndexJob(author, correlationId, specification.Id, jobId); }); // Save updated PublishedProviders to cosmos and increment version status if (existingPublishedProvidersToUpdate.Count > 0) { _logger.Information($"Saving updates to existing published providers. Total={existingPublishedProvidersToUpdate.Count}"); await _publishedProviderStatusUpdateService.UpdatePublishedProviderStatus(existingPublishedProvidersToUpdate.Values, author, PublishedProviderStatus.Updated, jobId, correlationId); _logger.Information("Indexing existing PublishedProviders"); await _publishedProviderIndexerService.IndexPublishedProviders(existingPublishedProvidersToUpdate.Values.Select(_ => _.Current)); } if (newProviders.Count > 0) { _logger.Information($"Saving new published providers. Total={newProviders.Count}"); await _publishedProviderStatusUpdateService.UpdatePublishedProviderStatus(newProviders.Values, author, PublishedProviderStatus.Draft, jobId, correlationId); _logger.Information("Indexing newly added PublishedProviders"); await _publishedProviderIndexerService.IndexPublishedProviders(newProviders.Values.Select(_ => _.Current)); } transaction.Complete(); } catch (Exception ex) { await transaction.Compensate(); throw; } } _logger.Information("Creating generate Csv jobs"); IGeneratePublishedFundingCsvJobsCreation generateCsvJobs = _generateCsvJobsLocator .GetService(GeneratePublishingCsvJobsCreationAction.Refresh); IEnumerable <string> fundingLineCodes = await _publishedFundingDataService.GetPublishedProviderFundingLines(specification.Id); IEnumerable <string> fundingStreamIds = Array.Empty <string>(); PublishedFundingCsvJobsRequest publishedFundingCsvJobsRequest = new PublishedFundingCsvJobsRequest { SpecificationId = specification.Id, CorrelationId = correlationId, User = author, FundingLineCodes = fundingLineCodes, FundingStreamIds = fundingStreamIds, FundingPeriodId = fundingPeriodId }; await generateCsvJobs.CreateJobs(publishedFundingCsvJobsRequest); } } }
public async Task <IActionResult> GetPublishedProviderProfileTotalsForSpecificationForProviderForFundingLine( string specificationId, string providerId, string fundingStreamId, string fundingLineCode) { Guard.IsNullOrWhiteSpace(specificationId, nameof(specificationId)); Guard.IsNullOrWhiteSpace(fundingStreamId, nameof(fundingStreamId)); Guard.IsNullOrWhiteSpace(fundingLineCode, nameof(fundingLineCode)); Guard.IsNullOrWhiteSpace(providerId, nameof(providerId)); PublishedProviderVersion latestPublishedProviderVersion = await _resilience.ExecuteAsync(() => _publishedFunding.GetLatestPublishedProviderVersionBySpecificationId( specificationId, fundingStreamId, providerId)); if (latestPublishedProviderVersion == null) { return(new NotFoundResult()); } IEnumerable <ProfileVariationPointer> profileVariationPointers = await _specificationService.GetProfileVariationPointers(specificationId); TemplateMetadataDistinctFundingLinesContents templateMetadataDistinctFundingLinesContents = await _policiesService.GetDistinctTemplateMetadataFundingLinesContents( fundingStreamId, latestPublishedProviderVersion.FundingPeriodId, latestPublishedProviderVersion.TemplateVersion); IEnumerable <FundingStreamPeriodProfilePattern> fundingStreamPeriodProfilePatterns = await _profilingService.GetProfilePatternsForFundingStreamAndFundingPeriod( latestPublishedProviderVersion.FundingStreamId, latestPublishedProviderVersion.FundingPeriodId); (string profilePatternKey, string profilePatternName, string profilePatternDescription) = GetProfilePatternDetails(fundingLineCode, latestPublishedProviderVersion, fundingStreamPeriodProfilePatterns); FundingLineProfile fundingLineProfile = new FundingLineProfile { FundingLineCode = fundingLineCode, FundingLineName = templateMetadataDistinctFundingLinesContents?.FundingLines? .FirstOrDefault(_ => _.FundingLineCode == fundingLineCode)?.Name, ProfilePatternKey = profilePatternKey, ProfilePatternName = profilePatternName, ProfilePatternDescription = profilePatternDescription, ProviderId = latestPublishedProviderVersion.ProviderId, UKPRN = latestPublishedProviderVersion.Provider.UKPRN, ProviderName = latestPublishedProviderVersion.Provider.Name, CarryOverAmount = latestPublishedProviderVersion.GetCarryOverTotalForFundingLine(fundingLineCode) ?? 0 }; ProfileVariationPointer currentProfileVariationPointer = profileVariationPointers?.SingleOrDefault(_ => _.FundingStreamId == fundingStreamId && _.FundingLineId == fundingLineCode); ProfileTotal[] profileTotals = new PaymentFundingLineProfileTotals(latestPublishedProviderVersion, fundingLineCode) .ToArray(); fundingLineProfile.TotalAllocation = latestPublishedProviderVersion .FundingLines .Where(_ => _.Type == FundingLineType.Payment) .SingleOrDefault(_ => _.FundingLineCode == fundingLineCode) ?.Value; fundingLineProfile.ProfileTotalAmount = profileTotals.Sum(_ => _.Value); FundingDate fundingDate = await _policiesService.GetFundingDate( fundingStreamId, latestPublishedProviderVersion.FundingPeriodId, fundingLineCode); for (int index = 0; index < profileTotals.Count(); index++) { ProfileTotal profileTotal = profileTotals[index]; profileTotal.InstallmentNumber = index + 1; profileTotal.IsPaid = IsProfileTotalPaid(currentProfileVariationPointer, profileTotal); profileTotal.ActualDate = fundingDate?.Patterns?.SingleOrDefault(_ => _.Occurrence == profileTotal.Occurrence && _.Period == profileTotal.TypeValue && _.PeriodYear == profileTotal.Year)?.PaymentDate; } fundingLineProfile.AmountAlreadyPaid = profileTotals.Where(_ => _.IsPaid).Sum(_ => _.Value); fundingLineProfile.RemainingAmount = fundingLineProfile.TotalAllocation - fundingLineProfile.AmountAlreadyPaid; foreach (ProfileTotal profileTotal in profileTotals.Where(_ => !_.IsPaid)) { profileTotal.ProfileRemainingPercentage = fundingLineProfile.TotalAllocation.HasValue && fundingLineProfile.TotalAllocation > 0 ? profileTotal.Value / (fundingLineProfile.TotalAllocation - fundingLineProfile.AmountAlreadyPaid) * 100 : 0; } fundingLineProfile.ProfileTotals = profileTotals; fundingLineProfile.LastUpdatedDate = latestPublishedProviderVersion.GetLatestFundingLineDate(fundingLineCode); fundingLineProfile.LastUpdatedUser = latestPublishedProviderVersion.GetLatestFundingLineUser(fundingLineCode); return(new OkObjectResult(fundingLineProfile)); }