protected void GivenThePublishedFundingVersionFeed(ICosmosDbFeedIterator <PublishedFundingVersion> feed) { Cosmos.Setup(_ => _.GetPublishedFundingVersions(TaskDetails.FundingStreamId, TaskDetails.FundingPeriodId, TaskDetails.TimeStamp)) .Returns(feed); }
private async Task <(bool isComplete, IEnumerable <PublishedProvider> items)> ProducePublishedProviders(CancellationToken cancellationToken, dynamic context) { try { ICosmosDbFeedIterator <PublishedProvider> feed = ((ISqlImportContext)context).Documents; if (!feed.HasMoreResults) { return(true, ArraySegment <PublishedProvider> .Empty); } IEnumerable <PublishedProvider> documents = await feed.ReadNext(cancellationToken); while (documents.IsNullOrEmpty() && feed.HasMoreResults) { documents = await feed.ReadNext(cancellationToken); } if (documents.IsNullOrEmpty() && !feed.HasMoreResults) { return(true, ArraySegment <PublishedProvider> .Empty); } return(false, documents.ToArray()); } catch { return(true, ArraySegment <PublishedProvider> .Empty); } }
public MergeContext(ICosmosDbFeedIterator <ProviderResult> feed, IEnumerable <FundingPeriod> fundingPeriods) { Feed = feed; FundingPeriods = new ConcurrentDictionary <string, FundingPeriod>( fundingPeriods.ToDictionary(_ => _.Id)); }
public void GetPublishedFundingVersionsFromVersion() { string fundingStreamId = NewRandomString(); string fundingPeriodId = NewRandomString(); decimal version = NewRandomInteger(); ICosmosDbFeedIterator <PublishedFundingVersion> expectedFeed = NewFeedIterator <PublishedFundingVersion>(); GivenTheFeedIterator(@"SELECT * FROM publishedFundingVersion p WHERE p.documentType = 'PublishedFundingVersion' AND StringToNumber(CONCAT(Tostring(p.content.majorVersion), '.', Tostring(p.content.minorVersion))) >= @version AND p.content.fundingStreamId = @fundingStreamId AND p.content.fundingPeriod.id = @fundingPeriodId AND p.deleted = false", expectedFeed, ("@fundingPeriodId", fundingPeriodId), ("@fundingStreamId", fundingStreamId), ("@version", version)); ICosmosDbFeedIterator <PublishedFundingVersion> actualFeedIterator = _repository.GetPublishedFundingVersionsFromVersion(fundingStreamId, fundingPeriodId, version); actualFeedIterator .Should() .BeSameAs(expectedFeed); }
private async Task <(bool Complete, IEnumerable <MergeSpecificationRequest> items)> ProduceSpecificationInformation(CancellationToken cancellationToken, dynamic context) { ICosmosDbFeedIterator <ProviderResult> feed = ((MergeContext)context).Feed; while (feed.HasMoreResults) { ProviderResult[] providerResults = (await feed.ReadNext(cancellationToken)).ToArray(); Console.WriteLine($"Processing next {providerResults.Length} provider results"); MergeSpecificationRequest[] requests = providerResults.Select(_ => { SpecificationSummary specificationSummary = GetSpecificationSummary(_.SpecificationId); Console.WriteLine($"Creating merge specification request for provider {_.Provider.Id} and specification {_.SpecificationId}"); return(new MergeSpecificationRequest(new SpecificationInformation { Id = specificationSummary.Id, Name = specificationSummary.Name, FundingPeriodId = specificationSummary.FundingPeriod.Id, LastEditDate = specificationSummary.LastEditedDate }, _.Provider.Id)); }).ToArray(); return(false, requests); } return(true, ArraySegment <MergeSpecificationRequest> .Empty); }
public void GetPublishedFundingVersions() { string fundingStreamId = NewRandomString(); string fundingPeriodId = NewRandomString(); long timeStamp = NewRandomTimeStamp(); ICosmosDbFeedIterator <PublishedFundingVersion> expectedFeed = NewFeedIterator <PublishedFundingVersion>(); GivenTheFeedIterator(@"SELECT * FROM publishedFundingVersion p WHERE p.documentType = 'PublishedFundingVersion' AND p._ts >= @sinceTimeStamp AND p.content.fundingStreamId = @fundingStreamId AND p.content.fundingPeriod.id = @fundingPeriodId AND p.deleted = false", expectedFeed, ("@fundingPeriodId", fundingPeriodId), ("@fundingStreamId", fundingStreamId), ("@sinceTimeStamp", timeStamp)); ICosmosDbFeedIterator <PublishedFundingVersion> actualFeedIterator = _repository.GetPublishedFundingVersions(fundingStreamId, fundingPeriodId, timeStamp); actualFeedIterator .Should() .BeSameAs(expectedFeed); }
public async Task <ISqlImportContext> CreateImportContext(string specificationId, string fundingStreamId, SchemaContext schemaContext) { ICosmosDbFeedIterator <PublishedProvider> publishedProviderFeed = GetPublishedProviderFeed(specificationId, fundingStreamId); TemplateMetadataContents template = await GetTemplateMetadataContents(specificationId, fundingStreamId); IEnumerable <FundingLine> allFundingLines = template.RootFundingLines.Flatten(_ => _.FundingLines); IEnumerable <Calculation> allCalculations = allFundingLines.SelectMany(_ => _.Calculations.Flatten(cal => cal.Calculations)); IEnumerable <Calculation> uniqueCalculations = allCalculations.DistinctBy(_ => _.TemplateCalculationId); IDictionary <uint, string> calculationNames = GetCalculationNames(uniqueCalculations); return(new SqlImportContext { Documents = publishedProviderFeed, CalculationNames = calculationNames, Calculations = new CalculationDataTableBuilder(uniqueCalculations), Providers = new ProviderDataTableBuilder(), Funding = new PublishedProviderVersionDataTableBuilder(), InformationFundingLines = new InformationFundingLineDataTableBuilder(), PaymentFundingLines = new PaymentFundingLineDataTableBuilder(), SchemaContext = schemaContext }); }
private async Task <(bool isComplete, IEnumerable <ProviderWithResultsForSpecifications> items)> ProduceProviderWithResultsForSpecifications(CancellationToken token, dynamic context) { ICosmosDbFeedIterator <ProviderWithResultsForSpecifications> feedIterator = ((MergeSpecificationInformationContext)context).FeedIterator; while (feedIterator.HasMoreResults) { ProviderWithResultsForSpecifications[] page = (await feedIterator.ReadNext(token)).ToArray(); LogInformation($"Producing next page of ProviderWithResultsForSpecifications with {page.Length} items"); return(false, page); } return(true, ArraySegment <ProviderWithResultsForSpecifications> .Empty); }
public async Task GetPublishedFundingVersions() { ICosmosDbFeedIterator <PublishedFundingVersion> feed = _repository.GetPublishedFundingVersions("DSG", "FY-2021", 1588684299); feed.HasMoreResults .Should() .BeTrue(); IEnumerable <PublishedFundingVersion> documents = await feed.ReadNext(); documents .Should() .NotBeEmpty(); }
public async Task GetPublishedProvidersFromVersion() { ICosmosDbFeedIterator <PublishedProvider> feed = _repository.GetPublishedProvidersFromVersion("DSG", "FY-2021", 2M); feed.HasMoreResults .Should() .BeTrue(); IEnumerable <PublishedProvider> documents = await feed.ReadNext(); documents .Should() .NotBeEmpty(); }
public async Task GetPublishedProviders() { ICosmosDbFeedIterator <PublishedProvider> feed = _repository.GetPublishedProviders("DSG", "FY-2021-7db621f6-ff28-4910-a3b2-5440c2cd80b0", 1588682808); feed.HasMoreResults .Should() .BeTrue(); IEnumerable <PublishedProvider> documents = await feed.ReadNext(); documents .Should() .NotBeEmpty(); }
public async Task Run() { try { await _cosmosRepository.SetThroughput(100000); Console.WriteLine("Set calcresults RU at 100000"); CosmosDbQuery query = new CosmosDbQuery { QueryText = @" SELECT * FROM c WHERE c.documentType = 'ProviderResult' AND c.deleted = false" }; ICosmosDbFeedIterator <ProviderResult> feed = _cosmosRepository.GetFeedIterator <ProviderResult>(query); ApiResponse <IEnumerable <FundingPeriod> > fundingPeriods = await _policiesPolicy.ExecuteAsync(() => _policies.GetFundingPeriods()); MergeContext context = new MergeContext(feed, fundingPeriods.Content); IProducerConsumer producerConsumer = _producerConsumerFactory.CreateProducerConsumer(ProduceSpecificationInformation, ConsumeSpecificationInformation, 8, 4, _logger); await producerConsumer.Run(context); Console.WriteLine($"Starting bulk upsert of {_cachedProviderWithResultsForSpecifications.Count} providers with results for specifications summaries"); await _cosmosRepository.BulkUpsertAsync(_cachedProviderWithResultsForSpecifications); } catch (Exception e) { _logger.Error(e, "Unable to complete provider version migration"); throw; } finally { await _cosmosRepository.SetThroughput(10000); Console.WriteLine("Set calcresults RU at 10000"); } }
public async Task GetPublishedFunding() { ICosmosDbFeedIterator <PublishedFunding> feed = _repository.GetPublishedFunding("DSG", "FY-2021", 1588685609); while (feed.HasMoreResults) { IEnumerable <PublishedFunding> documents = await feed.ReadNext(); documents .Should() .NotBeNull() .And .NotBeEmpty(); } }
private async Task MergeSpecificationInformationForAllProviders(SpecificationInformation specificationInformation) { string specificationId = specificationInformation.Id; LogInformation($"Merging specification information for specification {specificationId} into summary for all providers with results currently tracking it"); ICosmosDbFeedIterator <ProviderWithResultsForSpecifications> providersWithResultsForSpecifications = GetProviderWithResultsBySpecificationId(specificationId); await EnsureFundingPeriodEndDateQueried(specificationInformation); MergeSpecificationInformationContext context = new MergeSpecificationInformationContext(providersWithResultsForSpecifications, specificationInformation); IProducerConsumer producerConsumer = _producerConsumerFactory.CreateProducerConsumer(ProduceProviderWithResultsForSpecifications, MergeSpecificationInformation, 200, 2, _logger); await producerConsumer.Run(context); }
public async Task Run(PublishedFundingUndoTaskContext taskContext) { LogStartingTask(); Guard.ArgumentNotNull(taskContext?.PublishedFundingVersionDetails, nameof(taskContext.PublishedFundingVersionDetails)); UndoTaskDetails details = taskContext.PublishedFundingVersionDetails; ICosmosDbFeedIterator <PublishedFundingVersion> feed = GetPublishedFundingVersionsFeed(details); FeedContext <PublishedFundingVersion> feedContext = new FeedContext <PublishedFundingVersion>(taskContext, feed); IProducerConsumer producerConsumer = ProducerConsumerFactory.CreateProducerConsumer(ProducePublishedFundingVersions, UndoPublishedFundingVersions, 200, 4, Logger); await producerConsumer.Run(feedContext); await NotifyJobProgress(taskContext); LogCompletedTask(); }
public MergeSpecificationInformationContext(ICosmosDbFeedIterator <ProviderWithResultsForSpecifications> feedIterator, SpecificationInformation specificationInformation) { FeedIterator = feedIterator; SpecificationInformation = specificationInformation; }
private void AndTheProviderWithResultsForSpecifications(string specificationId, ICosmosDbFeedIterator <ProviderWithResultsForSpecifications> feed) { _calculationResults.Setup(_ => _.GetProvidersWithResultsForSpecificationBySpecificationId(specificationId)) .Returns(feed); }
public FeedContext(PublishedFundingUndoTaskContext taskContext, ICosmosDbFeedIterator <TDocument> feed) { TaskContext = taskContext; Feed = feed; }