protected void GivenThePublishedFundingVersionFeed(ICosmosDbFeedIterator <PublishedFundingVersion> feed)
{
Cosmos.Setup(_ => _.GetPublishedFundingVersions(TaskDetails.FundingStreamId,
TaskDetails.FundingPeriodId,
TaskDetails.TimeStamp))
.Returns(feed);
}
private async Task <(bool isComplete, IEnumerable <PublishedProvider> items)> ProducePublishedProviders(CancellationToken cancellationToken,
dynamic context)
{
try
{
ICosmosDbFeedIterator <PublishedProvider> feed = ((ISqlImportContext)context).Documents;
if (!feed.HasMoreResults)
{
return(true, ArraySegment <PublishedProvider> .Empty);
}
IEnumerable <PublishedProvider> documents = await feed.ReadNext(cancellationToken);
while (documents.IsNullOrEmpty() && feed.HasMoreResults)
{
documents = await feed.ReadNext(cancellationToken);
}
if (documents.IsNullOrEmpty() && !feed.HasMoreResults)
{
return(true, ArraySegment <PublishedProvider> .Empty);
}
return(false, documents.ToArray());
}
catch
{
return(true, ArraySegment <PublishedProvider> .Empty);
}
}
public MergeContext(ICosmosDbFeedIterator <ProviderResult> feed,
IEnumerable <FundingPeriod> fundingPeriods)
{
Feed = feed;
FundingPeriods = new ConcurrentDictionary <string, FundingPeriod>(
fundingPeriods.ToDictionary(_ => _.Id));
}
public void GetPublishedFundingVersionsFromVersion()
{
string fundingStreamId = NewRandomString();
string fundingPeriodId = NewRandomString();
decimal version = NewRandomInteger();
ICosmosDbFeedIterator <PublishedFundingVersion> expectedFeed = NewFeedIterator <PublishedFundingVersion>();
GivenTheFeedIterator(@"SELECT
*
FROM publishedFundingVersion p
WHERE p.documentType = 'PublishedFundingVersion'
AND StringToNumber(CONCAT(Tostring(p.content.majorVersion), '.', Tostring(p.content.minorVersion))) >= @version
AND p.content.fundingStreamId = @fundingStreamId
AND p.content.fundingPeriod.id = @fundingPeriodId
AND p.deleted = false",
expectedFeed,
("@fundingPeriodId", fundingPeriodId),
("@fundingStreamId", fundingStreamId),
("@version", version));
ICosmosDbFeedIterator <PublishedFundingVersion> actualFeedIterator = _repository.GetPublishedFundingVersionsFromVersion(fundingStreamId,
fundingPeriodId,
version);
actualFeedIterator
.Should()
.BeSameAs(expectedFeed);
}
private async Task <(bool Complete, IEnumerable <MergeSpecificationRequest> items)> ProduceSpecificationInformation(CancellationToken cancellationToken,
dynamic context)
{
ICosmosDbFeedIterator <ProviderResult> feed = ((MergeContext)context).Feed;
while (feed.HasMoreResults)
{
ProviderResult[] providerResults = (await feed.ReadNext(cancellationToken)).ToArray();
Console.WriteLine($"Processing next {providerResults.Length} provider results");
MergeSpecificationRequest[] requests = providerResults.Select(_ =>
{
SpecificationSummary specificationSummary = GetSpecificationSummary(_.SpecificationId);
Console.WriteLine($"Creating merge specification request for provider {_.Provider.Id} and specification {_.SpecificationId}");
return(new MergeSpecificationRequest(new SpecificationInformation
{
Id = specificationSummary.Id,
Name = specificationSummary.Name,
FundingPeriodId = specificationSummary.FundingPeriod.Id,
LastEditDate = specificationSummary.LastEditedDate
},
_.Provider.Id));
}).ToArray();
return(false, requests);
}
return(true, ArraySegment <MergeSpecificationRequest> .Empty);
}
public void GetPublishedFundingVersions()
{
string fundingStreamId = NewRandomString();
string fundingPeriodId = NewRandomString();
long timeStamp = NewRandomTimeStamp();
ICosmosDbFeedIterator <PublishedFundingVersion> expectedFeed = NewFeedIterator <PublishedFundingVersion>();
GivenTheFeedIterator(@"SELECT
*
FROM publishedFundingVersion p
WHERE p.documentType = 'PublishedFundingVersion'
AND p._ts >= @sinceTimeStamp
AND p.content.fundingStreamId = @fundingStreamId
AND p.content.fundingPeriod.id = @fundingPeriodId
AND p.deleted = false",
expectedFeed,
("@fundingPeriodId", fundingPeriodId),
("@fundingStreamId", fundingStreamId),
("@sinceTimeStamp", timeStamp));
ICosmosDbFeedIterator <PublishedFundingVersion> actualFeedIterator = _repository.GetPublishedFundingVersions(fundingStreamId,
fundingPeriodId,
timeStamp);
actualFeedIterator
.Should()
.BeSameAs(expectedFeed);
}
public async Task <ISqlImportContext> CreateImportContext(string specificationId,
string fundingStreamId,
SchemaContext schemaContext)
{
ICosmosDbFeedIterator <PublishedProvider> publishedProviderFeed = GetPublishedProviderFeed(specificationId, fundingStreamId);
TemplateMetadataContents template = await GetTemplateMetadataContents(specificationId, fundingStreamId);
IEnumerable <FundingLine> allFundingLines = template.RootFundingLines.Flatten(_ => _.FundingLines);
IEnumerable <Calculation> allCalculations = allFundingLines.SelectMany(_ => _.Calculations.Flatten(cal => cal.Calculations));
IEnumerable <Calculation> uniqueCalculations = allCalculations.DistinctBy(_ => _.TemplateCalculationId);
IDictionary <uint, string> calculationNames = GetCalculationNames(uniqueCalculations);
return(new SqlImportContext
{
Documents = publishedProviderFeed,
CalculationNames = calculationNames,
Calculations = new CalculationDataTableBuilder(uniqueCalculations),
Providers = new ProviderDataTableBuilder(),
Funding = new PublishedProviderVersionDataTableBuilder(),
InformationFundingLines = new InformationFundingLineDataTableBuilder(),
PaymentFundingLines = new PaymentFundingLineDataTableBuilder(),
SchemaContext = schemaContext
});
}
private async Task <(bool isComplete, IEnumerable <ProviderWithResultsForSpecifications> items)> ProduceProviderWithResultsForSpecifications(CancellationToken token,
dynamic context)
{
ICosmosDbFeedIterator <ProviderWithResultsForSpecifications> feedIterator = ((MergeSpecificationInformationContext)context).FeedIterator;
while (feedIterator.HasMoreResults)
{
ProviderWithResultsForSpecifications[] page = (await feedIterator.ReadNext(token)).ToArray();
LogInformation($"Producing next page of ProviderWithResultsForSpecifications with {page.Length} items");
return(false, page);
}
return(true, ArraySegment <ProviderWithResultsForSpecifications> .Empty);
}
public async Task GetPublishedFundingVersions()
{
ICosmosDbFeedIterator <PublishedFundingVersion> feed = _repository.GetPublishedFundingVersions("DSG",
"FY-2021",
1588684299);
feed.HasMoreResults
.Should()
.BeTrue();
IEnumerable <PublishedFundingVersion> documents = await feed.ReadNext();
documents
.Should()
.NotBeEmpty();
}
public async Task GetPublishedProvidersFromVersion()
{
ICosmosDbFeedIterator <PublishedProvider> feed = _repository.GetPublishedProvidersFromVersion("DSG",
"FY-2021",
2M);
feed.HasMoreResults
.Should()
.BeTrue();
IEnumerable <PublishedProvider> documents = await feed.ReadNext();
documents
.Should()
.NotBeEmpty();
}
public async Task GetPublishedProviders()
{
ICosmosDbFeedIterator <PublishedProvider> feed = _repository.GetPublishedProviders("DSG",
"FY-2021-7db621f6-ff28-4910-a3b2-5440c2cd80b0",
1588682808);
feed.HasMoreResults
.Should()
.BeTrue();
IEnumerable <PublishedProvider> documents = await feed.ReadNext();
documents
.Should()
.NotBeEmpty();
}
public async Task Run()
{
try
{
await _cosmosRepository.SetThroughput(100000);
Console.WriteLine("Set calcresults RU at 100000");
CosmosDbQuery query = new CosmosDbQuery
{
QueryText = @" SELECT *
FROM c
WHERE c.documentType = 'ProviderResult'
AND c.deleted = false"
};
ICosmosDbFeedIterator <ProviderResult> feed = _cosmosRepository.GetFeedIterator <ProviderResult>(query);
ApiResponse <IEnumerable <FundingPeriod> > fundingPeriods = await _policiesPolicy.ExecuteAsync(() => _policies.GetFundingPeriods());
MergeContext context = new MergeContext(feed, fundingPeriods.Content);
IProducerConsumer producerConsumer = _producerConsumerFactory.CreateProducerConsumer(ProduceSpecificationInformation,
ConsumeSpecificationInformation,
8,
4,
_logger);
await producerConsumer.Run(context);
Console.WriteLine($"Starting bulk upsert of {_cachedProviderWithResultsForSpecifications.Count} providers with results for specifications summaries");
await _cosmosRepository.BulkUpsertAsync(_cachedProviderWithResultsForSpecifications);
}
catch (Exception e)
{
_logger.Error(e, "Unable to complete provider version migration");
throw;
}
finally
{
await _cosmosRepository.SetThroughput(10000);
Console.WriteLine("Set calcresults RU at 10000");
}
}
public async Task GetPublishedFunding()
{
ICosmosDbFeedIterator <PublishedFunding> feed = _repository.GetPublishedFunding("DSG",
"FY-2021",
1588685609);
while (feed.HasMoreResults)
{
IEnumerable <PublishedFunding> documents = await feed.ReadNext();
documents
.Should()
.NotBeNull()
.And
.NotBeEmpty();
}
}
private async Task MergeSpecificationInformationForAllProviders(SpecificationInformation specificationInformation)
{
string specificationId = specificationInformation.Id;
LogInformation($"Merging specification information for specification {specificationId} into summary for all providers with results currently tracking it");
ICosmosDbFeedIterator <ProviderWithResultsForSpecifications> providersWithResultsForSpecifications = GetProviderWithResultsBySpecificationId(specificationId);
await EnsureFundingPeriodEndDateQueried(specificationInformation);
MergeSpecificationInformationContext context = new MergeSpecificationInformationContext(providersWithResultsForSpecifications, specificationInformation);
IProducerConsumer producerConsumer = _producerConsumerFactory.CreateProducerConsumer(ProduceProviderWithResultsForSpecifications,
MergeSpecificationInformation,
200,
2,
_logger);
await producerConsumer.Run(context);
}
public async Task Run(PublishedFundingUndoTaskContext taskContext)
{
LogStartingTask();
Guard.ArgumentNotNull(taskContext?.PublishedFundingVersionDetails, nameof(taskContext.PublishedFundingVersionDetails));
UndoTaskDetails details = taskContext.PublishedFundingVersionDetails;
ICosmosDbFeedIterator <PublishedFundingVersion> feed = GetPublishedFundingVersionsFeed(details);
FeedContext <PublishedFundingVersion> feedContext = new FeedContext <PublishedFundingVersion>(taskContext, feed);
IProducerConsumer producerConsumer = ProducerConsumerFactory.CreateProducerConsumer(ProducePublishedFundingVersions,
UndoPublishedFundingVersions,
200,
4,
Logger);
await producerConsumer.Run(feedContext);
await NotifyJobProgress(taskContext);
LogCompletedTask();
}
public MergeSpecificationInformationContext(ICosmosDbFeedIterator <ProviderWithResultsForSpecifications> feedIterator,
SpecificationInformation specificationInformation)
{
FeedIterator = feedIterator;
SpecificationInformation = specificationInformation;
}
private void AndTheProviderWithResultsForSpecifications(string specificationId,
ICosmosDbFeedIterator <ProviderWithResultsForSpecifications> feed)
{
_calculationResults.Setup(_ => _.GetProvidersWithResultsForSpecificationBySpecificationId(specificationId))
.Returns(feed);
}
public FeedContext(PublishedFundingUndoTaskContext taskContext,
ICosmosDbFeedIterator <TDocument> feed)
{
TaskContext = taskContext;
Feed = feed;
}
