public async Task WhenPublishedProvidersRequestedAndPublishedProvidersExistForSpecificationPublishedProvidersReturned()
{
Provider successor = NewProvider();
IEnumerable <PublishedProvider> publishedProviders = new[] {
NewPublishedProvider(_ => _.WithCurrent(NewPublishedProviderVersion(ppv => ppv.WithProvider(NewProvider())))),
NewPublishedProvider(_ => _.WithCurrent(NewPublishedProviderVersion(ppv => ppv.WithProvider(NewProvider())))),
NewPublishedProvider(_ => _.WithCurrent(NewPublishedProviderVersion(ppv => ppv.WithProvider(NewProvider(p => p.WithSuccessor(successor.ProviderId))))))
};
IEnumerable <ApiProvider> apiproviders = _mapper.Map <IEnumerable <ApiProvider> >(publishedProviders.Select(_ => _.Current.Provider));
ApiProvider excludedProvider = _mapper.Map <ApiProvider>(NewProvider());
apiproviders = apiproviders.Concat(new[] { excludedProvider });
publishedProviders = publishedProviders.Concat(new[] { NewPublishedProvider(_ => _.WithCurrent(NewPublishedProviderVersion(ppv => ppv.WithProvider(successor)))) });
string providerVersionId = NewRandomString();
string specificationId = NewRandomString();
SpecificationSummary specification = new SpecificationSummary {
Id = specificationId,
FundingPeriod = new Reference {
Id = FundingPeriodId
},
ProviderVersionId = providerVersionId
};
GivenTheApiResponseProviderVersionContainsTheProviders(providerVersionId, apiproviders.ToArray());
AndPublishedProvidersForFundingStreamAndFundingPeriod(publishedProviders);
AndTheSecondApiResponseScopedProviderIdsContainsProviderIds(specificationId, apiproviders.Select(_ => _.ProviderId));
_jobManagement
.QueueJobAndWait(Arg.Any <Func <Task <bool> > >(), JobConstants.DefinitionNames.PopulateScopedProvidersJob, specificationId, Arg.Any <string>(), ServiceBusConstants.TopicNames.JobNotifications)
.Returns(true);
(IDictionary <string, PublishedProvider> PublishedProvidersForFundingStream,
IDictionary <string, PublishedProvider> ScopedPublishedProviders) = await WhenPublishedProvidersAreReturned(specification);
PublishedProvidersForFundingStream.Count()
.Should()
.Be(4);
PublishedProvidersForFundingStream.Keys
.Should()
.BeEquivalentTo(publishedProviders.Select(_ => _.Current.ProviderId));
ScopedPublishedProviders.Count()
.Should()
.Be(4);
ScopedPublishedProviders.Keys
.Should()
.BeEquivalentTo(publishedProviders.Select(_ => _.Current.ProviderId));
}
public async Task <IDictionary <string, Provider> > GetScopedProvidersForSpecification(string specificationId, string providerVersionId)
{
Guard.IsNullOrWhiteSpace(specificationId, nameof(specificationId));
Guard.IsNullOrWhiteSpace(providerVersionId, nameof(providerVersionId));
ApiResponse <ApiProviderVersion> providerVersionsResponse =
await _providersApiClientPolicy.ExecuteAsync(() => _providersApiClient.GetProvidersByVersion(providerVersionId));
Guard.ArgumentNotNull(providerVersionsResponse?.Content, nameof(providerVersionsResponse));
ApiResponse <IEnumerable <string> > scopedProviderIdResponse =
await _providersApiClientPolicy.ExecuteAsync(() => _providersApiClient.GetScopedProviderIds(specificationId));
// fallback if redis cache has been invalidated queue a new scoped provider job
if (scopedProviderIdResponse?.Content == null || scopedProviderIdResponse.Content.IsNullOrEmpty())
{
string correlationId = Guid.NewGuid().ToString();
bool jobCompletedSuccessfully = await _jobManagement.QueueJobAndWait(async() =>
{
ApiResponse <bool> refreshCacheFromApi = await _providersApiClientPolicy.ExecuteAsync(() =>
_providersApiClient.RegenerateProviderSummariesForSpecification(specificationId, true));
if (!refreshCacheFromApi.StatusCode.IsSuccess())
{
string errorMessage = $"Unable to re-generate scoped providers doing refresh '{specificationId}' with status code: {refreshCacheFromApi.StatusCode}";
}
// returns true if job queued
return(refreshCacheFromApi.Content);
},
JobConstants.DefinitionNames.PopulateScopedProvidersJob,
specificationId,
correlationId,
ServiceBusConstants.TopicNames.JobNotifications);
if (jobCompletedSuccessfully)
{
scopedProviderIdResponse = await _providersApiClientPolicy.ExecuteAsync(() => _providersApiClient.GetScopedProviderIds(specificationId));
}
;
if (scopedProviderIdResponse?.Content == null)
{
return(null);
}
}
HashSet <string> scopedProviders = scopedProviderIdResponse?.Content.ToHashSet();
IDictionary <string, Provider> scopedProvidersInVersion = (_mapper.Map <IEnumerable <Provider> >(providerVersionsResponse.Content.Providers.Where(p => scopedProviders.Contains(p.ProviderId)))).ToDictionary(_ => _.ProviderId);
return(scopedProvidersInVersion);
}
public override async Task Process(Message message)
{
Guard.ArgumentNotNull(message, nameof(message));
// don't auto complete the job as this will be done through child notifications
AutoComplete = false;
IDictionary <string, string> properties = message.BuildMessageProperties();
string specificationId = message.UserProperties["specification-id"].ToString();
IEnumerable <CalculationEntity> circularDependencies = await _graphRepository.GetCircularDependencies(specificationId);
if (!circularDependencies.IsNullOrEmpty())
{
string errorMessage = $"circular dependencies exist for specification: '{specificationId}'";
foreach (CalculationEntity calculationEntity in circularDependencies)
{
int i = 0;
_logger.Information(new[] { calculationEntity.Node.CalculationName }.Concat(calculationEntity.Relationships.Reverse().Select(rel =>
{
try
{
return($"|--->{((object)rel.Two).AsJson().AsPoco<GraphCalculation>().CalculationName}".AddLeading(i * 3));
}
finally
{
i++;
}
})).Aggregate((partialLog, log) => $"{partialLog}\r\n{log}"));
}
_logger.Error(errorMessage);
throw new NonRetriableException(errorMessage);
}
if (message.UserProperties.ContainsKey("ignore-save-provider-results"))
{
properties.Add("ignore-save-provider-results", "true");
}
string specificationSummaryCachekey = message.UserProperties.ContainsKey("specification-summary-cache-key") ?
message.UserProperties["specification-summary-cache-key"].ToString() :
$"{CacheKeys.SpecificationSummaryById}{specificationId}";
bool specificationSummaryExists = await _cacheProvider.KeyExists <Models.Specs.SpecificationSummary>(specificationSummaryCachekey);
if (!specificationSummaryExists)
{
ApiResponse <SpecificationSummary> specificationSummaryResponse = await _specificationsApiClientPolicy.ExecuteAsync(() => _specificationsApiClient.GetSpecificationSummaryById(specificationId));
if (!specificationSummaryResponse.StatusCode.IsSuccess())
{
string errorMessage = $"Unable to get specification summary by id: '{specificationId}' with status code: {specificationSummaryResponse.StatusCode}";
_logger.Error(errorMessage);
throw new NonRetriableException(errorMessage);
}
}
string providerCacheKey = message.UserProperties.ContainsKey("provider-cache-key") ?
message.UserProperties["provider-cache-key"].ToString() :
$"{CacheKeys.ScopedProviderSummariesPrefix}{specificationId}";
bool summariesExist = await _cacheProvider.KeyExists <ProviderSummary>(providerCacheKey);
long?totalCount = await _cacheProvider.ListLengthAsync <ProviderSummary>(providerCacheKey);
bool refreshCachedScopedProviders = false;
if (summariesExist)
{
// if there are no provider results for specification then the call returns no content
ApiResponse <IEnumerable <string> > scopedProviderIds = await _providersApiClient.GetScopedProviderIds(specificationId);
if (scopedProviderIds?.Content != null)
{
if (scopedProviderIds.Content.Count() != totalCount)
{
refreshCachedScopedProviders = true;
}
else
{
IEnumerable <ProviderSummary> cachedScopedSummaries = await _cacheProvider.ListRangeAsync <ProviderSummary>(providerCacheKey, 0, (int)totalCount);
IEnumerable <string> differences = scopedProviderIds.Content.Except(cachedScopedSummaries.Select(m => m.Id));
refreshCachedScopedProviders = differences.AnyWithNullCheck();
}
}
else
{
// if there are no provider results then always refresh scoped providers
refreshCachedScopedProviders = true;
}
}
if (!summariesExist || refreshCachedScopedProviders)
{
string correlationId = Guid.NewGuid().ToString();
bool jobCompletedSuccessfully = await _jobManagement.QueueJobAndWait(async() =>
{
ApiResponse <bool> refreshCacheFromApi = await _providersApiClientPolicy.ExecuteAsync(() =>
_providersApiClient.RegenerateProviderSummariesForSpecification(specificationId, !summariesExist));
if (!refreshCacheFromApi.StatusCode.IsSuccess())
{
string errorMessage = $"Unable to re-generate scoped providers while building projects '{specificationId}' with status code: {refreshCacheFromApi.StatusCode}";
_logger.Error(errorMessage);
throw new NonRetriableException(errorMessage);
}
// returns true if job queued
return(refreshCacheFromApi.Content);
},
DefinitionNames.PopulateScopedProvidersJob,
specificationId,
correlationId,
ServiceBusConstants.TopicNames.JobNotifications);
// if scoped provider job not completed successfully
if (!jobCompletedSuccessfully)
{
string errorMessage = $"Unable to re-generate scoped providers while building projects '{specificationId}' job didn't complete successfully in time";
_logger.Error(errorMessage);
throw new NonRetriableException(errorMessage);
}
totalCount = await _cacheProvider.ListLengthAsync <ProviderSummary>(providerCacheKey);
}
const string providerSummariesPartitionIndex = "provider-summaries-partition-index";
const string providerSummariesPartitionSize = "provider-summaries-partition-size";
properties.Add(providerSummariesPartitionSize, _engineSettings.MaxPartitionSize.ToString());
properties.Add("provider-cache-key", providerCacheKey);
properties.Add("specification-id", specificationId);
properties.Add("specification-summary-cache-key", specificationSummaryCachekey);
string assemblyETag = await _sourceFileRepository.GetAssemblyETag(specificationId);
if (assemblyETag.IsNotNullOrWhitespace())
{
properties.Add("assembly-etag", assemblyETag);
}
IList <IDictionary <string, string> > allJobProperties = new List <IDictionary <string, string> >();
for (int partitionIndex = 0; partitionIndex < totalCount; partitionIndex += _engineSettings.MaxPartitionSize)
{
if (properties.ContainsKey(providerSummariesPartitionIndex))
{
properties[providerSummariesPartitionIndex] = partitionIndex.ToString();
}
else
{
properties.Add(providerSummariesPartitionIndex, partitionIndex.ToString());
}
IDictionary <string, string> jobProperties = new Dictionary <string, string>();
foreach (KeyValuePair <string, string> item in properties)
{
jobProperties.Add(item.Key, item.Value);
}
allJobProperties.Add(jobProperties);
}
try
{
if (!allJobProperties.Any())
{
_logger.Information($"No scoped providers set for specification '{specificationId}'");
AutoComplete = true;
Outcome = "Calculations not run as no scoped providers set for specification";
return;
}
IEnumerable <Job> newJobs = await CreateGenerateAllocationJobs(allJobProperties, specificationId);
int newJobsCount = newJobs.Count();
int batchCount = allJobProperties.Count;
if (newJobsCount != batchCount)
{
string errorMessage = $"Only {newJobsCount} child jobs from {batchCount} were created with parent id: '{Job.Id}'";
_logger.Error(errorMessage);
throw new Exception(errorMessage);
}
else
{
_logger.Information($"{newJobsCount} child jobs were created for parent id: '{Job.Id}'");
}
}
catch (RefreshJobRunningException ex)
{
throw new NonRetriableException(ex.Message, ex);
}
catch (Exception ex)
{
_logger.Error(ex, $"Failed to create child jobs for parent job: '{Job.Id}'");
throw;
}
}