private async Task <Dictionary <string, Provider> > GetAllProviders(string providerVersionId)
{
if (!_providers.ContainsKey(providerVersionId))
{
Common.ApiClient.Models.ApiResponse <ProviderVersion> providerResult = await _providersApiClientPolicy.ExecuteAsync(() => _providersApiClient.GetProvidersByVersion(providerVersionId));
if (providerResult == null)
{
throw new InvalidOperationException($"Provider lookup returned null for provider version id '{providerVersionId}'");
}
if (providerResult.Content == null)
{
throw new InvalidOperationException($"Provider lookup content returned null for provider version id '{providerVersionId}'");
}
if (providerResult.Content.Providers == null)
{
throw new InvalidOperationException($"Provider lookup provider content returned null for provider version id '{providerVersionId}'");
}
_providers.Add(providerVersionId, providerResult.Content.Providers.ToDictionary(c => c.ProviderId, p => p));
}
return(_providers[providerVersionId]);
}
public async Task <bool> SpecificationHasResults(string specificationId)
{
Common.ApiClient.Models.ApiResponse <bool> providerResults =
await _apiClientProxy.GetProviderHasResultsBySpecificationId(specificationId);
if (providerResults == null)
{
throw new InvalidOperationException("Provider results should not return null");
}
return(providerResults.Content);
}
public async Task <IActionResult> QueueCsvGeneration(string specificationId)
{
Common.ApiClient.Models.ApiResponse <SpecModel.SpecificationSummary> specificationApiResponse =
await _specificationsApiClientPolicy.ExecuteAsync(() => _specificationsApiClient.GetSpecificationSummaryById(specificationId));
if (!specificationApiResponse.StatusCode.IsSuccess() || specificationApiResponse.Content == null)
{
string errorMessage = "No specification summaries found to generate calculation results csv.";
_logger.Error(errorMessage);
throw new RetriableException(errorMessage);
}
Job job = await QueueCsvGenerationMessage(specificationId, specificationApiResponse.Content.Name);
return(new OkObjectResult(job));
}
public async Task <IActionResult> SaveVersion(CreateNewTestScenarioVersion scenarioVersion, Reference user, string correlationId)
{
if (scenarioVersion == null)
{
_logger.Error("A null scenario version was provided");
return(new BadRequestObjectResult("Null or empty calculation Id provided"));
}
BadRequestObjectResult validationResult = (await _createNewTestScenarioVersionValidator.ValidateAsync(scenarioVersion)).PopulateModelState();
if (validationResult != null)
{
return(validationResult);
}
TestScenario testScenario = null;
if (!string.IsNullOrEmpty(scenarioVersion.Id))
{
testScenario = await _scenariosRepository.GetTestScenarioById(scenarioVersion.Id);
}
bool saveAsVersion = true;
Common.ApiClient.Models.ApiResponse <SpecModel.SpecificationSummary> specificationApiResponse =
await _specificationsApiClientPolicy.ExecuteAsync(() => _specificationsApiClient.GetSpecificationSummaryById(scenarioVersion.SpecificationId));
if (!specificationApiResponse.StatusCode.IsSuccess() || specificationApiResponse.Content == null)
{
_logger.Error($"Unable to find a specification for specification id : {scenarioVersion.SpecificationId}");
return(new StatusCodeResult(412));
}
SpecModel.SpecificationSummary specification = specificationApiResponse.Content;
if (testScenario == null)
{
string Id = Guid.NewGuid().ToString();
testScenario = new TestScenario
{
Id = Id,
SpecificationId = specification.Id,
Name = scenarioVersion.Name,
Current = new TestScenarioVersion
{
Date = DateTimeOffset.Now.ToLocalTime(),
TestScenarioId = Id,
PublishStatus = PublishStatus.Draft,
Version = 1,
Author = user,
Gherkin = scenarioVersion.Scenario,
Description = scenarioVersion.Description,
FundingPeriodId = specification.FundingPeriod.Id,
FundingStreamIds = specification.FundingStreams.Select(s => s.Id).ToArraySafe(),
}
};
}
else
{
testScenario.Name = scenarioVersion.Name;
saveAsVersion = !string.Equals(scenarioVersion.Scenario, testScenario.Current.Gherkin) ||
scenarioVersion.Description != testScenario.Current.Description;
TestScenarioVersion newVersion = testScenario.Current.Clone() as TestScenarioVersion;
if (saveAsVersion == true)
{
newVersion.Author = user;
newVersion.Gherkin = scenarioVersion.Scenario;
newVersion.Description = scenarioVersion.Description;
newVersion.FundingStreamIds = specification.FundingStreams.Select(s => s.Id).ToArraySafe();
newVersion.FundingPeriodId = specification.FundingPeriod.Id;
newVersion = await _versionRepository.CreateVersion(newVersion, testScenario.Current);
testScenario.Current = newVersion;
}
}
HttpStatusCode statusCode = await _scenariosRepository.SaveTestScenario(testScenario);
if (!statusCode.IsSuccess())
{
_logger.Error($"Failed to save test scenario with status code: {statusCode}");
return(new StatusCodeResult((int)statusCode));
}
await _versionRepository.SaveVersion(testScenario.Current);
ScenarioIndex scenarioIndex = CreateScenarioIndexFromScenario(testScenario, specification);
await _searchRepository.Index(new List <ScenarioIndex> {
scenarioIndex
});
await _cacheProvider.RemoveAsync <List <TestScenario> >($"{CacheKeys.TestScenarios}{testScenario.SpecificationId}");
await _cacheProvider.RemoveAsync <GherkinParseResult>($"{CacheKeys.GherkinParseResult}{testScenario.Id}");
IEnumerable <Common.ApiClient.Calcs.Models.Calculation> calculations = await _calcsRepositoryPolicy.ExecuteAsync(() => _calcsRepository.GetCurrentCalculationsBySpecificationId(specification.Id));
if (calculations.IsNullOrEmpty())
{
_logger.Information($"No calculations found to test for specification id: '{specification.Id}'");
}
else
{
try
{
JobsModels.Trigger trigger = new JobsModels.Trigger
{
EntityId = testScenario.Id,
EntityType = nameof(TestScenario),
Message = $"Saving test scenario: '{testScenario.Id}'"
};
bool generateCalculationAggregations = SourceCodeHelpers.HasCalculationAggregateFunctionParameters(calculations.Select(m => m.SourceCode));
JobsModels.Job job = await SendInstructAllocationsToJobService(specification.Id, user, trigger, correlationId, generateCalculationAggregations);
_logger.Information($"New job of type '{job.JobDefinitionId}' created with id: '{job.Id}'");
}
catch (Exception ex)
{
_logger.Error(ex, $"Failed to create job of type '{JobConstants.DefinitionNames.CreateInstructAllocationJob}' on specification '{specification.Id}'");
return(new InternalServerErrorResult($"An error occurred attempting to execute calculations prior to running tests on specification '{specification.Id}'"));
}
}
CurrentTestScenario testScenarioResult = await _scenariosRepository.GetCurrentTestScenarioById(testScenario.Id);
return(new OkObjectResult(testScenarioResult));
}
public async Task QueueCsvGenerationMessages()
{
Common.ApiClient.Models.ApiResponse <IEnumerable <SpecModel.SpecificationSummary> > specificationApiResponse =
await _specificationsApiClientPolicy.ExecuteAsync(() => _specificationsApiClient.GetSpecificationSummaries());
if (!specificationApiResponse.StatusCode.IsSuccess() || specificationApiResponse.Content.IsNullOrEmpty())
{
string errorMessage = "No specification summaries found to generate calculation results csv.";
_logger.Error(errorMessage);
throw new RetriableException(errorMessage);
}
IEnumerable <SpecModel.SpecificationSummary> specificationSummaries = specificationApiResponse.Content;
Task[] queueCsvJobTasks = specificationSummaries.Select(_ =>
QueueCsvGenerationMessageIfNewCalculationResults(_.Id, _.Name))
.ToArray();
await TaskHelper.WhenAllAndThrow(queueCsvJobTasks);
}
public async Task <IActionResult> ReIndexCalculationProviderResults()
{
IEnumerable <DocumentEntity <ProviderResult> > providerResults = await _resultsRepositoryPolicy.ExecuteAsync(() => _resultsRepository.GetAllProviderResults());
IList <ProviderCalculationResultsIndex> searchItems = new List <ProviderCalculationResultsIndex>();
Dictionary <string, SpecModel.SpecificationSummary> specifications = new Dictionary <string, SpecModel.SpecificationSummary>();
foreach (DocumentEntity <ProviderResult> documentEntity in providerResults)
{
ProviderResult providerResult = documentEntity.Content;
foreach (CalculationResult calculationResult in providerResult.CalculationResults)
{
SpecModel.SpecificationSummary specificationSummary = null;
if (!specifications.ContainsKey(providerResult.SpecificationId))
{
Common.ApiClient.Models.ApiResponse <SpecModel.SpecificationSummary> specificationApiResponse =
await _specificationsApiClientPolicy.ExecuteAsync(() => _specificationsApiClient.GetSpecificationSummaryById(providerResult.SpecificationId));
if (!specificationApiResponse.StatusCode.IsSuccess() || specificationApiResponse.Content == null)
{
throw new InvalidOperationException($"Specification Summary returned null for specification ID '{providerResult.SpecificationId}'");
}
specificationSummary = specificationApiResponse.Content;
specifications.Add(providerResult.SpecificationId, specificationSummary);
}
else
{
specificationSummary = specifications[providerResult.SpecificationId];
}
ProviderCalculationResultsIndex searchItem = new ProviderCalculationResultsIndex
{
SpecificationId = providerResult.SpecificationId,
SpecificationName = specificationSummary?.Name,
CalculationName = providerResult.CalculationResults.Select(x => x.Calculation.Name).ToArraySafe(),
CalculationId = providerResult.CalculationResults.Select(x => x.Calculation.Id).ToArraySafe(),
ProviderId = providerResult.Provider.Id,
ProviderName = providerResult.Provider.Name,
ProviderType = providerResult.Provider.ProviderType,
ProviderSubType = providerResult.Provider.ProviderSubType,
LocalAuthority = providerResult.Provider.Authority,
LastUpdatedDate = documentEntity.UpdatedAt,
UKPRN = providerResult.Provider.UKPRN,
URN = providerResult.Provider.URN,
UPIN = providerResult.Provider.UPIN,
EstablishmentNumber = providerResult.Provider.EstablishmentNumber,
OpenDate = providerResult.Provider.DateOpened,
CalculationResult = providerResult.CalculationResults.Select(m => !string.IsNullOrEmpty(m.Value?.ToString()) ? m.Value.ToString() : "null").ToArraySafe()
};
if (_featureToggle.IsExceptionMessagesEnabled())
{
searchItem.CalculationException = providerResult.CalculationResults
.Where(m => !string.IsNullOrWhiteSpace(m.ExceptionType))
.Select(e => e.Calculation.Id)
.ToArraySafe();
searchItem.CalculationExceptionType = providerResult.CalculationResults
.Select(m => m.ExceptionType ?? string.Empty)
.ToArraySafe();
searchItem.CalculationExceptionMessage = providerResult.CalculationResults
.Select(m => m.ExceptionMessage ?? string.Empty)
.ToArraySafe();
}
searchItems.Add(searchItem);
}
}
const int partitionSize = 500;
for (int i = 0; i < searchItems.Count; i += partitionSize)
{
IEnumerable <ProviderCalculationResultsIndex> partitionedResults = searchItems
.Skip(i)
.Take(partitionSize);
IEnumerable <IndexError> errors = await _resultsSearchRepositoryPolicy.ExecuteAsync(() => _calculationProviderResultsSearchRepository.Index(partitionedResults));
if (errors.Any())
{
_logger.Error($"Failed to index calculation provider result documents with errors: { string.Join(";", errors.Select(m => m.ErrorMessage)) }");
return(new InternalServerErrorResult(null));
}
}
return(new NoContentResult());
}
public async Task <IActionResult> ReIndex()
{
IEnumerable <DocumentEntity <TestScenario> > testScenarios = await _scenariosRepository.GetAllTestScenarios();
List <ScenarioIndex> testScenarioIndexes = new List <ScenarioIndex>();
Dictionary <string, SpecModel.SpecificationSummary> specifications = new Dictionary <string, SpecModel.SpecificationSummary>();
foreach (DocumentEntity <TestScenario> entity in testScenarios)
{
TestScenario testScenario = entity.Content;
SpecModel.SpecificationSummary specificationSummary = null;
if (!specifications.ContainsKey(testScenario.SpecificationId))
{
Common.ApiClient.Models.ApiResponse <SpecModel.SpecificationSummary> specificationApiResponse =
await _specificationsApiClientPolicy.ExecuteAsync(() => _specificationsApiClient.GetSpecificationSummaryById(testScenario.SpecificationId));
specificationSummary = specificationApiResponse.Content;
specifications.Add(testScenario.SpecificationId, specificationSummary);
}
else
{
specificationSummary = specifications[testScenario.SpecificationId];
}
testScenarioIndexes.Add(new ScenarioIndex()
{
Id = testScenario.Id,
Name = testScenario.Name,
Description = testScenario.Current.Description,
LastUpdatedDate = entity.UpdatedAt,
FundingStreamIds = testScenario.Current?.FundingStreamIds.ToArray(),
FundingStreamNames = specificationSummary.FundingStreams.Select(s => s.Name).ToArray(),
FundingPeriodId = testScenario.Current?.FundingPeriodId,
FundingPeriodName = specificationSummary.FundingPeriod.Name,
SpecificationId = testScenario.SpecificationId,
SpecificationName = specificationSummary.Name,
Status = Enum.GetName(typeof(PublishStatus), testScenario.Current.PublishStatus),
});
}
await _searchRepository.Index(testScenarioIndexes);
return(new OkObjectResult($"Updated {testScenarioIndexes.Count} records"));
}