private async Task <Dictionary <string, Provider> > GetAllProviders(string providerVersionId) { if (!_providers.ContainsKey(providerVersionId)) { Common.ApiClient.Models.ApiResponse <ProviderVersion> providerResult = await _providersApiClientPolicy.ExecuteAsync(() => _providersApiClient.GetProvidersByVersion(providerVersionId)); if (providerResult == null) { throw new InvalidOperationException($"Provider lookup returned null for provider version id '{providerVersionId}'"); } if (providerResult.Content == null) { throw new InvalidOperationException($"Provider lookup content returned null for provider version id '{providerVersionId}'"); } if (providerResult.Content.Providers == null) { throw new InvalidOperationException($"Provider lookup provider content returned null for provider version id '{providerVersionId}'"); } _providers.Add(providerVersionId, providerResult.Content.Providers.ToDictionary(c => c.ProviderId, p => p)); } return(_providers[providerVersionId]); }
public async Task <bool> SpecificationHasResults(string specificationId) { Common.ApiClient.Models.ApiResponse <bool> providerResults = await _apiClientProxy.GetProviderHasResultsBySpecificationId(specificationId); if (providerResults == null) { throw new InvalidOperationException("Provider results should not return null"); } return(providerResults.Content); }
public async Task <IActionResult> QueueCsvGeneration(string specificationId) { Common.ApiClient.Models.ApiResponse <SpecModel.SpecificationSummary> specificationApiResponse = await _specificationsApiClientPolicy.ExecuteAsync(() => _specificationsApiClient.GetSpecificationSummaryById(specificationId)); if (!specificationApiResponse.StatusCode.IsSuccess() || specificationApiResponse.Content == null) { string errorMessage = "No specification summaries found to generate calculation results csv."; _logger.Error(errorMessage); throw new RetriableException(errorMessage); } Job job = await QueueCsvGenerationMessage(specificationId, specificationApiResponse.Content.Name); return(new OkObjectResult(job)); }
public async Task <IActionResult> SaveVersion(CreateNewTestScenarioVersion scenarioVersion, Reference user, string correlationId) { if (scenarioVersion == null) { _logger.Error("A null scenario version was provided"); return(new BadRequestObjectResult("Null or empty calculation Id provided")); } BadRequestObjectResult validationResult = (await _createNewTestScenarioVersionValidator.ValidateAsync(scenarioVersion)).PopulateModelState(); if (validationResult != null) { return(validationResult); } TestScenario testScenario = null; if (!string.IsNullOrEmpty(scenarioVersion.Id)) { testScenario = await _scenariosRepository.GetTestScenarioById(scenarioVersion.Id); } bool saveAsVersion = true; Common.ApiClient.Models.ApiResponse <SpecModel.SpecificationSummary> specificationApiResponse = await _specificationsApiClientPolicy.ExecuteAsync(() => _specificationsApiClient.GetSpecificationSummaryById(scenarioVersion.SpecificationId)); if (!specificationApiResponse.StatusCode.IsSuccess() || specificationApiResponse.Content == null) { _logger.Error($"Unable to find a specification for specification id : {scenarioVersion.SpecificationId}"); return(new StatusCodeResult(412)); } SpecModel.SpecificationSummary specification = specificationApiResponse.Content; if (testScenario == null) { string Id = Guid.NewGuid().ToString(); testScenario = new TestScenario { Id = Id, SpecificationId = specification.Id, Name = scenarioVersion.Name, Current = new TestScenarioVersion { Date = DateTimeOffset.Now.ToLocalTime(), TestScenarioId = Id, PublishStatus = PublishStatus.Draft, Version = 1, Author = user, Gherkin = scenarioVersion.Scenario, Description = scenarioVersion.Description, FundingPeriodId = specification.FundingPeriod.Id, FundingStreamIds = specification.FundingStreams.Select(s => s.Id).ToArraySafe(), } }; } else { testScenario.Name = scenarioVersion.Name; saveAsVersion = !string.Equals(scenarioVersion.Scenario, testScenario.Current.Gherkin) || scenarioVersion.Description != testScenario.Current.Description; TestScenarioVersion newVersion = testScenario.Current.Clone() as TestScenarioVersion; if (saveAsVersion == true) { newVersion.Author = user; newVersion.Gherkin = scenarioVersion.Scenario; newVersion.Description = scenarioVersion.Description; newVersion.FundingStreamIds = specification.FundingStreams.Select(s => s.Id).ToArraySafe(); newVersion.FundingPeriodId = specification.FundingPeriod.Id; newVersion = await _versionRepository.CreateVersion(newVersion, testScenario.Current); testScenario.Current = newVersion; } } HttpStatusCode statusCode = await _scenariosRepository.SaveTestScenario(testScenario); if (!statusCode.IsSuccess()) { _logger.Error($"Failed to save test scenario with status code: {statusCode}"); return(new StatusCodeResult((int)statusCode)); } await _versionRepository.SaveVersion(testScenario.Current); ScenarioIndex scenarioIndex = CreateScenarioIndexFromScenario(testScenario, specification); await _searchRepository.Index(new List <ScenarioIndex> { scenarioIndex }); await _cacheProvider.RemoveAsync <List <TestScenario> >($"{CacheKeys.TestScenarios}{testScenario.SpecificationId}"); await _cacheProvider.RemoveAsync <GherkinParseResult>($"{CacheKeys.GherkinParseResult}{testScenario.Id}"); IEnumerable <Common.ApiClient.Calcs.Models.Calculation> calculations = await _calcsRepositoryPolicy.ExecuteAsync(() => _calcsRepository.GetCurrentCalculationsBySpecificationId(specification.Id)); if (calculations.IsNullOrEmpty()) { _logger.Information($"No calculations found to test for specification id: '{specification.Id}'"); } else { try { JobsModels.Trigger trigger = new JobsModels.Trigger { EntityId = testScenario.Id, EntityType = nameof(TestScenario), Message = $"Saving test scenario: '{testScenario.Id}'" }; bool generateCalculationAggregations = SourceCodeHelpers.HasCalculationAggregateFunctionParameters(calculations.Select(m => m.SourceCode)); JobsModels.Job job = await SendInstructAllocationsToJobService(specification.Id, user, trigger, correlationId, generateCalculationAggregations); _logger.Information($"New job of type '{job.JobDefinitionId}' created with id: '{job.Id}'"); } catch (Exception ex) { _logger.Error(ex, $"Failed to create job of type '{JobConstants.DefinitionNames.CreateInstructAllocationJob}' on specification '{specification.Id}'"); return(new InternalServerErrorResult($"An error occurred attempting to execute calculations prior to running tests on specification '{specification.Id}'")); } } CurrentTestScenario testScenarioResult = await _scenariosRepository.GetCurrentTestScenarioById(testScenario.Id); return(new OkObjectResult(testScenarioResult)); }
public async Task QueueCsvGenerationMessages() { Common.ApiClient.Models.ApiResponse <IEnumerable <SpecModel.SpecificationSummary> > specificationApiResponse = await _specificationsApiClientPolicy.ExecuteAsync(() => _specificationsApiClient.GetSpecificationSummaries()); if (!specificationApiResponse.StatusCode.IsSuccess() || specificationApiResponse.Content.IsNullOrEmpty()) { string errorMessage = "No specification summaries found to generate calculation results csv."; _logger.Error(errorMessage); throw new RetriableException(errorMessage); } IEnumerable <SpecModel.SpecificationSummary> specificationSummaries = specificationApiResponse.Content; Task[] queueCsvJobTasks = specificationSummaries.Select(_ => QueueCsvGenerationMessageIfNewCalculationResults(_.Id, _.Name)) .ToArray(); await TaskHelper.WhenAllAndThrow(queueCsvJobTasks); }
public async Task <IActionResult> ReIndexCalculationProviderResults() { IEnumerable <DocumentEntity <ProviderResult> > providerResults = await _resultsRepositoryPolicy.ExecuteAsync(() => _resultsRepository.GetAllProviderResults()); IList <ProviderCalculationResultsIndex> searchItems = new List <ProviderCalculationResultsIndex>(); Dictionary <string, SpecModel.SpecificationSummary> specifications = new Dictionary <string, SpecModel.SpecificationSummary>(); foreach (DocumentEntity <ProviderResult> documentEntity in providerResults) { ProviderResult providerResult = documentEntity.Content; foreach (CalculationResult calculationResult in providerResult.CalculationResults) { SpecModel.SpecificationSummary specificationSummary = null; if (!specifications.ContainsKey(providerResult.SpecificationId)) { Common.ApiClient.Models.ApiResponse <SpecModel.SpecificationSummary> specificationApiResponse = await _specificationsApiClientPolicy.ExecuteAsync(() => _specificationsApiClient.GetSpecificationSummaryById(providerResult.SpecificationId)); if (!specificationApiResponse.StatusCode.IsSuccess() || specificationApiResponse.Content == null) { throw new InvalidOperationException($"Specification Summary returned null for specification ID '{providerResult.SpecificationId}'"); } specificationSummary = specificationApiResponse.Content; specifications.Add(providerResult.SpecificationId, specificationSummary); } else { specificationSummary = specifications[providerResult.SpecificationId]; } ProviderCalculationResultsIndex searchItem = new ProviderCalculationResultsIndex { SpecificationId = providerResult.SpecificationId, SpecificationName = specificationSummary?.Name, CalculationName = providerResult.CalculationResults.Select(x => x.Calculation.Name).ToArraySafe(), CalculationId = providerResult.CalculationResults.Select(x => x.Calculation.Id).ToArraySafe(), ProviderId = providerResult.Provider.Id, ProviderName = providerResult.Provider.Name, ProviderType = providerResult.Provider.ProviderType, ProviderSubType = providerResult.Provider.ProviderSubType, LocalAuthority = providerResult.Provider.Authority, LastUpdatedDate = documentEntity.UpdatedAt, UKPRN = providerResult.Provider.UKPRN, URN = providerResult.Provider.URN, UPIN = providerResult.Provider.UPIN, EstablishmentNumber = providerResult.Provider.EstablishmentNumber, OpenDate = providerResult.Provider.DateOpened, CalculationResult = providerResult.CalculationResults.Select(m => !string.IsNullOrEmpty(m.Value?.ToString()) ? m.Value.ToString() : "null").ToArraySafe() }; if (_featureToggle.IsExceptionMessagesEnabled()) { searchItem.CalculationException = providerResult.CalculationResults .Where(m => !string.IsNullOrWhiteSpace(m.ExceptionType)) .Select(e => e.Calculation.Id) .ToArraySafe(); searchItem.CalculationExceptionType = providerResult.CalculationResults .Select(m => m.ExceptionType ?? string.Empty) .ToArraySafe(); searchItem.CalculationExceptionMessage = providerResult.CalculationResults .Select(m => m.ExceptionMessage ?? string.Empty) .ToArraySafe(); } searchItems.Add(searchItem); } } const int partitionSize = 500; for (int i = 0; i < searchItems.Count; i += partitionSize) { IEnumerable <ProviderCalculationResultsIndex> partitionedResults = searchItems .Skip(i) .Take(partitionSize); IEnumerable <IndexError> errors = await _resultsSearchRepositoryPolicy.ExecuteAsync(() => _calculationProviderResultsSearchRepository.Index(partitionedResults)); if (errors.Any()) { _logger.Error($"Failed to index calculation provider result documents with errors: { string.Join(";", errors.Select(m => m.ErrorMessage)) }"); return(new InternalServerErrorResult(null)); } } return(new NoContentResult()); }
public async Task <IActionResult> ReIndex() { IEnumerable <DocumentEntity <TestScenario> > testScenarios = await _scenariosRepository.GetAllTestScenarios(); List <ScenarioIndex> testScenarioIndexes = new List <ScenarioIndex>(); Dictionary <string, SpecModel.SpecificationSummary> specifications = new Dictionary <string, SpecModel.SpecificationSummary>(); foreach (DocumentEntity <TestScenario> entity in testScenarios) { TestScenario testScenario = entity.Content; SpecModel.SpecificationSummary specificationSummary = null; if (!specifications.ContainsKey(testScenario.SpecificationId)) { Common.ApiClient.Models.ApiResponse <SpecModel.SpecificationSummary> specificationApiResponse = await _specificationsApiClientPolicy.ExecuteAsync(() => _specificationsApiClient.GetSpecificationSummaryById(testScenario.SpecificationId)); specificationSummary = specificationApiResponse.Content; specifications.Add(testScenario.SpecificationId, specificationSummary); } else { specificationSummary = specifications[testScenario.SpecificationId]; } testScenarioIndexes.Add(new ScenarioIndex() { Id = testScenario.Id, Name = testScenario.Name, Description = testScenario.Current.Description, LastUpdatedDate = entity.UpdatedAt, FundingStreamIds = testScenario.Current?.FundingStreamIds.ToArray(), FundingStreamNames = specificationSummary.FundingStreams.Select(s => s.Name).ToArray(), FundingPeriodId = testScenario.Current?.FundingPeriodId, FundingPeriodName = specificationSummary.FundingPeriod.Name, SpecificationId = testScenario.SpecificationId, SpecificationName = specificationSummary.Name, Status = Enum.GetName(typeof(PublishStatus), testScenario.Current.PublishStatus), }); } await _searchRepository.Index(testScenarioIndexes); return(new OkObjectResult($"Updated {testScenarioIndexes.Count} records")); }