public async Task <(long saveToCosmosElapsedMs, long saveToSearchElapsedMs)> SaveProviderResults(IEnumerable <ProviderResult> providerResults, int degreeOfParallelism = 5) { if (providerResults == null || providerResults.Count() == 0) { return(0, 0); } IEnumerable <KeyValuePair <string, ProviderResult> > results = providerResults.Select(m => new KeyValuePair <string, ProviderResult>(m.Provider.Id, m)); IEnumerable <string> specificationIds = providerResults.Select(s => s.SpecificationId).Distinct(); Dictionary <string, SpecificationSummary> specifications = new Dictionary <string, SpecificationSummary>(); foreach (string specificationId in specificationIds) { SpecificationSummary specification = await _specificationsRepository.GetSpecificationSummaryById(specificationId); if (specification == null) { throw new InvalidOperationException($"Result for Specification Summary lookup was null with ID '{specificationId}'"); } specifications.Add(specificationId, specification); } Task <long> cosmosSaveTask = BulkSaveProviderResults(results, degreeOfParallelism); Task <long> searchSaveTask = UpdateSearch(providerResults, specifications); await TaskHelper.WhenAllAndThrow(cosmosSaveTask, searchSaveTask); return(cosmosSaveTask.Result, searchSaveTask.Result); }
public async Task <IActionResult> ReIndex(HttpRequest request) { IEnumerable <DocumentEntity <TestScenario> > testScenarios = await _scenariosRepository.GetAllTestScenarios(); List <ScenarioIndex> testScenarioIndexes = new List <ScenarioIndex>(); Dictionary <string, Models.Specs.SpecificationSummary> specifications = new Dictionary <string, Models.Specs.SpecificationSummary>(); foreach (DocumentEntity <TestScenario> entity in testScenarios) { TestScenario testScenario = entity.Content; Models.Specs.SpecificationSummary specificationSummary = null; if (!specifications.ContainsKey(testScenario.SpecificationId)) { specificationSummary = await _specificationsRepository.GetSpecificationSummaryById(testScenario.SpecificationId); specifications.Add(testScenario.SpecificationId, specificationSummary); } else { specificationSummary = specifications[testScenario.SpecificationId]; } testScenarioIndexes.Add(new ScenarioIndex() { Id = testScenario.Id, Name = testScenario.Name, Description = testScenario.Current.Description, LastUpdatedDate = entity.UpdatedAt, FundingStreamIds = testScenario.Current?.FundingStreamIds.ToArray(), FundingStreamNames = specificationSummary.FundingStreams.Select(s => s.Name).ToArray(), FundingPeriodId = testScenario.Current?.FundingPeriodId, FundingPeriodName = specificationSummary.FundingPeriod.Name, SpecificationId = testScenario.SpecificationId, SpecificationName = specificationSummary.Name, Status = Enum.GetName(typeof(PublishStatus), testScenario.Current.PublishStatus), }); } await _searchRepository.Index(testScenarioIndexes); return(new OkObjectResult($"Updated {testScenarioIndexes.Count} records")); }
public async Task <IActionResult> SaveVersion(HttpRequest request) { string json = await request.GetRawBodyStringAsync(); CreateNewTestScenarioVersion scenarioVersion = JsonConvert.DeserializeObject <CreateNewTestScenarioVersion>(json); if (scenarioVersion == null) { _logger.Error("A null scenario version was provided"); return(new BadRequestObjectResult("Null or empty calculation Id provided")); } BadRequestObjectResult validationResult = (await _createNewTestScenarioVersionValidator.ValidateAsync(scenarioVersion)).PopulateModelState(); if (validationResult != null) { return(validationResult); } TestScenario testScenario = null; if (!string.IsNullOrEmpty(scenarioVersion.Id)) { testScenario = await _scenariosRepository.GetTestScenarioById(scenarioVersion.Id); } bool saveAsVersion = true; SpecificationSummary specification = await _specificationsRepository.GetSpecificationSummaryById(scenarioVersion.SpecificationId); if (specification == null) { _logger.Error($"Unable to find a specification for specification id : {scenarioVersion.SpecificationId}"); return(new StatusCodeResult(412)); } Reference user = request.GetUserOrDefault(); if (testScenario == null) { string Id = Guid.NewGuid().ToString(); testScenario = new TestScenario { Id = Id, SpecificationId = specification.Id, Name = scenarioVersion.Name, Current = new TestScenarioVersion { Date = DateTimeOffset.Now.ToLocalTime(), TestScenarioId = Id, PublishStatus = PublishStatus.Draft, Version = 1, Author = user, Gherkin = scenarioVersion.Scenario, Description = scenarioVersion.Description, FundingPeriodId = specification.FundingPeriod.Id, FundingStreamIds = specification.FundingStreams.Select(s => s.Id).ToArraySafe(), } }; } else { testScenario.Name = scenarioVersion.Name; saveAsVersion = !string.Equals(scenarioVersion.Scenario, testScenario.Current.Gherkin) || scenarioVersion.Description != testScenario.Current.Description; TestScenarioVersion newVersion = testScenario.Current.Clone() as TestScenarioVersion; if (saveAsVersion == true) { newVersion.Author = user; newVersion.Gherkin = scenarioVersion.Scenario; newVersion.Description = scenarioVersion.Description; newVersion.FundingStreamIds = specification.FundingStreams.Select(s => s.Id).ToArraySafe(); newVersion.FundingPeriodId = specification.FundingPeriod.Id; newVersion = await _versionRepository.CreateVersion(newVersion, testScenario.Current); testScenario.Current = newVersion; } } HttpStatusCode statusCode = await _scenariosRepository.SaveTestScenario(testScenario); if (!statusCode.IsSuccess()) { _logger.Error($"Failed to save test scenario with status code: {statusCode.ToString()}"); return(new StatusCodeResult((int)statusCode)); } await _versionRepository.SaveVersion(testScenario.Current); ScenarioIndex scenarioIndex = CreateScenarioIndexFromScenario(testScenario, specification); await _searchRepository.Index(new List <ScenarioIndex> { scenarioIndex }); await _cacheProvider.RemoveAsync <List <TestScenario> >($"{CacheKeys.TestScenarios}{testScenario.SpecificationId}"); await _cacheProvider.RemoveAsync <GherkinParseResult>($"{CacheKeys.GherkinParseResult}{testScenario.Id}"); IEnumerable <Models.Calcs.CalculationCurrentVersion> calculations = await _calcsRepositoryPolicy.ExecuteAsync(() => _calcsRepository.GetCurrentCalculationsBySpecificationId(specification.Id)); if (calculations.IsNullOrEmpty()) { _logger.Information($"No calculations found to test for specification id: '{specification.Id}'"); } else { string correlationId = request.GetCorrelationId(); try { Trigger trigger = new Trigger { EntityId = testScenario.Id, EntityType = nameof(TestScenario), Message = $"Saving test scenario: '{testScenario.Id}'" }; bool generateCalculationAggregations = SourceCodeHelpers.HasCalculationAggregateFunctionParameters(calculations.Select(m => m.SourceCode)); Job job = await SendInstructAllocationsToJobService(specification.Id, user, trigger, correlationId, generateCalculationAggregations); _logger.Information($"New job of type '{job.JobDefinitionId}' created with id: '{job.Id}'"); } catch (Exception ex) { _logger.Error(ex, $"Failed to create job of type '{JobConstants.DefinitionNames.CreateInstructAllocationJob}' on specification '{specification.Id}'"); return(new InternalServerErrorResult($"An error occurred attempting to execute calculations prior to running tests on specification '{specification.Id}'")); } } CurrentTestScenario testScenarioResult = await _scenariosRepository.GetCurrentTestScenarioById(testScenario.Id); return(new OkObjectResult(testScenarioResult)); }
public async Task SaveProviderResults_WhenExcludedResultsAndIsNewProviderCalculationResultsIndexEnabled_ThenResultsSavedToCosmosSavesNull() { // Arrange ICosmosRepository cosmosRepository = CreateCosmosRepository(); ISearchRepository <ProviderCalculationResultsIndex> searchRepository = CreateProviderCalculationResultsSearchRepository(); ISpecificationsRepository specificationsRepository = CreateSpecificationsRepository(); IFeatureToggle featureToggle = CreateFeatureToggle(); featureToggle .IsNewProviderCalculationResultsIndexEnabled() .Returns(true); ProviderResultsRepository repo = CreateProviderResultsRepository(cosmosRepository, specificationsRepository: specificationsRepository, providerCalculationResultsSearchRepository: searchRepository, featureToggle: featureToggle); specificationsRepository.GetSpecificationSummaryById(Arg.Any <string>()).Returns(new SpecificationSummary { Name = "Specification 1" }); IEnumerable <ProviderResult> results = new List <ProviderResult> { new ProviderResult { AllocationLineResults = new List <AllocationLineResult> { new AllocationLineResult { AllocationLine = new Reference { Id = "alloc 1", Name = "Allocation one" }, Value = 1112.3M } }, CalculationResults = new List <CalculationResult> { new CalculationResult { AllocationLine = new Reference { Id = "alloc1", Name = "Allocation one" }, Calculation = new Reference { Id = "calc1", Name = "calculation one" }, CalculationSpecification = new Reference { Id = "calc1", Name = "calculation one" }, CalculationType = Models.Calcs.CalculationType.Funding, Value = null } }, Id = Guid.NewGuid().ToString(), Provider = new ProviderSummary { Id = "prov1", Name = "Provider 1", ProviderType = "TYpe 1", ProviderSubType = "Sub type 1", Authority = "Authority", UKPRN = "ukprn123", URN = "urn123", EstablishmentNumber = "en123", UPIN = "upin123", DateOpened = DateTime.Now }, SpecificationId = "spec1" } }; // Act await repo.SaveProviderResults(results); // Assert await cosmosRepository.Received().BulkUpsertAsync(Arg.Is <IEnumerable <KeyValuePair <string, ProviderResult> > >(r => r.Count() == 1), Arg.Any <int>(), Arg.Any <bool>(), Arg.Is <bool>(false)); await searchRepository.Received(1).Index(Arg.Is <IEnumerable <ProviderCalculationResultsIndex> >(r => r.First().SpecificationId == results.First().SpecificationId&& r.First().SpecificationName == "Specification 1" && r.First().CalculationId.Any() && r.First().CalculationId.First() == results.First().CalculationResults.First().Calculation.Id&& r.First().CalculationName.Any() && r.First().CalculationName.First() == results.First().CalculationResults.First().Calculation.Name&& r.First().ProviderId == results.First().Provider.Id&& r.First().ProviderName == results.First().Provider.Name&& r.First().ProviderType == results.First().Provider.ProviderType&& r.First().ProviderSubType == results.First().Provider.ProviderSubType&& r.First().LocalAuthority == results.First().Provider.Authority&& r.First().UKPRN == results.First().Provider.UKPRN&& r.First().URN == results.First().Provider.URN&& r.First().UPIN == results.First().Provider.UPIN&& r.First().EstablishmentNumber == results.First().Provider.EstablishmentNumber&& r.First().OpenDate == results.First().Provider.DateOpened&& r.First().CalculationResult.Any() && r.First().CalculationResult.First() == "null")); }
public async Task SaveProviderResults_WhenResults_ThenResultsSavedToCosmos() { // Arrange ICosmosRepository cosmosRepository = CreateCosmosRepository(); ISearchRepository <CalculationProviderResultsIndex> searchRepository = CreateCalculationProviderResultsSearchRepository(); ISpecificationsRepository specificationsRepository = CreateSpecificationsRepository(); ProviderResultsRepository repo = CreateProviderResultsRepository(cosmosRepository, searchRepository, specificationsRepository); specificationsRepository.GetSpecificationSummaryById(Arg.Any <string>()).Returns(new SpecificationSummary { Name = "Specification 1" }); IEnumerable <ProviderResult> results = new List <ProviderResult> { new ProviderResult { AllocationLineResults = new List <AllocationLineResult> { new AllocationLineResult { AllocationLine = new Reference { Id = "alloc 1", Name = "Allocation one" }, Value = 1112.3M } }, CalculationResults = new List <CalculationResult> { new CalculationResult { AllocationLine = new Reference { Id = "alloc1", Name = "Allocation one" }, Calculation = new Reference { Id = "calc1", Name = "calculation one" }, CalculationSpecification = new Reference { Id = "calc1", Name = "calculation one" }, CalculationType = Models.Calcs.CalculationType.Funding, Value = 1112.3M } }, Id = Guid.NewGuid().ToString(), Provider = new ProviderSummary { Id = "prov1", Name = "Provider 1", ProviderType = "TYpe 1", ProviderSubType = "Sub type 1", Authority = "Authority", UKPRN = "ukprn123", URN = "urn123", EstablishmentNumber = "en123", UPIN = "upin123", DateOpened = DateTime.Now }, SpecificationId = "spec1" } }; // Act await repo.SaveProviderResults(results); // Assert await cosmosRepository.Received().BulkUpsertAsync(Arg.Is <IEnumerable <KeyValuePair <string, ProviderResult> > >(r => r.Count() == 1), Arg.Any <int>(), Arg.Any <bool>(), Arg.Is <bool>(false)); }
public async Task SaveProviderResults_WhenResults_ThenResultsSavedToSearch() { // Arrange ICosmosRepository cosmosRepository = CreateCosmosRepository(); ISearchRepository <CalculationProviderResultsIndex> searchRepository = CreateCalculationProviderResultsSearchRepository(); ISpecificationsRepository specificationsRepository = CreateSpecificationsRepository(); ProviderResultsRepository repo = CreateProviderResultsRepository(cosmosRepository, searchRepository, specificationsRepository); specificationsRepository.GetSpecificationSummaryById(Arg.Any <string>()).Returns(new SpecificationSummary { Name = "Specification 1" }); IEnumerable <ProviderResult> results = new List <ProviderResult> { new ProviderResult { AllocationLineResults = new List <AllocationLineResult> { new AllocationLineResult { AllocationLine = new Reference { Id = "alloc 1", Name = "Allocation one" }, Value = 1112.3M } }, CalculationResults = new List <CalculationResult> { new CalculationResult { AllocationLine = new Reference { Id = "alloc1", Name = "Allocation one" }, Calculation = new Reference { Id = "calc1", Name = "calculation one" }, CalculationSpecification = new Reference { Id = "calc1", Name = "calculation one" }, CalculationType = Models.Calcs.CalculationType.Funding, Value = 1112.3M } }, Id = Guid.NewGuid().ToString(), Provider = new ProviderSummary { Id = "prov1", Name = "Provider 1", ProviderType = "TYpe 1", ProviderSubType = "Sub type 1", Authority = "Authority", UKPRN = "ukprn123", URN = "urn123", EstablishmentNumber = "en123", UPIN = "upin123", DateOpened = DateTime.Now }, SpecificationId = "spec1" } }; // Act await repo.SaveProviderResults(results); // Assert await searchRepository.Received(1).Index(Arg.Is <IEnumerable <CalculationProviderResultsIndex> >(r => r.Count() == 1)); await searchRepository.Received(1).Index(Arg.Is <IEnumerable <CalculationProviderResultsIndex> >(r => r.First().SpecificationId == results.First().SpecificationId&& r.First().SpecificationName == "Specification 1" && r.First().CalculationSpecificationId == results.First().CalculationResults.First().CalculationSpecification.Id&& r.First().CalculationSpecificationName == results.First().CalculationResults.First().CalculationSpecification.Name&& r.First().CalculationName == results.First().CalculationResults.First().Calculation.Name&& r.First().CalculationId == results.First().CalculationResults.First().Calculation.Id&& r.First().CalculationType == results.First().CalculationResults.First().CalculationType.ToString() && r.First().ProviderId == results.First().Provider.Id&& r.First().ProviderName == results.First().Provider.Name&& r.First().ProviderType == results.First().Provider.ProviderType&& r.First().ProviderSubType == results.First().Provider.ProviderSubType&& r.First().LocalAuthority == results.First().Provider.Authority&& r.First().UKPRN == results.First().Provider.UKPRN&& r.First().URN == results.First().Provider.URN&& r.First().UPIN == results.First().Provider.UPIN&& r.First().EstablishmentNumber == results.First().Provider.EstablishmentNumber&& r.First().OpenDate == results.First().Provider.DateOpened&& r.First().CalculationResult == Convert.ToDouble(results.First().CalculationResults.First().Value) && r.First().IsExcluded == false)); }
public async Task <IActionResult> CreateRelationship(HttpRequest request) { string json = await request.GetRawBodyStringAsync(); CreateDefinitionSpecificationRelationshipModel model = JsonConvert.DeserializeObject <CreateDefinitionSpecificationRelationshipModel>(json); if (model == null) { _logger.Error("Null CreateDefinitionSpecificationRelationshipModel was provided to CreateRelationship"); return(new BadRequestObjectResult("Null CreateDefinitionSpecificationRelationshipModel was provided")); } BadRequestObjectResult validationResult = (await _relationshipModelValidator.ValidateAsync(model)).PopulateModelState(); if (validationResult != null) { return(validationResult); } DatasetDefinition definition = await _datasetRepository.GetDatasetDefinition(model.DatasetDefinitionId); if (definition == null) { _logger.Error($"Datset definition was not found for id {model.DatasetDefinitionId}"); return(new StatusCodeResult(412)); } SpecificationSummary specification = await _specificationsRepository.GetSpecificationSummaryById(model.SpecificationId); if (specification == null) { _logger.Error($"Specification was not found for id {model.SpecificationId}"); return(new StatusCodeResult(412)); } string relationshipId = Guid.NewGuid().ToString(); DefinitionSpecificationRelationship relationship = new DefinitionSpecificationRelationship { Name = model.Name, DatasetDefinition = new Reference(definition.Id, definition.Name), Specification = new Reference(specification.Id, specification.Name), Description = model.Description, Id = relationshipId, IsSetAsProviderData = model.IsSetAsProviderData, UsedInDataAggregations = model.UsedInDataAggregations }; HttpStatusCode statusCode = await _datasetRepository.SaveDefinitionSpecificationRelationship(relationship); if (!statusCode.IsSuccess()) { _logger.Error($"Failed to save relationship with status code: {statusCode.ToString()}"); return(new StatusCodeResult((int)statusCode)); } IDictionary <string, string> properties = request.BuildMessageProperties(); await _messengerService.SendToQueue(ServiceBusConstants.QueueNames.AddDefinitionRelationshipToSpecification, new AssignDefinitionRelationshipMessage { SpecificationId = specification.Id, RelationshipId = relationshipId }, properties); DatasetRelationshipSummary relationshipSummary = new DatasetRelationshipSummary { Name = relationship.Name, Id = Guid.NewGuid().ToString(), Relationship = new Reference(relationship.Id, relationship.Name), DatasetDefinition = definition, DatasetDefinitionId = definition.Id, DataGranularity = relationship.UsedInDataAggregations ? DataGranularity.MultipleRowsPerProvider : DataGranularity.SingleRowPerProvider, DefinesScope = relationship.IsSetAsProviderData }; BuildProject buildProject = await _calcsRepository.UpdateBuildProjectRelationships(specification.Id, relationshipSummary); await _cacheProvider.RemoveAsync <IEnumerable <DatasetSchemaRelationshipModel> >($"{CacheKeys.DatasetRelationshipFieldsForSpecification}{specification.Id}"); return(new OkObjectResult(relationship)); }
public async Task <IActionResult> ReIndexCalculationProviderResults() { IEnumerable <DocumentEntity <ProviderResult> > providerResults = await _resultsRepositoryPolicy.ExecuteAsync(() => _resultsRepository.GetAllProviderResults()); IList <ProviderCalculationResultsIndex> searchItems = new List <ProviderCalculationResultsIndex>(); Dictionary <string, SpecificationSummary> specifications = new Dictionary <string, SpecificationSummary>(); foreach (DocumentEntity <ProviderResult> documentEntity in providerResults) { ProviderResult providerResult = documentEntity.Content; foreach (CalculationResult calculationResult in providerResult.CalculationResults) { SpecificationSummary specificationSummary = null; if (!specifications.ContainsKey(providerResult.SpecificationId)) { specificationSummary = await _specificationsRepositoryPolicy.ExecuteAsync(() => _specificationsRepository.GetSpecificationSummaryById(providerResult.SpecificationId)); if (specificationSummary == null) { throw new InvalidOperationException($"Specification Summary returned null for specification ID '{providerResult.SpecificationId}'"); } specifications.Add(providerResult.SpecificationId, specificationSummary); } else { specificationSummary = specifications[providerResult.SpecificationId]; } ProviderCalculationResultsIndex searchItem = new ProviderCalculationResultsIndex { SpecificationId = providerResult.SpecificationId, SpecificationName = specificationSummary?.Name, CalculationName = providerResult.CalculationResults.Select(x => x.Calculation.Name).ToArraySafe(), CalculationId = providerResult.CalculationResults.Select(x => x.Calculation.Id).ToArraySafe(), ProviderId = providerResult.Provider.Id, ProviderName = providerResult.Provider.Name, ProviderType = providerResult.Provider.ProviderType, ProviderSubType = providerResult.Provider.ProviderSubType, LocalAuthority = providerResult.Provider.Authority, LastUpdatedDate = documentEntity.UpdatedAt, UKPRN = providerResult.Provider.UKPRN, URN = providerResult.Provider.URN, UPIN = providerResult.Provider.UPIN, EstablishmentNumber = providerResult.Provider.EstablishmentNumber, OpenDate = providerResult.Provider.DateOpened, CalculationResult = providerResult.CalculationResults.Select(m => m.Value.HasValue ? m.Value.ToString() : "null").ToArraySafe() }; if (_featureToggle.IsExceptionMessagesEnabled()) { searchItem.CalculationException = providerResult.CalculationResults .Where(m => !string.IsNullOrWhiteSpace(m.ExceptionType)) .Select(e => e.Calculation.Id) .ToArraySafe(); searchItem.CalculationExceptionType = providerResult.CalculationResults .Select(m => m.ExceptionType ?? string.Empty) .ToArraySafe(); searchItem.CalculationExceptionMessage = providerResult.CalculationResults .Select(m => m.ExceptionMessage ?? string.Empty) .ToArraySafe(); } searchItems.Add(searchItem); } } const int partitionSize = 500; for (int i = 0; i < searchItems.Count; i += partitionSize) { IEnumerable <ProviderCalculationResultsIndex> partitionedResults = searchItems.Skip(i).Take(partitionSize); IEnumerable <IndexError> errors = await _resultsSearchRepositoryPolicy.ExecuteAsync(() => _calculationProviderResultsSearchRepository.Index(partitionedResults)); if (errors.Any()) { _logger.Error($"Failed to index calculation provider result documents with errors: { string.Join(";", errors.Select(m => m.ErrorMessage)) }"); return(new InternalServerErrorResult(null)); } } return(new NoContentResult()); }