public void HasCalculationAggregateFunctionParameters_GivenSourceDoesNotContainAggregateParameterButCalcNameStartsWithAggregateFunction_ReturnsFalse() { //Arrange IEnumerable <string> sourceCodes = new[] { "Return SumTest()" }; //Act bool result = SourceCodeHelpers.HasCalculationAggregateFunctionParameters(sourceCodes); //Assert result .Should() .BeFalse(); }
public void HasCalculationAggregateFunctionParameters_GivenSourceDoesNotContainAggregateParameterButCalcNameContainsAggregateFunctionCaseInsensitive_ReturnsFalse() { //Arrange IEnumerable <string> sourceCodes = new[] { "Return TESTSUM()" }; //Act bool result = SourceCodeHelpers.HasCalculationAggregateFunctionParameters(sourceCodes); //Assert result .Should() .BeFalse(); }
public void HasCalculationAggregateFunctionParameters_GivenSourceContainsAggregateParameterCaseInsensitiveAndSpaces_ReturnsTrue() { //Arrange IEnumerable <string> sourceCodes = new[] { //"Return SuM (Datasets.Testing1)", "Return mIn(Datasets.Testing1) + AVG(Calc1) + SuM (Calc2)" }; //Act bool result = SourceCodeHelpers.HasCalculationAggregateFunctionParameters(sourceCodes); //Assert result .Should() .BeTrue(); }
public void HasCalculationAggregateFunctionParameters_GivenSourceDoesNotContainAggregateParameter_ReturnsFalse() { //Arrange IEnumerable <string> sourceCodes = new[] { "Return Sum(Datasets.Testing1", "Return Sum(Datasets.Testing2)" }; //Act bool result = SourceCodeHelpers.HasCalculationAggregateFunctionParameters(sourceCodes); //Assert result .Should() .BeFalse(); }
public async Task ProcessDataset(Message message) { Guard.ArgumentNotNull(message, nameof(message)); IDictionary <string, object> properties = message.UserProperties; Dataset dataset = message.GetPayloadAsInstanceOf <Dataset>(); string jobId = message.UserProperties["jobId"].ToString(); await UpdateJobStatus(jobId, null, 0); if (dataset == null) { _logger.Error("A null dataset was provided to ProcessData"); await UpdateJobStatus(jobId, false, 100, "Failed to Process - null dataset provided"); return; } if (!message.UserProperties.ContainsKey("specification-id")) { _logger.Error("Specification Id key is missing in ProcessDataset message properties"); await UpdateJobStatus(jobId, false, 100, "Failed to Process - specification id not provided"); return; } string specificationId = message.UserProperties["specification-id"].ToString(); if (string.IsNullOrWhiteSpace(specificationId)) { _logger.Error("A null or empty specification id was provided to ProcessData"); await UpdateJobStatus(jobId, false, 100, "Failed to Process - specification if is null or empty"); return; } if (!message.UserProperties.ContainsKey("relationship-id")) { _logger.Error("Relationship Id key is missing in ProcessDataset message properties"); await UpdateJobStatus(jobId, false, 100, "Failed to Process - relationship id not provided"); return; } string relationshipId = message.UserProperties["relationship-id"].ToString(); if (string.IsNullOrWhiteSpace(relationshipId)) { _logger.Error("A null or empty relationship id was provided to ProcessDataset"); await UpdateJobStatus(jobId, false, 100, "Failed to Process - relationship id is null or empty"); return; } DefinitionSpecificationRelationship relationship = await _datasetRepository.GetDefinitionSpecificationRelationshipById(relationshipId); if (relationship == null) { _logger.Error($"Relationship not found for relationship id: {relationshipId}"); await UpdateJobStatus(jobId, false, 100, "Failed to Process - relationship not found"); return; } BuildProject buildProject = null; Reference user = message.GetUserDetails(); try { buildProject = await ProcessDataset(dataset, specificationId, relationshipId, relationship.DatasetVersion.Version, user); await UpdateJobStatus(jobId, true, 100, "Processed Dataset"); } catch (NonRetriableException argEx) { // This type of exception is not retriable so fail _logger.Error(argEx, $"Failed to run ProcessDataset with exception: {argEx.Message} for relationship ID '{relationshipId}'"); await UpdateJobStatus(jobId, false, 100, $"Failed to run Process - {argEx.Message}"); return; } catch (Exception exception) { // Unknown exception occurred so allow it to be retried _logger.Error(exception, $"Failed to run ProcessDataset with exception: {exception.Message} for relationship ID '{relationshipId}'"); throw; } if (buildProject != null && !buildProject.DatasetRelationships.IsNullOrEmpty() && buildProject.DatasetRelationships.Any(m => m.DefinesScope)) { string userId = user != null ? user.Id : ""; string userName = user != null ? user.Name : ""; try { HttpStatusCode statusCode = await _calcsRepository.CompileAndSaveAssembly(specificationId); if (!statusCode.IsSuccess()) { string errorMessage = $"Failed to compile and save assembly for specification id '{specificationId}' with status code '{statusCode}'"; _logger.Error(errorMessage); throw new NonRetriableException(errorMessage); } Trigger trigger = new Trigger { EntityId = relationshipId, EntityType = nameof(DefinitionSpecificationRelationship), Message = $"Processed dataset relationship: '{relationshipId}' for specification: '{specificationId}'" }; string correlationId = message.GetCorrelationId(); IEnumerable <CalculationCurrentVersion> allCalculations = await _calcsRepository.GetCurrentCalculationsBySpecificationId(specificationId); bool generateCalculationAggregations = allCalculations.IsNullOrEmpty() ? false : SourceCodeHelpers.HasCalculationAggregateFunctionParameters(allCalculations.Select(m => m.SourceCode)); await SendInstructAllocationsToJobService($"{CacheKeys.ScopedProviderSummariesPrefix}{specificationId}", specificationId, userId, userName, trigger, correlationId, generateCalculationAggregations); } catch (Exception ex) { _logger.Error(ex, $"Failed to create job of type '{JobConstants.DefinitionNames.CreateInstructAllocationJob}' on specification '{specificationId}'"); throw new Exception($"Failed to create job of type '{JobConstants.DefinitionNames.CreateInstructAllocationJob}' on specification '{specificationId}'", ex); } } }
public async Task <IActionResult> SaveVersion(HttpRequest request) { string json = await request.GetRawBodyStringAsync(); CreateNewTestScenarioVersion scenarioVersion = JsonConvert.DeserializeObject <CreateNewTestScenarioVersion>(json); if (scenarioVersion == null) { _logger.Error("A null scenario version was provided"); return(new BadRequestObjectResult("Null or empty calculation Id provided")); } BadRequestObjectResult validationResult = (await _createNewTestScenarioVersionValidator.ValidateAsync(scenarioVersion)).PopulateModelState(); if (validationResult != null) { return(validationResult); } TestScenario testScenario = null; if (!string.IsNullOrEmpty(scenarioVersion.Id)) { testScenario = await _scenariosRepository.GetTestScenarioById(scenarioVersion.Id); } bool saveAsVersion = true; SpecificationSummary specification = await _specificationsRepository.GetSpecificationSummaryById(scenarioVersion.SpecificationId); if (specification == null) { _logger.Error($"Unable to find a specification for specification id : {scenarioVersion.SpecificationId}"); return(new StatusCodeResult(412)); } Reference user = request.GetUserOrDefault(); if (testScenario == null) { string Id = Guid.NewGuid().ToString(); testScenario = new TestScenario { Id = Id, SpecificationId = specification.Id, Name = scenarioVersion.Name, Current = new TestScenarioVersion { Date = DateTimeOffset.Now.ToLocalTime(), TestScenarioId = Id, PublishStatus = PublishStatus.Draft, Version = 1, Author = user, Gherkin = scenarioVersion.Scenario, Description = scenarioVersion.Description, FundingPeriodId = specification.FundingPeriod.Id, FundingStreamIds = specification.FundingStreams.Select(s => s.Id).ToArraySafe(), } }; } else { testScenario.Name = scenarioVersion.Name; saveAsVersion = !string.Equals(scenarioVersion.Scenario, testScenario.Current.Gherkin) || scenarioVersion.Description != testScenario.Current.Description; TestScenarioVersion newVersion = testScenario.Current.Clone() as TestScenarioVersion; if (saveAsVersion == true) { newVersion.Author = user; newVersion.Gherkin = scenarioVersion.Scenario; newVersion.Description = scenarioVersion.Description; newVersion.FundingStreamIds = specification.FundingStreams.Select(s => s.Id).ToArraySafe(); newVersion.FundingPeriodId = specification.FundingPeriod.Id; newVersion = await _versionRepository.CreateVersion(newVersion, testScenario.Current); testScenario.Current = newVersion; } } HttpStatusCode statusCode = await _scenariosRepository.SaveTestScenario(testScenario); if (!statusCode.IsSuccess()) { _logger.Error($"Failed to save test scenario with status code: {statusCode.ToString()}"); return(new StatusCodeResult((int)statusCode)); } await _versionRepository.SaveVersion(testScenario.Current); ScenarioIndex scenarioIndex = CreateScenarioIndexFromScenario(testScenario, specification); await _searchRepository.Index(new List <ScenarioIndex> { scenarioIndex }); await _cacheProvider.RemoveAsync <List <TestScenario> >($"{CacheKeys.TestScenarios}{testScenario.SpecificationId}"); await _cacheProvider.RemoveAsync <GherkinParseResult>($"{CacheKeys.GherkinParseResult}{testScenario.Id}"); IEnumerable <Models.Calcs.CalculationCurrentVersion> calculations = await _calcsRepositoryPolicy.ExecuteAsync(() => _calcsRepository.GetCurrentCalculationsBySpecificationId(specification.Id)); if (calculations.IsNullOrEmpty()) { _logger.Information($"No calculations found to test for specification id: '{specification.Id}'"); } else { string correlationId = request.GetCorrelationId(); try { Trigger trigger = new Trigger { EntityId = testScenario.Id, EntityType = nameof(TestScenario), Message = $"Saving test scenario: '{testScenario.Id}'" }; bool generateCalculationAggregations = SourceCodeHelpers.HasCalculationAggregateFunctionParameters(calculations.Select(m => m.SourceCode)); Job job = await SendInstructAllocationsToJobService(specification.Id, user, trigger, correlationId, generateCalculationAggregations); _logger.Information($"New job of type '{job.JobDefinitionId}' created with id: '{job.Id}'"); } catch (Exception ex) { _logger.Error(ex, $"Failed to create job of type '{JobConstants.DefinitionNames.CreateInstructAllocationJob}' on specification '{specification.Id}'"); return(new InternalServerErrorResult($"An error occurred attempting to execute calculations prior to running tests on specification '{specification.Id}'")); } } CurrentTestScenario testScenarioResult = await _scenariosRepository.GetCurrentTestScenarioById(testScenario.Id); return(new OkObjectResult(testScenarioResult)); }
public async Task <Job> SendInstructAllocationsToJobService(string specificationId, string userId, string userName, Trigger trigger, string correlationId, bool initiateCalcRUn = true) { Job parentJob = null; IEnumerable <Calculation> allCalculations = await _calculationRepositoryPolicy.ExecuteAsync(() => _calculationsRepository.GetCalculationsBySpecificationId(specificationId)); bool generateCalculationAggregations = SourceCodeHelpers.HasCalculationAggregateFunctionParameters(allCalculations.Select(m => m.Current.SourceCode)); string jobDefinitionId = generateCalculationAggregations ? JobConstants.DefinitionNames.CreateInstructGenerateAggregationsAllocationJob : JobConstants.DefinitionNames.CreateInstructAllocationJob; if (await GraphEnabled()) { // if the graph is enabled then we need to queue a reindex of the graph jobDefinitionId = JobConstants.DefinitionNames.ReIndexSpecificationCalculationRelationshipsJob; } JobCreateModel job = new JobCreateModel { InvokerUserDisplayName = userName, InvokerUserId = userId, JobDefinitionId = jobDefinitionId, SpecificationId = specificationId, Properties = new Dictionary <string, string> { { "specification-id", specificationId } }, Trigger = trigger, CorrelationId = correlationId }; if (await GraphEnabled()) { string parentJobDefinition = generateCalculationAggregations ? JobConstants.DefinitionNames.GenerateGraphAndInstructGenerateAggregationAllocationJob : JobConstants.DefinitionNames.GenerateGraphAndInstructAllocationJob; try { JobCreateModel instructJob = new JobCreateModel { InvokerUserDisplayName = userName, InvokerUserId = userId, JobDefinitionId = parentJobDefinition, SpecificationId = specificationId, Properties = new Dictionary <string, string> { { "specification-id", specificationId } }, Trigger = trigger, CorrelationId = correlationId }; parentJob = await _jobManagement.QueueJob(instructJob); _logger.Information($"New job of type '{parentJob.JobDefinitionId}' created with id: '{parentJob.Id}'"); } catch (Exception ex) { string errorMessage = $"Failed to create job of type '{parentJobDefinition}' on specification '{specificationId}'"; _logger.Error(ex, errorMessage); throw new RetriableException(errorMessage, ex); } } if (parentJob != null) { job.ParentJobId = parentJob.Id; } try { return(await _jobManagement.QueueJob(job)); } catch (Exception ex) { string errorMessage = $"Failed to create job of type '{job.JobDefinitionId}' on specification '{specificationId}'"; _logger.Error(ex, errorMessage); throw new RetriableException(errorMessage, ex); } }
public async Task <IActionResult> AssignDatasourceVersionToRelationship(AssignDatasourceModel model, Reference user, string correlationId) { if (model == null) { _logger.Error("Null AssignDatasourceModel was provided to AssignDatasourceVersionToRelationship"); return(new BadRequestObjectResult("Null AssignDatasourceModel was provided")); } Dataset dataset = await _datasetRepository.GetDatasetByDatasetId(model.DatasetId); if (dataset == null) { _logger.Error($"Dataset not found for dataset id: {model.DatasetId}"); return(new StatusCodeResult(412)); } DefinitionSpecificationRelationship relationship = await _datasetRepository.GetDefinitionSpecificationRelationshipById(model.RelationshipId); if (relationship == null) { _logger.Error($"Relationship not found for relationship id: {model.RelationshipId}"); return(new StatusCodeResult(412)); } relationship.Author = user; relationship.LastUpdated = _dateTimeProvider.UtcNow; relationship.DatasetVersion = new DatasetRelationshipVersion { Id = model.DatasetId, Version = model.Version }; HttpStatusCode statusCode = await _datasetRepository.UpdateDefinitionSpecificationRelationship(relationship); if (!statusCode.IsSuccess()) { _logger.Error($"Failed to assign data source to relationship : {model.RelationshipId} with status code {statusCode.ToString()}"); return(new StatusCodeResult((int)statusCode)); } IEnumerable <CalculationResponseModel> allCalculations = await _calcsRepository.GetCurrentCalculationsBySpecificationId(relationship.Specification.Id); bool generateCalculationAggregations = !allCalculations.IsNullOrEmpty() && SourceCodeHelpers.HasCalculationAggregateFunctionParameters(allCalculations.Select(m => m.SourceCode)); Trigger trigger = new Trigger { EntityId = dataset.Id, EntityType = nameof(Dataset), Message = $"Mapping dataset: '{dataset.Id}'" }; JobCreateModel job = new JobCreateModel { InvokerUserDisplayName = user?.Name, InvokerUserId = user?.Id, JobDefinitionId = JobConstants.DefinitionNames.MapDatasetJob, MessageBody = JsonConvert.SerializeObject(dataset), Properties = new Dictionary <string, string> { { "specification-id", relationship.Specification.Id }, { "relationship-id", relationship.Id }, { "user-id", user?.Id }, { "user-name", user?.Name }, }, SpecificationId = relationship.Specification.Id, Trigger = trigger, CorrelationId = correlationId }; Job parentJob = null; if (relationship.IsSetAsProviderData) { string parentJobDefinition = generateCalculationAggregations ? JobConstants.DefinitionNames.MapScopedDatasetJobWithAggregation : JobConstants.DefinitionNames.MapScopedDatasetJob; parentJob = await _jobManagement.QueueJob(new JobCreateModel { InvokerUserDisplayName = user?.Name, InvokerUserId = user?.Id, JobDefinitionId = parentJobDefinition, Properties = new Dictionary <string, string> { { "specification-id", relationship.Specification.Id }, { "provider-cache-key", $"{CacheKeys.ScopedProviderSummariesPrefix}{relationship.Specification.Id}" }, { "specification-summary-cache-key", $"{CacheKeys.SpecificationSummaryById}{relationship.Specification.Id}" } }, SpecificationId = relationship.Specification.Id, Trigger = trigger, CorrelationId = correlationId }); } if (parentJob != null) { job.ParentJobId = parentJob.Id; job.Properties.Add("parentJobId", parentJob.Id); } await _jobManagement.QueueJob(job); return(new NoContentResult()); }