private async Task <Build> CheckDatasetValidAggregations(PreviewRequest previewRequest)
{
Build build = null;
IEnumerable <string> aggregateParameters = SourceCodeHelpers.GetDatasetAggregateFunctionParameters(previewRequest.SourceCode);
if (aggregateParameters.IsNullOrEmpty())
{
return(build);
}
string cacheKey = $"{CacheKeys.DatasetRelationshipFieldsForSpecification}{previewRequest.SpecificationId}";
IEnumerable <DatasetSchemaRelationshipModel> datasetSchemaRelationshipModels = Enumerable.Empty <DatasetSchemaRelationshipModel>();
datasetSchemaRelationshipModels = await _cacheProvider.GetAsync <List <DatasetSchemaRelationshipModel> >(cacheKey);
if (datasetSchemaRelationshipModels.IsNullOrEmpty())
{
ApiResponse <IEnumerable <Common.ApiClient.DataSets.Models.DatasetSchemaRelationshipModel> > datasetsApiClientResponse = await _datasetsApiClientPolicy.ExecuteAsync(() => _datasetsApiClient.GetDatasetSchemaRelationshipModelsForSpecificationId(previewRequest.SpecificationId));
if (!datasetsApiClientResponse.StatusCode.IsSuccess())
{
string message = $"No dataset schema relationship found for specificationId '{previewRequest.SpecificationId}'.";
_logger.Error(message);
throw new RetriableException(message);
}
if (datasetsApiClientResponse.Content != null)
{
datasetSchemaRelationshipModels = _mapper.Map <IEnumerable <DatasetSchemaRelationshipModel> >(datasetsApiClientResponse.Content);
}
await _cacheProvider.SetAsync <List <DatasetSchemaRelationshipModel> >(cacheKey, datasetSchemaRelationshipModels.ToList());
}
HashSet <string> compilerErrors = new HashSet <string>();
IEnumerable <string> datasetAggregationFields = datasetSchemaRelationshipModels?.SelectMany(m => m.Fields?.Where(f => f.IsAggregable).Select(f => f.FullyQualifiedSourceName));
foreach (string aggregateParameter in aggregateParameters)
{
if (datasetAggregationFields.IsNullOrEmpty() || !datasetAggregationFields.Any(m => string.Equals(m.Trim(), aggregateParameter.Trim(), StringComparison.CurrentCultureIgnoreCase)))
{
compilerErrors.Add($"{aggregateParameter} is not an aggregable field");
}
}
if (compilerErrors.Any())
{
build = new Build
{
CompilerMessages = compilerErrors.Select(m => new CompilerMessage {
Message = m, Severity = Severity.Error
}).ToList()
};
}
return(build);
}
public void GetReferencedCalculations_GivenListOfCalcsAndSourceCodeContainsTwoCalcs_ReturnsTwoCalcs()
{
//Arrange
string sourceCode = " calc1() + calc2()";
IEnumerable <string> calcNames = new[] { "calc1", "calc2", "calc3" };
//Act
IEnumerable <string> results = SourceCodeHelpers.GetReferencedCalculations(calcNames, sourceCode);
//Assert
results
.Count()
.Should()
.Be(2);
results
.ElementAt(0)
.Should()
.Be("calc1");
results
.ElementAt(1)
.Should()
.Be("calc2");
}
public void GetCalculationAggregateFunctionParameters_GivenSourceHasZeroDatasetParameters_ReturnsEmptyList()
{
//Arrange
string sourceCode = "Return Sum(Datasets.Testing1)";
//Act
IEnumerable <string> aggregateParameters = SourceCodeHelpers.GetCalculationAggregateFunctionParameters(sourceCode);
//Assert
aggregateParameters
.Should()
.BeEmpty();
}
public void CommentOutCode_GivenSourceCodeWithNoReasonForCommentingAndNoExceptionMessageWithHash_CommentsCode()
{
//Arrange
string originalSourceCode = CreateSourceCode();
string commentedSourceCode = CreateCommentedSourceCodeWithHash();
//Act
string result = SourceCodeHelpers.CommentOutCode(originalSourceCode, commentSymbol: "#");
//Assert
result
.Should()
.Be(commentedSourceCode);
}
public void CodeContainsFullyQualifiedDatasetFieldIdentifier_GivenListOfIdentifiersAndSourceCodeDoesContainAnyFromListInsideAnAggregate_ReturnsTrue()
{
//Arrange
string sourceCode = "return Sum(Datasets.Ds1.TestField1) + Max(Datasets.Ds1.TestField2)";
IEnumerable <string> datasetFieldIdentifiers = new[] { "Datasets.Ds1.TestField2" };
//Act
bool result = SourceCodeHelpers.CodeContainsFullyQualifiedDatasetFieldIdentifier(sourceCode, datasetFieldIdentifiers);
//Assert
result
.Should()
.BeTrue();
}
public void CodeContainsFullyQualifiedDatasetFieldIdentifier_GivenListOfIdentifiersAndSourceCodeDoesContainAnyFromListInAnyCase_ReturnsTrue()
{
//Arrange
string sourceCode = "return Datasets.Ds1.TestField1 + Datasets.Ds1.TestField2";
IEnumerable <string> datasetFieldIdentifiers = new[] { "DAtASetS.DS1.testfield2" };
//Act
bool result = SourceCodeHelpers.CodeContainsFullyQualifiedDatasetFieldIdentifier(sourceCode, datasetFieldIdentifiers);
//Assert
result
.Should()
.BeTrue();
}
public void CodeContainsDatasetFieldIdentifier_GivenListOfIdentifiersAndSourceCodeDoesContainAnyFromListButWithExtraCharacters_ReturnsFalse1()
{
//Arrange
string sourceCode = "return Datasets.Ds1.TestField1 + Datasets.Ds1.TestField2";
IEnumerable <string> datasetFieldIdentifiers = new[] { "Datasets.Ds1.TestField23" };
//Act
bool result = SourceCodeHelpers.CodeContainsFullyQualifiedDatasetFieldIdentifier(sourceCode, datasetFieldIdentifiers);
//Assert
result
.Should()
.BeFalse();
}
private async Task <Build> CheckDatasetValidAggregations(PreviewRequest previewRequest)
{
Build build = null;
IEnumerable <string> aggregateParameters = SourceCodeHelpers.GetDatasetAggregateFunctionParameters(previewRequest.SourceCode);
if (aggregateParameters.IsNullOrEmpty())
{
return(build);
}
string cacheKey = $"{CacheKeys.DatasetRelationshipFieldsForSpecification}{previewRequest.SpecificationId}";
IEnumerable <DatasetSchemaRelationshipModel> datasetSchemaRelationshipModels = Enumerable.Empty <DatasetSchemaRelationshipModel>();
datasetSchemaRelationshipModels = await _cacheProvider.GetAsync <List <DatasetSchemaRelationshipModel> >(cacheKey);
if (datasetSchemaRelationshipModels.IsNullOrEmpty())
{
datasetSchemaRelationshipModels = await _datasetRepository.GetDatasetSchemaRelationshipModelsForSpecificationId(previewRequest.SpecificationId);
await _cacheProvider.SetAsync <List <DatasetSchemaRelationshipModel> >(cacheKey, datasetSchemaRelationshipModels.ToList());
}
HashSet <string> compilerErrors = new HashSet <string>();
IEnumerable <string> datasetAggregationFields = datasetSchemaRelationshipModels?.SelectMany(m => m.Fields?.Where(f => f.IsAggregable).Select(f => f.FullyQualifiedSourceName));
foreach (string aggregateParameter in aggregateParameters)
{
if (datasetAggregationFields.IsNullOrEmpty() || !datasetAggregationFields.Any(m => string.Equals(m.Trim(), aggregateParameter.Trim(), StringComparison.CurrentCultureIgnoreCase)))
{
compilerErrors.Add($"{aggregateParameter} is not an aggregable field");
}
}
if (compilerErrors.Any())
{
build = new Build
{
CompilerMessages = compilerErrors.Select(m => new CompilerMessage {
Message = m, Severity = Severity.Error
}).ToList()
};
}
return(build);
}
public void GetReferencedCalculations_GivenListOfCalcsAndSourceCodeDoesNotContainCalc_ReturnsEmptyList()
{
//Arrange
string sourceCode = " calc4() + calc5()";
IEnumerable <string> calcNames = new[] { "calc1", "calc2", "calc3" };
//Act
IEnumerable <string> results = SourceCodeHelpers.GetReferencedCalculations(calcNames, sourceCode);
//Assert
results
.Any()
.Should()
.BeFalse();
}
public void HasCalculationAggregateFunctionParameters_GivenSourceDoesNotContainAggregateParameterButCalcNameStartsWithAggregateFunction_ReturnsFalse()
{
//Arrange
IEnumerable <string> sourceCodes = new[]
{
"Return SumTest()"
};
//Act
bool result = SourceCodeHelpers.HasCalculationAggregateFunctionParameters(sourceCodes);
//Assert
result
.Should()
.BeFalse();
}
public void HasCalculationAggregateFunctionParameters_GivenSourceDoesNotContainAggregateParameterButCalcNameContainsAggregateFunctionCaseInsensitive_ReturnsFalse()
{
//Arrange
IEnumerable <string> sourceCodes = new[]
{
"Return TESTSUM()"
};
//Act
bool result = SourceCodeHelpers.HasCalculationAggregateFunctionParameters(sourceCodes);
//Assert
result
.Should()
.BeFalse();
}
public void GetReferencedCalculations_GivenAnEmptySourceCodeString_ReturnsEmptyList()
{
//Arrange
string sourceCode = "";
IEnumerable <string> calcNames = new[] { "calc1", "calc2", "calc3" };
//Act
IEnumerable <string> results = SourceCodeHelpers.GetReferencedCalculations(calcNames, sourceCode);
//Assert
results
.Any()
.Should()
.BeFalse();
}
public void CodeContainsFullyQualifiedDatasetFieldIdentifier_GivenListOfIdentifiersAndSourceCodeDoesContainAnyFromListAsGherkinWhithMultiSpaces_ReturnsTrue()
{
//Arrange
string sourceCode = "Given the dataset 'ADULT Advanced Learner Loans Bursary Changed The Name Again' field 'Bursary_CurrentYear_PostPMPChange1' is equal to '101" + Environment.NewLine +
"Then the result for 'AB Calc 0804-001' is greater than '10'";
IEnumerable <string> datasetFieldIdentifiers = new[] { "ADULT Advanced Learner Loans Bursary Changed The Name Again field Bursary_CurrentYear_PostPMPChange1" };
//Act
bool result = SourceCodeHelpers.CodeContainsFullyQualifiedDatasetFieldIdentifier(sourceCode.RemoveAllQuotes(), datasetFieldIdentifiers);
//Assert
result
.Should()
.BeTrue();
}
public void HasCalculationAggregateFunctionParameters_GivenSourceDoesNotContainAggregateParameter_ReturnsFalse()
{
//Arrange
IEnumerable <string> sourceCodes = new[]
{
"Return Sum(Datasets.Testing1",
"Return Sum(Datasets.Testing2)"
};
//Act
bool result = SourceCodeHelpers.HasCalculationAggregateFunctionParameters(sourceCodes);
//Assert
result
.Should()
.BeFalse();
}
public void CommentOutCode_GivenSourceCodeAlreadyCommented_DoesntCommentCode()
{
//Arrange
const string systemCommented = "'System Commented\r\n\r\n";
string originalSourceCode = CreateCommentedSourceCode();
string commentedSourceCode = CreateCommentedSourceCode();
//Act
string result = SourceCodeHelpers.CommentOutCode(originalSourceCode);
//Assert
result
.Should()
.Be($"{systemCommented}{commentedSourceCode}");
}
public void CommentOutCode_GivenSourceCodeWithReasonForCommentingAndNoExceptionMessage_CommentsCode()
{
//Arrange
string reason = "Just for test the unit tests";
string originalSourceCode = CreateSourceCode();
string commentedSourceCode = CreateCommentedSourceCode();
//Act
string result = SourceCodeHelpers.CommentOutCode(originalSourceCode, reason);
//Assert
result
.Should()
.Be($"'System Commented\r\n\r\n'{reason}\r\n\r\n\r\n{commentedSourceCode}");
}
public void CommentOutCode_GivenSourceCodeWithNoReasonForCommentingAndNoExceptionMessage_CommentsCode()
{
//Arrange
const string systemCommented = "'System Commented\r\n\r\n";
string originalSourceCode = CreateSourceCode();
string commentedSourceCode = CreateCommentedSourceCode();
//Act
string result = SourceCodeHelpers.CommentOutCode(originalSourceCode);
//Assert
result
.Should()
.Be($"{systemCommented}{commentedSourceCode}");
}
public void HasCalculationAggregateFunctionParameters_GivenSourceContainsAggregateParameterCaseInsensitiveAndSpaces_ReturnsTrue()
{
//Arrange
IEnumerable <string> sourceCodes = new[]
{
//"Return SuM (Datasets.Testing1)",
"Return mIn(Datasets.Testing1) + AVG(Calc1) + SuM (Calc2)"
};
//Act
bool result = SourceCodeHelpers.HasCalculationAggregateFunctionParameters(sourceCodes);
//Assert
result
.Should()
.BeTrue();
}
public void GetCalculationAggregateFunctionParameters_GivenSourceHasOneCalculationParameter_ReturnsOneParameter()
{
//Arrange
string sourceCode = "Return Sum(Datasets.Testing1) + Sum(Calc1)";
//Act
IEnumerable <string> aggregateParameters = SourceCodeHelpers.GetCalculationAggregateFunctionParameters(sourceCode);
//Assert
aggregateParameters
.Count()
.Should()
.Be(1);
aggregateParameters
.First()
.Should()
.Be("Calc1");
}
public void IsCalcfReferencedInAnAggregate_GivenCalcIsNotReferencedInAnAggregate_ReturnsFalse()
{
//Arrange
Dictionary <string, string> functions = new Dictionary <string, string>
{
{ "Calc1", "Return Calc2()" },
{ "Calc2", "Return 1" },
{ "Calc3", "Return Sum(Calc2) + 5" },
{ "Calc4", "Return Sum(Calc1) + Avg(Calc3) + 5" }
};
string calcToCheck = "Calc4";
//Act
bool result = SourceCodeHelpers.IsCalcReferencedInAnAggregate(functions, calcToCheck);
//Assert
result
.Should()
.BeFalse();
}
public async Task ResetScenarioForFieldDefinitionChanges(IEnumerable <DatasetSpecificationRelationshipViewModel> relationships, string specificationId, IEnumerable <string> currentFieldDefinitionNames)
{
Guard.ArgumentNotNull(relationships, nameof(relationships));
Guard.IsNullOrWhiteSpace(specificationId, nameof(specificationId));
Guard.ArgumentNotNull(currentFieldDefinitionNames, nameof(currentFieldDefinitionNames));
IEnumerable <TestScenario> scenarios = await _scenariosRepositoryPolicy.ExecuteAsync(() => _scenariosRepository.GetTestScenariosBySpecificationId(specificationId));
if (scenarios.IsNullOrEmpty())
{
_logger.Information($"No scenarios found for specification id '{specificationId}'");
return;
}
List <string> fieldIdentifiers = new List <string>();
foreach (DatasetSpecificationRelationshipViewModel datasetSpecificationRelationshipViewModel in relationships)
{
fieldIdentifiers.AddRange(currentFieldDefinitionNames.Select(m => $"dataset {datasetSpecificationRelationshipViewModel.Name} field {VisualBasicTypeGenerator.GenerateIdentifier(m)}"));
}
IEnumerable <TestScenario> scenariosToUpdate = scenarios.Where(m => SourceCodeHelpers.CodeContainsFullyQualifiedDatasetFieldIdentifier(m.Current.Gherkin.RemoveAllQuotes(), fieldIdentifiers));
if (scenariosToUpdate.IsNullOrEmpty())
{
_logger.Information($"No test scenarios required resetting for specification id '{specificationId}'");
return;
}
const string reasonForCommenting = "The dataset definition referenced by this scenario/spec has been updated and subsequently the code has been commented out";
foreach (TestScenario scenario in scenariosToUpdate)
{
string gherkin = scenario.Current.Gherkin;
string updatedGherkin = SourceCodeHelpers.CommentOutCode(gherkin, reasonForCommenting, commentSymbol: "#");
await SaveVersion(scenario, updatedGherkin);
}
}
public IEnumerable <CalculationRelationship> DetermineRelationshipsBetweenCalculations(IEnumerable <Calculation> calculations)
{
Guard.IsNotEmpty(calculations, nameof(calculations));
Dictionary <string, string> calculationIdsBySourceCodeName = calculations
.ToDictionary(_ => $"{_.Namespace}.{_.Current.SourceCodeName}", _ => _.Id);
string[] calculationNames = calculations.Select(_ => $"{_.Namespace}.{_.Current.SourceCodeName}")
.ToArray();
return(calculations.SelectMany(_ =>
{
IEnumerable <string> relatedCalculationNames = SourceCodeHelpers.GetReferencedCalculations(calculationNames, _.Current.SourceCode);
return relatedCalculationNames.Select(rel => new CalculationRelationship
{
CalculationOneId = _.Id,
CalculationTwoId = calculationIdsBySourceCodeName.TryGetValue(rel, out string twoId) ?
twoId :
throw new InvalidOperationException($"Could not locate a calculation id for sourceCodeName {rel}")
});
}));
public void CommentOutCode_GivenSourceCodeWithReasonForCommentingAndExceptionMessage_CommentsCode()
{
//Arrange
const string reason = "Just for test the unit tests";
const string exceptionMessage = "Exception thrown, just for test the unit tests";
const string exceptionType = "DatasetReferenceChangeException";
string originalSourceCode = CreateSourceCode();
string commentedSourceCode = CreateCommentedSourceCode();
string expected = $"'System Commented\r\n\r\n'{reason}\r\n\r\n\r\nThrow New {exceptionType}(\"{exceptionMessage}\")\r\n\r\n\r\n{commentedSourceCode}";
//Act
string result = SourceCodeHelpers.CommentOutCode(originalSourceCode, reason, exceptionMessage, exceptionType);
//Assert
result
.Should()
.Be(expected);
}
public void CheckSourceForExistingCalculationAggregates_GivenCalcIsReferencedInAnAggregateWithDeepNestingAndFurtherLevelsAndIncludeWhitespaceInAggregateCall_ReturnsTrue()
{
//Arrange
Dictionary <string, string> functions = new Dictionary <string, string>
{
{ "Calc1", "Return Calc2() + Min( Calc6)" },
{ "Calc2", "Return 1" },
{ "Calc3", "Return Sum(Calc2) + 5" },
{ "Calc4", "Return Sum(Calc2) + Avg(Calc2) + Min( Calc3)" },
{ "Calc5", "Return Calc1()" },
{ "Calc6", "Return Calc5()" },
{ "Calc7", "Return Calc6()" }
};
string calcToCheck = "Return Sum(Calc2) + Avg(Calc2) + Sum(Calc3)";
//Act
bool result = SourceCodeHelpers.CheckSourceForExistingCalculationAggregates(functions, calcToCheck);
//Assert
result
.Should()
.BeTrue();
}
public void CheckSourceForExistingCalculationAggregates_GivenCalcContainsCalcReferenceThatIncludesAvgInTheNameButNoOtherAggregateRefrences_ReturnsFalse()
{
//Arrange
Dictionary <string, string> functions = new Dictionary <string, string>
{
{ "Calc1", "Return Calc2() + Min( Calc6)" },
{ "Calc2", "Return 1" },
{ "Calc3", "Return Sum(Calc2) + 5" },
{ "Calc4", "Return Sum(Calc2) + Avg(Calc2) + Min( Calc3)" },
{ "Calc5", "Return CalcAvg1()" },
{ "Calc6", "Return Calc5()" },
{ "Calc7", "Return Calc6()" }
};
string calcToCheck = "Return CalcAvg1() = Calc6()";
//Act
bool result = SourceCodeHelpers.CheckSourceForExistingCalculationAggregates(functions, calcToCheck);
//Assert
result
.Should()
.BeFalse();
}
public void GetCalculationAggregateFunctionParameters_GivenSourceHasTwoCalculationParameters_ReturnsTwoParameters()
{
//Arrange
string sourceCode = "Return Sum(Datasets.Testing1) + Avg(Calc1) + Sum(Calc2)";
//Act
IEnumerable <string> aggregateParameters = SourceCodeHelpers.GetCalculationAggregateFunctionParameters(sourceCode);
//Assert
aggregateParameters
.Count()
.Should()
.Be(2);
aggregateParameters
.ElementAt(0)
.Should()
.Be("Calc1");
aggregateParameters
.ElementAt(1)
.Should()
.Be("Calc2");
}
private async Task ProcessFieldChanges(string datasetDefinitionId, IEnumerable <FieldDefinitionChanges> fieldChanges, IEnumerable <string> relationshipSpecificationIds)
{
Guard.IsNullOrWhiteSpace(datasetDefinitionId, nameof(datasetDefinitionId));
Guard.ArgumentNotNull(fieldChanges, nameof(fieldChanges));
Guard.ArgumentNotNull(relationshipSpecificationIds, nameof(relationshipSpecificationIds));
IEnumerable <IGrouping <string, FieldDefinitionChanges> > groupedFieldChanges = fieldChanges.GroupBy(f => f.FieldDefinition.Id);
IList <FieldDefinitionChanges> fieldDefinitionChanges = new List <FieldDefinitionChanges>();
bool shouldResetCalculation = false;
foreach (IGrouping <string, FieldDefinitionChanges> grouping in groupedFieldChanges)
{
FieldDefinitionChanges fieldDefinitionChange = grouping.FirstOrDefault(m => m.ChangeTypes.Any(
c => c == FieldDefinitionChangeType.FieldName) || m.RequiresRemap);
if (fieldDefinitionChange != null)
{
fieldDefinitionChanges.Add(fieldDefinitionChange);
}
shouldResetCalculation = true;
}
if (!shouldResetCalculation)
{
return;
}
foreach (string specificationId in relationshipSpecificationIds)
{
IEnumerable <DatasetSpecificationRelationshipViewModel> relationships = await _datasetRepositoryPolicy.ExecuteAsync(() => _datasetRepository.GetCurrentRelationshipsBySpecificationIdAndDatasetDefinitionId(specificationId, datasetDefinitionId));
if (relationships.IsNullOrEmpty())
{
throw new RetriableException($"No relationships found for specificationId '{specificationId}' and dataset definition id '{datasetDefinitionId}'");
}
IEnumerable <Calculation> calculations = (await _calculationsRepositoryPolicy.ExecuteAsync(() => _calculationsRepository.GetCalculationsBySpecificationId(specificationId))).ToList();
IEnumerable <string> aggregateParameters = calculations.SelectMany(m => SourceCodeHelpers.GetDatasetAggregateFunctionParameters(m.Current.SourceCode));
HashSet <string> fieldNames = new HashSet <string>();
foreach (FieldDefinitionChanges changes in fieldDefinitionChanges)
{
//Check if only aggregable changes
if (!changes.ChangeTypes.Contains(FieldDefinitionChangeType.FieldType) && !changes.ChangeTypes.Contains(FieldDefinitionChangeType.FieldName))
{
foreach (DatasetSpecificationRelationshipViewModel datasetSpecificationRelationshipViewModel in relationships)
{
if (aggregateParameters.Contains($"Datasets.{VisualBasicTypeGenerator.GenerateIdentifier(datasetSpecificationRelationshipViewModel.Name)}.{VisualBasicTypeGenerator.GenerateIdentifier(changes.ExistingFieldDefinition.Name)}"))
{
fieldNames.Add(changes.ExistingFieldDefinition.Name);
}
}
}
else
{
fieldNames.Add(changes.ExistingFieldDefinition.Name);
}
}
if (fieldNames.Any())
{
await _calculationService.ResetCalculationForFieldDefinitionChanges(relationships, specificationId, fieldNames);
}
}
}
private async Task <IActionResult> GenerateAndCompile(BuildProject buildProject,
Calculation calculationToPreview,
IEnumerable <Calculation> calculations,
CompilerOptions compilerOptions,
PreviewRequest previewRequest)
{
PreviewProviderCalculationResponseModel previewProviderCalculation = null;
Build compilerOutput = _sourceCodeService.Compile(buildProject, calculations, compilerOptions);
if (compilerOutput.SourceFiles != null)
{
await _sourceCodeService.SaveSourceFiles(compilerOutput.SourceFiles, buildProject.SpecificationId, SourceCodeType.Preview);
}
if (compilerOutput.Success)
{
_logger.Information($"Build compiled successfully for calculation id {calculationToPreview.Id}");
string calculationIdentifier = $"{_typeIdentifierGenerator.GenerateIdentifier(calculationToPreview.Namespace)}.{_typeIdentifierGenerator.GenerateIdentifier(calculationToPreview.Name)}";
IDictionary <string, string> functions = _sourceCodeService.GetCalculationFunctions(compilerOutput.SourceFiles);
IDictionary <string, string> calculationIdentifierMap = calculations
.Select(_ => new
{
Identifier = $"{_typeIdentifierGenerator.GenerateIdentifier(_.Namespace)}.{_typeIdentifierGenerator.GenerateIdentifier(_.Name)}",
CalcName = _.Name
})
.ToDictionary(d => d.Identifier, d => d.CalcName);
if (!functions.ContainsKey(calculationIdentifier))
{
compilerOutput.Success = false;
compilerOutput.CompilerMessages.Add(new CompilerMessage {
Message = $"{calculationIdentifier} is not an aggregable field", Severity = Severity.Error
});
}
else
{
if (previewRequest != null)
{
if (!SourceCodeHelpers.HasReturn(previewRequest.SourceCode))
{
compilerOutput.Success = false;
compilerOutput.CompilerMessages.Add(new CompilerMessage {
Message = $"{calculationIdentifier} must have a return statement so that a calculation result will be returned", Severity = Severity.Error
});
}
else
{
IEnumerable <string> aggregateParameters = SourceCodeHelpers.GetCalculationAggregateFunctionParameters(previewRequest.SourceCode);
bool continueChecking = true;
if (!aggregateParameters.IsNullOrEmpty())
{
foreach (string aggregateParameter in aggregateParameters)
{
if (!functions.ContainsKey(aggregateParameter))
{
compilerOutput.Success = false;
compilerOutput.CompilerMessages.Add(new CompilerMessage {
Message = $"{aggregateParameter} is not an aggregable field", Severity = Severity.Error
});
continueChecking = false;
break;
}
if (calculationIdentifierMap.ContainsKey(aggregateParameter))
{
Calculation calculation = calculations.SingleOrDefault(_ => _.Name == calculationIdentifierMap[aggregateParameter]);
if (calculation.Current.DataType != CalculationDataType.Decimal)
{
compilerOutput.Success = false;
compilerOutput.CompilerMessages.Add(new CompilerMessage
{
Message =
$"Only decimal fields can be used on aggregation. {aggregateParameter} has data type of {calculation.Current.DataType}",
Severity = Severity.Error
});
continueChecking = false;
break;
}
}
}
if (continueChecking)
{
if (SourceCodeHelpers.IsCalcReferencedInAnAggregate(functions, calculationIdentifier))
{
compilerOutput.Success = false;
compilerOutput.CompilerMessages.Add(new CompilerMessage {
Message = $"{calculationIdentifier} is already referenced in an aggregation that would cause nesting", Severity = Severity.Error
});
}
else if (SourceCodeHelpers.CheckSourceForExistingCalculationAggregates(functions, previewRequest.SourceCode))
{
compilerOutput.Success = false;
compilerOutput.CompilerMessages.Add(new CompilerMessage {
Message = $"{calculationIdentifier} cannot reference another calc that is being aggregated", Severity = Severity.Error
});
}
}
}
}
}
}
}
else
{
_logger.Information($"Build did not compile successfully for calculation id {calculationToPreview.Id}");
}
LogMessages(compilerOutput, buildProject, calculationToPreview);
if (!string.IsNullOrEmpty(previewRequest.ProviderId))
{
CalculationSummaryModel calculationSummaryModel = calculationToPreview.ToSummaryModel();
CalcEngineModels.CalculationSummaryModel model = _mapper.Map <CalcEngineModels.CalculationSummaryModel>(calculationSummaryModel);
CalcEngineModels.PreviewCalculationRequest previewCalculationRequest = new CalcEngineModels.PreviewCalculationRequest
{
AssemblyContent = compilerOutput.Assembly,
PreviewCalculationSummaryModel = model
};
ApiResponse <CalcEngineProviderResult> previewCalcResultApiResponse =
await _calcEngineApiClientPolicy.ExecuteAsync(
() => _calcEngineApiClient.PreviewCalculationResults(
previewRequest.SpecificationId,
previewRequest.ProviderId,
previewCalculationRequest));
if (previewCalcResultApiResponse.StatusCode.IsSuccess())
{
CalcEngineProviderResult calcEngineProviderResult = previewCalcResultApiResponse.Content;
previewProviderCalculation = new PreviewProviderCalculationResponseModel
{
ProviderName = calcEngineProviderResult.Provider.Name,
CalculationResult = _mapper.Map <CalculationResult>(
calcEngineProviderResult.CalculationResults.SingleOrDefault(_ => _.Calculation?.Id == calculationToPreview.Id)),
};
}
}
return(new OkObjectResult(new PreviewResponse
{
Calculation = calculationToPreview.ToResponseModel(),
CompilerOutput = compilerOutput,
PreviewProviderCalculation = previewProviderCalculation
}));
}
public async Task <IActionResult> SaveVersion(HttpRequest request)
{
string json = await request.GetRawBodyStringAsync();
CreateNewTestScenarioVersion scenarioVersion = JsonConvert.DeserializeObject <CreateNewTestScenarioVersion>(json);
if (scenarioVersion == null)
{
_logger.Error("A null scenario version was provided");
return(new BadRequestObjectResult("Null or empty calculation Id provided"));
}
BadRequestObjectResult validationResult = (await _createNewTestScenarioVersionValidator.ValidateAsync(scenarioVersion)).PopulateModelState();
if (validationResult != null)
{
return(validationResult);
}
TestScenario testScenario = null;
if (!string.IsNullOrEmpty(scenarioVersion.Id))
{
testScenario = await _scenariosRepository.GetTestScenarioById(scenarioVersion.Id);
}
bool saveAsVersion = true;
SpecificationSummary specification = await _specificationsRepository.GetSpecificationSummaryById(scenarioVersion.SpecificationId);
if (specification == null)
{
_logger.Error($"Unable to find a specification for specification id : {scenarioVersion.SpecificationId}");
return(new StatusCodeResult(412));
}
Reference user = request.GetUserOrDefault();
if (testScenario == null)
{
string Id = Guid.NewGuid().ToString();
testScenario = new TestScenario
{
Id = Id,
SpecificationId = specification.Id,
Name = scenarioVersion.Name,
Current = new TestScenarioVersion
{
Date = DateTimeOffset.Now.ToLocalTime(),
TestScenarioId = Id,
PublishStatus = PublishStatus.Draft,
Version = 1,
Author = user,
Gherkin = scenarioVersion.Scenario,
Description = scenarioVersion.Description,
FundingPeriodId = specification.FundingPeriod.Id,
FundingStreamIds = specification.FundingStreams.Select(s => s.Id).ToArraySafe(),
}
};
}
else
{
testScenario.Name = scenarioVersion.Name;
saveAsVersion = !string.Equals(scenarioVersion.Scenario, testScenario.Current.Gherkin) ||
scenarioVersion.Description != testScenario.Current.Description;
TestScenarioVersion newVersion = testScenario.Current.Clone() as TestScenarioVersion;
if (saveAsVersion == true)
{
newVersion.Author = user;
newVersion.Gherkin = scenarioVersion.Scenario;
newVersion.Description = scenarioVersion.Description;
newVersion.FundingStreamIds = specification.FundingStreams.Select(s => s.Id).ToArraySafe();
newVersion.FundingPeriodId = specification.FundingPeriod.Id;
newVersion = await _versionRepository.CreateVersion(newVersion, testScenario.Current);
testScenario.Current = newVersion;
}
}
HttpStatusCode statusCode = await _scenariosRepository.SaveTestScenario(testScenario);
if (!statusCode.IsSuccess())
{
_logger.Error($"Failed to save test scenario with status code: {statusCode.ToString()}");
return(new StatusCodeResult((int)statusCode));
}
await _versionRepository.SaveVersion(testScenario.Current);
ScenarioIndex scenarioIndex = CreateScenarioIndexFromScenario(testScenario, specification);
await _searchRepository.Index(new List <ScenarioIndex> {
scenarioIndex
});
await _cacheProvider.RemoveAsync <List <TestScenario> >($"{CacheKeys.TestScenarios}{testScenario.SpecificationId}");
await _cacheProvider.RemoveAsync <GherkinParseResult>($"{CacheKeys.GherkinParseResult}{testScenario.Id}");
IEnumerable <Models.Calcs.CalculationCurrentVersion> calculations = await _calcsRepositoryPolicy.ExecuteAsync(() => _calcsRepository.GetCurrentCalculationsBySpecificationId(specification.Id));
if (calculations.IsNullOrEmpty())
{
_logger.Information($"No calculations found to test for specification id: '{specification.Id}'");
}
else
{
string correlationId = request.GetCorrelationId();
try
{
Trigger trigger = new Trigger
{
EntityId = testScenario.Id,
EntityType = nameof(TestScenario),
Message = $"Saving test scenario: '{testScenario.Id}'"
};
bool generateCalculationAggregations = SourceCodeHelpers.HasCalculationAggregateFunctionParameters(calculations.Select(m => m.SourceCode));
Job job = await SendInstructAllocationsToJobService(specification.Id, user, trigger, correlationId, generateCalculationAggregations);
_logger.Information($"New job of type '{job.JobDefinitionId}' created with id: '{job.Id}'");
}
catch (Exception ex)
{
_logger.Error(ex, $"Failed to create job of type '{JobConstants.DefinitionNames.CreateInstructAllocationJob}' on specification '{specification.Id}'");
return(new InternalServerErrorResult($"An error occurred attempting to execute calculations prior to running tests on specification '{specification.Id}'"));
}
}
CurrentTestScenario testScenarioResult = await _scenariosRepository.GetCurrentTestScenarioById(testScenario.Id);
return(new OkObjectResult(testScenarioResult));
}
public async Task ProcessDataset(Message message)
{
Guard.ArgumentNotNull(message, nameof(message));
IDictionary <string, object> properties = message.UserProperties;
Dataset dataset = message.GetPayloadAsInstanceOf <Dataset>();
string jobId = message.UserProperties["jobId"].ToString();
await UpdateJobStatus(jobId, null, 0);
if (dataset == null)
{
_logger.Error("A null dataset was provided to ProcessData");
await UpdateJobStatus(jobId, false, 100, "Failed to Process - null dataset provided");
return;
}
if (!message.UserProperties.ContainsKey("specification-id"))
{
_logger.Error("Specification Id key is missing in ProcessDataset message properties");
await UpdateJobStatus(jobId, false, 100, "Failed to Process - specification id not provided");
return;
}
string specificationId = message.UserProperties["specification-id"].ToString();
if (string.IsNullOrWhiteSpace(specificationId))
{
_logger.Error("A null or empty specification id was provided to ProcessData");
await UpdateJobStatus(jobId, false, 100, "Failed to Process - specification if is null or empty");
return;
}
if (!message.UserProperties.ContainsKey("relationship-id"))
{
_logger.Error("Relationship Id key is missing in ProcessDataset message properties");
await UpdateJobStatus(jobId, false, 100, "Failed to Process - relationship id not provided");
return;
}
string relationshipId = message.UserProperties["relationship-id"].ToString();
if (string.IsNullOrWhiteSpace(relationshipId))
{
_logger.Error("A null or empty relationship id was provided to ProcessDataset");
await UpdateJobStatus(jobId, false, 100, "Failed to Process - relationship id is null or empty");
return;
}
DefinitionSpecificationRelationship relationship = await _datasetRepository.GetDefinitionSpecificationRelationshipById(relationshipId);
if (relationship == null)
{
_logger.Error($"Relationship not found for relationship id: {relationshipId}");
await UpdateJobStatus(jobId, false, 100, "Failed to Process - relationship not found");
return;
}
BuildProject buildProject = null;
Reference user = message.GetUserDetails();
try
{
buildProject = await ProcessDataset(dataset, specificationId, relationshipId, relationship.DatasetVersion.Version, user);
await UpdateJobStatus(jobId, true, 100, "Processed Dataset");
}
catch (NonRetriableException argEx)
{
// This type of exception is not retriable so fail
_logger.Error(argEx, $"Failed to run ProcessDataset with exception: {argEx.Message} for relationship ID '{relationshipId}'");
await UpdateJobStatus(jobId, false, 100, $"Failed to run Process - {argEx.Message}");
return;
}
catch (Exception exception)
{
// Unknown exception occurred so allow it to be retried
_logger.Error(exception, $"Failed to run ProcessDataset with exception: {exception.Message} for relationship ID '{relationshipId}'");
throw;
}
if (buildProject != null && !buildProject.DatasetRelationships.IsNullOrEmpty() && buildProject.DatasetRelationships.Any(m => m.DefinesScope))
{
string userId = user != null ? user.Id : "";
string userName = user != null ? user.Name : "";
try
{
HttpStatusCode statusCode = await _calcsRepository.CompileAndSaveAssembly(specificationId);
if (!statusCode.IsSuccess())
{
string errorMessage = $"Failed to compile and save assembly for specification id '{specificationId}' with status code '{statusCode}'";
_logger.Error(errorMessage);
throw new NonRetriableException(errorMessage);
}
Trigger trigger = new Trigger
{
EntityId = relationshipId,
EntityType = nameof(DefinitionSpecificationRelationship),
Message = $"Processed dataset relationship: '{relationshipId}' for specification: '{specificationId}'"
};
string correlationId = message.GetCorrelationId();
IEnumerable <CalculationCurrentVersion> allCalculations = await _calcsRepository.GetCurrentCalculationsBySpecificationId(specificationId);
bool generateCalculationAggregations = allCalculations.IsNullOrEmpty() ? false : SourceCodeHelpers.HasCalculationAggregateFunctionParameters(allCalculations.Select(m => m.SourceCode));
await SendInstructAllocationsToJobService($"{CacheKeys.ScopedProviderSummariesPrefix}{specificationId}", specificationId, userId, userName, trigger, correlationId, generateCalculationAggregations);
}
catch (Exception ex)
{
_logger.Error(ex, $"Failed to create job of type '{JobConstants.DefinitionNames.CreateInstructAllocationJob}' on specification '{specificationId}'");
throw new Exception($"Failed to create job of type '{JobConstants.DefinitionNames.CreateInstructAllocationJob}' on specification '{specificationId}'", ex);
}
}
}
