public void Execute_GivenLogicResultReturnsFalse_ReturnsParseResulAndAbort()
{
//Arrange
ProviderResult providerResult = new ProviderResult
{
Provider = new ProviderSummary
{
Id = "p1"
}
};
IEnumerable <ProviderSourceDataset> datasets = new List <ProviderSourceDataset>();
AndProviderIs andProviderIs = new AndProviderIs {
ProviderId = "p2", Operator = ComparisonOperator.EqualTo
};
//Act
GherkinParseResult parseResult = andProviderIs.Execute(providerResult, datasets);
//Assert
parseResult
.Abort
.Should()
.BeTrue();
}
private async Task <GherkinParseResult> GetGherkinParseResult(TestScenario testScenario, BuildProject buildProject)
{
Guard.ArgumentNotNull(testScenario, nameof(testScenario));
Guard.ArgumentNotNull(buildProject, nameof(buildProject));
string cacheKey = $"{CacheKeys.GherkinParseResult}{testScenario.Id}";
JsonSerializerSettings jsonSerializerSettings = new JsonSerializerSettings()
{
TypeNameHandling = TypeNameHandling.All
};
GherkinParseResult gherkinParseResult = await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.GetAsync <GherkinParseResult>(cacheKey, jsonSerializerSettings));
if (gherkinParseResult == null)
{
gherkinParseResult = await _parser.Parse(testScenario.SpecificationId, testScenario.Current.Gherkin, buildProject);
if (gherkinParseResult != null && !gherkinParseResult.StepActions.IsNullOrEmpty())
{
await _cacheProviderPolicy.ExecuteAsync(() => _cacheProvider.SetAsync <GherkinParseResult>(cacheKey, gherkinParseResult, TimeSpan.FromHours(24), true, jsonSerializerSettings));
}
}
return(gherkinParseResult);
}
public async Task <IEnumerable <ScenarioResult> > Execute(ProviderResult providerResult, IEnumerable <ProviderSourceDataset> datasets,
IEnumerable <TestScenario> testScenarios, BuildProject buildProject)
{
Guard.ArgumentNotNull(providerResult, nameof(providerResult));
Guard.ArgumentNotNull(datasets, nameof(datasets));
Guard.ArgumentNotNull(testScenarios, nameof(testScenarios));
Guard.ArgumentNotNull(buildProject, nameof(buildProject));
IList <ScenarioResult> scenarioResults = new List <ScenarioResult>();
foreach (TestScenario scenario in testScenarios)
{
ScenarioResult scenarioResult = new ScenarioResult
{
Scenario = new Reference(scenario.Id, scenario.Name)
};
GherkinParseResult parseResult = await GetGherkinParseResult(scenario, buildProject);
if (parseResult != null && !parseResult.StepActions.IsNullOrEmpty())
{
scenarioResult.TotalSteps = parseResult.StepActions.Count;
scenarioResult.StepsExecuted = 0;
foreach (IStepAction action in parseResult.StepActions)
{
GherkinParseResult result = action.Execute(providerResult, datasets);
if (result.Abort)
{
break;
}
if (!result.Dependencies.IsNullOrEmpty())
{
foreach (Dependency resultDependency in result.Dependencies)
{
if (!scenarioResult.Dependencies.Contains(resultDependency))
{
scenarioResult.Dependencies.Add(resultDependency);
}
}
}
if (result.HasErrors)
{
scenarioResult.Errors.AddRange(result.Errors);
}
scenarioResult.StepsExecuted++;
}
}
scenarioResults.Add(scenarioResult);
}
return(scenarioResults);
}
public Task Parse(Step step, string stepExpression, GherkinParseResult parseResult, BuildProject buildProject)
{
if (buildProject.Build.Assembly.IsNullOrEmpty())
{
parseResult.AddError("No valid assembly to test", step.Location.Line, step.Location.Column);
}
else
{
byte[] assembly = buildProject.Build.Assembly;
if (assembly == null)
{
parseResult.AddError("No calculations available for this test", step.Location.Line, step.Location.Column);
}
else
{
string[] matches = Regex.Split(step.Text, stepExpression, RegexOptions.IgnoreCase);
string calcName = matches[7];
string comparison = matches[9];
string value = matches[11];
MethodInformation calculation = FindCalculationMethod(assembly, calcName);
if (calculation == null)
{
parseResult.AddError($"Calculation: '{calcName}' was not found to test", step.Location.Line, step.Location.Column);
}
if (!ComparisonOperators.Values.Contains(comparison.ToLower()))
{
parseResult.AddError($"'{comparison}' is not a valid comparison", step.Location.Line, step.Location.Column);
}
if (!Decimal.TryParse(value, out var result))
{
parseResult.AddError($"'{value}' is not a valid decimal", step.Location.Line, step.Location.Column);
}
parseResult.StepActions.Add(new ThenCalculationValue
{
CalculationName = calcName,
Operator = ComparisonOperators.FirstOrDefault(x => x.Value == comparison).Key,
Value = value
});
}
}
return(Task.CompletedTask);
}
public async Task Execute_WhenGherkinParseResultIsInCacheButNoStepActions_DoesNotCallParser()
{
//Arrange
ProviderResult providerResult = new ProviderResult();
IEnumerable <ProviderSourceDataset> datasets = new[]
{
new ProviderSourceDataset()
};
IEnumerable <TestScenario> testScenarios = new[]
{
new TestScenario
{
Id = "scenario-1"
}
};
BuildProject buildProject = new BuildProject();
IGherkinParser gherkinParser = CreateGherkinParser();
GherkinParseResult gherkinParseResult = new GherkinParseResult();
string cacheKey = $"{CacheKeys.GherkinParseResult}scenario-1";
ICacheProvider cacheProvider = CreateCacheProvider();
cacheProvider
.GetAsync <GherkinParseResult>(Arg.Is(cacheKey), Arg.Any <JsonSerializerSettings>())
.Returns(gherkinParseResult);
GherkinExecutor gherkinExecutor = CreateGherkinExecutor(gherkinParser, cacheProvider);
//Act
IEnumerable <ScenarioResult> scenarioResults = await gherkinExecutor.Execute(providerResult, datasets, testScenarios, buildProject);
//Assert
await
gherkinParser
.DidNotReceive()
.Parse(Arg.Any <string>(), Arg.Any <string>(), Arg.Any <BuildProject>());
}
public void Execute_GivenProviderSourceDatasetIsNull_ReturnsParseResultWithError()
{
//Arrange
ProviderResult providerResult = new ProviderResult
{
Provider = new ProviderSummary
{
Id = "p1"
}
};
IEnumerable <ProviderSourceDataset> datasets = new List <ProviderSourceDataset>();
GivenSourceField givenSourceField = new GivenSourceField
{
DatasetName = "ds1",
FieldName = "f1",
Operator = ComparisonOperator.GreaterThan,
Value = "5"
};
//Act
GherkinParseResult parseResult = givenSourceField.Execute(providerResult, datasets);
//Assert
parseResult
.Abort
.Should()
.BeFalse();
parseResult
.HasErrors
.Should()
.BeTrue();
parseResult
.Errors
.First()
.ErrorMessage
.Should()
.Be("f1 in ds1 was not found");
}
public async Task <IActionResult> ValidateGherkin(HttpRequest request)
{
string json = await request.GetRawBodyStringAsync();
ValidateGherkinRequestModel model = JsonConvert.DeserializeObject <ValidateGherkinRequestModel>(json);
if (model == null)
{
_logger.Error("Null model was provided to ValidateGherkin");
return(new BadRequestObjectResult("Null or empty specification id provided"));
}
if (string.IsNullOrWhiteSpace(model.SpecificationId))
{
_logger.Error("No specification id was provided to ValidateGherkin");
return(new BadRequestObjectResult("Null or empty specification id provided"));
}
if (string.IsNullOrWhiteSpace(model.Gherkin))
{
_logger.Error("Null or empty gherkin was provided to ValidateGherkin");
return(new BadRequestObjectResult("Null or empty gherkin name provided"));
}
BuildProject buildProject = await _buildProjectRepositoryPolicy.ExecuteAsync(() => _buildProjectRepository.GetBuildProjectBySpecificationId(model.SpecificationId));
if (buildProject == null || buildProject.Build == null)
{
_logger.Error($"Failed to find a valid build project for specification id: {model.SpecificationId}");
return(new StatusCodeResult(412));
}
GherkinParseResult parseResult = await _gherkinParser.Parse(model.SpecificationId, model.Gherkin, buildProject);
if (parseResult.HasErrors)
{
_logger.Information($"Gherkin parser failed validation with ");
}
return(new OkObjectResult(parseResult.Errors));
}
async public Task Parse(Step step, string stepExpression, GherkinParseResult parseResult, BuildProject buildProject)
{
string[] matches = Regex.Split(step.Text, stepExpression, RegexOptions.IgnoreCase);
string comparison = matches[5];
string providerId = matches[7];
ProviderResult providerResult = await _providerResultsRepositoryPolicy.ExecuteAsync(() => _providerResultsRepository.GetProviderResultByProviderIdAndSpecificationId(providerId, buildProject.SpecificationId));
if (providerResult == null)
{
parseResult.AddError($"Provider results for provider id : '{providerId}' could not be found", step.Location.Line, step.Location.Column);
}
parseResult.StepActions.Add(new AndProviderIs
{
ProviderId = providerId,
Operator = ComparisonOperators.FirstOrDefault(x => x.Value == comparison).Key,
});
}
public async Task <IActionResult> ValidateGherkin(ValidateGherkinRequestModel model)
{
if (model == null)
{
_logger.Error("Null model was provided to ValidateGherkin");
return(new BadRequestObjectResult("Null or empty specification id provided"));
}
if (string.IsNullOrWhiteSpace(model.SpecificationId))
{
_logger.Error("No specification id was provided to ValidateGherkin");
return(new BadRequestObjectResult("Null or empty specification id provided"));
}
if (string.IsNullOrWhiteSpace(model.Gherkin))
{
_logger.Error("Null or empty gherkin was provided to ValidateGherkin");
return(new BadRequestObjectResult("Null or empty gherkin name provided"));
}
BuildProject buildProject = _mapper.Map <BuildProject>(await _calcsApiClientPolicy.ExecuteAsync(() => _calcsApiClient.GetBuildProjectBySpecificationId(model.SpecificationId)));
if (buildProject == null || buildProject.Build == null)
{
_logger.Error($"Failed to find a valid build project for specification id: {model.SpecificationId}");
return(new StatusCodeResult((int)HttpStatusCode.PreconditionFailed));
}
GherkinParseResult parseResult = await _gherkinParser.Parse(model.SpecificationId, model.Gherkin, buildProject);
if (parseResult.HasErrors)
{
_logger.Information($"Gherkin parser failed validation with ");
}
return(new OkObjectResult(parseResult.Errors));
}
public Task Parse(Step step, string stepExpression, GherkinParseResult parseResult, BuildProject buildProject)
{
if (buildProject.Build.Assembly.IsNullOrEmpty())
{
parseResult.AddError("No valid assembly to test", step.Location.Line, step.Location.Column);
}
else
{
byte[] assembly = buildProject.Build.Assembly;
if (assembly == null)
{
parseResult.AddError("No datasets available for this test", step.Location.Line, step.Location.Column);
}
else
{
string[] matches = Regex.Split(step.Text, stepExpression, RegexOptions.IgnoreCase);
string datasetName = matches[5];
string fieldName = matches[9];
string comparisonOperator = matches[13];
string value = matches[15];
KeyValuePair <ComparisonOperator, string> comparisonOperatorValue = ComparisonOperators.FirstOrDefault(x => x.Value == comparisonOperator);
if (string.IsNullOrEmpty(comparisonOperatorValue.Value))
{
parseResult.AddError("Invalid comparison operator", step.Location.Line, step.Location.Column);
}
else
{
PropertyInformation dataset = FindCalculationProperty(assembly, datasetName, "Datasets");
if (dataset == null)
{
parseResult.AddError($"No dataset with the name '{datasetName}' could be found for this test", step.Location.Line, step.Location.Column);
}
else
{
PropertyInformation fieldInfo = FindCalculationProperty(assembly, fieldName, dataset.Type);
if (fieldInfo == null)
{
parseResult.AddError($"'{fieldName}' does not exist in the dataset '{datasetName}'", step.Location.Line, step.Location.Column);
}
else
{
try
{
Type destinationType = null;
switch (fieldInfo.Type)
{
case "String":
destinationType = typeof(string);
break;
case "Decimal":
destinationType = typeof(decimal);
break;
case "Integer":
destinationType = typeof(int);
break;
default:
parseResult.AddError($"Unknown input datatype of '{fieldInfo.Type}' for field '{fieldName}' in the dataset '{datasetName}'", step.Location.Line, step.Location.Column);
break;
}
if (destinationType != null)
{
object actualValue = Convert.ChangeType(value, destinationType);
}
}
catch (FormatException)
{
parseResult.AddError($"Data type mismatch for '{fieldName}' in the dataset '{datasetName}'", step.Location.Line, step.Location.Column);
}
}
}
parseResult.StepActions.Add(new GivenSourceField
{
FieldName = fieldName,
DatasetName = datasetName,
Value = value,
Operator = comparisonOperatorValue.Key,
});
}
}
}
return(Task.CompletedTask);
}
public async Task <GherkinParseResult> Parse(string specificationId, string gherkin, BuildProject buildProject)
{
Guard.IsNullOrWhiteSpace(specificationId, nameof(specificationId));
Guard.IsNullOrWhiteSpace(gherkin, nameof(gherkin));
Guard.ArgumentNotNull(buildProject, nameof(buildProject));
buildProject.Build.Assembly = await _calculationsRepository.GetAssemblyBySpecificationId(specificationId);
GherkinParseResult result = new GherkinParseResult();
Parser parser = new Parser();
try
{
StringBuilder builder = new StringBuilder();
builder.AppendLine("Feature: Feature Wrapper");
builder.AppendLine(" Scenario: Scenario Wrapper");
builder.Append(gherkin);
using (StringReader reader = new StringReader(builder.ToString()))
{
GherkinDocument document = null;
try
{
document = parser.Parse(reader);
}
catch (InvalidOperationException ex)
{
string buildProjectId = buildProject.Id;
_logger.Error(ex, $"Gherkin parser error for build project {{buildProjectId}}: {builder.ToString()}", buildProjectId);
throw;
}
if (document.Feature?.Children != null)
{
foreach (ScenarioDefinition scenario in document.Feature?.Children)
{
if (!scenario.Steps.IsNullOrEmpty())
{
foreach (Step step in scenario.Steps)
{
IEnumerable <KeyValuePair <StepType, string> > expression = stepExpressions.Where(m => Regex.IsMatch(step.Text, m.Value, RegexOptions.IgnoreCase));
if (expression.Any())
{
IStepParser stepParser = _stepParserFactory.GetStepParser(expression.First().Key);
if (stepParser == null)
{
result.AddError("The supplied gherkin could not be parsed", step.Location.Line, step.Location.Column);
}
else
{
await stepParser.Parse(step, expression.First().Value, result, buildProject);
}
}
else
{
result.AddError("The supplied gherkin could not be parsed", step.Location.Line, step.Location.Column);
}
string keyword = step.Keyword?.ToLowerInvariant().Trim();
}
}
else
{
result.AddError("The supplied gherkin could not be parsed", 0, 0);
}
}
}
}
}
catch (CompositeParserException exception)
{
foreach (ParserException error in exception.Errors)
{
result.AddError(error.Message, error.Location.Line, error.Location.Column);
}
}
return(result);
}
public async Task Execute_WhenGherkinParseResultIsInCacheWithStepActionButAborted_DoesNotCallParserDoesNotAddDependencies()
{
//Arrange
ProviderResult providerResult = new ProviderResult();
IEnumerable <ProviderSourceDataset> datasets = new[]
{
new ProviderSourceDataset()
};
IEnumerable <TestScenario> testScenarios = new[]
{
new TestScenario
{
Id = "scenario-1"
}
};
BuildProject buildProject = new BuildProject();
IGherkinParser gherkinParser = CreateGherkinParser();
GherkinParseResult stepActionherkinParseResult = new GherkinParseResult {
Abort = true
};
IStepAction stepAction = Substitute.For <IStepAction>();
stepAction
.Execute(Arg.Is(providerResult), Arg.Is(datasets))
.Returns(stepActionherkinParseResult);
GherkinParseResult gherkinParseResult = new GherkinParseResult();
gherkinParseResult
.StepActions
.Add(stepAction);
string cacheKey = $"{CacheKeys.GherkinParseResult}scenario-1";
ICacheProvider cacheProvider = CreateCacheProvider();
cacheProvider
.GetAsync <GherkinParseResult>(Arg.Is(cacheKey), Arg.Any <JsonSerializerSettings>())
.Returns(gherkinParseResult);
GherkinExecutor gherkinExecutor = CreateGherkinExecutor(gherkinParser, cacheProvider);
//Act
IEnumerable <ScenarioResult> scenarioResults = await gherkinExecutor.Execute(providerResult, datasets, testScenarios, buildProject);
//Assert
await
gherkinParser
.DidNotReceive()
.Parse(Arg.Any <string>(), Arg.Any <string>(), Arg.Any <BuildProject>());
scenarioResults
.Count()
.Should()
.Be(1);
scenarioResults
.First()
.StepsExecuted
.Should()
.Be(0);
}
public async Task Execute_WhenGherkinParseResultIsNotInCacheWithTeoStepActionAndResultHasError_CreatesResultWithErrors()
{
//Arrange
ProviderResult providerResult = new ProviderResult();
IEnumerable <ProviderSourceDataset> datasets = new[]
{
new ProviderSourceDataset()
};
IEnumerable <TestScenario> testScenarios = new[]
{
new TestScenario
{
Id = "scenario-1",
Current = new TestScenarioVersion
{
Gherkin = "gherkin"
},
SpecificationId = "spec1"
}
};
BuildProject buildProject = new BuildProject();
GherkinParseResult stepActionherkinParseResult1 = new GherkinParseResult("An error");
stepActionherkinParseResult1
.Dependencies
.Add(new Dependency("ds1", "f1", "value"));
GherkinParseResult stepActionherkinParseResult2 = new GherkinParseResult();
stepActionherkinParseResult2
.Dependencies
.Add(new Dependency("ds1", "f1", "value"));
IStepAction stepAction1 = Substitute.For <IStepAction>();
stepAction1
.Execute(Arg.Is(providerResult), Arg.Is(datasets))
.Returns(stepActionherkinParseResult1);
IStepAction stepAction2 = Substitute.For <IStepAction>();
stepAction2
.Execute(Arg.Is(providerResult), Arg.Is(datasets))
.Returns(stepActionherkinParseResult2);
GherkinParseResult gherkinParseResult = new GherkinParseResult();
gherkinParseResult
.StepActions
.AddRange(new[] { stepAction1, stepAction2 });
IGherkinParser gherkinParser = CreateGherkinParser();
gherkinParser
.Parse(Arg.Is("spec1"), Arg.Is("gherkin"), Arg.Is(buildProject))
.Returns(gherkinParseResult);
string cacheKey = $"{CacheKeys.GherkinParseResult}scenario-1";
ICacheProvider cacheProvider = CreateCacheProvider();
cacheProvider
.GetAsync <GherkinParseResult>(Arg.Is(cacheKey), Arg.Any <JsonSerializerSettings>())
.Returns((GherkinParseResult)null);
GherkinExecutor gherkinExecutor = CreateGherkinExecutor(gherkinParser, cacheProvider);
//Act
IEnumerable <ScenarioResult> scenarioResults = await gherkinExecutor.Execute(providerResult, datasets, testScenarios, buildProject);
//Assert
scenarioResults
.Count()
.Should()
.Be(1);
scenarioResults
.First()
.Dependencies
.Count()
.Should()
.Be(2);
scenarioResults
.First()
.HasErrors
.Should()
.BeTrue();
scenarioResults
.First()
.StepsExecuted
.Should()
.Be(2);
}
public Task Parse(Step step, string stepExpression, GherkinParseResult parseResult, BuildProject buildProject)
{
if (buildProject.Build.Assembly.IsNullOrEmpty())
{
parseResult.AddError("No valid assembly to test", step.Location.Line, step.Location.Column);
}
else
{
byte[] assembly = buildProject.Build.Assembly;
if (assembly == null)
{
parseResult.AddError("No calculations available for this test", step.Location.Line, step.Location.Column);
}
else
{
string[] matches = Regex.Split(step.Text, stepExpression, RegexOptions.IgnoreCase);
string calcName = matches[7];
string fieldName = matches[19];
string comparison = matches[9];
string datasetName = matches[15];
MethodInformation calculation = FindCalculationMethod(assembly, calcName);
if (calculation == null)
{
parseResult.AddError($"Calculation: '{calcName}' was not found to test", step.Location.Line, step.Location.Column);
}
PropertyInformation dataset = FindCalculationProperty(assembly, datasetName, "Datasets");
if (dataset == null)
{
parseResult.AddError($"No dataset with the name '{datasetName}' could be found for this test", step.Location.Line, step.Location.Column);
}
else
{
PropertyInformation fieldInfo = FindCalculationProperty(assembly, fieldName, dataset.Type);
if (fieldInfo == null)
{
parseResult.AddError($"'{fieldName}' does not exis in the dataset '{datasetName}'", step.Location.Line, step.Location.Column);
}
}
if (!ComparisonOperators.Values.Contains(comparison.ToLower()))
{
parseResult.AddError($"'{comparison}' is not a valid comparison", step.Location.Line, step.Location.Column);
}
parseResult.StepActions.Add(new ThenSourceField
{
CalculationName = calcName,
Operator = ComparisonOperators.FirstOrDefault(x => x.Value == comparison).Key,
FieldName = fieldName,
DatasetName = datasetName
});
}
}
return(Task.CompletedTask);
}
