/// <summary>
/// Generates a test suite of test cases based on the test specification.
/// Returns true if successful.
/// </summary>
public bool PopulateTestSuite(TestSpecification testSpecification, out string testSuiteStatus)
{
int numberConsistent = 0;
GeneratedTestsInSuite = new TestSuite();
GeneratedTestsInSuite.Set(testSpecification.Name, testSpecification.Text, new List <TestCase>());
CoverageMetricsInSuite.Initialize(testSpecification.CoverageGroups, testSpecification.InputParameters);
foreach (CoverageGroup coverageGroup in CoverageMetricsInSuite.SortedCoverageGroups(testSpecification))
{
while (!CoverageMetricsInSuite.CoverageComplete(coverageGroup.Name))
{
TestCase testCase = PopulateTestCase(testSpecification, coverageGroup);
if (testCase == null)
{
testSuiteStatus =
$"Aborted after generating {GeneratedTestsInSuite.TestCases.Count} test cases for test specification {testSpecification.Name}";
return(false);
}
CoverageMetricsInSuite.Record(testSpecification.CoverageGroups, testCase);
GeneratedTestsInSuite.TestCases.Add(testCase);
numberConsistent += testCase.Valuation ? 1 : 0;
}
}
int numberContradictory = GeneratedTestsInSuite.TestCases.Count - numberConsistent;
testSuiteStatus = $"Generated {GeneratedTestsInSuite.TestCases.Count} test cases for test specification {testSpecification.Name} ({numberConsistent} consistent, {numberContradictory} contradictory)";
return(true);
}
/// <summary>
/// Creates and returns an assignment for a parameter in the coverage group
/// based on a specific combination that has not yet been covered in a test case.
/// </summary>
private bool AssignCoveredParameter(
IDictionary <string, ParameterAssignment> parameterAssignments,
TestSpecification testSpecification,
CoverageGroup coverageGroup,
IList <string> uncoveredCombination,
int groupIndex)
{
string selectedParameter = coverageGroup.Parameters[groupIndex];
string selectedEquivalenceClass = uncoveredCombination[groupIndex];
if ((selectedParameter == null) || (selectedEquivalenceClass == null))
{
return(false);
}
InputParameter inputParameter = testSpecification.InputParameters[selectedParameter];
if (inputParameter == null)
{
return(false);
}
ParameterAssignment parameterAssignment = new ParameterAssignment();
parameterAssignment.Set(
inputParameter.Given,
inputParameter.Name,
inputParameter.Text,
inputParameter.EquivalenceClasses[selectedEquivalenceClass]);
parameterAssignments.Add(parameterAssignment.Name, parameterAssignment);
return(true);
}
/// <summary>
/// Reads the test specification from an XML input file, generates a test suite
/// from that test specification, and writes the test suite to an output file.
/// Returns true if successful.
/// </summary>
public bool CreateSuiteFromSpecificationAndWrite(string inputFilenameWithPath, string outputFilenameWithPath)
{
if (!Set(inputFilenameWithPath, outputFilenameWithPath))
{
return(false);
}
TestSpecification testSpecification = Read(InputFilenameWithPath, _inputFormat);
if (testSpecification == null)
{
return(false);
}
string testSuiteStatus;
GeneratedTestsInSuite = new TestSuite();
GeneratedTestsInSuite.Set(testSpecification.Name, testSpecification.Text, new List <TestCase>());
if (!PopulateTestSuite(testSpecification, out testSuiteStatus))
{
return(false);
}
return(WriteToFile(OutputFilenameWithPath, testSpecification.Given));
}
/// <summary>
/// Reads the XML element for the relational expression and returns true if successful.
/// </summary>
public override bool ReadAsXml(XElement xExpression)
{
string relationalOperator = TestSpecification.XmlField(xExpression, "Operator");
string parameterName = TestSpecification.XmlField(xExpression, "Parameter");
string equivalenceClassName = TestSpecification.XmlField(xExpression, "EquivalenceClass");
return(Set(relationalOperator, parameterName, equivalenceClassName));
}
/// <summary>
/// Reads the XML element for the equivalence class and returns true if successful.
/// </summary>
public bool ReadAsXml(XElement xEquivalenceClass)
{
string given = TestSpecification.XmlField(xEquivalenceClass, "Given");
string name = TestSpecification.XmlField(xEquivalenceClass, "Name");
string text = TestSpecification.XmlField(xEquivalenceClass, "Text");
IList <Expression> conditions = Expression.AllExpressonsAsXml(xEquivalenceClass);
return(Set(given, name, text, conditions));
}
/// <summary>
/// Generates a single test case that selects an "uncovered" combination from the coverage group.
/// Returns the test case if successful, otherwise null.
/// </summary>
private TestCase PopulateTestCase(TestSpecification testSpecification, CoverageGroup coverageGroup)
{
IList <string> uncoveredCombination = CoverageMetricsInSuite.FirstUncovered(coverageGroup.Name)
.Split(new char[] { '.' }, StringSplitOptions.None);
IDictionary <string, ParameterAssignment> parameterAssignments = new Dictionary <string, ParameterAssignment>();
for (int groupIndex = 0; groupIndex < coverageGroup.Parameters.Count; groupIndex++)
{
if (!AssignCoveredParameter(parameterAssignments, testSpecification, coverageGroup, uncoveredCombination, groupIndex))
{
return(null);
}
}
foreach (InputParameter inputParameter in testSpecification.InputParameters.Values
.Where(x => !coverageGroup.Parameters.Contains(x.Name)))
{
if (!AssignUncoveredParameter(parameterAssignments, inputParameter))
{
return(null);
}
}
bool valuation = true;
foreach (ParameterAssignment parameterAssignment in parameterAssignments.Values)
{
valuation = valuation &&
AssociatedParameterValuation(parameterAssignments,
testSpecification.InputParameters[parameterAssignment.Name]) &&
AssociatedEquivalenceClassValuation(parameterAssignments,
parameterAssignment.SelectedEquivalenceClass);
}
IList <ExpectedResult> expectedResults = new List <ExpectedResult>();
foreach (ExpectedResult expectedResult in testSpecification.ExpectedResults.Values
.Where(x => x.ConditionExpression.Match(parameterAssignments)))
{
expectedResults.Add(expectedResult);
}
valuation = (expectedResults.Count == 0) ? false : valuation;
_testCaseNumber++;
string given = testSpecification.Given;
string name = $"{testSpecification.Name}-{_testCaseNumber}";
string text = $"Test {_testCaseNumber}";
TestCase testCase = new TestCase();
testCase.Set(given, name, text, parameterAssignments, expectedResults, valuation);
return(testCase);
}
/// <summary>
/// Reads the test specification in its given format and returns the resulting object.
/// </summary>
private TestSpecification Read(string inputFilenameWithPath, FormatEnum inputFormat)
{
TestSpecification testSpecification = new TestSpecification();
if (((inputFormat == FormatEnum.Xml) && testSpecification.ReadAsXml(inputFilenameWithPath)) ||
((inputFormat == FormatEnum.Json) && testSpecification.ReadAsJson(inputFilenameWithPath)))
{
return(testSpecification);
}
return(null);
}
/// <summary>
/// Reads the XML element for the unary expression and returns true if successful.
/// </summary>
public override bool ReadAsXml(XElement xExpression)
{
string unaryOperator = TestSpecification.XmlField(xExpression, "Operator");
IList <Expression> expressions = AllExpressonsAsXml(xExpression);
if ((expressions == null) || (expressions.Count != 1))
{
return(false);
}
return(Set(unaryOperator, expressions[0]));
}
/// <summary>
/// Reads the XML element for the expected result and returns true if successful.
/// </summary>
public bool ReadAsXml(XElement xExpectedResult)
{
string given = TestSpecification.XmlField(xExpectedResult, "Given");
string name = TestSpecification.XmlField(xExpectedResult, "Name");
string text = TestSpecification.XmlField(xExpectedResult, "Text");
IList <Expression> expressions = Expression.AllExpressonsAsXml(xExpectedResult);
if ((expressions == null) || (expressions.Count != 1))
{
return(false);
}
return(Set(given, name, text, expressions[0]));
}
/// <summary>
/// Reads the XML element for the coverage group and returns true if successful.
/// </summary>
public bool ReadAsXml(XElement xCoverageGroup)
{
string name = TestSpecification.XmlField(xCoverageGroup, "Name");
IList <string> parameters = new List <string>();
foreach (XElement xParameter in XmlHelper.Children(xCoverageGroup, "Parameter"))
{
string parameterName = TestSpecification.FieldsAsAttributes
? XmlHelper.AttributeOf(xParameter, "Name")
: xParameter.Value;
parameters.Add(parameterName);
}
return(Set(name, parameters));
}
/// <summary>
/// Reads the JSON contents of the file into the test specification and returns true if successful.
/// </summary>
public bool ReadAsJson(string filenameWithPath)
{
string contentsAsString = FileHelper.ContentsAsString(filenameWithPath);
TestSpecification specification = JsonConvert.DeserializeObject <TestSpecification>(contentsAsString);
if (specification == null)
{
return(false);
}
bool compositeRead = Set(
specification.Given, specification.Name, specification.Text, specification.CoverageGroups,
specification.InputParameters, specification.ExpectedResults);
compositeRead = compositeRead && Normalize();
compositeRead = compositeRead && UpdateDependencies();
return(compositeRead);
}
/// <summary>
/// Returns the list of coverage groups, SORTED by number of combinations DESCENDING.
/// </summary>
public IList <CoverageGroup> SortedCoverageGroups(TestSpecification testSpecification)
{
IDictionary <int, IList <CoverageGroup> > combinationMappings = CombinationMappings(testSpecification);
List <int> combinationKeys = combinationMappings.Keys.ToList();
combinationKeys.Sort();
combinationKeys.Reverse();
IList <CoverageGroup> sortedCoverageGroups = new List <CoverageGroup>();
foreach (int combinationKey in combinationKeys)
{
foreach (CoverageGroup coverageGroup in combinationMappings[combinationKey])
{
sortedCoverageGroups.Add(coverageGroup);
}
}
return(sortedCoverageGroups);
}
/// <summary>
/// Returns a dictionary mapping a number of combinations onto the list of 1 or more coverage
/// groups whose parameters have that specific number of exhaustive combinations.
///
/// For example, if coverage group has parameters (A, B) and A has 3 equivalence classes and B
/// has 4 equivalence classes, then a mapping is made from 12 onto a list that includes this coverage
/// group.
/// </summary>
private IDictionary <int, IList <CoverageGroup> > CombinationMappings(TestSpecification testSpecification)
{
IDictionary <int, IList <CoverageGroup> > combinationMappings = new Dictionary <int, IList <CoverageGroup> >();
foreach (CoverageGroup coverageGroup in testSpecification.CoverageGroups)
{
int combinations = 1;
foreach (string parameterName in coverageGroup.Parameters)
{
combinations *= testSpecification.InputParameters[parameterName].EquivalenceClasses.Count;
}
if (!combinationMappings.ContainsKey(combinations))
{
combinationMappings.Add(combinations, new List <CoverageGroup>());
}
combinationMappings[combinations].Add(coverageGroup);
}
return(combinationMappings);
}
/// <summary>
/// Reads the XML element for the input parameter and returns true if successful.
/// </summary>
public bool ReadAsXml(XElement xInputParameter)
{
string given = TestSpecification.XmlField(xInputParameter, "Given");
string name = TestSpecification.XmlField(xInputParameter, "Name");
string text = TestSpecification.XmlField(xInputParameter, "Text");
IDictionary <string, EquivalenceClass> equivalenceClasses = new Dictionary <string, EquivalenceClass>();
foreach (XElement xEquivalenceClass in XmlHelper.Children(xInputParameter, "EquivalenceClass"))
{
EquivalenceClass equivalenceClass = new EquivalenceClass();
if (!equivalenceClass.ReadAsXml(xEquivalenceClass))
{
return(false);
}
equivalenceClasses.Add(equivalenceClass.Name, equivalenceClass);
}
IList <Expression> conditions = Expression.AllExpressonsAsXml(xInputParameter);
return(Set(given, name, text, equivalenceClasses, conditions));
}
