/// <summary> /// Generates a test suite of test cases based on the test specification. /// Returns true if successful. /// </summary> public bool PopulateTestSuite(TestSpecification testSpecification, out string testSuiteStatus) { int numberConsistent = 0; GeneratedTestsInSuite = new TestSuite(); GeneratedTestsInSuite.Set(testSpecification.Name, testSpecification.Text, new List <TestCase>()); CoverageMetricsInSuite.Initialize(testSpecification.CoverageGroups, testSpecification.InputParameters); foreach (CoverageGroup coverageGroup in CoverageMetricsInSuite.SortedCoverageGroups(testSpecification)) { while (!CoverageMetricsInSuite.CoverageComplete(coverageGroup.Name)) { TestCase testCase = PopulateTestCase(testSpecification, coverageGroup); if (testCase == null) { testSuiteStatus = $"Aborted after generating {GeneratedTestsInSuite.TestCases.Count} test cases for test specification {testSpecification.Name}"; return(false); } CoverageMetricsInSuite.Record(testSpecification.CoverageGroups, testCase); GeneratedTestsInSuite.TestCases.Add(testCase); numberConsistent += testCase.Valuation ? 1 : 0; } } int numberContradictory = GeneratedTestsInSuite.TestCases.Count - numberConsistent; testSuiteStatus = $"Generated {GeneratedTestsInSuite.TestCases.Count} test cases for test specification {testSpecification.Name} ({numberConsistent} consistent, {numberContradictory} contradictory)"; return(true); }
/// <summary> /// Creates and returns an assignment for a parameter in the coverage group /// based on a specific combination that has not yet been covered in a test case. /// </summary> private bool AssignCoveredParameter( IDictionary <string, ParameterAssignment> parameterAssignments, TestSpecification testSpecification, CoverageGroup coverageGroup, IList <string> uncoveredCombination, int groupIndex) { string selectedParameter = coverageGroup.Parameters[groupIndex]; string selectedEquivalenceClass = uncoveredCombination[groupIndex]; if ((selectedParameter == null) || (selectedEquivalenceClass == null)) { return(false); } InputParameter inputParameter = testSpecification.InputParameters[selectedParameter]; if (inputParameter == null) { return(false); } ParameterAssignment parameterAssignment = new ParameterAssignment(); parameterAssignment.Set( inputParameter.Given, inputParameter.Name, inputParameter.Text, inputParameter.EquivalenceClasses[selectedEquivalenceClass]); parameterAssignments.Add(parameterAssignment.Name, parameterAssignment); return(true); }
/// <summary> /// Reads the test specification from an XML input file, generates a test suite /// from that test specification, and writes the test suite to an output file. /// Returns true if successful. /// </summary> public bool CreateSuiteFromSpecificationAndWrite(string inputFilenameWithPath, string outputFilenameWithPath) { if (!Set(inputFilenameWithPath, outputFilenameWithPath)) { return(false); } TestSpecification testSpecification = Read(InputFilenameWithPath, _inputFormat); if (testSpecification == null) { return(false); } string testSuiteStatus; GeneratedTestsInSuite = new TestSuite(); GeneratedTestsInSuite.Set(testSpecification.Name, testSpecification.Text, new List <TestCase>()); if (!PopulateTestSuite(testSpecification, out testSuiteStatus)) { return(false); } return(WriteToFile(OutputFilenameWithPath, testSpecification.Given)); }
/// <summary> /// Reads the XML element for the relational expression and returns true if successful. /// </summary> public override bool ReadAsXml(XElement xExpression) { string relationalOperator = TestSpecification.XmlField(xExpression, "Operator"); string parameterName = TestSpecification.XmlField(xExpression, "Parameter"); string equivalenceClassName = TestSpecification.XmlField(xExpression, "EquivalenceClass"); return(Set(relationalOperator, parameterName, equivalenceClassName)); }
/// <summary> /// Reads the XML element for the equivalence class and returns true if successful. /// </summary> public bool ReadAsXml(XElement xEquivalenceClass) { string given = TestSpecification.XmlField(xEquivalenceClass, "Given"); string name = TestSpecification.XmlField(xEquivalenceClass, "Name"); string text = TestSpecification.XmlField(xEquivalenceClass, "Text"); IList <Expression> conditions = Expression.AllExpressonsAsXml(xEquivalenceClass); return(Set(given, name, text, conditions)); }
/// <summary> /// Generates a single test case that selects an "uncovered" combination from the coverage group. /// Returns the test case if successful, otherwise null. /// </summary> private TestCase PopulateTestCase(TestSpecification testSpecification, CoverageGroup coverageGroup) { IList <string> uncoveredCombination = CoverageMetricsInSuite.FirstUncovered(coverageGroup.Name) .Split(new char[] { '.' }, StringSplitOptions.None); IDictionary <string, ParameterAssignment> parameterAssignments = new Dictionary <string, ParameterAssignment>(); for (int groupIndex = 0; groupIndex < coverageGroup.Parameters.Count; groupIndex++) { if (!AssignCoveredParameter(parameterAssignments, testSpecification, coverageGroup, uncoveredCombination, groupIndex)) { return(null); } } foreach (InputParameter inputParameter in testSpecification.InputParameters.Values .Where(x => !coverageGroup.Parameters.Contains(x.Name))) { if (!AssignUncoveredParameter(parameterAssignments, inputParameter)) { return(null); } } bool valuation = true; foreach (ParameterAssignment parameterAssignment in parameterAssignments.Values) { valuation = valuation && AssociatedParameterValuation(parameterAssignments, testSpecification.InputParameters[parameterAssignment.Name]) && AssociatedEquivalenceClassValuation(parameterAssignments, parameterAssignment.SelectedEquivalenceClass); } IList <ExpectedResult> expectedResults = new List <ExpectedResult>(); foreach (ExpectedResult expectedResult in testSpecification.ExpectedResults.Values .Where(x => x.ConditionExpression.Match(parameterAssignments))) { expectedResults.Add(expectedResult); } valuation = (expectedResults.Count == 0) ? false : valuation; _testCaseNumber++; string given = testSpecification.Given; string name = $"{testSpecification.Name}-{_testCaseNumber}"; string text = $"Test {_testCaseNumber}"; TestCase testCase = new TestCase(); testCase.Set(given, name, text, parameterAssignments, expectedResults, valuation); return(testCase); }
/// <summary> /// Reads the test specification in its given format and returns the resulting object. /// </summary> private TestSpecification Read(string inputFilenameWithPath, FormatEnum inputFormat) { TestSpecification testSpecification = new TestSpecification(); if (((inputFormat == FormatEnum.Xml) && testSpecification.ReadAsXml(inputFilenameWithPath)) || ((inputFormat == FormatEnum.Json) && testSpecification.ReadAsJson(inputFilenameWithPath))) { return(testSpecification); } return(null); }
/// <summary> /// Reads the XML element for the unary expression and returns true if successful. /// </summary> public override bool ReadAsXml(XElement xExpression) { string unaryOperator = TestSpecification.XmlField(xExpression, "Operator"); IList <Expression> expressions = AllExpressonsAsXml(xExpression); if ((expressions == null) || (expressions.Count != 1)) { return(false); } return(Set(unaryOperator, expressions[0])); }
/// <summary> /// Reads the XML element for the expected result and returns true if successful. /// </summary> public bool ReadAsXml(XElement xExpectedResult) { string given = TestSpecification.XmlField(xExpectedResult, "Given"); string name = TestSpecification.XmlField(xExpectedResult, "Name"); string text = TestSpecification.XmlField(xExpectedResult, "Text"); IList <Expression> expressions = Expression.AllExpressonsAsXml(xExpectedResult); if ((expressions == null) || (expressions.Count != 1)) { return(false); } return(Set(given, name, text, expressions[0])); }
/// <summary> /// Reads the XML element for the coverage group and returns true if successful. /// </summary> public bool ReadAsXml(XElement xCoverageGroup) { string name = TestSpecification.XmlField(xCoverageGroup, "Name"); IList <string> parameters = new List <string>(); foreach (XElement xParameter in XmlHelper.Children(xCoverageGroup, "Parameter")) { string parameterName = TestSpecification.FieldsAsAttributes ? XmlHelper.AttributeOf(xParameter, "Name") : xParameter.Value; parameters.Add(parameterName); } return(Set(name, parameters)); }
/// <summary> /// Reads the JSON contents of the file into the test specification and returns true if successful. /// </summary> public bool ReadAsJson(string filenameWithPath) { string contentsAsString = FileHelper.ContentsAsString(filenameWithPath); TestSpecification specification = JsonConvert.DeserializeObject <TestSpecification>(contentsAsString); if (specification == null) { return(false); } bool compositeRead = Set( specification.Given, specification.Name, specification.Text, specification.CoverageGroups, specification.InputParameters, specification.ExpectedResults); compositeRead = compositeRead && Normalize(); compositeRead = compositeRead && UpdateDependencies(); return(compositeRead); }
/// <summary> /// Returns the list of coverage groups, SORTED by number of combinations DESCENDING. /// </summary> public IList <CoverageGroup> SortedCoverageGroups(TestSpecification testSpecification) { IDictionary <int, IList <CoverageGroup> > combinationMappings = CombinationMappings(testSpecification); List <int> combinationKeys = combinationMappings.Keys.ToList(); combinationKeys.Sort(); combinationKeys.Reverse(); IList <CoverageGroup> sortedCoverageGroups = new List <CoverageGroup>(); foreach (int combinationKey in combinationKeys) { foreach (CoverageGroup coverageGroup in combinationMappings[combinationKey]) { sortedCoverageGroups.Add(coverageGroup); } } return(sortedCoverageGroups); }
/// <summary> /// Returns a dictionary mapping a number of combinations onto the list of 1 or more coverage /// groups whose parameters have that specific number of exhaustive combinations. /// /// For example, if coverage group has parameters (A, B) and A has 3 equivalence classes and B /// has 4 equivalence classes, then a mapping is made from 12 onto a list that includes this coverage /// group. /// </summary> private IDictionary <int, IList <CoverageGroup> > CombinationMappings(TestSpecification testSpecification) { IDictionary <int, IList <CoverageGroup> > combinationMappings = new Dictionary <int, IList <CoverageGroup> >(); foreach (CoverageGroup coverageGroup in testSpecification.CoverageGroups) { int combinations = 1; foreach (string parameterName in coverageGroup.Parameters) { combinations *= testSpecification.InputParameters[parameterName].EquivalenceClasses.Count; } if (!combinationMappings.ContainsKey(combinations)) { combinationMappings.Add(combinations, new List <CoverageGroup>()); } combinationMappings[combinations].Add(coverageGroup); } return(combinationMappings); }
/// <summary> /// Reads the XML element for the input parameter and returns true if successful. /// </summary> public bool ReadAsXml(XElement xInputParameter) { string given = TestSpecification.XmlField(xInputParameter, "Given"); string name = TestSpecification.XmlField(xInputParameter, "Name"); string text = TestSpecification.XmlField(xInputParameter, "Text"); IDictionary <string, EquivalenceClass> equivalenceClasses = new Dictionary <string, EquivalenceClass>(); foreach (XElement xEquivalenceClass in XmlHelper.Children(xInputParameter, "EquivalenceClass")) { EquivalenceClass equivalenceClass = new EquivalenceClass(); if (!equivalenceClass.ReadAsXml(xEquivalenceClass)) { return(false); } equivalenceClasses.Add(equivalenceClass.Name, equivalenceClass); } IList <Expression> conditions = Expression.AllExpressonsAsXml(xInputParameter); return(Set(given, name, text, equivalenceClasses, conditions)); }