/// <summary>TESTS RUNNER</summary> public virtual void TestAll() { Assert.True("Number of tests has to be greater then zero", testsFromConfigFile .Count > 0); Log.Info("TestAll"); // Run the tests defined in the testConf.xml config file. for (int index = 0; index < testsFromConfigFile.Count; index++) { CLITestData testdata = testsFromConfigFile[index]; // Execute the test commands AList <CLICommand> testCommands = testdata.GetTestCommands(); CommandExecutor.Result cmdResult = null; foreach (CLICommand cmd in testCommands) { try { cmdResult = Execute(cmd); } catch (Exception e) { NUnit.Framework.Assert.Fail(StringUtils.StringifyException(e)); } } bool overallTCResult = true; // Run comparators AList <ComparatorData> compdata = testdata.GetComparatorData(); foreach (ComparatorData cd in compdata) { string comptype = cd.GetComparatorType(); bool compareOutput = false; if (!Runtime.EqualsIgnoreCase(comptype, "none")) { compareOutput = CompareTestOutput(cd, cmdResult); overallTCResult &= compareOutput; } cd.SetExitCode(cmdResult.GetExitCode()); cd.SetActualOutput(cmdResult.GetCommandOutput()); cd.SetTestResult(compareOutput); } testdata.SetTestResult(overallTCResult); // Execute the cleanup commands AList <CLICommand> cleanupCommands = testdata.GetCleanupCommands(); foreach (CLICommand cmd_1 in cleanupCommands) { try { Execute(cmd_1); } catch (Exception e) { NUnit.Framework.Assert.Fail(StringUtils.StringifyException(e)); } } } }
/// <exception cref="Org.Xml.Sax.SAXException"/> public override void StartElement(string uri, string localName, string qName, Attributes attributes) { if (qName.Equals("test")) { this.td = new CLITestData(); } else { if (qName.Equals("test-commands")) { this.testCommands = new AList <CLICommand>(); } else { if (qName.Equals("cleanup-commands")) { this.cleanupCommands = new AList <CLICommand>(); } else { if (qName.Equals("comparators")) { this._enclosing.testComparators = new AList <ComparatorData>(); } else { if (qName.Equals("comparator")) { this._enclosing.comparatorData = new ComparatorData(); } } } } } this.charString = string.Empty; }
/// <exception cref="Org.Xml.Sax.SAXException"/> public override void EndElement(string uri, string localName, string qName) { if (qName.Equals("description")) { this.td.SetTestDesc(this.charString); } else { if (qName.Equals("windows")) { this.runOnWindows = System.Boolean.Parse(this.charString); } else { if (qName.Equals("test-commands")) { this.td.SetTestCommands(this.testCommands); this.testCommands = null; } else { if (qName.Equals("cleanup-commands")) { this.td.SetCleanupCommands(this.cleanupCommands); this.cleanupCommands = null; } else { if (qName.Equals("command")) { if (this.testCommands != null) { this.testCommands.AddItem(new CLITestCmd(this.charString, new CLICommandFS())); } else { if (this.cleanupCommands != null) { this.cleanupCommands.AddItem(new CLITestCmd(this.charString, new CLICommandFS())); } } } else { if (qName.Equals("comparators")) { this.td.SetComparatorData(this._enclosing.testComparators); } else { if (qName.Equals("comparator")) { this._enclosing.testComparators.AddItem(this._enclosing.comparatorData); } else { if (qName.Equals("type")) { this._enclosing.comparatorData.SetComparatorType(this.charString); } else { if (qName.Equals("expected-output")) { this._enclosing.comparatorData.SetExpectedOutput(this.charString); } else { if (qName.Equals("test")) { if (!Shell.Windows || this.runOnWindows) { this._enclosing.testsFromConfigFile.AddItem(this.td); } this.td = null; this.runOnWindows = true; } else { if (qName.Equals("mode")) { this._enclosing.testMode = this.charString; if (!this._enclosing.testMode.Equals(CLITestHelper.TestmodeNocompare) && !this._enclosing .testMode.Equals(CLITestHelper.TestmodeTest)) { this._enclosing.testMode = CLITestHelper.TestmodeTest; } } } } } } } } } } } } }
/// <summary>Display the summarized results</summary> private void DisplayResults() { Log.Info("Detailed results:"); Log.Info("----------------------------------\n"); for (int i = 0; i < testsFromConfigFile.Count; i++) { CLITestData td = testsFromConfigFile[i]; bool testResult = td.GetTestResult(); // Display the details only if there is a failure if (!testResult) { Log.Info("-------------------------------------------"); Log.Info(" Test ID: [" + (i + 1) + "]"); Log.Info(" Test Description: [" + td.GetTestDesc() + "]"); Log.Info(string.Empty); AList <CLICommand> testCommands = td.GetTestCommands(); foreach (CLICommand cmd in testCommands) { Log.Info(" Test Commands: [" + ExpandCommand(cmd.GetCmd()) + "]"); } Log.Info(string.Empty); AList <CLICommand> cleanupCommands = td.GetCleanupCommands(); foreach (CLICommand cmd_1 in cleanupCommands) { Log.Info(" Cleanup Commands: [" + ExpandCommand(cmd_1.GetCmd()) + "]"); } Log.Info(string.Empty); AList <ComparatorData> compdata = td.GetComparatorData(); foreach (ComparatorData cd in compdata) { bool resultBoolean = cd.GetTestResult(); Log.Info(" Comparator: [" + cd.GetComparatorType() + "]"); Log.Info(" Comparision result: [" + (resultBoolean ? "pass" : "fail") + "]"); Log.Info(" Expected output: [" + ExpandCommand(cd.GetExpectedOutput( )) + "]"); Log.Info(" Actual output: [" + cd.GetActualOutput() + "]"); } Log.Info(string.Empty); } } Log.Info("Summary results:"); Log.Info("----------------------------------\n"); bool overallResults = true; int totalPass = 0; int totalFail = 0; int totalComparators = 0; for (int i_1 = 0; i_1 < testsFromConfigFile.Count; i_1++) { CLITestData td = testsFromConfigFile[i_1]; totalComparators += testsFromConfigFile[i_1].GetComparatorData().Count; bool resultBoolean = td.GetTestResult(); if (resultBoolean) { totalPass++; } else { totalFail++; } overallResults &= resultBoolean; } Log.Info(" Testing mode: " + testMode); Log.Info(string.Empty); Log.Info(" Overall result: " + (overallResults ? "+++ PASS +++" : "--- FAIL ---" )); if ((totalPass + totalFail) == 0) { Log.Info(" # Tests pass: "******" # Tests fail: " + 0); } else { Log.Info(" # Tests pass: "******" (" + (100 * totalPass / (totalPass + totalFail)) + "%)"); Log.Info(" # Tests fail: " + totalFail + " (" + (100 * totalFail / (totalPass + totalFail)) + "%)"); } Log.Info(" # Validations done: " + totalComparators + " (each test may do multiple validations)" ); Log.Info(string.Empty); Log.Info("Failing tests:"); Log.Info("--------------"); int i_2 = 0; bool foundTests = false; for (i_2 = 0; i_2 < testsFromConfigFile.Count; i_2++) { bool resultBoolean = testsFromConfigFile[i_2].GetTestResult(); if (!resultBoolean) { Log.Info((i_2 + 1) + ": " + testsFromConfigFile[i_2].GetTestDesc()); foundTests = true; } } if (!foundTests) { Log.Info("NONE"); } foundTests = false; Log.Info(string.Empty); Log.Info("Passing tests:"); Log.Info("--------------"); for (i_2 = 0; i_2 < testsFromConfigFile.Count; i_2++) { bool resultBoolean = testsFromConfigFile[i_2].GetTestResult(); if (resultBoolean) { Log.Info((i_2 + 1) + ": " + testsFromConfigFile[i_2].GetTestDesc()); foundTests = true; } } if (!foundTests) { Log.Info("NONE"); } Assert.True("One of the tests failed. " + "See the Detailed results to identify " + "the command that failed", overallResults); }