/// <summary>TESTS RUNNER</summary>
public virtual void TestAll()
{
Assert.True("Number of tests has to be greater then zero", testsFromConfigFile
.Count > 0);
Log.Info("TestAll");
// Run the tests defined in the testConf.xml config file.
for (int index = 0; index < testsFromConfigFile.Count; index++)
{
CLITestData testdata = testsFromConfigFile[index];
// Execute the test commands
AList <CLICommand> testCommands = testdata.GetTestCommands();
CommandExecutor.Result cmdResult = null;
foreach (CLICommand cmd in testCommands)
{
try
{
cmdResult = Execute(cmd);
}
catch (Exception e)
{
NUnit.Framework.Assert.Fail(StringUtils.StringifyException(e));
}
}
bool overallTCResult = true;
// Run comparators
AList <ComparatorData> compdata = testdata.GetComparatorData();
foreach (ComparatorData cd in compdata)
{
string comptype = cd.GetComparatorType();
bool compareOutput = false;
if (!Runtime.EqualsIgnoreCase(comptype, "none"))
{
compareOutput = CompareTestOutput(cd, cmdResult);
overallTCResult &= compareOutput;
}
cd.SetExitCode(cmdResult.GetExitCode());
cd.SetActualOutput(cmdResult.GetCommandOutput());
cd.SetTestResult(compareOutput);
}
testdata.SetTestResult(overallTCResult);
// Execute the cleanup commands
AList <CLICommand> cleanupCommands = testdata.GetCleanupCommands();
foreach (CLICommand cmd_1 in cleanupCommands)
{
try
{
Execute(cmd_1);
}
catch (Exception e)
{
NUnit.Framework.Assert.Fail(StringUtils.StringifyException(e));
}
}
}
}
/// <summary>Display the summarized results</summary>
private void DisplayResults()
{
Log.Info("Detailed results:");
Log.Info("----------------------------------\n");
for (int i = 0; i < testsFromConfigFile.Count; i++)
{
CLITestData td = testsFromConfigFile[i];
bool testResult = td.GetTestResult();
// Display the details only if there is a failure
if (!testResult)
{
Log.Info("-------------------------------------------");
Log.Info(" Test ID: [" + (i + 1) + "]");
Log.Info(" Test Description: [" + td.GetTestDesc() + "]");
Log.Info(string.Empty);
AList <CLICommand> testCommands = td.GetTestCommands();
foreach (CLICommand cmd in testCommands)
{
Log.Info(" Test Commands: [" + ExpandCommand(cmd.GetCmd()) + "]");
}
Log.Info(string.Empty);
AList <CLICommand> cleanupCommands = td.GetCleanupCommands();
foreach (CLICommand cmd_1 in cleanupCommands)
{
Log.Info(" Cleanup Commands: [" + ExpandCommand(cmd_1.GetCmd()) + "]");
}
Log.Info(string.Empty);
AList <ComparatorData> compdata = td.GetComparatorData();
foreach (ComparatorData cd in compdata)
{
bool resultBoolean = cd.GetTestResult();
Log.Info(" Comparator: [" + cd.GetComparatorType() + "]");
Log.Info(" Comparision result: [" + (resultBoolean ? "pass" : "fail") +
"]");
Log.Info(" Expected output: [" + ExpandCommand(cd.GetExpectedOutput(
)) + "]");
Log.Info(" Actual output: [" + cd.GetActualOutput() + "]");
}
Log.Info(string.Empty);
}
}
Log.Info("Summary results:");
Log.Info("----------------------------------\n");
bool overallResults = true;
int totalPass = 0;
int totalFail = 0;
int totalComparators = 0;
for (int i_1 = 0; i_1 < testsFromConfigFile.Count; i_1++)
{
CLITestData td = testsFromConfigFile[i_1];
totalComparators += testsFromConfigFile[i_1].GetComparatorData().Count;
bool resultBoolean = td.GetTestResult();
if (resultBoolean)
{
totalPass++;
}
else
{
totalFail++;
}
overallResults &= resultBoolean;
}
Log.Info(" Testing mode: " + testMode);
Log.Info(string.Empty);
Log.Info(" Overall result: " + (overallResults ? "+++ PASS +++" : "--- FAIL ---"
));
if ((totalPass + totalFail) == 0)
{
Log.Info(" # Tests pass: "******" # Tests fail: " + 0);
}
else
{
Log.Info(" # Tests pass: "******" (" + (100 * totalPass /
(totalPass + totalFail)) + "%)");
Log.Info(" # Tests fail: " + totalFail + " (" + (100 * totalFail /
(totalPass + totalFail)) + "%)");
}
Log.Info(" # Validations done: " + totalComparators + " (each test may do multiple validations)"
);
Log.Info(string.Empty);
Log.Info("Failing tests:");
Log.Info("--------------");
int i_2 = 0;
bool foundTests = false;
for (i_2 = 0; i_2 < testsFromConfigFile.Count; i_2++)
{
bool resultBoolean = testsFromConfigFile[i_2].GetTestResult();
if (!resultBoolean)
{
Log.Info((i_2 + 1) + ": " + testsFromConfigFile[i_2].GetTestDesc());
foundTests = true;
}
}
if (!foundTests)
{
Log.Info("NONE");
}
foundTests = false;
Log.Info(string.Empty);
Log.Info("Passing tests:");
Log.Info("--------------");
for (i_2 = 0; i_2 < testsFromConfigFile.Count; i_2++)
{
bool resultBoolean = testsFromConfigFile[i_2].GetTestResult();
if (resultBoolean)
{
Log.Info((i_2 + 1) + ": " + testsFromConfigFile[i_2].GetTestDesc());
foundTests = true;
}
}
if (!foundTests)
{
Log.Info("NONE");
}
Assert.True("One of the tests failed. " + "See the Detailed results to identify "
+ "the command that failed", overallResults);
}