/// <summary>
/// Checks the output produced when the converted solution
/// was run by cygwin.
/// </summary>
private static TestResults checkResults(SolutionInfo solutionInfo)
{
TestResults results = new TestResults();
results.Result = TestResults.PassFail.PASSED;
// The solution folder should contain a file called testExpectedResults.txt.
// This contains lines like:
// [output-file-name] = [expected-output]
// (There may be multiple lines so that we can test multiple configurations
// of the solutions.)
// We read each line from the expected-results file...
string expectedResultsFilename = String.Format("{0}/testExpectedResults.txt", solutionInfo.Folder);
string[] lines = File.ReadAllLines(expectedResultsFilename);
foreach (string line in lines)
{
// We get the filename and expected result from the line...
List <string> tokens = Utils.split(line, '=');
if (tokens.Count != 2)
{
throw new Exception(String.Format("Lines should be in the format [output-file-name] = [expected-output]. File={0}", expectedResultsFilename));
}
string file = solutionInfo.Folder + "/" + tokens[0].Trim();
string expectedResult = tokens[1].Trim();
// We read the data from the output file, and compare it
// with the expected results...
string actualResult = File.ReadAllText(file);
if (actualResult != expectedResult)
{
results.Result = TestResults.PassFail.FAILED;
results.Description += String.Format("Expected '{0}', got '{1}'.", expectedResult, actualResult);
}
}
return(results);
}
/// <summary>
/// Checks the output produced when the converted solution
/// was run by cygwin.
/// </summary>
private static TestResults checkResults(SolutionInfo solutionInfo)
{
TestResults results = new TestResults();
results.Result = TestResults.PassFail.PASSED;
// The solution folder should contain a file called testExpectedResults.txt.
// This contains lines like:
// [output-file-name] = [expected-output]
// (There may be multiple lines so that we can test multiple configurations
// of the solutions.)
// We read each line from the expected-results file...
string expectedResultsFilename = String.Format("{0}/testExpectedResults.txt", solutionInfo.Folder);
string[] lines = File.ReadAllLines(expectedResultsFilename);
foreach (string line in lines)
{
// We get the filename and expected result from the line...
List<string> tokens = Utils.split(line, '=');
if (tokens.Count != 2)
{
throw new Exception(String.Format("Lines should be in the format [output-file-name] = [expected-output]. File={0}", expectedResultsFilename) );
}
string file = solutionInfo.Folder + "/" + tokens[0].Trim();
string expectedResult = tokens[1].Trim();
// We read the data from the output file, and compare it
// with the expected results...
string actualResult = File.ReadAllText(file);
if (actualResult != expectedResult)
{
results.Result = TestResults.PassFail.FAILED;
results.Description += String.Format("Expected '{0}', got '{1}'.", expectedResult, actualResult);
}
}
return results;
}
public static int Main(string[] args)
{
Console.WriteLine("PlayScript TestRunner");
// Process arguments
for (int arg = 0; arg < args.Length; arg++)
{
if (args [arg].StartsWith("-v:"))
{
int.TryParse(args[arg].Substring(3), out Verbosity);
}
else if (args [arg].ToLower().EndsWith(".xml"))
{
TestXmlFile = args [arg];
}
}
// Set the current working directory to the directory the test.xml file is in
string fullTestXmlPath = Path.GetFullPath(TestXmlFile);
string testDir = Path.GetDirectoryName(fullTestXmlPath);
Directory.SetCurrentDirectory(testDir);
if (!SetupBinFolder())
{
return(ERROR);
}
TestConfig.Load(fullTestXmlPath);
var childNodes = TestConfig ["tests"].ChildNodes;
foreach (XmlNode folderNode in childNodes)
{
if (folderNode is XmlElement && ((XmlElement)folderNode).Name == "folder")
{
XmlElement folderElem = (XmlElement)folderNode;
string path = folderElem.Attributes ["path"].Value;
if (path == null)
{
Console.WriteLine("ERROR: No 'path' attribute found in <folder> tag.");
}
Console.WriteLine("Folder: {0}", path);
var folderChildNodes = folderElem.ChildNodes;
foreach (XmlNode testNode in folderChildNodes)
{
if (testNode is XmlElement && ((XmlElement)testNode).Name == "test")
{
XmlElement testElem = (XmlElement)testNode;
TestResults results = RunTest(path, testElem);
string passFail = results.TestPassed ? "[PASSED]" : "[FAILED]";
if (results.TestPassed)
{
TotalPassed++;
}
else
{
TotalFailed++;
}
Console.WriteLine(" {0} Test '{1}': Passed {2} Failed {3} ExpectedFailed {4} Skipped {5}",
passFail, results.TestName, results.Passed, results.Failed, results.ExpectedFailed, results.Skipped);
}
else
{
Console.WriteLine("Error, unrecognized XML tag " + testNode.Name);
}
}
}
else
{
Console.WriteLine("Error, unrecognized XML tag " + folderNode.Name);
}
}
Console.WriteLine("Total Passed {0} Total Failed {1}", TotalPassed, TotalFailed);
if (TotalFailed > 0)
{
Console.WriteLine("RESULT: FAILED!");
}
else
{
Console.WriteLine("RESULT: PASSED!");
}
return(TotalFailed > 0 ? ERROR : OK);
}
public static TestResults RunTest(string path, XmlElement test)
{
TestResults results = new TestResults();
Results.Add(results);
string testName = test.Attributes ["testName"].Value;
results.TestName = testName;
if (test.Attributes["expectedFailed"] != null)
{
results.ExpectedFailed = Convert.ToInt32(test.Attributes["expectedFailed"].Value, 10);
}
string extraFilesPath = Path.GetFullPath(Path.Combine(path, testName));
string[] extraFiles = {};
if (Directory.Exists(extraFilesPath))
{
extraFiles = Directory.GetFiles(extraFilesPath, "*.*");
}
string outputPath = Path.GetFullPath(Path.Combine(BinFolder, "test.exe"));
if (BuildTest(path, testName, extraFiles, outputPath) == ERROR)
{
results.TestPassed = false;
return(results);
}
if (RunCommand(Path.GetFullPath(MonoPath), outputPath) == ERROR)
{
results.TestPassed = false;
}
else
{
results.TestPassed = true;
}
if (Output != null)
{
int passedPos = Output.LastIndexOf("Passed: ");
if (passedPos != -1)
{
int.TryParse(Output.Substring(passedPos + 8, 3), out results.Passed);
}
int failedPos = Output.LastIndexOf("Failed: ");
if (failedPos != -1)
{
int.TryParse(Output.Substring(failedPos + 8, 3), out results.Failed);
}
int skippedPos = Output.LastIndexOf("Skipped: ");
if (skippedPos != -1)
{
int.TryParse(Output.Substring(skippedPos + 9, 3), out results.Skipped);
}
// Allow failures if we expect them..
if (results.Failed > 1 && results.Failed < results.ExpectedFailed)
{
results.TestPassed = true;
}
}
if (((Verbosity > 0 && results.TestPassed == false) || (Verbosity > 1)) && Output != null)
{
Console.WriteLine(Output);
}
return(results);
}
/// <summary>
/// Runs a test on a worker thread.
/// </summary>
private void runTestOnWorkerThread(object context)
{
try
{
// We test one solution:
// - Convert it with MakeItSo
// - Run a bash script with cygwin that builds and runs it
// - Check the output
SolutionInfo solutionInfo = (SolutionInfo)context;
TestResults results;
try
{
// We run MakeItSo for this solution...
runMakeItSo(solutionInfo);
// We build and run the project using cygwin...
cygwinBuildAndRun(solutionInfo);
// We check the expected results...
results = checkResults(solutionInfo);
}
catch (Exception ex)
{
results = new TestResults();
results.Result = TestResults.PassFail.FAILED;
results.Description = "Exception caught: " + ex.Message;
}
// We update the display with the results...
switch (results.Result)
{
case TestResults.PassFail.PASSED:
solutionInfo.TestResult = "PASS";
solutionInfo.BackgroundColor = Color.LightGreen;
break;
case TestResults.PassFail.FAILED:
solutionInfo.TestResult = String.Format("FAIL ({0})", results.Description);
solutionInfo.BackgroundColor = Color.LightPink;
break;
}
// We notify the UI thread that the test has completed,
// and that it can update the screen...
Utils.raiseEvent(TestCompleted, this, null);
}
catch (Exception ex)
{
MessageBox.Show(ex.Message, "TestRunner");
}
}
public static TestResults RunTest(string path, XmlElement test)
{
TestResults results = new TestResults ();
Results.Add (results);
string testName = test.Attributes ["testName"].Value;
results.TestName = testName;
if (test.Attributes["expectedFailed"] != null) {
results.ExpectedFailed = Convert.ToInt32 (test.Attributes["expectedFailed"].Value, 10);
}
string extraFilesPath = Path.GetFullPath(Path.Combine (path, testName));
string[] extraFiles = {};
if (Directory.Exists (extraFilesPath)) {
extraFiles = Directory.GetFiles (extraFilesPath, "*.*");
}
string outputPath = Path.GetFullPath (Path.Combine (BinFolder, "test.exe"));
if (BuildTest (path, testName, extraFiles, outputPath) == ERROR) {
results.TestPassed = false;
return results;
}
if (RunCommand (Path.GetFullPath(MonoPath), outputPath) == ERROR) {
results.TestPassed = false;
} else {
results.TestPassed = true;
}
if (Output != null) {
int passedPos = Output.LastIndexOf ("Passed: ");
if (passedPos != -1) {
int.TryParse (Output.Substring (passedPos + 8, 3), out results.Passed);
}
int failedPos = Output.LastIndexOf ("Failed: ");
if (failedPos != -1) {
int.TryParse (Output.Substring (failedPos + 8, 3), out results.Failed);
}
int skippedPos = Output.LastIndexOf ("Skipped: ");
if (skippedPos != -1) {
int.TryParse (Output.Substring (skippedPos + 9, 3), out results.Skipped);
}
// Allow failures if we expect them..
if (results.Failed > 1 && results.Failed < results.ExpectedFailed)
results.TestPassed = true;
}
if (((Verbosity > 0 && results.TestPassed == false) || (Verbosity > 1)) && Output != null) {
Console.WriteLine (Output);
}
return results;
}
