public bool TestsSucceeded(Log listener_log, bool timed_out, bool crashed)
{
// parsing the result is different if we are in jenkins or not.
// When in Jenkins, Touch.Unit produces an xml file instead of a console log (so that we can get better test reporting).
// However, for our own reporting, we still want the console-based log. This log is embedded inside the xml produced
// by Touch.Unit, so we need to extract it and write it to disk. We also need to re-save the xml output, since Touch.Unit
// wraps the NUnit xml output with additional information, which we need to unwrap so that Jenkins understands it.
if (Harness.InJenkins)
{
// use a tmp file so that we can use the reader in xml and write the humman version
var tmpFile = Path.GetTempFileName();
// we have to parse the xml result
crashed = false;
var xmldoc = new XmlDocument();
XmlNode mainResultNode;
try {
using (var reader = listener_log.GetReader()) {
xmldoc.Load(reader);
var testsResults = new XmlDocument();
using (var writer = new StreamWriter(tmpFile)) {
if (IsTouchUnitResult(xmldoc))
{
testsResults = ParseTouchUnitXml(xmldoc, writer);
}
else
{
testsResults = ParseNUnitXml(xmldoc, writer);
}
}
mainResultNode = testsResults.SelectSingleNode("test-results");
}
if (mainResultNode == null)
{
Harness.LogWrench($"Node is null.");
}
else
{
// update the information of the main node to add information about the mode and the test that is excuted. This will later create
// nicer reports in jenkins
mainResultNode.Attributes ["name"].Value = Target.AsString();
// store a clean version of the logs, later this will be used by the bots to show results in github/web
var path = listener_log.FullPath;
path = Path.ChangeExtension(path, "xml");
// we already have all the data needed in the listerner, rather than saving from the doc, copy
File.Copy(listener_log.FullPath, path, true);
Logs.AddFile(path, "Test xml");
}
// write on the log
File.Copy(tmpFile, listener_log.FullPath, true);
File.Delete(tmpFile);
} catch (Exception e) {
main_log.WriteLine("Could not parse xml result file: {0}", e);
if (timed_out)
{
Harness.LogWrench($"@MonkeyWrench: AddSummary: <b><i>{mode} timed out</i></b><br/>");
return(false);
}
else
{
Harness.LogWrench($"@MonkeyWrench: AddSummary: <b><i>{mode} crashed</i></b><br/>");
main_log.WriteLine("Test run crashed");
crashed = true;
return(false);
}
}
}
// read until the end and decide if we got results, do not store them
// we do not want to use too much memory
string resultLine = null;
using (var reader = new StreamReader(listener_log.FullPath)) {
string line;
while ((line = reader.ReadLine()) != null)
{
if (line.Contains("Tests run"))
{
resultLine = line;
}
}
}
// read the parsed logs in a human readable way
if (resultLine != null)
{
var tests_run = string.Empty;
var failed = false;
using (var reader = listener_log.GetReader()) {
string line;
while ((line = reader.ReadLine()) != null)
{
if (line.Contains("Tests run:"))
{
Console.WriteLine(line);
tests_run = line.Replace("Tests run: ", "");
break;
}
else if (line.Contains("[FAIL]"))
{
Console.WriteLine(line);
failed = true;
}
}
}
if (failed)
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b>{0} failed: {1}</b><br/>", mode, tests_run);
main_log.WriteLine("Test run failed");
return(false);
}
else
{
Harness.LogWrench("@MonkeyWrench: AddSummary: {0} succeeded: {1}<br/>", mode, tests_run);
main_log.WriteLine("Test run succeeded");
return(true);
}
}
else if (timed_out)
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} timed out</i></b><br/>", mode);
return(false);
}
else
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} crashed</i></b><br/>", mode);
main_log.WriteLine("Test run crashed");
crashed = true;
return(false);
}
}
public bool TestsSucceeded(Log listener_log, bool timed_out, bool crashed)
{
string log;
using (var reader = listener_log.GetReader())
log = reader.ReadToEnd();
// parsing the result is different if we are in jenkins or not.
// When in Jenkins, Touch.Unit produces an xml file instead of a console log (so that we can get better test reporting).
// However, for our own reporting, we still want the console-based log. This log is embedded inside the xml produced
// by Touch.Unit, so we need to extract it and write it to disk. We also need to re-save the xml output, since Touch.Unit
// wraps the NUnit xml output with additional information, which we need to unwrap so that Jenkins understands it.
if (Harness.InJenkins)
{
// we have to parse the xml result
crashed = false;
var xmldoc = new XmlDocument();
try {
xmldoc.LoadXml(log);
var nunit_output = xmldoc.SelectSingleNode("/TouchUnitTestRun/NUnitOutput");
var xmllog = nunit_output.InnerXml;
var extra_output = xmldoc.SelectSingleNode("/TouchUnitTestRun/TouchUnitExtraData");
log = extra_output.InnerText;
File.WriteAllText(listener_log.FullPath, log);
var testsResults = new XmlDocument();
testsResults.LoadXml(xmllog);
var mainResultNode = testsResults.SelectSingleNode("test-results");
if (mainResultNode == null)
{
Harness.LogWrench($"Node is null.");
}
else
{
// update the information of the main node to add information about the mode and the test that is excuted. This will later create
// nicer reports in jenkins
mainResultNode.Attributes ["name"].Value = Target.AsString();
// store a clean version of the logs, later this will be used by the bots to show results in github/web
var path = listener_log.FullPath;
path = Path.ChangeExtension(path, "xml");
testsResults.Save(path);
Logs.AddFile(path, "Test xml");
}
} catch (Exception e) {
main_log.WriteLine("Could not parse xml result file: {0}", e);
if (timed_out)
{
Harness.LogWrench($"@MonkeyWrench: AddSummary: <b><i>{mode} timed out</i></b><br/>");
return(false);
}
else
{
Harness.LogWrench($"@MonkeyWrench: AddSummary: <b><i>{mode} crashed</i></b><br/>");
main_log.WriteLine("Test run crashed");
crashed = true;
return(false);
}
}
}
// parsing the human readable results
if (log.Contains("Tests run"))
{
var tests_run = string.Empty;
var log_lines = log.Split('\n');
var failed = false;
foreach (var line in log_lines)
{
if (line.Contains("Tests run:"))
{
Console.WriteLine(line);
tests_run = line.Replace("Tests run: ", "");
break;
}
else if (line.Contains("FAIL"))
{
Console.WriteLine(line);
failed = true;
}
}
if (failed)
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b>{0} failed: {1}</b><br/>", mode, tests_run);
main_log.WriteLine("Test run failed");
return(false);
}
else
{
Harness.LogWrench("@MonkeyWrench: AddSummary: {0} succeeded: {1}<br/>", mode, tests_run);
main_log.WriteLine("Test run succeeded");
return(true);
}
}
else if (timed_out)
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} timed out</i></b><br/>", mode);
return(false);
}
else
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} crashed</i></b><br/>", mode);
main_log.WriteLine("Test run crashed");
crashed = true;
return(false);
}
}
public bool TestsSucceeded(LogStream listener_log, bool timed_out, bool crashed)
{
string log;
using (var reader = listener_log.GetReader())
log = reader.ReadToEnd();
// parsing the result is different if we are in jenkins or nor.
if (Harness.InJenkins)
{
// we have to parse the xml result
crashed = false;
if (log.Contains("test-results"))
{
// remove any possible extra info
var index = log.IndexOf("<test-results");
var header = log.Substring(0, log.IndexOf('<'));
log = log.Remove(0, index - 1);
var testsResults = new XmlDocument();
testsResults.LoadXml(log);
var mainResultNode = testsResults.SelectSingleNode("test-results");
if (mainResultNode == null)
{
Harness.LogWrench($"Node is null.");
crashed = true;
return(false);
}
// update the information of the main node to add information about the mode and the test that is excuted. This will later create
// nicer reports in jenkins
mainResultNode.Attributes["name"].Value = Target.AsString();
// store a clean version of the logs, later this will be used by the bots to show results in github/web
var path = listener_log.FullPath;
path = path.Replace(".log", ".xml");
testsResults.Save(path);
Logs.Add(new LogFile("Test xml", path));
// we want to keep the old TestResult page,
GenerateHumanReadableLogs(listener_log.FullPath, header, testsResults);
int ignored = Convert.ToInt16(mainResultNode.Attributes["ignored"].Value);
int invalid = Convert.ToInt16(mainResultNode.Attributes["invalid"].Value);
int inconclusive = Convert.ToInt16(mainResultNode.Attributes["inconclusive"].Value);
int errors = Convert.ToInt16(mainResultNode.Attributes["errors"].Value);
int failures = Convert.ToInt16(mainResultNode.Attributes["failures"].Value);
int totalTests = Convert.ToInt16(mainResultNode.Attributes["total"].Value);
// generate human readable logs
var failed = errors != 0 || failures != 0;
if (failed)
{
Harness.LogWrench($"@MonkeyWrench: AddSummary: <b>{mode} failed: Test run: {totalTests} Passed: {totalTests - invalid - inconclusive - ignored} Inconclusive: {inconclusive} Failed: {errors + failures} Ignored: {ignored}</b><br/>");
main_log.WriteLine("Test run failed");
return(false);
}
else
{
Harness.LogWrench($"@MonkeyWrench: AddSummary: {mode} succeeded: Test run: {totalTests} Passed: {totalTests - invalid - inconclusive - ignored} Inconclusive: {inconclusive} Failed: 0 Ignored: {ignored}<br/>");
main_log.WriteLine("Test run succeeded");
return(true);
}
}
else if (timed_out)
{
Harness.LogWrench($"@MonkeyWrench: AddSummary: <b><i>{mode} timed out</i></b><br/>");
return(false);
}
else
{
Harness.LogWrench($"@MonkeyWrench: AddSummary: <b><i>{mode} crashed</i></b><br/>");
main_log.WriteLine("Test run crashed");
crashed = true;
return(false);
}
}
else
{
// parsing the human readable results
if (log.Contains("Tests run"))
{
var tests_run = string.Empty;
var log_lines = log.Split('\n');
var failed = false;
foreach (var line in log_lines)
{
if (line.Contains("Tests run:"))
{
Console.WriteLine(line);
tests_run = line.Replace("Tests run: ", "");
break;
}
else if (line.Contains("FAIL"))
{
Console.WriteLine(line);
failed = true;
}
}
if (failed)
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b>{0} failed: {1}</b><br/>", mode, tests_run);
main_log.WriteLine("Test run failed");
return(false);
}
else
{
Harness.LogWrench("@MonkeyWrench: AddSummary: {0} succeeded: {1}<br/>", mode, tests_run);
main_log.WriteLine("Test run succeeded");
return(true);
}
}
else if (timed_out)
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} timed out</i></b><br/>", mode);
return(false);
}
else
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} crashed</i></b><br/>", mode);
main_log.WriteLine("Test run crashed");
crashed = true;
return(false);
}
}
}