public bool TestsSucceeded(Log listener_log, bool timed_out, bool crashed)
{
string log;
using (var reader = listener_log.GetReader())
log = reader.ReadToEnd();
// parsing the result is different if we are in jenkins or not.
// When in Jenkins, Touch.Unit produces an xml file instead of a console log (so that we can get better test reporting).
// However, for our own reporting, we still want the console-based log. This log is embedded inside the xml produced
// by Touch.Unit, so we need to extract it and write it to disk. We also need to re-save the xml output, since Touch.Unit
// wraps the NUnit xml output with additional information, which we need to unwrap so that Jenkins understands it.
if (Harness.InJenkins)
{
// we have to parse the xml result
crashed = false;
var xmldoc = new XmlDocument();
try {
xmldoc.LoadXml(log);
var nunit_output = xmldoc.SelectSingleNode("/TouchUnitTestRun/NUnitOutput");
var xmllog = nunit_output.InnerXml;
var extra_output = xmldoc.SelectSingleNode("/TouchUnitTestRun/TouchUnitExtraData");
log = extra_output.InnerText;
File.WriteAllText(listener_log.FullPath, log);
var testsResults = new XmlDocument();
testsResults.LoadXml(xmllog);
var mainResultNode = testsResults.SelectSingleNode("test-results");
if (mainResultNode == null)
{
Harness.LogWrench($"Node is null.");
}
else
{
// update the information of the main node to add information about the mode and the test that is excuted. This will later create
// nicer reports in jenkins
mainResultNode.Attributes ["name"].Value = Target.AsString();
// store a clean version of the logs, later this will be used by the bots to show results in github/web
var path = listener_log.FullPath;
path = Path.ChangeExtension(path, "xml");
testsResults.Save(path);
Logs.AddFile(path, "Test xml");
}
} catch (Exception e) {
main_log.WriteLine("Could not parse xml result file: {0}", e);
if (timed_out)
{
Harness.LogWrench($"@MonkeyWrench: AddSummary: <b><i>{mode} timed out</i></b><br/>");
return(false);
}
else
{
Harness.LogWrench($"@MonkeyWrench: AddSummary: <b><i>{mode} crashed</i></b><br/>");
main_log.WriteLine("Test run crashed");
crashed = true;
return(false);
}
}
}
// parsing the human readable results
if (log.Contains("Tests run"))
{
var tests_run = string.Empty;
var log_lines = log.Split('\n');
var failed = false;
foreach (var line in log_lines)
{
if (line.Contains("Tests run:"))
{
Console.WriteLine(line);
tests_run = line.Replace("Tests run: ", "");
break;
}
else if (line.Contains("FAIL"))
{
Console.WriteLine(line);
failed = true;
}
}
if (failed)
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b>{0} failed: {1}</b><br/>", mode, tests_run);
main_log.WriteLine("Test run failed");
return(false);
}
else
{
Harness.LogWrench("@MonkeyWrench: AddSummary: {0} succeeded: {1}<br/>", mode, tests_run);
main_log.WriteLine("Test run succeeded");
return(true);
}
}
else if (timed_out)
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} timed out</i></b><br/>", mode);
return(false);
}
else
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} crashed</i></b><br/>", mode);
main_log.WriteLine("Test run crashed");
crashed = true;
return(false);
}
}
public async Task EndCaptureAsync(TimeSpan timeout)
{
// Check for crash reports
var crash_report_search_done = false;
var crash_report_search_timeout = timeout.TotalSeconds;
var watch = new Stopwatch();
watch.Start();
do
{
var end_crashes = await Harness.CreateCrashReportsSnapshotAsync(Log, !Device, DeviceName);
end_crashes.ExceptWith(InitialSet);
Reports = end_crashes;
if (end_crashes.Count > 0)
{
Log.WriteLine("Found {0} new crash report(s)", end_crashes.Count);
List <LogFile> crash_reports;
if (!Device)
{
crash_reports = new List <LogFile> (end_crashes.Count);
foreach (var path in end_crashes)
{
Logs.AddFile(path, $"Crash report: {Path.GetFileName (path)}");
}
}
else
{
// Download crash reports from the device. We put them in the project directory so that they're automatically deleted on wrench
// (if we put them in /tmp, they'd never be deleted).
var downloaded_crash_reports = new List <LogFile> ();
foreach (var file in end_crashes)
{
var name = Path.GetFileName(file);
var crash_report_target = Logs.Create(name, $"Crash report: {name}");
var sb = new StringBuilder();
sb.Append(" --download-crash-report=").Append(StringUtils.Quote(file));
sb.Append(" --download-crash-report-to=").Append(StringUtils.Quote(crash_report_target.Path));
sb.Append(" --sdkroot ").Append(StringUtils.Quote(Harness.XcodeRoot));
if (!string.IsNullOrEmpty(DeviceName))
{
sb.Append(" --devname ").Append(StringUtils.Quote(DeviceName));
}
var result = await ProcessHelper.ExecuteCommandAsync(Harness.MlaunchPath, sb.ToString(), Log, TimeSpan.FromMinutes(1));
if (result.Succeeded)
{
Log.WriteLine("Downloaded crash report {0} to {1}", file, crash_report_target.Path);
crash_report_target = await Harness.SymbolicateCrashReportAsync(Logs, Log, crash_report_target);
downloaded_crash_reports.Add(crash_report_target);
}
else
{
Log.WriteLine("Could not download crash report {0}", file);
}
}
crash_reports = downloaded_crash_reports;
}
foreach (var cp in crash_reports)
{
Harness.LogWrench("@MonkeyWrench: AddFile: {0}", cp.Path);
Log.WriteLine(" {0}", cp.Path);
}
crash_report_search_done = true;
}
else
{
if (watch.Elapsed.TotalSeconds > crash_report_search_timeout)
{
crash_report_search_done = true;
}
else
{
Log.WriteLine("No crash reports, waiting a second to see if the crash report service just didn't complete in time ({0})", (int)(crash_report_search_timeout - watch.Elapsed.TotalSeconds));
Thread.Sleep(TimeSpan.FromSeconds(1));
}
}
} while (!crash_report_search_done);
}
public bool TestsSucceeded(Log listener_log, bool timed_out, bool crashed)
{
// parsing the result is different if we are in jenkins or not.
// When in Jenkins, Touch.Unit produces an xml file instead of a console log (so that we can get better test reporting).
// However, for our own reporting, we still want the console-based log. This log is embedded inside the xml produced
// by Touch.Unit, so we need to extract it and write it to disk. We also need to re-save the xml output, since Touch.Unit
// wraps the NUnit xml output with additional information, which we need to unwrap so that Jenkins understands it.
if (Harness.InJenkins)
{
// use a tmp file so that we can use the reader in xml and write the humman version
var tmpFile = Path.GetTempFileName();
// we have to parse the xml result
crashed = false;
var xmldoc = new XmlDocument();
XmlNode mainResultNode;
try {
using (var reader = listener_log.GetReader()) {
xmldoc.Load(reader);
var testsResults = new XmlDocument();
using (var writer = new StreamWriter(tmpFile)) {
if (IsTouchUnitResult(xmldoc))
{
testsResults = ParseTouchUnitXml(xmldoc, writer);
}
else
{
testsResults = ParseNUnitXml(xmldoc, writer);
}
}
mainResultNode = testsResults.SelectSingleNode("test-results");
}
if (mainResultNode == null)
{
Harness.LogWrench($"Node is null.");
}
else
{
// update the information of the main node to add information about the mode and the test that is excuted. This will later create
// nicer reports in jenkins
mainResultNode.Attributes ["name"].Value = Target.AsString();
// store a clean version of the logs, later this will be used by the bots to show results in github/web
var path = listener_log.FullPath;
path = Path.ChangeExtension(path, "xml");
// we already have all the data needed in the listerner, rather than saving from the doc, copy
File.Copy(listener_log.FullPath, path, true);
Logs.AddFile(path, "Test xml");
}
// write on the log
File.Copy(tmpFile, listener_log.FullPath, true);
File.Delete(tmpFile);
} catch (Exception e) {
main_log.WriteLine("Could not parse xml result file: {0}", e);
if (timed_out)
{
Harness.LogWrench($"@MonkeyWrench: AddSummary: <b><i>{mode} timed out</i></b><br/>");
return(false);
}
else
{
Harness.LogWrench($"@MonkeyWrench: AddSummary: <b><i>{mode} crashed</i></b><br/>");
main_log.WriteLine("Test run crashed");
crashed = true;
return(false);
}
}
}
// read until the end and decide if we got results, do not store them
// we do not want to use too much memory
string resultLine = null;
using (var reader = new StreamReader(listener_log.FullPath)) {
string line;
while ((line = reader.ReadLine()) != null)
{
if (line.Contains("Tests run"))
{
resultLine = line;
}
}
}
// read the parsed logs in a human readable way
if (resultLine != null)
{
var tests_run = string.Empty;
var failed = false;
using (var reader = listener_log.GetReader()) {
string line;
while ((line = reader.ReadLine()) != null)
{
if (line.Contains("Tests run:"))
{
Console.WriteLine(line);
tests_run = line.Replace("Tests run: ", "");
break;
}
else if (line.Contains("[FAIL]"))
{
Console.WriteLine(line);
failed = true;
}
}
}
if (failed)
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b>{0} failed: {1}</b><br/>", mode, tests_run);
main_log.WriteLine("Test run failed");
return(false);
}
else
{
Harness.LogWrench("@MonkeyWrench: AddSummary: {0} succeeded: {1}<br/>", mode, tests_run);
main_log.WriteLine("Test run succeeded");
return(true);
}
}
else if (timed_out)
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} timed out</i></b><br/>", mode);
return(false);
}
else
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} crashed</i></b><br/>", mode);
main_log.WriteLine("Test run crashed");
crashed = true;
return(false);
}
}
