private async Task <(bool Succeeded, bool Crashed, string ResultLine)> TestsSucceeded(AppBundleInformation appInfo, string test_log_path, bool timed_out)
{
var(resultLine, failed, crashed) = await ParseResultFile(appInfo, test_log_path, timed_out);
// read the parsed logs in a human readable way
if (resultLine != null)
{
var tests_run = resultLine.Replace("Tests run: ", "");
if (failed)
{
WrenchLog.WriteLine("AddSummary: <b>{0} failed: {1}</b><br/>", _runMode, tests_run);
_mainLog.WriteLine("Test run failed");
return(false, crashed, resultLine);
}
else
{
WrenchLog.WriteLine("AddSummary: {0} succeeded: {1}<br/>", _runMode, tests_run);
_mainLog.WriteLine("Test run succeeded");
return(true, crashed, resultLine);
}
}
else if (timed_out)
{
WrenchLog.WriteLine("AddSummary: <b><i>{0} timed out</i></b><br/>", _runMode);
_mainLog.WriteLine("Test run timed out");
return(false, false, "Test run timed out");
}
else
{
WrenchLog.WriteLine("AddSummary: <b><i>{0} crashed</i></b><br/>", _runMode);
_mainLog.WriteLine("Test run crashed");
return(false, true, "Test run crashed");
}
}
public async Task EndCaptureAsync(TimeSpan timeout)
{
// Check for crash reports
var stopwatch = Stopwatch.StartNew();
do
{
var newCrashFiles = await CreateCrashReportsSnapshotAsync();
newCrashFiles.ExceptWith(_initialCrashes);
if (newCrashFiles.Count == 0)
{
if (stopwatch.Elapsed.TotalSeconds > timeout.TotalSeconds)
{
break;
}
else
{
_log.WriteLine(
"No crash reports, waiting a second to see if the crash report service just didn't complete in time ({0})",
(int)(timeout.TotalSeconds - stopwatch.Elapsed.TotalSeconds));
Thread.Sleep(TimeSpan.FromSeconds(1));
}
continue;
}
_log.WriteLine("Found {0} new crash report(s)", newCrashFiles.Count);
IEnumerable <IFileBackedLog> crashReports;
if (!_isDevice)
{
crashReports = new List <IFileBackedLog>(newCrashFiles.Count);
foreach (var path in newCrashFiles)
{
_logs.AddFile(path, $"Crash report: {Path.GetFileName(path)}");
}
}
else
{
// Download crash reports from the device. We put them in the project directory so that they're automatically deleted on wrench
// (if we put them in /tmp, they'd never be deleted).
crashReports = newCrashFiles
.Select(async crash => await ProcessCrash(crash))
.Select(t => t.Result)
.Where(c => c != null);
}
foreach (var cp in crashReports)
{
WrenchLog.WriteLine("AddFile: {0}", cp.FullPath);
_log.WriteLine(" {0}", cp.FullPath);
}
break;
} while (true);
}
public async Task <(TestExecutingResult ExecutingResult, string?ResultMessage)> ParseResult()
{
(TestExecutingResult ExecutingResult, string?ResultMessage)result = (ExecutingResult : TestExecutingResult.Finished, ResultMessage : null);
var crashed = false;
if (File.Exists(_listener.TestLog.FullPath))
{
WrenchLog.WriteLine("AddFile: {0}", _listener.TestLog.FullPath);
(Success, crashed, result.ResultMessage) = await TestsSucceeded(_appInfo, _listener.TestLog.FullPath, _timedout);
}
else if (_timedout)
{
WrenchLog.WriteLine("AddSummary: <b><i>{0} never launched</i></b><br/>", _runMode);
_mainLog.WriteLine("Test run never launched");
result.ResultMessage = "Test runner never started";
Success = false;
}
else if (_launchFailure)
{
WrenchLog.WriteLine("AddSummary: <b><i>{0} failed to launch</i></b><br/>", _runMode);
_mainLog.WriteLine("Test run failed to launch");
result.ResultMessage = "Test runner failed to launch";
Success = false;
}
else
{
WrenchLog.WriteLine("AddSummary: <b><i>{0} crashed at startup (no log)</i></b><br/>", _runMode);
_mainLog.WriteLine("Test run crashed before it started (no log file produced)");
result.ResultMessage = "No test log file was produced";
crashed = true;
Success = false;
}
if (!Success.HasValue)
{
Success = false;
}
var crashLogWaitTime = 0;
if (!Success.Value)
{
crashLogWaitTime = 5;
}
if (crashed)
{
crashLogWaitTime = 30;
}
await _crashReporter.EndCaptureAsync(TimeSpan.FromSeconds(crashLogWaitTime));
if (_timedout)
{
result.ExecutingResult = TestExecutingResult.TimedOut;
}
else if (_launchFailure)
{
result.ExecutingResult = TestExecutingResult.LaunchFailure;
}
else if (_launchFailure)
{
result.ExecutingResult = TestExecutingResult.Crashed;
}
else if (Success.Value)
{
result.ExecutingResult = TestExecutingResult.Succeeded;
}
else
{
result.ExecutingResult = TestExecutingResult.Failed;
}
// Check crash reports to see if any of them explains why the test run crashed.
if (!Success.Value)
{
int pid = -1;
string?crashReason = null;
foreach (var crashLog in _crashLogs)
{
try
{
_logs.Add(crashLog);
if (pid == -1)
{
// Find the pid
pid = await GetPidFromMainLog();
}
GetCrashReason(pid, crashLog, out crashReason);
if (crashReason != null)
{
break;
}
}
catch (Exception e)
{
var message = string.Format("Failed to process crash report '{1}': {0}", e.Message, crashLog.Description);
_mainLog.WriteLine(message);
_exceptionLogger?.Invoke(2, message);
}
}
if (!string.IsNullOrEmpty(crashReason))
{
if (crashReason == "per-process-limit")
{
result.ResultMessage = "Killed due to using too much memory (per-process-limit).";
}
else
{
result.ResultMessage = $"Killed by the OS ({crashReason})";
}
}
else if (_launchFailure)
{
// same as with a crash
result.ResultMessage = $"Launch failure";
}
await GenerateXmlFailures(result.ResultMessage, crashed, crashReason);
}
return(result);
}
private async Task <(string?resultLine, bool failed, bool crashed)> ParseResultFile(AppBundleInformation appInfo, string test_log_path, bool timed_out)
{
(string?resultLine, bool failed, bool crashed)parseResult = (null, false, false);
if (!File.Exists(test_log_path))
{
parseResult.crashed = true; // if we do not have a log file, the test crashes
return(parseResult);
}
// parsing the result is different if we are in jenkins or not.
// When in Jenkins, Touch.Unit produces an xml file instead of a console log (so that we can get better test reporting).
// However, for our own reporting, we still want the console-based log. This log is embedded inside the xml produced
// by Touch.Unit, so we need to extract it and write it to disk. We also need to re-save the xml output, since Touch.Unit
// wraps the NUnit xml output with additional information, which we need to unwrap so that Jenkins understands it.
//
// On the other hand, the nunit and xunit do not have that data and have to be parsed.
//
// This if statement has a small trick, we found out that internet sharing in some of the bots (VSTS) does not work, in
// that case, we cannot do a TCP connection to xharness to get the log, this is a problem since if we did not get the xml
// from the TCP connection, we are going to fail when trying to read it and not parse it. Therefore, we are not only
// going to check if we are in CI, but also if the listener_log is valid.
var path = Path.ChangeExtension(test_log_path, "xml");
_resultParser.CleanXml(test_log_path, path);
if (ResultsUseXml && _resultParser.IsValidXml(path, out var xmlType))
{
try
{
var newFilename = _resultParser.GetXmlFilePath(path, xmlType);
// at this point, we have the test results, but we want to be able to have attachments in vsts, so if the format is
// the right one (NUnitV3) add the nodes. ATM only TouchUnit uses V3.
var testRunName = $"{appInfo.AppName} {appInfo.Variation}";
if (xmlType == XmlResultJargon.NUnitV3)
{
var logFiles = new List <string>();
// add our logs AND the logs of the previous task, which is the build task
logFiles.AddRange(Directory.GetFiles(_crashLogs.Directory));
if (_additionalLogsDirectory != null) // when using the run command, we do not have a build task, ergo, there are no logs to add.
{
logFiles.AddRange(Directory.GetFiles(_additionalLogsDirectory));
}
// add the attachments and write in the new filename
// add a final prefix to the file name to make sure that the VSTS test uploaded just pick
// the final version, else we will upload tests more than once
newFilename = XmlResultParser.GetVSTSFilename(newFilename);
_resultParser.UpdateMissingData(path, newFilename, testRunName, logFiles);
}
else
{
// rename the path to the correct value
File.Move(path, newFilename);
}
path = newFilename;
if (_generateHtml)
{
// write the human readable results in a tmp file, which we later use to step on the logs
var tmpFile = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
(parseResult.resultLine, parseResult.failed) = _resultParser.ParseResults(path, xmlType, tmpFile);
File.Copy(tmpFile, test_log_path, true);
File.Delete(tmpFile);
}
else
{
(parseResult.resultLine, parseResult.failed) = _resultParser.ParseResults(path, xmlType);
}
// we do not longer need the tmp file
_logs.AddFile(path, LogType.XmlLog.ToString());
return(parseResult);
}
catch (Exception e)
{
_mainLog.WriteLine("Could not parse xml result file: {0}", e);
// print file for better debugging
_mainLog.WriteLine("File data is:");
_mainLog.WriteLine(new string('#', 10));
using (var stream = new StreamReader(path))
{
string?line;
while ((line = await stream.ReadLineAsync()) != null)
{
_mainLog.WriteLine(line);
}
}
_mainLog.WriteLine(new string('#', 10));
_mainLog.WriteLine("End of xml results.");
if (timed_out)
{
WrenchLog.WriteLine($"AddSummary: <b><i>{_runMode} timed out</i></b><br/>");
return(parseResult);
}
else
{
WrenchLog.WriteLine($"AddSummary: <b><i>{_runMode} crashed</i></b><br/>");
_mainLog.WriteLine("Test run crashed");
parseResult.crashed = true;
return(parseResult);
}
}
}
// delete not needed copy
File.Delete(path);
// not the most efficient way but this just happens when we run
// the tests locally and we usually do not run all tests, we are
// more interested to be efficent on the bots
(parseResult.resultLine, parseResult.failed) = await GetResultLine(test_log_path);
return(parseResult);
}
public async Task EndCaptureAsync(TimeSpan timeout)
{
if (_initialCrashes == null)
{
throw new InvalidOperationException("CrashSnapshotReport capturing was ended without being started first!");
}
_log.WriteLine($"No crash reports, waiting {(int)timeout.TotalSeconds} seconds for the crash report service...");
// Check for crash reports
var stopwatch = Stopwatch.StartNew();
do
{
var newCrashFiles = await CreateCrashReportsSnapshotAsync();
newCrashFiles.ExceptWith(_initialCrashes);
if (newCrashFiles.Count == 0)
{
if (stopwatch.Elapsed.TotalSeconds > timeout.TotalSeconds)
{
break;
}
else
{
await Task.Delay(TimeSpan.FromSeconds(1));
}
continue;
}
_log.WriteLine("Found {0} new crash report(s)", newCrashFiles.Count);
IEnumerable <IFileBackedLog> crashReports;
if (!_isDevice)
{
crashReports = new List <IFileBackedLog>(newCrashFiles.Count);
foreach (var path in newCrashFiles)
{
// It can happen that the crash log is still being written to so we have to retry
int retry = 1;
while (true)
{
try
{
var fileName = Path.GetFileName(path);
_log.WriteLine($" - Adding {path}");
_logs.AddFile(path, $"Crash report: {fileName}");
_log.WriteLine($" Successfully copied {fileName}");
break;
}
catch (Exception e)
{
_log.WriteLine($" Attempt {retry} to copy a crash report failed: {e.Message}");
}
if (retry == 3)
{
_log.WriteLine($" Failed to copy a crash report after {retry} retries");
break;
}
++retry;
await Task.Delay(TimeSpan.FromSeconds(2 * retry));
}
}
}
else
{
// Download crash reports from the device. We put them in the project directory so that they're automatically deleted on wrench
// (if we put them in /tmp, they'd never be deleted).
crashReports = newCrashFiles
.Select(async crash => await ProcessCrash(crash))
.Select(t => t.Result)
.Where(c => c != null);
}
foreach (var cp in crashReports)
{
WrenchLog.WriteLine("AddFile: {0}", cp.FullPath);
_log.WriteLine(" {0}", cp.FullPath);
}
break;
} while (true);
}