public ITestReporter Create(IFileBackedLog mainLog,
IReadableLog runLog,
ILogs logs,
ICrashSnapshotReporter crashReporter,
ISimpleListener simpleListener,
IResultParser parser,
AppBundleInformation appInformation,
RunMode runMode,
XmlResultJargon xmlJargon,
string?device,
TimeSpan timeout,
string?additionalLogsDirectory = null,
ExceptionLogger?exceptionLogger = null,
bool generateHtml = false) => new TestReporter(_processManager,
mainLog,
runLog,
logs,
crashReporter,
simpleListener,
parser,
appInformation,
runMode,
xmlJargon,
device,
timeout,
additionalLogsDirectory,
exceptionLogger,
generateHtml);
async Task <(bool Succeeded, bool Crashed)> TestsSucceeded(AppBundleInformation appInfo, string test_log_path, bool timed_out)
{
var(resultLine, failed, crashed) = await ParseResultFile(appInfo, test_log_path, timed_out);
// read the parsed logs in a human readable way
if (resultLine != null)
{
var tests_run = resultLine.Replace("Tests run: ", "");
if (failed)
{
WrenchLog.WriteLine("AddSummary: <b>{0} failed: {1}</b><br/>", runMode, tests_run);
mainLog.WriteLine("Test run failed");
return(false, crashed);
}
else
{
WrenchLog.WriteLine("AddSummary: {0} succeeded: {1}<br/>", runMode, tests_run);
mainLog.WriteLine("Test run succeeded");
return(true, crashed);
}
}
else if (timed_out)
{
WrenchLog.WriteLine("AddSummary: <b><i>{0} timed out</i></b><br/>", runMode);
return(false, false);
}
else
{
WrenchLog.WriteLine("AddSummary: <b><i>{0} crashed</i></b><br/>", runMode);
mainLog.WriteLine("Test run crashed");
return(false, true);
}
}
public ITestReporter Create(ILog mainLog,
ILog runLog,
ILogs logs,
ICrashSnapshotReporter crashReporter,
ISimpleListener simpleListener,
IResultParser parser,
AppBundleInformation appInformation,
RunMode runMode,
XmlResultJargon xmlJargon,
string device,
TimeSpan timeout,
string additionalLogsDirectory = null,
ExceptionLogger exceptionLogger = null)
{
return(new TestReporter(processManager,
mainLog,
runLog,
logs,
crashReporter,
simpleListener,
parser,
appInformation,
runMode,
xmlJargon,
device,
timeout,
additionalLogsDirectory,
exceptionLogger));
}
public TestReporter(IMlaunchProcessManager processManager,
IFileBackedLog mainLog,
IReadableLog runLog,
ILogs logs,
ICrashSnapshotReporter crashReporter,
ISimpleListener simpleListener,
IResultParser parser,
AppBundleInformation appInformation,
RunMode runMode,
XmlResultJargon xmlJargon,
string?device,
TimeSpan timeout,
string?additionalLogsDirectory = null,
ExceptionLogger?exceptionLogger = null,
bool generateHtml = false)
{
_processManager = processManager ?? throw new ArgumentNullException(nameof(processManager));
_deviceName = device; // can be null on simulators
_listener = simpleListener ?? throw new ArgumentNullException(nameof(simpleListener));
_mainLog = mainLog ?? throw new ArgumentNullException(nameof(mainLog));
_runLog = runLog ?? throw new ArgumentNullException(nameof(runLog));
_logs = logs ?? throw new ArgumentNullException(nameof(logs));
_crashReporter = crashReporter ?? throw new ArgumentNullException(nameof(crashReporter));
_crashLogs = new Logs(logs.Directory);
_resultParser = parser ?? throw new ArgumentNullException(nameof(parser));
_appInfo = appInformation ?? throw new ArgumentNullException(nameof(appInformation));
_runMode = runMode;
_xmlJargon = xmlJargon;
_timeout = timeout;
_additionalLogsDirectory = additionalLogsDirectory;
_exceptionLogger = exceptionLogger;
_timeoutWatch = Stopwatch.StartNew();
_generateHtml = generateHtml;
CallbackLog = new CallbackLog(line =>
{
// MT1111: Application launched successfully, but it's not possible to wait for the app to exit as
// requested because it's not possible to detect app termination when launching using gdbserver
_waitedForExit &= line?.Contains("MT1111: ") != true;
if (line?.Contains("error MT1007") == true)
{
_launchFailure = true;
}
});
}
public TestReporter(IProcessManager processManager,
ILog mainLog,
ILog runLog,
ILogs logs,
ICrashSnapshotReporter crashReporter,
ISimpleListener simpleListener,
IResultParser parser,
AppBundleInformation appInformation,
RunMode runMode,
XmlResultJargon xmlJargon,
string device,
TimeSpan timeout,
double launchTimeout,
string additionalLogsDirectory = null,
ExceptionLogger exceptionLogger = null)
{
this.processManager = processManager ?? throw new ArgumentNullException(nameof(processManager));
this.deviceName = device; // can be null on simulators
this.listener = simpleListener ?? throw new ArgumentNullException(nameof(simpleListener));
this.mainLog = mainLog ?? throw new ArgumentNullException(nameof(mainLog));
this.runLog = runLog ?? throw new ArgumentNullException(nameof(runLog));
this.logs = logs ?? throw new ArgumentNullException(nameof(logs));
this.crashReporter = crashReporter ?? throw new ArgumentNullException(nameof(crashReporter));
this.crashLogs = new Logs(logs.Directory);
this.resultParser = parser ?? throw new ArgumentNullException(nameof(parser));
this.appInfo = appInformation ?? throw new ArgumentNullException(nameof(appInformation));
this.runMode = runMode;
this.xmlJargon = xmlJargon;
this.timeout = timeout;
this.launchTimeout = launchTimeout;
this.additionalLogsDirectory = additionalLogsDirectory;
this.exceptionLogger = exceptionLogger;
this.timeoutWatch = Stopwatch.StartNew();
CallbackLog = new CallbackLog((line) => {
// MT1111: Application launched successfully, but it's not possible to wait for the app to exit as requested because it's not possible to detect app termination when launching using gdbserver
waitedForExit &= line?.Contains("MT1111: ") != true;
if (line?.Contains("error MT1007") == true)
{
launchFailure = true;
}
});
}
private async Task <(string?resultLine, bool failed, bool crashed)> ParseResultFile(AppBundleInformation appInfo, string test_log_path, bool timed_out)
{
(string?resultLine, bool failed, bool crashed)parseResult = (null, false, false);
if (!File.Exists(test_log_path))
{
parseResult.crashed = true; // if we do not have a log file, the test crashes
return(parseResult);
}
// parsing the result is different if we are in jenkins or not.
// When in Jenkins, Touch.Unit produces an xml file instead of a console log (so that we can get better test reporting).
// However, for our own reporting, we still want the console-based log. This log is embedded inside the xml produced
// by Touch.Unit, so we need to extract it and write it to disk. We also need to re-save the xml output, since Touch.Unit
// wraps the NUnit xml output with additional information, which we need to unwrap so that Jenkins understands it.
//
// On the other hand, the nunit and xunit do not have that data and have to be parsed.
//
// This if statement has a small trick, we found out that internet sharing in some of the bots (VSTS) does not work, in
// that case, we cannot do a TCP connection to xharness to get the log, this is a problem since if we did not get the xml
// from the TCP connection, we are going to fail when trying to read it and not parse it. Therefore, we are not only
// going to check if we are in CI, but also if the listener_log is valid.
var path = Path.ChangeExtension(test_log_path, "xml");
_resultParser.CleanXml(test_log_path, path);
if (ResultsUseXml && _resultParser.IsValidXml(path, out var xmlType))
{
try
{
var newFilename = _resultParser.GetXmlFilePath(path, xmlType);
// at this point, we have the test results, but we want to be able to have attachments in vsts, so if the format is
// the right one (NUnitV3) add the nodes. ATM only TouchUnit uses V3.
var testRunName = $"{appInfo.AppName} {appInfo.Variation}";
if (xmlType == XmlResultJargon.NUnitV3)
{
var logFiles = new List <string>();
// add our logs AND the logs of the previous task, which is the build task
logFiles.AddRange(Directory.GetFiles(_crashLogs.Directory));
if (_additionalLogsDirectory != null) // when using the run command, we do not have a build task, ergo, there are no logs to add.
{
logFiles.AddRange(Directory.GetFiles(_additionalLogsDirectory));
}
// add the attachments and write in the new filename
// add a final prefix to the file name to make sure that the VSTS test uploaded just pick
// the final version, else we will upload tests more than once
newFilename = XmlResultParser.GetVSTSFilename(newFilename);
_resultParser.UpdateMissingData(path, newFilename, testRunName, logFiles);
}
else
{
// rename the path to the correct value
File.Move(path, newFilename);
}
path = newFilename;
if (_generateHtml)
{
// write the human readable results in a tmp file, which we later use to step on the logs
var tmpFile = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
(parseResult.resultLine, parseResult.failed) = _resultParser.ParseResults(path, xmlType, tmpFile);
File.Copy(tmpFile, test_log_path, true);
File.Delete(tmpFile);
}
else
{
(parseResult.resultLine, parseResult.failed) = _resultParser.ParseResults(path, xmlType);
}
// we do not longer need the tmp file
_logs.AddFile(path, LogType.XmlLog.ToString());
return(parseResult);
}
catch (Exception e)
{
_mainLog.WriteLine("Could not parse xml result file: {0}", e);
// print file for better debugging
_mainLog.WriteLine("File data is:");
_mainLog.WriteLine(new string('#', 10));
using (var stream = new StreamReader(path))
{
string?line;
while ((line = await stream.ReadLineAsync()) != null)
{
_mainLog.WriteLine(line);
}
}
_mainLog.WriteLine(new string('#', 10));
_mainLog.WriteLine("End of xml results.");
if (timed_out)
{
WrenchLog.WriteLine($"AddSummary: <b><i>{_runMode} timed out</i></b><br/>");
return(parseResult);
}
else
{
WrenchLog.WriteLine($"AddSummary: <b><i>{_runMode} crashed</i></b><br/>");
_mainLog.WriteLine("Test run crashed");
parseResult.crashed = true;
return(parseResult);
}
}
}
// delete not needed copy
File.Delete(path);
// not the most efficient way but this just happens when we run
// the tests locally and we usually do not run all tests, we are
// more interested to be efficent on the bots
(parseResult.resultLine, parseResult.failed) = await GetResultLine(test_log_path);
return(parseResult);
}
