public async Task EndCaptureAsync(TimeSpan timeout)
{
// Check for crash reports
var crash_report_search_done = false;
var crash_report_search_timeout = timeout.TotalSeconds;
var watch = new Stopwatch();
watch.Start();
do
{
var end_crashes = await Harness.CreateCrashReportsSnapshotAsync(Log, !Device, DeviceName);
end_crashes.ExceptWith(InitialSet);
Reports = end_crashes;
if (end_crashes.Count > 0)
{
Log.WriteLine("Found {0} new crash report(s)", end_crashes.Count);
List <LogFile> crash_reports;
if (!Device)
{
crash_reports = new List <LogFile> (end_crashes.Count);
foreach (var path in end_crashes)
{
Logs.AddFile(path, $"Crash report: {Path.GetFileName (path)}");
}
}
else
{
// Download crash reports from the device. We put them in the project directory so that they're automatically deleted on wrench
// (if we put them in /tmp, they'd never be deleted).
var downloaded_crash_reports = new List <LogFile> ();
foreach (var file in end_crashes)
{
var name = Path.GetFileName(file);
var crash_report_target = Logs.Create(name, $"Crash report: {name}");
var sb = new StringBuilder();
sb.Append(" --download-crash-report=").Append(StringUtils.Quote(file));
sb.Append(" --download-crash-report-to=").Append(StringUtils.Quote(crash_report_target.Path));
sb.Append(" --sdkroot ").Append(StringUtils.Quote(Harness.XcodeRoot));
if (!string.IsNullOrEmpty(DeviceName))
{
sb.Append(" --devname ").Append(StringUtils.Quote(DeviceName));
}
var result = await ProcessHelper.ExecuteCommandAsync(Harness.MlaunchPath, sb.ToString(), Log, TimeSpan.FromMinutes(1));
if (result.Succeeded)
{
Log.WriteLine("Downloaded crash report {0} to {1}", file, crash_report_target.Path);
crash_report_target = await Harness.SymbolicateCrashReportAsync(Logs, Log, crash_report_target);
downloaded_crash_reports.Add(crash_report_target);
}
else
{
Log.WriteLine("Could not download crash report {0}", file);
}
}
crash_reports = downloaded_crash_reports;
}
foreach (var cp in crash_reports)
{
Harness.LogWrench("@MonkeyWrench: AddFile: {0}", cp.Path);
Log.WriteLine(" {0}", cp.Path);
}
crash_report_search_done = true;
}
else
{
if (watch.Elapsed.TotalSeconds > crash_report_search_timeout)
{
crash_report_search_done = true;
}
else
{
Log.WriteLine("No crash reports, waiting a second to see if the crash report service just didn't complete in time ({0})", (int)(crash_report_search_timeout - watch.Elapsed.TotalSeconds));
Thread.Sleep(TimeSpan.FromSeconds(1));
}
}
} while (!crash_report_search_done);
}
public bool TestsSucceeded(Log listener_log, bool timed_out, bool crashed)
{
string log;
using (var reader = listener_log.GetReader())
log = reader.ReadToEnd();
// parsing the result is different if we are in jenkins or not.
// When in Jenkins, Touch.Unit produces an xml file instead of a console log (so that we can get better test reporting).
// However, for our own reporting, we still want the console-based log. This log is embedded inside the xml produced
// by Touch.Unit, so we need to extract it and write it to disk. We also need to re-save the xml output, since Touch.Unit
// wraps the NUnit xml output with additional information, which we need to unwrap so that Jenkins understands it.
if (Harness.InJenkins)
{
// we have to parse the xml result
crashed = false;
var xmldoc = new XmlDocument();
try {
xmldoc.LoadXml(log);
var nunit_output = xmldoc.SelectSingleNode("/TouchUnitTestRun/NUnitOutput");
var xmllog = nunit_output.InnerXml;
var extra_output = xmldoc.SelectSingleNode("/TouchUnitTestRun/TouchUnitExtraData");
log = extra_output.InnerText;
File.WriteAllText(listener_log.FullPath, log);
var testsResults = new XmlDocument();
testsResults.LoadXml(xmllog);
var mainResultNode = testsResults.SelectSingleNode("test-results");
if (mainResultNode == null)
{
Harness.LogWrench($"Node is null.");
}
else
{
// update the information of the main node to add information about the mode and the test that is excuted. This will later create
// nicer reports in jenkins
mainResultNode.Attributes ["name"].Value = Target.AsString();
// store a clean version of the logs, later this will be used by the bots to show results in github/web
var path = listener_log.FullPath;
path = Path.ChangeExtension(path, "xml");
testsResults.Save(path);
Logs.AddFile(path, "Test xml");
}
} catch (Exception e) {
main_log.WriteLine("Could not parse xml result file: {0}", e);
if (timed_out)
{
Harness.LogWrench($"@MonkeyWrench: AddSummary: <b><i>{mode} timed out</i></b><br/>");
return(false);
}
else
{
Harness.LogWrench($"@MonkeyWrench: AddSummary: <b><i>{mode} crashed</i></b><br/>");
main_log.WriteLine("Test run crashed");
crashed = true;
return(false);
}
}
}
// parsing the human readable results
if (log.Contains("Tests run"))
{
var tests_run = string.Empty;
var log_lines = log.Split('\n');
var failed = false;
foreach (var line in log_lines)
{
if (line.Contains("Tests run:"))
{
Console.WriteLine(line);
tests_run = line.Replace("Tests run: ", "");
break;
}
else if (line.Contains("FAIL"))
{
Console.WriteLine(line);
failed = true;
}
}
if (failed)
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b>{0} failed: {1}</b><br/>", mode, tests_run);
main_log.WriteLine("Test run failed");
return(false);
}
else
{
Harness.LogWrench("@MonkeyWrench: AddSummary: {0} succeeded: {1}<br/>", mode, tests_run);
main_log.WriteLine("Test run succeeded");
return(true);
}
}
else if (timed_out)
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} timed out</i></b><br/>", mode);
return(false);
}
else
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} crashed</i></b><br/>", mode);
main_log.WriteLine("Test run crashed");
crashed = true;
return(false);
}
}
public bool TestsSucceeded(Log listener_log, bool timed_out, bool crashed)
{
// parsing the result is different if we are in jenkins or not.
// When in Jenkins, Touch.Unit produces an xml file instead of a console log (so that we can get better test reporting).
// However, for our own reporting, we still want the console-based log. This log is embedded inside the xml produced
// by Touch.Unit, so we need to extract it and write it to disk. We also need to re-save the xml output, since Touch.Unit
// wraps the NUnit xml output with additional information, which we need to unwrap so that Jenkins understands it.
if (Harness.InJenkins)
{
// use a tmp file so that we can use the reader in xml and write the humman version
var tmpFile = Path.GetTempFileName();
// we have to parse the xml result
crashed = false;
var xmldoc = new XmlDocument();
XmlNode mainResultNode;
try {
using (var reader = listener_log.GetReader()) {
xmldoc.Load(reader);
var testsResults = new XmlDocument();
using (var writer = new StreamWriter(tmpFile)) {
if (IsTouchUnitResult(xmldoc))
{
testsResults = ParseTouchUnitXml(xmldoc, writer);
}
else
{
testsResults = ParseNUnitXml(xmldoc, writer);
}
}
mainResultNode = testsResults.SelectSingleNode("test-results");
}
if (mainResultNode == null)
{
Harness.LogWrench($"Node is null.");
}
else
{
// update the information of the main node to add information about the mode and the test that is excuted. This will later create
// nicer reports in jenkins
mainResultNode.Attributes ["name"].Value = Target.AsString();
// store a clean version of the logs, later this will be used by the bots to show results in github/web
var path = listener_log.FullPath;
path = Path.ChangeExtension(path, "xml");
// we already have all the data needed in the listerner, rather than saving from the doc, copy
File.Copy(listener_log.FullPath, path, true);
Logs.AddFile(path, "Test xml");
}
// write on the log
File.Copy(tmpFile, listener_log.FullPath, true);
File.Delete(tmpFile);
} catch (Exception e) {
main_log.WriteLine("Could not parse xml result file: {0}", e);
if (timed_out)
{
Harness.LogWrench($"@MonkeyWrench: AddSummary: <b><i>{mode} timed out</i></b><br/>");
return(false);
}
else
{
Harness.LogWrench($"@MonkeyWrench: AddSummary: <b><i>{mode} crashed</i></b><br/>");
main_log.WriteLine("Test run crashed");
crashed = true;
return(false);
}
}
}
// read until the end and decide if we got results, do not store them
// we do not want to use too much memory
string resultLine = null;
using (var reader = new StreamReader(listener_log.FullPath)) {
string line;
while ((line = reader.ReadLine()) != null)
{
if (line.Contains("Tests run"))
{
resultLine = line;
}
}
}
// read the parsed logs in a human readable way
if (resultLine != null)
{
var tests_run = string.Empty;
var failed = false;
using (var reader = listener_log.GetReader()) {
string line;
while ((line = reader.ReadLine()) != null)
{
if (line.Contains("Tests run:"))
{
Console.WriteLine(line);
tests_run = line.Replace("Tests run: ", "");
break;
}
else if (line.Contains("[FAIL]"))
{
Console.WriteLine(line);
failed = true;
}
}
}
if (failed)
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b>{0} failed: {1}</b><br/>", mode, tests_run);
main_log.WriteLine("Test run failed");
return(false);
}
else
{
Harness.LogWrench("@MonkeyWrench: AddSummary: {0} succeeded: {1}<br/>", mode, tests_run);
main_log.WriteLine("Test run succeeded");
return(true);
}
}
else if (timed_out)
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} timed out</i></b><br/>", mode);
return(false);
}
else
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} crashed</i></b><br/>", mode);
main_log.WriteLine("Test run crashed");
crashed = true;
return(false);
}
}
public int Run()
{
HashSet <string> start_crashes = null;
string device_system_log = null;
Initialize();
var args = new StringBuilder();
if (!string.IsNullOrEmpty(Harness.XcodeRoot))
{
args.Append(" --sdkroot ").Append(Harness.XcodeRoot);
}
for (int i = -1; i < Harness.Verbosity; i++)
{
args.Append(" -v ");
}
args.Append(" -argument=-connection-mode -argument=none"); // This will prevent the app from trying to connect to any IDEs
args.Append(" -argument=-app-arg:-autostart");
args.Append(" -setenv=NUNIT_AUTOSTART=true");
args.Append(" -argument=-app-arg:-autoexit");
args.Append(" -setenv=NUNIT_AUTOEXIT=true");
args.Append(" -argument=-app-arg:-enablenetwork");
args.Append(" -setenv=NUNIT_ENABLE_NETWORK=true");
if (isSimulator)
{
args.Append(" -argument=-app-arg:-hostname:127.0.0.1");
args.Append(" -setenv=NUNIT_HOSTNAME=127.0.0.1");
}
else
{
var ips = new StringBuilder();
var ipAddresses = System.Net.Dns.GetHostEntry(System.Net.Dns.GetHostName()).AddressList;
for (int i = 0; i < ipAddresses.Length; i++)
{
if (i > 0)
{
ips.Append(',');
}
ips.Append(ipAddresses [i].ToString());
}
args.AppendFormat(" -argument=-app-arg:-hostname:{0}", ips.ToString());
args.AppendFormat(" -setenv=NUNIT_HOSTNAME={0}", ips.ToString());
}
var transport = mode == "watchos" ? "HTTP" : "TCP";
args.AppendFormat(" -argument=-app-arg:-transport:{0}", transport);
args.AppendFormat(" -setenv=NUNIT_TRANSPORT={0}", transport);
SimpleListener listener;
switch (transport)
{
case "HTTP":
listener = new SimpleHttpListener();
break;
case "TCP":
listener = new SimpleTcpListener();
break;
default:
throw new NotImplementedException();
}
listener.LogPath = Path.GetDirectoryName(Harness.LogFile);
listener.LogFile = Path.GetFileName(Harness.LogFile);
listener.AutoExit = true;
listener.Address = System.Net.IPAddress.Any;
listener.Initialize();
args.AppendFormat(" -argument=-app-arg:-hostport:{0}", listener.Port);
args.AppendFormat(" -setenv=NUNIT_HOSTPORT={0}", listener.Port);
if (File.Exists(Harness.LogFile))
{
File.Delete(Harness.LogFile);
}
bool?success = null;
bool timed_out = false;
if (isSimulator)
{
FindSimulator();
Harness.Log("*** Executing {0}/{1} in the simulator ***", appName, mode);
PrepareSimulator();
args.Append(" --launchsim");
args.AppendFormat(" \"{0}\" ", launchAppPath);
args.Append(" --device=:v2:udid=").Append(simulator.udid).Append(" ");
start_crashes = CreateCrashReportsSnapshot(true);
listener.StartAsync();
Harness.Log("Starting test run");
var proc = new XProcess()
{
Harness = Harness,
FileName = Harness.MlaunchPath,
Arguments = args.ToString(),
VerbosityLevel = 0,
};
proc.Start();
var launchState = 0; // 0: launching, 1: launch timed out, 2: run timed out, 3: completed
var launchMutex = new Mutex();
var runCompleted = new ManualResetEvent(false);
ThreadPool.QueueUserWorkItem((v) => {
if (!listener.WaitForConnection(TimeSpan.FromMinutes(Harness.LaunchTimeout)))
{
lock (launchMutex) {
if (launchState == 0)
{
launchState = 1;
runCompleted.Set();
}
}
Harness.Log("Test launch timed out after {0} minute(s).", Harness.LaunchTimeout);
}
else
{
Harness.Log("Test run started");
}
});
ThreadPool.QueueUserWorkItem((v) => {
var rv = proc.WaitForExit(TimeSpan.FromMinutes(Harness.Timeout));
lock (launchMutex) {
if (launchState == 0)
{
launchState = rv ? 3 : 2;
}
runCompleted.Set();
}
if (rv)
{
Harness.Log("Test run completed");
}
else
{
Harness.Log("Test run timed out after {0} minute(s).", Harness.Timeout);
}
});
runCompleted.WaitOne();
switch (launchState)
{
case 1:
case 2:
success = false;
timed_out = true;
// find pid
var pid = -1;
var output = proc.ReadCurrentOutput();
foreach (var line in output.ToString().Split('\n'))
{
if (line.StartsWith("Application launched. PID = ", StringComparison.Ordinal))
{
var pidstr = line.Substring("Application launched. PID = ".Length);
if (!int.TryParse(pidstr, out pid))
{
Harness.Log("Could not parse pid: {0}", pidstr);
}
}
else if (line.Contains("Xamarin.Hosting: Launched ") && line.Contains(" with pid "))
{
var pidstr = line.Substring(line.LastIndexOf(' '));
if (!int.TryParse(pidstr, out pid))
{
Harness.Log("Could not parse pid: {0}", pidstr);
}
}
}
if (pid > 0)
{
KillPid(proc, pid, TimeSpan.FromSeconds(5), TimeSpan.FromMinutes(launchState == 1 ? Harness.LaunchTimeout : Harness.Timeout), launchState == 1 ? "Launch" : "Completion");
}
else
{
Harness.Log("Could not find pid in mtouch output.");
}
// kill mtouch too
kill(proc.Id, 9);
break;
case 3:
// Success!
break;
case 0: // shouldn't happen ever
default:
throw new Exception($"Invalid launch state: {launchState}");
}
listener.Cancel();
// cleanup after us
KillEverything();
}
else
{
FindDevice();
Harness.Log("*** Executing {0}/{1} on device ***", appName, mode);
args.Append(" --launchdev");
args.AppendFormat(" \"{0}\" ", launchAppPath);
AddDeviceName(args);
device_system_log = Harness.LogFile + ".device.log";
var logdev = new DeviceLogCapturer()
{
Harness = Harness,
LogPath = device_system_log,
DeviceName = device_name,
};
logdev.StartCapture();
start_crashes = CreateCrashReportsSnapshot(false);
listener.StartAsync();
Harness.Log("Starting test run");
ExecuteCommand(Harness.MlaunchPath, args.ToString());
if (listener.WaitForCompletion(TimeSpan.FromMinutes(Harness.Timeout)))
{
Harness.Log("Test run completed");
}
else
{
Harness.Log("Test run did not complete in {0} minutes.", Harness.Timeout);
listener.Cancel();
success = false;
timed_out = true;
}
logdev.StopCapture();
// Upload the system log
if (File.Exists(device_system_log))
{
Harness.Log(1, "A capture of the device log is: {0}", device_system_log);
if (Harness.InWrench)
{
Harness.LogWrench("@MonkeyWrench: AddFile: {0}", Path.GetFullPath(device_system_log));
}
}
}
listener.Dispose();
// check the final status
var crashed = false;
if (File.Exists(Harness.LogFile))
{
Harness.LogWrench("@MonkeyWrench: AddFile: {0}", Path.GetFullPath(Harness.LogFile));
var log = File.ReadAllText(Harness.LogFile);
if (log.Contains("Tests run"))
{
var tests_run = string.Empty;
var log_lines = File.ReadAllLines(Harness.LogFile);
var failed = false;
foreach (var line in log_lines)
{
if (line.Contains("Tests run:"))
{
Console.WriteLine(line);
tests_run = line.Replace("Tests run: ", "");
break;
}
else if (line.Contains("FAIL"))
{
Console.WriteLine(line);
failed = true;
}
}
if (failed)
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b>{0} failed: {1}</b><br/>", mode, tests_run);
Harness.Log("Test run failed");
}
else
{
Harness.LogWrench("@MonkeyWrench: AddSummary: {0} succeeded: {1}<br/>", mode, tests_run);
Harness.Log("Test run succeeded");
success = true;
}
}
else if (timed_out)
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} timed out</i></b><br/>", mode);
}
else
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} crashed</i></b><br/>", mode);
Harness.Log("Test run crashed");
crashed = true;
}
}
else if (timed_out)
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} never launched</i></b><br/>", mode);
Harness.Log("Test run never launched");
}
else
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} crashed at startup (no log)</i></b><br/>", mode);
Harness.Log("Test run crashed before it started (no log file produced)");
crashed = true;
}
// Check for crash reports
var crash_report_search_done = false;
var crash_report_search_timeout = 5;
var watch = new Stopwatch();
watch.Start();
do
{
var end_crashes = CreateCrashReportsSnapshot(isSimulator);
end_crashes.ExceptWith(start_crashes);
if (end_crashes.Count > 0)
{
Harness.Log("Found {0} new crash report(s)", end_crashes.Count);
if (!isSimulator)
{
// Download crash reports from the device. We put them in the project directory so that they're automatically deleted on wrench
// (if we put them in /tmp, they'd never be deleted).
var downloaded_crash_reports = new HashSet <string> ();
foreach (var file in end_crashes)
{
var crash_report_target = Path.Combine(Path.GetDirectoryName(ProjectFile), Path.GetFileName(file));
if (ExecuteCommand(Harness.MlaunchPath, "--download-crash-report=" + file + " --download-crash-report-to=" + crash_report_target + " --sdkroot " + Harness.XcodeRoot))
{
Harness.Log("Downloaded crash report {0} to {1}", file, crash_report_target);
crash_report_target = SymbolicateCrashReport(crash_report_target);
downloaded_crash_reports.Add(crash_report_target);
}
else
{
Harness.Log("Could not download crash report {0}", file);
}
}
end_crashes = downloaded_crash_reports;
}
foreach (var cp in end_crashes)
{
Harness.LogWrench("@MonkeyWrench: AddFile: {0}", Path.GetFullPath(cp));
Harness.Log(" {0}", cp);
}
crash_report_search_done = true;
}
else if (!crashed && !timed_out)
{
crash_report_search_done = true;
}
else
{
if (watch.Elapsed.TotalSeconds > crash_report_search_timeout)
{
crash_report_search_done = true;
}
else
{
Harness.Log("No crash reports, waiting a second to see if the crash report service just didn't complete in time ({0})", (int)(crash_report_search_timeout - watch.Elapsed.TotalSeconds));
Thread.Sleep(TimeSpan.FromSeconds(1));
}
}
} while (!crash_report_search_done);
if (!success.HasValue)
{
success = false;
}
if (isSimulator)
{
foreach (var sim in simulators)
{
// Upload the system log
if (File.Exists(sim.system_log))
{
Harness.Log(success.Value ? 1 : 0, "System log for the '{1}' simulator is: {0}", sim.system_log, sim.name);
if (Harness.InWrench)
{
var syslog = Harness.LogFile + (sim == simulator ? ".system.log" : ".companion.system.log");
File.Copy(sim.system_log, syslog, true);
Harness.LogWrench("@MonkeyWrench: AddFile: {0}", Path.GetFullPath(syslog));
}
}
}
}
return(success.Value ? 0 : 1);
}
public bool TestsSucceeded(LogStream listener_log, bool timed_out, bool crashed)
{
string log;
using (var reader = listener_log.GetReader())
log = reader.ReadToEnd();
// parsing the result is different if we are in jenkins or nor.
if (Harness.InJenkins)
{
// we have to parse the xml result
crashed = false;
if (log.Contains("test-results"))
{
// remove any possible extra info
var index = log.IndexOf("<test-results");
var header = log.Substring(0, log.IndexOf('<'));
log = log.Remove(0, index - 1);
var testsResults = new XmlDocument();
testsResults.LoadXml(log);
var mainResultNode = testsResults.SelectSingleNode("test-results");
if (mainResultNode == null)
{
Harness.LogWrench($"Node is null.");
crashed = true;
return(false);
}
// update the information of the main node to add information about the mode and the test that is excuted. This will later create
// nicer reports in jenkins
mainResultNode.Attributes["name"].Value = Target.AsString();
// store a clean version of the logs, later this will be used by the bots to show results in github/web
var path = listener_log.FullPath;
path = path.Replace(".log", ".xml");
testsResults.Save(path);
Logs.Add(new LogFile("Test xml", path));
// we want to keep the old TestResult page,
GenerateHumanReadableLogs(listener_log.FullPath, header, testsResults);
int ignored = Convert.ToInt16(mainResultNode.Attributes["ignored"].Value);
int invalid = Convert.ToInt16(mainResultNode.Attributes["invalid"].Value);
int inconclusive = Convert.ToInt16(mainResultNode.Attributes["inconclusive"].Value);
int errors = Convert.ToInt16(mainResultNode.Attributes["errors"].Value);
int failures = Convert.ToInt16(mainResultNode.Attributes["failures"].Value);
int totalTests = Convert.ToInt16(mainResultNode.Attributes["total"].Value);
// generate human readable logs
var failed = errors != 0 || failures != 0;
if (failed)
{
Harness.LogWrench($"@MonkeyWrench: AddSummary: <b>{mode} failed: Test run: {totalTests} Passed: {totalTests - invalid - inconclusive - ignored} Inconclusive: {inconclusive} Failed: {errors + failures} Ignored: {ignored}</b><br/>");
main_log.WriteLine("Test run failed");
return(false);
}
else
{
Harness.LogWrench($"@MonkeyWrench: AddSummary: {mode} succeeded: Test run: {totalTests} Passed: {totalTests - invalid - inconclusive - ignored} Inconclusive: {inconclusive} Failed: 0 Ignored: {ignored}<br/>");
main_log.WriteLine("Test run succeeded");
return(true);
}
}
else if (timed_out)
{
Harness.LogWrench($"@MonkeyWrench: AddSummary: <b><i>{mode} timed out</i></b><br/>");
return(false);
}
else
{
Harness.LogWrench($"@MonkeyWrench: AddSummary: <b><i>{mode} crashed</i></b><br/>");
main_log.WriteLine("Test run crashed");
crashed = true;
return(false);
}
}
else
{
// parsing the human readable results
if (log.Contains("Tests run"))
{
var tests_run = string.Empty;
var log_lines = log.Split('\n');
var failed = false;
foreach (var line in log_lines)
{
if (line.Contains("Tests run:"))
{
Console.WriteLine(line);
tests_run = line.Replace("Tests run: ", "");
break;
}
else if (line.Contains("FAIL"))
{
Console.WriteLine(line);
failed = true;
}
}
if (failed)
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b>{0} failed: {1}</b><br/>", mode, tests_run);
main_log.WriteLine("Test run failed");
return(false);
}
else
{
Harness.LogWrench("@MonkeyWrench: AddSummary: {0} succeeded: {1}<br/>", mode, tests_run);
main_log.WriteLine("Test run succeeded");
return(true);
}
}
else if (timed_out)
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} timed out</i></b><br/>", mode);
return(false);
}
else
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} crashed</i></b><br/>", mode);
main_log.WriteLine("Test run crashed");
crashed = true;
return(false);
}
}
}
public async Task <int> RunAsync()
{
CrashReportSnapshot crash_reports;
LogStream device_system_log = null;
LogStream listener_log = null;
Log run_log = main_log;
Initialize();
if (!isSimulator)
{
FindDevice();
}
crash_reports = new CrashReportSnapshot()
{
Device = !isSimulator,
DeviceName = device_name,
Harness = Harness,
Log = main_log,
Logs = Logs,
LogDirectory = LogDirectory,
};
var args = new StringBuilder();
if (!string.IsNullOrEmpty(Harness.XcodeRoot))
{
args.Append(" --sdkroot ").Append(Harness.XcodeRoot);
}
for (int i = -1; i < Harness.Verbosity; i++)
{
args.Append(" -v ");
}
args.Append(" -argument=-connection-mode -argument=none"); // This will prevent the app from trying to connect to any IDEs
args.Append(" -argument=-app-arg:-autostart");
args.Append(" -setenv=NUNIT_AUTOSTART=true");
args.Append(" -argument=-app-arg:-autoexit");
args.Append(" -setenv=NUNIT_AUTOEXIT=true");
args.Append(" -argument=-app-arg:-enablenetwork");
args.Append(" -setenv=NUNIT_ENABLE_NETWORK=true");
if (isSimulator)
{
args.Append(" -argument=-app-arg:-hostname:127.0.0.1");
args.Append(" -setenv=NUNIT_HOSTNAME=127.0.0.1");
}
else
{
var ips = new StringBuilder();
var ipAddresses = System.Net.Dns.GetHostEntry(System.Net.Dns.GetHostName()).AddressList;
for (int i = 0; i < ipAddresses.Length; i++)
{
if (i > 0)
{
ips.Append(',');
}
ips.Append(ipAddresses [i].ToString());
}
args.AppendFormat(" -argument=-app-arg:-hostname:{0}", ips.ToString());
args.AppendFormat(" -setenv=NUNIT_HOSTNAME={0}", ips.ToString());
}
var transport = mode == "watchos" ? "HTTP" : "TCP";
args.AppendFormat(" -argument=-app-arg:-transport:{0}", transport);
args.AppendFormat(" -setenv=NUNIT_TRANSPORT={0}", transport);
SimpleListener listener;
switch (transport)
{
case "HTTP":
listener = new SimpleHttpListener();
break;
case "TCP":
listener = new SimpleTcpListener();
break;
default:
throw new NotImplementedException();
}
listener_log = Logs.CreateStream(LogDirectory, string.Format("test-{0:yyyyMMdd_HHmmss}.log", DateTime.Now), "Test log");
listener.TestLog = listener_log;
listener.Log = main_log;
listener.AutoExit = true;
listener.Address = System.Net.IPAddress.Any;
listener.Initialize();
args.AppendFormat(" -argument=-app-arg:-hostport:{0}", listener.Port);
args.AppendFormat(" -setenv=NUNIT_HOSTPORT={0}", listener.Port);
foreach (var kvp in Harness.EnvironmentVariables)
{
args.AppendFormat(" -setenv={0}={1}", kvp.Key, kvp.Value);
}
bool?success = null;
bool timed_out = false;
if (isSimulator)
{
FindSimulator();
var systemLogs = new List <CaptureLog> ();
foreach (var sim in simulators)
{
// Upload the system log
main_log.WriteLine("System log for the '{1}' simulator is: {0}", sim.SystemLog, sim.Name);
bool isCompanion = sim != simulator;
var log = new CaptureLog(sim.SystemLog)
{
Path = Path.Combine(LogDirectory, sim.UDID + ".log"),
Description = isCompanion ? "System log (companion)" : "System log",
};
log.StartCapture();
Logs.Add(log);
systemLogs.Add(log);
Harness.LogWrench("@MonkeyWrench: AddFile: {0}", log.Path);
}
main_log.WriteLine("*** Executing {0}/{1} in the simulator ***", appName, mode);
if (EnsureCleanSimulatorState)
{
foreach (var sim in simulators)
{
await sim.PrepareSimulatorAsync(main_log, bundle_identifier);
}
}
args.Append(" --launchsim");
args.AppendFormat(" \"{0}\" ", launchAppPath);
args.Append(" --device=:v2:udid=").Append(simulator.UDID).Append(" ");
await crash_reports.StartCaptureAsync();
listener.StartAsync();
main_log.WriteLine("Starting test run");
var cancellation_source = new CancellationTokenSource();
ThreadPool.QueueUserWorkItem((v) => {
if (!listener.WaitForConnection(TimeSpan.FromMinutes(Harness.LaunchTimeout)))
{
cancellation_source.Cancel();
main_log.WriteLine("Test launch timed out after {0} minute(s).", Harness.LaunchTimeout);
timed_out = true;
}
else
{
main_log.WriteLine("Test run started");
}
});
var result = await ProcessHelper.ExecuteCommandAsync(Harness.MlaunchPath, args.ToString(), run_log, TimeSpan.FromMinutes(Harness.Timeout), cancellation_token : cancellation_source.Token);
if (result.TimedOut)
{
timed_out = true;
success = false;
main_log.WriteLine("Test run timed out after {0} minute(s).", Harness.Timeout);
}
else if (result.Succeeded)
{
main_log.WriteLine("Test run completed");
success = true;
}
else
{
main_log.WriteLine("Test run failed");
success = false;
}
if (!success.Value)
{
// find pid
var pid = -1;
using (var reader = run_log.GetReader()) {
while (!reader.EndOfStream)
{
var line = reader.ReadLine();
if (line.StartsWith("Application launched. PID = ", StringComparison.Ordinal))
{
var pidstr = line.Substring("Application launched. PID = ".Length);
if (!int.TryParse(pidstr, out pid))
{
main_log.WriteLine("Could not parse pid: {0}", pidstr);
}
}
else if (line.Contains("Xamarin.Hosting: Launched ") && line.Contains(" with pid "))
{
var pidstr = line.Substring(line.LastIndexOf(' '));
if (!int.TryParse(pidstr, out pid))
{
main_log.WriteLine("Could not parse pid: {0}", pidstr);
}
}
}
}
if (pid > 0)
{
var launchTimedout = cancellation_source.IsCancellationRequested;
await KillPidAsync(main_log, pid, TimeSpan.FromSeconds(5), TimeSpan.FromMinutes(launchTimedout ? Harness.LaunchTimeout : Harness.Timeout), launchTimedout? "Launch" : "Completion");
}
else
{
main_log.WriteLine("Could not find pid in mtouch output.");
}
}
listener.Cancel();
// cleanup after us
if (EnsureCleanSimulatorState)
{
await SimDevice.KillEverythingAsync(main_log);
}
foreach (var log in systemLogs)
{
log.StopCapture();
}
}
else
{
main_log.WriteLine("*** Executing {0}/{1} on device '{2}' ***", appName, mode, device_name);
args.Append(" --launchdev");
args.AppendFormat(" \"{0}\" ", launchAppPath);
var waits_for_exit = false;
if (mode == "watchos")
{
args.Append(" --attach-native-debugger"); // this prevents the watch from backgrounding the app.
waits_for_exit = true;
}
AddDeviceName(args);
device_system_log = Logs.CreateStream(LogDirectory, "device.log", "Device log");
var logdev = new DeviceLogCapturer()
{
Harness = Harness,
Log = device_system_log,
DeviceName = device_name,
};
logdev.StartCapture();
await crash_reports.StartCaptureAsync();
listener.StartAsync();
main_log.WriteLine("Starting test run");
double launch_timeout = waits_for_exit ? Harness.Timeout : 1;
double listener_timeout = waits_for_exit ? 0.2 : Harness.Timeout;
await ProcessHelper.ExecuteCommandAsync(Harness.MlaunchPath, args.ToString(), main_log, TimeSpan.FromMinutes(launch_timeout));
if (listener.WaitForCompletion(TimeSpan.FromMinutes(listener_timeout)))
{
main_log.WriteLine("Test run completed");
}
else
{
main_log.WriteLine("Test run did not complete in {0} minutes.", Harness.Timeout);
listener.Cancel();
success = false;
timed_out = true;
}
logdev.StopCapture();
// Upload the system log
if (File.Exists(device_system_log.FullPath))
{
main_log.WriteLine("A capture of the device log is: {0}", device_system_log.FullPath);
if (Harness.InWrench)
{
Harness.LogWrench("@MonkeyWrench: AddFile: {0}", device_system_log.FullPath);
}
}
}
listener.Dispose();
// check the final status
var crashed = false;
if (File.Exists(listener_log.FullPath))
{
Harness.LogWrench("@MonkeyWrench: AddFile: {0}", listener_log.FullPath);
string log;
using (var reader = listener_log.GetReader())
log = reader.ReadToEnd();
if (log.Contains("Tests run"))
{
var tests_run = string.Empty;
var log_lines = log.Split('\n');
var failed = false;
foreach (var line in log_lines)
{
if (line.Contains("Tests run:"))
{
Console.WriteLine(line);
tests_run = line.Replace("Tests run: ", "");
break;
}
else if (line.Contains("FAIL"))
{
Console.WriteLine(line);
failed = true;
}
}
if (failed)
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b>{0} failed: {1}</b><br/>", mode, tests_run);
main_log.WriteLine("Test run failed");
success = false;
}
else
{
Harness.LogWrench("@MonkeyWrench: AddSummary: {0} succeeded: {1}<br/>", mode, tests_run);
main_log.WriteLine("Test run succeeded");
success = true;
}
}
else if (timed_out)
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} timed out</i></b><br/>", mode);
success = false;
}
else
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} crashed</i></b><br/>", mode);
main_log.WriteLine("Test run crashed");
crashed = true;
success = false;
}
}
else if (timed_out)
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} never launched</i></b><br/>", mode);
main_log.WriteLine("Test run never launched");
success = false;
}
else
{
Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} crashed at startup (no log)</i></b><br/>", mode);
main_log.WriteLine("Test run crashed before it started (no log file produced)");
crashed = true;
success = false;
}
if (!success.HasValue)
{
success = false;
}
await crash_reports.EndCaptureAsync(TimeSpan.FromSeconds(success.Value ? 0 : 5));
if (timed_out)
{
Result = TestExecutingResult.TimedOut;
}
else if (crashed)
{
Result = TestExecutingResult.Crashed;
}
else if (success.Value)
{
Result = TestExecutingResult.Succeeded;
}
else
{
Result = TestExecutingResult.Failed;
}
return(success.Value ? 0 : 1);
}