void KillPid(XProcess proc, int pid, TimeSpan kill_separation, TimeSpan timeout, string type) { Harness.Log("{2} timeout ({1} s) reached, will now send SIGQUIT to the app (PID: {0})", pid, timeout.TotalSeconds, type); kill(pid, 3 /* SIGQUIT */); // print managed stack traces. if (!proc.WaitForExit(kill_separation /* wait for at most 5 seconds to see if something happens */)) { Harness.Log("{2} timeout ({1} s) reached, will now send SIGABRT to the app (PID: {0})", pid, timeout.TotalSeconds, type); kill(pid, 6 /* SIGABRT */); // print native stack traces. if (!proc.WaitForExit(kill_separation /* wait another 5 seconds */)) { Harness.Log("{2} timeout ({1} s) reached, will now send SIGKILL to the app (PID: {0})", pid, timeout.TotalSeconds, type); kill(pid, 9 /* SIGKILL */); // terminate unconditionally. } } }
public int Run() { HashSet <string> start_crashes = null; string device_system_log = null; Initialize(); var args = new StringBuilder(); if (!string.IsNullOrEmpty(Harness.XcodeRoot)) { args.Append(" --sdkroot ").Append(Harness.XcodeRoot); } for (int i = -1; i < Harness.Verbosity; i++) { args.Append(" -v "); } args.Append(" -argument=-connection-mode -argument=none"); // This will prevent the app from trying to connect to any IDEs args.Append(" -argument=-app-arg:-autostart"); args.Append(" -setenv=NUNIT_AUTOSTART=true"); args.Append(" -argument=-app-arg:-autoexit"); args.Append(" -setenv=NUNIT_AUTOEXIT=true"); args.Append(" -argument=-app-arg:-enablenetwork"); args.Append(" -setenv=NUNIT_ENABLE_NETWORK=true"); if (isSimulator) { args.Append(" -argument=-app-arg:-hostname:127.0.0.1"); args.Append(" -setenv=NUNIT_HOSTNAME=127.0.0.1"); } else { var ips = new StringBuilder(); var ipAddresses = System.Net.Dns.GetHostEntry(System.Net.Dns.GetHostName()).AddressList; for (int i = 0; i < ipAddresses.Length; i++) { if (i > 0) { ips.Append(','); } ips.Append(ipAddresses [i].ToString()); } args.AppendFormat(" -argument=-app-arg:-hostname:{0}", ips.ToString()); args.AppendFormat(" -setenv=NUNIT_HOSTNAME={0}", ips.ToString()); } var transport = mode == "watchos" ? "HTTP" : "TCP"; args.AppendFormat(" -argument=-app-arg:-transport:{0}", transport); args.AppendFormat(" -setenv=NUNIT_TRANSPORT={0}", transport); SimpleListener listener; switch (transport) { case "HTTP": listener = new SimpleHttpListener(); break; case "TCP": listener = new SimpleTcpListener(); break; default: throw new NotImplementedException(); } listener.LogPath = Path.GetDirectoryName(Harness.LogFile); listener.LogFile = Path.GetFileName(Harness.LogFile); listener.AutoExit = true; listener.Address = System.Net.IPAddress.Any; listener.Initialize(); args.AppendFormat(" -argument=-app-arg:-hostport:{0}", listener.Port); args.AppendFormat(" -setenv=NUNIT_HOSTPORT={0}", listener.Port); if (File.Exists(Harness.LogFile)) { File.Delete(Harness.LogFile); } bool?success = null; bool timed_out = false; if (isSimulator) { FindSimulator(); Harness.Log("*** Executing {0}/{1} in the simulator ***", appName, mode); PrepareSimulator(); args.Append(" --launchsim"); args.AppendFormat(" \"{0}\" ", launchAppPath); args.Append(" --device=:v2:udid=").Append(simulator.udid).Append(" "); start_crashes = CreateCrashReportsSnapshot(true); listener.StartAsync(); Harness.Log("Starting test run"); var proc = new XProcess() { Harness = Harness, FileName = Harness.MlaunchPath, Arguments = args.ToString(), VerbosityLevel = 0, }; proc.Start(); var launchState = 0; // 0: launching, 1: launch timed out, 2: run timed out, 3: completed var launchMutex = new Mutex(); var runCompleted = new ManualResetEvent(false); ThreadPool.QueueUserWorkItem((v) => { if (!listener.WaitForConnection(TimeSpan.FromMinutes(Harness.LaunchTimeout))) { lock (launchMutex) { if (launchState == 0) { launchState = 1; runCompleted.Set(); } } Harness.Log("Test launch timed out after {0} minute(s).", Harness.LaunchTimeout); } else { Harness.Log("Test run started"); } }); ThreadPool.QueueUserWorkItem((v) => { var rv = proc.WaitForExit(TimeSpan.FromMinutes(Harness.Timeout)); lock (launchMutex) { if (launchState == 0) { launchState = rv ? 3 : 2; } runCompleted.Set(); } if (rv) { Harness.Log("Test run completed"); } else { Harness.Log("Test run timed out after {0} minute(s).", Harness.Timeout); } }); runCompleted.WaitOne(); switch (launchState) { case 1: case 2: success = false; timed_out = true; // find pid var pid = -1; var output = proc.ReadCurrentOutput(); foreach (var line in output.ToString().Split('\n')) { if (line.StartsWith("Application launched. PID = ", StringComparison.Ordinal)) { var pidstr = line.Substring("Application launched. PID = ".Length); if (!int.TryParse(pidstr, out pid)) { Harness.Log("Could not parse pid: {0}", pidstr); } } else if (line.Contains("Xamarin.Hosting: Launched ") && line.Contains(" with pid ")) { var pidstr = line.Substring(line.LastIndexOf(' ')); if (!int.TryParse(pidstr, out pid)) { Harness.Log("Could not parse pid: {0}", pidstr); } } } if (pid > 0) { KillPid(proc, pid, TimeSpan.FromSeconds(5), TimeSpan.FromMinutes(launchState == 1 ? Harness.LaunchTimeout : Harness.Timeout), launchState == 1 ? "Launch" : "Completion"); } else { Harness.Log("Could not find pid in mtouch output."); } // kill mtouch too kill(proc.Id, 9); break; case 3: // Success! break; case 0: // shouldn't happen ever default: throw new Exception($"Invalid launch state: {launchState}"); } listener.Cancel(); // cleanup after us KillEverything(); } else { FindDevice(); Harness.Log("*** Executing {0}/{1} on device ***", appName, mode); args.Append(" --launchdev"); args.AppendFormat(" \"{0}\" ", launchAppPath); AddDeviceName(args); device_system_log = Harness.LogFile + ".device.log"; var logdev = new DeviceLogCapturer() { Harness = Harness, LogPath = device_system_log, DeviceName = device_name, }; logdev.StartCapture(); start_crashes = CreateCrashReportsSnapshot(false); listener.StartAsync(); Harness.Log("Starting test run"); ExecuteCommand(Harness.MlaunchPath, args.ToString()); if (listener.WaitForCompletion(TimeSpan.FromMinutes(Harness.Timeout))) { Harness.Log("Test run completed"); } else { Harness.Log("Test run did not complete in {0} minutes.", Harness.Timeout); listener.Cancel(); success = false; timed_out = true; } logdev.StopCapture(); // Upload the system log if (File.Exists(device_system_log)) { Harness.Log(1, "A capture of the device log is: {0}", device_system_log); if (Harness.InWrench) { Harness.LogWrench("@MonkeyWrench: AddFile: {0}", Path.GetFullPath(device_system_log)); } } } listener.Dispose(); // check the final status var crashed = false; if (File.Exists(Harness.LogFile)) { Harness.LogWrench("@MonkeyWrench: AddFile: {0}", Path.GetFullPath(Harness.LogFile)); var log = File.ReadAllText(Harness.LogFile); if (log.Contains("Tests run")) { var tests_run = string.Empty; var log_lines = File.ReadAllLines(Harness.LogFile); var failed = false; foreach (var line in log_lines) { if (line.Contains("Tests run:")) { Console.WriteLine(line); tests_run = line.Replace("Tests run: ", ""); break; } else if (line.Contains("FAIL")) { Console.WriteLine(line); failed = true; } } if (failed) { Harness.LogWrench("@MonkeyWrench: AddSummary: <b>{0} failed: {1}</b><br/>", mode, tests_run); Harness.Log("Test run failed"); } else { Harness.LogWrench("@MonkeyWrench: AddSummary: {0} succeeded: {1}<br/>", mode, tests_run); Harness.Log("Test run succeeded"); success = true; } } else if (timed_out) { Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} timed out</i></b><br/>", mode); } else { Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} crashed</i></b><br/>", mode); Harness.Log("Test run crashed"); crashed = true; } } else if (timed_out) { Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} never launched</i></b><br/>", mode); Harness.Log("Test run never launched"); } else { Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} crashed at startup (no log)</i></b><br/>", mode); Harness.Log("Test run crashed before it started (no log file produced)"); crashed = true; } // Check for crash reports var crash_report_search_done = false; var crash_report_search_timeout = 5; var watch = new Stopwatch(); watch.Start(); do { var end_crashes = CreateCrashReportsSnapshot(isSimulator); end_crashes.ExceptWith(start_crashes); if (end_crashes.Count > 0) { Harness.Log("Found {0} new crash report(s)", end_crashes.Count); if (!isSimulator) { // Download crash reports from the device. We put them in the project directory so that they're automatically deleted on wrench // (if we put them in /tmp, they'd never be deleted). var downloaded_crash_reports = new HashSet <string> (); foreach (var file in end_crashes) { var crash_report_target = Path.Combine(Path.GetDirectoryName(ProjectFile), Path.GetFileName(file)); if (ExecuteCommand(Harness.MlaunchPath, "--download-crash-report=" + file + " --download-crash-report-to=" + crash_report_target + " --sdkroot " + Harness.XcodeRoot)) { Harness.Log("Downloaded crash report {0} to {1}", file, crash_report_target); crash_report_target = SymbolicateCrashReport(crash_report_target); downloaded_crash_reports.Add(crash_report_target); } else { Harness.Log("Could not download crash report {0}", file); } } end_crashes = downloaded_crash_reports; } foreach (var cp in end_crashes) { Harness.LogWrench("@MonkeyWrench: AddFile: {0}", Path.GetFullPath(cp)); Harness.Log(" {0}", cp); } crash_report_search_done = true; } else if (!crashed && !timed_out) { crash_report_search_done = true; } else { if (watch.Elapsed.TotalSeconds > crash_report_search_timeout) { crash_report_search_done = true; } else { Harness.Log("No crash reports, waiting a second to see if the crash report service just didn't complete in time ({0})", (int)(crash_report_search_timeout - watch.Elapsed.TotalSeconds)); Thread.Sleep(TimeSpan.FromSeconds(1)); } } } while (!crash_report_search_done); if (!success.HasValue) { success = false; } if (isSimulator) { foreach (var sim in simulators) { // Upload the system log if (File.Exists(sim.system_log)) { Harness.Log(success.Value ? 1 : 0, "System log for the '{1}' simulator is: {0}", sim.system_log, sim.name); if (Harness.InWrench) { var syslog = Harness.LogFile + (sim == simulator ? ".system.log" : ".companion.system.log"); File.Copy(sim.system_log, syslog, true); Harness.LogWrench("@MonkeyWrench: AddFile: {0}", Path.GetFullPath(syslog)); } } } } return(success.Value ? 0 : 1); }