public async Task <int> RunAsync() { CrashReportSnapshot crash_reports; LogStream device_system_log = null; LogStream listener_log = null; Log run_log = main_log; Initialize(); crash_reports = new CrashReportSnapshot() { Device = !isSimulator, Harness = Harness, Log = main_log, Logs = Logs, LogDirectory = LogDirectory }; var args = new StringBuilder(); if (!string.IsNullOrEmpty(Harness.XcodeRoot)) { args.Append(" --sdkroot ").Append(Harness.XcodeRoot); } for (int i = -1; i < Harness.Verbosity; i++) { args.Append(" -v "); } args.Append(" -argument=-connection-mode -argument=none"); // This will prevent the app from trying to connect to any IDEs args.Append(" -argument=-app-arg:-autostart"); args.Append(" -setenv=NUNIT_AUTOSTART=true"); args.Append(" -argument=-app-arg:-autoexit"); args.Append(" -setenv=NUNIT_AUTOEXIT=true"); args.Append(" -argument=-app-arg:-enablenetwork"); args.Append(" -setenv=NUNIT_ENABLE_NETWORK=true"); if (isSimulator) { args.Append(" -argument=-app-arg:-hostname:127.0.0.1"); args.Append(" -setenv=NUNIT_HOSTNAME=127.0.0.1"); } else { var ips = new StringBuilder(); var ipAddresses = System.Net.Dns.GetHostEntry(System.Net.Dns.GetHostName()).AddressList; for (int i = 0; i < ipAddresses.Length; i++) { if (i > 0) { ips.Append(','); } ips.Append(ipAddresses [i].ToString()); } args.AppendFormat(" -argument=-app-arg:-hostname:{0}", ips.ToString()); args.AppendFormat(" -setenv=NUNIT_HOSTNAME={0}", ips.ToString()); } var transport = mode == "watchos" ? "HTTP" : "TCP"; args.AppendFormat(" -argument=-app-arg:-transport:{0}", transport); args.AppendFormat(" -setenv=NUNIT_TRANSPORT={0}", transport); SimpleListener listener; switch (transport) { case "HTTP": listener = new SimpleHttpListener(); break; case "TCP": listener = new SimpleTcpListener(); break; default: throw new NotImplementedException(); } listener_log = Logs.CreateStream(LogDirectory, string.Format("test-{0:yyyyMMdd_HHmmss}.log", DateTime.Now), "Test log"); listener.TestLog = listener_log; listener.Log = main_log; listener.AutoExit = true; listener.Address = System.Net.IPAddress.Any; listener.Initialize(); args.AppendFormat(" -argument=-app-arg:-hostport:{0}", listener.Port); args.AppendFormat(" -setenv=NUNIT_HOSTPORT={0}", listener.Port); foreach (var kvp in Harness.EnvironmentVariables) { args.AppendFormat(" -setenv={0}={1}", kvp.Key, kvp.Value); } bool?success = null; bool timed_out = false; if (isSimulator) { FindSimulator(); var systemLogs = new List <CaptureLog> (); foreach (var sim in simulators) { // Upload the system log main_log.WriteLine("System log for the '{1}' simulator is: {0}", sim.SystemLog, sim.Name); bool isCompanion = sim != simulator; var log = new CaptureLog(sim.SystemLog) { Path = Path.Combine(LogDirectory, sim.UDID + ".log"), Description = isCompanion ? "System log (companion)" : "System log", }; log.StartCapture(); Logs.Add(log); systemLogs.Add(log); Harness.LogWrench("@MonkeyWrench: AddFile: {0}", log.Path); } main_log.WriteLine("*** Executing {0}/{1} in the simulator ***", appName, mode); if (EnsureCleanSimulatorState) { foreach (var sim in simulators) { await sim.PrepareSimulatorAsync(main_log, bundle_identifier); } } args.Append(" --launchsim"); args.AppendFormat(" \"{0}\" ", launchAppPath); args.Append(" --device=:v2:udid=").Append(simulator.UDID).Append(" "); await crash_reports.StartCaptureAsync(); listener.StartAsync(); main_log.WriteLine("Starting test run"); var cancellation_source = new CancellationTokenSource(); ThreadPool.QueueUserWorkItem((v) => { if (!listener.WaitForConnection(TimeSpan.FromMinutes(Harness.LaunchTimeout))) { cancellation_source.Cancel(); main_log.WriteLine("Test launch timed out after {0} minute(s).", Harness.LaunchTimeout); timed_out = true; } else { main_log.WriteLine("Test run started"); } }); var result = await ProcessHelper.ExecuteCommandAsync(Harness.MlaunchPath, args.ToString(), run_log, TimeSpan.FromMinutes(Harness.Timeout), cancellation_token : cancellation_source.Token); if (result.TimedOut) { timed_out = true; success = false; main_log.WriteLine("Test run timed out after {0} minute(s).", Harness.Timeout); } else if (result.Succeeded) { main_log.WriteLine("Test run completed"); success = true; } else { main_log.WriteLine("Test run failed"); success = false; } if (!success.Value) { // find pid var pid = -1; using (var reader = run_log.GetReader()) { while (!reader.EndOfStream) { var line = reader.ReadLine(); if (line.StartsWith("Application launched. PID = ", StringComparison.Ordinal)) { var pidstr = line.Substring("Application launched. PID = ".Length); if (!int.TryParse(pidstr, out pid)) { main_log.WriteLine("Could not parse pid: {0}", pidstr); } } else if (line.Contains("Xamarin.Hosting: Launched ") && line.Contains(" with pid ")) { var pidstr = line.Substring(line.LastIndexOf(' ')); if (!int.TryParse(pidstr, out pid)) { main_log.WriteLine("Could not parse pid: {0}", pidstr); } } } } if (pid > 0) { var launchTimedout = cancellation_source.IsCancellationRequested; await KillPidAsync(main_log, pid, TimeSpan.FromSeconds(5), TimeSpan.FromMinutes(launchTimedout ? Harness.LaunchTimeout : Harness.Timeout), launchTimedout? "Launch" : "Completion"); } else { main_log.WriteLine("Could not find pid in mtouch output."); } } listener.Cancel(); // cleanup after us if (EnsureCleanSimulatorState) { await SimDevice.KillEverythingAsync(main_log); } foreach (var log in systemLogs) { log.StopCapture(); } } else { FindDevice(); main_log.WriteLine("*** Executing {0}/{1} on device ***", appName, mode); args.Append(" --launchdev"); args.AppendFormat(" \"{0}\" ", launchAppPath); var waits_for_exit = false; if (mode == "watchos") { args.Append(" --attach-native-debugger"); // this prevents the watch from backgrounding the app. waits_for_exit = true; } AddDeviceName(args); device_system_log = Logs.CreateStream(LogDirectory, "device.log", "Device log"); var logdev = new DeviceLogCapturer() { Harness = Harness, Log = device_system_log, DeviceName = device_name, }; logdev.StartCapture(); await crash_reports.StartCaptureAsync(); listener.StartAsync(); main_log.WriteLine("Starting test run"); double launch_timeout = waits_for_exit ? Harness.Timeout : 1; double listener_timeout = waits_for_exit ? 0.2 : Harness.Timeout; await ProcessHelper.ExecuteCommandAsync(Harness.MlaunchPath, args.ToString(), main_log, TimeSpan.FromMinutes(launch_timeout)); if (listener.WaitForCompletion(TimeSpan.FromMinutes(listener_timeout))) { main_log.WriteLine("Test run completed"); } else { main_log.WriteLine("Test run did not complete in {0} minutes.", Harness.Timeout); listener.Cancel(); success = false; timed_out = true; } logdev.StopCapture(); // Upload the system log if (File.Exists(device_system_log.FullPath)) { main_log.WriteLine("A capture of the device log is: {0}", device_system_log.FullPath); if (Harness.InWrench) { Harness.LogWrench("@MonkeyWrench: AddFile: {0}", device_system_log.FullPath); } } } listener.Dispose(); // check the final status var crashed = false; if (File.Exists(listener_log.FullPath)) { Harness.LogWrench("@MonkeyWrench: AddFile: {0}", listener_log.FullPath); string log; using (var reader = listener_log.GetReader()) log = reader.ReadToEnd(); if (log.Contains("Tests run")) { var tests_run = string.Empty; var log_lines = log.Split('\n'); var failed = false; foreach (var line in log_lines) { if (line.Contains("Tests run:")) { Console.WriteLine(line); tests_run = line.Replace("Tests run: ", ""); break; } else if (line.Contains("FAIL")) { Console.WriteLine(line); failed = true; } } if (failed) { Harness.LogWrench("@MonkeyWrench: AddSummary: <b>{0} failed: {1}</b><br/>", mode, tests_run); main_log.WriteLine("Test run failed"); success = false; } else { Harness.LogWrench("@MonkeyWrench: AddSummary: {0} succeeded: {1}<br/>", mode, tests_run); main_log.WriteLine("Test run succeeded"); success = true; } } else if (timed_out) { Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} timed out</i></b><br/>", mode); } else { Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} crashed</i></b><br/>", mode); main_log.WriteLine("Test run crashed"); crashed = true; } } else if (timed_out) { Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} never launched</i></b><br/>", mode); main_log.WriteLine("Test run never launched"); } else { Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} crashed at startup (no log)</i></b><br/>", mode); main_log.WriteLine("Test run crashed before it started (no log file produced)"); crashed = true; } if (!success.HasValue) { success = false; } await crash_reports.EndCaptureAsync(TimeSpan.FromSeconds(success.Value ? 0 : 5)); if (timed_out) { Result = TestExecutingResult.TimedOut; } else if (crashed) { Result = TestExecutingResult.Crashed; } else if (success.Value) { Result = TestExecutingResult.Succeeded; } else { Result = TestExecutingResult.Failed; } return(success.Value ? 0 : 1); }
public SimDevice FindCompanionDevice(Log log, SimDevice device) { var pair = available_device_pairs.Where((v) => v.Gizmo == device.UDID).Single(); return(available_devices.Single((v) => v.UDID == pair.Companion)); }
public async Task <SimDevice []> FindAsync(AppRunnerTarget target, Log log) { SimDevice [] simulators = null; string [] simulator_devicetypes; string simulator_runtime; string [] companion_devicetypes = null; string companion_runtime = null; switch (target) { case AppRunnerTarget.Simulator_iOS32: simulator_devicetypes = new string [] { "com.apple.CoreSimulator.SimDeviceType.iPhone-5" }; simulator_runtime = "com.apple.CoreSimulator.SimRuntime.iOS-10-3"; break; case AppRunnerTarget.Simulator_iOS64: simulator_devicetypes = new string [] { "com.apple.CoreSimulator.SimDeviceType.iPhone-6" }; simulator_runtime = "com.apple.CoreSimulator.SimRuntime.iOS-" + Xamarin.SdkVersions.iOS.Replace('.', '-'); break; case AppRunnerTarget.Simulator_iOS: simulator_devicetypes = new string [] { "com.apple.CoreSimulator.SimDeviceType.iPhone-5" }; simulator_runtime = "com.apple.CoreSimulator.SimRuntime.iOS-" + Xamarin.SdkVersions.iOS.Replace('.', '-'); break; case AppRunnerTarget.Simulator_tvOS: simulator_devicetypes = new string [] { "com.apple.CoreSimulator.SimDeviceType.Apple-TV-1080p" }; simulator_runtime = "com.apple.CoreSimulator.SimRuntime.tvOS-" + Xamarin.SdkVersions.TVOS.Replace('.', '-'); break; case AppRunnerTarget.Simulator_watchOS: simulator_devicetypes = new string [] { "com.apple.CoreSimulator.SimDeviceType.Apple-Watch-38mm", "com.apple.CoreSimulator.SimDeviceType.Apple-Watch-Series-2-38mm" }; simulator_runtime = "com.apple.CoreSimulator.SimRuntime.watchOS-" + Xamarin.SdkVersions.WatchOS.Replace('.', '-'); companion_devicetypes = new string [] { "com.apple.CoreSimulator.SimDeviceType.iPhone-6s" }; companion_runtime = "com.apple.CoreSimulator.SimRuntime.iOS-" + Xamarin.SdkVersions.iOS.Replace('.', '-'); break; default: throw new Exception(string.Format("Unknown simulator target: {0}", target)); } var devices = AvailableDevices.Where((SimDevice v) => { if (v.SimRuntime != simulator_runtime) { return(false); } if (!simulator_devicetypes.Contains(v.SimDeviceType)) { return(false); } if (target == AppRunnerTarget.Simulator_watchOS) { return(AvailableDevicePairs.Any((SimDevicePair pair) => pair.Companion == v.UDID || pair.Gizmo == v.UDID)); } return(true); }); SimDevice candidate = null; foreach (var device in devices) { var data = device; var secondaryData = (SimDevice)null; var nodeCompanions = AvailableDevicePairs.Where((SimDevicePair v) => v.Companion == device.UDID); var nodeGizmos = AvailableDevicePairs.Where((SimDevicePair v) => v.Gizmo == device.UDID); if (nodeCompanions.Any()) { var gizmo_udid = nodeCompanions.First().Gizmo; var node = AvailableDevices.Where((SimDevice v) => v.UDID == gizmo_udid); secondaryData = node.First(); } else if (nodeGizmos.Any()) { var companion_udid = nodeGizmos.First().Companion; var node = AvailableDevices.Where((SimDevice v) => v.UDID == companion_udid); secondaryData = node.First(); } if (secondaryData != null) { simulators = new SimDevice [] { data, secondaryData }; break; } else { candidate = data; } } if (simulators == null && candidate == null && target == AppRunnerTarget.Simulator_watchOS) { // We might be only missing device pairs to match phone + watch. var watchDevices = AvailableDevices.Where((SimDevice v) => { return(v.SimRuntime == simulator_runtime && simulator_devicetypes.Contains(v.SimDeviceType)); }); var companionDevices = AvailableDevices.Where((SimDevice v) => { return(v.SimRuntime == companion_runtime && companion_devicetypes.Contains(v.SimDeviceType)); }); if (!watchDevices.Any() || !companionDevices.Any()) { log.WriteLine($"Could not find both watch devices for <runtime={simulator_runtime} and device type={string.Join (";", simulator_devicetypes)}> and companion device for <runtime={companion_runtime} and device type {string.Join (";", companion_devicetypes)}>"); return(null); } var watchDevice = watchDevices.First(); var companionDevice = companionDevices.First(); log.WriteLine($"Creating device pair for '{watchDevice.Name}' and '{companionDevice.Name}'"); var rv = await Harness.ExecuteXcodeCommandAsync("simctl", $"pair {watchDevice.UDID} {companionDevice.UDID}", log, TimeSpan.FromMinutes(1)); if (!rv.Succeeded) { log.WriteLine($"Could not create device pair, so could not find simulator for runtime={simulator_runtime} and device type={string.Join ("; ", simulator_devicetypes)}."); return(null); } available_device_pairs.Add(new SimDevicePair() { Companion = companionDevice.UDID, Gizmo = watchDevice.UDID, UDID = $"<created for {companionDevice.UDID} and {watchDevice.UDID}", }); simulators = new SimDevice [] { watchDevice, companionDevice }; } if (simulators == null) { if (candidate == null) { log.WriteLine($"Could not find simulator for runtime={simulator_runtime} and device type={string.Join (";", simulator_devicetypes)}."); return(null); } simulators = new SimDevice [] { candidate }; } if (simulators == null) { log.WriteLine("Could not find simulator"); return(null); } log.WriteLine("Found simulator: {0} {1}", simulators [0].Name, simulators [0].UDID); if (simulators.Length > 1) { log.WriteLine("Found companion simulator: {0} {1}", simulators [1].Name, simulators [1].UDID); } return(simulators); }
public async Task <SimDevice []> FindAsync(AppRunnerTarget target, Log log, bool create_if_needed = true) { SimDevice [] simulators = null; string simulator_devicetype; string simulator_runtime; string companion_devicetype = null; string companion_runtime = null; switch (target) { case AppRunnerTarget.Simulator_iOS32: simulator_devicetype = "com.apple.CoreSimulator.SimDeviceType.iPhone-5"; simulator_runtime = "com.apple.CoreSimulator.SimRuntime.iOS-10-3"; break; case AppRunnerTarget.Simulator_iOS64: simulator_devicetype = "com.apple.CoreSimulator.SimDeviceType.iPhone-X"; simulator_runtime = "com.apple.CoreSimulator.SimRuntime.iOS-" + Xamarin.SdkVersions.iOS.Replace('.', '-'); break; case AppRunnerTarget.Simulator_iOS: simulator_devicetype = "com.apple.CoreSimulator.SimDeviceType.iPhone-5"; simulator_runtime = "com.apple.CoreSimulator.SimRuntime.iOS-" + Xamarin.SdkVersions.iOS.Replace('.', '-'); break; case AppRunnerTarget.Simulator_tvOS: simulator_devicetype = "com.apple.CoreSimulator.SimDeviceType.Apple-TV-1080p"; simulator_runtime = "com.apple.CoreSimulator.SimRuntime.tvOS-" + Xamarin.SdkVersions.TVOS.Replace('.', '-'); break; case AppRunnerTarget.Simulator_watchOS: simulator_devicetype = "com.apple.CoreSimulator.SimDeviceType.Apple-Watch-Series-3-38mm"; simulator_runtime = "com.apple.CoreSimulator.SimRuntime.watchOS-" + Xamarin.SdkVersions.WatchOS.Replace('.', '-'); companion_devicetype = "com.apple.CoreSimulator.SimDeviceType.iPhone-X"; companion_runtime = "com.apple.CoreSimulator.SimRuntime.iOS-" + Xamarin.SdkVersions.iOS.Replace('.', '-'); break; default: throw new Exception(string.Format("Unknown simulator target: {0}", target)); } var devices = await FindOrCreateDevicesAsync(log, simulator_runtime, simulator_devicetype); var companion_devices = await FindOrCreateDevicesAsync(log, companion_runtime, companion_devicetype); if (devices?.Any() != true) { log.WriteLine($"Could not find or create devices runtime={simulator_runtime} and device type={simulator_devicetype}."); return(null); } if (companion_runtime == null) { simulators = new SimDevice [] { devices.First() }; } else { if (companion_devices?.Any() != true) { log.WriteLine($"Could not find or create companion devices runtime={companion_runtime} and device type={companion_devicetype}."); return(null); } var pair = await FindOrCreateDevicePairAsync(log, devices, companion_devices); if (pair == null) { log.WriteLine($"Could not find or create device pair runtime={companion_runtime} and device type={companion_devicetype}."); return(null); } simulators = new SimDevice [] { devices.First((v) => v.UDID == pair.Gizmo), companion_devices.First((v) => v.UDID == pair.Companion), }; } if (simulators == null) { log.WriteLine($"Could not find simulator for runtime={simulator_runtime} and device type={simulator_devicetype}."); return(null); } log.WriteLine("Found simulator: {0} {1}", simulators [0].Name, simulators [0].UDID); if (simulators.Length > 1) { log.WriteLine("Found companion simulator: {0} {1}", simulators [1].Name, simulators [1].UDID); } return(simulators); }
public async Task <int> RunAsync() { CrashReportSnapshot crash_reports; Log device_system_log = null; Log listener_log = null; Log run_log = main_log; Initialize(); if (!isSimulator) { FindDevice(); } crash_reports = new CrashReportSnapshot() { Device = !isSimulator, DeviceName = device_name, Harness = Harness, Log = main_log, Logs = Logs, LogDirectory = LogDirectory, }; var args = new StringBuilder(); if (!string.IsNullOrEmpty(Harness.XcodeRoot)) { args.Append(" --sdkroot ").Append(Harness.XcodeRoot); } for (int i = -1; i < Harness.Verbosity; i++) { args.Append(" -v "); } args.Append(" -argument=-connection-mode -argument=none"); // This will prevent the app from trying to connect to any IDEs args.Append(" -argument=-app-arg:-autostart"); args.Append(" -setenv=NUNIT_AUTOSTART=true"); args.Append(" -argument=-app-arg:-autoexit"); args.Append(" -setenv=NUNIT_AUTOEXIT=true"); args.Append(" -argument=-app-arg:-enablenetwork"); args.Append(" -setenv=NUNIT_ENABLE_NETWORK=true"); // detect if we are using a jenkins bot. var useXmlOutput = Harness.InJenkins; if (useXmlOutput) { args.Append(" -setenv=NUNIT_ENABLE_XML_OUTPUT=true"); args.Append(" -setenv=NUNIT_ENABLE_XML_MODE=wrapped"); } if (!Harness.IncludeSystemPermissionTests) { args.Append(" -setenv=DISABLE_SYSTEM_PERMISSION_TESTS=1"); } if (isSimulator) { args.Append(" -argument=-app-arg:-hostname:127.0.0.1"); args.Append(" -setenv=NUNIT_HOSTNAME=127.0.0.1"); } else { var ips = new StringBuilder(); var ipAddresses = System.Net.Dns.GetHostEntry(System.Net.Dns.GetHostName()).AddressList; for (int i = 0; i < ipAddresses.Length; i++) { if (i > 0) { ips.Append(','); } ips.Append(ipAddresses [i].ToString()); } args.AppendFormat(" -argument=-app-arg:-hostname:{0}", ips.ToString()); args.AppendFormat(" -setenv=NUNIT_HOSTNAME={0}", ips.ToString()); } string transport; if (mode == "watchos") { transport = isSimulator ? "FILE" : "HTTP"; } else { transport = "TCP"; } args.AppendFormat(" -argument=-app-arg:-transport:{0}", transport); args.AppendFormat(" -setenv=NUNIT_TRANSPORT={0}", transport); listener_log = Logs.Create($"test-{mode}-{Harness.Timestamp}.log", "Test log"); SimpleListener listener; switch (transport) { case "FILE": var fn = listener_log.FullPath + ".tmp"; listener = new SimpleFileListener(fn); args.Append(" -setenv=NUNIT_LOG_FILE=").Append(StringUtils.Quote(fn)); break; case "HTTP": listener = new SimpleHttpListener(); break; case "TCP": listener = new SimpleTcpListener(); break; default: throw new NotImplementedException(); } listener.TestLog = listener_log; listener.Log = main_log; listener.AutoExit = true; listener.Address = System.Net.IPAddress.Any; listener.XmlOutput = useXmlOutput; listener.Initialize(); args.AppendFormat(" -argument=-app-arg:-hostport:{0}", listener.Port); args.AppendFormat(" -setenv=NUNIT_HOSTPORT={0}", listener.Port); listener.StartAsync(); var cancellation_source = new CancellationTokenSource(); var timed_out = false; ThreadPool.QueueUserWorkItem((v) => { if (!listener.WaitForConnection(TimeSpan.FromMinutes(Harness.LaunchTimeout))) { cancellation_source.Cancel(); main_log.WriteLine("Test launch timed out after {0} minute(s).", Harness.LaunchTimeout); timed_out = true; } else { main_log.WriteLine("Test run started"); } }); foreach (var kvp in Harness.EnvironmentVariables) { args.AppendFormat(" -setenv={0}={1}", kvp.Key, kvp.Value); } bool?success = null; bool launch_failure = false; if (isExtension) { switch (extension) { case Extension.TodayExtension: args.Append(isSimulator ? " --launchsimbundleid" : " --launchdevbundleid"); args.Append(" todayviewforextensions:"); args.Append(BundleIdentifier); args.Append(" --observe-extension "); args.Append(StringUtils.Quote(launchAppPath)); break; case Extension.WatchKit2: default: throw new NotImplementedException(); } } else { args.Append(isSimulator ? " --launchsim " : " --launchdev "); args.Append(StringUtils.Quote(launchAppPath)); } if (!isSimulator) { args.Append(" --disable-memory-limits"); } if (isSimulator) { if (!await FindSimulatorAsync()) { return(1); } if (mode != "watchos") { var stderr_tty = Marshal.PtrToStringAuto(ttyname(2)); if (!string.IsNullOrEmpty(stderr_tty)) { args.Append(" --stdout=").Append(StringUtils.Quote(stderr_tty)); args.Append(" --stderr=").Append(StringUtils.Quote(stderr_tty)); } else { var stdout_log = Logs.CreateFile($"stdout-{Harness.Timestamp}.log", "Standard output"); var stderr_log = Logs.CreateFile($"stderr-{Harness.Timestamp}.log", "Standard error"); args.Append(" --stdout=").Append(StringUtils.Quote(stdout_log)); args.Append(" --stderr=").Append(StringUtils.Quote(stderr_log)); } } var systemLogs = new List <CaptureLog> (); foreach (var sim in simulators) { // Upload the system log main_log.WriteLine("System log for the '{1}' simulator is: {0}", sim.SystemLog, sim.Name); bool isCompanion = sim != simulator; var log = new CaptureLog(Logs, sim.SystemLog, entire_file: Harness.Action != HarnessAction.Jenkins) { Path = Path.Combine(LogDirectory, sim.Name + ".log"), Description = isCompanion ? "System log (companion)" : "System log", }; log.StartCapture(); Logs.Add(log); systemLogs.Add(log); Harness.LogWrench("@MonkeyWrench: AddFile: {0}", log.Path); } main_log.WriteLine("*** Executing {0}/{1} in the simulator ***", appName, mode); if (EnsureCleanSimulatorState) { foreach (var sim in simulators) { await sim.PrepareSimulatorAsync(main_log, bundle_identifier); } } args.Append(" --device=:v2:udid=").Append(simulator.UDID).Append(" "); await crash_reports.StartCaptureAsync(); main_log.WriteLine("Starting test run"); var result = await ProcessHelper.ExecuteCommandAsync(Harness.MlaunchPath, args.ToString(), run_log, TimeSpan.FromMinutes(Harness.Timeout), cancellation_token : cancellation_source.Token); if (result.TimedOut) { timed_out = true; success = false; main_log.WriteLine("Test run timed out after {0} minute(s).", Harness.Timeout); } else if (result.Succeeded) { main_log.WriteLine("Test run completed"); success = true; } else { main_log.WriteLine("Test run failed"); success = false; } if (!success.Value) { // find pid var pid = -1; using (var reader = run_log.GetReader()) { while (!reader.EndOfStream) { var line = reader.ReadLine(); if (line.StartsWith("Application launched. PID = ", StringComparison.Ordinal)) { var pidstr = line.Substring("Application launched. PID = ".Length); if (!int.TryParse(pidstr, out pid)) { main_log.WriteLine("Could not parse pid: {0}", pidstr); } } else if (line.Contains("Xamarin.Hosting: Launched ") && line.Contains(" with pid ")) { var pidstr = line.Substring(line.LastIndexOf(' ')); if (!int.TryParse(pidstr, out pid)) { main_log.WriteLine("Could not parse pid: {0}", pidstr); } } else if (line.Contains("error MT1008")) { launch_failure = true; } } } if (pid > 0) { var launchTimedout = cancellation_source.IsCancellationRequested; var timeoutType = launchTimedout ? "Launch" : "Completion"; var timeoutValue = launchTimedout ? Harness.LaunchTimeout : Harness.Timeout; main_log.WriteLine($"{timeoutType} timed out after {timeoutValue}"); await Process_Extensions.KillTreeAsync(pid, main_log, true); } else { main_log.WriteLine("Could not find pid in mtouch output."); } } // cleanup after us if (EnsureCleanSimulatorState) { await SimDevice.KillEverythingAsync(main_log); } foreach (var log in systemLogs) { log.StopCapture(); } } else { main_log.WriteLine("*** Executing {0}/{1} on device '{2}' ***", appName, mode, device_name); if (mode == "watchos") { args.Append(" --attach-native-debugger"); // this prevents the watch from backgrounding the app. } else { args.Append(" --wait-for-exit"); } AddDeviceName(args); device_system_log = Logs.Create($"device-{device_name}-{Harness.Timestamp}.log", "Device log"); var logdev = new DeviceLogCapturer() { Harness = Harness, Log = device_system_log, DeviceName = device_name, }; logdev.StartCapture(); await crash_reports.StartCaptureAsync(); main_log.WriteLine("Starting test run"); bool waitedForExit = true; // We need to check for MT1111 (which means that mlaunch won't wait for the app to exit). var callbackLog = new CallbackLog((line) => { // MT1111: Application launched successfully, but it's not possible to wait for the app to exit as requested because it's not possible to detect app termination when launching using gdbserver waitedForExit &= line?.Contains("MT1111: ") != true; if (line?.Contains("error MT1007") == true) { launch_failure = true; } }); var runLog = Log.CreateAggregatedLog(callbackLog, main_log); var timeout = TimeSpan.FromMinutes(Harness.Timeout); var timeoutWatch = Stopwatch.StartNew(); var result = await ProcessHelper.ExecuteCommandAsync(Harness.MlaunchPath, args.ToString(), runLog, timeout, cancellation_token : cancellation_source.Token); if (!waitedForExit && !result.TimedOut) { // mlaunch couldn't wait for exit for some reason. Let's assume the app exits when the test listener completes. main_log.WriteLine("Waiting for listener to complete, since mlaunch won't tell."); if (!await listener.CompletionTask.TimeoutAfter(timeout - timeoutWatch.Elapsed)) { result.TimedOut = true; } } if (result.TimedOut) { timed_out = true; success = false; main_log.WriteLine("Test run timed out after {0} minute(s).", Harness.Timeout); } else if (result.Succeeded) { main_log.WriteLine("Test run completed"); success = true; } else { main_log.WriteLine("Test run failed"); success = false; } logdev.StopCapture(); device_system_log.Dispose(); // Upload the system log if (File.Exists(device_system_log.FullPath)) { main_log.WriteLine("A capture of the device log is: {0}", device_system_log.FullPath); Harness.LogWrench("@MonkeyWrench: AddFile: {0}", device_system_log.FullPath); } } listener.Cancel(); listener.Dispose(); // check the final status var crashed = false; if (File.Exists(listener_log.FullPath)) { Harness.LogWrench("@MonkeyWrench: AddFile: {0}", listener_log.FullPath); success = TestsSucceeded(listener_log, timed_out, crashed); } else if (timed_out) { Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} never launched</i></b><br/>", mode); main_log.WriteLine("Test run never launched"); success = false; } else if (launch_failure) { Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} failed to launch</i></b><br/>", mode); main_log.WriteLine("Test run failed to launch"); success = false; } else { Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} crashed at startup (no log)</i></b><br/>", mode); main_log.WriteLine("Test run crashed before it started (no log file produced)"); crashed = true; success = false; } if (!success.HasValue) { success = false; } await crash_reports.EndCaptureAsync(TimeSpan.FromSeconds(success.Value ? 0 : 5)); if (timed_out) { Result = TestExecutingResult.TimedOut; } else if (crashed) { Result = TestExecutingResult.Crashed; } else if (success.Value) { Result = TestExecutingResult.Succeeded; } else { Result = TestExecutingResult.Failed; } // Check crash reports to see if any of them explains why the test run crashed. if (!success.Value) { int pid = 0; string crash_reason = null; foreach (var crash in crash_reports.Logs) { try { if (pid == 0) { // Find the pid using (var log_reader = main_log.GetReader()) { string line; while ((line = log_reader.ReadLine()) != null) { const string str = "was launched with pid '"; var idx = line.IndexOf(str, StringComparison.Ordinal); if (idx > 0) { idx += str.Length; var next_idx = line.IndexOf('\'', idx); if (next_idx > idx) { int.TryParse(line.Substring(idx, next_idx - idx), out pid); } } if (pid != 0) { break; } } } } using (var crash_reader = crash.GetReader()) { var text = crash_reader.ReadToEnd(); var reader = System.Runtime.Serialization.Json.JsonReaderWriterFactory.CreateJsonReader(Encoding.UTF8.GetBytes(text), new XmlDictionaryReaderQuotas()); var doc = new XmlDocument(); doc.Load(reader); foreach (XmlNode node in doc.SelectNodes($"/root/processes/item[pid = '" + pid + "']")) { Console.WriteLine(node?.InnerXml); Console.WriteLine(node?.SelectSingleNode("reason")?.InnerText); crash_reason = node?.SelectSingleNode("reason")?.InnerText; } } if (crash_reason != null) { break; } } catch (Exception e) { Harness.Log(2, "Failed to process crash report '{1}': {0}", e.Message, crash.Description); } } if (!string.IsNullOrEmpty(crash_reason)) { if (crash_reason == "per-process-limit") { FailureMessage = "Killed due to using too much memory (per-process-limit)."; } else { FailureMessage = $"Killed by the OS ({crash_reason})"; } } else if (launch_failure) { FailureMessage = $"Launch failure"; } } return(success.Value ? 0 : 1); }
void FindSimulator() { if (simulators != null) { return; } string [] simulator_devicetypes; string simulator_runtime; switch (Target) { case "ios-simulator-32": simulator_devicetypes = new string [] { "com.apple.CoreSimulator.SimDeviceType.iPhone-5" }; simulator_runtime = "com.apple.CoreSimulator.SimRuntime.iOS-" + Xamarin.SdkVersions.iOS.Replace('.', '-'); break; case "ios-simulator-64": simulator_devicetypes = new string [] { "com.apple.CoreSimulator.SimDeviceType.iPhone-5s" }; simulator_runtime = "com.apple.CoreSimulator.SimRuntime.iOS-" + Xamarin.SdkVersions.iOS.Replace('.', '-'); break; case "ios-simulator": simulator_devicetypes = new string [] { "com.apple.CoreSimulator.SimDeviceType.iPhone-5" }; simulator_runtime = "com.apple.CoreSimulator.SimRuntime.iOS-" + Xamarin.SdkVersions.iOS.Replace('.', '-'); break; case "tvos-simulator": simulator_devicetypes = new string [] { "com.apple.CoreSimulator.SimDeviceType.Apple-TV-1080p" }; simulator_runtime = "com.apple.CoreSimulator.SimRuntime.tvOS-" + Xamarin.SdkVersions.TVOS.Replace('.', '-'); break; case "watchos-simulator": simulator_devicetypes = new string [] { "com.apple.CoreSimulator.SimDeviceType.Apple-Watch-38mm", "com.apple.CoreSimulator.SimDeviceType.Apple-Watch-Series-2-38mm" }; simulator_runtime = "com.apple.CoreSimulator.SimRuntime.watchOS-" + Xamarin.SdkVersions.WatchOS.Replace('.', '-'); break; default: throw new Exception(string.Format("Unknown simulator target: {0}", Harness.Target)); } var sims = new Simulators() { Harness = Harness, }; Task.Run(async() => { await sims.LoadAsync(Logs.CreateStream(LogDirectory, "simulator-list.log", "Simulator list")); }).Wait(); var devices = sims.AvailableDevices.Where((SimDevice v) => v.SimRuntime == simulator_runtime && simulator_devicetypes.Contains(v.SimDeviceType)); SimDevice candidate = null; simulators = null; foreach (var device in devices) { var data = device; var secondaryData = (SimDevice)null; var nodeCompanions = sims.AvailableDevicePairs.Where((SimDevicePair v) => v.Companion == device.UDID); var nodeGizmos = sims.AvailableDevicePairs.Where((SimDevicePair v) => v.Gizmo == device.UDID); if (nodeCompanions.Any()) { var gizmo_udid = nodeCompanions.First().Gizmo; var node = sims.AvailableDevices.Where((SimDevice v) => v.UDID == gizmo_udid); secondaryData = node.First(); } else if (nodeGizmos.Any()) { var companion_udid = nodeGizmos.First().Companion; var node = sims.AvailableDevices.Where((SimDevice v) => v.UDID == companion_udid); secondaryData = node.First(); } if (secondaryData != null) { simulators = new SimDevice [] { data, secondaryData }; break; } else { candidate = data; } } if (simulators == null) { if (candidate == null) { throw new Exception($"Could not find simulator for runtime={simulator_runtime} and device type={string.Join (";", simulator_devicetypes)}."); } simulators = new SimDevice [] { candidate }; } if (simulators == null) { throw new Exception("Could not find simulator"); } main_log.WriteLine("Found simulator: {0} {1}", simulators [0].Name, simulators [0].UDID); if (simulators.Length > 1) { main_log.WriteLine("Found companion simulator: {0} {1}", simulators [1].Name, simulators [1].UDID); } }