public async Task <(string deviceName, ProcessExecutionResult result)> InstallApp(string appPath, TestTarget target, string deviceName = null, CancellationToken cancellationToken = default) { if (target.IsSimulator()) { // We reset the simulator when running, so a separate install step does not make much sense. throw new InvalidOperationException("Installing to a simulator is not supported."); } if (!Directory.Exists(appPath)) { throw new DirectoryNotFoundException("Failed to find the app bundle directory"); } if (deviceName == null) { var device = await _deviceLoader.FindDevice(target.ToRunMode(), _mainLog, false, false); if (target.IsWatchOSTarget()) { deviceName = (await _deviceLoader.FindCompanionDevice(_mainLog, device)).Name; } else { deviceName = device.Name; } } if (deviceName == null) { throw new NoDeviceFoundException(); } var args = new MlaunchArguments(); for (int i = -1; i < _verbosity; i++) { args.Add(new VerbosityArgument()); } args.Add(new InstallAppOnDeviceArgument(appPath)); args.Add(new DeviceNameArgument(deviceName)); if (target.IsWatchOSTarget()) { args.Add(new DeviceArgument("ios,watchos")); } var totalSize = Directory.GetFiles(appPath, "*", SearchOption.AllDirectories).Select((v) => new FileInfo(v).Length).Sum(); _mainLog.WriteLine($"Installing '{appPath}' to '{deviceName}' ({totalSize / 1024.0 / 1024.0:N2} MB)"); ProcessExecutionResult result = await _processManager.ExecuteCommandAsync(args, _mainLog, TimeSpan.FromHours(1), cancellation_token : cancellationToken); return(deviceName, result); }
protected async Task <string> FindDevice(TestTargetOs target) { IHardwareDevice?companionDevice = null; IHardwareDevice device = await _hardwareDeviceLoader.FindDevice(target.Platform.ToRunMode(), _mainLog, includeLocked : false, force : false); if (target.Platform.IsWatchOSTarget()) { companionDevice = await _hardwareDeviceLoader.FindCompanionDevice(_mainLog, device); } return(companionDevice?.Name ?? device.Name); }
public async Task<(string DeviceName, TestExecutingResult Result, string ResultMessage)> RunApp( AppBundleInformation appInformation, TestTarget target, TimeSpan timeout, TimeSpan testLaunchTimeout, string? deviceName = null, string? companionDeviceName = null, bool ensureCleanSimulatorState = false, int verbosity = 1, XmlResultJargon xmlResultJargon = XmlResultJargon.xUnit, CancellationToken cancellationToken = default) { var args = new MlaunchArguments { new SetAppArgumentArgument("-connection-mode"), new SetAppArgumentArgument("none"), // This will prevent the app from trying to connect to any IDEs new SetAppArgumentArgument("-autostart", true), new SetEnvVariableArgument(EnviromentVariables.AutoStart, true), new SetAppArgumentArgument("-autoexit", true), new SetEnvVariableArgument(EnviromentVariables.AutoExit, true), new SetAppArgumentArgument("-enablenetwork", true), new SetEnvVariableArgument(EnviromentVariables.EnableNetwork, true), // On macOS we can't edit the TCC database easily // (it requires adding the mac has to be using MDM: https://carlashley.com/2018/09/28/tcc-round-up/) // So by default ignore any tests that would pop up permission dialogs in CI. new SetEnvVariableArgument(EnviromentVariables.DisableSystemPermissionTests, 1), }; for (int i = -1; i < verbosity; i++) { args.Add(new VerbosityArgument()); } var isSimulator = target.IsSimulator(); if (isSimulator) { args.Add(new SetAppArgumentArgument("-hostname:127.0.0.1", true)); args.Add(new SetEnvVariableArgument(EnviromentVariables.HostName, "127.0.0.1")); } else { var ipAddresses = System.Net.Dns.GetHostEntry(System.Net.Dns.GetHostName()).AddressList.Select(ip => ip.ToString()); var ips = string.Join(",", ipAddresses); args.Add(new SetAppArgumentArgument($"-hostname:{ips}", true)); args.Add(new SetEnvVariableArgument(EnviromentVariables.HostName, ips)); } var listenerLog = _logs.Create($"test-{target.AsString()}-{_helpers.Timestamp}.log", LogType.TestLog.ToString(), timestamp: true); var (transport, listener, listenerTmpFile) = _listenerFactory.Create(target.ToRunMode(), log: _mainLog, testLog: listenerLog, isSimulator: isSimulator, autoExit: true, xmlOutput: true); // cli always uses xml // Initialize has to be called before we try to get Port (internal implementation of the listener says so) // TODO: Improve this to not get into a broken state - it was really hard to debug when I moved this lower listener.Initialize(); args.Add(new SetAppArgumentArgument($"-transport:{transport}", true)); args.Add(new SetEnvVariableArgument(EnviromentVariables.Transport, transport.ToString().ToUpper())); if (transport == ListenerTransport.File) { args.Add(new SetEnvVariableArgument(EnviromentVariables.LogFilePath, listenerTmpFile)); } args.Add(new SetAppArgumentArgument($"-hostport:{listener.Port}", true)); args.Add(new SetEnvVariableArgument(EnviromentVariables.HostPort, listener.Port)); if (_listenerFactory.UseTunnel && !isSimulator) // simulators do not support tunnels { args.Add(new SetEnvVariableArgument(EnviromentVariables.UseTcpTunnel, true)); } if (_useXmlOutput) { // let the runner now via envars that we want to get a xml output, else the runner will default to plain text args.Add (new SetEnvVariableArgument (EnviromentVariables.EnableXmlOutput, true)); args.Add (new SetEnvVariableArgument (EnviromentVariables.XmlMode, "wrapped")); args.Add (new SetEnvVariableArgument (EnviromentVariables.XmlVersion, $"{xmlResultJargon}")); } listener.StartAsync(); var crashLogs = new Logs(_logs.Directory); if (appInformation.Extension.HasValue) { switch (appInformation.Extension) { case Extension.TodayExtension: args.Add(isSimulator ? (MlaunchArgument)new LaunchSimulatorExtensionArgument(appInformation.LaunchAppPath, appInformation.BundleIdentifier) : new LaunchDeviceExtensionArgument(appInformation.LaunchAppPath, appInformation.BundleIdentifier)); break; case Extension.WatchKit2: default: throw new NotImplementedException(); } } else { args.Add(isSimulator ? (MlaunchArgument)new LaunchSimulatorArgument(appInformation.LaunchAppPath) : new LaunchDeviceArgument(appInformation.LaunchAppPath)); } var runMode = target.ToRunMode(); ICrashSnapshotReporter crashReporter; ITestReporter testReporter; if (isSimulator) { crashReporter = _snapshotReporterFactory.Create(_mainLog, crashLogs, isDevice: !isSimulator, deviceName: null!); testReporter = _testReporterFactory.Create(_mainLog, _mainLog, _logs, crashReporter, listener, new XmlResultParser(), appInformation, runMode, xmlResultJargon, device: null, timeout, null, (level, message) => _mainLog.WriteLine(message)); using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(testReporter.CancellationToken, cancellationToken); listener.ConnectedTask .TimeoutAfter(testLaunchTimeout) .ContinueWith(testReporter.LaunchCallback) .DoNotAwait(); await _simulatorLoader.LoadDevices(_logs.Create($"simulator-list-{_helpers.Timestamp}.log", "Simulator list"), false, false); var simulators = await _simulatorLoader.FindSimulators(target, _mainLog); if (!(simulators?.Any() ?? false)) { _mainLog.WriteLine("Didn't find any suitable simulators"); throw new NoDeviceFoundException(); } var simulator = string.IsNullOrEmpty(deviceName) ? simulators.FirstOrDefault() : simulators.FirstOrDefault(s => string.Equals(s.Name, deviceName, StringComparison.InvariantCultureIgnoreCase)); if (simulator == null) { throw new NoDeviceFoundException(); } deviceName = simulator.Name; if (!target.IsWatchOSTarget()) { var stderrTty = _helpers.GetTerminalName(2); if (!string.IsNullOrEmpty(stderrTty)) { args.Add(new SetStderrArgument(stderrTty)); } else { var stdoutLog = _logs.CreateFile($"mlaunch-stdout-{_helpers.Timestamp}.log", "Standard output"); var stderrLog = _logs.CreateFile($"mlaunch-stderr-{_helpers.Timestamp}.log", "Standard error"); args.Add(new SetStdoutArgument(stdoutLog)); args.Add(new SetStderrArgument(stderrLog)); } } var systemLogs = new List<ICaptureLog>(); foreach (var sim in simulators) { // Upload the system log _mainLog.WriteLine("System log for the '{1}' simulator is: {0}", sim.SystemLog, sim.Name); bool isCompanion = sim != simulator; var logDescription = isCompanion ? LogType.CompanionSystemLog.ToString() : LogType.SystemLog.ToString(); var log = _captureLogFactory.Create( Path.Combine(_logs.Directory, sim.Name + ".log"), sim.SystemLog, true, logDescription); log.StartCapture(); _logs.Add(log); systemLogs.Add(log); } _mainLog.WriteLine("*** Executing {0}/{1} in the simulator ***", appInformation.AppName, target); if (ensureCleanSimulatorState) { foreach (var sim in simulators) { await sim.PrepareSimulator(_mainLog, appInformation.BundleIdentifier); } } args.Add(new SimulatorUDIDArgument(simulator.UDID)); await crashReporter.StartCaptureAsync(); _mainLog.WriteLine("Starting test run"); var result = _processManager.ExecuteCommandAsync(args, _mainLog, timeout, cancellationToken: linkedCts.Token); await testReporter.CollectSimulatorResult(result); // cleanup after us if (ensureCleanSimulatorState) { await simulator.KillEverything(_mainLog); } foreach (var log in systemLogs) { log.StopCapture(); } } else { args.Add(new DisableMemoryLimitsArgument()); if (deviceName == null) { IHardwareDevice? companionDevice = null; IHardwareDevice device = await _hardwareDeviceLoader.FindDevice(runMode, _mainLog, includeLocked: false, force: false); if (target.IsWatchOSTarget()) { companionDevice = await _hardwareDeviceLoader.FindCompanionDevice(_mainLog, device); } deviceName = companionDevice?.Name ?? device.Name; } if (deviceName == null) { throw new NoDeviceFoundException(); } crashReporter = _snapshotReporterFactory.Create(_mainLog, crashLogs, isDevice: !isSimulator, deviceName); testReporter = _testReporterFactory.Create(_mainLog, _mainLog, _logs, crashReporter, listener, new XmlResultParser(), appInformation, runMode, xmlResultJargon, deviceName, timeout, null, (level, message) => _mainLog.WriteLine(message)); using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(testReporter.CancellationToken, cancellationToken); listener.ConnectedTask .TimeoutAfter(testLaunchTimeout) .ContinueWith(testReporter.LaunchCallback) .DoNotAwait(); _mainLog.WriteLine("*** Executing {0}/{1} on device '{2}' ***", appInformation.AppName, target, deviceName); if (target.IsWatchOSTarget()) { args.Add(new AttachNativeDebuggerArgument()); // this prevents the watch from backgrounding the app. } else { args.Add(new WaitForExitArgument()); } args.Add(new DeviceNameArgument(deviceName)); var deviceSystemLog = _logs.Create($"device-{deviceName}-{_helpers.Timestamp}.log", "Device log"); var deviceLogCapturer = _deviceLogCapturerFactory.Create(_mainLog, deviceSystemLog, deviceName); deviceLogCapturer.StartCapture(); try { await crashReporter.StartCaptureAsync(); // create a tunnel to communicate with the device if (transport == ListenerTransport.Tcp && _listenerFactory.UseTunnel && listener is SimpleTcpListener tcpListener) { // create a new tunnel using the listener var tunnel = _listenerFactory.TunnelBore.Create(deviceName, _mainLog); tunnel.Open(deviceName, tcpListener, timeout, _mainLog); // wait until we started the tunnel await tunnel.Started; } _mainLog.WriteLine("Starting test run"); // We need to check for MT1111 (which means that mlaunch won't wait for the app to exit). var aggregatedLog = Log.CreateAggregatedLog(testReporter.CallbackLog, _mainLog); Task<ProcessExecutionResult> runTestTask = _processManager.ExecuteCommandAsync( args, aggregatedLog, timeout, cancellationToken: linkedCts.Token); await testReporter.CollectDeviceResult(runTestTask); } finally { deviceLogCapturer.StopCapture(); deviceSystemLog.Dispose(); // close a tunnel if it was created if (!isSimulator && _listenerFactory.UseTunnel) await _listenerFactory.TunnelBore.Close(deviceName); } // Upload the system log if (File.Exists(deviceSystemLog.FullPath)) { _mainLog.WriteLine("A capture of the device log is: {0}", deviceSystemLog.FullPath); } } listener.Cancel(); listener.Dispose(); // check the final status, copy all the required data var (testResult, resultMessage) = await testReporter.ParseResult(); return (deviceName, testResult, resultMessage); }
public async Task RunTestAsync() { mainLog.WriteLine("Running '{0}' on device (candidates: '{1}')", testTask.ProjectFile, string.Join("', '", testTask.Candidates.Select((v) => v.Name).ToArray())); var uninstall_log = testTask.Logs.Create($"uninstall-{Helpers.Timestamp}.log", "Uninstall log"); using (var device_resource = await testTask.NotifyBlockingWaitAsync(resourceManager.GetDeviceResources(testTask.Candidates).AcquireAnyConcurrentAsync())) { try { // Set the device we acquired. testTask.Device = testTask.Candidates.First((d) => d.UDID == device_resource.Resource.Name); if (testTask.Device.DevicePlatform == DevicePlatform.watchOS) { testTask.CompanionDevice = await devices.FindCompanionDevice(deviceLoadLog, testTask.Device); } mainLog.WriteLine("Acquired device '{0}' for '{1}'", testTask.Device.Name, testTask.ProjectFile); ITunnelBore tunnelBore = null; if (useTcpTunnel && testTask.Device.DevicePlatform != DevicePlatform.iOS && testTask.Device.DevicePlatform != DevicePlatform.tvOS) { mainLog.WriteLine("Ignoring request to use a tunnel because it is not supported by the specified platform"); } else if (useTcpTunnel && (testTask.Device.DevicePlatform == DevicePlatform.iOS || testTask.Device.DevicePlatform == DevicePlatform.tvOS)) { tunnelBore = testTask.TunnelBore; mainLog.WriteLine("Using tunnel to communicate with device."); } testTask.Runner = new AppRunner(testTask.ProcessManager, new AppBundleInformationParser(), new SimulatorLoaderFactory(testTask.ProcessManager), new SimpleListenerFactory(tunnelBore), new DeviceLoaderFactory(testTask.ProcessManager), new CrashSnapshotReporterFactory(testTask.ProcessManager), new CaptureLogFactory(), new DeviceLogCapturerFactory(testTask.ProcessManager), new TestReporterFactory(testTask.ProcessManager), testTask.AppRunnerTarget, testTask.Harness, projectFilePath: testTask.ProjectFile, mainLog: uninstall_log, logs: new Logs(testTask.LogDirectory ?? defaultLogDirectory), buildConfiguration: testTask.ProjectConfiguration, deviceName: testTask.Device.Name, companionDeviceName: testTask.CompanionDevice?.Name, timeoutMultiplier: testTask.TimeoutMultiplier, variation: testTask.Variation, buildTask: testTask.BuildTask); await testTask.Runner.InitializeAsync(); // Sometimes devices can't upgrade (depending on what has changed), so make sure to uninstall any existing apps first. if (uninstallTestApp) { testTask.Runner.MainLog = uninstall_log; var uninstall_result = await testTask.Runner.UninstallAsync(); if (!uninstall_result.Succeeded) { mainLog.WriteLine($"Pre-run uninstall failed, exit code: {uninstall_result.ExitCode} (this hopefully won't affect the test result)"); } } else { uninstall_log.WriteLine($"Pre-run uninstall skipped."); } if (!testTask.Failed) { // Install the app InstallLog = new AppInstallMonitorLog(testTask.Logs.Create($"install-{Helpers.Timestamp}.log", "Install log")); try { testTask.Runner.MainLog = this.InstallLog; var install_result = await testTask.Runner.InstallAsync(InstallLog.CancellationToken); if (!install_result.Succeeded) { testTask.FailureMessage = $"Install failed, exit code: {install_result.ExitCode}."; testTask.ExecutionResult = TestExecutingResult.Failed; if (generateXmlFailures) { resultParser.GenerateFailure( testTask.Logs, "install", testTask.Runner.AppInformation.AppName, testTask.Variation, $"AppInstallation on {testTask.Device.Name}", $"Install failed on {testTask.Device.Name}, exit code: {install_result.ExitCode}", InstallLog.FullPath, xmlResultJargon); } } } finally { InstallLog.Dispose(); InstallLog = null; } } if (!testTask.Failed) { // Run the app testTask.Runner.MainLog = testTask.Logs.Create($"run-{testTask.Device.UDID}-{Helpers.Timestamp}.log", "Run log"); await testTask.Runner.RunAsync(); if (!string.IsNullOrEmpty(testTask.Runner.FailureMessage)) { testTask.FailureMessage = testTask.Runner.FailureMessage; } else if (testTask.Runner.Result != TestExecutingResult.Succeeded) { testTask.FailureMessage = testTask.GuessFailureReason(testTask.Runner.MainLog); } if (string.IsNullOrEmpty(testTask.FailureMessage) && errorKnowledgeBase.IsKnownTestIssue(testTask.Runner.MainLog, out var failure)) { testTask.KnownFailure = failure; mainLog.WriteLine($"Test run has a known failure: '{testTask.KnownFailure}'"); } if (testTask.Runner.Result == TestExecutingResult.Succeeded && testTask.Platform == TestPlatform.iOS_TodayExtension64) { // For the today extension, the main app is just a single test. // This is because running the today extension will not wake up the device, // nor will it close & reopen the today app (but launching the main app // will do both of these things, preparing the device for launching the today extension). AppRunner todayRunner = new AppRunner(testTask.ProcessManager, new AppBundleInformationParser(), new SimulatorLoaderFactory(testTask.ProcessManager), new SimpleListenerFactory(tunnelBore), new DeviceLoaderFactory(testTask.ProcessManager), new CrashSnapshotReporterFactory(testTask.ProcessManager), new CaptureLogFactory(), new DeviceLogCapturerFactory(testTask.ProcessManager), new TestReporterFactory(testTask.ProcessManager), testTask.AppRunnerTarget, testTask.Harness, projectFilePath: testTask.ProjectFile, mainLog: testTask.Logs.Create($"extension-run-{testTask.Device.UDID}-{Helpers.Timestamp}.log", "Extension run log"), logs: new Logs(testTask.LogDirectory ?? defaultLogDirectory), buildConfiguration: testTask.ProjectConfiguration, deviceName: testTask.Device.Name, companionDeviceName: testTask.CompanionDevice?.Name, timeoutMultiplier: testTask.TimeoutMultiplier, variation: testTask.Variation, buildTask: testTask.BuildTask); await todayRunner.InitializeAsync(); testTask.AdditionalRunner = todayRunner; await todayRunner.RunAsync(); foreach (var log in todayRunner.Logs.Where((v) => !v.Description.StartsWith("Extension ", StringComparison.Ordinal))) { log.Description = "Extension " + log.Description [0].ToString().ToLower() + log.Description.Substring(1); } testTask.ExecutionResult = todayRunner.Result; if (!string.IsNullOrEmpty(todayRunner.FailureMessage)) { testTask.FailureMessage = todayRunner.FailureMessage; } } else { testTask.ExecutionResult = testTask.Runner.Result; } } } finally { // Uninstall again, so that we don't leave junk behind and fill up the device. if (uninstallTestApp) { testTask.Runner.MainLog = uninstall_log; var uninstall_result = await testTask.Runner.UninstallAsync(); if (!uninstall_result.Succeeded) { mainLog.WriteLine($"Post-run uninstall failed, exit code: {uninstall_result.ExitCode} (this won't affect the test result)"); } } else { uninstall_log.WriteLine($"Post-run uninstall skipped."); } // Also clean up after us locally. if (inCI || cleanSuccessfulTestRuns && testTask.Succeeded) { await testTask.BuildTask.CleanAsync(); } } } }
public async Task <(string deviceName, ProcessExecutionResult result)> InstallApp(AppBundleInformation appBundleInformation, TestTargetOs target, string?deviceName = null, CancellationToken cancellationToken = default) { if (target.Platform.IsSimulator()) { // We reset the simulator when running, so a separate install step does not make much sense. throw new InvalidOperationException("Installing to a simulator is not supported."); } if (!Directory.Exists(appBundleInformation.LaunchAppPath)) { throw new DirectoryNotFoundException("Failed to find the app bundle directory"); } if (deviceName == null) { // the _deviceLoader.FindDevice will return the fist device of the type, but we want to make sure that // the device we use is if the correct arch, therefore, we will use the LoadDevices and return the // correct one await _deviceLoader.LoadDevices(_mainLog, false, false); IHardwareDevice?device = null; if (appBundleInformation.Supports32Bit) { // we only support 32b on iOS, therefore we can ignore the target device = _deviceLoader.Connected32BitIOS.FirstOrDefault(); } else { device = target.Platform switch { TestTarget.Device_iOS => _deviceLoader.Connected64BitIOS.FirstOrDefault(), TestTarget.Device_tvOS => _deviceLoader.ConnectedTV.FirstOrDefault(), _ => device }; } deviceName = target.Platform.IsWatchOSTarget() ? (await _deviceLoader.FindCompanionDevice(_mainLog, device)).Name : device?.Name; } if (deviceName == null) { throw new NoDeviceFoundException(); } var args = new MlaunchArguments(); for (int i = -1; i < _verbosity; i++) { args.Add(new VerbosityArgument()); } args.Add(new InstallAppOnDeviceArgument(appBundleInformation.LaunchAppPath)); args.Add(new DeviceNameArgument(deviceName)); if (target.Platform.IsWatchOSTarget()) { args.Add(new DeviceArgument("ios,watchos")); } var totalSize = Directory.GetFiles(appBundleInformation.LaunchAppPath, "*", SearchOption.AllDirectories).Select((v) => new FileInfo(v).Length).Sum(); _mainLog.WriteLine($"Installing '{appBundleInformation.LaunchAppPath}' to '{deviceName}' ({totalSize / 1024.0 / 1024.0:N2} MB)"); ProcessExecutionResult result = await _processManager.ExecuteCommandAsync(args, _mainLog, TimeSpan.FromHours(1), cancellationToken : cancellationToken); return(deviceName, result); } }
protected override async Task RunTestAsync() { Jenkins.MainLog.WriteLine("Running '{0}' on device (candidates: '{1}')", ProjectFile, string.Join("', '", Candidates.Select((v) => v.Name).ToArray())); var uninstall_log = Logs.Create($"uninstall-{Timestamp}.log", "Uninstall log"); using (var device_resource = await NotifyBlockingWaitAsync(Jenkins.GetDeviceResources(Candidates).AcquireAnyConcurrentAsync())) { try { // Set the device we acquired. Device = Candidates.First((d) => d.UDID == device_resource.Resource.Name); if (Device.DevicePlatform == DevicePlatform.watchOS) { CompanionDevice = await devices.FindCompanionDevice(Jenkins.DeviceLoadLog, Device); } Jenkins.MainLog.WriteLine("Acquired device '{0}' for '{1}'", Device.Name, ProjectFile); runner = new AppRunner(ProcessManager, new AppBundleInformationParser(), new SimulatorLoaderFactory(ProcessManager), new SimpleListenerFactory(), new DeviceLoaderFactory(ProcessManager), new CrashSnapshotReporterFactory(ProcessManager), new CaptureLogFactory(), new DeviceLogCapturerFactory(ProcessManager), new TestReporterFactory(ProcessManager), AppRunnerTarget, Harness, projectFilePath: ProjectFile, mainLog: uninstall_log, logs: new Logs(LogDirectory ?? Harness.LogDirectory), buildConfiguration: ProjectConfiguration, deviceName: Device.Name, companionDeviceName: CompanionDevice?.Name, timeoutMultiplier: TimeoutMultiplier, variation: Variation, buildTask: BuildTask); // Sometimes devices can't upgrade (depending on what has changed), so make sure to uninstall any existing apps first. if (Jenkins.UninstallTestApp) { runner.MainLog = uninstall_log; var uninstall_result = await runner.UninstallAsync(); if (!uninstall_result.Succeeded) { MainLog.WriteLine($"Pre-run uninstall failed, exit code: {uninstall_result.ExitCode} (this hopefully won't affect the test result)"); } } else { uninstall_log.WriteLine($"Pre-run uninstall skipped."); } if (!Failed) { // Install the app this.install_log = new AppInstallMonitorLog(Logs.Create($"install-{Timestamp}.log", "Install log")); try { runner.MainLog = this.install_log; var install_result = await runner.InstallAsync(install_log.CancellationToken); if (!install_result.Succeeded) { FailureMessage = $"Install failed, exit code: {install_result.ExitCode}."; ExecutionResult = TestExecutingResult.Failed; if (Harness.InCI) { resultParser.GenerateFailure(Logs, "install", runner.AppInformation.AppName, Variation, $"AppInstallation on {Device.Name}", $"Install failed on {Device.Name}, exit code: {install_result.ExitCode}", install_log.FullPath, Harness.XmlJargon); } } } finally { this.install_log.Dispose(); this.install_log = null; } } if (!Failed) { // Run the app runner.MainLog = Logs.Create($"run-{Device.UDID}-{Timestamp}.log", "Run log"); await runner.RunAsync(); if (!string.IsNullOrEmpty(runner.FailureMessage)) { FailureMessage = runner.FailureMessage; } else if (runner.Result != TestExecutingResult.Succeeded) { FailureMessage = GuessFailureReason(runner.MainLog); } if (runner.Result == TestExecutingResult.Succeeded && Platform == TestPlatform.iOS_TodayExtension64) { // For the today extension, the main app is just a single test. // This is because running the today extension will not wake up the device, // nor will it close & reopen the today app (but launching the main app // will do both of these things, preparing the device for launching the today extension). AppRunner todayRunner = new AppRunner(ProcessManager, new AppBundleInformationParser(), new SimulatorLoaderFactory(ProcessManager), new SimpleListenerFactory(), new DeviceLoaderFactory(ProcessManager), new CrashSnapshotReporterFactory(ProcessManager), new CaptureLogFactory(), new DeviceLogCapturerFactory(ProcessManager), new TestReporterFactory(ProcessManager), AppRunnerTarget, Harness, projectFilePath: ProjectFile, mainLog: Logs.Create($"extension-run-{Device.UDID}-{Timestamp}.log", "Extension run log"), logs: new Logs(LogDirectory ?? Harness.LogDirectory), buildConfiguration: ProjectConfiguration, deviceName: Device.Name, companionDeviceName: CompanionDevice?.Name, timeoutMultiplier: TimeoutMultiplier, variation: Variation, buildTask: BuildTask); additional_runner = todayRunner; await todayRunner.RunAsync(); foreach (var log in todayRunner.Logs.Where((v) => !v.Description.StartsWith("Extension ", StringComparison.Ordinal))) { log.Description = "Extension " + log.Description [0].ToString().ToLower() + log.Description.Substring(1); } ExecutionResult = todayRunner.Result; if (!string.IsNullOrEmpty(todayRunner.FailureMessage)) { FailureMessage = todayRunner.FailureMessage; } } else { ExecutionResult = runner.Result; } } } finally { // Uninstall again, so that we don't leave junk behind and fill up the device. if (Jenkins.UninstallTestApp) { runner.MainLog = uninstall_log; var uninstall_result = await runner.UninstallAsync(); if (!uninstall_result.Succeeded) { MainLog.WriteLine($"Post-run uninstall failed, exit code: {uninstall_result.ExitCode} (this won't affect the test result)"); } } else { uninstall_log.WriteLine($"Post-run uninstall skipped."); } // Also clean up after us locally. if (Harness.InCI || Jenkins.CleanSuccessfulTestRuns && Succeeded) { await BuildTask.CleanAsync(); } } } }
public async Task <DevicePair> FindDevice( TestTargetOs target, string?deviceName, ILog log, bool includeWirelessDevices = true, CancellationToken cancellationToken = default) { IDevice?device; IDevice?companionDevice = null; bool IsMatchingDevice(IDevice device) => device.Name.Equals(deviceName, StringComparison.InvariantCultureIgnoreCase) || device.UDID.Equals(deviceName, StringComparison.InvariantCultureIgnoreCase); if (target.Platform.IsSimulator()) { if (deviceName == null) { (device, companionDevice) = await _simulatorLoader.FindSimulators(target, log, retryCount : 3, cancellationToken : cancellationToken); } else { await _simulatorLoader.LoadDevices(log, includeLocked : false, cancellationToken : cancellationToken); device = _simulatorLoader.AvailableDevices.FirstOrDefault(IsMatchingDevice) ?? throw new NoDeviceFoundException($"Failed to find a simulator '{deviceName}'"); } } else { // The DeviceLoader.FindDevice will return the fist device of the type, but we want to make sure that // the device we use is of the correct arch, therefore, we will use the LoadDevices and handpick one await _deviceLoader.LoadDevices( log, includeLocked : false, forceRefresh : false, includeWirelessDevices : includeWirelessDevices, cancellationToken : cancellationToken); if (deviceName == null) { IHardwareDevice?hardwareDevice = target.Platform switch { TestTarget.Simulator_iOS32 => _deviceLoader.Connected32BitIOS.FirstOrDefault(), TestTarget.Device_iOS => _deviceLoader.Connected64BitIOS.FirstOrDefault(), TestTarget.Device_tvOS => _deviceLoader.ConnectedTV.FirstOrDefault(), _ => throw new ArgumentOutOfRangeException(nameof(target), $"Unrecognized device platform {target.Platform}") }; if (target.Platform.IsWatchOSTarget() && hardwareDevice != null) { companionDevice = await _deviceLoader.FindCompanionDevice(log, hardwareDevice, cancellationToken : cancellationToken); } device = hardwareDevice; } else { device = _deviceLoader.ConnectedDevices.FirstOrDefault(IsMatchingDevice) ?? throw new NoDeviceFoundException($"Failed to find a device '{deviceName}'. " + "Please make sure the device is connected and unlocked."); } } if (device == null) { throw new NoDeviceFoundException($"Failed to find a suitable device for target {target.AsString()}"); } return(new DevicePair(device, companionDevice)); }