protected async Task <AppBundleInformation> GetAppBundleFromId(TestTargetOs target, IDevice device, string bundleIdentifier, CancellationToken cancellationToken) { // We can exchange bundle ID for path where the bundle is on the simulator and get that if (device is ISimulatorDevice simulator) { _logger.LogInformation($"Querying simulator for app bundle information.."); await simulator.Boot(_mainLog, cancellationToken); var appBundlePath = await simulator.GetAppBundlePath(_mainLog, bundleIdentifier, cancellationToken); return(await GetAppBundleFromPath(target, appBundlePath, cancellationToken)); } // We're unable to do this for real devices / or MacCatalyst // It is not ideal but doesn't matter much at the moment as we don't need the full list of properties there _logger.LogDebug("Supplemented full app bundle information with bundle identifier"); return(AppBundleInformation.FromBundleId(bundleIdentifier)); }
public async Task OrchestrateFailedDeviceInstallationTest() { // Setup SetupInstall(_device.Object, 1); SetupUninstall(_device.Object, 1); // This can fail as this is the first purge of the app before we install it var failure = new KnownIssue("Some failure", suggestedExitCode: (int)ExitCode.APP_NOT_SIGNED); _errorKnowledgeBase .Setup(x => x.IsKnownInstallIssue(It.IsAny <IFileBackedLog>(), out failure)) .Returns(true) .Verifiable(); var testTarget = new TestTargetOs(TestTarget.Device_iOS, "14.2"); // Act var result = await _installOrchestrator.OrchestrateInstall( testTarget, null, AppPath, TimeSpan.FromMinutes(30), includeWirelessDevices : false, resetSimulator : false, enableLldb : true, new CancellationToken()); // Verify Assert.Equal(ExitCode.APP_NOT_SIGNED, result); _deviceFinder.Verify( x => x.FindDevice(testTarget, null, It.IsAny <ILog>(), false, It.IsAny <CancellationToken>()), Times.Once); VerifySimulatorReset(false); VerifySimulatorCleanUp(false); VerifyDiagnosticData(testTarget); _appInstaller.Verify( x => x.InstallApp(_appBundleInformation, testTarget, _device.Object, It.IsAny <CancellationToken>()), Times.Once); }
public async Task TryMacCatalystResetTest() { // Setup var testTarget = new TestTargetOs(TestTarget.MacCatalyst, null); _deviceFinder.Reset(); // Act var result = await _simulatorResetOrchestrator.OrchestrateSimulatorReset( testTarget, null, TimeSpan.FromMinutes(30), new CancellationToken()); // Verify Assert.Equal(ExitCode.INVALID_ARGUMENTS, result); _deviceFinder.VerifyNoOtherCalls(); _appInstaller.VerifyNoOtherCalls(); _appUninstaller.VerifyNoOtherCalls(); }
public async Task <(string DeviceName, TestExecutingResult Result, string ResultMessage)> RunApp( AppBundleInformation appInformation, TestTargetOs target, TimeSpan timeout, TimeSpan testLaunchTimeout, string?deviceName = null, string?companionDeviceName = null, bool ensureCleanSimulatorState = false, int verbosity = 1, XmlResultJargon xmlResultJargon = XmlResultJargon.xUnit, string[]?skippedMethods = null, string[]?skippedTestClasses = null, CancellationToken cancellationToken = default) { var runMode = target.Platform.ToRunMode(); bool isSimulator = target.Platform.IsSimulator(); var deviceListenerLog = _logs.Create($"test-{target.AsString()}-{_helpers.Timestamp}.log", LogType.TestLog.ToString(), timestamp: true); var(deviceListenerTransport, deviceListener, deviceListenerTmpFile) = _listenerFactory.Create( runMode, log: _mainLog, testLog: deviceListenerLog, isSimulator: isSimulator, autoExit: true, xmlOutput: true); // cli always uses xml ISimulatorDevice?simulator = null; ISimulatorDevice?companionSimulator = null; // Find devices if (isSimulator) { int attempt = 1; const int maxAttempts = 3; while (true) { try { (simulator, companionSimulator) = await _simulatorLoader.FindSimulators(target, _mainLog); break; } catch (Exception e) { _mainLog.WriteLine($"Failed to find/create simulator (attempt {attempt}/{maxAttempts}):" + Environment.NewLine + e); if (attempt == maxAttempts) { throw new NoDeviceFoundException("Failed to find/create suitable simulator"); } } finally { attempt++; } } deviceName = companionSimulator?.Name ?? simulator.Name; } else { deviceName ??= await FindDevice(target) ?? throw new NoDeviceFoundException(); } int deviceListenerPort = deviceListener.InitializeAndGetPort(); deviceListener.StartAsync(); var crashLogs = new Logs(_logs.Directory); ICrashSnapshotReporter crashReporter = _snapshotReporterFactory.Create(_mainLog, crashLogs, isDevice: !isSimulator, deviceName); ITestReporter testReporter = _testReporterFactory.Create(_mainLog, _mainLog, _logs, crashReporter, deviceListener, _resultParser, appInformation, runMode, xmlResultJargon, deviceName, timeout, null, (level, message) => _mainLog.WriteLine(message)); deviceListener.ConnectedTask .TimeoutAfter(testLaunchTimeout) .ContinueWith(testReporter.LaunchCallback) .DoNotAwait(); _mainLog.WriteLine($"*** Executing '{appInformation.AppName}' on {target} '{deviceName}' ***"); try { using var combinedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(testReporter.CancellationToken, cancellationToken); if (isSimulator) { if (simulator == null) { _mainLog.WriteLine("Didn't find any suitable simulator"); throw new NoDeviceFoundException(); } var mlaunchArguments = GetSimulatorArguments( appInformation, simulator, verbosity, xmlResultJargon, skippedMethods, skippedTestClasses, deviceListenerTransport, deviceListenerPort, deviceListenerTmpFile); await RunSimulatorTests( mlaunchArguments, appInformation, crashReporter, testReporter, simulator, companionSimulator, ensureCleanSimulatorState, timeout, combinedCancellationToken.Token); } else { var mlaunchArguments = GetDeviceArguments( appInformation, deviceName, target.Platform.IsWatchOSTarget(), verbosity, xmlResultJargon, skippedMethods, skippedTestClasses, deviceListenerTransport, deviceListenerPort, deviceListenerTmpFile); await RunDeviceTests( mlaunchArguments, crashReporter, testReporter, deviceListener, deviceName, timeout, combinedCancellationToken.Token); } } finally { deviceListener.Cancel(); deviceListener.Dispose(); } // Check the final status, copy all the required data var(testResult, resultMessage) = await testReporter.ParseResult(); return(deviceName, testResult, resultMessage); }
protected Task <AppBundleInformation> GetAppBundleFromPath(TestTargetOs target, string appBundlePath, CancellationToken cancellationToken) { appBundlePath = Path.GetFullPath(appBundlePath); _logger.LogInformation($"Getting app bundle information from '{appBundlePath}'.."); return(_appBundleInformationParser.ParseFromAppBundle(appBundlePath, target.Platform, _mainLog, cancellationToken)); }
private async Task <ExitCode> OrchestrateOperationInternal( TestTargetOs target, string?deviceName, bool includeWirelessDevices, bool resetSimulator, bool enableLldb, GetAppBundleInfoFunc getAppBundle, ExecuteMacCatalystAppFunc executeMacCatalystApp, ExecuteAppFunc executeApp, CancellationToken cancellationToken) { _lldbFileCreated = false; var isLldbEnabled = IsLldbEnabled(); if (isLldbEnabled && !enableLldb) { // the file is present, but the user did not set it, warn him about it _logger.LogWarning("Lldb will be used since '~/.mtouch-launch-with-lldb' was found in the system but it was not created by xharness."); } else if (enableLldb) { if (!File.Exists(s_mlaunchLldbConfigFile)) { // create empty file File.WriteAllText(s_mlaunchLldbConfigFile, string.Empty); _lldbFileCreated = true; } } if (includeWirelessDevices && target.Platform.IsSimulator()) { _logger.LogWarning("Including wireless devices while targeting a simulator has no effect"); } if (resetSimulator && !target.Platform.IsSimulator()) { _logger.LogWarning("Targeting device but requesting simulator reset has no effect"); resetSimulator = false; } ExitCode exitCode; IDevice device; IDevice? companionDevice; AppBundleInformation appBundleInfo; if (target.Platform == TestTarget.MacCatalyst) { try { appBundleInfo = await getAppBundle(target, null !, cancellationToken); } catch (Exception e) { cancellationToken.ThrowIfCancellationRequested(); _logger.LogError(e.Message); return(ExitCode.PACKAGE_NOT_FOUND); } try { return(await executeMacCatalystApp(appBundleInfo)); } catch (Exception e) { var message = new StringBuilder().AppendLine("Application run failed:"); exitCode = ExitCode.APP_LAUNCH_FAILURE; if (_errorKnowledgeBase.IsKnownTestIssue(_mainLog, out var failure)) { message.Append(failure.HumanMessage); if (failure.IssueLink != null) { message.AppendLine($" Find more information at {failure.IssueLink}"); } if (failure.SuggestedExitCode.HasValue) { exitCode = (ExitCode)failure.SuggestedExitCode.Value; } } else { message.AppendLine(e.ToString()); } _logger.LogError(message.ToString()); return(exitCode); } } try { _logger.LogInformation($"Looking for available {target.AsString()} {(target.Platform.IsSimulator() ? "simulators" : "devices")}.."); var finderLogName = $"list-{target.AsString()}-{_helpers.Timestamp}.log"; using var finderLog = _logs.Create(finderLogName, "DeviceList", true); _mainLog.WriteLine( $"Looking for available {target.AsString()} {(target.Platform.IsSimulator() ? "simulators" : "devices")}. " + $"Storing logs into {finderLogName}"); (device, companionDevice) = await _deviceFinder.FindDevice(target, deviceName, finderLog, includeWirelessDevices, cancellationToken); _logger.LogInformation($"Found {(target.Platform.IsSimulator() ? "simulator" : "physical")} device '{device.Name}'"); if (companionDevice != null) { _logger.LogInformation($"Found companion {(target.Platform.IsSimulator() ? "simulator" : "physical")} device '{companionDevice.Name}'"); } } catch (NoDeviceFoundException e) { _logger.LogError(e.Message); return(ExitCode.DEVICE_NOT_FOUND); } cancellationToken.ThrowIfCancellationRequested(); try { appBundleInfo = await getAppBundle(target, device, cancellationToken); } catch (Exception e) { cancellationToken.ThrowIfCancellationRequested(); _logger.LogError(e.Message); return(ExitCode.PACKAGE_NOT_FOUND); } cancellationToken.ThrowIfCancellationRequested(); if (target.Platform.IsSimulator() && resetSimulator) { try { var simulator = (ISimulatorDevice)device; var bundleIds = appBundleInfo.BundleIdentifier == string.Empty ? Array.Empty <string>() : new[] { appBundleInfo.BundleIdentifier }; _logger.LogInformation($"Reseting simulator '{device.Name}'"); await simulator.PrepareSimulator(_mainLog, bundleIds); if (companionDevice != null) { _logger.LogInformation($"Reseting companion simulator '{companionDevice.Name}'"); var companionSimulator = (ISimulatorDevice)companionDevice; await companionSimulator.PrepareSimulator(_mainLog, bundleIds); } _logger.LogInformation("Simulator reset finished"); } catch (Exception e) { _logger.LogError($"Failed to reset simulator: " + Environment.NewLine + e); return(ExitCode.SIMULATOR_FAILURE); } } cancellationToken.ThrowIfCancellationRequested(); // Note down the actual test target // For simulators (e.g. "iOS 13.4"), we strip the iOS part and keep the version only, for devices there's no OS _diagnosticsData.TargetOS = target.Platform.IsSimulator() ? device.OSVersion.Split(' ', 2).Last() : device.OSVersion; _diagnosticsData.Device = device.Name ?? device.UDID; // Uninstall the app first to get a clean state if (!resetSimulator) { await UninstallApp(target.Platform, appBundleInfo.BundleIdentifier, device, isPreparation : true, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); } exitCode = await InstallApp(appBundleInfo, device, target, cancellationToken); if (exitCode != ExitCode.SUCCESS) { _logger.LogInformation($"Cleaning up the failed installation from '{device.Name}'"); var uninstallResult = await UninstallApp(target.Platform, appBundleInfo.BundleIdentifier, device, isPreparation : false, new CancellationToken()); if (uninstallResult == ExitCode.SIMULATOR_FAILURE) { // Sometimes the simulator gets in a bad shape and we won't be able to install the app, we can tell here return(ExitCode.SIMULATOR_FAILURE); } return(exitCode); } try { exitCode = await executeApp(appBundleInfo, device, companionDevice); } catch (Exception e) { exitCode = ExitCode.GENERAL_FAILURE; var message = new StringBuilder().AppendLine("Application run failed:"); if (_errorKnowledgeBase.IsKnownTestIssue(_mainLog, out var failure)) { message.Append(failure.HumanMessage); if (failure.IssueLink != null) { message.AppendLine($" Find more information at {failure.IssueLink}"); } if (failure.SuggestedExitCode.HasValue) { exitCode = (ExitCode)failure.SuggestedExitCode.Value; } } else { message.AppendLine(e.ToString()); } _logger.LogError(message.ToString()); } finally { if (target.Platform.IsSimulator() && resetSimulator) { await CleanUpSimulators(device, companionDevice); } else if (device != null) // Do not uninstall if device was cleaned up { var uninstallResult = await UninstallApp(target.Platform, appBundleInfo.BundleIdentifier, device, false, new CancellationToken()); // We are able to detect a case when simulator is in a bad shape // If it also failed the test/run, we should present that as the failure if (uninstallResult == ExitCode.SIMULATOR_FAILURE && exitCode != ExitCode.SUCCESS && exitCode != ExitCode.TESTS_FAILED) { exitCode = ExitCode.SIMULATOR_FAILURE; } } } return(exitCode); }
public async Task <(ISimulatorDevice Simulator, ISimulatorDevice?CompanionSimulator)> FindSimulators(TestTargetOs target, ILog log, bool createIfNeeded = true, bool minVersion = false) { var runtimePrefix = target.Platform switch { TestTarget.Simulator_iOS32 => "com.apple.CoreSimulator.SimRuntime.iOS-", TestTarget.Simulator_iOS64 => "com.apple.CoreSimulator.SimRuntime.iOS-", TestTarget.Simulator_iOS => "com.apple.CoreSimulator.SimRuntime.iOS-", TestTarget.Simulator_tvOS => "com.apple.CoreSimulator.SimRuntime.tvOS-", TestTarget.Simulator_watchOS => "com.apple.CoreSimulator.SimRuntime.watchOS-", _ => throw new Exception(string.Format("Invalid simulator target: {0}", target)) }; var runtimeVersion = target.OSVersion; if (runtimeVersion == null) { if (!_loaded) { await LoadDevices(log); } string?firstOsVersion = _supportedRuntimes .Where(r => r.Identifier.StartsWith(runtimePrefix)) .OrderByDescending(r => r.Identifier) .FirstOrDefault()? .Identifier .Substring(runtimePrefix.Length); runtimeVersion = firstOsVersion ?? throw new NoDeviceFoundException($"Failed to find a suitable OS runtime version for {target.AsString()}"); } string simulatorRuntime = runtimePrefix + runtimeVersion.Replace('.', '-'); string simulatorDeviceType = target.Platform switch { TestTarget.Simulator_iOS => "com.apple.CoreSimulator.SimDeviceType.iPhone-5", TestTarget.Simulator_iOS32 => "com.apple.CoreSimulator.SimDeviceType.iPhone-5", TestTarget.Simulator_iOS64 => "com.apple.CoreSimulator.SimDeviceType." + (minVersion ? "iPhone-6" : "iPhone-X"), TestTarget.Simulator_tvOS => "com.apple.CoreSimulator.SimDeviceType.Apple-TV-1080p", TestTarget.Simulator_watchOS => "com.apple.CoreSimulator.SimDeviceType." + (minVersion ? "Apple-Watch-38mm" : "Apple-Watch-Series-3-38mm"), _ => throw new Exception(string.Format("Invalid simulator target: {0}", target)) }; string?companionDeviceType = null; string?companionRuntime = null; if (target.Platform == TestTarget.Simulator_watchOS) { // TODO: Allow to specify companion runtime companionRuntime = "com.apple.CoreSimulator.SimRuntime.iOS-" + (minVersion ? SdkVersions.MinWatchOSCompanionSimulator : SdkVersions.MaxWatchOSCompanionSimulator).Replace('.', '-'); companionDeviceType = "com.apple.CoreSimulator.SimDeviceType." + (minVersion ? "iPhone-6" : "iPhone-X"); } var devices = await FindOrCreateDevicesAsync(log, simulatorRuntime, simulatorDeviceType); IEnumerable <ISimulatorDevice>?companionDevices = null; if (companionRuntime != null && companionDeviceType != null) { companionDevices = await FindOrCreateDevicesAsync(log, companionRuntime, companionDeviceType); } if (devices?.Any() != true) { throw new Exception($"Could not find or create devices{Environment.NewLine}" + $"runtime: {simulatorRuntime}{Environment.NewLine}" + $"device type: {simulatorDeviceType}"); } ISimulatorDevice?simulator = null; ISimulatorDevice?companionSimulator = null; if (companionRuntime == null) { simulator = devices.First(); } else { if (companionDevices?.Any() != true) { throw new Exception($"Could not find or create companion devices{Environment.NewLine}" + $"runtime: {companionRuntime}{Environment.NewLine}" + $"device type: {companionDeviceType}"); } var pair = await FindOrCreateDevicePairAsync(log, devices, companionDevices); if (pair == null) { throw new Exception($"Could not find or create device pair{Environment.NewLine}" + $"runtime: {companionRuntime}{Environment.NewLine}" + $"device type: {companionDeviceType}"); } simulator = devices.First(v => v.UDID == pair.Gizmo); companionSimulator = companionDevices.First(v => v.UDID == pair.Companion); } if (simulator == null) { throw new Exception($"Could not find simulator{Environment.NewLine}" + $"runtime: {simulatorRuntime}{Environment.NewLine}" + $"device type: {simulatorDeviceType}"); } log.WriteLine("Found simulator: {0} {1}", simulator.Name, simulator.UDID); if (companionSimulator != null) { log.WriteLine("Found companion simulator: {0} {1}", companionSimulator.Name, companionSimulator.UDID); } return(simulator, companionSimulator); }
public async Task <(string deviceName, ProcessExecutionResult result)> InstallApp(AppBundleInformation appBundleInformation, TestTargetOs target, string?deviceName = null, CancellationToken cancellationToken = default) { if (target.Platform.IsSimulator()) { // We reset the simulator when running, so a separate install step does not make much sense. throw new InvalidOperationException("Installing to a simulator is not supported."); } if (!Directory.Exists(appBundleInformation.LaunchAppPath)) { throw new DirectoryNotFoundException("Failed to find the app bundle directory"); } if (deviceName == null) { // the _deviceLoader.FindDevice will return the fist device of the type, but we want to make sure that // the device we use is if the correct arch, therefore, we will use the LoadDevices and return the // correct one await _deviceLoader.LoadDevices(_mainLog, false, false); IHardwareDevice?device = null; if (appBundleInformation.Supports32Bit) { // we only support 32b on iOS, therefore we can ignore the target device = _deviceLoader.Connected32BitIOS.FirstOrDefault(); } else { device = target.Platform switch { TestTarget.Device_iOS => _deviceLoader.Connected64BitIOS.FirstOrDefault(), TestTarget.Device_tvOS => _deviceLoader.ConnectedTV.FirstOrDefault(), _ => device }; } deviceName = target.Platform.IsWatchOSTarget() ? (await _deviceLoader.FindCompanionDevice(_mainLog, device)).Name : device?.Name; } if (deviceName == null) { throw new NoDeviceFoundException(); } var args = new MlaunchArguments(); for (int i = -1; i < _verbosity; i++) { args.Add(new VerbosityArgument()); } args.Add(new InstallAppOnDeviceArgument(appBundleInformation.LaunchAppPath)); args.Add(new DeviceNameArgument(deviceName)); if (target.Platform.IsWatchOSTarget()) { args.Add(new DeviceArgument("ios,watchos")); } var totalSize = Directory.GetFiles(appBundleInformation.LaunchAppPath, "*", SearchOption.AllDirectories).Select((v) => new FileInfo(v).Length).Sum(); _mainLog.WriteLine($"Installing '{appBundleInformation.LaunchAppPath}' to '{deviceName}' ({totalSize / 1024.0 / 1024.0:N2} MB)"); ProcessExecutionResult result = await _processManager.ExecuteCommandAsync(args, _mainLog, TimeSpan.FromHours(1), cancellationToken : cancellationToken); return(deviceName, result); } }
public async Task <(string DeviceName, int?exitCode)> RunApp( AppBundleInformation appInformation, TestTargetOs target, TimeSpan timeout, string?deviceName = null, string?companionDeviceName = null, bool ensureCleanSimulatorState = false, int verbosity = 1, CancellationToken cancellationToken = default) { bool isSimulator = target.Platform.IsSimulator(); ISimulatorDevice?simulator = null; ISimulatorDevice?companionSimulator = null; // Find devices if (isSimulator) { (simulator, companionSimulator) = await _simulatorLoader.FindSimulators(target, _mainLog, 3); deviceName = companionSimulator?.Name ?? simulator.Name; } else { deviceName ??= await FindDevice(target) ?? throw new NoDeviceFoundException(); } var crashLogs = new Logs(_logs.Directory); ICrashSnapshotReporter crashReporter = _snapshotReporterFactory.Create(_mainLog, crashLogs, isDevice: !isSimulator, deviceName); _mainLog.WriteLine($"*** Executing '{appInformation.AppName}' on {target.AsString()} '{deviceName}' ***"); if (isSimulator) { if (simulator == null) { _mainLog.WriteLine("Didn't find any suitable simulator"); throw new NoDeviceFoundException(); } var mlaunchArguments = GetSimulatorArguments(appInformation, simulator, verbosity); await RunSimulatorApp( mlaunchArguments, appInformation, crashReporter, simulator, companionSimulator, ensureCleanSimulatorState, timeout, cancellationToken); } else { var mlaunchArguments = GetDeviceArguments(appInformation, deviceName, target.Platform.IsWatchOSTarget(), verbosity); await RunDeviceApp( mlaunchArguments, crashReporter, deviceName, timeout, cancellationToken); } var systemLog = _logs.FirstOrDefault(log => log.Description == LogType.SystemLog.ToString()); if (systemLog == null) { _mainLog.WriteLine("App run ended but failed to detect exit code (no system log found)"); return(deviceName, null); } var exitCode = _exitCodeDetector.DetectExitCode(appInformation, systemLog); _mainLog.WriteLine($"App run ended with {exitCode}"); return(deviceName, exitCode); }
protected override async Task <ExitCode> RunAppInternal( AppBundleInformation appBundleInfo, string?deviceName, ILogger logger, TestTargetOs target, Logs logs, IFileBackedLog mainLog, CancellationToken cancellationToken) { var tunnelBore = (_arguments.CommunicationChannel == CommunicationChannel.UsbTunnel && !target.Platform.IsSimulator()) ? new TunnelBore(ProcessManager) : null; // only add the extra callback if we do know that the feature was indeed enabled Action <string>?logCallback = IsLldbEnabled() ? (l) => NotifyUserLldbCommand(logger, l) : (Action <string>?)null; var appTester = new AppTester( ProcessManager, DeviceLoader, SimulatorLoader, new SimpleListenerFactory(tunnelBore), new CrashSnapshotReporterFactory(ProcessManager), new CaptureLogFactory(), new DeviceLogCapturerFactory(ProcessManager), new TestReporterFactory(ProcessManager), new XmlResultParser(), mainLog, logs, new Helpers(), logCallback: logCallback, appArguments: PassThroughArguments); string resultMessage; TestExecutingResult testResult; (deviceName, testResult, resultMessage) = await appTester.TestApp(appBundleInfo, target, _arguments.Timeout, _arguments.LaunchTimeout, deviceName, verbosity : GetMlaunchVerbosity(_arguments.Verbosity), xmlResultJargon : _arguments.XmlResultJargon, cancellationToken : cancellationToken, skippedMethods : _arguments.SingleMethodFilters?.ToArray(), skippedTestClasses : _arguments.ClassMethodFilters?.ToArray()); switch (testResult) { case TestExecutingResult.Succeeded: logger.LogInformation($"Application finished the test run successfully"); logger.LogInformation(resultMessage); return(ExitCode.SUCCESS); case TestExecutingResult.Failed: logger.LogInformation($"Application finished the test run successfully with some failed tests"); logger.LogInformation(resultMessage); return(ExitCode.TESTS_FAILED); case TestExecutingResult.LaunchFailure: if (resultMessage != null) { logger.LogError($"Failed to launch the application:{Environment.NewLine}" + $"{resultMessage}{Environment.NewLine}{Environment.NewLine}" + $"Check logs for more information."); } else { logger.LogError($"Failed to launch the application. Check logs for more information"); } return(ExitCode.APP_LAUNCH_FAILURE); case TestExecutingResult.Crashed: if (resultMessage != null) { logger.LogError($"Application run crashed:{Environment.NewLine}" + $"{resultMessage}{Environment.NewLine}{Environment.NewLine}" + $"Check logs for more information."); } else { logger.LogError($"Application test run crashed. Check logs for more information"); } return(ExitCode.APP_CRASH); case TestExecutingResult.TimedOut: logger.LogWarning($"Application test run timed out"); return(ExitCode.TIMED_OUT); default: if (resultMessage != null) { logger.LogError($"Application test run ended in an unexpected way: '{testResult}'{Environment.NewLine}" + $"{resultMessage}{Environment.NewLine}{Environment.NewLine}" + $"Check logs for more information."); } else { logger.LogError($"Application test run ended in an unexpected way: '{testResult}'. Check logs for more information"); } return(ExitCode.GENERAL_FAILURE); } }
public async Task <(string DeviceName, ProcessExecutionResult result)> RunApp( AppBundleInformation appInformation, TestTargetOs target, TimeSpan timeout, string?deviceName = null, string?companionDeviceName = null, bool ensureCleanSimulatorState = false, int verbosity = 1, CancellationToken cancellationToken = default) { var isSimulator = target.Platform.IsSimulator(); ProcessExecutionResult result; ISimulatorDevice? simulator = null; ISimulatorDevice? companionSimulator = null; if (target.Platform == TestTarget.MacCatalyst) { _mainLog.WriteLine($"*** Executing '{appInformation.AppName}' on MacCatalyst ***"); result = await RunMacCatalystApp(appInformation, timeout, _appArguments, new Dictionary <string, object>(), cancellationToken); return("MacCatalyst", result); } // Find devices if (isSimulator) { (simulator, companionSimulator) = await _simulatorLoader.FindSimulators(target, _mainLog, 3); deviceName = companionSimulator?.Name ?? simulator.Name; } else { deviceName ??= await FindDevice(target) ?? throw new NoDeviceFoundException(); } var crashLogs = new Logs(_logs.Directory); ICrashSnapshotReporter crashReporter = _snapshotReporterFactory.Create( _mainLog, crashLogs, isDevice: !isSimulator, deviceName); _mainLog.WriteLine($"*** Executing '{appInformation.AppName}' on {target.AsString()} '{deviceName}' ***"); if (isSimulator) { if (simulator == null) { _mainLog.WriteLine("Didn't find any suitable simulator"); throw new NoDeviceFoundException(); } var mlaunchArguments = GetSimulatorArguments(appInformation, simulator, verbosity); result = await RunSimulatorApp( mlaunchArguments, appInformation, crashReporter, simulator, companionSimulator, ensureCleanSimulatorState, timeout, cancellationToken); } else { var mlaunchArguments = GetDeviceArguments(appInformation, deviceName, target.Platform.IsWatchOSTarget(), verbosity); result = await RunDeviceApp( mlaunchArguments, crashReporter, deviceName, timeout, cancellationToken); } return(deviceName, result); }
public async Task <(ISimulatorDevice Simulator, ISimulatorDevice?CompanionSimulator)> FindSimulators(TestTargetOs target, ILog log, int retryCount, bool createIfNeeded = true, bool minVersion = false) { if (retryCount < 1) { throw new ArgumentOutOfRangeException(nameof(retryCount)); } int attempt = 1; while (true) { try { return(await FindSimulators(target, log)); } catch (Exception e) { log.WriteLine($"Failed to find/create simulator (attempt {attempt}/{retryCount}):" + Environment.NewLine + e); if (attempt == retryCount) { throw new NoDeviceFoundException("Failed to find/create suitable simulator"); } } finally { attempt++; } } }
public async Task <DevicePair> FindDevice( TestTargetOs target, string?deviceName, ILog log, bool includeWirelessDevices = true, CancellationToken cancellationToken = default) { IDevice?device; IDevice?companionDevice = null; bool IsMatchingDevice(IDevice device) => device.Name.Equals(deviceName, StringComparison.InvariantCultureIgnoreCase) || device.UDID.Equals(deviceName, StringComparison.InvariantCultureIgnoreCase); if (target.Platform.IsSimulator()) { if (deviceName == null) { (device, companionDevice) = await _simulatorLoader.FindSimulators(target, log, retryCount : 3, cancellationToken : cancellationToken); } else { await _simulatorLoader.LoadDevices(log, includeLocked : false, cancellationToken : cancellationToken); device = _simulatorLoader.AvailableDevices.FirstOrDefault(IsMatchingDevice) ?? throw new NoDeviceFoundException($"Failed to find a simulator '{deviceName}'"); } } else { // The DeviceLoader.FindDevice will return the fist device of the type, but we want to make sure that // the device we use is of the correct arch, therefore, we will use the LoadDevices and handpick one await _deviceLoader.LoadDevices( log, includeLocked : false, forceRefresh : false, includeWirelessDevices : includeWirelessDevices, cancellationToken : cancellationToken); if (deviceName == null) { IHardwareDevice?hardwareDevice = target.Platform switch { TestTarget.Simulator_iOS32 => _deviceLoader.Connected32BitIOS.FirstOrDefault(), TestTarget.Device_iOS => _deviceLoader.Connected64BitIOS.FirstOrDefault(), TestTarget.Device_tvOS => _deviceLoader.ConnectedTV.FirstOrDefault(), _ => throw new ArgumentOutOfRangeException(nameof(target), $"Unrecognized device platform {target.Platform}") }; if (target.Platform.IsWatchOSTarget() && hardwareDevice != null) { companionDevice = await _deviceLoader.FindCompanionDevice(log, hardwareDevice, cancellationToken : cancellationToken); } device = hardwareDevice; } else { device = _deviceLoader.ConnectedDevices.FirstOrDefault(IsMatchingDevice) ?? throw new NoDeviceFoundException($"Failed to find a device '{deviceName}'. " + "Please make sure the device is connected and unlocked."); } } if (device == null) { throw new NoDeviceFoundException($"Failed to find a suitable device for target {target.AsString()}"); } return(new DevicePair(device, companionDevice)); }
private async Task <ExitCode> RunTest( ILogger logger, TestTargetOs target, Logs logs, MLaunchProcessManager processManager, IHardwareDeviceLoader deviceLoader, ISimulatorLoader simulatorLoader, ITunnelBore?tunnelBore, CancellationToken cancellationToken = default) { var isLldbEnabled = IsLldbEnabled(); if (isLldbEnabled && !_arguments.EnableLldb) { // the file is present, but the user did not set it, warn him about it logger.LogWarning("Lldb will be used since '~/.mtouch-launch-with-lldb' was found in the system but it was not created by xharness."); } else if (_arguments.EnableLldb) { if (!File.Exists(s_mlaunchLldbConfigFile)) { // create empty file File.WriteAllText(s_mlaunchLldbConfigFile, string.Empty); _createdLldbFile = true; } } logger.LogInformation($"Starting test for {target.AsString()}{ (_arguments.DeviceName != null ? " targeting " + _arguments.DeviceName : null) }.."); string mainLogFile = Path.Join(_arguments.OutputDirectory, $"run-{target.AsString()}{(_arguments.DeviceName != null ? "-" + _arguments.DeviceName : null)}.log"); IFileBackedLog mainLog = Log.CreateReadableAggregatedLog( logs.Create(mainLogFile, LogType.ExecutionLog.ToString(), true), new CallbackLog(message => logger.LogDebug(message.Trim())) { Timestamp = false }); int verbosity = GetMlaunchVerbosity(_arguments.Verbosity); string?deviceName = _arguments.DeviceName; var appBundleInformationParser = new AppBundleInformationParser(processManager); AppBundleInformation appBundleInfo; try { appBundleInfo = await appBundleInformationParser.ParseFromAppBundle(_arguments.AppPackagePath, target.Platform, mainLog, cancellationToken); } catch (Exception e) { logger.LogError($"Failed to get bundle information: {e.Message}"); return(ExitCode.FAILED_TO_GET_BUNDLE_INFO); } if (!target.Platform.IsSimulator()) { logger.LogInformation($"Installing application '{appBundleInfo.AppName}' on " + (deviceName != null ? $" on device '{deviceName}'" : target.AsString())); var appInstaller = new AppInstaller(processManager, deviceLoader, mainLog, verbosity); ProcessExecutionResult result; try { (deviceName, result) = await appInstaller.InstallApp(appBundleInfo, target, cancellationToken : cancellationToken); } catch (NoDeviceFoundException) { logger.LogError($"Failed to find suitable device for target {target.AsString()}" + Environment.NewLine + "Please make sure the device is connected and unlocked."); return(ExitCode.DEVICE_NOT_FOUND); } catch (Exception e) { logger.LogError($"Failed to install the app bundle:{Environment.NewLine}{e}"); return(ExitCode.PACKAGE_INSTALLATION_FAILURE); } if (!result.Succeeded) { // use the knowledge base class to decide if the error is known, if it is, let the user know // the failure reason if (_errorKnowledgeBase.IsKnownInstallIssue(mainLog, out var errorMessage)) { var msg = $"Failed to install the app bundle (exit code={result.ExitCode}): {errorMessage.Value.HumanMessage}."; if (errorMessage.Value.IssueLink != null) { msg += $" Find more information at {errorMessage.Value.IssueLink}"; } logger.LogError(msg); } else { logger.LogError($"Failed to install the app bundle (exit code={result.ExitCode})"); } return(ExitCode.PACKAGE_INSTALLATION_FAILURE); } logger.LogInformation($"Application '{appBundleInfo.AppName}' was installed successfully on device '{deviceName}'"); } logger.LogInformation($"Starting application '{appBundleInfo.AppName}' on " + (deviceName != null ? $"device '{deviceName}'" : target.AsString())); // only add the extra callback if we do know that the feature was indeed enabled Action <string>?logCallback = isLldbEnabled ? (l) => NotifyUserLldbCommand(logger, l) : (Action <string>?)null; var appRunner = new AppRunner( processManager, deviceLoader, simulatorLoader, new SimpleListenerFactory(tunnelBore), new CrashSnapshotReporterFactory(processManager), new CaptureLogFactory(), new DeviceLogCapturerFactory(processManager), new TestReporterFactory(processManager), new XmlResultParser(), mainLog, logs, new Helpers(), logCallback: logCallback, appArguments: PassThroughArguments); try { string resultMessage; TestExecutingResult testResult; (deviceName, testResult, resultMessage) = await appRunner.RunApp(appBundleInfo, target, _arguments.Timeout, _arguments.LaunchTimeout, deviceName, verbosity : verbosity, xmlResultJargon : _arguments.XmlResultJargon, cancellationToken : cancellationToken, skippedMethods : _arguments.SingleMethodFilters?.ToArray(), skippedTestClasses : _arguments.ClassMethodFilters?.ToArray()); switch (testResult) { case TestExecutingResult.Succeeded: logger.LogInformation($"Application finished the test run successfully"); logger.LogInformation(resultMessage); return(ExitCode.SUCCESS); case TestExecutingResult.Failed: logger.LogInformation($"Application finished the test run successfully with some failed tests"); logger.LogInformation(resultMessage); return(ExitCode.TESTS_FAILED); case TestExecutingResult.LaunchFailure: if (resultMessage != null) { logger.LogError($"Failed to launch the application:{Environment.NewLine}" + $"{resultMessage}{Environment.NewLine}{Environment.NewLine}" + $"Check logs for more information."); } else { logger.LogError($"Failed to launch the application. Check logs for more information"); } return(ExitCode.APP_LAUNCH_FAILURE); case TestExecutingResult.Crashed: if (resultMessage != null) { logger.LogError($"Application run crashed:{Environment.NewLine}" + $"{resultMessage}{Environment.NewLine}{Environment.NewLine}" + $"Check logs for more information."); } else { logger.LogError($"Application run crashed. Check logs for more information"); } return(ExitCode.APP_CRASH); case TestExecutingResult.TimedOut: logger.LogWarning($"Application run timed out"); return(ExitCode.TIMED_OUT); default: if (resultMessage != null) { logger.LogError($"Application run ended in an unexpected way: '{testResult}'{Environment.NewLine}" + $"{resultMessage}{Environment.NewLine}{Environment.NewLine}" + $"Check logs for more information."); } else { logger.LogError($"Application run ended in an unexpected way: '{testResult}'. Check logs for more information"); } return(ExitCode.GENERAL_FAILURE); } } catch (NoDeviceFoundException) { logger.LogError($"Failed to find suitable device for target {target.AsString()}" + (target.Platform.IsSimulator() ? Environment.NewLine + "Please make sure suitable Simulator is installed in Xcode" : string.Empty)); return(ExitCode.DEVICE_NOT_FOUND); } catch (Exception e) { if (_errorKnowledgeBase.IsKnownTestIssue(mainLog, out var failureMessage)) { var msg = $"Application run failed:{Environment.NewLine}{failureMessage.Value.HumanMessage}."; if (failureMessage.Value.IssueLink != null) { msg += $" Find more information at {failureMessage.Value.IssueLink}"; } logger.LogError(msg); } else { logger.LogError($"Application run failed:{Environment.NewLine}{e}"); } return(ExitCode.APP_CRASH); } finally { mainLog.Dispose(); if (!target.Platform.IsSimulator() && deviceName != null) { logger.LogInformation($"Uninstalling the application '{appBundleInfo.AppName}' from device '{deviceName}'"); var appUninstaller = new AppUninstaller(processManager, mainLog, verbosity); var uninstallResult = await appUninstaller.UninstallApp(deviceName, appBundleInfo.BundleIdentifier, cancellationToken); if (!uninstallResult.Succeeded) { logger.LogError($"Failed to uninstall the app bundle with exit code: {uninstallResult.ExitCode}"); } else { logger.LogInformation($"Application '{appBundleInfo.AppName}' was uninstalled successfully"); } } if (_createdLldbFile) // clean after the setting { File.Delete(s_mlaunchLldbConfigFile); } } }
public async Task OrchestrateSimulatorJustTestTest() { // Setup var testTarget = new TestTargetOs(TestTarget.Simulator_iOS64, "13.5"); var envVars = new[] { ("envVar1", "value1"), ("envVar2", "value2") };
protected override async Task <ExitCode> RunAppInternal( AppBundleInformation appBundleInfo, string?deviceName, ILogger logger, TestTargetOs target, Logs logs, IFileBackedLog mainLog, CancellationToken cancellationToken) { // only add the extra callback if we do know that the feature was indeed enabled Action <string>?logCallback = IsLldbEnabled() ? (l) => NotifyUserLldbCommand(logger, l) : (Action <string>?)null; var appRunner = new AppRunner( ProcessManager, DeviceLoader, SimulatorLoader, new CrashSnapshotReporterFactory(ProcessManager), new CaptureLogFactory(), new DeviceLogCapturerFactory(ProcessManager), mainLog, logs, new Helpers(), PassThroughArguments, logCallback); ProcessExecutionResult result; (deviceName, result) = await appRunner.RunApp( appBundleInfo, target, _arguments.Timeout, deviceName, verbosity : GetMlaunchVerbosity(_arguments.Verbosity), cancellationToken : cancellationToken); if (!result.Succeeded) { if (result.TimedOut) { logger.LogError($"App run has timed out"); return(ExitCode.TIMED_OUT); } logger.LogError($"App run has failed. mlaunch exited with {result.ExitCode}"); return(ExitCode.APP_LAUNCH_FAILURE); } var systemLog = logs.FirstOrDefault(log => log.Description == LogType.SystemLog.ToString()); if (systemLog == null) { logger.LogError("Application has finished but no system log found. Failed to determine the exit code!"); return(ExitCode.RETURN_CODE_NOT_SET); } var exitCode = new ExitCodeDetector().DetectExitCode(appBundleInfo, systemLog); logger.LogInformation($"App run ended with {exitCode}"); if (_arguments.ExpectedExitCode != exitCode) { logger.LogError($"Application has finished with exit code {exitCode} but {_arguments.ExpectedExitCode} was expected"); return(ExitCode.GENERAL_FAILURE); } logger.LogInformation("Application has finished with exit code: " + exitCode + (_arguments.ExpectedExitCode != 0 ? " (as expected)" : null)); return(ExitCode.SUCCESS); }
protected void VerifyDiagnosticData(TestTargetOs target) { Assert.Equal(target.Platform.IsSimulator() ? _simulator.Object.Name : _device.Object.Name, _diagnosticsData.Device); Assert.Contains(target.OSVersion, _diagnosticsData.TargetOS); }