public void RunOnDeviceWithNoAvailableSimulatorTest() { _hardwareDeviceLoader = new Mock <IHardwareDeviceLoader>(); _hardwareDeviceLoader .Setup(x => x.FindDevice(RunMode.iOS, _mainLog.Object, false, false)) .ThrowsAsync(new NoDeviceFoundException()); // Act var appRunner = new AppRunner(_processManager.Object, _hardwareDeviceLoader.Object, _simulatorLoader.Object, _listenerFactory, _snapshotReporterFactory, Mock.Of <ICaptureLogFactory>(), Mock.Of <IDeviceLogCapturerFactory>(), _testReporterFactory, _mainLog.Object, _logs.Object, _helpers.Object); var appInformation = new AppBundleInformation(AppName, AppBundleIdentifier, s_appPath, s_appPath, null); Assert.ThrowsAsync <NoDeviceFoundException>( async() => await appRunner.RunApp( appInformation, TestTarget.Device_iOS, TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(30), ensureCleanSimulatorState: true), "Running requires connected devices"); }
public int?DetectExitCode(AppBundleInformation appBundleInfo, IReadableLog log) { StreamReader reader; try { reader = log.GetReader(); } catch (FileNotFoundException e) { throw new Exception("Failed to detect application's exit code. The log file was empty / not found at " + e.FileName); } using (reader) while (!reader.EndOfStream) { if (reader.ReadLine() is string line && IsSignalLine(appBundleInfo, line) is Match match && match.Success && int.TryParse(match.Groups["exitCode"].Value, out var exitCode)) { return(exitCode); } } return(null); }
public ITestReporter Create(ILog mainLog, ILog runLog, ILogs logs, ICrashSnapshotReporter crashReporter, ISimpleListener simpleListener, IResultParser parser, AppBundleInformation appInformation, RunMode runMode, XmlResultJargon xmlJargon, string device, TimeSpan timeout, double launchTimeout, string additionalLogsDirectory = null, ExceptionLogger exceptionLogger = null) { return(new TestReporter(processManager, mainLog, runLog, logs, crashReporter, simpleListener, parser, appInformation, runMode, xmlJargon, device, timeout, launchTimeout, additionalLogsDirectory, exceptionLogger)); }
protected virtual async Task <ExitCode> InstallApp( AppBundleInformation appBundleInfo, IDevice device, TestTargetOs target, CancellationToken cancellationToken) { _logger.LogInformation($"Installing application '{appBundleInfo.AppName}' on '{device.Name}'"); ProcessExecutionResult result; try { result = await _appInstaller.InstallApp(appBundleInfo, target, device, cancellationToken); } catch (Exception e) { _logger.LogError($"Failed to install the app bundle:{Environment.NewLine}{e}"); return(ExitCode.PACKAGE_INSTALLATION_FAILURE); } if (!result.Succeeded) { cancellationToken.ThrowIfCancellationRequested(); var exitCode = ExitCode.PACKAGE_INSTALLATION_FAILURE; // use the knowledge base class to decide if the error is known, if it is, let the user know // the failure reason if (_errorKnowledgeBase.IsKnownInstallIssue(_mainLog, out var failure)) { var error = new StringBuilder() .AppendLine("Failed to install the application") .AppendLine(failure.HumanMessage); if (failure.IssueLink != null) { error .AppendLine() .AppendLine($" Find more information at {failure.IssueLink}"); } if (failure.SuggestedExitCode.HasValue) { exitCode = (ExitCode)failure.SuggestedExitCode.Value; } _logger.LogError(error.ToString()); } else { _logger.LogError($"Failed to install the application"); } return(exitCode); } _logger.LogInformation($"Application '{appBundleInfo.AppName}' was installed successfully on '{device.Name}'"); return(ExitCode.SUCCESS); }
public AppInstallerTests() { _mainLog = new Mock <ILog>(); _processManager = new Mock <IProcessManager>(); _processManager.SetReturnsDefault(Task.FromResult(new ProcessExecutionResult() { ExitCode = 0 })); _hardwareDeviceLoader = new Mock <IHardwareDeviceLoader>(); _hardwareDeviceLoader .Setup(x => x.Connected64BitIOS).Returns(new List <IHardwareDevice> { s_mockDevice }); Directory.CreateDirectory(s_appPath); _appBundleInformation = new AppBundleInformation( appName: "AppName", bundleIdentifier: s_appIdentifier, appPath: s_appPath, launchAppPath: s_appPath, supports32b: false, extension: null); }
public async Task <ProcessExecutionResult> InstallApp( AppBundleInformation appBundleInformation, TestTargetOs target, IDevice device, CancellationToken cancellationToken = default) { if (!Directory.Exists(appBundleInformation.LaunchAppPath)) { throw new DirectoryNotFoundException("Failed to find the app bundle directory"); } var args = new MlaunchArguments(); if (target.Platform.IsSimulator()) { args.Add(new SimulatorUDIDArgument(device)); args.Add(new InstallAppOnSimulatorArgument(appBundleInformation.LaunchAppPath)); } else { args.Add(new DeviceNameArgument(device)); args.Add(new InstallAppOnDeviceArgument(appBundleInformation.LaunchAppPath)); if (target.Platform.IsWatchOSTarget()) { args.Add(new DeviceArgument("ios,watchos")); } } var totalSize = Directory.GetFiles(appBundleInformation.LaunchAppPath, "*", SearchOption.AllDirectories).Select((v) => new FileInfo(v).Length).Sum(); _mainLog.WriteLine($"Installing '{appBundleInformation.LaunchAppPath}' to '{device.Name}' ({totalSize / 1024.0 / 1024.0:N2} MB)"); return(await _processManager.ExecuteCommandAsync(args, _mainLog, TimeSpan.FromMinutes(15), verbosity : 2, cancellationToken : cancellationToken)); }
public void iOSDeviceCodeIsDetectedTest() { var appBundleInformation = new AppBundleInformation("iOS.Simulator.PInvoke.Test", "net.dot.iOS.Simulator.PInvoke.Test", "some/path", "some/path", false, null); var detector = new iOSExitCodeDetector(); var log = new[] { "[07:02:15.9749990] Xamarin.Hosting: Mounting developer image on 'DNCENGOSX-003'", "[07:02:15.9752160] Xamarin.Hosting: Mounted developer image on 'DNCENGOSX-003'", "[07:02:16.5177370] Xamarin.Hosting: Launched net.dot.Some.Other.App with PID: 13942", "[07:02:16.5181560] Launched application 'net.dot.Some.Other.App' on 'DNCENGOSX-003' with pid 13942", "[07:02:16.6150270] 2022-03-30 07:02:16.601 Some.Other.App[13942:136284382] Done!", "[07:02:21.6632630] Xamarin.Hosting: Process '13942' exited with exit code 143 or crashing signal .", "[07:02:21.6637600] Application 'net.dot.Some.Other.App' terminated (with exit code '143' and/or crashing signal ').", // We care about this run "[07:02:15.9749990] Xamarin.Hosting: Mounting developer image on 'DNCENGOSX-003'", "[07:02:15.9752160] Xamarin.Hosting: Mounted developer image on 'DNCENGOSX-003'", "[07:02:16.5177370] Xamarin.Hosting: Launched net.dot.iOS.Simulator.PInvoke.Test with PID: 83937", "[07:02:16.5181560] Launched application 'net.dot.iOS.Simulator.PInvoke.Test' on 'DNCENGOSX-003' with pid 83937", "[07:02:16.6150270] 2022-03-30 07:02:16.601 iOS.Simulator.PInvoke.Test[83937:136284382] Done!", "[07:02:21.6632630] Xamarin.Hosting: Process '83937' exited with exit code 42 or crashing signal .", "[07:02:21.6637600] Application 'net.dot.iOS.Simulator.PInvoke.Test' terminated (with exit code '42' and/or crashing signal ').", }; var exitCode = detector.DetectExitCode(appBundleInformation, GetLogMock(log)); Assert.Equal(42, exitCode); }
public ITestReporter Create(IFileBackedLog mainLog, IReadableLog runLog, ILogs logs, ICrashSnapshotReporter crashReporter, ISimpleListener simpleListener, IResultParser parser, AppBundleInformation appInformation, RunMode runMode, XmlResultJargon xmlJargon, string?device, TimeSpan timeout, string?additionalLogsDirectory = null, ExceptionLogger?exceptionLogger = null, bool generateHtml = false) => new TestReporter(_processManager, mainLog, runLog, logs, crashReporter, simpleListener, parser, appInformation, runMode, xmlJargon, device, timeout, additionalLogsDirectory, exceptionLogger, generateHtml);
private MlaunchArguments GetSimulatorArguments( AppBundleInformation appInformation, ISimulatorDevice simulator, int verbosity) { var args = GetCommonArguments(verbosity); args.Add(new SimulatorUDIDArgument(simulator.UDID)); if (appInformation.Extension.HasValue) { switch (appInformation.Extension) { case Extension.TodayExtension: args.Add(new LaunchSimulatorExtensionArgument(appInformation.LaunchAppPath, appInformation.BundleIdentifier)); break; case Extension.WatchKit2: default: throw new NotImplementedException(); } } else { args.Add(new LaunchSimulatorArgument(appInformation.LaunchAppPath)); } return(args); }
public void ExitCodeIsDetectedOnMacCatalystTest() { var appBundleInformation = new AppBundleInformation("System.Buffers.Tests", "net.dot.System.Buffers.Tests", "some/path", "some/path", false, null); var detector = new MacCatalystExitCodeDetector(); var log = new[] { "Feb 18 06:40:16 Admins-Mac-Mini System.Buffers.Tests[59229]: CDN - client insert callback function client = 0 type = 17 function = 0x7fff3b262246 local_olny = false", "Feb 18 06:40:16 Admins-Mac-Mini System.Buffers.Tests[59229]: CDN - client setup_remote_port", "Feb 18 06:40:16 Admins-Mac-Mini System.Buffers.Tests[59229]: CDN - Bootstrap Port: 1799", "Feb 18 06:40:16 Admins-Mac-Mini System.Buffers.Tests[59229]: CDN - Remote Port: 56835 (com.apple.CoreDisplay.Notification)", "Feb 18 06:40:16 Admins-Mac-Mini System.Buffers.Tests[59229]: CDN - client setup_local_port", "Feb 18 06:40:16 Admins-Mac-Mini System.Buffers.Tests[59229]: CDN - Local Port: 78339", "Feb 18 06:40:16 Admins-Mac-Mini com.apple.xpc.launchd[1] (net.dot.System.Buffers.Tests.15140[59229]): Service exited with abnormal code: 74", "Feb 18 06:40:49 Admins-Mac-Mini com.apple.xpc.launchd[1] (com.apple.mdworker.shared.09000000-0600-0000-0000-000000000000[59231]): Service exited due to SIGKILL | sent by mds[88]", "Feb 18 06:40:58 Admins-Mac-Mini com.apple.xpc.launchd[1] (com.apple.mdworker.shared.02000000-0100-0000-0000-000000000000[59232]): Service exited due to SIGKILL | sent by mds[88]", "Feb 18 06:41:01 Admins-Mac-Mini com.apple.xpc.launchd[1] (com.apple.mdworker.shared.0D000000-0000-0000-0000-000000000000[59237]): Service exited due to SIGKILL | sent by mds[88]", "Feb 18 06:41:23 Admins-Mac-Mini System.Buffers.Tests[59248]: CDN - client insert callback function client = 0 type = 17 function = 0x7fff3b262246 local_olny = false", "Feb 18 06:41:23 Admins-Mac-Mini System.Buffers.Tests[59248]: CDN - client setup_remote_port", "Feb 18 06:41:23 Admins-Mac-Mini System.Buffers.Tests[59248]: CDN - Bootstrap Port: 1799", "Feb 18 06:41:23 Admins-Mac-Mini System.Buffers.Tests[59248]: CDN - Remote Port: 75271 (com.apple.CoreDisplay.Notification)", "Feb 18 06:41:23 Admins-Mac-Mini System.Buffers.Tests[59248]: CDN - client setup_local_port", "Feb 18 06:41:23 Admins-Mac-Mini System.Buffers.Tests[59248]: CDN - Local Port: 52995", }; var exitCode = detector.DetectExitCode(appBundleInformation, GetLogMock(log)); Assert.Equal(74, exitCode); }
public void ExitCodeFromPreviousRunIsIgnored() { var previousLog = "Nov 18 04:31:44 dci-mac-build-053 CloudKeychainProxy[67121]: objc[67121]: Class MockAKSRefKeyObject is implemented in both /Applications/Xcode115.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Security.framework/Security (0x10350b738) and /Applications/Xcode115.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Security.framework/CloudKeychainProxy.bundle/CloudKeychainProxy (0x103259970). One of the two will be used. Which one is undefined." + Environment.NewLine + "Nov 18 04:31:44 dci-mac-build-053 CloudKeychainProxy[67121]: objc[67121]: Class MockAKSOptionalParameters is implemented in both /Applications/Xcode115.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Security.framework/Security (0x10350b788) and /Applications/Xcode115.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Security.framework/CloudKeychainProxy.bundle/CloudKeychainProxy (0x1032599c0). One of the two will be used. Which one is undefined." + Environment.NewLine + "Nov 18 04:31:44 dci-mac-build-053 CloudKeychainProxy[67121]: objc[67121]: Class SecXPCHelper is implemented in both /Applications/Xcode115.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Security.framework/Security (0x10350b918) and /Applications/Xcode115.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Security.framework/CloudKeychainProxy.bundle/CloudKeychainProxy (0x103259a60). One of the two will be used. Which one is undefined." + Environment.NewLine + "Nov 18 04:31:44 dci-mac-build-053 ML-MacVM com.apple.CoreSimulator.SimDevice.2E1EE736-5672-4220-89B5-B7C77DB6AF18[55655] (UIKitApplication:net.dot.HelloiOS[9a0b][rb-legacy][57331]): Some other error message" + Environment.NewLine + "Nov 18 04:31:44 dci-mac-build-053 ML-MacVM com.apple.CoreSimulator.SimDevice.2E1EE736-5672-4220-89B5-B7C77DB6AF18[55655] (UIKitApplication:net.dot.HelloiOS[9a0b][rb-legacy][57331]): Service exited with abnormal code: 55" + Environment.NewLine + "Nov 18 04:31:44 dci-mac-build-053 com.apple.CoreSimulator.SimDevice.F67392D9-A327-4217-B924-5DA0918415E5[811] (com.apple.security.cloudkeychainproxy3): Service only ran for 0 seconds. Pushing respawn out by 10 seconds." + Environment.NewLine; var currentRunLog = "Nov 18 04:31:44 dci-mac-build-053 CloudKeychainProxy[67121]: objc[67121]: Class MockAKSRefKeyObject is implemented in both /Applications/Xcode115.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Security.framework/Security (0x10350b738) and /Applications/Xcode115.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Security.framework/CloudKeychainProxy.bundle/CloudKeychainProxy (0x103259970). One of the two will be used. Which one is undefined." + Environment.NewLine + "Nov 18 04:31:44 dci-mac-build-053 CloudKeychainProxy[67121]: objc[67121]: Class MockAKSOptionalParameters is implemented in both /Applications/Xcode115.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Security.framework/Security (0x10350b788) and /Applications/Xcode115.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Security.framework/CloudKeychainProxy.bundle/CloudKeychainProxy (0x1032599c0). One of the two will be used. Which one is undefined." + Environment.NewLine + "Nov 18 04:31:44 dci-mac-build-053 CloudKeychainProxy[67121]: objc[67121]: Class SecXPCHelper is implemented in both /Applications/Xcode115.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Security.framework/Security (0x10350b918) and /Applications/Xcode115.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Security.framework/CloudKeychainProxy.bundle/CloudKeychainProxy (0x103259a60). One of the two will be used. Which one is undefined." + Environment.NewLine + "Nov 18 04:31:44 dci-mac-build-053 ML-MacVM com.apple.CoreSimulator.SimDevice.2E1EE736-5672-4220-89B5-B7C77DB6AF18[55655] (UIKitApplication:net.dot.HelloiOS[9a0b][rb-legacy][57331]): Some other error message" + Environment.NewLine + "Nov 18 04:31:44 dci-mac-build-053 ML-MacVM com.apple.CoreSimulator.SimDevice.2E1EE736-5672-4220-89B5-B7C77DB6AF18[55655] (UIKitApplication:net.dot.HelloiOS[9a0b][rb-legacy][57331]): Service exited with abnormal code: 72" + Environment.NewLine + "Nov 18 04:31:44 dci-mac-build-053 com.apple.CoreSimulator.SimDevice.F67392D9-A327-4217-B924-5DA0918415E5[811] (com.apple.security.cloudkeychainproxy3): Service only ran for 0 seconds. Pushing respawn out by 10 seconds." + Environment.NewLine; var tempFilename = Path.GetTempFileName(); File.WriteAllText(tempFilename, previousLog); var capturedFilename = Path.GetTempFileName(); using var captureLog = new CaptureLog(capturedFilename, tempFilename, false); captureLog.StartCapture(); File.AppendAllText(tempFilename, currentRunLog); captureLog.StopCapture(); var appBundleInformation = new AppBundleInformation("net.dot.HelloiOS", "net.dot.HelloiOS", "some/path", "some/path", false, null); var exitCode = new iOSExitCodeDetector().DetectExitCode(appBundleInformation, captureLog); Assert.Equal(72, exitCode); }
protected virtual Match?IsSignalLine(AppBundleInformation appBundleInfo, string logLine) { if (IsAbnormalExitLine(appBundleInfo, logLine) || IsStdoutExitLine(appBundleInfo, logLine)) { return(EoLExitCodeRegex.Match(logLine)); } return(null); }
protected override async Task <ExitCode> RunAppInternal( AppBundleInformation appBundleInfo, string?deviceName, ILogger logger, TestTargetOs target, Logs logs, IFileBackedLog mainLog, CancellationToken cancellationToken) { // only add the extra callback if we do know that the feature was indeed enabled Action <string>?logCallback = IsLldbEnabled() ? (l) => NotifyUserLldbCommand(logger, l) : (Action <string>?)null; var appRunner = new AppRunner( ProcessManager, DeviceLoader, SimulatorLoader, new CrashSnapshotReporterFactory(ProcessManager), new CaptureLogFactory(), new DeviceLogCapturerFactory(ProcessManager), new ExitCodeDetector(), mainLog, logs, new Helpers(), PassThroughArguments, logCallback); int?exitCode = null; (deviceName, exitCode) = await appRunner.RunApp( appBundleInfo, target, _arguments.Timeout, deviceName, verbosity : GetMlaunchVerbosity(_arguments.Verbosity), cancellationToken : cancellationToken); if (exitCode.HasValue) { if (_arguments.ExpectedExitCode != exitCode) { logger.LogError($"Application has finished with exit code {exitCode} but {_arguments.ExpectedExitCode} was expected"); return(ExitCode.GENERAL_FAILURE); } logger.LogInformation("Application has finished with exit code: " + exitCode + (_arguments.ExpectedExitCode != 0 ? " (as expected)" : null)); return(ExitCode.SUCCESS); } else { logger.LogError("Application has finished but no system log found. Failed to determine the exit code!"); return(ExitCode.RETURN_CODE_NOT_SET); } }
protected override Match?IsSignalLine(AppBundleInformation appBundleInfo, string logLine) { if (base.IsSignalLine(appBundleInfo, logLine) is Match match && match.Success) { return(match); } if (logLine.Contains(appBundleInfo.BundleIdentifier)) { return(DeviceExitCodeRegex.Match(logLine)); } return(null); }
public async Task InstallOn32bMissingDevice() { var appBundle32b = new AppBundleInformation( appName: "AppName", bundleIdentifier: s_appIdentifier, appPath: s_appPath, launchAppPath: s_appPath, supports32b: true, extension: null); var appInstaller = new AppInstaller(_processManager.Object, _hardwareDeviceLoader.Object, _mainLog.Object, 1); await Assert.ThrowsAsync <NoDeviceFoundException>( async() => await appInstaller.InstallApp(appBundle32b, TestTarget.Device_iOS)); }
public TestReporterTests() { _crashReporter = new Mock <ICrashSnapshotReporter>(); _processManager = new Mock <IProcessManager>(); _parser = new XmlResultParser(); _runLog = new Mock <ILog>(); _mainLog = new Mock <ILog>(); _logs = new Mock <ILogs>(); _listener = new Mock <ISimpleListener>(); _appInformation = new AppBundleInformation("test app", "my.id.com", "/path/to/app", "/launch/app/path", null) { Variation = "Debug" }; _logsDirectory = Path.GetTempFileName(); File.Delete(_logsDirectory); Directory.CreateDirectory(_logsDirectory); }
protected async Task <AppBundleInformation> GetAppBundleFromId(TestTargetOs target, IDevice device, string bundleIdentifier, CancellationToken cancellationToken) { // We can exchange bundle ID for path where the bundle is on the simulator and get that if (device is ISimulatorDevice simulator) { _logger.LogInformation($"Querying simulator for app bundle information.."); await simulator.Boot(_mainLog, cancellationToken); var appBundlePath = await simulator.GetAppBundlePath(_mainLog, bundleIdentifier, cancellationToken); return(await GetAppBundleFromPath(target, appBundlePath, cancellationToken)); } // We're unable to do this for real devices / or MacCatalyst // It is not ideal but doesn't matter much at the moment as we don't need the full list of properties there _logger.LogDebug("Supplemented full app bundle information with bundle identifier"); return(AppBundleInformation.FromBundleId(bundleIdentifier)); }
private MlaunchArguments GetSimulatorArguments( AppBundleInformation appInformation, ISimulatorDevice simulator, int verbosity, XmlResultJargon xmlResultJargon, string[]?skippedMethods, string[]?skippedTestClasses, ListenerTransport deviceListenerTransport, int deviceListenerPort, string deviceListenerTmpFile) { var args = GetCommonArguments( verbosity, xmlResultJargon, skippedMethods, skippedTestClasses, deviceListenerTransport, deviceListenerPort, deviceListenerTmpFile); args.Add(new SetAppArgumentArgument("-hostname:127.0.0.1", true)); args.Add(new SetEnvVariableArgument(EnviromentVariables.HostName, "127.0.0.1")); args.Add(new SimulatorUDIDArgument(simulator.UDID)); if (appInformation.Extension.HasValue) { switch (appInformation.Extension) { case Extension.TodayExtension: args.Add(new LaunchSimulatorExtensionArgument(appInformation.LaunchAppPath, appInformation.BundleIdentifier)); break; case Extension.WatchKit2: default: throw new NotImplementedException(); } } else { args.Add(new LaunchSimulatorArgument(appInformation.LaunchAppPath)); } return(args); }
protected async Task <ProcessExecutionResult> RunMacCatalystApp( AppBundleInformation appInfo, TimeSpan timeout, IEnumerable <string> appArguments, Dictionary <string, object> environmentVariables, CancellationToken cancellationToken) { using var systemLog = _captureLogFactory.Create( path: _logs.CreateFile("MacCatalyst.system.log", LogType.SystemLog), systemLogPath: SystemLogPath, entireFile: false, LogType.SystemLog); // We need to make the binary executable var binaryPath = Path.Combine(appInfo.AppPath, "Contents", "MacOS", appInfo.BundleExecutable ?? appInfo.AppName); if (File.Exists(binaryPath)) { await _processManager.ExecuteCommandAsync("chmod", new[] { "+x", binaryPath }, _mainLog, TimeSpan.FromSeconds(10), cancellationToken : cancellationToken); } var arguments = new List <string> { "-W", appInfo.LaunchAppPath }; arguments.AddRange(appArguments); var envVars = environmentVariables.ToDictionary( p => p.Key, p => p.Value is bool?p.Value.ToString().ToLowerInvariant() : p.Value.ToString()); // turns "True" to "true" systemLog.StartCapture(); try { return(await _processManager.ExecuteCommandAsync("open", arguments, _mainLog, timeout, envVars, cancellationToken)); } finally { systemLog.StopCapture(waitIfEmpty: TimeSpan.FromSeconds(10)); } }
public AppInstallerTests() { _mainLog = new Mock <ILog>(); _processManager = new Mock <IMlaunchProcessManager>(); _processManager.SetReturnsDefault(Task.FromResult(new ProcessExecutionResult() { ExitCode = 0 })); Directory.CreateDirectory(s_appPath); _appBundleInformation = new AppBundleInformation( appName: "AppName", bundleIdentifier: s_appIdentifier, appPath: s_appPath, launchAppPath: s_appPath, supports32b: false, extension: null); }
public int DetectExitCode(AppBundleInformation appBundleInfo, IReadableLog systemLog) { using var reader = systemLog.GetReader(); while (!reader.EndOfStream) { var line = reader.ReadLine(); if (IsSignalLine(appBundleInfo, line)) { var match = ExitCodeRegex.Match(line); if (match.Success && int.TryParse(match.Captures.First().Value, out var exitCode)) { return(exitCode); } } } return(0); }
public void ExitCodeIsNotDetectedTest() { var appBundleInformation = new AppBundleInformation("HelloiOS", "HelloiOS", "some/path", "some/path", false, null); var detector = new iOSExitCodeDetector(); var log = new[] { "Nov 18 04:31:44 dci-mac-build-053 CloudKeychainProxy[67121]: objc[67121]: Class MockAKSRefKeyObject is implemented in both /Applications/Xcode115.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Security.framework/Security (0x10350b738) and /Applications/Xcode115.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Security.framework/CloudKeychainProxy.bundle/CloudKeychainProxy (0x103259970). One of the two will be used. Which one is undefined.", "Nov 18 04:31:44 dci-mac-build-053 CloudKeychainProxy[67121]: objc[67121]: Class MockAKSOptionalParameters is implemented in both /Applications/Xcode115.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Security.framework/Security (0x10350b788) and /Applications/Xcode115.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Security.framework/CloudKeychainProxy.bundle/CloudKeychainProxy (0x1032599c0). One of the two will be used. Which one is undefined.", "Nov 18 04:31:44 dci-mac-build-053 CloudKeychainProxy[67121]: objc[67121]: Class SecXPCHelper is implemented in both /Applications/Xcode115.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Security.framework/Security (0x10350b918) and /Applications/Xcode115.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Security.framework/CloudKeychainProxy.bundle/CloudKeychainProxy (0x103259a60). One of the two will be used. Which one is undefined.", "Nov 18 04:31:44 ML-MacVM com.apple.CoreSimulator.SimDevice.2E1EE736-5672-4220-89B5-B7C77DB6AF18[55655] (UIKitApplication:net.dot.HelloiOS[9a0b][rb-legacy][57331]): Some other error message", "Nov 18 04:31:44 dci-mac-build-053 com.apple.CoreSimulator.SimDevice.F67392D9-A327-4217-B924-5DA0918415E5[811] (com.apple.security.cloudkeychainproxy3[67121]): Service exited with abnormal code: 1", "Nov 18 04:31:44 dci-mac-build-053 com.apple.CoreSimulator.SimDevice.F67392D9-A327-4217-B924-5DA0918415E5[811] (com.apple.security.cloudkeychainproxy3): Service only ran for 0 seconds. Pushing respawn out by 10 seconds.", }; var exitCode = detector.DetectExitCode(appBundleInformation, GetLogMock(log)); Assert.Equal(0, exitCode); }
public async Task TestOnDeviceWithNoAvailableSimulatorTest() { _hardwareDeviceLoader = new Mock <IHardwareDeviceLoader>(); _hardwareDeviceLoader .Setup(x => x.FindDevice(RunMode.iOS, _mainLog.Object, false, false)) .ThrowsAsync(new NoDeviceFoundException()); _listenerFactory.Setup(f => f.UseTunnel).Returns(false); // Act var appTester = new AppTester(_processManager.Object, _hardwareDeviceLoader.Object, _simulatorLoader.Object, _listenerFactory.Object, _snapshotReporterFactory, Mock.Of <ICaptureLogFactory>(), Mock.Of <IDeviceLogCapturerFactory>(), _testReporterFactory, new XmlResultParser(), _mainLog.Object, _logs.Object, _helpers.Object, Enumerable.Empty <string>()); var appInformation = new AppBundleInformation( appName: AppName, bundleIdentifier: AppBundleIdentifier, appPath: s_appPath, launchAppPath: s_appPath, supports32b: false, extension: null); await Assert.ThrowsAsync <NoDeviceFoundException>( async() => await appTester.TestApp( appInformation, new TestTargetOs(TestTarget.Device_iOS, null), TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(30), ensureCleanSimulatorState: true)); }
private MlaunchArguments GetDeviceArguments( AppBundleInformation appInformation, string deviceName, bool isWatchTarget, int verbosity) { var args = GetCommonArguments(verbosity); args.Add(new DisableMemoryLimitsArgument()); args.Add(new DeviceNameArgument(deviceName)); if (appInformation.Extension.HasValue) { switch (appInformation.Extension) { case Extension.TodayExtension: args.Add(new LaunchDeviceExtensionArgument(appInformation.LaunchAppPath, appInformation.BundleIdentifier)); break; case Extension.WatchKit2: default: throw new NotImplementedException(); } } else { args.Add(new LaunchDeviceArgument(appInformation.LaunchAppPath)); } if (isWatchTarget) { args.Add(new AttachNativeDebuggerArgument()); // this prevents the watch from backgrounding the app. } else { args.Add(new WaitForExitArgument()); } return(args); }
public int DetectExitCode(AppBundleInformation appBundleInfo, IReadableLog systemLog) { using var reader = systemLog.GetReader(); while (!reader.EndOfStream) { string line = reader.ReadLine(); // This will be improved when we find out it differes for new iOS versions if (line.Contains("UIKitApplication:") && line.Contains(appBundleInfo.AppName) && line.Contains("Service exited with abnormal code")) { var regex = new Regex(" (\\-?[0-9]+)$"); var match = regex.Match(line); if (match.Success && int.TryParse(match.Captures.First().Value, out var exitCode)) { return(exitCode); } } } return(0); }
public TestReporterTests() { _crashReporter = new Mock <ICrashSnapshotReporter>(); _processManager = new Mock <IMLaunchProcessManager>(); _parser = new XmlResultParser(); _runLog = new Mock <IReadableLog>(); _mainLog = new Mock <IFileBackedLog>(); _logs = new Mock <ILogs>(); _listener = new Mock <ISimpleListener>(); _appInformation = new AppBundleInformation( appName: "test app", bundleIdentifier: "my.id.com", appPath: "/path/to/app", launchAppPath: "/launch/app/path", supports32b: false, extension: null) { Variation = "Debug" }; _logsDirectory = Path.GetTempFileName(); File.Delete(_logsDirectory); Directory.CreateDirectory(_logsDirectory); }
// Example line // Feb 18 06:40:16 Admins-Mac-Mini com.apple.xpc.launchd[1] (net.dot.System.Buffers.Tests.15140[59229]): Service exited with abnormal code: 74 protected override bool IsSignalLine(AppBundleInformation appBundleInfo, string logLine) => logLine.Contains(appBundleInfo.AppName) && logLine.Contains("Service exited with abnormal code");
protected abstract bool IsSignalLine(AppBundleInformation appBundleInfo, string logLine);
protected override async Task <ExitCode> RunAppInternal( AppBundleInformation appBundleInfo, string?deviceName, ILogger logger, TestTargetOs target, Logs logs, IFileBackedLog mainLog, CancellationToken cancellationToken) { // only add the extra callback if we do know that the feature was indeed enabled Action <string>?logCallback = IsLldbEnabled() ? (l) => NotifyUserLldbCommand(logger, l) : (Action <string>?)null; var appRunner = new AppRunner( ProcessManager, DeviceLoader, SimulatorLoader, new CrashSnapshotReporterFactory(ProcessManager), new CaptureLogFactory(), new DeviceLogCapturerFactory(ProcessManager), mainLog, logs, new Helpers(), PassThroughArguments, logCallback); ProcessExecutionResult result; (deviceName, result) = await appRunner.RunApp( appBundleInfo, target, _arguments.Timeout, deviceName, verbosity : GetMlaunchVerbosity(_arguments.Verbosity), cancellationToken : cancellationToken); if (!result.Succeeded) { if (result.TimedOut) { logger.LogError($"App run has timed out"); return(ExitCode.TIMED_OUT); } logger.LogError($"App run has failed. mlaunch exited with {result.ExitCode}"); return(ExitCode.APP_LAUNCH_FAILURE); } var systemLog = logs.FirstOrDefault(log => log.Description == LogType.SystemLog.ToString()); if (systemLog == null) { logger.LogError("Application has finished but no system log found. Failed to determine the exit code!"); return(ExitCode.RETURN_CODE_NOT_SET); } var exitCode = new ExitCodeDetector().DetectExitCode(appBundleInfo, systemLog); logger.LogInformation($"App run ended with {exitCode}"); if (_arguments.ExpectedExitCode != exitCode) { logger.LogError($"Application has finished with exit code {exitCode} but {_arguments.ExpectedExitCode} was expected"); return(ExitCode.GENERAL_FAILURE); } logger.LogInformation("Application has finished with exit code: " + exitCode + (_arguments.ExpectedExitCode != 0 ? " (as expected)" : null)); return(ExitCode.SUCCESS); }
public async Task<(string DeviceName, TestExecutingResult Result, string ResultMessage)> RunApp( AppBundleInformation appInformation, TestTarget target, TimeSpan timeout, TimeSpan testLaunchTimeout, string? deviceName = null, string? companionDeviceName = null, bool ensureCleanSimulatorState = false, int verbosity = 1, XmlResultJargon xmlResultJargon = XmlResultJargon.xUnit, CancellationToken cancellationToken = default) { var args = new MlaunchArguments { new SetAppArgumentArgument("-connection-mode"), new SetAppArgumentArgument("none"), // This will prevent the app from trying to connect to any IDEs new SetAppArgumentArgument("-autostart", true), new SetEnvVariableArgument(EnviromentVariables.AutoStart, true), new SetAppArgumentArgument("-autoexit", true), new SetEnvVariableArgument(EnviromentVariables.AutoExit, true), new SetAppArgumentArgument("-enablenetwork", true), new SetEnvVariableArgument(EnviromentVariables.EnableNetwork, true), // On macOS we can't edit the TCC database easily // (it requires adding the mac has to be using MDM: https://carlashley.com/2018/09/28/tcc-round-up/) // So by default ignore any tests that would pop up permission dialogs in CI. new SetEnvVariableArgument(EnviromentVariables.DisableSystemPermissionTests, 1), }; for (int i = -1; i < verbosity; i++) { args.Add(new VerbosityArgument()); } var isSimulator = target.IsSimulator(); if (isSimulator) { args.Add(new SetAppArgumentArgument("-hostname:127.0.0.1", true)); args.Add(new SetEnvVariableArgument(EnviromentVariables.HostName, "127.0.0.1")); } else { var ipAddresses = System.Net.Dns.GetHostEntry(System.Net.Dns.GetHostName()).AddressList.Select(ip => ip.ToString()); var ips = string.Join(",", ipAddresses); args.Add(new SetAppArgumentArgument($"-hostname:{ips}", true)); args.Add(new SetEnvVariableArgument(EnviromentVariables.HostName, ips)); } var listenerLog = _logs.Create($"test-{target.AsString()}-{_helpers.Timestamp}.log", LogType.TestLog.ToString(), timestamp: true); var (transport, listener, listenerTmpFile) = _listenerFactory.Create(target.ToRunMode(), log: _mainLog, testLog: listenerLog, isSimulator: isSimulator, autoExit: true, xmlOutput: true); // cli always uses xml // Initialize has to be called before we try to get Port (internal implementation of the listener says so) // TODO: Improve this to not get into a broken state - it was really hard to debug when I moved this lower listener.Initialize(); args.Add(new SetAppArgumentArgument($"-transport:{transport}", true)); args.Add(new SetEnvVariableArgument(EnviromentVariables.Transport, transport.ToString().ToUpper())); if (transport == ListenerTransport.File) { args.Add(new SetEnvVariableArgument(EnviromentVariables.LogFilePath, listenerTmpFile)); } args.Add(new SetAppArgumentArgument($"-hostport:{listener.Port}", true)); args.Add(new SetEnvVariableArgument(EnviromentVariables.HostPort, listener.Port)); if (_listenerFactory.UseTunnel && !isSimulator) // simulators do not support tunnels { args.Add(new SetEnvVariableArgument(EnviromentVariables.UseTcpTunnel, true)); } if (_useXmlOutput) { // let the runner now via envars that we want to get a xml output, else the runner will default to plain text args.Add (new SetEnvVariableArgument (EnviromentVariables.EnableXmlOutput, true)); args.Add (new SetEnvVariableArgument (EnviromentVariables.XmlMode, "wrapped")); args.Add (new SetEnvVariableArgument (EnviromentVariables.XmlVersion, $"{xmlResultJargon}")); } listener.StartAsync(); var crashLogs = new Logs(_logs.Directory); if (appInformation.Extension.HasValue) { switch (appInformation.Extension) { case Extension.TodayExtension: args.Add(isSimulator ? (MlaunchArgument)new LaunchSimulatorExtensionArgument(appInformation.LaunchAppPath, appInformation.BundleIdentifier) : new LaunchDeviceExtensionArgument(appInformation.LaunchAppPath, appInformation.BundleIdentifier)); break; case Extension.WatchKit2: default: throw new NotImplementedException(); } } else { args.Add(isSimulator ? (MlaunchArgument)new LaunchSimulatorArgument(appInformation.LaunchAppPath) : new LaunchDeviceArgument(appInformation.LaunchAppPath)); } var runMode = target.ToRunMode(); ICrashSnapshotReporter crashReporter; ITestReporter testReporter; if (isSimulator) { crashReporter = _snapshotReporterFactory.Create(_mainLog, crashLogs, isDevice: !isSimulator, deviceName: null!); testReporter = _testReporterFactory.Create(_mainLog, _mainLog, _logs, crashReporter, listener, new XmlResultParser(), appInformation, runMode, xmlResultJargon, device: null, timeout, null, (level, message) => _mainLog.WriteLine(message)); using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(testReporter.CancellationToken, cancellationToken); listener.ConnectedTask .TimeoutAfter(testLaunchTimeout) .ContinueWith(testReporter.LaunchCallback) .DoNotAwait(); await _simulatorLoader.LoadDevices(_logs.Create($"simulator-list-{_helpers.Timestamp}.log", "Simulator list"), false, false); var simulators = await _simulatorLoader.FindSimulators(target, _mainLog); if (!(simulators?.Any() ?? false)) { _mainLog.WriteLine("Didn't find any suitable simulators"); throw new NoDeviceFoundException(); } var simulator = string.IsNullOrEmpty(deviceName) ? simulators.FirstOrDefault() : simulators.FirstOrDefault(s => string.Equals(s.Name, deviceName, StringComparison.InvariantCultureIgnoreCase)); if (simulator == null) { throw new NoDeviceFoundException(); } deviceName = simulator.Name; if (!target.IsWatchOSTarget()) { var stderrTty = _helpers.GetTerminalName(2); if (!string.IsNullOrEmpty(stderrTty)) { args.Add(new SetStderrArgument(stderrTty)); } else { var stdoutLog = _logs.CreateFile($"mlaunch-stdout-{_helpers.Timestamp}.log", "Standard output"); var stderrLog = _logs.CreateFile($"mlaunch-stderr-{_helpers.Timestamp}.log", "Standard error"); args.Add(new SetStdoutArgument(stdoutLog)); args.Add(new SetStderrArgument(stderrLog)); } } var systemLogs = new List<ICaptureLog>(); foreach (var sim in simulators) { // Upload the system log _mainLog.WriteLine("System log for the '{1}' simulator is: {0}", sim.SystemLog, sim.Name); bool isCompanion = sim != simulator; var logDescription = isCompanion ? LogType.CompanionSystemLog.ToString() : LogType.SystemLog.ToString(); var log = _captureLogFactory.Create( Path.Combine(_logs.Directory, sim.Name + ".log"), sim.SystemLog, true, logDescription); log.StartCapture(); _logs.Add(log); systemLogs.Add(log); } _mainLog.WriteLine("*** Executing {0}/{1} in the simulator ***", appInformation.AppName, target); if (ensureCleanSimulatorState) { foreach (var sim in simulators) { await sim.PrepareSimulator(_mainLog, appInformation.BundleIdentifier); } } args.Add(new SimulatorUDIDArgument(simulator.UDID)); await crashReporter.StartCaptureAsync(); _mainLog.WriteLine("Starting test run"); var result = _processManager.ExecuteCommandAsync(args, _mainLog, timeout, cancellationToken: linkedCts.Token); await testReporter.CollectSimulatorResult(result); // cleanup after us if (ensureCleanSimulatorState) { await simulator.KillEverything(_mainLog); } foreach (var log in systemLogs) { log.StopCapture(); } } else { args.Add(new DisableMemoryLimitsArgument()); if (deviceName == null) { IHardwareDevice? companionDevice = null; IHardwareDevice device = await _hardwareDeviceLoader.FindDevice(runMode, _mainLog, includeLocked: false, force: false); if (target.IsWatchOSTarget()) { companionDevice = await _hardwareDeviceLoader.FindCompanionDevice(_mainLog, device); } deviceName = companionDevice?.Name ?? device.Name; } if (deviceName == null) { throw new NoDeviceFoundException(); } crashReporter = _snapshotReporterFactory.Create(_mainLog, crashLogs, isDevice: !isSimulator, deviceName); testReporter = _testReporterFactory.Create(_mainLog, _mainLog, _logs, crashReporter, listener, new XmlResultParser(), appInformation, runMode, xmlResultJargon, deviceName, timeout, null, (level, message) => _mainLog.WriteLine(message)); using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(testReporter.CancellationToken, cancellationToken); listener.ConnectedTask .TimeoutAfter(testLaunchTimeout) .ContinueWith(testReporter.LaunchCallback) .DoNotAwait(); _mainLog.WriteLine("*** Executing {0}/{1} on device '{2}' ***", appInformation.AppName, target, deviceName); if (target.IsWatchOSTarget()) { args.Add(new AttachNativeDebuggerArgument()); // this prevents the watch from backgrounding the app. } else { args.Add(new WaitForExitArgument()); } args.Add(new DeviceNameArgument(deviceName)); var deviceSystemLog = _logs.Create($"device-{deviceName}-{_helpers.Timestamp}.log", "Device log"); var deviceLogCapturer = _deviceLogCapturerFactory.Create(_mainLog, deviceSystemLog, deviceName); deviceLogCapturer.StartCapture(); try { await crashReporter.StartCaptureAsync(); // create a tunnel to communicate with the device if (transport == ListenerTransport.Tcp && _listenerFactory.UseTunnel && listener is SimpleTcpListener tcpListener) { // create a new tunnel using the listener var tunnel = _listenerFactory.TunnelBore.Create(deviceName, _mainLog); tunnel.Open(deviceName, tcpListener, timeout, _mainLog); // wait until we started the tunnel await tunnel.Started; } _mainLog.WriteLine("Starting test run"); // We need to check for MT1111 (which means that mlaunch won't wait for the app to exit). var aggregatedLog = Log.CreateAggregatedLog(testReporter.CallbackLog, _mainLog); Task<ProcessExecutionResult> runTestTask = _processManager.ExecuteCommandAsync( args, aggregatedLog, timeout, cancellationToken: linkedCts.Token); await testReporter.CollectDeviceResult(runTestTask); } finally { deviceLogCapturer.StopCapture(); deviceSystemLog.Dispose(); // close a tunnel if it was created if (!isSimulator && _listenerFactory.UseTunnel) await _listenerFactory.TunnelBore.Close(deviceName); } // Upload the system log if (File.Exists(deviceSystemLog.FullPath)) { _mainLog.WriteLine("A capture of the device log is: {0}", deviceSystemLog.FullPath); } } listener.Cancel(); listener.Dispose(); // check the final status, copy all the required data var (testResult, resultMessage) = await testReporter.ParseResult(); return (deviceName, testResult, resultMessage); }