public ITestReporter Create(IFileBackedLog mainLog, IReadableLog runLog, ILogs logs, ICrashSnapshotReporter crashReporter, ISimpleListener simpleListener, IResultParser parser, AppBundleInformation appInformation, RunMode runMode, XmlResultJargon xmlJargon, string?device, TimeSpan timeout, string?additionalLogsDirectory = null, ExceptionLogger?exceptionLogger = null, bool generateHtml = false) => new TestReporter(_processManager, mainLog, runLog, logs, crashReporter, simpleListener, parser, appInformation, runMode, xmlJargon, device, timeout, additionalLogsDirectory, exceptionLogger, generateHtml);
public ITestReporter Create(ILog mainLog, ILog runLog, ILogs logs, ICrashSnapshotReporter crashReporter, ISimpleListener simpleListener, IResultParser parser, AppBundleInformation appInformation, RunMode runMode, XmlResultJargon xmlJargon, string device, TimeSpan timeout, string additionalLogsDirectory = null, ExceptionLogger exceptionLogger = null) { return(new TestReporter(processManager, mainLog, runLog, logs, crashReporter, simpleListener, parser, appInformation, runMode, xmlJargon, device, timeout, additionalLogsDirectory, exceptionLogger)); }
private async Task <ProcessExecutionResult> RunDeviceApp( MlaunchArguments mlaunchArguments, ICrashSnapshotReporter crashReporter, string deviceName, TimeSpan timeout, CancellationToken cancellationToken) { var deviceSystemLog = _logs.Create($"device-{deviceName}-{_helpers.Timestamp}.log", LogType.SystemLog.ToString()); var deviceLogCapturer = _deviceLogCapturerFactory.Create(_mainLog, deviceSystemLog, deviceName); deviceLogCapturer.StartCapture(); try { await crashReporter.StartCaptureAsync(); _mainLog.WriteLine("Starting the app"); return(await _processManager.ExecuteCommandAsync( mlaunchArguments, _mainLog, timeout, cancellationToken : cancellationToken)); } finally { deviceLogCapturer.StopCapture(); deviceSystemLog.Dispose(); } }
private async Task RunDeviceTests( MlaunchArguments mlaunchArguments, ICrashSnapshotReporter crashReporter, ITestReporter testReporter, ISimpleListener deviceListener, string deviceName, TimeSpan timeout, CancellationToken cancellationToken) { var deviceSystemLog = _logs.Create($"device-{deviceName}-{_helpers.Timestamp}.log", "Device log"); var deviceLogCapturer = _deviceLogCapturerFactory.Create(_mainLog, deviceSystemLog, deviceName); deviceLogCapturer.StartCapture(); try { await crashReporter.StartCaptureAsync(); // create a tunnel to communicate with the device if (_listenerFactory.UseTunnel && deviceListener is SimpleTcpListener tcpListener) { // create a new tunnel using the listener var tunnel = _listenerFactory.TunnelBore.Create(deviceName, _mainLog); tunnel.Open(deviceName, tcpListener, timeout, _mainLog); // wait until we started the tunnel await tunnel.Started; } _mainLog.WriteLine("Starting test run"); // We need to check for MT1111 (which means that mlaunch won't wait for the app to exit). var aggregatedLog = Log.CreateReadableAggregatedLog(_mainLog, testReporter.CallbackLog); var result = _processManager.ExecuteCommandAsync( mlaunchArguments, aggregatedLog, timeout, cancellationToken: cancellationToken); await testReporter.CollectDeviceResult(result); } finally { deviceLogCapturer.StopCapture(); deviceSystemLog.Dispose(); // close a tunnel if it was created if (_listenerFactory.UseTunnel) { await _listenerFactory.TunnelBore.Close(deviceName); } } // Upload the system log if (File.Exists(deviceSystemLog.FullPath)) { _mainLog.WriteLine("A capture of the device log is: {0}", deviceSystemLog.FullPath); } }
public TestReporter(IMlaunchProcessManager processManager, IFileBackedLog mainLog, IReadableLog runLog, ILogs logs, ICrashSnapshotReporter crashReporter, ISimpleListener simpleListener, IResultParser parser, AppBundleInformation appInformation, RunMode runMode, XmlResultJargon xmlJargon, string?device, TimeSpan timeout, string?additionalLogsDirectory = null, ExceptionLogger?exceptionLogger = null, bool generateHtml = false) { _processManager = processManager ?? throw new ArgumentNullException(nameof(processManager)); _deviceName = device; // can be null on simulators _listener = simpleListener ?? throw new ArgumentNullException(nameof(simpleListener)); _mainLog = mainLog ?? throw new ArgumentNullException(nameof(mainLog)); _runLog = runLog ?? throw new ArgumentNullException(nameof(runLog)); _logs = logs ?? throw new ArgumentNullException(nameof(logs)); _crashReporter = crashReporter ?? throw new ArgumentNullException(nameof(crashReporter)); _crashLogs = new Logs(logs.Directory); _resultParser = parser ?? throw new ArgumentNullException(nameof(parser)); _appInfo = appInformation ?? throw new ArgumentNullException(nameof(appInformation)); _runMode = runMode; _xmlJargon = xmlJargon; _timeout = timeout; _additionalLogsDirectory = additionalLogsDirectory; _exceptionLogger = exceptionLogger; _timeoutWatch = Stopwatch.StartNew(); _generateHtml = generateHtml; CallbackLog = new CallbackLog(line => { // MT1111: Application launched successfully, but it's not possible to wait for the app to exit as // requested because it's not possible to detect app termination when launching using gdbserver _waitedForExit &= line?.Contains("MT1111: ") != true; if (line?.Contains("error MT1007") == true) { _launchFailure = true; } }); }
public TestReporter(IProcessManager processManager, ILog mainLog, ILog runLog, ILogs logs, ICrashSnapshotReporter crashReporter, ISimpleListener simpleListener, IResultParser parser, AppBundleInformation appInformation, RunMode runMode, XmlResultJargon xmlJargon, string device, TimeSpan timeout, double launchTimeout, string additionalLogsDirectory = null, ExceptionLogger exceptionLogger = null) { this.processManager = processManager ?? throw new ArgumentNullException(nameof(processManager)); this.deviceName = device; // can be null on simulators this.listener = simpleListener ?? throw new ArgumentNullException(nameof(simpleListener)); this.mainLog = mainLog ?? throw new ArgumentNullException(nameof(mainLog)); this.runLog = runLog ?? throw new ArgumentNullException(nameof(runLog)); this.logs = logs ?? throw new ArgumentNullException(nameof(logs)); this.crashReporter = crashReporter ?? throw new ArgumentNullException(nameof(crashReporter)); this.crashLogs = new Logs(logs.Directory); this.resultParser = parser ?? throw new ArgumentNullException(nameof(parser)); this.appInfo = appInformation ?? throw new ArgumentNullException(nameof(appInformation)); this.runMode = runMode; this.xmlJargon = xmlJargon; this.timeout = timeout; this.launchTimeout = launchTimeout; this.additionalLogsDirectory = additionalLogsDirectory; this.exceptionLogger = exceptionLogger; this.timeoutWatch = Stopwatch.StartNew(); CallbackLog = new CallbackLog((line) => { // MT1111: Application launched successfully, but it's not possible to wait for the app to exit as requested because it's not possible to detect app termination when launching using gdbserver waitedForExit &= line?.Contains("MT1111: ") != true; if (line?.Contains("error MT1007") == true) { launchFailure = true; } }); }
private async Task RunDeviceApp( MlaunchArguments mlaunchArguments, ICrashSnapshotReporter crashReporter, string deviceName, TimeSpan timeout, CancellationToken cancellationToken) { var deviceSystemLog = _logs.Create($"device-{deviceName}-{_helpers.Timestamp}.log", LogType.SystemLog.ToString()); var deviceLogCapturer = _deviceLogCapturerFactory.Create(_mainLog, deviceSystemLog, deviceName); deviceLogCapturer.StartCapture(); try { await crashReporter.StartCaptureAsync(); _mainLog.WriteLine("Starting test run"); await _processManager.ExecuteCommandAsync( mlaunchArguments, _mainLog, timeout, cancellationToken : cancellationToken); } finally { deviceLogCapturer.StopCapture(); deviceSystemLog.Dispose(); } // Upload the system log if (File.Exists(deviceSystemLog.FullPath)) { _mainLog.WriteLine("A capture of the device log is: {0}", deviceSystemLog.FullPath); } }
public override async Task RunTestAsync() { var projectDir = System.IO.Path.GetDirectoryName(ProjectFile); var name = System.IO.Path.GetFileName(projectDir); if (string.Equals("mac", name, StringComparison.OrdinalIgnoreCase)) { name = System.IO.Path.GetFileName(System.IO.Path.GetDirectoryName(projectDir)); } var suffix = string.Empty; switch (Platform) { case TestPlatform.Mac_Modern: suffix = "-modern"; break; case TestPlatform.Mac_Full: suffix = "-full"; break; case TestPlatform.Mac_System: suffix = "-system"; break; } if (ProjectFile.EndsWith(".sln", StringComparison.Ordinal)) { Path = System.IO.Path.Combine(System.IO.Path.GetDirectoryName(ProjectFile), "bin", BuildTask.ProjectPlatform, BuildTask.ProjectConfiguration + suffix, name + ".app", "Contents", "MacOS", name); } else { var project = new XmlDocument(); project.LoadWithoutNetworkAccess(ProjectFile); string outputPath; if (TestProject?.IsDotNetProject == true) { outputPath = await Harness.AppBundleLocator.LocateAppBundle(project, ProjectFile, TestTarget.None, BuildTask.ProjectConfiguration); } else { outputPath = project.GetOutputPath(BuildTask.ProjectPlatform, BuildTask.ProjectConfiguration).Replace('\\', '/'); } var assemblyName = project.GetAssemblyName(); Path = System.IO.Path.Combine(System.IO.Path.GetDirectoryName(ProjectFile), outputPath, assemblyName + ".app", "Contents", "MacOS", assemblyName); } using (var resource = await NotifyAndAcquireDesktopResourceAsync()) { using (var proc = new Process()) { proc.StartInfo.FileName = Path; var arguments = new List <string> (); IFileBackedLog xmlLog = null; var useXmlOutput = Harness.InCI || true; if (IsUnitTest) { var extension = useXmlOutput ? "xml" : "log"; var type = useXmlOutput ? LogType.XmlLog : LogType.NUnitResult; xmlLog = Logs.Create($"test-{Platform}-{Timestamp}.{extension}", type.ToString()); arguments.Add($"-transport:FILE"); proc.StartInfo.EnvironmentVariables ["NUNIT_TRANSPORT"] = "FILE"; arguments.Add($"--logfile:{xmlLog.FullPath}"); proc.StartInfo.EnvironmentVariables ["NUNIT_LOG_FILE"] = xmlLog.FullPath; if (useXmlOutput) { arguments.Add("--enablexml"); proc.StartInfo.EnvironmentVariables ["NUNIT_ENABLE_XML_OUTPUT"] = "true"; arguments.Add("--xmlmode=wrapped"); proc.StartInfo.EnvironmentVariables ["NUNIT_ENABLE_XML_MODE"] = "wrapped"; arguments.Add("--xmlversion=nunitv3"); proc.StartInfo.EnvironmentVariables ["NUNIT_XML_VERSION"] = "nunitv3"; } arguments.Add("--autostart"); proc.StartInfo.EnvironmentVariables ["NUNIT_AUTOSTART"] = "true"; arguments.Add("--autoexit"); proc.StartInfo.EnvironmentVariables ["NUNIT_AUTOEXIT"] = "true"; } if (!Harness.GetIncludeSystemPermissionTests(Platform, false)) { proc.StartInfo.EnvironmentVariables ["DISABLE_SYSTEM_PERMISSION_TESTS"] = "1"; } proc.StartInfo.EnvironmentVariables ["MONO_DEBUG"] = "no-gdb-backtrace"; proc.StartInfo.EnvironmentVariables.Remove("DYLD_FALLBACK_LIBRARY_PATH"); // VSMac might set this, and the test may end up crashing proc.StartInfo.Arguments = StringUtils.FormatArguments(arguments); Jenkins.MainLog.WriteLine("Executing {0} ({1})", TestName, Mode); var log = Logs.Create($"execute-{Platform}-{Timestamp}.txt", LogType.ExecutionLog.ToString()); ICrashSnapshotReporter snapshot = null; if (!Jenkins.Harness.DryRun) { ExecutionResult = TestExecutingResult.Running; snapshot = CrashReportSnapshotFactory.Create(log, Logs, isDevice: false, deviceName: null); await snapshot.StartCaptureAsync(); ProcessExecutionResult result = null; try { var timeout = TimeSpan.FromMinutes(20); result = await ProcessManager.RunAsync(proc, log, timeout); if (result.TimedOut) { FailureMessage = $"Execution timed out after {timeout.TotalSeconds} seconds."; log.WriteLine(FailureMessage); ExecutionResult = TestExecutingResult.TimedOut; } else if (result.Succeeded) { ExecutionResult = TestExecutingResult.Succeeded; } else { ExecutionResult = TestExecutingResult.Failed; FailureMessage = result.ExitCode != 1 ? $"Test run crashed (exit code: {result.ExitCode})." : "Test run failed."; log.WriteLine(FailureMessage); } } finally { await snapshot.EndCaptureAsync(TimeSpan.FromSeconds(Succeeded ? 0 : result?.ExitCode > 1 ? 120 : 5)); } } Jenkins.MainLog.WriteLine("Executed {0} ({1})", TestName, Mode); if (IsUnitTest) { var reporterFactory = new TestReporterFactory(ProcessManager); var listener = new Microsoft.DotNet.XHarness.iOS.Shared.Listeners.SimpleFileListener(xmlLog.FullPath, log, xmlLog, useXmlOutput); var reporter = reporterFactory.Create(Harness.HarnessLog, log, Logs, snapshot, listener, Harness.ResultParser, new AppBundleInformation("N/A", "N/A", "N/A", "N/A", true, null), RunMode.MacOS, Harness.XmlJargon, "no device here", TimeSpan.Zero); var rv = await reporter.ParseResult(); if (ExecutionResult == TestExecutingResult.Succeeded) { // The process might have crashed, timed out at exit, or otherwise returned a non-zero exit code when all the unit tests passed, in which we shouldn't override the execution result here, ExecutionResult = rv.ExecutingResult; } // Set or replace the failure message, depending on whether there already is a failure message or not. if (string.IsNullOrEmpty(FailureMessage)) { FailureMessage = rv.ResultMessage; } else if (!string.IsNullOrEmpty(rv.ResultMessage)) { FailureMessage += "\n" + rv.ResultMessage; } } } } }
public override async Task RunTestAsync() { var projectDir = System.IO.Path.GetDirectoryName(ProjectFile); var name = System.IO.Path.GetFileName(projectDir); if (string.Equals("mac", name, StringComparison.OrdinalIgnoreCase)) { name = System.IO.Path.GetFileName(System.IO.Path.GetDirectoryName(projectDir)); } var suffix = string.Empty; switch (Platform) { case TestPlatform.Mac_Modern: suffix = "-modern"; break; case TestPlatform.Mac_Full: suffix = "-full"; break; case TestPlatform.Mac_System: suffix = "-system"; break; } if (ProjectFile.EndsWith(".sln", StringComparison.Ordinal)) { Path = System.IO.Path.Combine(System.IO.Path.GetDirectoryName(ProjectFile), "bin", BuildTask.ProjectPlatform, BuildTask.ProjectConfiguration + suffix, name + ".app", "Contents", "MacOS", name); } else { var project = new XmlDocument(); project.LoadWithoutNetworkAccess(ProjectFile); var outputPath = project.GetOutputPath(BuildTask.ProjectPlatform, BuildTask.ProjectConfiguration).Replace('\\', '/'); var assemblyName = project.GetAssemblyName(); Path = System.IO.Path.Combine(System.IO.Path.GetDirectoryName(ProjectFile), outputPath, assemblyName + ".app", "Contents", "MacOS", assemblyName); } using (var resource = await NotifyAndAcquireDesktopResourceAsync()) { using (var proc = new Process()) { proc.StartInfo.FileName = Path; var arguments = new List <string> (); IFileBackedLog xmlLog = null; var useXmlOutput = Harness.InCI || true; if (IsUnitTest) { var extension = useXmlOutput ? "xml" : "log"; var type = useXmlOutput ? LogType.XmlLog : LogType.NUnitResult; xmlLog = Logs.Create($"test-{Platform}-{Timestamp}.{extension}", type.ToString()); arguments.Add($"-transport:FILE"); arguments.Add($"--logfile:{xmlLog.FullPath}"); if (useXmlOutput) { arguments.Add("--enablexml"); arguments.Add("--xmlmode=wrapped"); arguments.Add("--xmlversion=nunitv3"); } } if (!Harness.GetIncludeSystemPermissionTests(Platform, false)) { proc.StartInfo.EnvironmentVariables ["DISABLE_SYSTEM_PERMISSION_TESTS"] = "1"; } proc.StartInfo.EnvironmentVariables ["MONO_DEBUG"] = "no-gdb-backtrace"; proc.StartInfo.Arguments = StringUtils.FormatArguments(arguments); Jenkins.MainLog.WriteLine("Executing {0} ({1})", TestName, Mode); var log = Logs.Create($"execute-{Platform}-{Timestamp}.txt", LogType.ExecutionLog.ToString()); ICrashSnapshotReporter snapshot = null; if (!Jenkins.Harness.DryRun) { ExecutionResult = TestExecutingResult.Running; snapshot = CrashReportSnapshotFactory.Create(log, Logs, isDevice: false, deviceName: null); await snapshot.StartCaptureAsync(); ProcessExecutionResult result = null; try { var timeout = TimeSpan.FromMinutes(20); result = await ProcessManager.RunAsync(proc, log, timeout); if (result.TimedOut) { FailureMessage = $"Execution timed out after {timeout.TotalSeconds} seconds."; log.WriteLine(FailureMessage); ExecutionResult = TestExecutingResult.TimedOut; } else if (result.Succeeded) { ExecutionResult = TestExecutingResult.Succeeded; } else { ExecutionResult = TestExecutingResult.Failed; FailureMessage = result.ExitCode != 1 ? $"Test run crashed (exit code: {result.ExitCode})." : "Test run failed."; log.WriteLine(FailureMessage); } } finally { await snapshot.EndCaptureAsync(TimeSpan.FromSeconds(Succeeded ? 0 : result?.ExitCode > 1 ? 120 : 5)); } } Jenkins.MainLog.WriteLine("Executed {0} ({1})", TestName, Mode); if (IsUnitTest) { var reporterFactory = new TestReporterFactory(ProcessManager); var listener = new Microsoft.DotNet.XHarness.iOS.Shared.Listeners.SimpleFileListener(xmlLog.FullPath, log, xmlLog, useXmlOutput); var reporter = reporterFactory.Create(Harness.HarnessLog, log, Logs, snapshot, listener, Harness.ResultParser, new AppBundleInformation("N/A", "N/A", "N/A", "N/A", true, null), RunMode.MacOS, Harness.XmlJargon, "no device here", TimeSpan.Zero); var rv = await reporter.ParseResult(); ExecutionResult = rv.ExecutingResult; FailureMessage = rv.ExecutingResult == TestExecutingResult.Succeeded ? null : rv.ResultMessage; } } } }
public async Task <(string DeviceName, TestExecutingResult Result, string ResultMessage)> RunApp( AppBundleInformation appInformation, TestTargetOs target, TimeSpan timeout, TimeSpan testLaunchTimeout, string?deviceName = null, string?companionDeviceName = null, bool ensureCleanSimulatorState = false, int verbosity = 1, XmlResultJargon xmlResultJargon = XmlResultJargon.xUnit, string[]?skippedMethods = null, string[]?skippedTestClasses = null, CancellationToken cancellationToken = default) { var runMode = target.Platform.ToRunMode(); bool isSimulator = target.Platform.IsSimulator(); var deviceListenerLog = _logs.Create($"test-{target.AsString()}-{_helpers.Timestamp}.log", LogType.TestLog.ToString(), timestamp: true); var(deviceListenerTransport, deviceListener, deviceListenerTmpFile) = _listenerFactory.Create( runMode, log: _mainLog, testLog: deviceListenerLog, isSimulator: isSimulator, autoExit: true, xmlOutput: true); // cli always uses xml ISimulatorDevice?simulator = null; ISimulatorDevice?companionSimulator = null; // Find devices if (isSimulator) { int attempt = 1; const int maxAttempts = 3; while (true) { try { (simulator, companionSimulator) = await _simulatorLoader.FindSimulators(target, _mainLog); break; } catch (Exception e) { _mainLog.WriteLine($"Failed to find/create simulator (attempt {attempt}/{maxAttempts}):" + Environment.NewLine + e); if (attempt == maxAttempts) { throw new NoDeviceFoundException("Failed to find/create suitable simulator"); } } finally { attempt++; } } deviceName = companionSimulator?.Name ?? simulator.Name; } else { deviceName ??= await FindDevice(target) ?? throw new NoDeviceFoundException(); } int deviceListenerPort = deviceListener.InitializeAndGetPort(); deviceListener.StartAsync(); var crashLogs = new Logs(_logs.Directory); ICrashSnapshotReporter crashReporter = _snapshotReporterFactory.Create(_mainLog, crashLogs, isDevice: !isSimulator, deviceName); ITestReporter testReporter = _testReporterFactory.Create(_mainLog, _mainLog, _logs, crashReporter, deviceListener, _resultParser, appInformation, runMode, xmlResultJargon, deviceName, timeout, null, (level, message) => _mainLog.WriteLine(message)); deviceListener.ConnectedTask .TimeoutAfter(testLaunchTimeout) .ContinueWith(testReporter.LaunchCallback) .DoNotAwait(); _mainLog.WriteLine($"*** Executing '{appInformation.AppName}' on {target} '{deviceName}' ***"); try { using var combinedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(testReporter.CancellationToken, cancellationToken); if (isSimulator) { if (simulator == null) { _mainLog.WriteLine("Didn't find any suitable simulator"); throw new NoDeviceFoundException(); } var mlaunchArguments = GetSimulatorArguments( appInformation, simulator, verbosity, xmlResultJargon, skippedMethods, skippedTestClasses, deviceListenerTransport, deviceListenerPort, deviceListenerTmpFile); await RunSimulatorTests( mlaunchArguments, appInformation, crashReporter, testReporter, simulator, companionSimulator, ensureCleanSimulatorState, timeout, combinedCancellationToken.Token); } else { var mlaunchArguments = GetDeviceArguments( appInformation, deviceName, target.Platform.IsWatchOSTarget(), verbosity, xmlResultJargon, skippedMethods, skippedTestClasses, deviceListenerTransport, deviceListenerPort, deviceListenerTmpFile); await RunDeviceTests( mlaunchArguments, crashReporter, testReporter, deviceListener, deviceName, timeout, combinedCancellationToken.Token); } } finally { deviceListener.Cancel(); deviceListener.Dispose(); } // Check the final status, copy all the required data var(testResult, resultMessage) = await testReporter.ParseResult(); return(deviceName, testResult, resultMessage); }
private async Task RunSimulatorTests( MlaunchArguments mlaunchArguments, AppBundleInformation appInformation, ICrashSnapshotReporter crashReporter, ITestReporter testReporter, ISimulatorDevice simulator, ISimulatorDevice?companionSimulator, bool ensureCleanSimulatorState, TimeSpan timeout, CancellationToken cancellationToken) { var systemLogs = new List <ICaptureLog>(); try { _mainLog.WriteLine("System log for the '{1}' simulator is: {0}", simulator.SystemLog, simulator.Name); var simulatorLog = _captureLogFactory.Create( path: Path.Combine(_logs.Directory, simulator.Name + ".log"), systemLogPath: simulator.SystemLog, entireFile: true, LogType.SystemLog.ToString()); simulatorLog.StartCapture(); _logs.Add(simulatorLog); systemLogs.Add(simulatorLog); if (companionSimulator != null) { _mainLog.WriteLine("System log for the '{1}' companion simulator is: {0}", companionSimulator.SystemLog, companionSimulator.Name); var companionLog = _captureLogFactory.Create( path: Path.Combine(_logs.Directory, companionSimulator.Name + ".log"), systemLogPath: companionSimulator.SystemLog, entireFile: true, LogType.CompanionSystemLog.ToString()); companionLog.StartCapture(); _logs.Add(companionLog); systemLogs.Add(companionLog); } if (ensureCleanSimulatorState) { await simulator.PrepareSimulator(_mainLog, appInformation.BundleIdentifier); if (companionSimulator != null) { await companionSimulator.PrepareSimulator(_mainLog, appInformation.BundleIdentifier); } } await crashReporter.StartCaptureAsync(); _mainLog.WriteLine("Starting test run"); var result = _processManager.ExecuteCommandAsync(mlaunchArguments, _mainLog, timeout, cancellationToken: cancellationToken); await testReporter.CollectSimulatorResult(result); // cleanup after us if (ensureCleanSimulatorState) { await simulator.KillEverything(_mainLog); if (companionSimulator != null) { await companionSimulator.KillEverything(_mainLog); } } } finally { foreach (ICaptureLog?log in systemLogs) { log.StopCapture(); log.Dispose(); } } }
public async Task <(string DeviceName, int?exitCode)> RunApp( AppBundleInformation appInformation, TestTargetOs target, TimeSpan timeout, string?deviceName = null, string?companionDeviceName = null, bool ensureCleanSimulatorState = false, int verbosity = 1, CancellationToken cancellationToken = default) { bool isSimulator = target.Platform.IsSimulator(); ISimulatorDevice?simulator = null; ISimulatorDevice?companionSimulator = null; // Find devices if (isSimulator) { (simulator, companionSimulator) = await _simulatorLoader.FindSimulators(target, _mainLog, 3); deviceName = companionSimulator?.Name ?? simulator.Name; } else { deviceName ??= await FindDevice(target) ?? throw new NoDeviceFoundException(); } var crashLogs = new Logs(_logs.Directory); ICrashSnapshotReporter crashReporter = _snapshotReporterFactory.Create(_mainLog, crashLogs, isDevice: !isSimulator, deviceName); _mainLog.WriteLine($"*** Executing '{appInformation.AppName}' on {target.AsString()} '{deviceName}' ***"); if (isSimulator) { if (simulator == null) { _mainLog.WriteLine("Didn't find any suitable simulator"); throw new NoDeviceFoundException(); } var mlaunchArguments = GetSimulatorArguments(appInformation, simulator, verbosity); await RunSimulatorApp( mlaunchArguments, appInformation, crashReporter, simulator, companionSimulator, ensureCleanSimulatorState, timeout, cancellationToken); } else { var mlaunchArguments = GetDeviceArguments(appInformation, deviceName, target.Platform.IsWatchOSTarget(), verbosity); await RunDeviceApp( mlaunchArguments, crashReporter, deviceName, timeout, cancellationToken); } var systemLog = _logs.FirstOrDefault(log => log.Description == LogType.SystemLog.ToString()); if (systemLog == null) { _mainLog.WriteLine("App run ended but failed to detect exit code (no system log found)"); return(deviceName, null); } var exitCode = _exitCodeDetector.DetectExitCode(appInformation, systemLog); _mainLog.WriteLine($"App run ended with {exitCode}"); return(deviceName, exitCode); }
/// <summary> /// Runs the MacCatalyst app by executing its binary (or if not found, via `open -W path.to.app`). /// </summary> private async Task <(TestExecutingResult Result, string ResultMessage)> RunMacCatalystTests( ListenerTransport deviceListenerTransport, ISimpleListener deviceListener, string deviceListenerTmpFile, AppBundleInformation appInformation, TimeSpan timeout, TimeSpan testLaunchTimeout, XmlResultJargon xmlResultJargon, string[]?skippedMethods, string[]?skippedTestClasses, CancellationToken cancellationToken) { var deviceListenerPort = deviceListener.InitializeAndGetPort(); deviceListener.StartAsync(); var crashLogs = new Logs(_logs.Directory); ICrashSnapshotReporter crashReporter = _snapshotReporterFactory.Create(_mainLog, crashLogs, isDevice: false, null); ITestReporter testReporter = _testReporterFactory.Create( _mainLog, _mainLog, _logs, crashReporter, deviceListener, _resultParser, appInformation, RunMode.MacOS, xmlResultJargon, null, timeout, null, (level, message) => _mainLog.WriteLine(message)); deviceListener.ConnectedTask .TimeoutAfter(testLaunchTimeout) .ContinueWith(testReporter.LaunchCallback) .DoNotAwait(); _mainLog.WriteLine($"*** Executing '{appInformation.AppName}' on MacCatalyst ***"); try { using var combinedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(testReporter.CancellationToken, cancellationToken); var envVariables = GetEnvVariables( xmlResultJargon, skippedMethods, skippedTestClasses, deviceListenerTransport, deviceListenerPort, deviceListenerTmpFile); envVariables[EnviromentVariables.HostName] = "127.0.0.1"; var arguments = new List <string> { "-W", appInformation.LaunchAppPath, }; arguments.AddRange(_appArguments); await crashReporter.StartCaptureAsync(); var result = await RunMacCatalystApp(appInformation, timeout, _appArguments, envVariables, combinedCancellationToken.Token); await testReporter.CollectSimulatorResult(result); } finally { deviceListener.Cancel(); deviceListener.Dispose(); } return(await testReporter.ParseResult()); }
public async Task <(string DeviceName, ProcessExecutionResult result)> RunApp( AppBundleInformation appInformation, TestTargetOs target, TimeSpan timeout, string?deviceName = null, string?companionDeviceName = null, bool ensureCleanSimulatorState = false, int verbosity = 1, CancellationToken cancellationToken = default) { var isSimulator = target.Platform.IsSimulator(); ProcessExecutionResult result; ISimulatorDevice? simulator = null; ISimulatorDevice? companionSimulator = null; if (target.Platform == TestTarget.MacCatalyst) { _mainLog.WriteLine($"*** Executing '{appInformation.AppName}' on MacCatalyst ***"); result = await RunMacCatalystApp(appInformation, timeout, _appArguments, new Dictionary <string, object>(), cancellationToken); return("MacCatalyst", result); } // Find devices if (isSimulator) { (simulator, companionSimulator) = await _simulatorLoader.FindSimulators(target, _mainLog, 3); deviceName = companionSimulator?.Name ?? simulator.Name; } else { deviceName ??= await FindDevice(target) ?? throw new NoDeviceFoundException(); } var crashLogs = new Logs(_logs.Directory); ICrashSnapshotReporter crashReporter = _snapshotReporterFactory.Create( _mainLog, crashLogs, isDevice: !isSimulator, deviceName); _mainLog.WriteLine($"*** Executing '{appInformation.AppName}' on {target.AsString()} '{deviceName}' ***"); if (isSimulator) { if (simulator == null) { _mainLog.WriteLine("Didn't find any suitable simulator"); throw new NoDeviceFoundException(); } var mlaunchArguments = GetSimulatorArguments(appInformation, simulator, verbosity); result = await RunSimulatorApp( mlaunchArguments, appInformation, crashReporter, simulator, companionSimulator, ensureCleanSimulatorState, timeout, cancellationToken); } else { var mlaunchArguments = GetDeviceArguments(appInformation, deviceName, target.Platform.IsWatchOSTarget(), verbosity); result = await RunDeviceApp( mlaunchArguments, crashReporter, deviceName, timeout, cancellationToken); } return(deviceName, result); }
protected async Task <ProcessExecutionResult> RunSimulatorApp( AppBundleInformation appInformation, MlaunchArguments mlaunchArguments, ICrashSnapshotReporter crashReporter, ISimulatorDevice simulator, ISimulatorDevice?companionSimulator, TimeSpan timeout, bool waitForExit, CancellationToken cancellationToken) { _mainLog.WriteLine("System log for the '{1}' simulator is: {0}", simulator.SystemLog, simulator.Name); var simulatorLog = _captureLogFactory.Create( path: Path.Combine(_logs.Directory, simulator.Name + ".log"), systemLogPath: simulator.SystemLog, entireFile: false, LogType.SystemLog); simulatorLog.StartCapture(); _logs.Add(simulatorLog); var simulatorScanToken = await CaptureSimulatorLog(simulator, appInformation, cancellationToken); using var systemLogs = new DisposableList <ICaptureLog> { simulatorLog }; if (companionSimulator != null) { _mainLog.WriteLine("System log for the '{1}' companion simulator is: {0}", companionSimulator.SystemLog, companionSimulator.Name); var companionLog = _captureLogFactory.Create( path: Path.Combine(_logs.Directory, companionSimulator.Name + ".log"), systemLogPath: companionSimulator.SystemLog, entireFile: false, LogType.CompanionSystemLog); companionLog.StartCapture(); _logs.Add(companionLog); systemLogs.Add(companionLog); var companionScanToken = await CaptureSimulatorLog(companionSimulator, appInformation, cancellationToken); if (companionScanToken != null) { simulatorScanToken = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, companionScanToken.Token); } } await crashReporter.StartCaptureAsync(); _mainLog.WriteLine("Launching the app"); if (waitForExit) { var result = await _processManager.ExecuteCommandAsync(mlaunchArguments, _mainLog, timeout, cancellationToken : cancellationToken); simulatorScanToken?.Cancel(); return(result); } TaskCompletionSource appLaunched = new(); var scanLog = new ScanLog($"Launched {appInformation.BundleIdentifier} with pid", () => { _mainLog.WriteLine("App launch detected"); appLaunched.SetResult(); }); _mainLog.WriteLine("Waiting for the app to launch.."); var runTask = _processManager.ExecuteCommandAsync(mlaunchArguments, Log.CreateAggregatedLog(_mainLog, scanLog), timeout, cancellationToken: cancellationToken); await Task.WhenAny(runTask, appLaunched.Task); if (!appLaunched.Task.IsCompleted) { // In case the other task completes first, it is because one of these scenarios happened: // - The app crashed and never launched // - We missed the launch signal somehow and the app timed out // - The app launched and quit immediately and race condition noticed that before the scan log did its job // In all cases, we should return the result of the run task, it will be most likely 137 + Timeout (killed by us) // If not, it will be a success because the app ran for a super short amount of time _mainLog.WriteLine("App launch was not detected in time"); return(runTask.Result); } _mainLog.WriteLine("Not waiting for the app to exit"); return(new ProcessExecutionResult { ExitCode = 0 }); }