public async Task TestOnSimulatorTest(bool useTunnel) { var testResultFilePath = Path.GetTempFileName(); var listenerLogFile = Mock.Of <IFileBackedLog>(x => x.FullPath == testResultFilePath); File.WriteAllLines(testResultFilePath, new[] { "Some result here", "Tests run: 124", "Some result there" }); _logs .Setup(x => x.Create("test-ios-simulator-64-mocked_timestamp.log", "TestLog", It.IsAny <bool?>())) .Returns(listenerLogFile); var captureLog = new Mock <ICaptureLog>(); captureLog.SetupGet(x => x.FullPath).Returns(_simulatorLogPath); var captureLogFactory = new Mock <ICaptureLogFactory>(); captureLogFactory .Setup(x => x.Create( Path.Combine(_logs.Object.Directory, _mockSimulator.Name + ".log"), _mockSimulator.SystemLog, false, It.IsAny <LogType>())) .Returns(captureLog.Object); _listenerFactory.Setup(f => f.UseTunnel).Returns(useTunnel); // Act var appTester = new AppTester( _processManager.Object, _listenerFactory.Object, _snapshotReporterFactory, captureLogFactory.Object, Mock.Of <IDeviceLogCapturerFactory>(), _testReporterFactory, new XmlResultParser(), _mainLog.Object, _logs.Object, _helpers.Object); var(result, resultMessage) = await appTester.TestApp( _appBundleInfo, new TestTargetOs(TestTarget.Simulator_tvOS, null), _mockSimulator, null, TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(30), signalAppEnd : false, extraAppArguments : new string[] { "--foo=bar", "--xyz" }, extraEnvVariables : new[] { ("appArg1", "value1") });
public async Task TestOnDeviceWithNoAvailableSimulatorTest() { _hardwareDeviceLoader = new Mock <IHardwareDeviceLoader>(); _hardwareDeviceLoader .Setup(x => x.FindDevice(RunMode.iOS, _mainLog.Object, false, false)) .ThrowsAsync(new NoDeviceFoundException()); _listenerFactory.Setup(f => f.UseTunnel).Returns(false); // Act var appTester = new AppTester(_processManager.Object, _hardwareDeviceLoader.Object, _simulatorLoader.Object, _listenerFactory.Object, _snapshotReporterFactory, Mock.Of <ICaptureLogFactory>(), Mock.Of <IDeviceLogCapturerFactory>(), _testReporterFactory, new XmlResultParser(), _mainLog.Object, _logs.Object, _helpers.Object, Enumerable.Empty <string>()); var appInformation = new AppBundleInformation( appName: AppName, bundleIdentifier: AppBundleIdentifier, appPath: s_appPath, launchAppPath: s_appPath, supports32b: false, extension: null); await Assert.ThrowsAsync <NoDeviceFoundException>( async() => await appTester.TestApp( appInformation, new TestTargetOs(TestTarget.Device_iOS, null), TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(30), ensureCleanSimulatorState: true)); }
public async Task TestOnMacCatalystSuccessfullyTest() { var testResultFilePath = Path.GetTempFileName(); var listenerLogFile = Mock.Of <IFileBackedLog>(x => x.FullPath == testResultFilePath); File.WriteAllLines(testResultFilePath, new[] { "Some result here", "Tests run: 124", "Some result there" }); _logs .Setup(x => x.Create("test-maccatalyst-mocked_timestamp.log", "TestLog", It.IsAny <bool?>())) .Returns(listenerLogFile); var captureLog = new Mock <ICaptureLog>(); captureLog.SetupGet(x => x.FullPath).Returns(_simulatorLogPath); var captureLogFactory = new Mock <ICaptureLogFactory>(); captureLogFactory .Setup(x => x.Create( It.IsAny <string>(), "/var/log/system.log", false, It.IsAny <LogType>())) .Returns(captureLog.Object); // Act var appTester = new AppTester(_processManager.Object, _hardwareDeviceLoader.Object, _simulatorLoader.Object, _listenerFactory.Object, _snapshotReporterFactory, captureLogFactory.Object, Mock.Of <IDeviceLogCapturerFactory>(), _testReporterFactory, new XmlResultParser(), _mainLog.Object, _logs.Object, _helpers.Object, Array.Empty <string>()); var appInformation = new AppBundleInformation( appName: AppName, bundleIdentifier: AppBundleIdentifier, appPath: s_appPath, launchAppPath: s_appPath, supports32b: false, extension: null); var(deviceName, result, resultMessage) = await appTester.TestApp( appInformation, new TestTargetOs(TestTarget.MacCatalyst, null), TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(30), ensureCleanSimulatorState : true); // Verify Assert.Equal(TestExecutingResult.Succeeded, result); Assert.Equal("Tests run: 1194 Passed: 1191 Inconclusive: 0 Failed: 0 Ignored: 0", resultMessage); _processManager .Verify( x => x.ExecuteCommandAsync( "open", It.Is <IList <string> >(args => args.Contains(s_appPath)), _mainLog.Object, It.IsAny <TimeSpan>(), It.Is <Dictionary <string, string> >(envVars => envVars["NUNIT_HOSTNAME"] == "127.0.0.1" && envVars["NUNIT_HOSTPORT"] == Port.ToString() && envVars["NUNIT_AUTOEXIT"] == "true" && envVars["NUNIT_XML_VERSION"] == "xUnit" && envVars["NUNIT_ENABLE_XML_OUTPUT"] == "true"), It.IsAny <CancellationToken>()), Times.Once); _listener.Verify(x => x.InitializeAndGetPort(), Times.AtLeastOnce); _listener.Verify(x => x.StartAsync(), Times.AtLeastOnce); _listener.Verify(x => x.Cancel(), Times.AtLeastOnce); _listener.Verify(x => x.Dispose(), Times.AtLeastOnce); }
public async Task TestOnDeviceWithSkippedClassesTestTest(params string[] skippedClasses) { var deviceSystemLog = new Mock <IFileBackedLog>(); deviceSystemLog.SetupGet(x => x.FullPath).Returns(Path.GetTempFileName()); var deviceLogCapturer = new Mock <IDeviceLogCapturer>(); var deviceLogCapturerFactory = new Mock <IDeviceLogCapturerFactory>(); deviceLogCapturerFactory .Setup(x => x.Create(_mainLog.Object, deviceSystemLog.Object, DeviceName)) .Returns(deviceLogCapturer.Object); var testResultFilePath = Path.GetTempFileName(); var listenerLogFile = Mock.Of <IFileBackedLog>(x => x.FullPath == testResultFilePath); File.WriteAllLines(testResultFilePath, new[] { "Some result here", "Tests run: 124", "Some result there" }); _logs .Setup(x => x.Create("test-Device_iOS-mocked_timestamp.log", "TestLog", It.IsAny <bool?>())) .Returns(listenerLogFile); _logs .Setup(x => x.Create($"device-{DeviceName}-mocked_timestamp.log", LogType.SystemLog.ToString(), It.IsAny <bool?>())) .Returns(deviceSystemLog.Object); // Act var appTester = new AppTester(_processManager.Object, _hardwareDeviceLoader.Object, _simulatorLoader.Object, _listenerFactory.Object, _snapshotReporterFactory, Mock.Of <ICaptureLogFactory>(), deviceLogCapturerFactory.Object, _testReporterFactory, new XmlResultParser(), _mainLog.Object, _logs.Object, _helpers.Object, Enumerable.Empty <string>()); var appInformation = new AppBundleInformation( appName: AppName, bundleIdentifier: AppBundleIdentifier, appPath: s_appPath, launchAppPath: s_appPath, supports32b: false, extension: null); var(deviceName, result, resultMessage) = await appTester.TestApp( appInformation, new TestTargetOs(TestTarget.Device_iOS, null), timeout : TimeSpan.FromSeconds(30), testLaunchTimeout : TimeSpan.FromSeconds(30), skippedTestClasses : skippedClasses); // Verify Assert.Equal(DeviceName, deviceName); Assert.Equal(TestExecutingResult.Succeeded, result); Assert.Equal("Tests run: 1194 Passed: 1191 Inconclusive: 0 Failed: 0 Ignored: 0", resultMessage); var skippedTestsArg = $"-setenv=NUNIT_RUN_ALL=false -setenv=NUNIT_SKIPPED_CLASSES={string.Join(',', skippedClasses)} "; var expectedArgs = GetExpectedDeviceMlaunchArgs(skippedTestsArg); _processManager .Verify( x => x.ExecuteCommandAsync( It.Is <MlaunchArguments>(args => args.AsCommandLine() == expectedArgs), It.IsAny <ILog>(), It.IsAny <TimeSpan>(), null, It.IsAny <CancellationToken>()), Times.Once); _listener.Verify(x => x.InitializeAndGetPort(), Times.AtLeastOnce); _listener.Verify(x => x.StartAsync(), Times.AtLeastOnce); _listener.Verify(x => x.Cancel(), Times.AtLeastOnce); _listener.Verify(x => x.Dispose(), Times.AtLeastOnce); _hardwareDeviceLoader.VerifyAll(); _snapshotReporter.Verify(x => x.StartCaptureAsync(), Times.AtLeastOnce); _snapshotReporter.Verify(x => x.StartCaptureAsync(), Times.AtLeastOnce); deviceSystemLog.Verify(x => x.Dispose(), Times.AtLeastOnce); }
public async Task TestOnDeviceSuccessfullyTest(bool useTunnel) { var deviceSystemLog = new Mock <IFileBackedLog>(); deviceSystemLog.SetupGet(x => x.FullPath).Returns(Path.GetTempFileName()); var deviceLogCapturer = new Mock <IDeviceLogCapturer>(); var deviceLogCapturerFactory = new Mock <IDeviceLogCapturerFactory>(); deviceLogCapturerFactory .Setup(x => x.Create(_mainLog.Object, deviceSystemLog.Object, DeviceName)) .Returns(deviceLogCapturer.Object); var testResultFilePath = Path.GetTempFileName(); var listenerLogFile = Mock.Of <IFileBackedLog>(x => x.FullPath == testResultFilePath); File.WriteAllLines(testResultFilePath, new[] { "Some result here", "Tests run: 124", "Some result there" }); _logs .Setup(x => x.Create("test-Device_iOS-mocked_timestamp.log", "TestLog", It.IsAny <bool?>())) .Returns(listenerLogFile); _logs .Setup(x => x.Create($"device-{DeviceName}-mocked_timestamp.log", LogType.SystemLog.ToString(), It.IsAny <bool?>())) .Returns(deviceSystemLog.Object); // set tunnel bore expectation if (useTunnel) { _tunnelBore.Setup(t => t.Create(DeviceName, It.IsAny <ILog>())); } _listenerFactory.Setup(f => f.UseTunnel).Returns((useTunnel)); // Act var appTester = new AppTester(_processManager.Object, _hardwareDeviceLoader.Object, _simulatorLoader.Object, _listenerFactory.Object, _snapshotReporterFactory, Mock.Of <ICaptureLogFactory>(), deviceLogCapturerFactory.Object, _testReporterFactory, new XmlResultParser(), _mainLog.Object, _logs.Object, _helpers.Object, new[] { "--appArg1=value1", "-f" }); var appInformation = new AppBundleInformation( appName: AppName, bundleIdentifier: AppBundleIdentifier, appPath: s_appPath, launchAppPath: s_appPath, supports32b: false, extension: null); var(deviceName, result, resultMessage) = await appTester.TestApp( appInformation, new TestTargetOs(TestTarget.Device_iOS, null), TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(30)); // Verify Assert.Equal(DeviceName, deviceName); Assert.Equal(TestExecutingResult.Succeeded, result); Assert.Equal("Tests run: 1194 Passed: 1191 Inconclusive: 0 Failed: 0 Ignored: 0", resultMessage); var expectedArgs = GetExpectedDeviceMlaunchArgs( useTunnel: useTunnel, extraArgs: "-argument=--appArg1=value1 -argument=-f "); _processManager .Verify( x => x.ExecuteCommandAsync( It.Is <MlaunchArguments>(args => args.AsCommandLine() == expectedArgs), It.IsAny <ILog>(), It.IsAny <TimeSpan>(), null, It.IsAny <CancellationToken>()), Times.Once); _listener.Verify(x => x.InitializeAndGetPort(), Times.AtLeastOnce); _listener.Verify(x => x.StartAsync(), Times.AtLeastOnce); _listener.Verify(x => x.Cancel(), Times.AtLeastOnce); _listener.Verify(x => x.Dispose(), Times.AtLeastOnce); // verify that we do close the tunnel when it was used // we dont want to leak a process if (useTunnel) { _tunnelBore.Verify(t => t.Close(DeviceName)); } _hardwareDeviceLoader.VerifyAll(); _snapshotReporter.Verify(x => x.StartCaptureAsync(), Times.AtLeastOnce); _snapshotReporter.Verify(x => x.StartCaptureAsync(), Times.AtLeastOnce); deviceSystemLog.Verify(x => x.Dispose(), Times.AtLeastOnce); }
public async Task TestOnSimulatorSuccessfullyTest(bool useTunnel) { _simulatorLoader .Setup(x => x.FindSimulators(It.Is <TestTargetOs>(t => t.Platform == TestTarget.Simulator_tvOS), _mainLog.Object, It.IsAny <int>(), true, false)) .ReturnsAsync((_mockSimulator.Object, null)); var testResultFilePath = Path.GetTempFileName(); var listenerLogFile = Mock.Of <IFileBackedLog>(x => x.FullPath == testResultFilePath); File.WriteAllLines(testResultFilePath, new[] { "Some result here", "Tests run: 124", "Some result there" }); _logs .Setup(x => x.Create("test-Simulator_tvOS-mocked_timestamp.log", "TestLog", It.IsAny <bool?>())) .Returns(listenerLogFile); var captureLog = new Mock <ICaptureLog>(); captureLog.SetupGet(x => x.FullPath).Returns(_simulatorLogPath); var captureLogFactory = new Mock <ICaptureLogFactory>(); captureLogFactory .Setup(x => x.Create( Path.Combine(_logs.Object.Directory, _mockSimulator.Object.Name + ".log"), _mockSimulator.Object.SystemLog, false, It.IsAny <LogType>())) .Returns(captureLog.Object); _listenerFactory.Setup(f => f.UseTunnel).Returns(useTunnel); // Act var appTester = new AppTester(_processManager.Object, _hardwareDeviceLoader.Object, _simulatorLoader.Object, _listenerFactory.Object, _snapshotReporterFactory, captureLogFactory.Object, Mock.Of <IDeviceLogCapturerFactory>(), _testReporterFactory, new XmlResultParser(), _mainLog.Object, _logs.Object, _helpers.Object, Array.Empty <string>()); var appInformation = new AppBundleInformation( appName: AppName, bundleIdentifier: AppBundleIdentifier, appPath: s_appPath, launchAppPath: s_appPath, supports32b: false, extension: null); var(deviceName, result, resultMessage) = await appTester.TestApp( appInformation, new TestTargetOs(TestTarget.Simulator_tvOS, null), TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(30), ensureCleanSimulatorState : true); // Verify Assert.Equal(SimulatorDeviceName, deviceName); Assert.Equal(TestExecutingResult.Succeeded, result); Assert.Equal("Tests run: 1194 Passed: 1191 Inconclusive: 0 Failed: 0 Ignored: 0", resultMessage); var expectedArgs = GetExpectedSimulatorMlaunchArgs(); _processManager .Verify( x => x.ExecuteCommandAsync( It.Is <MlaunchArguments>(args => args.AsCommandLine() == expectedArgs), _mainLog.Object, It.IsAny <TimeSpan>(), null, It.IsAny <CancellationToken>()), Times.Once); _listener.Verify(x => x.InitializeAndGetPort(), Times.AtLeastOnce); _listener.Verify(x => x.StartAsync(), Times.AtLeastOnce); _listener.Verify(x => x.Cancel(), Times.AtLeastOnce); _listener.Verify(x => x.Dispose(), Times.AtLeastOnce); _simulatorLoader.VerifyAll(); captureLog.Verify(x => x.StartCapture(), Times.AtLeastOnce); // When ensureCleanSimulatorState == true _mockSimulator.Verify(x => x.PrepareSimulator(_mainLog.Object, AppBundleIdentifier)); _mockSimulator.Verify(x => x.KillEverything(_mainLog.Object)); }
public async Task TestOnSimulatorWithNoAvailableSimulatorTest(bool useTcpTunnel) { // Mock finding simulators string simulatorLogPath = Path.Combine(Path.GetTempPath(), "simulator-logs"); _simulatorLoader .Setup(x => x.FindSimulators(It.Is <TestTargetOs>(t => t.Platform == TestTarget.Simulator_tvOS), _mainLog.Object, It.IsAny <int>(), true, false)) .ThrowsAsync(new NoDeviceFoundException("Failed to find simulator")); var listenerLogFile = new Mock <IFileBackedLog>(); _logs .Setup(x => x.Create(It.IsAny <string>(), "TestLog", It.IsAny <bool>())) .Returns(listenerLogFile.Object); var captureLog = new Mock <ICaptureLog>(); captureLog .SetupGet(x => x.FullPath) .Returns(simulatorLogPath); var captureLogFactory = new Mock <ICaptureLogFactory>(); captureLogFactory .Setup(x => x.Create( Path.Combine(_logs.Object.Directory, "tvos.log"), "/path/to/_mockSimulator.log", false, It.IsAny <LogType>())) .Returns(captureLog.Object); _listenerFactory.Setup(f => f.UseTunnel).Returns(useTcpTunnel); // Act var appTester = new AppTester(_processManager.Object, _hardwareDeviceLoader.Object, _simulatorLoader.Object, _listenerFactory.Object, _snapshotReporterFactory, captureLogFactory.Object, Mock.Of <IDeviceLogCapturerFactory>(), _testReporterFactory, new XmlResultParser(), _mainLog.Object, _logs.Object, _helpers.Object, Enumerable.Empty <string>()); var appInformation = new AppBundleInformation( appName: AppName, bundleIdentifier: AppBundleIdentifier, appPath: s_appPath, launchAppPath: s_appPath, supports32b: false, extension: null); await Assert.ThrowsAsync <NoDeviceFoundException>( async() => await appTester.TestApp( appInformation, new TestTargetOs(TestTarget.Simulator_tvOS, null), TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(30))); // Verify _mainLog.Verify(x => x.WriteLine("Test run completed"), Times.Never); _simulatorLoader.VerifyAll(); _listener.Verify(x => x.StartAsync(), Times.Never); _tunnelBore.Verify(t => t.Create(It.IsAny <string>(), It.IsAny <ILog>()), Times.Never); // never create tunnels on simulators }
protected override async Task <ExitCode> RunAppInternal( AppBundleInformation appBundleInfo, string?deviceName, ILogger logger, TestTargetOs target, Logs logs, IFileBackedLog mainLog, CancellationToken cancellationToken) { var tunnelBore = (_arguments.CommunicationChannel == CommunicationChannel.UsbTunnel && !target.Platform.IsSimulator()) ? new TunnelBore(ProcessManager) : null; // only add the extra callback if we do know that the feature was indeed enabled Action <string>?logCallback = IsLldbEnabled() ? (l) => NotifyUserLldbCommand(logger, l) : (Action <string>?)null; var appTester = new AppTester( ProcessManager, DeviceLoader, SimulatorLoader, new SimpleListenerFactory(tunnelBore), new CrashSnapshotReporterFactory(ProcessManager), new CaptureLogFactory(), new DeviceLogCapturerFactory(ProcessManager), new TestReporterFactory(ProcessManager), new XmlResultParser(), mainLog, logs, new Helpers(), logCallback: logCallback, appArguments: PassThroughArguments); string resultMessage; TestExecutingResult testResult; (deviceName, testResult, resultMessage) = await appTester.TestApp(appBundleInfo, target, _arguments.Timeout, _arguments.LaunchTimeout, deviceName, verbosity : GetMlaunchVerbosity(_arguments.Verbosity), xmlResultJargon : _arguments.XmlResultJargon, cancellationToken : cancellationToken, skippedMethods : _arguments.SingleMethodFilters?.ToArray(), skippedTestClasses : _arguments.ClassMethodFilters?.ToArray()); switch (testResult) { case TestExecutingResult.Succeeded: logger.LogInformation($"Application finished the test run successfully"); logger.LogInformation(resultMessage); return(ExitCode.SUCCESS); case TestExecutingResult.Failed: logger.LogInformation($"Application finished the test run successfully with some failed tests"); logger.LogInformation(resultMessage); return(ExitCode.TESTS_FAILED); case TestExecutingResult.LaunchFailure: if (resultMessage != null) { logger.LogError($"Failed to launch the application:{Environment.NewLine}" + $"{resultMessage}{Environment.NewLine}{Environment.NewLine}" + $"Check logs for more information."); } else { logger.LogError($"Failed to launch the application. Check logs for more information"); } return(ExitCode.APP_LAUNCH_FAILURE); case TestExecutingResult.Crashed: if (resultMessage != null) { logger.LogError($"Application run crashed:{Environment.NewLine}" + $"{resultMessage}{Environment.NewLine}{Environment.NewLine}" + $"Check logs for more information."); } else { logger.LogError($"Application test run crashed. Check logs for more information"); } return(ExitCode.APP_CRASH); case TestExecutingResult.TimedOut: logger.LogWarning($"Application test run timed out"); return(ExitCode.TIMED_OUT); default: if (resultMessage != null) { logger.LogError($"Application test run ended in an unexpected way: '{testResult}'{Environment.NewLine}" + $"{resultMessage}{Environment.NewLine}{Environment.NewLine}" + $"Check logs for more information."); } else { logger.LogError($"Application test run ended in an unexpected way: '{testResult}'. Check logs for more information"); } return(ExitCode.GENERAL_FAILURE); } }