public async Task TestOnSimulatorTest(bool useTunnel)
{
var testResultFilePath = Path.GetTempFileName();
var listenerLogFile = Mock.Of <IFileBackedLog>(x => x.FullPath == testResultFilePath);
File.WriteAllLines(testResultFilePath, new[] { "Some result here", "Tests run: 124", "Some result there" });
_logs
.Setup(x => x.Create("test-ios-simulator-64-mocked_timestamp.log", "TestLog", It.IsAny <bool?>()))
.Returns(listenerLogFile);
var captureLog = new Mock <ICaptureLog>();
captureLog.SetupGet(x => x.FullPath).Returns(_simulatorLogPath);
var captureLogFactory = new Mock <ICaptureLogFactory>();
captureLogFactory
.Setup(x => x.Create(
Path.Combine(_logs.Object.Directory, _mockSimulator.Name + ".log"),
_mockSimulator.SystemLog,
false,
It.IsAny <LogType>()))
.Returns(captureLog.Object);
_listenerFactory.Setup(f => f.UseTunnel).Returns(useTunnel);
// Act
var appTester = new AppTester(
_processManager.Object,
_listenerFactory.Object,
_snapshotReporterFactory,
captureLogFactory.Object,
Mock.Of <IDeviceLogCapturerFactory>(),
_testReporterFactory,
new XmlResultParser(),
_mainLog.Object,
_logs.Object,
_helpers.Object);
var(result, resultMessage) = await appTester.TestApp(
_appBundleInfo,
new TestTargetOs(TestTarget.Simulator_tvOS, null),
_mockSimulator,
null,
TimeSpan.FromSeconds(30),
TimeSpan.FromSeconds(30),
signalAppEnd : false,
extraAppArguments : new string[] { "--foo=bar", "--xyz" },
extraEnvVariables : new[] { ("appArg1", "value1") });
public async Task TestOnDeviceWithNoAvailableSimulatorTest()
{
_hardwareDeviceLoader = new Mock <IHardwareDeviceLoader>();
_hardwareDeviceLoader
.Setup(x => x.FindDevice(RunMode.iOS, _mainLog.Object, false, false))
.ThrowsAsync(new NoDeviceFoundException());
_listenerFactory.Setup(f => f.UseTunnel).Returns(false);
// Act
var appTester = new AppTester(_processManager.Object,
_hardwareDeviceLoader.Object,
_simulatorLoader.Object,
_listenerFactory.Object,
_snapshotReporterFactory,
Mock.Of <ICaptureLogFactory>(),
Mock.Of <IDeviceLogCapturerFactory>(),
_testReporterFactory,
new XmlResultParser(),
_mainLog.Object,
_logs.Object,
_helpers.Object,
Enumerable.Empty <string>());
var appInformation = new AppBundleInformation(
appName: AppName,
bundleIdentifier: AppBundleIdentifier,
appPath: s_appPath,
launchAppPath: s_appPath,
supports32b: false,
extension: null);
await Assert.ThrowsAsync <NoDeviceFoundException>(
async() => await appTester.TestApp(
appInformation,
new TestTargetOs(TestTarget.Device_iOS, null),
TimeSpan.FromSeconds(30),
TimeSpan.FromSeconds(30),
ensureCleanSimulatorState: true));
}
public async Task TestOnMacCatalystSuccessfullyTest()
{
var testResultFilePath = Path.GetTempFileName();
var listenerLogFile = Mock.Of <IFileBackedLog>(x => x.FullPath == testResultFilePath);
File.WriteAllLines(testResultFilePath, new[] { "Some result here", "Tests run: 124", "Some result there" });
_logs
.Setup(x => x.Create("test-maccatalyst-mocked_timestamp.log", "TestLog", It.IsAny <bool?>()))
.Returns(listenerLogFile);
var captureLog = new Mock <ICaptureLog>();
captureLog.SetupGet(x => x.FullPath).Returns(_simulatorLogPath);
var captureLogFactory = new Mock <ICaptureLogFactory>();
captureLogFactory
.Setup(x => x.Create(
It.IsAny <string>(),
"/var/log/system.log",
false,
It.IsAny <LogType>()))
.Returns(captureLog.Object);
// Act
var appTester = new AppTester(_processManager.Object,
_hardwareDeviceLoader.Object,
_simulatorLoader.Object,
_listenerFactory.Object,
_snapshotReporterFactory,
captureLogFactory.Object,
Mock.Of <IDeviceLogCapturerFactory>(),
_testReporterFactory,
new XmlResultParser(),
_mainLog.Object,
_logs.Object,
_helpers.Object,
Array.Empty <string>());
var appInformation = new AppBundleInformation(
appName: AppName,
bundleIdentifier: AppBundleIdentifier,
appPath: s_appPath,
launchAppPath: s_appPath,
supports32b: false,
extension: null);
var(deviceName, result, resultMessage) = await appTester.TestApp(
appInformation,
new TestTargetOs(TestTarget.MacCatalyst, null),
TimeSpan.FromSeconds(30),
TimeSpan.FromSeconds(30),
ensureCleanSimulatorState : true);
// Verify
Assert.Equal(TestExecutingResult.Succeeded, result);
Assert.Equal("Tests run: 1194 Passed: 1191 Inconclusive: 0 Failed: 0 Ignored: 0", resultMessage);
_processManager
.Verify(
x => x.ExecuteCommandAsync(
"open",
It.Is <IList <string> >(args => args.Contains(s_appPath)),
_mainLog.Object,
It.IsAny <TimeSpan>(),
It.Is <Dictionary <string, string> >(envVars =>
envVars["NUNIT_HOSTNAME"] == "127.0.0.1" &&
envVars["NUNIT_HOSTPORT"] == Port.ToString() &&
envVars["NUNIT_AUTOEXIT"] == "true" &&
envVars["NUNIT_XML_VERSION"] == "xUnit" &&
envVars["NUNIT_ENABLE_XML_OUTPUT"] == "true"),
It.IsAny <CancellationToken>()),
Times.Once);
_listener.Verify(x => x.InitializeAndGetPort(), Times.AtLeastOnce);
_listener.Verify(x => x.StartAsync(), Times.AtLeastOnce);
_listener.Verify(x => x.Cancel(), Times.AtLeastOnce);
_listener.Verify(x => x.Dispose(), Times.AtLeastOnce);
}
public async Task TestOnDeviceWithSkippedClassesTestTest(params string[] skippedClasses)
{
var deviceSystemLog = new Mock <IFileBackedLog>();
deviceSystemLog.SetupGet(x => x.FullPath).Returns(Path.GetTempFileName());
var deviceLogCapturer = new Mock <IDeviceLogCapturer>();
var deviceLogCapturerFactory = new Mock <IDeviceLogCapturerFactory>();
deviceLogCapturerFactory
.Setup(x => x.Create(_mainLog.Object, deviceSystemLog.Object, DeviceName))
.Returns(deviceLogCapturer.Object);
var testResultFilePath = Path.GetTempFileName();
var listenerLogFile = Mock.Of <IFileBackedLog>(x => x.FullPath == testResultFilePath);
File.WriteAllLines(testResultFilePath, new[] { "Some result here", "Tests run: 124", "Some result there" });
_logs
.Setup(x => x.Create("test-Device_iOS-mocked_timestamp.log", "TestLog", It.IsAny <bool?>()))
.Returns(listenerLogFile);
_logs
.Setup(x => x.Create($"device-{DeviceName}-mocked_timestamp.log", LogType.SystemLog.ToString(), It.IsAny <bool?>()))
.Returns(deviceSystemLog.Object);
// Act
var appTester = new AppTester(_processManager.Object,
_hardwareDeviceLoader.Object,
_simulatorLoader.Object,
_listenerFactory.Object,
_snapshotReporterFactory,
Mock.Of <ICaptureLogFactory>(),
deviceLogCapturerFactory.Object,
_testReporterFactory,
new XmlResultParser(),
_mainLog.Object,
_logs.Object,
_helpers.Object,
Enumerable.Empty <string>());
var appInformation = new AppBundleInformation(
appName: AppName,
bundleIdentifier: AppBundleIdentifier,
appPath: s_appPath,
launchAppPath: s_appPath,
supports32b: false,
extension: null);
var(deviceName, result, resultMessage) = await appTester.TestApp(
appInformation,
new TestTargetOs(TestTarget.Device_iOS, null),
timeout : TimeSpan.FromSeconds(30),
testLaunchTimeout : TimeSpan.FromSeconds(30),
skippedTestClasses : skippedClasses);
// Verify
Assert.Equal(DeviceName, deviceName);
Assert.Equal(TestExecutingResult.Succeeded, result);
Assert.Equal("Tests run: 1194 Passed: 1191 Inconclusive: 0 Failed: 0 Ignored: 0", resultMessage);
var skippedTestsArg = $"-setenv=NUNIT_RUN_ALL=false -setenv=NUNIT_SKIPPED_CLASSES={string.Join(',', skippedClasses)} ";
var expectedArgs = GetExpectedDeviceMlaunchArgs(skippedTestsArg);
_processManager
.Verify(
x => x.ExecuteCommandAsync(
It.Is <MlaunchArguments>(args => args.AsCommandLine() == expectedArgs),
It.IsAny <ILog>(),
It.IsAny <TimeSpan>(),
null,
It.IsAny <CancellationToken>()),
Times.Once);
_listener.Verify(x => x.InitializeAndGetPort(), Times.AtLeastOnce);
_listener.Verify(x => x.StartAsync(), Times.AtLeastOnce);
_listener.Verify(x => x.Cancel(), Times.AtLeastOnce);
_listener.Verify(x => x.Dispose(), Times.AtLeastOnce);
_hardwareDeviceLoader.VerifyAll();
_snapshotReporter.Verify(x => x.StartCaptureAsync(), Times.AtLeastOnce);
_snapshotReporter.Verify(x => x.StartCaptureAsync(), Times.AtLeastOnce);
deviceSystemLog.Verify(x => x.Dispose(), Times.AtLeastOnce);
}
public async Task TestOnDeviceSuccessfullyTest(bool useTunnel)
{
var deviceSystemLog = new Mock <IFileBackedLog>();
deviceSystemLog.SetupGet(x => x.FullPath).Returns(Path.GetTempFileName());
var deviceLogCapturer = new Mock <IDeviceLogCapturer>();
var deviceLogCapturerFactory = new Mock <IDeviceLogCapturerFactory>();
deviceLogCapturerFactory
.Setup(x => x.Create(_mainLog.Object, deviceSystemLog.Object, DeviceName))
.Returns(deviceLogCapturer.Object);
var testResultFilePath = Path.GetTempFileName();
var listenerLogFile = Mock.Of <IFileBackedLog>(x => x.FullPath == testResultFilePath);
File.WriteAllLines(testResultFilePath, new[] { "Some result here", "Tests run: 124", "Some result there" });
_logs
.Setup(x => x.Create("test-Device_iOS-mocked_timestamp.log", "TestLog", It.IsAny <bool?>()))
.Returns(listenerLogFile);
_logs
.Setup(x => x.Create($"device-{DeviceName}-mocked_timestamp.log", LogType.SystemLog.ToString(), It.IsAny <bool?>()))
.Returns(deviceSystemLog.Object);
// set tunnel bore expectation
if (useTunnel)
{
_tunnelBore.Setup(t => t.Create(DeviceName, It.IsAny <ILog>()));
}
_listenerFactory.Setup(f => f.UseTunnel).Returns((useTunnel));
// Act
var appTester = new AppTester(_processManager.Object,
_hardwareDeviceLoader.Object,
_simulatorLoader.Object,
_listenerFactory.Object,
_snapshotReporterFactory,
Mock.Of <ICaptureLogFactory>(),
deviceLogCapturerFactory.Object,
_testReporterFactory,
new XmlResultParser(),
_mainLog.Object,
_logs.Object,
_helpers.Object,
new[] { "--appArg1=value1", "-f" });
var appInformation = new AppBundleInformation(
appName: AppName,
bundleIdentifier: AppBundleIdentifier,
appPath: s_appPath,
launchAppPath: s_appPath,
supports32b: false,
extension: null);
var(deviceName, result, resultMessage) = await appTester.TestApp(
appInformation,
new TestTargetOs(TestTarget.Device_iOS, null),
TimeSpan.FromSeconds(30),
TimeSpan.FromSeconds(30));
// Verify
Assert.Equal(DeviceName, deviceName);
Assert.Equal(TestExecutingResult.Succeeded, result);
Assert.Equal("Tests run: 1194 Passed: 1191 Inconclusive: 0 Failed: 0 Ignored: 0", resultMessage);
var expectedArgs = GetExpectedDeviceMlaunchArgs(
useTunnel: useTunnel,
extraArgs: "-argument=--appArg1=value1 -argument=-f ");
_processManager
.Verify(
x => x.ExecuteCommandAsync(
It.Is <MlaunchArguments>(args => args.AsCommandLine() == expectedArgs),
It.IsAny <ILog>(),
It.IsAny <TimeSpan>(),
null,
It.IsAny <CancellationToken>()),
Times.Once);
_listener.Verify(x => x.InitializeAndGetPort(), Times.AtLeastOnce);
_listener.Verify(x => x.StartAsync(), Times.AtLeastOnce);
_listener.Verify(x => x.Cancel(), Times.AtLeastOnce);
_listener.Verify(x => x.Dispose(), Times.AtLeastOnce);
// verify that we do close the tunnel when it was used
// we dont want to leak a process
if (useTunnel)
{
_tunnelBore.Verify(t => t.Close(DeviceName));
}
_hardwareDeviceLoader.VerifyAll();
_snapshotReporter.Verify(x => x.StartCaptureAsync(), Times.AtLeastOnce);
_snapshotReporter.Verify(x => x.StartCaptureAsync(), Times.AtLeastOnce);
deviceSystemLog.Verify(x => x.Dispose(), Times.AtLeastOnce);
}
public async Task TestOnSimulatorSuccessfullyTest(bool useTunnel)
{
_simulatorLoader
.Setup(x => x.FindSimulators(It.Is <TestTargetOs>(t => t.Platform == TestTarget.Simulator_tvOS), _mainLog.Object, It.IsAny <int>(), true, false))
.ReturnsAsync((_mockSimulator.Object, null));
var testResultFilePath = Path.GetTempFileName();
var listenerLogFile = Mock.Of <IFileBackedLog>(x => x.FullPath == testResultFilePath);
File.WriteAllLines(testResultFilePath, new[] { "Some result here", "Tests run: 124", "Some result there" });
_logs
.Setup(x => x.Create("test-Simulator_tvOS-mocked_timestamp.log", "TestLog", It.IsAny <bool?>()))
.Returns(listenerLogFile);
var captureLog = new Mock <ICaptureLog>();
captureLog.SetupGet(x => x.FullPath).Returns(_simulatorLogPath);
var captureLogFactory = new Mock <ICaptureLogFactory>();
captureLogFactory
.Setup(x => x.Create(
Path.Combine(_logs.Object.Directory, _mockSimulator.Object.Name + ".log"),
_mockSimulator.Object.SystemLog,
false,
It.IsAny <LogType>()))
.Returns(captureLog.Object);
_listenerFactory.Setup(f => f.UseTunnel).Returns(useTunnel);
// Act
var appTester = new AppTester(_processManager.Object,
_hardwareDeviceLoader.Object,
_simulatorLoader.Object,
_listenerFactory.Object,
_snapshotReporterFactory,
captureLogFactory.Object,
Mock.Of <IDeviceLogCapturerFactory>(),
_testReporterFactory,
new XmlResultParser(),
_mainLog.Object,
_logs.Object,
_helpers.Object,
Array.Empty <string>());
var appInformation = new AppBundleInformation(
appName: AppName,
bundleIdentifier: AppBundleIdentifier,
appPath: s_appPath,
launchAppPath: s_appPath,
supports32b: false,
extension: null);
var(deviceName, result, resultMessage) = await appTester.TestApp(
appInformation,
new TestTargetOs(TestTarget.Simulator_tvOS, null),
TimeSpan.FromSeconds(30),
TimeSpan.FromSeconds(30),
ensureCleanSimulatorState : true);
// Verify
Assert.Equal(SimulatorDeviceName, deviceName);
Assert.Equal(TestExecutingResult.Succeeded, result);
Assert.Equal("Tests run: 1194 Passed: 1191 Inconclusive: 0 Failed: 0 Ignored: 0", resultMessage);
var expectedArgs = GetExpectedSimulatorMlaunchArgs();
_processManager
.Verify(
x => x.ExecuteCommandAsync(
It.Is <MlaunchArguments>(args => args.AsCommandLine() == expectedArgs),
_mainLog.Object,
It.IsAny <TimeSpan>(),
null,
It.IsAny <CancellationToken>()),
Times.Once);
_listener.Verify(x => x.InitializeAndGetPort(), Times.AtLeastOnce);
_listener.Verify(x => x.StartAsync(), Times.AtLeastOnce);
_listener.Verify(x => x.Cancel(), Times.AtLeastOnce);
_listener.Verify(x => x.Dispose(), Times.AtLeastOnce);
_simulatorLoader.VerifyAll();
captureLog.Verify(x => x.StartCapture(), Times.AtLeastOnce);
// When ensureCleanSimulatorState == true
_mockSimulator.Verify(x => x.PrepareSimulator(_mainLog.Object, AppBundleIdentifier));
_mockSimulator.Verify(x => x.KillEverything(_mainLog.Object));
}
public async Task TestOnSimulatorWithNoAvailableSimulatorTest(bool useTcpTunnel)
{
// Mock finding simulators
string simulatorLogPath = Path.Combine(Path.GetTempPath(), "simulator-logs");
_simulatorLoader
.Setup(x => x.FindSimulators(It.Is <TestTargetOs>(t => t.Platform == TestTarget.Simulator_tvOS), _mainLog.Object, It.IsAny <int>(), true, false))
.ThrowsAsync(new NoDeviceFoundException("Failed to find simulator"));
var listenerLogFile = new Mock <IFileBackedLog>();
_logs
.Setup(x => x.Create(It.IsAny <string>(), "TestLog", It.IsAny <bool>()))
.Returns(listenerLogFile.Object);
var captureLog = new Mock <ICaptureLog>();
captureLog
.SetupGet(x => x.FullPath)
.Returns(simulatorLogPath);
var captureLogFactory = new Mock <ICaptureLogFactory>();
captureLogFactory
.Setup(x => x.Create(
Path.Combine(_logs.Object.Directory, "tvos.log"),
"/path/to/_mockSimulator.log",
false,
It.IsAny <LogType>()))
.Returns(captureLog.Object);
_listenerFactory.Setup(f => f.UseTunnel).Returns(useTcpTunnel);
// Act
var appTester = new AppTester(_processManager.Object,
_hardwareDeviceLoader.Object,
_simulatorLoader.Object,
_listenerFactory.Object,
_snapshotReporterFactory,
captureLogFactory.Object,
Mock.Of <IDeviceLogCapturerFactory>(),
_testReporterFactory,
new XmlResultParser(),
_mainLog.Object,
_logs.Object,
_helpers.Object,
Enumerable.Empty <string>());
var appInformation = new AppBundleInformation(
appName: AppName,
bundleIdentifier: AppBundleIdentifier,
appPath: s_appPath,
launchAppPath: s_appPath,
supports32b: false,
extension: null);
await Assert.ThrowsAsync <NoDeviceFoundException>(
async() => await appTester.TestApp(
appInformation,
new TestTargetOs(TestTarget.Simulator_tvOS, null),
TimeSpan.FromSeconds(30),
TimeSpan.FromSeconds(30)));
// Verify
_mainLog.Verify(x => x.WriteLine("Test run completed"), Times.Never);
_simulatorLoader.VerifyAll();
_listener.Verify(x => x.StartAsync(), Times.Never);
_tunnelBore.Verify(t => t.Create(It.IsAny <string>(), It.IsAny <ILog>()), Times.Never); // never create tunnels on simulators
}
protected override async Task <ExitCode> RunAppInternal(
AppBundleInformation appBundleInfo,
string?deviceName,
ILogger logger,
TestTargetOs target,
Logs logs,
IFileBackedLog mainLog,
CancellationToken cancellationToken)
{
var tunnelBore = (_arguments.CommunicationChannel == CommunicationChannel.UsbTunnel && !target.Platform.IsSimulator())
? new TunnelBore(ProcessManager)
: null;
// only add the extra callback if we do know that the feature was indeed enabled
Action <string>?logCallback = IsLldbEnabled() ? (l) => NotifyUserLldbCommand(logger, l) : (Action <string>?)null;
var appTester = new AppTester(
ProcessManager,
DeviceLoader,
SimulatorLoader,
new SimpleListenerFactory(tunnelBore),
new CrashSnapshotReporterFactory(ProcessManager),
new CaptureLogFactory(),
new DeviceLogCapturerFactory(ProcessManager),
new TestReporterFactory(ProcessManager),
new XmlResultParser(),
mainLog,
logs,
new Helpers(),
logCallback: logCallback,
appArguments: PassThroughArguments);
string resultMessage;
TestExecutingResult testResult;
(deviceName, testResult, resultMessage) = await appTester.TestApp(appBundleInfo,
target,
_arguments.Timeout,
_arguments.LaunchTimeout,
deviceName,
verbosity : GetMlaunchVerbosity(_arguments.Verbosity),
xmlResultJargon : _arguments.XmlResultJargon,
cancellationToken : cancellationToken,
skippedMethods : _arguments.SingleMethodFilters?.ToArray(),
skippedTestClasses : _arguments.ClassMethodFilters?.ToArray());
switch (testResult)
{
case TestExecutingResult.Succeeded:
logger.LogInformation($"Application finished the test run successfully");
logger.LogInformation(resultMessage);
return(ExitCode.SUCCESS);
case TestExecutingResult.Failed:
logger.LogInformation($"Application finished the test run successfully with some failed tests");
logger.LogInformation(resultMessage);
return(ExitCode.TESTS_FAILED);
case TestExecutingResult.LaunchFailure:
if (resultMessage != null)
{
logger.LogError($"Failed to launch the application:{Environment.NewLine}" +
$"{resultMessage}{Environment.NewLine}{Environment.NewLine}" +
$"Check logs for more information.");
}
else
{
logger.LogError($"Failed to launch the application. Check logs for more information");
}
return(ExitCode.APP_LAUNCH_FAILURE);
case TestExecutingResult.Crashed:
if (resultMessage != null)
{
logger.LogError($"Application run crashed:{Environment.NewLine}" +
$"{resultMessage}{Environment.NewLine}{Environment.NewLine}" +
$"Check logs for more information.");
}
else
{
logger.LogError($"Application test run crashed. Check logs for more information");
}
return(ExitCode.APP_CRASH);
case TestExecutingResult.TimedOut:
logger.LogWarning($"Application test run timed out");
return(ExitCode.TIMED_OUT);
default:
if (resultMessage != null)
{
logger.LogError($"Application test run ended in an unexpected way: '{testResult}'{Environment.NewLine}" +
$"{resultMessage}{Environment.NewLine}{Environment.NewLine}" +
$"Check logs for more information.");
}
else
{
logger.LogError($"Application test run ended in an unexpected way: '{testResult}'. Check logs for more information");
}
return(ExitCode.GENERAL_FAILURE);
}
}