public void RunOnDeviceWithNoAvailableSimulatorTest()
{
_hardwareDeviceLoader = new Mock <IHardwareDeviceLoader>();
_hardwareDeviceLoader
.Setup(x => x.FindDevice(RunMode.iOS, _mainLog.Object, false, false))
.ThrowsAsync(new NoDeviceFoundException());
// Act
var appRunner = new AppRunner(_processManager.Object,
_hardwareDeviceLoader.Object,
_simulatorLoader.Object,
_listenerFactory,
_snapshotReporterFactory,
Mock.Of <ICaptureLogFactory>(),
Mock.Of <IDeviceLogCapturerFactory>(),
_testReporterFactory,
_mainLog.Object,
_logs.Object,
_helpers.Object);
var appInformation = new AppBundleInformation(AppName, AppBundleIdentifier, s_appPath, s_appPath, null);
Assert.ThrowsAsync <NoDeviceFoundException>(
async() => await appRunner.RunApp(
appInformation,
TestTarget.Device_iOS,
TimeSpan.FromSeconds(30),
TimeSpan.FromSeconds(30),
ensureCleanSimulatorState: true),
"Running requires connected devices");
}
public async Task RunOnDeviceWithNoAvailableSimulatorTest()
{
_hardwareDeviceLoader = new Mock <IHardwareDeviceLoader>();
_hardwareDeviceLoader
.Setup(x => x.FindDevice(RunMode.iOS, _mainLog.Object, false, false))
.ThrowsAsync(new NoDeviceFoundException());
// Act
var appRunner = new AppRunner(_processManager.Object,
_hardwareDeviceLoader.Object,
_simulatorLoader.Object,
_snapshotReporterFactory,
Mock.Of <ICaptureLogFactory>(),
Mock.Of <IDeviceLogCapturerFactory>(),
_mainLog.Object,
_logs.Object,
_helpers.Object,
Enumerable.Empty <string>());
var appInformation = GetMockedAppBundleInfo();
await Assert.ThrowsAsync <NoDeviceFoundException>(
async() => await appRunner.RunApp(
appInformation,
new TestTargetOs(TestTarget.Device_iOS, null),
TimeSpan.FromSeconds(30),
ensureCleanSimulatorState: true));
}
protected override async Task <ExitCode> RunAppInternal(
AppBundleInformation appBundleInfo,
string?deviceName,
ILogger logger,
TestTargetOs target,
Logs logs,
IFileBackedLog mainLog,
CancellationToken cancellationToken)
{
// only add the extra callback if we do know that the feature was indeed enabled
Action <string>?logCallback = IsLldbEnabled() ? (l) => NotifyUserLldbCommand(logger, l) : (Action <string>?)null;
var appRunner = new AppRunner(
ProcessManager,
DeviceLoader,
SimulatorLoader,
new CrashSnapshotReporterFactory(ProcessManager),
new CaptureLogFactory(),
new DeviceLogCapturerFactory(ProcessManager),
new ExitCodeDetector(),
mainLog,
logs,
new Helpers(),
PassThroughArguments,
logCallback);
int?exitCode = null;
(deviceName, exitCode) = await appRunner.RunApp(
appBundleInfo,
target,
_arguments.Timeout,
deviceName,
verbosity : GetMlaunchVerbosity(_arguments.Verbosity),
cancellationToken : cancellationToken);
if (exitCode.HasValue)
{
if (_arguments.ExpectedExitCode != exitCode)
{
logger.LogError($"Application has finished with exit code {exitCode} but {_arguments.ExpectedExitCode} was expected");
return(ExitCode.GENERAL_FAILURE);
}
logger.LogInformation("Application has finished with exit code: " + exitCode + (_arguments.ExpectedExitCode != 0 ? " (as expected)" : null));
return(ExitCode.SUCCESS);
}
else
{
logger.LogError("Application has finished but no system log found. Failed to determine the exit code!");
return(ExitCode.RETURN_CODE_NOT_SET);
}
}
public async Task RunOnSimulatorWithNoAvailableSimulatorTest()
{
_simulatorLoader
.Setup(x => x.FindSimulators(It.Is <TestTargetOs>(t => t.Platform == TestTarget.Simulator_tvOS), _mainLog.Object, It.IsAny <int>(), true, false))
.ThrowsAsync(new NoDeviceFoundException("Failed to find simulator"));
var captureLog = new Mock <ICaptureLog>();
captureLog
.SetupGet(x => x.FullPath)
.Returns(_simulatorLogPath);
var captureLogFactory = new Mock <ICaptureLogFactory>();
captureLogFactory
.Setup(x => x.Create(
Path.Combine(_logs.Object.Directory, "tvos.log"),
"/path/to/simulator.log",
false,
It.IsAny <string>()))
.Returns(captureLog.Object);
// Act
var appRunner = new AppRunner(_processManager.Object,
_hardwareDeviceLoader.Object,
_simulatorLoader.Object,
_snapshotReporterFactory,
captureLogFactory.Object,
Mock.Of <IDeviceLogCapturerFactory>(),
_exitCodeDetector.Object,
_mainLog.Object,
_logs.Object,
_helpers.Object,
Enumerable.Empty <string>());
var appInformation = GetMockedAppBundleInfo();
await Assert.ThrowsAsync <NoDeviceFoundException>(
async() => await appRunner.RunApp(
appInformation,
new TestTargetOs(TestTarget.Simulator_tvOS, null),
TimeSpan.FromSeconds(30)));
// Verify
_mainLog.Verify(x => x.WriteLine("App run ended with 0"), Times.Never);
_simulatorLoader.VerifyAll();
}
public async Task RunOnDeviceWithNoAvailableSimulatorTest()
{
_hardwareDeviceLoader = new Mock <IHardwareDeviceLoader>();
_hardwareDeviceLoader
.Setup(x => x.FindDevice(RunMode.iOS, _mainLog.Object, false, false))
.ThrowsAsync(new NoDeviceFoundException());
_listenerFactory.Setup(f => f.UseTunnel).Returns(false);
// Act
var appRunner = new AppRunner(_processManager.Object,
_hardwareDeviceLoader.Object,
_simulatorLoader.Object,
_listenerFactory.Object,
_snapshotReporterFactory,
Mock.Of <ICaptureLogFactory>(),
Mock.Of <IDeviceLogCapturerFactory>(),
_testReporterFactory,
new XmlResultParser(),
_mainLog.Object,
_logs.Object,
_helpers.Object,
Enumerable.Empty <string>());
var appInformation = new AppBundleInformation(
appName: AppName,
bundleIdentifier: AppBundleIdentifier,
appPath: s_appPath,
launchAppPath: s_appPath,
supports32b: false,
extension: null);
await Assert.ThrowsAsync <NoDeviceFoundException>(
async() => await appRunner.RunApp(
appInformation,
new TestTargetOs(TestTarget.Device_iOS, null),
TimeSpan.FromSeconds(30),
TimeSpan.FromSeconds(30),
ensureCleanSimulatorState: true));
}
public async Task RunOnSimulatorTest()
{
var captureLog = new Mock <ICaptureLog>();
captureLog.SetupGet(x => x.FullPath).Returns(_simulatorLogPath);
captureLog.SetupGet(x => x.Description).Returns(LogType.SystemLog.ToString());
var captureLogFactory = new Mock <ICaptureLogFactory>();
captureLogFactory
.Setup(x => x.Create(
Path.Combine(_logs.Object.Directory, _mockSimulator.Name + ".log"),
_mockSimulator.SystemLog,
false,
LogType.SystemLog))
.Returns(captureLog.Object);
SetupLogList(new[] { captureLog.Object });
// Act
var appRunner = new AppRunner(
_processManager.Object,
_snapshotReporterFactory,
captureLogFactory.Object,
Mock.Of <IDeviceLogCapturerFactory>(),
_mainLog.Object,
_logs.Object,
_helpers.Object);
var result = await appRunner.RunApp(
_appBundleInfo,
new TestTargetOs(TestTarget.Simulator_tvOS, null),
_mockSimulator,
null,
timeout : TimeSpan.FromSeconds(30),
signalAppEnd : false,
waitForExit : true,
extraAppArguments : new[] { "--foo=bar", "--xyz" },
extraEnvVariables : new[] { ("appArg1", "value1") });
public async Task RunOnSimulatorSuccessfullyTest()
{
_simulatorLoader
.Setup(x => x.FindSimulators(It.Is <TestTargetOs>(t => t.Platform == TestTarget.Simulator_tvOS), _mainLog.Object, It.IsAny <int>(), true, false))
.ReturnsAsync((_mockSimulator.Object, null));
var captureLog = new Mock <ICaptureLog>();
captureLog.SetupGet(x => x.FullPath).Returns(_simulatorLogPath);
captureLog.SetupGet(x => x.Description).Returns(LogType.SystemLog.ToString());
var captureLogFactory = new Mock <ICaptureLogFactory>();
captureLogFactory
.Setup(x => x.Create(
Path.Combine(_logs.Object.Directory, _mockSimulator.Object.Name + ".log"),
_mockSimulator.Object.SystemLog,
false,
LogType.SystemLog))
.Returns(captureLog.Object);
SetupLogList(new[] { captureLog.Object });
var appInformation = GetMockedAppBundleInfo();
// Act
var appRunner = new AppRunner(_processManager.Object,
_hardwareDeviceLoader.Object,
_simulatorLoader.Object,
_snapshotReporterFactory,
captureLogFactory.Object,
Mock.Of <IDeviceLogCapturerFactory>(),
_mainLog.Object,
_logs.Object,
_helpers.Object,
new[] { "--appArg1=value1", "-g" });
var(deviceName, result) = await appRunner.RunApp(
appInformation,
new TestTargetOs(TestTarget.Simulator_tvOS, null),
TimeSpan.FromSeconds(30),
ensureCleanSimulatorState : true);
// Verify
Assert.Equal(SimulatorDeviceName, deviceName);
Assert.True(result.Succeeded);
var expectedArgs = GetExpectedSimulatorMlaunchArgs();
_processManager
.Verify(
x => x.ExecuteCommandAsync(
It.Is <MlaunchArguments>(args => args.AsCommandLine() == expectedArgs),
_mainLog.Object,
It.IsAny <TimeSpan>(),
null,
It.IsAny <CancellationToken>()),
Times.Once);
_simulatorLoader.VerifyAll();
captureLog.Verify(x => x.StartCapture(), Times.AtLeastOnce);
// When ensureCleanSimulatorState == true
_mockSimulator.Verify(x => x.PrepareSimulator(_mainLog.Object, AppBundleIdentifier));
_mockSimulator.Verify(x => x.KillEverything(_mainLog.Object));
}
protected override async Task <ExitCode> RunAppInternal(
AppBundleInformation appBundleInfo,
string?deviceName,
ILogger logger,
TestTargetOs target,
Logs logs,
IFileBackedLog mainLog,
CancellationToken cancellationToken)
{
// only add the extra callback if we do know that the feature was indeed enabled
Action <string>?logCallback = IsLldbEnabled() ? (l) => NotifyUserLldbCommand(logger, l) : (Action <string>?)null;
var appRunner = new AppRunner(
ProcessManager,
DeviceLoader,
SimulatorLoader,
new CrashSnapshotReporterFactory(ProcessManager),
new CaptureLogFactory(),
new DeviceLogCapturerFactory(ProcessManager),
mainLog,
logs,
new Helpers(),
PassThroughArguments,
logCallback);
ProcessExecutionResult result;
(deviceName, result) = await appRunner.RunApp(
appBundleInfo,
target,
_arguments.Timeout,
deviceName,
verbosity : GetMlaunchVerbosity(_arguments.Verbosity),
cancellationToken : cancellationToken);
if (!result.Succeeded)
{
if (result.TimedOut)
{
logger.LogError($"App run has timed out");
return(ExitCode.TIMED_OUT);
}
logger.LogError($"App run has failed. mlaunch exited with {result.ExitCode}");
return(ExitCode.APP_LAUNCH_FAILURE);
}
var systemLog = logs.FirstOrDefault(log => log.Description == LogType.SystemLog.ToString());
if (systemLog == null)
{
logger.LogError("Application has finished but no system log found. Failed to determine the exit code!");
return(ExitCode.RETURN_CODE_NOT_SET);
}
var exitCode = new ExitCodeDetector().DetectExitCode(appBundleInfo, systemLog);
logger.LogInformation($"App run ended with {exitCode}");
if (_arguments.ExpectedExitCode != exitCode)
{
logger.LogError($"Application has finished with exit code {exitCode} but {_arguments.ExpectedExitCode} was expected");
return(ExitCode.GENERAL_FAILURE);
}
logger.LogInformation("Application has finished with exit code: " + exitCode +
(_arguments.ExpectedExitCode != 0 ? " (as expected)" : null));
return(ExitCode.SUCCESS);
}
private async Task <ExitCode> RunTest(
ILogger logger,
TestTarget target,
Logs logs,
ProcessManager processManager,
IHardwareDeviceLoader deviceLoader,
ISimulatorLoader simulatorLoader,
ITunnelBore?tunnelBore,
CancellationToken cancellationToken = default)
{
logger.LogInformation($"Starting test for {target.AsString()}{ (_arguments.DeviceName != null ? " targeting " + _arguments.DeviceName : null) }..");
string mainLogFile = Path.Join(_arguments.OutputDirectory, $"run-{target}{(_arguments.DeviceName != null ? "-" + _arguments.DeviceName : null)}.log");
ILog mainLog = logs.Create(mainLogFile, LogType.ExecutionLog.ToString(), true);
int verbosity = GetMlaunchVerbosity(_arguments.Verbosity);
string?deviceName = _arguments.DeviceName;
var appBundleInformationParser = new AppBundleInformationParser(processManager);
AppBundleInformation appBundleInfo;
try
{
appBundleInfo = await appBundleInformationParser.ParseFromAppBundle(_arguments.AppPackagePath, target, mainLog, cancellationToken);
}
catch (Exception e)
{
logger.LogError($"Failed to get bundle information: {e.Message}");
return(ExitCode.FAILED_TO_GET_BUNDLE_INFO);
}
if (!target.IsSimulator())
{
logger.LogInformation($"Installing application '{appBundleInfo.AppName}' on " + (deviceName != null ? $" on device '{deviceName}'" : target.AsString()));
var appInstaller = new AppInstaller(processManager, deviceLoader, mainLog, verbosity);
ProcessExecutionResult result;
try
{
(deviceName, result) = await appInstaller.InstallApp(_arguments.AppPackagePath, target, cancellationToken : cancellationToken);
}
catch (NoDeviceFoundException)
{
logger.LogError($"Failed to find suitable device for target {target.AsString()}");
return(ExitCode.DEVICE_NOT_FOUND);
}
catch (Exception e)
{
logger.LogError($"Failed to install the app bundle:{Environment.NewLine}{e}");
return(ExitCode.PACKAGE_INSTALLATION_FAILURE);
}
if (!result.Succeeded)
{
// use the knowledge base class to decide if the error is known, if it is, let the user know
// the failure reason
if (_errorKnowledgeBase.IsKnownInstallIssue(mainLog, out var errorMessage))
{
logger.LogError($"Failed to install the app bundle (exit code={result.ExitCode}): {errorMessage}");
}
else
{
logger.LogError($"Failed to install the app bundle (exit code={result.ExitCode})");
}
return(ExitCode.PACKAGE_INSTALLATION_FAILURE);
}
logger.LogInformation($"Application '{appBundleInfo.AppName}' was installed successfully on device '{deviceName}'");
}
logger.LogInformation($"Starting application '{appBundleInfo.AppName}' on " + (deviceName != null ? $"device '{deviceName}'" : target.AsString()));
var appRunner = new AppRunner(
processManager,
deviceLoader,
simulatorLoader,
new SimpleListenerFactory(tunnelBore),
new CrashSnapshotReporterFactory(processManager),
new CaptureLogFactory(),
new DeviceLogCapturerFactory(processManager),
new TestReporterFactory(processManager),
mainLog,
logs,
new Helpers(),
useXmlOutput: true); // the cli ALWAYS will get the output as xml
try
{
string resultMessage;
TestExecutingResult testResult;
(deviceName, testResult, resultMessage) = await appRunner.RunApp(appBundleInfo,
target,
_arguments.Timeout,
_arguments.LaunchTimeout,
deviceName,
verbosity : verbosity,
xmlResultJargon : _arguments.XmlResultJargon,
cancellationToken : cancellationToken);
switch (testResult)
{
case TestExecutingResult.Succeeded:
logger.LogInformation($"Application finished the test run successfully");
logger.LogInformation(resultMessage);
return(ExitCode.SUCCESS);
case TestExecutingResult.Failed:
logger.LogInformation($"Application finished the test run successfully with some failed tests");
logger.LogInformation(resultMessage);
return(ExitCode.TESTS_FAILED);
case TestExecutingResult.Crashed:
if (resultMessage != null)
{
logger.LogError($"Application run crashed:{Environment.NewLine}" +
$"{resultMessage}{Environment.NewLine}{Environment.NewLine}" +
$"Check logs for more information.");
}
else
{
logger.LogError($"Application run crashed. Check logs for more information");
}
return(ExitCode.APP_CRASH);
case TestExecutingResult.TimedOut:
logger.LogWarning($"Application run timed out");
return(ExitCode.TIMED_OUT);
default:
if (resultMessage != null)
{
logger.LogError($"Application run ended in an unexpected way: '{testResult}'{Environment.NewLine}" +
$"{resultMessage}{Environment.NewLine}{Environment.NewLine}" +
$"Check logs for more information.");
}
else
{
logger.LogError($"Application run ended in an unexpected way: '{testResult}'. Check logs for more information");
}
return(ExitCode.GENERAL_FAILURE);
}
}
catch (NoDeviceFoundException)
{
logger.LogError($"Failed to find suitable device for target {target.AsString()}");
return(ExitCode.DEVICE_NOT_FOUND);
}
catch (Exception e)
{
if (_errorKnowledgeBase.IsKnownTestIssue(mainLog, out var failureMessage))
{
logger.LogError($"Application run failed:{Environment.NewLine}{failureMessage}");
}
else
{
logger.LogError($"Application run failed:{Environment.NewLine}{e}");
}
return(ExitCode.APP_CRASH);
}
finally
{
if (!target.IsSimulator() && deviceName != null)
{
logger.LogInformation($"Uninstalling the application '{appBundleInfo.AppName}' from device '{deviceName}'");
var appUninstaller = new AppUninstaller(processManager, mainLog, verbosity);
var uninstallResult = await appUninstaller.UninstallApp(deviceName, appBundleInfo.BundleIdentifier, cancellationToken);
if (!uninstallResult.Succeeded)
{
logger.LogError($"Failed to uninstall the app bundle with exit code: {uninstallResult.ExitCode}");
}
else
{
logger.LogInformation($"Application '{appBundleInfo.AppName}' was uninstalled successfully");
}
}
}
}
[InlineData(true, true)] // tunnel and xml
public async Task RunOnDeviceSuccessfullyTest(bool useTunnel, bool useXml)
{
var deviceSystemLog = new Mock <ILog>();
deviceSystemLog.SetupGet(x => x.FullPath).Returns(Path.GetTempFileName());
var deviceLogCapturer = new Mock <IDeviceLogCapturer>();
var deviceLogCapturerFactory = new Mock <IDeviceLogCapturerFactory>();
deviceLogCapturerFactory
.Setup(x => x.Create(_mainLog.Object, deviceSystemLog.Object, "Test iPhone"))
.Returns(deviceLogCapturer.Object);
var testResultFilePath = Path.GetTempFileName();
var listenerLogFile = Mock.Of <ILog>(x => x.FullPath == testResultFilePath);
File.WriteAllLines(testResultFilePath, new[] { "Some result here", "Tests run: 124", "Some result there" });
_logs
.Setup(x => x.Create("test-Device_iOS-mocked_timestamp.log", "TestLog", It.IsAny <bool?>()))
.Returns(listenerLogFile);
_logs
.Setup(x => x.Create("device-Test iPhone-mocked_timestamp.log", "Device log", It.IsAny <bool?>()))
.Returns(deviceSystemLog.Object);
// set tunnel bore expectation
if (useTunnel)
{
_tunnelBore.Setup(t => t.Create("Test iPhone", It.IsAny <ILog>()));
}
_listenerFactory.Setup(f => f.UseTunnel).Returns((useTunnel));
// Act
var appRunner = new AppRunner(_processManager.Object,
_hardwareDeviceLoader.Object,
_simulatorLoader.Object,
_listenerFactory.Object,
_snapshotReporterFactory,
Mock.Of <ICaptureLogFactory>(),
deviceLogCapturerFactory.Object,
_testReporterFactory,
_mainLog.Object,
_logs.Object,
_helpers.Object,
useXml);
var appInformation = new AppBundleInformation(
appName: AppName,
bundleIdentifier: AppBundleIdentifier,
appPath: s_appPath,
launchAppPath: s_appPath,
supports32b: false,
extension: null);
var(deviceName, result, resultMessage) = await appRunner.RunApp(
appInformation,
TestTarget.Device_iOS,
TimeSpan.FromSeconds(30),
TimeSpan.FromSeconds(30));
// Verify
Assert.Equal("Test iPhone", deviceName);
Assert.Equal(TestExecutingResult.Succeeded, result);
Assert.Equal("Tests run: 1194 Passed: 1191 Inconclusive: 0 Failed: 0 Ignored: 0", resultMessage);
var ipAddresses = System.Net.Dns.GetHostEntry(System.Net.Dns.GetHostName()).AddressList.Select(ip => ip.ToString());
var ips = string.Join(",", ipAddresses);
var tunnelParam = useTunnel ? "-setenv=USE_TCP_TUNNEL=true " : "";
var xmlParam = useXml ? "-setenv=NUNIT_ENABLE_XML_OUTPUT=true -setenv=NUNIT_ENABLE_XML_MODE=wrapped -setenv=NUNIT_XML_VERSION=xUnit " : "";
var expectedArgs = $"-argument=-connection-mode -argument=none -argument=-app-arg:-autostart " +
$"-setenv=NUNIT_AUTOSTART=true -argument=-app-arg:-autoexit -setenv=NUNIT_AUTOEXIT=true " +
$"-argument=-app-arg:-enablenetwork -setenv=NUNIT_ENABLE_NETWORK=true -setenv=DISABLE_SYSTEM_PERMISSION_TESTS=1 -v -v " +
$"-argument=-app-arg:-hostname:{ips} -setenv=NUNIT_HOSTNAME={ips} -argument=-app-arg:-transport:Tcp " +
$"-setenv=NUNIT_TRANSPORT=TCP -argument=-app-arg:-hostport:{_listener.Object.Port} " +
$"-setenv=NUNIT_HOSTPORT={_listener.Object.Port} {tunnelParam}{xmlParam}--launchdev {StringUtils.FormatArguments(s_appPath)} " +
$"--disable-memory-limits --wait-for-exit --devname \"Test iPhone\"";
_processManager
.Verify(
x => x.ExecuteCommandAsync(
It.Is <MlaunchArguments>(args => args.AsCommandLine() == expectedArgs),
It.IsAny <ILog>(),
It.IsAny <TimeSpan>(),
null,
It.IsAny <CancellationToken>()),
Times.Once);
_listener.Verify(x => x.Initialize(), Times.AtLeastOnce);
_listener.Verify(x => x.StartAsync(), Times.AtLeastOnce);
_listener.Verify(x => x.Cancel(), Times.AtLeastOnce);
_listener.Verify(x => x.Dispose(), Times.AtLeastOnce);
// verify that we do close the tunnel when it was used
// we dont want to leak a process
if (useTunnel)
{
_tunnelBore.Verify(t => t.Close("Test iPhone"));
}
_hardwareDeviceLoader.VerifyAll();
_snapshotReporter.Verify(x => x.StartCaptureAsync(), Times.AtLeastOnce);
_snapshotReporter.Verify(x => x.StartCaptureAsync(), Times.AtLeastOnce);
deviceSystemLog.Verify(x => x.Dispose(), Times.AtLeastOnce);
}
[InlineData(true, true)] // tunnel and xml
public async Task RunOnSimulatorSuccessfullyTest(bool useTunnel, bool useXml)
{
string simulatorLogPath = Path.Combine(Path.GetTempPath(), "simulator-logs");
var simulator = new Mock <ISimulatorDevice>();
simulator.SetupGet(x => x.Name).Returns("Test iPhone simulator");
simulator.SetupGet(x => x.UDID).Returns("58F21118E4D34FD69EAB7860BB9B38A0");
simulator.SetupGet(x => x.LogPath).Returns(simulatorLogPath);
simulator.SetupGet(x => x.SystemLog).Returns(Path.Combine(simulatorLogPath, "system.log"));
_simulatorLoader
.Setup(x => x.FindSimulators(TestTarget.Simulator_tvOS, _mainLog.Object, true, false))
.ReturnsAsync(new ISimulatorDevice[] { simulator.Object });
var testResultFilePath = Path.GetTempFileName();
var listenerLogFile = Mock.Of <ILog>(x => x.FullPath == testResultFilePath);
File.WriteAllLines(testResultFilePath, new[] { "Some result here", "Tests run: 124", "Some result there" });
_logs
.Setup(x => x.Create("test-Simulator_tvOS-mocked_timestamp.log", "TestLog", It.IsAny <bool?>()))
.Returns(listenerLogFile);
var captureLog = new Mock <ICaptureLog>();
captureLog.SetupGet(x => x.FullPath).Returns(simulatorLogPath);
var captureLogFactory = new Mock <ICaptureLogFactory>();
captureLogFactory
.Setup(x => x.Create(
Path.Combine(_logs.Object.Directory, simulator.Object.Name + ".log"),
simulator.Object.SystemLog,
true,
It.IsAny <string>()))
.Returns(captureLog.Object);
_listenerFactory.Setup(f => f.UseTunnel).Returns((useTunnel));
// Act
var appRunner = new AppRunner(_processManager.Object,
_hardwareDeviceLoader.Object,
_simulatorLoader.Object,
_listenerFactory.Object,
_snapshotReporterFactory,
captureLogFactory.Object,
Mock.Of <IDeviceLogCapturerFactory>(),
_testReporterFactory,
_mainLog.Object,
_logs.Object,
_helpers.Object,
useXml);
var appInformation = new AppBundleInformation(
appName: AppName,
bundleIdentifier: AppBundleIdentifier,
appPath: s_appPath,
launchAppPath: s_appPath,
supports32b: false,
extension: null);
var(deviceName, result, resultMessage) = await appRunner.RunApp(
appInformation,
TestTarget.Simulator_tvOS,
TimeSpan.FromSeconds(30),
TimeSpan.FromSeconds(30),
ensureCleanSimulatorState : true);
// Verify
Assert.Equal("Test iPhone simulator", deviceName);
Assert.Equal(TestExecutingResult.Succeeded, result);
Assert.Equal("Tests run: 1194 Passed: 1191 Inconclusive: 0 Failed: 0 Ignored: 0", resultMessage);
var xmlParam = useXml ? "-setenv=NUNIT_ENABLE_XML_OUTPUT=true -setenv=NUNIT_ENABLE_XML_MODE=wrapped -setenv=NUNIT_XML_VERSION=xUnit " : "";
var expectedArgs = $"-argument=-connection-mode -argument=none -argument=-app-arg:-autostart " +
$"-setenv=NUNIT_AUTOSTART=true -argument=-app-arg:-autoexit -setenv=NUNIT_AUTOEXIT=true " +
$"-argument=-app-arg:-enablenetwork -setenv=NUNIT_ENABLE_NETWORK=true -setenv=DISABLE_SYSTEM_PERMISSION_TESTS=1 -v -v " +
$"-argument=-app-arg:-hostname:127.0.0.1 -setenv=NUNIT_HOSTNAME=127.0.0.1 -argument=-app-arg:-transport:Tcp " +
$"-setenv=NUNIT_TRANSPORT=TCP -argument=-app-arg:-hostport:{_listener.Object.Port} " +
$"-setenv=NUNIT_HOSTPORT={_listener.Object.Port} {xmlParam}--launchsim {StringUtils.FormatArguments(s_appPath)} " +
$"--stderr=tty1 --device=:v2:udid={simulator.Object.UDID}";
_processManager
.Verify(
x => x.ExecuteCommandAsync(
It.Is <MlaunchArguments>(args => args.AsCommandLine() == expectedArgs),
_mainLog.Object,
It.IsAny <TimeSpan>(),
null,
It.IsAny <CancellationToken>()),
Times.Once);
_listener.Verify(x => x.Initialize(), Times.AtLeastOnce);
_listener.Verify(x => x.StartAsync(), Times.AtLeastOnce);
_listener.Verify(x => x.Cancel(), Times.AtLeastOnce);
_listener.Verify(x => x.Dispose(), Times.AtLeastOnce);
_simulatorLoader.VerifyAll();
captureLog.Verify(x => x.StartCapture(), Times.AtLeastOnce);
captureLog.Verify(x => x.StopCapture(), Times.AtLeastOnce);
// When ensureCleanSimulatorState == true
simulator.Verify(x => x.PrepareSimulator(_mainLog.Object, AppBundleIdentifier));
simulator.Verify(x => x.KillEverything(_mainLog.Object));
}
[InlineData(true, true)] // tunnel and xml
public async Task RunOnSimulatorWithNoAvailableSimulatorTest(bool useTcpTunnel, bool useXmlOutput)
{
// Mock finding simulators
string simulatorLogPath = Path.Combine(Path.GetTempPath(), "simulator-logs");
_simulatorLoader
.Setup(x => x.FindSimulators(TestTarget.Simulator_tvOS, _mainLog.Object, true, false))
.ReturnsAsync(new ISimulatorDevice[0]);
var listenerLogFile = new Mock <ILog>();
_logs
.Setup(x => x.Create(It.IsAny <string>(), "TestLog", It.IsAny <bool>()))
.Returns(listenerLogFile.Object);
var captureLog = new Mock <ICaptureLog>();
captureLog
.SetupGet(x => x.FullPath)
.Returns(simulatorLogPath);
var captureLogFactory = new Mock <ICaptureLogFactory>();
captureLogFactory
.Setup(x => x.Create(
Path.Combine(_logs.Object.Directory, "tvos.log"),
"/path/to/simulator.log",
true,
It.IsAny <string>()))
.Returns(captureLog.Object);
_listenerFactory.Setup(f => f.UseTunnel).Returns(useTcpTunnel);
// Act
var appRunner = new AppRunner(_processManager.Object,
_hardwareDeviceLoader.Object,
_simulatorLoader.Object,
_listenerFactory.Object,
_snapshotReporterFactory,
captureLogFactory.Object,
Mock.Of <IDeviceLogCapturerFactory>(),
_testReporterFactory,
_mainLog.Object,
_logs.Object,
_helpers.Object,
useXmlOutput);
var appInformation = new AppBundleInformation(
appName: AppName,
bundleIdentifier: AppBundleIdentifier,
appPath: s_appPath,
launchAppPath: s_appPath,
supports32b: false,
extension: null);
await Assert.ThrowsAsync <NoDeviceFoundException>(
async() => await appRunner.RunApp(
appInformation,
TestTarget.Simulator_tvOS,
TimeSpan.FromSeconds(30),
TimeSpan.FromSeconds(30)));
// Verify
_mainLog.Verify(x => x.WriteLine("Test run completed"), Times.Never);
_simulatorLoader.VerifyAll();
_listener.Verify(x => x.Initialize(), Times.AtLeastOnce);
_listener.Verify(x => x.StartAsync(), Times.AtLeastOnce);
_tunnelBore.Verify(t => t.Create(It.IsAny <string>(), It.IsAny <ILog>()), Times.Never); // never create tunnels on simulators
}
private async Task <ExitCode> RunTest(TestTarget target,
Logs logs,
ProcessManager processManager,
IHardwareDeviceLoader deviceLoader,
ISimulatorLoader simulatorLoader,
CancellationToken cancellationToken = default)
{
_log.LogInformation($"Starting test for {target.AsString()}{ (_arguments.DeviceName != null ? " targeting " + _arguments.DeviceName : null) }..");
string mainLogFile = Path.Join(_arguments.OutputDirectory, $"run-{target}{(_arguments.DeviceName != null ? "-" + _arguments.DeviceName : null)}.log");
ILog mainLog = logs.Create(mainLogFile, LogType.ExecutionLog.ToString(), true);
int verbosity = _arguments.Verbosity.ToInt();
string deviceName = _arguments.DeviceName;
var appBundleInformationParser = new AppBundleInformationParser(processManager);
AppBundleInformation appBundleInfo;
try
{
appBundleInfo = await appBundleInformationParser.ParseFromAppBundle(_arguments.AppPackagePath, target, mainLog, cancellationToken);
}
catch (Exception e)
{
_log.LogError($"Failed to get bundle information: {e.Message}");
return(ExitCode.FAILED_TO_GET_BUNDLE_INFO);
}
if (!target.IsSimulator())
{
_log.LogInformation($"Installing application '{appBundleInfo.AppName}' on " + (deviceName != null ? " on device '{deviceName}'" : target.AsString()));
var appInstaller = new AppInstaller(processManager, deviceLoader, mainLog, verbosity);
ProcessExecutionResult result;
try
{
(deviceName, result) = await appInstaller.InstallApp(_arguments.AppPackagePath, target, cancellationToken : cancellationToken);
}
catch (NoDeviceFoundException)
{
_log.LogError($"Failed to find suitable device for target {target.AsString()}");
return(ExitCode.DEVICE_NOT_FOUND);
}
catch (Exception e)
{
_log.LogError($"Failed to install the app bundle:{Environment.NewLine}{e}");
return(ExitCode.PACKAGE_INSTALLATION_FAILURE);
}
if (!result.Succeeded)
{
_log.LogError($"Failed to install the app bundle (exit code={result.ExitCode})");
return(ExitCode.PACKAGE_INSTALLATION_FAILURE);
}
_log.LogInformation($"Application '{appBundleInfo.AppName}' was installed successfully on device '{deviceName}'");
}
_log.LogInformation($"Starting application '{appBundleInfo.AppName}' on " + (deviceName != null ? " on device '{deviceName}'" : target.AsString()));
int exitCode;
try
{
var appRunner = new AppRunner(
processManager,
deviceLoader,
simulatorLoader,
new SimpleListenerFactory(),
new CrashSnapshotReporterFactory(processManager),
new CaptureLogFactory(),
new DeviceLogCapturerFactory(processManager),
new TestReporterFactory(processManager),
mainLog,
logs,
new Helpers());
(deviceName, exitCode) = await appRunner.RunApp(appBundleInfo,
target,
_arguments.Timeout,
_arguments.LaunchTimeout,
deviceName,
verbosity : verbosity,
xmlResultJargon : XmlResultJargon.xUnit);
if (exitCode != 0)
{
_log.LogError($"App bundle run failed with exit code {exitCode}. Check logs for more information");
}
else
{
_log.LogInformation("Application finished the run successfully");
}
return(0);
}
catch (NoDeviceFoundException)
{
_log.LogError($"Failed to find suitable device for target {target.AsString()}");
return(ExitCode.DEVICE_NOT_FOUND);
}
catch (Exception e)
{
_log.LogError($"Application run failed:{Environment.NewLine}{e}");
return(ExitCode.APP_CRASH);
}
finally
{
if (!target.IsSimulator())
{
_log.LogInformation($"Uninstalling the application '{appBundleInfo.AppName}' from device '{deviceName}'");
var appUninstaller = new AppUninstaller(processManager, mainLog, verbosity);
var uninstallResult = await appUninstaller.UninstallApp(deviceName, appBundleInfo.BundleIdentifier, cancellationToken);
if (!uninstallResult.Succeeded)
{
_log.LogError($"Failed to uninstall the app bundle with exit code: {uninstallResult.ExitCode}");
}
else
{
_log.LogInformation($"Application '{appBundleInfo.AppName}' was uninstalled successfully");
}
}
}
}
public async Task RunOnDeviceSuccessfullyTest()
{
var deviceSystemLog = new Mock <ILog>();
deviceSystemLog.SetupGet(x => x.FullPath).Returns(Path.GetTempFileName());
var deviceLogCapturer = new Mock <IDeviceLogCapturer>();
var deviceLogCapturerFactory = new Mock <IDeviceLogCapturerFactory>();
deviceLogCapturerFactory
.Setup(x => x.Create(_mainLog.Object, deviceSystemLog.Object, "Test iPhone"))
.Returns(deviceLogCapturer.Object);
var testResultFilePath = Path.GetTempFileName();
var listenerLogFile = Mock.Of <ILog>(x => x.FullPath == testResultFilePath);
File.WriteAllLines(testResultFilePath, new[] { "Some result here", "Tests run: 124", "Some result there" });
_logs
.Setup(x => x.Create("test-Device_iOS-mocked_timestamp.log", "TestLog", It.IsAny <bool?>()))
.Returns(listenerLogFile);
_logs
.Setup(x => x.Create("device-Test iPhone-mocked_timestamp.log", "Device log", It.IsAny <bool?>()))
.Returns(deviceSystemLog.Object);
// Act
var appRunner = new AppRunner(_processManager.Object,
_hardwareDeviceLoader.Object,
_simulatorLoader.Object,
_listenerFactory,
_snapshotReporterFactory,
Mock.Of <ICaptureLogFactory>(),
deviceLogCapturerFactory.Object,
_testReporterFactory,
_mainLog.Object,
_logs.Object,
_helpers.Object);
var appInformation = new AppBundleInformation(AppName, AppBundleIdentifier, s_appPath, s_appPath, null);
var(deviceName, exitCode) = await appRunner.RunApp(
appInformation,
TestTarget.Device_iOS,
TimeSpan.FromSeconds(30),
TimeSpan.FromSeconds(30));
// Verify
Assert.AreEqual("Test iPhone", deviceName);
Assert.AreEqual(0, exitCode);
var ips = string.Join(",", System.Net.Dns.GetHostEntry(System.Net.Dns.GetHostName()).AddressList.Select(ip => ip.ToString()));
var expectedArgs = $"-argument=-connection-mode -argument=none -argument=-app-arg:-autostart " +
$"-setenv=NUNIT_AUTOSTART=true -argument=-app-arg:-autoexit -setenv=NUNIT_AUTOEXIT=true " +
$"-argument=-app-arg:-enablenetwork -setenv=NUNIT_ENABLE_NETWORK=true -setenv=DISABLE_SYSTEM_PERMISSION_TESTS=1 -v -v " +
$"-argument=-app-arg:-hostname:{ips} -setenv=NUNIT_HOSTNAME={ips} -argument=-app-arg:-transport:Tcp " +
$"-setenv=NUNIT_TRANSPORT=TCP -argument=-app-arg:-hostport:{_listener.Object.Port} " +
$"-setenv=NUNIT_HOSTPORT={_listener.Object.Port} --launchdev {StringUtils.FormatArguments(s_appPath)} " +
$"--disable-memory-limits --wait-for-exit --devname \"Test iPhone\"";
_processManager
.Verify(
x => x.ExecuteCommandAsync(
It.Is <MlaunchArguments>(args => args.AsCommandLine() == expectedArgs),
It.IsAny <ILog>(),
It.IsAny <TimeSpan>(),
null,
It.IsAny <CancellationToken>()),
Times.Once);
_listener.Verify(x => x.Initialize(), Times.AtLeastOnce);
_listener.Verify(x => x.StartAsync(), Times.AtLeastOnce);
_listener.Verify(x => x.Cancel(), Times.AtLeastOnce);
_listener.Verify(x => x.Dispose(), Times.AtLeastOnce);
_hardwareDeviceLoader.VerifyAll();
_snapshotReporter.Verify(x => x.StartCaptureAsync(), Times.AtLeastOnce);
_snapshotReporter.Verify(x => x.StartCaptureAsync(), Times.AtLeastOnce);
deviceSystemLog.Verify(x => x.Dispose(), Times.AtLeastOnce);
}
public void RunOnSimulatorWithNoAvailableSimulatorTest()
{
// Mock finding simulators
string simulatorLogPath = Path.Combine(Path.GetTempPath(), "simulator-logs");
_simulatorLoader
.Setup(x => x.FindSimulators(TestTarget.Simulator_tvOS, _mainLog.Object, true, false))
.ReturnsAsync(new ISimulatorDevice[0]);
var listenerLogFile = new Mock <ILog>();
_logs
.Setup(x => x.Create(It.IsAny <string>(), "TestLog", It.IsAny <bool>()))
.Returns(listenerLogFile.Object);
var captureLog = new Mock <ICaptureLog>();
captureLog
.SetupGet(x => x.FullPath)
.Returns(simulatorLogPath);
var captureLogFactory = new Mock <ICaptureLogFactory>();
captureLogFactory
.Setup(x => x.Create(
Path.Combine(_logs.Object.Directory, "tvos.log"),
"/path/to/simulator.log",
true,
It.IsAny <string>()))
.Returns(captureLog.Object);
// Act
var appRunner = new AppRunner(_processManager.Object,
_hardwareDeviceLoader.Object,
_simulatorLoader.Object,
_listenerFactory,
_snapshotReporterFactory,
captureLogFactory.Object,
Mock.Of <IDeviceLogCapturerFactory>(),
_testReporterFactory,
_mainLog.Object,
_logs.Object,
_helpers.Object);
var appInformation = new AppBundleInformation(AppName, AppBundleIdentifier, s_appPath, s_appPath, null);
Assert.ThrowsAsync <NoDeviceFoundException>(
async() => await appRunner.RunApp(
appInformation,
TestTarget.Simulator_tvOS,
TimeSpan.FromSeconds(30),
TimeSpan.FromSeconds(30)),
"Running requires installed simulators");
// Verify
_mainLog.Verify(x => x.WriteLine("Test run completed"), Times.Never);
_simulatorLoader.VerifyAll();
_listener.Verify(x => x.Initialize(), Times.AtLeastOnce);
_listener.Verify(x => x.StartAsync(), Times.AtLeastOnce);
}
public async Task RunOnDeviceWithSkippedClassesTestTest(params string[] skippedClasses)
{
var deviceSystemLog = new Mock <IFileBackedLog>();
deviceSystemLog.SetupGet(x => x.FullPath).Returns(Path.GetTempFileName());
var deviceLogCapturer = new Mock <IDeviceLogCapturer>();
var deviceLogCapturerFactory = new Mock <IDeviceLogCapturerFactory>();
deviceLogCapturerFactory
.Setup(x => x.Create(_mainLog.Object, deviceSystemLog.Object, "Test iPhone"))
.Returns(deviceLogCapturer.Object);
var testResultFilePath = Path.GetTempFileName();
var listenerLogFile = Mock.Of <IFileBackedLog>(x => x.FullPath == testResultFilePath);
File.WriteAllLines(testResultFilePath, new[] { "Some result here", "Tests run: 124", "Some result there" });
_logs
.Setup(x => x.Create("test-Device_iOS-mocked_timestamp.log", "TestLog", It.IsAny <bool?>()))
.Returns(listenerLogFile);
_logs
.Setup(x => x.Create("device-Test iPhone-mocked_timestamp.log", "Device log", It.IsAny <bool?>()))
.Returns(deviceSystemLog.Object);
// Act
var appRunner = new AppRunner(_processManager.Object,
_hardwareDeviceLoader.Object,
_simulatorLoader.Object,
_listenerFactory.Object,
_snapshotReporterFactory,
Mock.Of <ICaptureLogFactory>(),
deviceLogCapturerFactory.Object,
_testReporterFactory,
new XmlResultParser(),
_mainLog.Object,
_logs.Object,
_helpers.Object,
Enumerable.Empty <string>());
var appInformation = new AppBundleInformation(
appName: AppName,
bundleIdentifier: AppBundleIdentifier,
appPath: s_appPath,
launchAppPath: s_appPath,
supports32b: false,
extension: null);
var(deviceName, result, resultMessage) = await appRunner.RunApp(
appInformation,
new TestTargetOs(TestTarget.Device_iOS, null),
timeout : TimeSpan.FromSeconds(30),
testLaunchTimeout : TimeSpan.FromSeconds(30),
skippedTestClasses : skippedClasses);
// Verify
Assert.Equal("Test iPhone", deviceName);
Assert.Equal(TestExecutingResult.Succeeded, result);
Assert.Equal("Tests run: 1194 Passed: 1191 Inconclusive: 0 Failed: 0 Ignored: 0", resultMessage);
var skippedTestsArg = $"-setenv=NUNIT_RUN_ALL=false -setenv=NUNIT_SKIPPED_CLASSES={string.Join(',', skippedClasses)} ";
var expectedArgs = "-argument=-connection-mode " +
"-argument=none " +
"-argument=-app-arg:-autostart " +
"-setenv=NUNIT_AUTOSTART=true " +
"-argument=-app-arg:-autoexit " +
"-setenv=NUNIT_AUTOEXIT=true " +
"-argument=-app-arg:-enablenetwork " +
"-setenv=NUNIT_ENABLE_NETWORK=true " +
"-setenv=DISABLE_SYSTEM_PERMISSION_TESTS=1 " +
skippedTestsArg +
"-v " +
"-v " +
"-setenv=NUNIT_ENABLE_XML_OUTPUT=true " +
"-setenv=NUNIT_ENABLE_XML_MODE=wrapped " +
"-setenv=NUNIT_XML_VERSION=xUnit " +
"-argument=-app-arg:-transport:Tcp " +
"-setenv=NUNIT_TRANSPORT=TCP " +
$"-argument=-app-arg:-hostport:{Port} " +
$"-setenv=NUNIT_HOSTPORT={Port} " +
"-argument=-app-arg:-hostname:127.0.0.1,::1 " +
"-setenv=NUNIT_HOSTNAME=127.0.0.1,::1 " +
"--disable-memory-limits " +
"--devname \"Test iPhone\" " +
$"--launchdev {StringUtils.FormatArguments(s_appPath)} " +
"--wait-for-exit";
_processManager
.Verify(
x => x.ExecuteCommandAsync(
It.Is <MlaunchArguments>(args => args.AsCommandLine() == expectedArgs),
It.IsAny <ILog>(),
It.IsAny <TimeSpan>(),
null,
It.IsAny <CancellationToken>()),
Times.Once);
_listener.Verify(x => x.InitializeAndGetPort(), Times.AtLeastOnce);
_listener.Verify(x => x.StartAsync(), Times.AtLeastOnce);
_listener.Verify(x => x.Cancel(), Times.AtLeastOnce);
_listener.Verify(x => x.Dispose(), Times.AtLeastOnce);
_hardwareDeviceLoader.VerifyAll();
_snapshotReporter.Verify(x => x.StartCaptureAsync(), Times.AtLeastOnce);
_snapshotReporter.Verify(x => x.StartCaptureAsync(), Times.AtLeastOnce);
deviceSystemLog.Verify(x => x.Dispose(), Times.AtLeastOnce);
}
public async Task RunOnDeviceSuccessfullyTest()
{
var deviceSystemLog = new Mock <IFileBackedLog>();
deviceSystemLog.SetupGet(x => x.FullPath).Returns(Path.GetTempFileName());
deviceSystemLog.SetupGet(x => x.Description).Returns(LogType.SystemLog.ToString());
SetupLogList(new[] { deviceSystemLog.Object });
_logs
.Setup(x => x.Create("device-" + DeviceName + "-mocked_timestamp.log", LogType.SystemLog.ToString(), It.IsAny <bool?>()))
.Returns(deviceSystemLog.Object);
var deviceLogCapturer = new Mock <IDeviceLogCapturer>();
var deviceLogCapturerFactory = new Mock <IDeviceLogCapturerFactory>();
deviceLogCapturerFactory
.Setup(x => x.Create(_mainLog.Object, deviceSystemLog.Object, DeviceName))
.Returns(deviceLogCapturer.Object);
var x = _logs.Object.First();
var appInformation = GetMockedAppBundleInfo();
// Act
var appRunner = new AppRunner(_processManager.Object,
_hardwareDeviceLoader.Object,
_simulatorLoader.Object,
_snapshotReporterFactory,
Mock.Of <ICaptureLogFactory>(),
deviceLogCapturerFactory.Object,
_mainLog.Object,
_logs.Object,
_helpers.Object,
new[] { "--appArg1=value1", "-g" });
var(deviceName, result) = await appRunner.RunApp(
appInformation,
new TestTargetOs(TestTarget.Device_iOS, null),
TimeSpan.FromSeconds(30));
// Verify
Assert.Equal(DeviceName, deviceName);
Assert.True(result.Succeeded);
var expectedArgs = GetExpectedDeviceMlaunchArgs();
_processManager
.Verify(
x => x.ExecuteCommandAsync(
It.Is <MlaunchArguments>(args => args.AsCommandLine() == expectedArgs),
It.IsAny <ILog>(),
It.IsAny <TimeSpan>(),
null,
It.IsAny <CancellationToken>()),
Times.Once);
_hardwareDeviceLoader.VerifyAll();
_snapshotReporter.Verify(x => x.StartCaptureAsync(), Times.AtLeastOnce);
_snapshotReporter.Verify(x => x.StartCaptureAsync(), Times.AtLeastOnce);
deviceSystemLog.Verify(x => x.Dispose(), Times.AtLeastOnce);
}
private async Task <ExitCode> RunTest(
ILogger logger,
TestTargetOs target,
Logs logs,
MLaunchProcessManager processManager,
IHardwareDeviceLoader deviceLoader,
ISimulatorLoader simulatorLoader,
ITunnelBore?tunnelBore,
CancellationToken cancellationToken = default)
{
var isLldbEnabled = IsLldbEnabled();
if (isLldbEnabled && !_arguments.EnableLldb)
{
// the file is present, but the user did not set it, warn him about it
logger.LogWarning("Lldb will be used since '~/.mtouch-launch-with-lldb' was found in the system but it was not created by xharness.");
}
else if (_arguments.EnableLldb)
{
if (!File.Exists(s_mlaunchLldbConfigFile))
{
// create empty file
File.WriteAllText(s_mlaunchLldbConfigFile, string.Empty);
_createdLldbFile = true;
}
}
logger.LogInformation($"Starting test for {target.AsString()}{ (_arguments.DeviceName != null ? " targeting " + _arguments.DeviceName : null) }..");
string mainLogFile = Path.Join(_arguments.OutputDirectory, $"run-{target.AsString()}{(_arguments.DeviceName != null ? "-" + _arguments.DeviceName : null)}.log");
IFileBackedLog mainLog = Log.CreateReadableAggregatedLog(
logs.Create(mainLogFile, LogType.ExecutionLog.ToString(), true),
new CallbackLog(message => logger.LogDebug(message.Trim()))
{
Timestamp = false
});
int verbosity = GetMlaunchVerbosity(_arguments.Verbosity);
string?deviceName = _arguments.DeviceName;
var appBundleInformationParser = new AppBundleInformationParser(processManager);
AppBundleInformation appBundleInfo;
try
{
appBundleInfo = await appBundleInformationParser.ParseFromAppBundle(_arguments.AppPackagePath, target.Platform, mainLog, cancellationToken);
}
catch (Exception e)
{
logger.LogError($"Failed to get bundle information: {e.Message}");
return(ExitCode.FAILED_TO_GET_BUNDLE_INFO);
}
if (!target.Platform.IsSimulator())
{
logger.LogInformation($"Installing application '{appBundleInfo.AppName}' on " + (deviceName != null ? $" on device '{deviceName}'" : target.AsString()));
var appInstaller = new AppInstaller(processManager, deviceLoader, mainLog, verbosity);
ProcessExecutionResult result;
try
{
(deviceName, result) = await appInstaller.InstallApp(appBundleInfo, target, cancellationToken : cancellationToken);
}
catch (NoDeviceFoundException)
{
logger.LogError($"Failed to find suitable device for target {target.AsString()}" + Environment.NewLine +
"Please make sure the device is connected and unlocked.");
return(ExitCode.DEVICE_NOT_FOUND);
}
catch (Exception e)
{
logger.LogError($"Failed to install the app bundle:{Environment.NewLine}{e}");
return(ExitCode.PACKAGE_INSTALLATION_FAILURE);
}
if (!result.Succeeded)
{
// use the knowledge base class to decide if the error is known, if it is, let the user know
// the failure reason
if (_errorKnowledgeBase.IsKnownInstallIssue(mainLog, out var errorMessage))
{
var msg = $"Failed to install the app bundle (exit code={result.ExitCode}): {errorMessage.Value.HumanMessage}.";
if (errorMessage.Value.IssueLink != null)
{
msg += $" Find more information at {errorMessage.Value.IssueLink}";
}
logger.LogError(msg);
}
else
{
logger.LogError($"Failed to install the app bundle (exit code={result.ExitCode})");
}
return(ExitCode.PACKAGE_INSTALLATION_FAILURE);
}
logger.LogInformation($"Application '{appBundleInfo.AppName}' was installed successfully on device '{deviceName}'");
}
logger.LogInformation($"Starting application '{appBundleInfo.AppName}' on " + (deviceName != null ? $"device '{deviceName}'" : target.AsString()));
// only add the extra callback if we do know that the feature was indeed enabled
Action <string>?logCallback = isLldbEnabled ? (l) => NotifyUserLldbCommand(logger, l) : (Action <string>?)null;
var appRunner = new AppRunner(
processManager,
deviceLoader,
simulatorLoader,
new SimpleListenerFactory(tunnelBore),
new CrashSnapshotReporterFactory(processManager),
new CaptureLogFactory(),
new DeviceLogCapturerFactory(processManager),
new TestReporterFactory(processManager),
new XmlResultParser(),
mainLog,
logs,
new Helpers(),
logCallback: logCallback,
appArguments: PassThroughArguments);
try
{
string resultMessage;
TestExecutingResult testResult;
(deviceName, testResult, resultMessage) = await appRunner.RunApp(appBundleInfo,
target,
_arguments.Timeout,
_arguments.LaunchTimeout,
deviceName,
verbosity : verbosity,
xmlResultJargon : _arguments.XmlResultJargon,
cancellationToken : cancellationToken,
skippedMethods : _arguments.SingleMethodFilters?.ToArray(),
skippedTestClasses : _arguments.ClassMethodFilters?.ToArray());
switch (testResult)
{
case TestExecutingResult.Succeeded:
logger.LogInformation($"Application finished the test run successfully");
logger.LogInformation(resultMessage);
return(ExitCode.SUCCESS);
case TestExecutingResult.Failed:
logger.LogInformation($"Application finished the test run successfully with some failed tests");
logger.LogInformation(resultMessage);
return(ExitCode.TESTS_FAILED);
case TestExecutingResult.LaunchFailure:
if (resultMessage != null)
{
logger.LogError($"Failed to launch the application:{Environment.NewLine}" +
$"{resultMessage}{Environment.NewLine}{Environment.NewLine}" +
$"Check logs for more information.");
}
else
{
logger.LogError($"Failed to launch the application. Check logs for more information");
}
return(ExitCode.APP_LAUNCH_FAILURE);
case TestExecutingResult.Crashed:
if (resultMessage != null)
{
logger.LogError($"Application run crashed:{Environment.NewLine}" +
$"{resultMessage}{Environment.NewLine}{Environment.NewLine}" +
$"Check logs for more information.");
}
else
{
logger.LogError($"Application run crashed. Check logs for more information");
}
return(ExitCode.APP_CRASH);
case TestExecutingResult.TimedOut:
logger.LogWarning($"Application run timed out");
return(ExitCode.TIMED_OUT);
default:
if (resultMessage != null)
{
logger.LogError($"Application run ended in an unexpected way: '{testResult}'{Environment.NewLine}" +
$"{resultMessage}{Environment.NewLine}{Environment.NewLine}" +
$"Check logs for more information.");
}
else
{
logger.LogError($"Application run ended in an unexpected way: '{testResult}'. Check logs for more information");
}
return(ExitCode.GENERAL_FAILURE);
}
}
catch (NoDeviceFoundException)
{
logger.LogError($"Failed to find suitable device for target {target.AsString()}" +
(target.Platform.IsSimulator() ? Environment.NewLine + "Please make sure suitable Simulator is installed in Xcode" : string.Empty));
return(ExitCode.DEVICE_NOT_FOUND);
}
catch (Exception e)
{
if (_errorKnowledgeBase.IsKnownTestIssue(mainLog, out var failureMessage))
{
var msg = $"Application run failed:{Environment.NewLine}{failureMessage.Value.HumanMessage}.";
if (failureMessage.Value.IssueLink != null)
{
msg += $" Find more information at {failureMessage.Value.IssueLink}";
}
logger.LogError(msg);
}
else
{
logger.LogError($"Application run failed:{Environment.NewLine}{e}");
}
return(ExitCode.APP_CRASH);
}
finally
{
mainLog.Dispose();
if (!target.Platform.IsSimulator() && deviceName != null)
{
logger.LogInformation($"Uninstalling the application '{appBundleInfo.AppName}' from device '{deviceName}'");
var appUninstaller = new AppUninstaller(processManager, mainLog, verbosity);
var uninstallResult = await appUninstaller.UninstallApp(deviceName, appBundleInfo.BundleIdentifier, cancellationToken);
if (!uninstallResult.Succeeded)
{
logger.LogError($"Failed to uninstall the app bundle with exit code: {uninstallResult.ExitCode}");
}
else
{
logger.LogInformation($"Application '{appBundleInfo.AppName}' was uninstalled successfully");
}
}
if (_createdLldbFile) // clean after the setting
{
File.Delete(s_mlaunchLldbConfigFile);
}
}
}