protected async Task <AppBundleInformation> GetAppBundleFromId(TestTargetOs target, IDevice device, string bundleIdentifier, CancellationToken cancellationToken)
{
// We can exchange bundle ID for path where the bundle is on the simulator and get that
if (device is ISimulatorDevice simulator)
{
_logger.LogInformation($"Querying simulator for app bundle information..");
await simulator.Boot(_mainLog, cancellationToken);
var appBundlePath = await simulator.GetAppBundlePath(_mainLog, bundleIdentifier, cancellationToken);
return(await GetAppBundleFromPath(target, appBundlePath, cancellationToken));
}
// We're unable to do this for real devices / or MacCatalyst
// It is not ideal but doesn't matter much at the moment as we don't need the full list of properties there
_logger.LogDebug("Supplemented full app bundle information with bundle identifier");
return(AppBundleInformation.FromBundleId(bundleIdentifier));
}
public async Task OrchestrateFailedDeviceInstallationTest()
{
// Setup
SetupInstall(_device.Object, 1);
SetupUninstall(_device.Object, 1); // This can fail as this is the first purge of the app before we install it
var failure = new KnownIssue("Some failure", suggestedExitCode: (int)ExitCode.APP_NOT_SIGNED);
_errorKnowledgeBase
.Setup(x => x.IsKnownInstallIssue(It.IsAny <IFileBackedLog>(), out failure))
.Returns(true)
.Verifiable();
var testTarget = new TestTargetOs(TestTarget.Device_iOS, "14.2");
// Act
var result = await _installOrchestrator.OrchestrateInstall(
testTarget,
null,
AppPath,
TimeSpan.FromMinutes(30),
includeWirelessDevices : false,
resetSimulator : false,
enableLldb : true,
new CancellationToken());
// Verify
Assert.Equal(ExitCode.APP_NOT_SIGNED, result);
_deviceFinder.Verify(
x => x.FindDevice(testTarget, null, It.IsAny <ILog>(), false, It.IsAny <CancellationToken>()),
Times.Once);
VerifySimulatorReset(false);
VerifySimulatorCleanUp(false);
VerifyDiagnosticData(testTarget);
_appInstaller.Verify(
x => x.InstallApp(_appBundleInformation, testTarget, _device.Object, It.IsAny <CancellationToken>()),
Times.Once);
}
public async Task TryMacCatalystResetTest()
{
// Setup
var testTarget = new TestTargetOs(TestTarget.MacCatalyst, null);
_deviceFinder.Reset();
// Act
var result = await _simulatorResetOrchestrator.OrchestrateSimulatorReset(
testTarget,
null,
TimeSpan.FromMinutes(30),
new CancellationToken());
// Verify
Assert.Equal(ExitCode.INVALID_ARGUMENTS, result);
_deviceFinder.VerifyNoOtherCalls();
_appInstaller.VerifyNoOtherCalls();
_appUninstaller.VerifyNoOtherCalls();
}
public async Task <(string DeviceName, TestExecutingResult Result, string ResultMessage)> RunApp(
AppBundleInformation appInformation,
TestTargetOs target,
TimeSpan timeout,
TimeSpan testLaunchTimeout,
string?deviceName = null,
string?companionDeviceName = null,
bool ensureCleanSimulatorState = false,
int verbosity = 1,
XmlResultJargon xmlResultJargon = XmlResultJargon.xUnit,
string[]?skippedMethods = null,
string[]?skippedTestClasses = null,
CancellationToken cancellationToken = default)
{
var runMode = target.Platform.ToRunMode();
bool isSimulator = target.Platform.IsSimulator();
var deviceListenerLog = _logs.Create($"test-{target.AsString()}-{_helpers.Timestamp}.log", LogType.TestLog.ToString(), timestamp: true);
var(deviceListenerTransport, deviceListener, deviceListenerTmpFile) = _listenerFactory.Create(
runMode,
log: _mainLog,
testLog: deviceListenerLog,
isSimulator: isSimulator,
autoExit: true,
xmlOutput: true); // cli always uses xml
ISimulatorDevice?simulator = null;
ISimulatorDevice?companionSimulator = null;
// Find devices
if (isSimulator)
{
int attempt = 1;
const int maxAttempts = 3;
while (true)
{
try
{
(simulator, companionSimulator) = await _simulatorLoader.FindSimulators(target, _mainLog);
break;
}
catch (Exception e)
{
_mainLog.WriteLine($"Failed to find/create simulator (attempt {attempt}/{maxAttempts}):" + Environment.NewLine + e);
if (attempt == maxAttempts)
{
throw new NoDeviceFoundException("Failed to find/create suitable simulator");
}
}
finally
{
attempt++;
}
}
deviceName = companionSimulator?.Name ?? simulator.Name;
}
else
{
deviceName ??= await FindDevice(target) ?? throw new NoDeviceFoundException();
}
int deviceListenerPort = deviceListener.InitializeAndGetPort();
deviceListener.StartAsync();
var crashLogs = new Logs(_logs.Directory);
ICrashSnapshotReporter crashReporter = _snapshotReporterFactory.Create(_mainLog, crashLogs, isDevice: !isSimulator, deviceName);
ITestReporter testReporter = _testReporterFactory.Create(_mainLog,
_mainLog,
_logs,
crashReporter,
deviceListener,
_resultParser,
appInformation,
runMode,
xmlResultJargon,
deviceName,
timeout,
null,
(level, message) => _mainLog.WriteLine(message));
deviceListener.ConnectedTask
.TimeoutAfter(testLaunchTimeout)
.ContinueWith(testReporter.LaunchCallback)
.DoNotAwait();
_mainLog.WriteLine($"*** Executing '{appInformation.AppName}' on {target} '{deviceName}' ***");
try
{
using var combinedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(testReporter.CancellationToken, cancellationToken);
if (isSimulator)
{
if (simulator == null)
{
_mainLog.WriteLine("Didn't find any suitable simulator");
throw new NoDeviceFoundException();
}
var mlaunchArguments = GetSimulatorArguments(
appInformation,
simulator,
verbosity,
xmlResultJargon,
skippedMethods,
skippedTestClasses,
deviceListenerTransport,
deviceListenerPort,
deviceListenerTmpFile);
await RunSimulatorTests(
mlaunchArguments,
appInformation,
crashReporter,
testReporter,
simulator,
companionSimulator,
ensureCleanSimulatorState,
timeout,
combinedCancellationToken.Token);
}
else
{
var mlaunchArguments = GetDeviceArguments(
appInformation,
deviceName,
target.Platform.IsWatchOSTarget(),
verbosity,
xmlResultJargon,
skippedMethods,
skippedTestClasses,
deviceListenerTransport,
deviceListenerPort,
deviceListenerTmpFile);
await RunDeviceTests(
mlaunchArguments,
crashReporter,
testReporter,
deviceListener,
deviceName,
timeout,
combinedCancellationToken.Token);
}
}
finally
{
deviceListener.Cancel();
deviceListener.Dispose();
}
// Check the final status, copy all the required data
var(testResult, resultMessage) = await testReporter.ParseResult();
return(deviceName, testResult, resultMessage);
}
protected Task <AppBundleInformation> GetAppBundleFromPath(TestTargetOs target, string appBundlePath, CancellationToken cancellationToken)
{
appBundlePath = Path.GetFullPath(appBundlePath);
_logger.LogInformation($"Getting app bundle information from '{appBundlePath}'..");
return(_appBundleInformationParser.ParseFromAppBundle(appBundlePath, target.Platform, _mainLog, cancellationToken));
}
private async Task <ExitCode> OrchestrateOperationInternal(
TestTargetOs target,
string?deviceName,
bool includeWirelessDevices,
bool resetSimulator,
bool enableLldb,
GetAppBundleInfoFunc getAppBundle,
ExecuteMacCatalystAppFunc executeMacCatalystApp,
ExecuteAppFunc executeApp,
CancellationToken cancellationToken)
{
_lldbFileCreated = false;
var isLldbEnabled = IsLldbEnabled();
if (isLldbEnabled && !enableLldb)
{
// the file is present, but the user did not set it, warn him about it
_logger.LogWarning("Lldb will be used since '~/.mtouch-launch-with-lldb' was found in the system but it was not created by xharness.");
}
else if (enableLldb)
{
if (!File.Exists(s_mlaunchLldbConfigFile))
{
// create empty file
File.WriteAllText(s_mlaunchLldbConfigFile, string.Empty);
_lldbFileCreated = true;
}
}
if (includeWirelessDevices && target.Platform.IsSimulator())
{
_logger.LogWarning("Including wireless devices while targeting a simulator has no effect");
}
if (resetSimulator && !target.Platform.IsSimulator())
{
_logger.LogWarning("Targeting device but requesting simulator reset has no effect");
resetSimulator = false;
}
ExitCode exitCode;
IDevice device;
IDevice? companionDevice;
AppBundleInformation appBundleInfo;
if (target.Platform == TestTarget.MacCatalyst)
{
try
{
appBundleInfo = await getAppBundle(target, null !, cancellationToken);
}
catch (Exception e)
{
cancellationToken.ThrowIfCancellationRequested();
_logger.LogError(e.Message);
return(ExitCode.PACKAGE_NOT_FOUND);
}
try
{
return(await executeMacCatalystApp(appBundleInfo));
}
catch (Exception e)
{
var message = new StringBuilder().AppendLine("Application run failed:");
exitCode = ExitCode.APP_LAUNCH_FAILURE;
if (_errorKnowledgeBase.IsKnownTestIssue(_mainLog, out var failure))
{
message.Append(failure.HumanMessage);
if (failure.IssueLink != null)
{
message.AppendLine($" Find more information at {failure.IssueLink}");
}
if (failure.SuggestedExitCode.HasValue)
{
exitCode = (ExitCode)failure.SuggestedExitCode.Value;
}
}
else
{
message.AppendLine(e.ToString());
}
_logger.LogError(message.ToString());
return(exitCode);
}
}
try
{
_logger.LogInformation($"Looking for available {target.AsString()} {(target.Platform.IsSimulator() ? "simulators" : "devices")}..");
var finderLogName = $"list-{target.AsString()}-{_helpers.Timestamp}.log";
using var finderLog = _logs.Create(finderLogName, "DeviceList", true);
_mainLog.WriteLine(
$"Looking for available {target.AsString()} {(target.Platform.IsSimulator() ? "simulators" : "devices")}. " +
$"Storing logs into {finderLogName}");
(device, companionDevice) = await _deviceFinder.FindDevice(target, deviceName, finderLog, includeWirelessDevices, cancellationToken);
_logger.LogInformation($"Found {(target.Platform.IsSimulator() ? "simulator" : "physical")} device '{device.Name}'");
if (companionDevice != null)
{
_logger.LogInformation($"Found companion {(target.Platform.IsSimulator() ? "simulator" : "physical")} device '{companionDevice.Name}'");
}
}
catch (NoDeviceFoundException e)
{
_logger.LogError(e.Message);
return(ExitCode.DEVICE_NOT_FOUND);
}
cancellationToken.ThrowIfCancellationRequested();
try
{
appBundleInfo = await getAppBundle(target, device, cancellationToken);
}
catch (Exception e)
{
cancellationToken.ThrowIfCancellationRequested();
_logger.LogError(e.Message);
return(ExitCode.PACKAGE_NOT_FOUND);
}
cancellationToken.ThrowIfCancellationRequested();
if (target.Platform.IsSimulator() && resetSimulator)
{
try
{
var simulator = (ISimulatorDevice)device;
var bundleIds = appBundleInfo.BundleIdentifier == string.Empty ? Array.Empty <string>() : new[] { appBundleInfo.BundleIdentifier };
_logger.LogInformation($"Reseting simulator '{device.Name}'");
await simulator.PrepareSimulator(_mainLog, bundleIds);
if (companionDevice != null)
{
_logger.LogInformation($"Reseting companion simulator '{companionDevice.Name}'");
var companionSimulator = (ISimulatorDevice)companionDevice;
await companionSimulator.PrepareSimulator(_mainLog, bundleIds);
}
_logger.LogInformation("Simulator reset finished");
}
catch (Exception e)
{
_logger.LogError($"Failed to reset simulator: " + Environment.NewLine + e);
return(ExitCode.SIMULATOR_FAILURE);
}
}
cancellationToken.ThrowIfCancellationRequested();
// Note down the actual test target
// For simulators (e.g. "iOS 13.4"), we strip the iOS part and keep the version only, for devices there's no OS
_diagnosticsData.TargetOS = target.Platform.IsSimulator() ? device.OSVersion.Split(' ', 2).Last() : device.OSVersion;
_diagnosticsData.Device = device.Name ?? device.UDID;
// Uninstall the app first to get a clean state
if (!resetSimulator)
{
await UninstallApp(target.Platform, appBundleInfo.BundleIdentifier, device, isPreparation : true, cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
}
exitCode = await InstallApp(appBundleInfo, device, target, cancellationToken);
if (exitCode != ExitCode.SUCCESS)
{
_logger.LogInformation($"Cleaning up the failed installation from '{device.Name}'");
var uninstallResult = await UninstallApp(target.Platform, appBundleInfo.BundleIdentifier, device, isPreparation : false, new CancellationToken());
if (uninstallResult == ExitCode.SIMULATOR_FAILURE)
{
// Sometimes the simulator gets in a bad shape and we won't be able to install the app, we can tell here
return(ExitCode.SIMULATOR_FAILURE);
}
return(exitCode);
}
try
{
exitCode = await executeApp(appBundleInfo, device, companionDevice);
}
catch (Exception e)
{
exitCode = ExitCode.GENERAL_FAILURE;
var message = new StringBuilder().AppendLine("Application run failed:");
if (_errorKnowledgeBase.IsKnownTestIssue(_mainLog, out var failure))
{
message.Append(failure.HumanMessage);
if (failure.IssueLink != null)
{
message.AppendLine($" Find more information at {failure.IssueLink}");
}
if (failure.SuggestedExitCode.HasValue)
{
exitCode = (ExitCode)failure.SuggestedExitCode.Value;
}
}
else
{
message.AppendLine(e.ToString());
}
_logger.LogError(message.ToString());
}
finally
{
if (target.Platform.IsSimulator() && resetSimulator)
{
await CleanUpSimulators(device, companionDevice);
}
else if (device != null) // Do not uninstall if device was cleaned up
{
var uninstallResult = await UninstallApp(target.Platform, appBundleInfo.BundleIdentifier, device, false, new CancellationToken());
// We are able to detect a case when simulator is in a bad shape
// If it also failed the test/run, we should present that as the failure
if (uninstallResult == ExitCode.SIMULATOR_FAILURE && exitCode != ExitCode.SUCCESS && exitCode != ExitCode.TESTS_FAILED)
{
exitCode = ExitCode.SIMULATOR_FAILURE;
}
}
}
return(exitCode);
}
public async Task <(ISimulatorDevice Simulator, ISimulatorDevice?CompanionSimulator)> FindSimulators(TestTargetOs target, ILog log, bool createIfNeeded = true, bool minVersion = false)
{
var runtimePrefix = target.Platform switch
{
TestTarget.Simulator_iOS32 => "com.apple.CoreSimulator.SimRuntime.iOS-",
TestTarget.Simulator_iOS64 => "com.apple.CoreSimulator.SimRuntime.iOS-",
TestTarget.Simulator_iOS => "com.apple.CoreSimulator.SimRuntime.iOS-",
TestTarget.Simulator_tvOS => "com.apple.CoreSimulator.SimRuntime.tvOS-",
TestTarget.Simulator_watchOS => "com.apple.CoreSimulator.SimRuntime.watchOS-",
_ => throw new Exception(string.Format("Invalid simulator target: {0}", target))
};
var runtimeVersion = target.OSVersion;
if (runtimeVersion == null)
{
if (!_loaded)
{
await LoadDevices(log);
}
string?firstOsVersion = _supportedRuntimes
.Where(r => r.Identifier.StartsWith(runtimePrefix))
.OrderByDescending(r => r.Identifier)
.FirstOrDefault()?
.Identifier
.Substring(runtimePrefix.Length);
runtimeVersion = firstOsVersion ?? throw new NoDeviceFoundException($"Failed to find a suitable OS runtime version for {target.AsString()}");
}
string simulatorRuntime = runtimePrefix + runtimeVersion.Replace('.', '-');
string simulatorDeviceType = target.Platform switch
{
TestTarget.Simulator_iOS => "com.apple.CoreSimulator.SimDeviceType.iPhone-5",
TestTarget.Simulator_iOS32 => "com.apple.CoreSimulator.SimDeviceType.iPhone-5",
TestTarget.Simulator_iOS64 => "com.apple.CoreSimulator.SimDeviceType." + (minVersion ? "iPhone-6" : "iPhone-X"),
TestTarget.Simulator_tvOS => "com.apple.CoreSimulator.SimDeviceType.Apple-TV-1080p",
TestTarget.Simulator_watchOS => "com.apple.CoreSimulator.SimDeviceType." + (minVersion ? "Apple-Watch-38mm" : "Apple-Watch-Series-3-38mm"),
_ => throw new Exception(string.Format("Invalid simulator target: {0}", target))
};
string?companionDeviceType = null;
string?companionRuntime = null;
if (target.Platform == TestTarget.Simulator_watchOS)
{
// TODO: Allow to specify companion runtime
companionRuntime = "com.apple.CoreSimulator.SimRuntime.iOS-" + (minVersion ? SdkVersions.MinWatchOSCompanionSimulator : SdkVersions.MaxWatchOSCompanionSimulator).Replace('.', '-');
companionDeviceType = "com.apple.CoreSimulator.SimDeviceType." + (minVersion ? "iPhone-6" : "iPhone-X");
}
var devices = await FindOrCreateDevicesAsync(log, simulatorRuntime, simulatorDeviceType);
IEnumerable <ISimulatorDevice>?companionDevices = null;
if (companionRuntime != null && companionDeviceType != null)
{
companionDevices = await FindOrCreateDevicesAsync(log, companionRuntime, companionDeviceType);
}
if (devices?.Any() != true)
{
throw new Exception($"Could not find or create devices{Environment.NewLine}" +
$"runtime: {simulatorRuntime}{Environment.NewLine}" +
$"device type: {simulatorDeviceType}");
}
ISimulatorDevice?simulator = null;
ISimulatorDevice?companionSimulator = null;
if (companionRuntime == null)
{
simulator = devices.First();
}
else
{
if (companionDevices?.Any() != true)
{
throw new Exception($"Could not find or create companion devices{Environment.NewLine}" +
$"runtime: {companionRuntime}{Environment.NewLine}" +
$"device type: {companionDeviceType}");
}
var pair = await FindOrCreateDevicePairAsync(log, devices, companionDevices);
if (pair == null)
{
throw new Exception($"Could not find or create device pair{Environment.NewLine}" +
$"runtime: {companionRuntime}{Environment.NewLine}" +
$"device type: {companionDeviceType}");
}
simulator = devices.First(v => v.UDID == pair.Gizmo);
companionSimulator = companionDevices.First(v => v.UDID == pair.Companion);
}
if (simulator == null)
{
throw new Exception($"Could not find simulator{Environment.NewLine}" +
$"runtime: {simulatorRuntime}{Environment.NewLine}" +
$"device type: {simulatorDeviceType}");
}
log.WriteLine("Found simulator: {0} {1}", simulator.Name, simulator.UDID);
if (companionSimulator != null)
{
log.WriteLine("Found companion simulator: {0} {1}", companionSimulator.Name, companionSimulator.UDID);
}
return(simulator, companionSimulator);
}
public async Task <(string deviceName, ProcessExecutionResult result)> InstallApp(AppBundleInformation appBundleInformation, TestTargetOs target, string?deviceName = null, CancellationToken cancellationToken = default)
{
if (target.Platform.IsSimulator())
{
// We reset the simulator when running, so a separate install step does not make much sense.
throw new InvalidOperationException("Installing to a simulator is not supported.");
}
if (!Directory.Exists(appBundleInformation.LaunchAppPath))
{
throw new DirectoryNotFoundException("Failed to find the app bundle directory");
}
if (deviceName == null)
{
// the _deviceLoader.FindDevice will return the fist device of the type, but we want to make sure that
// the device we use is if the correct arch, therefore, we will use the LoadDevices and return the
// correct one
await _deviceLoader.LoadDevices(_mainLog, false, false);
IHardwareDevice?device = null;
if (appBundleInformation.Supports32Bit)
{
// we only support 32b on iOS, therefore we can ignore the target
device = _deviceLoader.Connected32BitIOS.FirstOrDefault();
}
else
{
device = target.Platform switch
{
TestTarget.Device_iOS => _deviceLoader.Connected64BitIOS.FirstOrDefault(),
TestTarget.Device_tvOS => _deviceLoader.ConnectedTV.FirstOrDefault(),
_ => device
};
}
deviceName = target.Platform.IsWatchOSTarget() ? (await _deviceLoader.FindCompanionDevice(_mainLog, device)).Name : device?.Name;
}
if (deviceName == null)
{
throw new NoDeviceFoundException();
}
var args = new MlaunchArguments();
for (int i = -1; i < _verbosity; i++)
{
args.Add(new VerbosityArgument());
}
args.Add(new InstallAppOnDeviceArgument(appBundleInformation.LaunchAppPath));
args.Add(new DeviceNameArgument(deviceName));
if (target.Platform.IsWatchOSTarget())
{
args.Add(new DeviceArgument("ios,watchos"));
}
var totalSize = Directory.GetFiles(appBundleInformation.LaunchAppPath, "*", SearchOption.AllDirectories).Select((v) => new FileInfo(v).Length).Sum();
_mainLog.WriteLine($"Installing '{appBundleInformation.LaunchAppPath}' to '{deviceName}' ({totalSize / 1024.0 / 1024.0:N2} MB)");
ProcessExecutionResult result = await _processManager.ExecuteCommandAsync(args, _mainLog, TimeSpan.FromHours(1), cancellationToken : cancellationToken);
return(deviceName, result);
}
}
public async Task <(string DeviceName, int?exitCode)> RunApp(
AppBundleInformation appInformation,
TestTargetOs target,
TimeSpan timeout,
string?deviceName = null,
string?companionDeviceName = null,
bool ensureCleanSimulatorState = false,
int verbosity = 1,
CancellationToken cancellationToken = default)
{
bool isSimulator = target.Platform.IsSimulator();
ISimulatorDevice?simulator = null;
ISimulatorDevice?companionSimulator = null;
// Find devices
if (isSimulator)
{
(simulator, companionSimulator) = await _simulatorLoader.FindSimulators(target, _mainLog, 3);
deviceName = companionSimulator?.Name ?? simulator.Name;
}
else
{
deviceName ??= await FindDevice(target) ?? throw new NoDeviceFoundException();
}
var crashLogs = new Logs(_logs.Directory);
ICrashSnapshotReporter crashReporter = _snapshotReporterFactory.Create(_mainLog, crashLogs, isDevice: !isSimulator, deviceName);
_mainLog.WriteLine($"*** Executing '{appInformation.AppName}' on {target.AsString()} '{deviceName}' ***");
if (isSimulator)
{
if (simulator == null)
{
_mainLog.WriteLine("Didn't find any suitable simulator");
throw new NoDeviceFoundException();
}
var mlaunchArguments = GetSimulatorArguments(appInformation, simulator, verbosity);
await RunSimulatorApp(
mlaunchArguments,
appInformation,
crashReporter,
simulator,
companionSimulator,
ensureCleanSimulatorState,
timeout,
cancellationToken);
}
else
{
var mlaunchArguments = GetDeviceArguments(appInformation, deviceName, target.Platform.IsWatchOSTarget(), verbosity);
await RunDeviceApp(
mlaunchArguments,
crashReporter,
deviceName,
timeout,
cancellationToken);
}
var systemLog = _logs.FirstOrDefault(log => log.Description == LogType.SystemLog.ToString());
if (systemLog == null)
{
_mainLog.WriteLine("App run ended but failed to detect exit code (no system log found)");
return(deviceName, null);
}
var exitCode = _exitCodeDetector.DetectExitCode(appInformation, systemLog);
_mainLog.WriteLine($"App run ended with {exitCode}");
return(deviceName, exitCode);
}
protected override async Task <ExitCode> RunAppInternal(
AppBundleInformation appBundleInfo,
string?deviceName,
ILogger logger,
TestTargetOs target,
Logs logs,
IFileBackedLog mainLog,
CancellationToken cancellationToken)
{
var tunnelBore = (_arguments.CommunicationChannel == CommunicationChannel.UsbTunnel && !target.Platform.IsSimulator())
? new TunnelBore(ProcessManager)
: null;
// only add the extra callback if we do know that the feature was indeed enabled
Action <string>?logCallback = IsLldbEnabled() ? (l) => NotifyUserLldbCommand(logger, l) : (Action <string>?)null;
var appTester = new AppTester(
ProcessManager,
DeviceLoader,
SimulatorLoader,
new SimpleListenerFactory(tunnelBore),
new CrashSnapshotReporterFactory(ProcessManager),
new CaptureLogFactory(),
new DeviceLogCapturerFactory(ProcessManager),
new TestReporterFactory(ProcessManager),
new XmlResultParser(),
mainLog,
logs,
new Helpers(),
logCallback: logCallback,
appArguments: PassThroughArguments);
string resultMessage;
TestExecutingResult testResult;
(deviceName, testResult, resultMessage) = await appTester.TestApp(appBundleInfo,
target,
_arguments.Timeout,
_arguments.LaunchTimeout,
deviceName,
verbosity : GetMlaunchVerbosity(_arguments.Verbosity),
xmlResultJargon : _arguments.XmlResultJargon,
cancellationToken : cancellationToken,
skippedMethods : _arguments.SingleMethodFilters?.ToArray(),
skippedTestClasses : _arguments.ClassMethodFilters?.ToArray());
switch (testResult)
{
case TestExecutingResult.Succeeded:
logger.LogInformation($"Application finished the test run successfully");
logger.LogInformation(resultMessage);
return(ExitCode.SUCCESS);
case TestExecutingResult.Failed:
logger.LogInformation($"Application finished the test run successfully with some failed tests");
logger.LogInformation(resultMessage);
return(ExitCode.TESTS_FAILED);
case TestExecutingResult.LaunchFailure:
if (resultMessage != null)
{
logger.LogError($"Failed to launch the application:{Environment.NewLine}" +
$"{resultMessage}{Environment.NewLine}{Environment.NewLine}" +
$"Check logs for more information.");
}
else
{
logger.LogError($"Failed to launch the application. Check logs for more information");
}
return(ExitCode.APP_LAUNCH_FAILURE);
case TestExecutingResult.Crashed:
if (resultMessage != null)
{
logger.LogError($"Application run crashed:{Environment.NewLine}" +
$"{resultMessage}{Environment.NewLine}{Environment.NewLine}" +
$"Check logs for more information.");
}
else
{
logger.LogError($"Application test run crashed. Check logs for more information");
}
return(ExitCode.APP_CRASH);
case TestExecutingResult.TimedOut:
logger.LogWarning($"Application test run timed out");
return(ExitCode.TIMED_OUT);
default:
if (resultMessage != null)
{
logger.LogError($"Application test run ended in an unexpected way: '{testResult}'{Environment.NewLine}" +
$"{resultMessage}{Environment.NewLine}{Environment.NewLine}" +
$"Check logs for more information.");
}
else
{
logger.LogError($"Application test run ended in an unexpected way: '{testResult}'. Check logs for more information");
}
return(ExitCode.GENERAL_FAILURE);
}
}
public async Task <(string DeviceName, ProcessExecutionResult result)> RunApp(
AppBundleInformation appInformation,
TestTargetOs target,
TimeSpan timeout,
string?deviceName = null,
string?companionDeviceName = null,
bool ensureCleanSimulatorState = false,
int verbosity = 1,
CancellationToken cancellationToken = default)
{
var isSimulator = target.Platform.IsSimulator();
ProcessExecutionResult result;
ISimulatorDevice? simulator = null;
ISimulatorDevice? companionSimulator = null;
if (target.Platform == TestTarget.MacCatalyst)
{
_mainLog.WriteLine($"*** Executing '{appInformation.AppName}' on MacCatalyst ***");
result = await RunMacCatalystApp(appInformation, timeout, _appArguments, new Dictionary <string, object>(), cancellationToken);
return("MacCatalyst", result);
}
// Find devices
if (isSimulator)
{
(simulator, companionSimulator) = await _simulatorLoader.FindSimulators(target, _mainLog, 3);
deviceName = companionSimulator?.Name ?? simulator.Name;
}
else
{
deviceName ??= await FindDevice(target) ?? throw new NoDeviceFoundException();
}
var crashLogs = new Logs(_logs.Directory);
ICrashSnapshotReporter crashReporter = _snapshotReporterFactory.Create(
_mainLog,
crashLogs,
isDevice: !isSimulator,
deviceName);
_mainLog.WriteLine($"*** Executing '{appInformation.AppName}' on {target.AsString()} '{deviceName}' ***");
if (isSimulator)
{
if (simulator == null)
{
_mainLog.WriteLine("Didn't find any suitable simulator");
throw new NoDeviceFoundException();
}
var mlaunchArguments = GetSimulatorArguments(appInformation, simulator, verbosity);
result = await RunSimulatorApp(
mlaunchArguments,
appInformation,
crashReporter,
simulator,
companionSimulator,
ensureCleanSimulatorState,
timeout,
cancellationToken);
}
else
{
var mlaunchArguments = GetDeviceArguments(appInformation, deviceName, target.Platform.IsWatchOSTarget(), verbosity);
result = await RunDeviceApp(
mlaunchArguments,
crashReporter,
deviceName,
timeout,
cancellationToken);
}
return(deviceName, result);
}
public async Task <(ISimulatorDevice Simulator, ISimulatorDevice?CompanionSimulator)> FindSimulators(TestTargetOs target, ILog log, int retryCount, bool createIfNeeded = true, bool minVersion = false)
{
if (retryCount < 1)
{
throw new ArgumentOutOfRangeException(nameof(retryCount));
}
int attempt = 1;
while (true)
{
try
{
return(await FindSimulators(target, log));
}
catch (Exception e)
{
log.WriteLine($"Failed to find/create simulator (attempt {attempt}/{retryCount}):" + Environment.NewLine + e);
if (attempt == retryCount)
{
throw new NoDeviceFoundException("Failed to find/create suitable simulator");
}
}
finally
{
attempt++;
}
}
}
public async Task <DevicePair> FindDevice(
TestTargetOs target,
string?deviceName,
ILog log,
bool includeWirelessDevices = true,
CancellationToken cancellationToken = default)
{
IDevice?device;
IDevice?companionDevice = null;
bool IsMatchingDevice(IDevice device) =>
device.Name.Equals(deviceName, StringComparison.InvariantCultureIgnoreCase) ||
device.UDID.Equals(deviceName, StringComparison.InvariantCultureIgnoreCase);
if (target.Platform.IsSimulator())
{
if (deviceName == null)
{
(device, companionDevice) = await _simulatorLoader.FindSimulators(target, log, retryCount : 3, cancellationToken : cancellationToken);
}
else
{
await _simulatorLoader.LoadDevices(log, includeLocked : false, cancellationToken : cancellationToken);
device = _simulatorLoader.AvailableDevices.FirstOrDefault(IsMatchingDevice)
?? throw new NoDeviceFoundException($"Failed to find a simulator '{deviceName}'");
}
}
else
{
// The DeviceLoader.FindDevice will return the fist device of the type, but we want to make sure that
// the device we use is of the correct arch, therefore, we will use the LoadDevices and handpick one
await _deviceLoader.LoadDevices(
log,
includeLocked : false,
forceRefresh : false,
includeWirelessDevices : includeWirelessDevices,
cancellationToken : cancellationToken);
if (deviceName == null)
{
IHardwareDevice?hardwareDevice = target.Platform switch
{
TestTarget.Simulator_iOS32 => _deviceLoader.Connected32BitIOS.FirstOrDefault(),
TestTarget.Device_iOS => _deviceLoader.Connected64BitIOS.FirstOrDefault(),
TestTarget.Device_tvOS => _deviceLoader.ConnectedTV.FirstOrDefault(),
_ => throw new ArgumentOutOfRangeException(nameof(target), $"Unrecognized device platform {target.Platform}")
};
if (target.Platform.IsWatchOSTarget() && hardwareDevice != null)
{
companionDevice = await _deviceLoader.FindCompanionDevice(log, hardwareDevice, cancellationToken : cancellationToken);
}
device = hardwareDevice;
}
else
{
device = _deviceLoader.ConnectedDevices.FirstOrDefault(IsMatchingDevice)
?? throw new NoDeviceFoundException($"Failed to find a device '{deviceName}'. " +
"Please make sure the device is connected and unlocked.");
}
}
if (device == null)
{
throw new NoDeviceFoundException($"Failed to find a suitable device for target {target.AsString()}");
}
return(new DevicePair(device, companionDevice));
}
private async Task <ExitCode> RunTest(
ILogger logger,
TestTargetOs target,
Logs logs,
MLaunchProcessManager processManager,
IHardwareDeviceLoader deviceLoader,
ISimulatorLoader simulatorLoader,
ITunnelBore?tunnelBore,
CancellationToken cancellationToken = default)
{
var isLldbEnabled = IsLldbEnabled();
if (isLldbEnabled && !_arguments.EnableLldb)
{
// the file is present, but the user did not set it, warn him about it
logger.LogWarning("Lldb will be used since '~/.mtouch-launch-with-lldb' was found in the system but it was not created by xharness.");
}
else if (_arguments.EnableLldb)
{
if (!File.Exists(s_mlaunchLldbConfigFile))
{
// create empty file
File.WriteAllText(s_mlaunchLldbConfigFile, string.Empty);
_createdLldbFile = true;
}
}
logger.LogInformation($"Starting test for {target.AsString()}{ (_arguments.DeviceName != null ? " targeting " + _arguments.DeviceName : null) }..");
string mainLogFile = Path.Join(_arguments.OutputDirectory, $"run-{target.AsString()}{(_arguments.DeviceName != null ? "-" + _arguments.DeviceName : null)}.log");
IFileBackedLog mainLog = Log.CreateReadableAggregatedLog(
logs.Create(mainLogFile, LogType.ExecutionLog.ToString(), true),
new CallbackLog(message => logger.LogDebug(message.Trim()))
{
Timestamp = false
});
int verbosity = GetMlaunchVerbosity(_arguments.Verbosity);
string?deviceName = _arguments.DeviceName;
var appBundleInformationParser = new AppBundleInformationParser(processManager);
AppBundleInformation appBundleInfo;
try
{
appBundleInfo = await appBundleInformationParser.ParseFromAppBundle(_arguments.AppPackagePath, target.Platform, mainLog, cancellationToken);
}
catch (Exception e)
{
logger.LogError($"Failed to get bundle information: {e.Message}");
return(ExitCode.FAILED_TO_GET_BUNDLE_INFO);
}
if (!target.Platform.IsSimulator())
{
logger.LogInformation($"Installing application '{appBundleInfo.AppName}' on " + (deviceName != null ? $" on device '{deviceName}'" : target.AsString()));
var appInstaller = new AppInstaller(processManager, deviceLoader, mainLog, verbosity);
ProcessExecutionResult result;
try
{
(deviceName, result) = await appInstaller.InstallApp(appBundleInfo, target, cancellationToken : cancellationToken);
}
catch (NoDeviceFoundException)
{
logger.LogError($"Failed to find suitable device for target {target.AsString()}" + Environment.NewLine +
"Please make sure the device is connected and unlocked.");
return(ExitCode.DEVICE_NOT_FOUND);
}
catch (Exception e)
{
logger.LogError($"Failed to install the app bundle:{Environment.NewLine}{e}");
return(ExitCode.PACKAGE_INSTALLATION_FAILURE);
}
if (!result.Succeeded)
{
// use the knowledge base class to decide if the error is known, if it is, let the user know
// the failure reason
if (_errorKnowledgeBase.IsKnownInstallIssue(mainLog, out var errorMessage))
{
var msg = $"Failed to install the app bundle (exit code={result.ExitCode}): {errorMessage.Value.HumanMessage}.";
if (errorMessage.Value.IssueLink != null)
{
msg += $" Find more information at {errorMessage.Value.IssueLink}";
}
logger.LogError(msg);
}
else
{
logger.LogError($"Failed to install the app bundle (exit code={result.ExitCode})");
}
return(ExitCode.PACKAGE_INSTALLATION_FAILURE);
}
logger.LogInformation($"Application '{appBundleInfo.AppName}' was installed successfully on device '{deviceName}'");
}
logger.LogInformation($"Starting application '{appBundleInfo.AppName}' on " + (deviceName != null ? $"device '{deviceName}'" : target.AsString()));
// only add the extra callback if we do know that the feature was indeed enabled
Action <string>?logCallback = isLldbEnabled ? (l) => NotifyUserLldbCommand(logger, l) : (Action <string>?)null;
var appRunner = new AppRunner(
processManager,
deviceLoader,
simulatorLoader,
new SimpleListenerFactory(tunnelBore),
new CrashSnapshotReporterFactory(processManager),
new CaptureLogFactory(),
new DeviceLogCapturerFactory(processManager),
new TestReporterFactory(processManager),
new XmlResultParser(),
mainLog,
logs,
new Helpers(),
logCallback: logCallback,
appArguments: PassThroughArguments);
try
{
string resultMessage;
TestExecutingResult testResult;
(deviceName, testResult, resultMessage) = await appRunner.RunApp(appBundleInfo,
target,
_arguments.Timeout,
_arguments.LaunchTimeout,
deviceName,
verbosity : verbosity,
xmlResultJargon : _arguments.XmlResultJargon,
cancellationToken : cancellationToken,
skippedMethods : _arguments.SingleMethodFilters?.ToArray(),
skippedTestClasses : _arguments.ClassMethodFilters?.ToArray());
switch (testResult)
{
case TestExecutingResult.Succeeded:
logger.LogInformation($"Application finished the test run successfully");
logger.LogInformation(resultMessage);
return(ExitCode.SUCCESS);
case TestExecutingResult.Failed:
logger.LogInformation($"Application finished the test run successfully with some failed tests");
logger.LogInformation(resultMessage);
return(ExitCode.TESTS_FAILED);
case TestExecutingResult.LaunchFailure:
if (resultMessage != null)
{
logger.LogError($"Failed to launch the application:{Environment.NewLine}" +
$"{resultMessage}{Environment.NewLine}{Environment.NewLine}" +
$"Check logs for more information.");
}
else
{
logger.LogError($"Failed to launch the application. Check logs for more information");
}
return(ExitCode.APP_LAUNCH_FAILURE);
case TestExecutingResult.Crashed:
if (resultMessage != null)
{
logger.LogError($"Application run crashed:{Environment.NewLine}" +
$"{resultMessage}{Environment.NewLine}{Environment.NewLine}" +
$"Check logs for more information.");
}
else
{
logger.LogError($"Application run crashed. Check logs for more information");
}
return(ExitCode.APP_CRASH);
case TestExecutingResult.TimedOut:
logger.LogWarning($"Application run timed out");
return(ExitCode.TIMED_OUT);
default:
if (resultMessage != null)
{
logger.LogError($"Application run ended in an unexpected way: '{testResult}'{Environment.NewLine}" +
$"{resultMessage}{Environment.NewLine}{Environment.NewLine}" +
$"Check logs for more information.");
}
else
{
logger.LogError($"Application run ended in an unexpected way: '{testResult}'. Check logs for more information");
}
return(ExitCode.GENERAL_FAILURE);
}
}
catch (NoDeviceFoundException)
{
logger.LogError($"Failed to find suitable device for target {target.AsString()}" +
(target.Platform.IsSimulator() ? Environment.NewLine + "Please make sure suitable Simulator is installed in Xcode" : string.Empty));
return(ExitCode.DEVICE_NOT_FOUND);
}
catch (Exception e)
{
if (_errorKnowledgeBase.IsKnownTestIssue(mainLog, out var failureMessage))
{
var msg = $"Application run failed:{Environment.NewLine}{failureMessage.Value.HumanMessage}.";
if (failureMessage.Value.IssueLink != null)
{
msg += $" Find more information at {failureMessage.Value.IssueLink}";
}
logger.LogError(msg);
}
else
{
logger.LogError($"Application run failed:{Environment.NewLine}{e}");
}
return(ExitCode.APP_CRASH);
}
finally
{
mainLog.Dispose();
if (!target.Platform.IsSimulator() && deviceName != null)
{
logger.LogInformation($"Uninstalling the application '{appBundleInfo.AppName}' from device '{deviceName}'");
var appUninstaller = new AppUninstaller(processManager, mainLog, verbosity);
var uninstallResult = await appUninstaller.UninstallApp(deviceName, appBundleInfo.BundleIdentifier, cancellationToken);
if (!uninstallResult.Succeeded)
{
logger.LogError($"Failed to uninstall the app bundle with exit code: {uninstallResult.ExitCode}");
}
else
{
logger.LogInformation($"Application '{appBundleInfo.AppName}' was uninstalled successfully");
}
}
if (_createdLldbFile) // clean after the setting
{
File.Delete(s_mlaunchLldbConfigFile);
}
}
}
public async Task OrchestrateSimulatorJustTestTest()
{
// Setup
var testTarget = new TestTargetOs(TestTarget.Simulator_iOS64, "13.5");
var envVars = new[] { ("envVar1", "value1"), ("envVar2", "value2") };
protected override async Task <ExitCode> RunAppInternal(
AppBundleInformation appBundleInfo,
string?deviceName,
ILogger logger,
TestTargetOs target,
Logs logs,
IFileBackedLog mainLog,
CancellationToken cancellationToken)
{
// only add the extra callback if we do know that the feature was indeed enabled
Action <string>?logCallback = IsLldbEnabled() ? (l) => NotifyUserLldbCommand(logger, l) : (Action <string>?)null;
var appRunner = new AppRunner(
ProcessManager,
DeviceLoader,
SimulatorLoader,
new CrashSnapshotReporterFactory(ProcessManager),
new CaptureLogFactory(),
new DeviceLogCapturerFactory(ProcessManager),
mainLog,
logs,
new Helpers(),
PassThroughArguments,
logCallback);
ProcessExecutionResult result;
(deviceName, result) = await appRunner.RunApp(
appBundleInfo,
target,
_arguments.Timeout,
deviceName,
verbosity : GetMlaunchVerbosity(_arguments.Verbosity),
cancellationToken : cancellationToken);
if (!result.Succeeded)
{
if (result.TimedOut)
{
logger.LogError($"App run has timed out");
return(ExitCode.TIMED_OUT);
}
logger.LogError($"App run has failed. mlaunch exited with {result.ExitCode}");
return(ExitCode.APP_LAUNCH_FAILURE);
}
var systemLog = logs.FirstOrDefault(log => log.Description == LogType.SystemLog.ToString());
if (systemLog == null)
{
logger.LogError("Application has finished but no system log found. Failed to determine the exit code!");
return(ExitCode.RETURN_CODE_NOT_SET);
}
var exitCode = new ExitCodeDetector().DetectExitCode(appBundleInfo, systemLog);
logger.LogInformation($"App run ended with {exitCode}");
if (_arguments.ExpectedExitCode != exitCode)
{
logger.LogError($"Application has finished with exit code {exitCode} but {_arguments.ExpectedExitCode} was expected");
return(ExitCode.GENERAL_FAILURE);
}
logger.LogInformation("Application has finished with exit code: " + exitCode +
(_arguments.ExpectedExitCode != 0 ? " (as expected)" : null));
return(ExitCode.SUCCESS);
}
protected void VerifyDiagnosticData(TestTargetOs target)
{
Assert.Equal(target.Platform.IsSimulator() ? _simulator.Object.Name : _device.Object.Name, _diagnosticsData.Device);
Assert.Contains(target.OSVersion, _diagnosticsData.TargetOS);
}
