public AppRunner(IMLaunchProcessManager processManager,
IHardwareDeviceLoader hardwareDeviceLoader,
ISimulatorLoader simulatorLoader,
ISimpleListenerFactory simpleListenerFactory,
ICrashSnapshotReporterFactory snapshotReporterFactory,
ICaptureLogFactory captureLogFactory,
IDeviceLogCapturerFactory deviceLogCapturerFactory,
ITestReporterFactory reporterFactory,
IFileBackedLog mainLog,
ILogs logs,
IHelpers helpers,
bool useXmlOutput,
Action <string>?logCallback = null)
{
_processManager = processManager ?? throw new ArgumentNullException(nameof(processManager));
_hardwareDeviceLoader = hardwareDeviceLoader ?? throw new ArgumentNullException(nameof(hardwareDeviceLoader));
_simulatorLoader = simulatorLoader ?? throw new ArgumentNullException(nameof(simulatorLoader));
_listenerFactory = simpleListenerFactory ?? throw new ArgumentNullException(nameof(simpleListenerFactory));
_snapshotReporterFactory = snapshotReporterFactory ?? throw new ArgumentNullException(nameof(snapshotReporterFactory));
_captureLogFactory = captureLogFactory ?? throw new ArgumentNullException(nameof(captureLogFactory));
_deviceLogCapturerFactory = deviceLogCapturerFactory ?? throw new ArgumentNullException(nameof(deviceLogCapturerFactory));
_testReporterFactory = reporterFactory ?? throw new ArgumentNullException(nameof(_testReporterFactory));
_logs = logs ?? throw new ArgumentNullException(nameof(logs));
_helpers = helpers ?? throw new ArgumentNullException(nameof(helpers));
_useXmlOutput = useXmlOutput;
if (logCallback == null)
{
_mainLog = mainLog ?? throw new ArgumentNullException(nameof(mainLog));
}
else
{
// create using the main as the default log
_mainLog = Log.CreateReadableAggregatedLog(mainLog, new CallbackLog(logCallback));
}
}
private readonly IEnumerable <string> _appArguments; // Arguments that will be passed to the iOS application
public AppRunner(
IMlaunchProcessManager processManager,
IHardwareDeviceLoader hardwareDeviceLoader,
ISimulatorLoader simulatorLoader,
ICrashSnapshotReporterFactory snapshotReporterFactory,
ICaptureLogFactory captureLogFactory,
IDeviceLogCapturerFactory deviceLogCapturerFactory,
IExitCodeDetector exitCodeDetector,
IFileBackedLog mainLog,
ILogs logs,
IHelpers helpers,
IEnumerable <string> appArguments,
Action <string>?logCallback = null)
: base(hardwareDeviceLoader, mainLog, logCallback)
{
_processManager = processManager ?? throw new ArgumentNullException(nameof(processManager));
_simulatorLoader = simulatorLoader ?? throw new ArgumentNullException(nameof(simulatorLoader));
_snapshotReporterFactory = snapshotReporterFactory ?? throw new ArgumentNullException(nameof(snapshotReporterFactory));
_captureLogFactory = captureLogFactory ?? throw new ArgumentNullException(nameof(captureLogFactory));
_deviceLogCapturerFactory = deviceLogCapturerFactory ?? throw new ArgumentNullException(nameof(deviceLogCapturerFactory));
_exitCodeDetector = exitCodeDetector ?? throw new ArgumentNullException(nameof(exitCodeDetector));
_logs = logs ?? throw new ArgumentNullException(nameof(logs));
_helpers = helpers ?? throw new ArgumentNullException(nameof(helpers));
_appArguments = appArguments;
}
private readonly IEnumerable <string> _appArguments; // Arguments that will be passed to the iOS application
public AppTester(
IMlaunchProcessManager processManager,
IHardwareDeviceLoader hardwareDeviceLoader,
ISimulatorLoader simulatorLoader,
ISimpleListenerFactory simpleListenerFactory,
ICrashSnapshotReporterFactory snapshotReporterFactory,
ICaptureLogFactory captureLogFactory,
IDeviceLogCapturerFactory deviceLogCapturerFactory,
ITestReporterFactory reporterFactory,
IResultParser resultParser,
IFileBackedLog mainLog,
ILogs logs,
IHelpers helpers,
IEnumerable <string> appArguments,
Action <string>?logCallback = null)
: base(processManager, hardwareDeviceLoader, captureLogFactory, logs, mainLog, logCallback)
{
_processManager = processManager ?? throw new ArgumentNullException(nameof(processManager));
_simulatorLoader = simulatorLoader ?? throw new ArgumentNullException(nameof(simulatorLoader));
_listenerFactory = simpleListenerFactory ?? throw new ArgumentNullException(nameof(simpleListenerFactory));
_snapshotReporterFactory = snapshotReporterFactory ?? throw new ArgumentNullException(nameof(snapshotReporterFactory));
_captureLogFactory = captureLogFactory ?? throw new ArgumentNullException(nameof(captureLogFactory));
_deviceLogCapturerFactory = deviceLogCapturerFactory ?? throw new ArgumentNullException(nameof(deviceLogCapturerFactory));
_testReporterFactory = reporterFactory ?? throw new ArgumentNullException(nameof(_testReporterFactory));
_resultParser = resultParser ?? throw new ArgumentNullException(nameof(resultParser));
_mainLog = mainLog ?? throw new ArgumentNullException(nameof(mainLog));
_logs = logs ?? throw new ArgumentNullException(nameof(logs));
_helpers = helpers ?? throw new ArgumentNullException(nameof(helpers));
_appArguments = appArguments;
}
public RunSimulator(IRunSimulatorTask testTask,
ISimulatorLoader simulators,
IErrorKnowledgeBase errorKnowledgeBase,
ILog mainLog,
ILog simulatorLoadLog)
{
this.testTask = testTask ?? throw new ArgumentNullException(nameof(testTask));
this.simulators = simulators ?? throw new ArgumentNullException(nameof(simulators));
this.errorKnowledgeBase = errorKnowledgeBase ?? throw new ArgumentNullException(nameof(errorKnowledgeBase));
this.mainLog = mainLog ?? throw new ArgumentNullException(nameof(mainLog));
this.simulatorLoadLog = simulatorLoadLog ?? throw new ArgumentNullException(nameof(simulatorLoadLog));
var project = Path.GetFileNameWithoutExtension(testTask.ProjectFile);
if (project.EndsWith("-tvos", StringComparison.Ordinal))
{
testTask.AppRunnerTarget = TestTarget.Simulator_tvOS;
}
else if (project.EndsWith("-watchos", StringComparison.Ordinal))
{
testTask.AppRunnerTarget = TestTarget.Simulator_watchOS;
}
else
{
testTask.AppRunnerTarget = TestTarget.Simulator_iOS;
}
}
public AppRunner(IProcessManager processManager,
IHardwareDeviceLoader hardwareDeviceLoader,
ISimulatorLoader simulatorLoader,
ISimpleListenerFactory simpleListenerFactory,
ICrashSnapshotReporterFactory snapshotReporterFactory,
ICaptureLogFactory captureLogFactory,
IDeviceLogCapturerFactory deviceLogCapturerFactory,
ITestReporterFactory reporterFactory,
ILog mainLog,
ILogs logs,
IHelpers helpers,
bool useXmlOutput)
{
_processManager = processManager ?? throw new ArgumentNullException(nameof(processManager));
_hardwareDeviceLoader = hardwareDeviceLoader ?? throw new ArgumentNullException(nameof(hardwareDeviceLoader));
_simulatorLoader = simulatorLoader ?? throw new ArgumentNullException(nameof(simulatorLoader));
_listenerFactory = simpleListenerFactory ?? throw new ArgumentNullException(nameof(simpleListenerFactory));
_snapshotReporterFactory = snapshotReporterFactory ?? throw new ArgumentNullException(nameof(snapshotReporterFactory));
_captureLogFactory = captureLogFactory ?? throw new ArgumentNullException(nameof(captureLogFactory));
_deviceLogCapturerFactory = deviceLogCapturerFactory ?? throw new ArgumentNullException(nameof(deviceLogCapturerFactory));
_testReporterFactory = reporterFactory ?? throw new ArgumentNullException(nameof(_testReporterFactory));
_mainLog = mainLog ?? throw new ArgumentNullException(nameof(mainLog));
_logs = logs ?? throw new ArgumentNullException(nameof(logs));
_helpers = helpers ?? throw new ArgumentNullException(nameof(helpers));
_useXmlOutput = useXmlOutput;
}
public async Task FoundSimulatorsTest(ISimulatorLoader simulators, IHardwareDeviceLoader devices, string expectedDescription)
{
var loader = new JenkinsDeviceLoader(simulators, devices, logs.Object);
await loader.LoadSimulatorsAsync();
// validate that the log description will be set as expected
log.VerifySet(l => l.Description = It.Is <string> (s => s.Equals(expectedDescription)));
}
public JenkinsDeviceLoader(ISimulatorLoader simulators, IHardwareDeviceLoader devices, ILogs logs)
{
if (logs == null)
{
throw new ArgumentNullException(nameof(logs));
}
this.simulators = simulators ?? throw new ArgumentNullException(nameof(simulators));
this.devices = devices ?? throw new ArgumentNullException(nameof(devices));
SimulatorLoadLog = logs.Create($"simulator-list-{Harness.Helpers.Timestamp}.log", $"Simulator Listing");
DeviceLoadLog = logs.Create($"device-list-{Harness.Helpers.Timestamp}.log", $"Device Listing");
}
public Jenkins(Harness harness, IProcessManager processManager, IResultParser resultParser, ITunnelBore tunnelBore)
{
this.processManager = processManager ?? throw new ArgumentNullException(nameof(processManager));
this.TunnelBore = tunnelBore ?? throw new ArgumentNullException(nameof(tunnelBore));
Harness = harness ?? throw new ArgumentNullException(nameof(harness));
Simulators = new SimulatorLoader(processManager);
Devices = new HardwareDeviceLoader(processManager);
testSelector = new TestSelector(this, processManager, new GitHub(harness, processManager));
testVariationsFactory = new TestVariationsFactory(this, processManager);
DeviceLoader = new JenkinsDeviceLoader(Simulators, Devices, Logs);
resourceManager = new ResourceManager();
htmlReportWriter = new HtmlReportWriter(jenkins: this, resourceManager: resourceManager, resultParser: resultParser);
markdownReportWriter = new MarkdownReportWriter();
}
public Jenkins(IHarness harness, IProcessManager processManager, IResultParser resultParser, ITunnelBore tunnelBore)
{
this.processManager = processManager ?? throw new ArgumentNullException(nameof(processManager));
this.TunnelBore = tunnelBore ?? throw new ArgumentNullException(nameof(tunnelBore));
Harness = harness ?? throw new ArgumentNullException(nameof(harness));
Simulators = new SimulatorLoader(processManager);
Devices = new HardwareDeviceLoader(processManager);
testSelector = new TestSelector(this, processManager, new GitHub(harness, processManager));
testVariationsFactory = new TestVariationsFactory(this, processManager);
DeviceLoader = new JenkinsDeviceLoader(Simulators, Devices, Logs);
resourceManager = new ResourceManager();
xamarinStorageHtmlReportWriter = new HtmlReportWriter(jenkins: this, resourceManager: resourceManager, resultParser: resultParser);
// we only care about the vsdrops writer if we are in the CI, locally makes no sense
if (harness.InCI && !string.IsNullOrEmpty(Harness.VSDropsUri))
{
vsdropsHtmlReportWriter = new HtmlReportWriter(this, resourceManager, resultParser, linksPrefix: Harness.VSDropsUri, embeddedResources: true);
}
markdownReportWriter = new MarkdownReportWriter();
}
public RunSimulatorTask(ISimulatorLoader simulators, MSBuildTask build_task, IProcessManager ProcessManager, IEnumerable <ISimulatorDevice> candidates = null)
: base(build_task, ProcessManager, candidates)
{
var project = Path.GetFileNameWithoutExtension(ProjectFile);
if (project.EndsWith("-tvos", StringComparison.Ordinal))
{
AppRunnerTarget = TestTarget.Simulator_tvOS;
}
else if (project.EndsWith("-watchos", StringComparison.Ordinal))
{
AppRunnerTarget = TestTarget.Simulator_watchOS;
}
else
{
AppRunnerTarget = TestTarget.Simulator_iOS;
}
this.simulators = simulators ?? throw new ArgumentNullException(nameof(simulators));
}
Task LoadAsync(ILog log, IDeviceLoader deviceManager, string name)
{
log.Description = $"{name} Listing (in progress)";
var capturedLog = log;
return(deviceManager.LoadDevices(capturedLog, includeLocked: false, forceRefresh: true).ContinueWith((v) => {
if (v.IsFaulted)
{
capturedLog.WriteLine("Failed to load:");
capturedLog.WriteLine(v.Exception.ToString());
capturedLog.Description = $"{name} Listing {v.Exception.Message})";
}
else if (v.IsCompleted)
{
capturedLog.Description = deviceManager switch
{
IHardwareDeviceLoader d => BuildDevicesDescription(d, name),
ISimulatorLoader s => BuildSimulatorsDescription(s, name),
_ => throw new NotImplementedException(),
};
}
}));
}
static string BuildSimulatorsDescription(ISimulatorLoader simulatorLoader, string name)
{
var simCount = simulatorLoader.AvailableDevices.Count();
return((simCount == 0) ? $"{name} Listing (ok - no simulators found)." : $"{name} Listing (ok - Found {simCount} simulators).");
}
private async Task <ExitCode> RunTest(
ILogger logger,
TestTarget target,
Logs logs,
ProcessManager processManager,
IHardwareDeviceLoader deviceLoader,
ISimulatorLoader simulatorLoader,
ITunnelBore?tunnelBore,
CancellationToken cancellationToken = default)
{
logger.LogInformation($"Starting test for {target.AsString()}{ (_arguments.DeviceName != null ? " targeting " + _arguments.DeviceName : null) }..");
string mainLogFile = Path.Join(_arguments.OutputDirectory, $"run-{target}{(_arguments.DeviceName != null ? "-" + _arguments.DeviceName : null)}.log");
ILog mainLog = logs.Create(mainLogFile, LogType.ExecutionLog.ToString(), true);
int verbosity = GetMlaunchVerbosity(_arguments.Verbosity);
string?deviceName = _arguments.DeviceName;
var appBundleInformationParser = new AppBundleInformationParser(processManager);
AppBundleInformation appBundleInfo;
try
{
appBundleInfo = await appBundleInformationParser.ParseFromAppBundle(_arguments.AppPackagePath, target, mainLog, cancellationToken);
}
catch (Exception e)
{
logger.LogError($"Failed to get bundle information: {e.Message}");
return(ExitCode.FAILED_TO_GET_BUNDLE_INFO);
}
if (!target.IsSimulator())
{
logger.LogInformation($"Installing application '{appBundleInfo.AppName}' on " + (deviceName != null ? $" on device '{deviceName}'" : target.AsString()));
var appInstaller = new AppInstaller(processManager, deviceLoader, mainLog, verbosity);
ProcessExecutionResult result;
try
{
(deviceName, result) = await appInstaller.InstallApp(_arguments.AppPackagePath, target, cancellationToken : cancellationToken);
}
catch (NoDeviceFoundException)
{
logger.LogError($"Failed to find suitable device for target {target.AsString()}");
return(ExitCode.DEVICE_NOT_FOUND);
}
catch (Exception e)
{
logger.LogError($"Failed to install the app bundle:{Environment.NewLine}{e}");
return(ExitCode.PACKAGE_INSTALLATION_FAILURE);
}
if (!result.Succeeded)
{
// use the knowledge base class to decide if the error is known, if it is, let the user know
// the failure reason
if (_errorKnowledgeBase.IsKnownInstallIssue(mainLog, out var errorMessage))
{
logger.LogError($"Failed to install the app bundle (exit code={result.ExitCode}): {errorMessage}");
}
else
{
logger.LogError($"Failed to install the app bundle (exit code={result.ExitCode})");
}
return(ExitCode.PACKAGE_INSTALLATION_FAILURE);
}
logger.LogInformation($"Application '{appBundleInfo.AppName}' was installed successfully on device '{deviceName}'");
}
logger.LogInformation($"Starting application '{appBundleInfo.AppName}' on " + (deviceName != null ? $"device '{deviceName}'" : target.AsString()));
var appRunner = new AppRunner(
processManager,
deviceLoader,
simulatorLoader,
new SimpleListenerFactory(tunnelBore),
new CrashSnapshotReporterFactory(processManager),
new CaptureLogFactory(),
new DeviceLogCapturerFactory(processManager),
new TestReporterFactory(processManager),
mainLog,
logs,
new Helpers(),
useXmlOutput: true); // the cli ALWAYS will get the output as xml
try
{
string resultMessage;
TestExecutingResult testResult;
(deviceName, testResult, resultMessage) = await appRunner.RunApp(appBundleInfo,
target,
_arguments.Timeout,
_arguments.LaunchTimeout,
deviceName,
verbosity : verbosity,
xmlResultJargon : _arguments.XmlResultJargon,
cancellationToken : cancellationToken);
switch (testResult)
{
case TestExecutingResult.Succeeded:
logger.LogInformation($"Application finished the test run successfully");
logger.LogInformation(resultMessage);
return(ExitCode.SUCCESS);
case TestExecutingResult.Failed:
logger.LogInformation($"Application finished the test run successfully with some failed tests");
logger.LogInformation(resultMessage);
return(ExitCode.TESTS_FAILED);
case TestExecutingResult.Crashed:
if (resultMessage != null)
{
logger.LogError($"Application run crashed:{Environment.NewLine}" +
$"{resultMessage}{Environment.NewLine}{Environment.NewLine}" +
$"Check logs for more information.");
}
else
{
logger.LogError($"Application run crashed. Check logs for more information");
}
return(ExitCode.APP_CRASH);
case TestExecutingResult.TimedOut:
logger.LogWarning($"Application run timed out");
return(ExitCode.TIMED_OUT);
default:
if (resultMessage != null)
{
logger.LogError($"Application run ended in an unexpected way: '{testResult}'{Environment.NewLine}" +
$"{resultMessage}{Environment.NewLine}{Environment.NewLine}" +
$"Check logs for more information.");
}
else
{
logger.LogError($"Application run ended in an unexpected way: '{testResult}'. Check logs for more information");
}
return(ExitCode.GENERAL_FAILURE);
}
}
catch (NoDeviceFoundException)
{
logger.LogError($"Failed to find suitable device for target {target.AsString()}");
return(ExitCode.DEVICE_NOT_FOUND);
}
catch (Exception e)
{
if (_errorKnowledgeBase.IsKnownTestIssue(mainLog, out var failureMessage))
{
logger.LogError($"Application run failed:{Environment.NewLine}{failureMessage}");
}
else
{
logger.LogError($"Application run failed:{Environment.NewLine}{e}");
}
return(ExitCode.APP_CRASH);
}
finally
{
if (!target.IsSimulator() && deviceName != null)
{
logger.LogInformation($"Uninstalling the application '{appBundleInfo.AppName}' from device '{deviceName}'");
var appUninstaller = new AppUninstaller(processManager, mainLog, verbosity);
var uninstallResult = await appUninstaller.UninstallApp(deviceName, appBundleInfo.BundleIdentifier, cancellationToken);
if (!uninstallResult.Succeeded)
{
logger.LogError($"Failed to uninstall the app bundle with exit code: {uninstallResult.ExitCode}");
}
else
{
logger.LogInformation($"Application '{appBundleInfo.AppName}' was uninstalled successfully");
}
}
}
}
public RunSimulatorTask(Jenkins jenkins, ISimulatorLoader simulators, MSBuildTask buildTask, IMlaunchProcessManager processManager, IEnumerable <ISimulatorDevice> candidates = null)
: base(jenkins, buildTask, processManager, candidates) => runSimulator = new RunSimulator(
public DeviceFinder(IHardwareDeviceLoader deviceLoader, ISimulatorLoader simulatorLoader)
{
_deviceLoader = deviceLoader ?? throw new ArgumentNullException(nameof(deviceLoader));
_simulatorLoader = simulatorLoader ?? throw new ArgumentNullException(nameof(simulatorLoader));
}
private async Task <ExitCode> RunTest(TestTarget target,
Logs logs,
ProcessManager processManager,
IHardwareDeviceLoader deviceLoader,
ISimulatorLoader simulatorLoader,
CancellationToken cancellationToken = default)
{
_log.LogInformation($"Starting test for {target.AsString()}{ (_arguments.DeviceName != null ? " targeting " + _arguments.DeviceName : null) }..");
string mainLogFile = Path.Join(_arguments.OutputDirectory, $"run-{target}{(_arguments.DeviceName != null ? "-" + _arguments.DeviceName : null)}.log");
ILog mainLog = logs.Create(mainLogFile, LogType.ExecutionLog.ToString(), true);
int verbosity = _arguments.Verbosity.ToInt();
string deviceName = _arguments.DeviceName;
var appBundleInformationParser = new AppBundleInformationParser(processManager);
AppBundleInformation appBundleInfo;
try
{
appBundleInfo = await appBundleInformationParser.ParseFromAppBundle(_arguments.AppPackagePath, target, mainLog, cancellationToken);
}
catch (Exception e)
{
_log.LogError($"Failed to get bundle information: {e.Message}");
return(ExitCode.FAILED_TO_GET_BUNDLE_INFO);
}
if (!target.IsSimulator())
{
_log.LogInformation($"Installing application '{appBundleInfo.AppName}' on " + (deviceName != null ? " on device '{deviceName}'" : target.AsString()));
var appInstaller = new AppInstaller(processManager, deviceLoader, mainLog, verbosity);
ProcessExecutionResult result;
try
{
(deviceName, result) = await appInstaller.InstallApp(_arguments.AppPackagePath, target, cancellationToken : cancellationToken);
}
catch (NoDeviceFoundException)
{
_log.LogError($"Failed to find suitable device for target {target.AsString()}");
return(ExitCode.DEVICE_NOT_FOUND);
}
catch (Exception e)
{
_log.LogError($"Failed to install the app bundle:{Environment.NewLine}{e}");
return(ExitCode.PACKAGE_INSTALLATION_FAILURE);
}
if (!result.Succeeded)
{
_log.LogError($"Failed to install the app bundle (exit code={result.ExitCode})");
return(ExitCode.PACKAGE_INSTALLATION_FAILURE);
}
_log.LogInformation($"Application '{appBundleInfo.AppName}' was installed successfully on device '{deviceName}'");
}
_log.LogInformation($"Starting application '{appBundleInfo.AppName}' on " + (deviceName != null ? " on device '{deviceName}'" : target.AsString()));
int exitCode;
try
{
var appRunner = new AppRunner(
processManager,
deviceLoader,
simulatorLoader,
new SimpleListenerFactory(),
new CrashSnapshotReporterFactory(processManager),
new CaptureLogFactory(),
new DeviceLogCapturerFactory(processManager),
new TestReporterFactory(processManager),
mainLog,
logs,
new Helpers());
(deviceName, exitCode) = await appRunner.RunApp(appBundleInfo,
target,
_arguments.Timeout,
_arguments.LaunchTimeout,
deviceName,
verbosity : verbosity,
xmlResultJargon : XmlResultJargon.xUnit);
if (exitCode != 0)
{
_log.LogError($"App bundle run failed with exit code {exitCode}. Check logs for more information");
}
else
{
_log.LogInformation("Application finished the run successfully");
}
return(0);
}
catch (NoDeviceFoundException)
{
_log.LogError($"Failed to find suitable device for target {target.AsString()}");
return(ExitCode.DEVICE_NOT_FOUND);
}
catch (Exception e)
{
_log.LogError($"Application run failed:{Environment.NewLine}{e}");
return(ExitCode.APP_CRASH);
}
finally
{
if (!target.IsSimulator())
{
_log.LogInformation($"Uninstalling the application '{appBundleInfo.AppName}' from device '{deviceName}'");
var appUninstaller = new AppUninstaller(processManager, mainLog, verbosity);
var uninstallResult = await appUninstaller.UninstallApp(deviceName, appBundleInfo.BundleIdentifier, cancellationToken);
if (!uninstallResult.Succeeded)
{
_log.LogError($"Failed to uninstall the app bundle with exit code: {uninstallResult.ExitCode}");
}
else
{
_log.LogInformation($"Application '{appBundleInfo.AppName}' was uninstalled successfully");
}
}
}
}
public SimulatorEnumerable(ISimulatorLoader simulators, TestTarget target, bool minVersion, ILog log)
{
_findTask = new Lazy <Task <(ISimulatorDevice, ISimulatorDevice?)> >(() => simulators.FindSimulators(target, log, minVersion: minVersion), LazyThreadSafetyMode.ExecutionAndPublication);
_toString = $"Simulators for {target} (MinVersion: {minVersion})";
}
private async Task <ExitCode> RunTest(
ILogger logger,
TestTargetOs target,
Logs logs,
MLaunchProcessManager processManager,
IHardwareDeviceLoader deviceLoader,
ISimulatorLoader simulatorLoader,
ITunnelBore?tunnelBore,
CancellationToken cancellationToken = default)
{
var isLldbEnabled = IsLldbEnabled();
if (isLldbEnabled && !_arguments.EnableLldb)
{
// the file is present, but the user did not set it, warn him about it
logger.LogWarning("Lldb will be used since '~/.mtouch-launch-with-lldb' was found in the system but it was not created by xharness.");
}
else if (_arguments.EnableLldb)
{
if (!File.Exists(s_mlaunchLldbConfigFile))
{
// create empty file
File.WriteAllText(s_mlaunchLldbConfigFile, string.Empty);
_createdLldbFile = true;
}
}
logger.LogInformation($"Starting test for {target.AsString()}{ (_arguments.DeviceName != null ? " targeting " + _arguments.DeviceName : null) }..");
string mainLogFile = Path.Join(_arguments.OutputDirectory, $"run-{target.AsString()}{(_arguments.DeviceName != null ? "-" + _arguments.DeviceName : null)}.log");
IFileBackedLog mainLog = Log.CreateReadableAggregatedLog(
logs.Create(mainLogFile, LogType.ExecutionLog.ToString(), true),
new CallbackLog(message => logger.LogDebug(message.Trim()))
{
Timestamp = false
});
int verbosity = GetMlaunchVerbosity(_arguments.Verbosity);
string?deviceName = _arguments.DeviceName;
var appBundleInformationParser = new AppBundleInformationParser(processManager);
AppBundleInformation appBundleInfo;
try
{
appBundleInfo = await appBundleInformationParser.ParseFromAppBundle(_arguments.AppPackagePath, target.Platform, mainLog, cancellationToken);
}
catch (Exception e)
{
logger.LogError($"Failed to get bundle information: {e.Message}");
return(ExitCode.FAILED_TO_GET_BUNDLE_INFO);
}
if (!target.Platform.IsSimulator())
{
logger.LogInformation($"Installing application '{appBundleInfo.AppName}' on " + (deviceName != null ? $" on device '{deviceName}'" : target.AsString()));
var appInstaller = new AppInstaller(processManager, deviceLoader, mainLog, verbosity);
ProcessExecutionResult result;
try
{
(deviceName, result) = await appInstaller.InstallApp(appBundleInfo, target, cancellationToken : cancellationToken);
}
catch (NoDeviceFoundException)
{
logger.LogError($"Failed to find suitable device for target {target.AsString()}" + Environment.NewLine +
"Please make sure the device is connected and unlocked.");
return(ExitCode.DEVICE_NOT_FOUND);
}
catch (Exception e)
{
logger.LogError($"Failed to install the app bundle:{Environment.NewLine}{e}");
return(ExitCode.PACKAGE_INSTALLATION_FAILURE);
}
if (!result.Succeeded)
{
// use the knowledge base class to decide if the error is known, if it is, let the user know
// the failure reason
if (_errorKnowledgeBase.IsKnownInstallIssue(mainLog, out var errorMessage))
{
var msg = $"Failed to install the app bundle (exit code={result.ExitCode}): {errorMessage.Value.HumanMessage}.";
if (errorMessage.Value.IssueLink != null)
{
msg += $" Find more information at {errorMessage.Value.IssueLink}";
}
logger.LogError(msg);
}
else
{
logger.LogError($"Failed to install the app bundle (exit code={result.ExitCode})");
}
return(ExitCode.PACKAGE_INSTALLATION_FAILURE);
}
logger.LogInformation($"Application '{appBundleInfo.AppName}' was installed successfully on device '{deviceName}'");
}
logger.LogInformation($"Starting application '{appBundleInfo.AppName}' on " + (deviceName != null ? $"device '{deviceName}'" : target.AsString()));
// only add the extra callback if we do know that the feature was indeed enabled
Action <string>?logCallback = isLldbEnabled ? (l) => NotifyUserLldbCommand(logger, l) : (Action <string>?)null;
var appRunner = new AppRunner(
processManager,
deviceLoader,
simulatorLoader,
new SimpleListenerFactory(tunnelBore),
new CrashSnapshotReporterFactory(processManager),
new CaptureLogFactory(),
new DeviceLogCapturerFactory(processManager),
new TestReporterFactory(processManager),
new XmlResultParser(),
mainLog,
logs,
new Helpers(),
logCallback: logCallback,
appArguments: PassThroughArguments);
try
{
string resultMessage;
TestExecutingResult testResult;
(deviceName, testResult, resultMessage) = await appRunner.RunApp(appBundleInfo,
target,
_arguments.Timeout,
_arguments.LaunchTimeout,
deviceName,
verbosity : verbosity,
xmlResultJargon : _arguments.XmlResultJargon,
cancellationToken : cancellationToken,
skippedMethods : _arguments.SingleMethodFilters?.ToArray(),
skippedTestClasses : _arguments.ClassMethodFilters?.ToArray());
switch (testResult)
{
case TestExecutingResult.Succeeded:
logger.LogInformation($"Application finished the test run successfully");
logger.LogInformation(resultMessage);
return(ExitCode.SUCCESS);
case TestExecutingResult.Failed:
logger.LogInformation($"Application finished the test run successfully with some failed tests");
logger.LogInformation(resultMessage);
return(ExitCode.TESTS_FAILED);
case TestExecutingResult.LaunchFailure:
if (resultMessage != null)
{
logger.LogError($"Failed to launch the application:{Environment.NewLine}" +
$"{resultMessage}{Environment.NewLine}{Environment.NewLine}" +
$"Check logs for more information.");
}
else
{
logger.LogError($"Failed to launch the application. Check logs for more information");
}
return(ExitCode.APP_LAUNCH_FAILURE);
case TestExecutingResult.Crashed:
if (resultMessage != null)
{
logger.LogError($"Application run crashed:{Environment.NewLine}" +
$"{resultMessage}{Environment.NewLine}{Environment.NewLine}" +
$"Check logs for more information.");
}
else
{
logger.LogError($"Application run crashed. Check logs for more information");
}
return(ExitCode.APP_CRASH);
case TestExecutingResult.TimedOut:
logger.LogWarning($"Application run timed out");
return(ExitCode.TIMED_OUT);
default:
if (resultMessage != null)
{
logger.LogError($"Application run ended in an unexpected way: '{testResult}'{Environment.NewLine}" +
$"{resultMessage}{Environment.NewLine}{Environment.NewLine}" +
$"Check logs for more information.");
}
else
{
logger.LogError($"Application run ended in an unexpected way: '{testResult}'. Check logs for more information");
}
return(ExitCode.GENERAL_FAILURE);
}
}
catch (NoDeviceFoundException)
{
logger.LogError($"Failed to find suitable device for target {target.AsString()}" +
(target.Platform.IsSimulator() ? Environment.NewLine + "Please make sure suitable Simulator is installed in Xcode" : string.Empty));
return(ExitCode.DEVICE_NOT_FOUND);
}
catch (Exception e)
{
if (_errorKnowledgeBase.IsKnownTestIssue(mainLog, out var failureMessage))
{
var msg = $"Application run failed:{Environment.NewLine}{failureMessage.Value.HumanMessage}.";
if (failureMessage.Value.IssueLink != null)
{
msg += $" Find more information at {failureMessage.Value.IssueLink}";
}
logger.LogError(msg);
}
else
{
logger.LogError($"Application run failed:{Environment.NewLine}{e}");
}
return(ExitCode.APP_CRASH);
}
finally
{
mainLog.Dispose();
if (!target.Platform.IsSimulator() && deviceName != null)
{
logger.LogInformation($"Uninstalling the application '{appBundleInfo.AppName}' from device '{deviceName}'");
var appUninstaller = new AppUninstaller(processManager, mainLog, verbosity);
var uninstallResult = await appUninstaller.UninstallApp(deviceName, appBundleInfo.BundleIdentifier, cancellationToken);
if (!uninstallResult.Succeeded)
{
logger.LogError($"Failed to uninstall the app bundle with exit code: {uninstallResult.ExitCode}");
}
else
{
logger.LogInformation($"Application '{appBundleInfo.AppName}' was uninstalled successfully");
}
}
if (_createdLldbFile) // clean after the setting
{
File.Delete(s_mlaunchLldbConfigFile);
}
}
}
