public async Task RunTestAsync()
{
mainLog.WriteLine("Running XI on '{0}' ({2}) for {1}", testTask.Device?.Name, testTask.ProjectFile, testTask.Device?.UDID);
testTask.ExecutionResult = testTask.ExecutionResult & ~TestExecutingResult.InProgressMask | TestExecutingResult.Running;
await testTask.BuildTask.RunAsync();
if (!testTask.BuildTask.Succeeded)
{
testTask.ExecutionResult = TestExecutingResult.BuildFailure;
return;
}
using (var resource = await testTask.NotifyBlockingWaitAsync(testTask.AcquireResourceAsync())) {
if (testTask.Runner == null)
{
await SelectSimulatorAsync();
}
await testTask.Runner.RunAsync();
}
testTask.ExecutionResult = testTask.Runner.Result;
testTask.KnownFailure = null;
if (errorKnowledgeBase.IsKnownTestIssue(testTask.Runner.MainLog, out var failure))
{
testTask.KnownFailure = failure;
mainLog.WriteLine($"Test run has a known failure: '{testTask.KnownFailure}'");
}
}
private async Task <ExitCode> OrchestrateOperationInternal(
TestTargetOs target,
string?deviceName,
bool includeWirelessDevices,
bool resetSimulator,
bool enableLldb,
GetAppBundleInfoFunc getAppBundle,
ExecuteMacCatalystAppFunc executeMacCatalystApp,
ExecuteAppFunc executeApp,
CancellationToken cancellationToken)
{
_lldbFileCreated = false;
var isLldbEnabled = IsLldbEnabled();
if (isLldbEnabled && !enableLldb)
{
// the file is present, but the user did not set it, warn him about it
_logger.LogWarning("Lldb will be used since '~/.mtouch-launch-with-lldb' was found in the system but it was not created by xharness.");
}
else if (enableLldb)
{
if (!File.Exists(s_mlaunchLldbConfigFile))
{
// create empty file
File.WriteAllText(s_mlaunchLldbConfigFile, string.Empty);
_lldbFileCreated = true;
}
}
if (includeWirelessDevices && target.Platform.IsSimulator())
{
_logger.LogWarning("Including wireless devices while targeting a simulator has no effect");
}
if (resetSimulator && !target.Platform.IsSimulator())
{
_logger.LogWarning("Targeting device but requesting simulator reset has no effect");
resetSimulator = false;
}
ExitCode exitCode;
IDevice device;
IDevice? companionDevice;
AppBundleInformation appBundleInfo;
if (target.Platform == TestTarget.MacCatalyst)
{
try
{
appBundleInfo = await getAppBundle(target, null !, cancellationToken);
}
catch (Exception e)
{
cancellationToken.ThrowIfCancellationRequested();
_logger.LogError(e.Message);
return(ExitCode.PACKAGE_NOT_FOUND);
}
try
{
return(await executeMacCatalystApp(appBundleInfo));
}
catch (Exception e)
{
var message = new StringBuilder().AppendLine("Application run failed:");
exitCode = ExitCode.APP_LAUNCH_FAILURE;
if (_errorKnowledgeBase.IsKnownTestIssue(_mainLog, out var failure))
{
message.Append(failure.HumanMessage);
if (failure.IssueLink != null)
{
message.AppendLine($" Find more information at {failure.IssueLink}");
}
if (failure.SuggestedExitCode.HasValue)
{
exitCode = (ExitCode)failure.SuggestedExitCode.Value;
}
}
else
{
message.AppendLine(e.ToString());
}
_logger.LogError(message.ToString());
return(exitCode);
}
}
try
{
_logger.LogInformation($"Looking for available {target.AsString()} {(target.Platform.IsSimulator() ? "simulators" : "devices")}..");
var finderLogName = $"list-{target.AsString()}-{_helpers.Timestamp}.log";
using var finderLog = _logs.Create(finderLogName, "DeviceList", true);
_mainLog.WriteLine(
$"Looking for available {target.AsString()} {(target.Platform.IsSimulator() ? "simulators" : "devices")}. " +
$"Storing logs into {finderLogName}");
(device, companionDevice) = await _deviceFinder.FindDevice(target, deviceName, finderLog, includeWirelessDevices, cancellationToken);
_logger.LogInformation($"Found {(target.Platform.IsSimulator() ? "simulator" : "physical")} device '{device.Name}'");
if (companionDevice != null)
{
_logger.LogInformation($"Found companion {(target.Platform.IsSimulator() ? "simulator" : "physical")} device '{companionDevice.Name}'");
}
}
catch (NoDeviceFoundException e)
{
_logger.LogError(e.Message);
return(ExitCode.DEVICE_NOT_FOUND);
}
cancellationToken.ThrowIfCancellationRequested();
try
{
appBundleInfo = await getAppBundle(target, device, cancellationToken);
}
catch (Exception e)
{
cancellationToken.ThrowIfCancellationRequested();
_logger.LogError(e.Message);
return(ExitCode.PACKAGE_NOT_FOUND);
}
cancellationToken.ThrowIfCancellationRequested();
if (target.Platform.IsSimulator() && resetSimulator)
{
try
{
var simulator = (ISimulatorDevice)device;
var bundleIds = appBundleInfo.BundleIdentifier == string.Empty ? Array.Empty <string>() : new[] { appBundleInfo.BundleIdentifier };
_logger.LogInformation($"Reseting simulator '{device.Name}'");
await simulator.PrepareSimulator(_mainLog, bundleIds);
if (companionDevice != null)
{
_logger.LogInformation($"Reseting companion simulator '{companionDevice.Name}'");
var companionSimulator = (ISimulatorDevice)companionDevice;
await companionSimulator.PrepareSimulator(_mainLog, bundleIds);
}
_logger.LogInformation("Simulator reset finished");
}
catch (Exception e)
{
_logger.LogError($"Failed to reset simulator: " + Environment.NewLine + e);
return(ExitCode.SIMULATOR_FAILURE);
}
}
cancellationToken.ThrowIfCancellationRequested();
// Note down the actual test target
// For simulators (e.g. "iOS 13.4"), we strip the iOS part and keep the version only, for devices there's no OS
_diagnosticsData.TargetOS = target.Platform.IsSimulator() ? device.OSVersion.Split(' ', 2).Last() : device.OSVersion;
_diagnosticsData.Device = device.Name ?? device.UDID;
// Uninstall the app first to get a clean state
if (!resetSimulator)
{
await UninstallApp(target.Platform, appBundleInfo.BundleIdentifier, device, isPreparation : true, cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
}
exitCode = await InstallApp(appBundleInfo, device, target, cancellationToken);
if (exitCode != ExitCode.SUCCESS)
{
_logger.LogInformation($"Cleaning up the failed installation from '{device.Name}'");
var uninstallResult = await UninstallApp(target.Platform, appBundleInfo.BundleIdentifier, device, isPreparation : false, new CancellationToken());
if (uninstallResult == ExitCode.SIMULATOR_FAILURE)
{
// Sometimes the simulator gets in a bad shape and we won't be able to install the app, we can tell here
return(ExitCode.SIMULATOR_FAILURE);
}
return(exitCode);
}
try
{
exitCode = await executeApp(appBundleInfo, device, companionDevice);
}
catch (Exception e)
{
exitCode = ExitCode.GENERAL_FAILURE;
var message = new StringBuilder().AppendLine("Application run failed:");
if (_errorKnowledgeBase.IsKnownTestIssue(_mainLog, out var failure))
{
message.Append(failure.HumanMessage);
if (failure.IssueLink != null)
{
message.AppendLine($" Find more information at {failure.IssueLink}");
}
if (failure.SuggestedExitCode.HasValue)
{
exitCode = (ExitCode)failure.SuggestedExitCode.Value;
}
}
else
{
message.AppendLine(e.ToString());
}
_logger.LogError(message.ToString());
}
finally
{
if (target.Platform.IsSimulator() && resetSimulator)
{
await CleanUpSimulators(device, companionDevice);
}
else if (device != null) // Do not uninstall if device was cleaned up
{
var uninstallResult = await UninstallApp(target.Platform, appBundleInfo.BundleIdentifier, device, false, new CancellationToken());
// We are able to detect a case when simulator is in a bad shape
// If it also failed the test/run, we should present that as the failure
if (uninstallResult == ExitCode.SIMULATOR_FAILURE && exitCode != ExitCode.SUCCESS && exitCode != ExitCode.TESTS_FAILED)
{
exitCode = ExitCode.SIMULATOR_FAILURE;
}
}
}
return(exitCode);
}
public async Task RunTestAsync()
{
mainLog.WriteLine("Running '{0}' on device (candidates: '{1}')", testTask.ProjectFile, string.Join("', '", testTask.Candidates.Select((v) => v.Name).ToArray()));
var uninstall_log = testTask.Logs.Create($"uninstall-{Helpers.Timestamp}.log", "Uninstall log");
using (var device_resource = await testTask.NotifyBlockingWaitAsync(resourceManager.GetDeviceResources(testTask.Candidates).AcquireAnyConcurrentAsync())) {
try {
// Set the device we acquired.
testTask.Device = testTask.Candidates.First((d) => d.UDID == device_resource.Resource.Name);
if (testTask.Device.DevicePlatform == DevicePlatform.watchOS)
{
testTask.CompanionDevice = await devices.FindCompanionDevice(deviceLoadLog, testTask.Device);
}
mainLog.WriteLine("Acquired device '{0}' for '{1}'", testTask.Device.Name, testTask.ProjectFile);
ITunnelBore tunnelBore = null;
if (useTcpTunnel && testTask.Device.DevicePlatform != DevicePlatform.iOS &&
testTask.Device.DevicePlatform != DevicePlatform.tvOS)
{
mainLog.WriteLine("Ignoring request to use a tunnel because it is not supported by the specified platform");
}
else if (useTcpTunnel && (testTask.Device.DevicePlatform == DevicePlatform.iOS ||
testTask.Device.DevicePlatform == DevicePlatform.tvOS))
{
tunnelBore = testTask.TunnelBore;
mainLog.WriteLine("Using tunnel to communicate with device.");
}
testTask.Runner = new AppRunner(testTask.ProcessManager,
new AppBundleInformationParser(),
new SimulatorLoaderFactory(testTask.ProcessManager),
new SimpleListenerFactory(tunnelBore),
new DeviceLoaderFactory(testTask.ProcessManager),
new CrashSnapshotReporterFactory(testTask.ProcessManager),
new CaptureLogFactory(),
new DeviceLogCapturerFactory(testTask.ProcessManager),
new TestReporterFactory(testTask.ProcessManager),
testTask.AppRunnerTarget,
testTask.Harness,
projectFilePath: testTask.ProjectFile,
mainLog: uninstall_log,
logs: new Logs(testTask.LogDirectory ?? defaultLogDirectory),
buildConfiguration: testTask.ProjectConfiguration,
deviceName: testTask.Device.Name,
companionDeviceName: testTask.CompanionDevice?.Name,
timeoutMultiplier: testTask.TimeoutMultiplier,
variation: testTask.Variation,
buildTask: testTask.BuildTask);
await testTask.Runner.InitializeAsync();
// Sometimes devices can't upgrade (depending on what has changed), so make sure to uninstall any existing apps first.
if (uninstallTestApp)
{
testTask.Runner.MainLog = uninstall_log;
var uninstall_result = await testTask.Runner.UninstallAsync();
if (!uninstall_result.Succeeded)
{
mainLog.WriteLine($"Pre-run uninstall failed, exit code: {uninstall_result.ExitCode} (this hopefully won't affect the test result)");
}
}
else
{
uninstall_log.WriteLine($"Pre-run uninstall skipped.");
}
if (!testTask.Failed)
{
// Install the app
InstallLog = new AppInstallMonitorLog(testTask.Logs.Create($"install-{Helpers.Timestamp}.log", "Install log"));
try {
testTask.Runner.MainLog = this.InstallLog;
var install_result = await testTask.Runner.InstallAsync(InstallLog.CancellationToken);
if (!install_result.Succeeded)
{
testTask.FailureMessage = $"Install failed, exit code: {install_result.ExitCode}.";
testTask.ExecutionResult = TestExecutingResult.Failed;
if (generateXmlFailures)
{
resultParser.GenerateFailure(
testTask.Logs,
"install",
testTask.Runner.AppInformation.AppName,
testTask.Variation,
$"AppInstallation on {testTask.Device.Name}",
$"Install failed on {testTask.Device.Name}, exit code: {install_result.ExitCode}",
InstallLog.FullPath,
xmlResultJargon);
}
}
} finally {
InstallLog.Dispose();
InstallLog = null;
}
}
if (!testTask.Failed)
{
// Run the app
testTask.Runner.MainLog = testTask.Logs.Create($"run-{testTask.Device.UDID}-{Helpers.Timestamp}.log", "Run log");
await testTask.Runner.RunAsync();
if (!string.IsNullOrEmpty(testTask.Runner.FailureMessage))
{
testTask.FailureMessage = testTask.Runner.FailureMessage;
}
else if (testTask.Runner.Result != TestExecutingResult.Succeeded)
{
testTask.FailureMessage = testTask.GuessFailureReason(testTask.Runner.MainLog);
}
if (string.IsNullOrEmpty(testTask.FailureMessage) && errorKnowledgeBase.IsKnownTestIssue(testTask.Runner.MainLog, out var failure))
{
testTask.KnownFailure = failure;
mainLog.WriteLine($"Test run has a known failure: '{testTask.KnownFailure}'");
}
if (testTask.Runner.Result == TestExecutingResult.Succeeded && testTask.Platform == TestPlatform.iOS_TodayExtension64)
{
// For the today extension, the main app is just a single test.
// This is because running the today extension will not wake up the device,
// nor will it close & reopen the today app (but launching the main app
// will do both of these things, preparing the device for launching the today extension).
AppRunner todayRunner = new AppRunner(testTask.ProcessManager,
new AppBundleInformationParser(),
new SimulatorLoaderFactory(testTask.ProcessManager),
new SimpleListenerFactory(tunnelBore),
new DeviceLoaderFactory(testTask.ProcessManager),
new CrashSnapshotReporterFactory(testTask.ProcessManager),
new CaptureLogFactory(),
new DeviceLogCapturerFactory(testTask.ProcessManager),
new TestReporterFactory(testTask.ProcessManager),
testTask.AppRunnerTarget,
testTask.Harness,
projectFilePath: testTask.ProjectFile,
mainLog: testTask.Logs.Create($"extension-run-{testTask.Device.UDID}-{Helpers.Timestamp}.log", "Extension run log"),
logs: new Logs(testTask.LogDirectory ?? defaultLogDirectory),
buildConfiguration: testTask.ProjectConfiguration,
deviceName: testTask.Device.Name,
companionDeviceName: testTask.CompanionDevice?.Name,
timeoutMultiplier: testTask.TimeoutMultiplier,
variation: testTask.Variation,
buildTask: testTask.BuildTask);
await todayRunner.InitializeAsync();
testTask.AdditionalRunner = todayRunner;
await todayRunner.RunAsync();
foreach (var log in todayRunner.Logs.Where((v) => !v.Description.StartsWith("Extension ", StringComparison.Ordinal)))
{
log.Description = "Extension " + log.Description [0].ToString().ToLower() + log.Description.Substring(1);
}
testTask.ExecutionResult = todayRunner.Result;
if (!string.IsNullOrEmpty(todayRunner.FailureMessage))
{
testTask.FailureMessage = todayRunner.FailureMessage;
}
}
else
{
testTask.ExecutionResult = testTask.Runner.Result;
}
}
} finally {
// Uninstall again, so that we don't leave junk behind and fill up the device.
if (uninstallTestApp)
{
testTask.Runner.MainLog = uninstall_log;
var uninstall_result = await testTask.Runner.UninstallAsync();
if (!uninstall_result.Succeeded)
{
mainLog.WriteLine($"Post-run uninstall failed, exit code: {uninstall_result.ExitCode} (this won't affect the test result)");
}
}
else
{
uninstall_log.WriteLine($"Post-run uninstall skipped.");
}
// Also clean up after us locally.
if (inCI || cleanSuccessfulTestRuns && testTask.Succeeded)
{
await testTask.BuildTask.CleanAsync();
}
}
}
}
