public int Install()
{
if (HarnessLog == null)
{
HarnessLog = new ConsoleLog();
}
foreach (var project in IOSTestProjects)
{
var runner = new AppRunner()
{
Harness = this,
ProjectFile = project.Path,
MainLog = HarnessLog,
};
using (var install_log = new AppInstallMonitorLog(runner.MainLog)) {
var rv = runner.InstallAsync(install_log.CancellationToken).Result;
if (!rv.Succeeded)
{
return(rv.ExitCode);
}
}
}
return(0);
}
public async Task RunTestAsync()
{
mainLog.WriteLine("Running '{0}' on device (candidates: '{1}')", testTask.ProjectFile, string.Join("', '", testTask.Candidates.Select((v) => v.Name).ToArray()));
var uninstall_log = testTask.Logs.Create($"uninstall-{Helpers.Timestamp}.log", "Uninstall log");
using (var device_resource = await testTask.NotifyBlockingWaitAsync(resourceManager.GetDeviceResources(testTask.Candidates).AcquireAnyConcurrentAsync())) {
try {
// Set the device we acquired.
testTask.Device = testTask.Candidates.First((d) => d.UDID == device_resource.Resource.Name);
if (testTask.Device.DevicePlatform == DevicePlatform.watchOS)
{
testTask.CompanionDevice = await devices.FindCompanionDevice(deviceLoadLog, testTask.Device);
}
mainLog.WriteLine("Acquired device '{0}' for '{1}'", testTask.Device.Name, testTask.ProjectFile);
ITunnelBore tunnelBore = null;
if (useTcpTunnel && testTask.Device.DevicePlatform != DevicePlatform.iOS &&
testTask.Device.DevicePlatform != DevicePlatform.tvOS)
{
mainLog.WriteLine("Ignoring request to use a tunnel because it is not supported by the specified platform");
}
else if (useTcpTunnel && (testTask.Device.DevicePlatform == DevicePlatform.iOS ||
testTask.Device.DevicePlatform == DevicePlatform.tvOS))
{
tunnelBore = testTask.TunnelBore;
mainLog.WriteLine("Using tunnel to communicate with device.");
}
testTask.Runner = new AppRunner(testTask.ProcessManager,
new AppBundleInformationParser(),
new SimulatorLoaderFactory(testTask.ProcessManager),
new SimpleListenerFactory(tunnelBore),
new DeviceLoaderFactory(testTask.ProcessManager),
new CrashSnapshotReporterFactory(testTask.ProcessManager),
new CaptureLogFactory(),
new DeviceLogCapturerFactory(testTask.ProcessManager),
new TestReporterFactory(testTask.ProcessManager),
testTask.AppRunnerTarget,
testTask.Harness,
projectFilePath: testTask.ProjectFile,
mainLog: uninstall_log,
logs: new Logs(testTask.LogDirectory ?? defaultLogDirectory),
buildConfiguration: testTask.ProjectConfiguration,
deviceName: testTask.Device.Name,
companionDeviceName: testTask.CompanionDevice?.Name,
timeoutMultiplier: testTask.TimeoutMultiplier,
variation: testTask.Variation,
buildTask: testTask.BuildTask);
await testTask.Runner.InitializeAsync();
// Sometimes devices can't upgrade (depending on what has changed), so make sure to uninstall any existing apps first.
if (uninstallTestApp)
{
testTask.Runner.MainLog = uninstall_log;
var uninstall_result = await testTask.Runner.UninstallAsync();
if (!uninstall_result.Succeeded)
{
mainLog.WriteLine($"Pre-run uninstall failed, exit code: {uninstall_result.ExitCode} (this hopefully won't affect the test result)");
}
}
else
{
uninstall_log.WriteLine($"Pre-run uninstall skipped.");
}
if (!testTask.Failed)
{
// Install the app
InstallLog = new AppInstallMonitorLog(testTask.Logs.Create($"install-{Helpers.Timestamp}.log", "Install log"));
try {
testTask.Runner.MainLog = this.InstallLog;
var install_result = await testTask.Runner.InstallAsync(InstallLog.CancellationToken);
if (!install_result.Succeeded)
{
testTask.FailureMessage = $"Install failed, exit code: {install_result.ExitCode}.";
testTask.ExecutionResult = TestExecutingResult.Failed;
if (generateXmlFailures)
{
resultParser.GenerateFailure(
testTask.Logs,
"install",
testTask.Runner.AppInformation.AppName,
testTask.Variation,
$"AppInstallation on {testTask.Device.Name}",
$"Install failed on {testTask.Device.Name}, exit code: {install_result.ExitCode}",
InstallLog.FullPath,
xmlResultJargon);
}
}
} finally {
InstallLog.Dispose();
InstallLog = null;
}
}
if (!testTask.Failed)
{
// Run the app
testTask.Runner.MainLog = testTask.Logs.Create($"run-{testTask.Device.UDID}-{Helpers.Timestamp}.log", "Run log");
await testTask.Runner.RunAsync();
if (!string.IsNullOrEmpty(testTask.Runner.FailureMessage))
{
testTask.FailureMessage = testTask.Runner.FailureMessage;
}
else if (testTask.Runner.Result != TestExecutingResult.Succeeded)
{
testTask.FailureMessage = testTask.GuessFailureReason(testTask.Runner.MainLog);
}
if (string.IsNullOrEmpty(testTask.FailureMessage) && errorKnowledgeBase.IsKnownTestIssue(testTask.Runner.MainLog, out var failure))
{
testTask.KnownFailure = failure;
mainLog.WriteLine($"Test run has a known failure: '{testTask.KnownFailure}'");
}
if (testTask.Runner.Result == TestExecutingResult.Succeeded && testTask.Platform == TestPlatform.iOS_TodayExtension64)
{
// For the today extension, the main app is just a single test.
// This is because running the today extension will not wake up the device,
// nor will it close & reopen the today app (but launching the main app
// will do both of these things, preparing the device for launching the today extension).
AppRunner todayRunner = new AppRunner(testTask.ProcessManager,
new AppBundleInformationParser(),
new SimulatorLoaderFactory(testTask.ProcessManager),
new SimpleListenerFactory(tunnelBore),
new DeviceLoaderFactory(testTask.ProcessManager),
new CrashSnapshotReporterFactory(testTask.ProcessManager),
new CaptureLogFactory(),
new DeviceLogCapturerFactory(testTask.ProcessManager),
new TestReporterFactory(testTask.ProcessManager),
testTask.AppRunnerTarget,
testTask.Harness,
projectFilePath: testTask.ProjectFile,
mainLog: testTask.Logs.Create($"extension-run-{testTask.Device.UDID}-{Helpers.Timestamp}.log", "Extension run log"),
logs: new Logs(testTask.LogDirectory ?? defaultLogDirectory),
buildConfiguration: testTask.ProjectConfiguration,
deviceName: testTask.Device.Name,
companionDeviceName: testTask.CompanionDevice?.Name,
timeoutMultiplier: testTask.TimeoutMultiplier,
variation: testTask.Variation,
buildTask: testTask.BuildTask);
await todayRunner.InitializeAsync();
testTask.AdditionalRunner = todayRunner;
await todayRunner.RunAsync();
foreach (var log in todayRunner.Logs.Where((v) => !v.Description.StartsWith("Extension ", StringComparison.Ordinal)))
{
log.Description = "Extension " + log.Description [0].ToString().ToLower() + log.Description.Substring(1);
}
testTask.ExecutionResult = todayRunner.Result;
if (!string.IsNullOrEmpty(todayRunner.FailureMessage))
{
testTask.FailureMessage = todayRunner.FailureMessage;
}
}
else
{
testTask.ExecutionResult = testTask.Runner.Result;
}
}
} finally {
// Uninstall again, so that we don't leave junk behind and fill up the device.
if (uninstallTestApp)
{
testTask.Runner.MainLog = uninstall_log;
var uninstall_result = await testTask.Runner.UninstallAsync();
if (!uninstall_result.Succeeded)
{
mainLog.WriteLine($"Post-run uninstall failed, exit code: {uninstall_result.ExitCode} (this won't affect the test result)");
}
}
else
{
uninstall_log.WriteLine($"Post-run uninstall skipped.");
}
// Also clean up after us locally.
if (inCI || cleanSuccessfulTestRuns && testTask.Succeeded)
{
await testTask.BuildTask.CleanAsync();
}
}
}
}
protected override async Task RunTestAsync()
{
Jenkins.MainLog.WriteLine("Running '{0}' on device (candidates: '{1}')", ProjectFile, string.Join("', '", Candidates.Select((v) => v.Name).ToArray()));
var uninstall_log = Logs.Create($"uninstall-{Timestamp}.log", "Uninstall log");
using (var device_resource = await NotifyBlockingWaitAsync(Jenkins.GetDeviceResources(Candidates).AcquireAnyConcurrentAsync())) {
try {
// Set the device we acquired.
Device = Candidates.First((d) => d.UDID == device_resource.Resource.Name);
if (Device.DevicePlatform == DevicePlatform.watchOS)
{
CompanionDevice = Jenkins.Devices.FindCompanionDevice(Jenkins.DeviceLoadLog, Device);
}
Jenkins.MainLog.WriteLine("Acquired device '{0}' for '{1}'", Device.Name, ProjectFile);
runner = new AppRunner {
Harness = Harness,
ProjectFile = ProjectFile,
Target = AppRunnerTarget,
LogDirectory = LogDirectory,
MainLog = uninstall_log,
DeviceName = Device.Name,
CompanionDeviceName = CompanionDevice?.Name,
Configuration = ProjectConfiguration,
TimeoutMultiplier = TimeoutMultiplier,
Variation = Variation,
BuildTask = BuildTask,
};
// Sometimes devices can't upgrade (depending on what has changed), so make sure to uninstall any existing apps first.
if (Jenkins.UninstallTestApp)
{
runner.MainLog = uninstall_log;
var uninstall_result = await runner.UninstallAsync();
if (!uninstall_result.Succeeded)
{
MainLog.WriteLine($"Pre-run uninstall failed, exit code: {uninstall_result.ExitCode} (this hopefully won't affect the test result)");
}
}
else
{
uninstall_log.WriteLine($"Pre-run uninstall skipped.");
}
if (!Failed)
{
// Install the app
this.install_log = new AppInstallMonitorLog(Logs.Create($"install-{Timestamp}.log", "Install log"));
try {
runner.MainLog = this.install_log;
var install_result = await runner.InstallAsync(install_log.CancellationToken);
if (!install_result.Succeeded)
{
FailureMessage = $"Install failed, exit code: {install_result.ExitCode}.";
ExecutionResult = TestExecutingResult.Failed;
if (Harness.InCI)
{
XmlResultParser.GenerateFailure(Logs, "install", runner.AppName, runner.Variation,
$"AppInstallation on {runner.DeviceName}", $"Install failed on {runner.DeviceName}, exit code: {install_result.ExitCode}",
install_log.FullPath, Harness.XmlJargon);
}
}
} finally {
this.install_log.Dispose();
this.install_log = null;
}
}
if (!Failed)
{
// Run the app
runner.MainLog = Logs.Create($"run-{Device.UDID}-{Timestamp}.log", "Run log");
await runner.RunAsync();
if (!string.IsNullOrEmpty(runner.FailureMessage))
{
FailureMessage = runner.FailureMessage;
}
else if (runner.Result != TestExecutingResult.Succeeded)
{
FailureMessage = GuessFailureReason(runner.MainLog);
}
if (runner.Result == TestExecutingResult.Succeeded && Platform == TestPlatform.iOS_TodayExtension64)
{
// For the today extension, the main app is just a single test.
// This is because running the today extension will not wake up the device,
// nor will it close & reopen the today app (but launching the main app
// will do both of these things, preparing the device for launching the today extension).
AppRunner todayRunner = new AppRunner {
Harness = Harness,
ProjectFile = TestProject.GetTodayExtension().Path,
Target = AppRunnerTarget,
LogDirectory = LogDirectory,
MainLog = Logs.Create($"extension-run-{Device.UDID}-{Timestamp}.log", "Extension run log"),
DeviceName = Device.Name,
CompanionDeviceName = CompanionDevice?.Name,
Configuration = ProjectConfiguration,
Variation = Variation,
BuildTask = BuildTask,
};
additional_runner = todayRunner;
await todayRunner.RunAsync();
foreach (var log in todayRunner.Logs.Where((v) => !v.Description.StartsWith("Extension ", StringComparison.Ordinal)))
{
log.Description = "Extension " + log.Description [0].ToString().ToLower() + log.Description.Substring(1);
}
ExecutionResult = todayRunner.Result;
if (!string.IsNullOrEmpty(todayRunner.FailureMessage))
{
FailureMessage = todayRunner.FailureMessage;
}
}
else
{
ExecutionResult = runner.Result;
}
}
} finally {
// Uninstall again, so that we don't leave junk behind and fill up the device.
if (Jenkins.UninstallTestApp)
{
runner.MainLog = uninstall_log;
var uninstall_result = await runner.UninstallAsync();
if (!uninstall_result.Succeeded)
{
MainLog.WriteLine($"Post-run uninstall failed, exit code: {uninstall_result.ExitCode} (this won't affect the test result)");
}
}
else
{
uninstall_log.WriteLine($"Post-run uninstall skipped.");
}
// Also clean up after us locally.
if (Harness.InCI || Jenkins.CleanSuccessfulTestRuns && Succeeded)
{
await BuildTask.CleanAsync();
}
}
}
}