public void InstallWhenNoDevicesTest()
{
var appRunner = new AppRunner(processManager.Object,
appBundleInformationParser,
simulatorsFactory,
listenerFactory,
devicesFactory,
snapshotReporterFactory,
Mock.Of <ICaptureLogFactory> (),
Mock.Of <IDeviceLogCapturerFactory> (),
Mock.Of <ITestReporterFactory> (),
TestTarget.Device_iOS,
Mock.Of <IHarness> (),
mainLog.Object,
logs.Object,
projectFilePath: projectFilePath,
buildConfiguration: "Debug");
appRunner.InitializeAsync().Wait();
devices.Setup(x => x.ConnectedDevices).Returns(new IHardwareDevice [0]);
Assert.ThrowsAsync <NoDeviceFoundException> (
async() => await appRunner.InstallAsync(new CancellationToken()),
"Install requires connected devices");
}
public void InitializeTest()
{
var appRunner = new AppRunner(processManager.Object,
appBundleInformationParser,
simulatorsFactory,
listenerFactory,
devicesFactory,
snapshotReporterFactory,
Mock.Of <ICaptureLogFactory> (),
Mock.Of <IDeviceLogCapturerFactory> (),
Mock.Of <ITestReporterFactory> (),
TestTarget.Simulator_iOS64,
Mock.Of <IHarness> (),
mainLog.Object,
logs.Object,
projectFilePath: projectFilePath,
buildConfiguration: "Debug");
appRunner.InitializeAsync().Wait();
Assert.AreEqual(appName, appRunner.AppInformation.AppName);
Assert.AreEqual(appPath, appRunner.AppInformation.AppPath);
Assert.AreEqual(appPath, appRunner.AppInformation.LaunchAppPath);
Assert.AreEqual(appName, appRunner.AppInformation.BundleIdentifier);
}
public void RunOnDeviceWithNoAvailableSimulatorTest()
{
devices.Setup(d => d.FindDevice(It.IsAny <RunMode> (), It.IsAny <ILog> (), false, false)).ReturnsAsync((IHardwareDevice)null);
simulators
.Setup(s => s.FindSimulators(It.IsAny <TestTarget> (), It.IsAny <ILog> (), true, false))
.ReturnsAsync((null, null));
// Crash reporter
var crashReporterFactory = new Mock <ICrashSnapshotReporterFactory> ();
crashReporterFactory
.Setup(x => x.Create(mainLog.Object, It.IsAny <ILogs> (), false, null))
.Returns(snapshotReporter.Object);
var listenerLogFile = new Mock <IFileBackedLog> ();
var testReporterFactory = new Mock <ITestReporterFactory> ();
var testReporter = new Mock <ITestReporter> ();
testReporterFactory.SetReturnsDefault(testReporter.Object);
testReporter.Setup(r => r.ParseResult()).Returns(() => {
return(Task.FromResult <(TestExecutingResult, string)> ((TestExecutingResult.Succeeded, null)));
});
testReporter.Setup(r => r.Success).Returns(true);
logs
.Setup(x => x.Create(It.IsAny <string> (), "TestLog", It.IsAny <bool> ()))
.Returns(listenerLogFile.Object);
simpleListener.SetupGet(x => x.ConnectedTask).Returns(Task.FromResult(true));
// Act
var appRunner = new AppRunner(processManager.Object,
appBundleInformationParser,
simulatorsFactory,
listenerFactory,
devicesFactory,
crashReporterFactory.Object,
Mock.Of <ICaptureLogFactory> (),
Mock.Of <IDeviceLogCapturerFactory> (),
testReporterFactory.Object,
TestTarget.Device_tvOS,
GetMockedHarness(),
mainLog.Object,
logs.Object,
projectFilePath: projectFilePath,
buildConfiguration: "Debug",
timeoutMultiplier: 2);
appRunner.InitializeAsync().Wait();
Assert.ThrowsAsync <NoDeviceFoundException> (
async() => await appRunner.RunAsync(),
"Running requires connected devices");
}
public async Task InstallOnDeviceTest()
{
var harness = Mock.Of <IHarness> (x => x.Verbosity == 2);
var processResult = new ProcessExecutionResult()
{
ExitCode = 1, TimedOut = false
};
processManager.SetReturnsDefault(Task.FromResult(processResult));
devices.Setup(d => d.FindDevice(It.IsAny <RunMode> (), It.IsAny <ILog> (), false, false)).ReturnsAsync(mockDevices [0]);
// Act
var appRunner = new AppRunner(processManager.Object,
appBundleInformationParser,
simulatorsFactory,
listenerFactory,
devicesFactory,
snapshotReporterFactory,
Mock.Of <ICaptureLogFactory> (),
Mock.Of <IDeviceLogCapturerFactory> (),
Mock.Of <ITestReporterFactory> (),
TestTarget.Device_iOS,
harness,
mainLog.Object,
logs.Object,
projectFilePath: projectFilePath,
buildConfiguration: "Debug");
appRunner.InitializeAsync().Wait();
CancellationToken cancellationToken = new CancellationToken();
var result = await appRunner.InstallAsync(cancellationToken);
// Verify
Assert.AreEqual(1, result.ExitCode);
var expectedArgs = $"-v -v -v --installdev {StringUtils.FormatArguments (appPath)} --devname \"{mockDevices[0].Name}\"";
processManager.Verify(x => x.ExecuteCommandAsync(
It.Is <MlaunchArguments> (args => args.AsCommandLine() == expectedArgs),
mainLog.Object,
TimeSpan.FromHours(1),
null,
5,
cancellationToken));
}
public async Task UninstallFromDeviceTest()
{
var harness = Mock.Of <IHarness> (x => x.Verbosity == 1);
var processResult = new ProcessExecutionResult()
{
ExitCode = 3, TimedOut = false
};
processManager.SetReturnsDefault(Task.FromResult(processResult));
devices.Setup(d => d.FindDevice(It.IsAny <RunMode> (), It.IsAny <ILog> (), false, false)).ReturnsAsync(mockDevices[1]);
// Act
var appRunner = new AppRunner(processManager.Object,
appBundleInformationParser,
simulatorsFactory,
listenerFactory,
devicesFactory,
snapshotReporterFactory,
Mock.Of <ICaptureLogFactory> (),
Mock.Of <IDeviceLogCapturerFactory> (),
Mock.Of <ITestReporterFactory> (),
TestTarget.Device_iOS,
harness,
mainLog.Object,
logs.Object,
projectFilePath: Path.Combine(sampleProjectPath, "SystemXunit.csproj"),
buildConfiguration: "Debug");
appRunner.InitializeAsync().Wait();
var result = await appRunner.UninstallAsync();
// Verify
Assert.AreEqual(3, result.ExitCode);
var expectedArgs = $"-v -v --uninstalldevbundleid {StringUtils.FormatArguments (appName)} --devname \"Test iPad\"";
processManager.Verify(x => x.ExecuteCommandAsync(
It.Is <MlaunchArguments> (args => args.AsCommandLine() == expectedArgs),
mainLog.Object,
TimeSpan.FromMinutes(1),
null,
5,
null));
}
public void UninstallToSimulatorTest()
{
var appRunner = new AppRunner(processManager.Object,
appBundleInformationParser,
simulatorsFactory,
listenerFactory,
devicesFactory,
snapshotReporterFactory,
Mock.Of <ICaptureLogFactory> (),
Mock.Of <IDeviceLogCapturerFactory> (),
Mock.Of <ITestReporterFactory> (),
TestTarget.Simulator_iOS64,
Mock.Of <IHarness> (),
mainLog.Object,
logs.Object,
projectFilePath: projectFilePath,
buildConfiguration: "Debug");
appRunner.InitializeAsync().Wait();
var exception = Assert.ThrowsAsync <InvalidOperationException> (
async() => await appRunner.UninstallAsync(),
"Uninstall should not be allowed on a simulator");
}
public async Task RunOnDeviceWithFailedTestsTest()
{
var harness = GetMockedHarness();
devices.Setup(d => d.FindDevice(It.IsAny <RunMode> (), It.IsAny <ILog> (), false, false)).ReturnsAsync(mockDevices[1]);
// Crash reporter
var crashReporterFactory = new Mock <ICrashSnapshotReporterFactory> ();
crashReporterFactory
.Setup(x => x.Create(mainLog.Object, It.IsAny <ILogs> (), true, "Test iPad"))
.Returns(snapshotReporter.Object);
var deviceSystemLog = new Mock <IFileBackedLog> ();
deviceSystemLog.SetupGet(x => x.FullPath).Returns(Path.GetTempFileName());
var testResultFilePath = Path.GetTempFileName();
var listenerLogFile = Mock.Of <IFileBackedLog> (x => x.FullPath == testResultFilePath);
File.WriteAllLines(testResultFilePath, new [] { "Some result here", "[FAIL] This test failed", "Some result there", "Tests run: 3" });
logs
.Setup(x => x.Create(It.Is <string> (s => s.StartsWith("test-ios-")), "TestLog", It.IsAny <bool?> ()))
.Returns(listenerLogFile);
logs
.Setup(x => x.Create(It.Is <string> (s => s.StartsWith("device-Test iPad-")), "Device log", It.IsAny <bool?> ()))
.Returns(deviceSystemLog.Object);
simpleListener.SetupGet(x => x.ConnectedTask).Returns(Task.FromResult(true));
var deviceLogCapturer = new Mock <IDeviceLogCapturer> ();
var deviceLogCapturerFactory = new Mock <IDeviceLogCapturerFactory> ();
deviceLogCapturerFactory
.Setup(x => x.Create(mainLog.Object, deviceSystemLog.Object, "Test iPad"))
.Returns(deviceLogCapturer.Object);
var ips = new StringBuilder();
var ipAddresses = System.Net.Dns.GetHostEntry(System.Net.Dns.GetHostName()).AddressList;
for (int i = 0; i < ipAddresses.Length; i++)
{
if (i > 0)
{
ips.Append(',');
}
ips.Append(ipAddresses [i].ToString());
}
var expectedArgs = $"-v -v -argument=-connection-mode -argument=none " +
$"-argument=-app-arg:-autostart -setenv=NUNIT_AUTOSTART=true -argument=-app-arg:-autoexit " +
$"-setenv=NUNIT_AUTOEXIT=true -argument=-app-arg:-enablenetwork -setenv=NUNIT_ENABLE_NETWORK=true " +
$"-setenv=DISABLE_SYSTEM_PERMISSION_TESTS=1 -argument=-app-arg:-hostname:{ips} -setenv=NUNIT_HOSTNAME={ips} " +
$"-argument=-app-arg:-transport:Tcp -setenv=NUNIT_TRANSPORT=TCP -argument=-app-arg:-hostport:{listenerPort} " +
$"-setenv=NUNIT_HOSTPORT={listenerPort} -setenv=env1=value1 -setenv=env2=value2 " +
$"--launchdev {StringUtils.FormatArguments (appPath)} --disable-memory-limits --wait-for-exit --devname \"Test iPad\"";
processManager
.Setup(x => x.ExecuteCommandAsync(
It.Is <MlaunchArguments> (args => args.AsCommandLine() == expectedArgs),
It.IsAny <ILog> (),
TimeSpan.FromMinutes(harness.Timeout * 2),
null,
It.IsAny <CancellationToken> ()))
.ReturnsAsync(new ProcessExecutionResult()
{
ExitCode = 0
});
var xmlResultFile = Path.ChangeExtension(testResultFilePath, "xml");
var testReporterFactory = new Mock <ITestReporterFactory> ();
var testReporter = new Mock <ITestReporter> ();
testReporterFactory.SetReturnsDefault(testReporter.Object);
testReporter.Setup(r => r.ParseResult()).Returns(() => {
return(Task.FromResult <(TestExecutingResult, string)> ((TestExecutingResult.Failed, null)));
});
testReporter.Setup(r => r.Success).Returns(false);
// Act
var appRunner = new AppRunner(processManager.Object,
appBundleInformationParser,
simulatorsFactory,
listenerFactory,
devicesFactory,
crashReporterFactory.Object,
Mock.Of <ICaptureLogFactory> (), // Used for simulators only
deviceLogCapturerFactory.Object,
testReporterFactory.Object,
TestTarget.Device_iOS,
harness,
mainLog.Object,
logs.Object,
projectFilePath: projectFilePath,
buildConfiguration: "Debug",
timeoutMultiplier: 2);
appRunner.InitializeAsync().Wait();
var result = await appRunner.RunAsync();
// Verify
Assert.AreEqual(1, result);
processManager.VerifyAll();
simpleListener.Verify(x => x.InitializeAndGetPort(), Times.AtLeastOnce);
simpleListener.Verify(x => x.StartAsync(), Times.AtLeastOnce);
simpleListener.Verify(x => x.Cancel(), Times.AtLeastOnce);
simpleListener.Verify(x => x.Dispose(), Times.AtLeastOnce);
snapshotReporter.Verify(x => x.StartCaptureAsync(), Times.AtLeastOnce);
snapshotReporter.Verify(x => x.StartCaptureAsync(), Times.AtLeastOnce);
deviceSystemLog.Verify(x => x.Dispose(), Times.AtLeastOnce);
}
public async Task RunOnSimulatorSuccessfullyTest()
{
var harness = GetMockedHarness();
devices.Setup(x => x.ConnectedDevices).Returns(mockDevices.Reverse());
// Crash reporter
var crashReporterFactory = new Mock <ICrashSnapshotReporterFactory> ();
crashReporterFactory
.Setup(x => x.Create(mainLog.Object, It.IsAny <ILogs> (), false, null))
.Returns(snapshotReporter.Object);
// Mock finding simulators
simulators
.Setup(x => x.LoadDevices(It.IsAny <ILog> (), false, false, false))
.Returns(Task.FromResult(true));
string simulatorLogPath = Path.Combine(Path.GetTempPath(), "simulator-logs");
var simulator = new Mock <ISimulatorDevice> ();
simulator.SetupGet(x => x.Name).Returns("Test iPhone simulator");
simulator.SetupGet(x => x.UDID).Returns("58F21118E4D34FD69EAB7860BB9B38A0");
simulator.SetupGet(x => x.LogPath).Returns(simulatorLogPath);
simulator.SetupGet(x => x.SystemLog).Returns(Path.Combine(simulatorLogPath, "system.log"));
simulators
.Setup(x => x.FindSimulators(TestTarget.Simulator_iOS64, mainLog.Object, true, false))
.ReturnsAsync((simulator.Object, null));
var testResultFilePath = Path.GetTempFileName();
var listenerLogFile = Mock.Of <IFileBackedLog> (x => x.FullPath == testResultFilePath);
File.WriteAllLines(testResultFilePath, new [] { "Some result here", "Tests run: 124", "Some result there" });
logs
.Setup(x => x.Create(It.IsAny <string> (), It.IsAny <string>(), It.IsAny <bool?> ()))
.Returns(listenerLogFile);
logs
.Setup(x => x.CreateFile(It.IsAny <string> (), It.IsAny <string> ()))
.Returns($"/path/to/log-{Guid.NewGuid ()}.log");
simpleListener.SetupGet(x => x.ConnectedTask).Returns(Task.FromResult(true));
var captureLog = new Mock <ICaptureLog> ();
captureLog.SetupGet(x => x.FullPath).Returns(simulatorLogPath);
var captureLogFactory = new Mock <ICaptureLogFactory> ();
captureLogFactory
.Setup(x => x.Create(
Path.Combine(logs.Object.Directory, simulator.Object.Name + ".log"),
simulator.Object.SystemLog,
true,
It.IsAny <string> ()))
.Returns(captureLog.Object);
var expectedArgs = $"--sdkroot {StringUtils.FormatArguments (xcodePath)} -v -v " +
$"-argument=-connection-mode -argument=none -argument=-app-arg:-autostart " +
$"-setenv=NUNIT_AUTOSTART=true -argument=-app-arg:-autoexit -setenv=NUNIT_AUTOEXIT=true " +
$"-argument=-app-arg:-enablenetwork -setenv=NUNIT_ENABLE_NETWORK=true " +
$"-setenv=DISABLE_SYSTEM_PERMISSION_TESTS=1 -argument=-app-arg:-hostname:127.0.0.1 " +
$"-setenv=NUNIT_HOSTNAME=127.0.0.1 -argument=-app-arg:-transport:Tcp -setenv=NUNIT_TRANSPORT=TCP " +
$"-argument=-app-arg:-hostport:{listenerPort} -setenv=NUNIT_HOSTPORT={listenerPort} " +
$"-setenv=env1=value1 -setenv=env2=value2 --launchsim {StringUtils.FormatArguments (appPath)} " +
$"--stdout=tty1 --stderr=tty1 --device=:v2:udid={simulator.Object.UDID}";
processManager
.Setup(x => x.ExecuteCommandAsync(
It.Is <MlaunchArguments> (args => args.AsCommandLine() == expectedArgs),
mainLog.Object,
TimeSpan.FromMinutes(harness.Timeout * 2),
null,
It.IsAny <CancellationToken> ()))
.ReturnsAsync(new ProcessExecutionResult()
{
ExitCode = 0
});
var xmlResultFile = Path.ChangeExtension(testResultFilePath, "xml");
var testReporterFactory = new Mock <ITestReporterFactory> ();
var testReporter = new Mock <ITestReporter> ();
testReporterFactory.SetReturnsDefault(testReporter.Object);
testReporter.Setup(r => r.ParseResult()).Returns(() => {
return(Task.FromResult <(TestExecutingResult, string)> ((TestExecutingResult.Succeeded, null)));
});
testReporter.Setup(r => r.Success).Returns(true);
// Act
var appRunner = new AppRunner(processManager.Object,
appBundleInformationParser,
simulatorsFactory,
listenerFactory,
devicesFactory,
crashReporterFactory.Object,
captureLogFactory.Object,
Mock.Of <IDeviceLogCapturerFactory> (), // Use for devices only
testReporterFactory.Object,
TestTarget.Simulator_iOS64,
harness,
mainLog.Object,
logs.Object,
projectFilePath: projectFilePath,
buildConfiguration: "Debug",
timeoutMultiplier: 2,
ensureCleanSimulatorState: true);
appRunner.InitializeAsync().Wait();
var result = await appRunner.RunAsync();
// Verify
Assert.AreEqual(0, result);
simpleListener.Verify(x => x.InitializeAndGetPort(), Times.AtLeastOnce);
simpleListener.Verify(x => x.StartAsync(), Times.AtLeastOnce);
simpleListener.Verify(x => x.Cancel(), Times.AtLeastOnce);
simpleListener.Verify(x => x.Dispose(), Times.AtLeastOnce);
simulators.VerifyAll();
captureLog.Verify(x => x.StartCapture(), Times.AtLeastOnce);
captureLog.Verify(x => x.StopCapture(), Times.AtLeastOnce);
// When ensureCleanSimulatorState == true
simulator.Verify(x => x.PrepareSimulator(It.IsAny <ILog> (), appName));
simulator.Verify(x => x.KillEverything(mainLog.Object));
}
public async Task RunOnSimulatorWithNoAvailableSimulatorTest()
{
devices.Setup(x => x.ConnectedDevices).Returns(mockDevices.Reverse());
// Crash reporter
var crashReporterFactory = new Mock <ICrashSnapshotReporterFactory> ();
crashReporterFactory
.Setup(x => x.Create(mainLog.Object, It.IsAny <ILogs> (), false, null))
.Returns(snapshotReporter.Object);
// Mock finding simulators
simulators
.Setup(x => x.LoadDevices(It.IsAny <ILog> (), false, false, false))
.Returns(Task.FromResult(true));
string simulatorLogPath = Path.Combine(Path.GetTempPath(), "simulator-logs");
simulators
.Setup(x => x.FindSimulators(TestTarget.Simulator_tvOS, mainLog.Object, true, false))
.ReturnsAsync((null, null));
var listenerLogFile = new Mock <IFileBackedLog> ();
logs
.Setup(x => x.Create(It.IsAny <string> (), "TestLog", It.IsAny <bool> ()))
.Returns(listenerLogFile.Object);
simpleListener.SetupGet(x => x.ConnectedTask).Returns(Task.FromResult(true));
var captureLog = new Mock <ICaptureLog> ();
captureLog.SetupGet(x => x.FullPath).Returns(simulatorLogPath);
var captureLogFactory = new Mock <ICaptureLogFactory> ();
captureLogFactory
.Setup(x => x.Create(
Path.Combine(logs.Object.Directory, "tvos.log"),
"/path/to/simulator.log",
true,
It.IsAny <string> ()))
.Returns(captureLog.Object);
var testReporterFactory = new Mock <ITestReporterFactory> ();
var testReporter = new Mock <ITestReporter> ();
testReporterFactory.SetReturnsDefault(testReporter.Object);
testReporter.Setup(r => r.ParseResult()).Returns(() => {
return(Task.FromResult <(TestExecutingResult, string)> ((TestExecutingResult.Succeeded, null)));
});
testReporter.Setup(r => r.Success).Returns(true);
// Act
var appRunner = new AppRunner(processManager.Object,
appBundleInformationParser,
simulatorsFactory,
listenerFactory,
devicesFactory,
crashReporterFactory.Object,
captureLogFactory.Object,
Mock.Of <IDeviceLogCapturerFactory> (),
testReporterFactory.Object,
TestTarget.Simulator_tvOS,
GetMockedHarness(),
mainLog.Object,
logs.Object,
projectFilePath: projectFilePath,
buildConfiguration: "Debug",
timeoutMultiplier: 2);
appRunner.InitializeAsync().Wait();
var result = await appRunner.RunAsync();
// Verify
Assert.AreEqual(1, result);
mainLog.Verify(x => x.WriteLine("Test run completed"), Times.Never);
simpleListener.Verify(x => x.InitializeAndGetPort(), Times.AtLeastOnce);
simpleListener.Verify(x => x.StartAsync(), Times.AtLeastOnce);
simulators.VerifyAll();
}
public async Task RunTestAsync()
{
mainLog.WriteLine("Running '{0}' on device (candidates: '{1}')", testTask.ProjectFile, string.Join("', '", testTask.Candidates.Select((v) => v.Name).ToArray()));
var uninstall_log = testTask.Logs.Create($"uninstall-{Helpers.Timestamp}.log", "Uninstall log");
using (var device_resource = await testTask.NotifyBlockingWaitAsync(resourceManager.GetDeviceResources(testTask.Candidates).AcquireAnyConcurrentAsync())) {
try {
// Set the device we acquired.
testTask.Device = testTask.Candidates.First((d) => d.UDID == device_resource.Resource.Name);
if (testTask.Device.DevicePlatform == DevicePlatform.watchOS)
{
testTask.CompanionDevice = await devices.FindCompanionDevice(deviceLoadLog, testTask.Device);
}
mainLog.WriteLine("Acquired device '{0}' for '{1}'", testTask.Device.Name, testTask.ProjectFile);
ITunnelBore tunnelBore = null;
if (useTcpTunnel && testTask.Device.DevicePlatform != DevicePlatform.iOS &&
testTask.Device.DevicePlatform != DevicePlatform.tvOS)
{
mainLog.WriteLine("Ignoring request to use a tunnel because it is not supported by the specified platform");
}
else if (useTcpTunnel && (testTask.Device.DevicePlatform == DevicePlatform.iOS ||
testTask.Device.DevicePlatform == DevicePlatform.tvOS))
{
tunnelBore = testTask.TunnelBore;
mainLog.WriteLine("Using tunnel to communicate with device.");
}
testTask.Runner = new AppRunner(testTask.ProcessManager,
new AppBundleInformationParser(),
new SimulatorLoaderFactory(testTask.ProcessManager),
new SimpleListenerFactory(tunnelBore),
new DeviceLoaderFactory(testTask.ProcessManager),
new CrashSnapshotReporterFactory(testTask.ProcessManager),
new CaptureLogFactory(),
new DeviceLogCapturerFactory(testTask.ProcessManager),
new TestReporterFactory(testTask.ProcessManager),
testTask.AppRunnerTarget,
testTask.Harness,
projectFilePath: testTask.ProjectFile,
mainLog: uninstall_log,
logs: new Logs(testTask.LogDirectory ?? defaultLogDirectory),
buildConfiguration: testTask.ProjectConfiguration,
deviceName: testTask.Device.Name,
companionDeviceName: testTask.CompanionDevice?.Name,
timeoutMultiplier: testTask.TimeoutMultiplier,
variation: testTask.Variation,
buildTask: testTask.BuildTask);
await testTask.Runner.InitializeAsync();
// Sometimes devices can't upgrade (depending on what has changed), so make sure to uninstall any existing apps first.
if (uninstallTestApp)
{
testTask.Runner.MainLog = uninstall_log;
var uninstall_result = await testTask.Runner.UninstallAsync();
if (!uninstall_result.Succeeded)
{
mainLog.WriteLine($"Pre-run uninstall failed, exit code: {uninstall_result.ExitCode} (this hopefully won't affect the test result)");
}
}
else
{
uninstall_log.WriteLine($"Pre-run uninstall skipped.");
}
if (!testTask.Failed)
{
// Install the app
InstallLog = new AppInstallMonitorLog(testTask.Logs.Create($"install-{Helpers.Timestamp}.log", "Install log"));
try {
testTask.Runner.MainLog = this.InstallLog;
var install_result = await testTask.Runner.InstallAsync(InstallLog.CancellationToken);
if (!install_result.Succeeded)
{
testTask.FailureMessage = $"Install failed, exit code: {install_result.ExitCode}.";
testTask.ExecutionResult = TestExecutingResult.Failed;
if (generateXmlFailures)
{
resultParser.GenerateFailure(
testTask.Logs,
"install",
testTask.Runner.AppInformation.AppName,
testTask.Variation,
$"AppInstallation on {testTask.Device.Name}",
$"Install failed on {testTask.Device.Name}, exit code: {install_result.ExitCode}",
InstallLog.FullPath,
xmlResultJargon);
}
}
} finally {
InstallLog.Dispose();
InstallLog = null;
}
}
if (!testTask.Failed)
{
// Run the app
testTask.Runner.MainLog = testTask.Logs.Create($"run-{testTask.Device.UDID}-{Helpers.Timestamp}.log", "Run log");
await testTask.Runner.RunAsync();
if (!string.IsNullOrEmpty(testTask.Runner.FailureMessage))
{
testTask.FailureMessage = testTask.Runner.FailureMessage;
}
else if (testTask.Runner.Result != TestExecutingResult.Succeeded)
{
testTask.FailureMessage = testTask.GuessFailureReason(testTask.Runner.MainLog);
}
if (string.IsNullOrEmpty(testTask.FailureMessage) && errorKnowledgeBase.IsKnownTestIssue(testTask.Runner.MainLog, out var failure))
{
testTask.KnownFailure = failure;
mainLog.WriteLine($"Test run has a known failure: '{testTask.KnownFailure}'");
}
if (testTask.Runner.Result == TestExecutingResult.Succeeded && testTask.Platform == TestPlatform.iOS_TodayExtension64)
{
// For the today extension, the main app is just a single test.
// This is because running the today extension will not wake up the device,
// nor will it close & reopen the today app (but launching the main app
// will do both of these things, preparing the device for launching the today extension).
AppRunner todayRunner = new AppRunner(testTask.ProcessManager,
new AppBundleInformationParser(),
new SimulatorLoaderFactory(testTask.ProcessManager),
new SimpleListenerFactory(tunnelBore),
new DeviceLoaderFactory(testTask.ProcessManager),
new CrashSnapshotReporterFactory(testTask.ProcessManager),
new CaptureLogFactory(),
new DeviceLogCapturerFactory(testTask.ProcessManager),
new TestReporterFactory(testTask.ProcessManager),
testTask.AppRunnerTarget,
testTask.Harness,
projectFilePath: testTask.ProjectFile,
mainLog: testTask.Logs.Create($"extension-run-{testTask.Device.UDID}-{Helpers.Timestamp}.log", "Extension run log"),
logs: new Logs(testTask.LogDirectory ?? defaultLogDirectory),
buildConfiguration: testTask.ProjectConfiguration,
deviceName: testTask.Device.Name,
companionDeviceName: testTask.CompanionDevice?.Name,
timeoutMultiplier: testTask.TimeoutMultiplier,
variation: testTask.Variation,
buildTask: testTask.BuildTask);
await todayRunner.InitializeAsync();
testTask.AdditionalRunner = todayRunner;
await todayRunner.RunAsync();
foreach (var log in todayRunner.Logs.Where((v) => !v.Description.StartsWith("Extension ", StringComparison.Ordinal)))
{
log.Description = "Extension " + log.Description [0].ToString().ToLower() + log.Description.Substring(1);
}
testTask.ExecutionResult = todayRunner.Result;
if (!string.IsNullOrEmpty(todayRunner.FailureMessage))
{
testTask.FailureMessage = todayRunner.FailureMessage;
}
}
else
{
testTask.ExecutionResult = testTask.Runner.Result;
}
}
} finally {
// Uninstall again, so that we don't leave junk behind and fill up the device.
if (uninstallTestApp)
{
testTask.Runner.MainLog = uninstall_log;
var uninstall_result = await testTask.Runner.UninstallAsync();
if (!uninstall_result.Succeeded)
{
mainLog.WriteLine($"Post-run uninstall failed, exit code: {uninstall_result.ExitCode} (this won't affect the test result)");
}
}
else
{
uninstall_log.WriteLine($"Post-run uninstall skipped.");
}
// Also clean up after us locally.
if (inCI || cleanSuccessfulTestRuns && testTask.Succeeded)
{
await testTask.BuildTask.CleanAsync();
}
}
}
}