static async Task <int> Main(string[] args)
{
try
{
Log.Info("Running application with parameters {@arguments}", args);
var runner = new AppRunner <ConsoleApplication>();
ConfigureApplication(runner);
var exitCode = await runner.RunAsync(args);
Log.Info("Application exited with error code {exitCode}.", exitCode);
return(exitCode);
}
catch (Exception e)
{
Log.Error(e, "Application terminated in an unexpected way.");
return(-1);
}
finally
{
LogManager.Flush(TimeSpan.FromSeconds(2));
#if DEBUG
Console.ReadKey();
#endif
}
}
public void RunOnDeviceWithNoAvailableSimulatorTest()
{
devices.Setup(d => d.FindDevice(It.IsAny <RunMode> (), It.IsAny <ILog> (), false, false)).ReturnsAsync((IHardwareDevice)null);
simulators
.Setup(s => s.FindSimulators(It.IsAny <TestTarget> (), It.IsAny <ILog> (), true, false))
.ReturnsAsync((null, null));
// Crash reporter
var crashReporterFactory = new Mock <ICrashSnapshotReporterFactory> ();
crashReporterFactory
.Setup(x => x.Create(mainLog.Object, It.IsAny <ILogs> (), false, null))
.Returns(snapshotReporter.Object);
var listenerLogFile = new Mock <IFileBackedLog> ();
var testReporterFactory = new Mock <ITestReporterFactory> ();
var testReporter = new Mock <ITestReporter> ();
testReporterFactory.SetReturnsDefault(testReporter.Object);
testReporter.Setup(r => r.ParseResult()).Returns(() => {
return(Task.FromResult <(TestExecutingResult, string)> ((TestExecutingResult.Succeeded, null)));
});
testReporter.Setup(r => r.Success).Returns(true);
logs
.Setup(x => x.Create(It.IsAny <string> (), "TestLog", It.IsAny <bool> ()))
.Returns(listenerLogFile.Object);
simpleListener.SetupGet(x => x.ConnectedTask).Returns(Task.FromResult(true));
// Act
var appRunner = new AppRunner(processManager.Object,
appBundleInformationParser,
simulatorsFactory,
listenerFactory,
devicesFactory,
crashReporterFactory.Object,
Mock.Of <ICaptureLogFactory> (),
Mock.Of <IDeviceLogCapturerFactory> (),
testReporterFactory.Object,
TestTarget.Device_tvOS,
GetMockedHarness(),
mainLog.Object,
logs.Object,
projectFilePath: projectFilePath,
buildConfiguration: "Debug",
timeoutMultiplier: 2);
appRunner.InitializeAsync().Wait();
Assert.ThrowsAsync <NoDeviceFoundException> (
async() => await appRunner.RunAsync(),
"Running requires connected devices");
}
public static void Main(string[] args)
{
// create service collection
var builder = Host.CreateDefaultBuilder(args).ConfigureServices(ConfigureServices).Build();
var appRunner = new AppRunner <OysterMain>();
appRunner.UseDefaultMiddleware();
appRunner.UseMicrosoftDependencyInjection(builder.Services);
// run app
Task.WaitAll(builder.RunAsync(), appRunner.RunAsync(args));
}
public static async Task <int> Main(string[] args)
{
var host = CreateHostBuilder().Build();
var logger = LogManager.GetCurrentClassLogger();
int result;
Console.WriteLine();
try
{
var appRunner = new AppRunner <EDRMethods>()
.Configure(a => a.AppSettings.Help.PrintHelpOption = true)
.UseDefaultMiddleware()
.UseMicrosoftDependencyInjection(host.Services);
result = await appRunner.RunAsync(args);
}
catch (CommandLineArgumentException ae)
{
logger.Info(ae.Message);
Console.WriteLine();
ae.GetCommandContext()?.PrintHelp();
result = 1;
}
catch (ConnectorConfigurationException ce)
{
logger.Error(ce);
ce.GetCommandContext()?.PrintHelp();
result = 1;
}
#pragma warning disable CA1031 // Do not catch general exception types
catch (Exception e)
{
logger.Error(e);
result = 1;
}
#pragma warning restore CA1031 // Do not catch general exception types
finally
{
LogManager.Shutdown();
}
return(result);
}
public void RunOnDeviceWithNoAvailableSimulatorTest()
{
devices.Setup(x => x.ConnectedDevices).Returns(mockDevices.Reverse());
// Crash reporter
var crashReporterFactory = new Mock <ICrashSnapshotReporterFactory> ();
crashReporterFactory
.Setup(x => x.Create(mainLog.Object, It.IsAny <ILogs> (), false, null))
.Returns(snapshotReporter.Object);
var listenerLogFile = new Mock <ILog> ();
logs
.Setup(x => x.Create(It.IsAny <string> (), "TestLog", It.IsAny <bool> ()))
.Returns(listenerLogFile.Object);
simpleListener.SetupGet(x => x.ConnectedTask).Returns(Task.CompletedTask);
// Act
var appRunner = new AppRunner(processManager.Object,
appBundleInformationParser,
simulatorsFactory,
listenerFactory,
devicesFactory,
crashReporterFactory.Object,
Mock.Of <ICaptureLogFactory> (),
Mock.Of <IDeviceLogCapturerFactory> (),
Mock.Of <ITestReporterFactory> (),
TestTarget.Device_tvOS,
GetMockedHarness(),
mainLog.Object,
logs.Object,
projectFilePath: projectFilePath,
buildConfiguration: "Debug",
timeoutMultiplier: 2);
Assert.ThrowsAsync <NoDeviceFoundException> (
async() => await appRunner.RunAsync(),
"Running requires connected devices");
}
public MainWindow()
{
InitializeComponent();
var scopeBuilder = new SkitanaScopeBuilder();
scopeBuilder.RegisterSkiaRenderer()
.RegisterWpfInputServices(skiaElement)
.RegisterSkitanaAppFramework()
.RegisterAllControlsServices()
.RegisterSingleton(skiaElement);
var services = scopeBuilder.Build();
onCloseDisposable = services as IDisposable;
var factory = (IIoCFactory)services.GetService(typeof(IIoCFactory));
appRunner = factory.Create <WpfAppRunner>();
var app = factory.Create <AllControlsApp>();
cancellationTokenSource = new CancellationTokenSource();
appLoopTask = appRunner.RunAsync(app, cancellationTokenSource.Token);
}
public async Task RunOnDeviceWithFailedTestsTest()
{
var harness = GetMockedHarness();
devices.Setup(d => d.FindDevice(It.IsAny <RunMode> (), It.IsAny <ILog> (), false, false)).ReturnsAsync(mockDevices[1]);
// Crash reporter
var crashReporterFactory = new Mock <ICrashSnapshotReporterFactory> ();
crashReporterFactory
.Setup(x => x.Create(mainLog.Object, It.IsAny <ILogs> (), true, "Test iPad"))
.Returns(snapshotReporter.Object);
var deviceSystemLog = new Mock <IFileBackedLog> ();
deviceSystemLog.SetupGet(x => x.FullPath).Returns(Path.GetTempFileName());
var testResultFilePath = Path.GetTempFileName();
var listenerLogFile = Mock.Of <IFileBackedLog> (x => x.FullPath == testResultFilePath);
File.WriteAllLines(testResultFilePath, new [] { "Some result here", "[FAIL] This test failed", "Some result there", "Tests run: 3" });
logs
.Setup(x => x.Create(It.Is <string> (s => s.StartsWith("test-ios-")), "TestLog", It.IsAny <bool?> ()))
.Returns(listenerLogFile);
logs
.Setup(x => x.Create(It.Is <string> (s => s.StartsWith("device-Test iPad-")), "Device log", It.IsAny <bool?> ()))
.Returns(deviceSystemLog.Object);
simpleListener.SetupGet(x => x.ConnectedTask).Returns(Task.FromResult(true));
var deviceLogCapturer = new Mock <IDeviceLogCapturer> ();
var deviceLogCapturerFactory = new Mock <IDeviceLogCapturerFactory> ();
deviceLogCapturerFactory
.Setup(x => x.Create(mainLog.Object, deviceSystemLog.Object, "Test iPad"))
.Returns(deviceLogCapturer.Object);
var ips = new StringBuilder();
var ipAddresses = System.Net.Dns.GetHostEntry(System.Net.Dns.GetHostName()).AddressList;
for (int i = 0; i < ipAddresses.Length; i++)
{
if (i > 0)
{
ips.Append(',');
}
ips.Append(ipAddresses [i].ToString());
}
var expectedArgs = $"-v -v -argument=-connection-mode -argument=none " +
$"-argument=-app-arg:-autostart -setenv=NUNIT_AUTOSTART=true -argument=-app-arg:-autoexit " +
$"-setenv=NUNIT_AUTOEXIT=true -argument=-app-arg:-enablenetwork -setenv=NUNIT_ENABLE_NETWORK=true " +
$"-setenv=DISABLE_SYSTEM_PERMISSION_TESTS=1 -argument=-app-arg:-hostname:{ips} -setenv=NUNIT_HOSTNAME={ips} " +
$"-argument=-app-arg:-transport:Tcp -setenv=NUNIT_TRANSPORT=TCP -argument=-app-arg:-hostport:{listenerPort} " +
$"-setenv=NUNIT_HOSTPORT={listenerPort} -setenv=env1=value1 -setenv=env2=value2 " +
$"--launchdev {StringUtils.FormatArguments (appPath)} --disable-memory-limits --wait-for-exit --devname \"Test iPad\"";
processManager
.Setup(x => x.ExecuteCommandAsync(
It.Is <MlaunchArguments> (args => args.AsCommandLine() == expectedArgs),
It.IsAny <ILog> (),
TimeSpan.FromMinutes(harness.Timeout * 2),
null,
It.IsAny <CancellationToken> ()))
.ReturnsAsync(new ProcessExecutionResult()
{
ExitCode = 0
});
var xmlResultFile = Path.ChangeExtension(testResultFilePath, "xml");
var testReporterFactory = new Mock <ITestReporterFactory> ();
var testReporter = new Mock <ITestReporter> ();
testReporterFactory.SetReturnsDefault(testReporter.Object);
testReporter.Setup(r => r.ParseResult()).Returns(() => {
return(Task.FromResult <(TestExecutingResult, string)> ((TestExecutingResult.Failed, null)));
});
testReporter.Setup(r => r.Success).Returns(false);
// Act
var appRunner = new AppRunner(processManager.Object,
appBundleInformationParser,
simulatorsFactory,
listenerFactory,
devicesFactory,
crashReporterFactory.Object,
Mock.Of <ICaptureLogFactory> (), // Used for simulators only
deviceLogCapturerFactory.Object,
testReporterFactory.Object,
TestTarget.Device_iOS,
harness,
mainLog.Object,
logs.Object,
projectFilePath: projectFilePath,
buildConfiguration: "Debug",
timeoutMultiplier: 2);
appRunner.InitializeAsync().Wait();
var result = await appRunner.RunAsync();
// Verify
Assert.AreEqual(1, result);
processManager.VerifyAll();
simpleListener.Verify(x => x.InitializeAndGetPort(), Times.AtLeastOnce);
simpleListener.Verify(x => x.StartAsync(), Times.AtLeastOnce);
simpleListener.Verify(x => x.Cancel(), Times.AtLeastOnce);
simpleListener.Verify(x => x.Dispose(), Times.AtLeastOnce);
snapshotReporter.Verify(x => x.StartCaptureAsync(), Times.AtLeastOnce);
snapshotReporter.Verify(x => x.StartCaptureAsync(), Times.AtLeastOnce);
deviceSystemLog.Verify(x => x.Dispose(), Times.AtLeastOnce);
}
public async Task RunOnSimulatorSuccessfullyTest()
{
var harness = GetMockedHarness();
devices.Setup(x => x.ConnectedDevices).Returns(mockDevices.Reverse());
// Crash reporter
var crashReporterFactory = new Mock <ICrashSnapshotReporterFactory> ();
crashReporterFactory
.Setup(x => x.Create(mainLog.Object, It.IsAny <ILogs> (), false, null))
.Returns(snapshotReporter.Object);
// Mock finding simulators
simulators
.Setup(x => x.LoadDevices(It.IsAny <ILog> (), false, false, false))
.Returns(Task.FromResult(true));
string simulatorLogPath = Path.Combine(Path.GetTempPath(), "simulator-logs");
var simulator = new Mock <ISimulatorDevice> ();
simulator.SetupGet(x => x.Name).Returns("Test iPhone simulator");
simulator.SetupGet(x => x.UDID).Returns("58F21118E4D34FD69EAB7860BB9B38A0");
simulator.SetupGet(x => x.LogPath).Returns(simulatorLogPath);
simulator.SetupGet(x => x.SystemLog).Returns(Path.Combine(simulatorLogPath, "system.log"));
simulators
.Setup(x => x.FindSimulators(TestTarget.Simulator_iOS64, mainLog.Object, true, false))
.ReturnsAsync((simulator.Object, null));
var testResultFilePath = Path.GetTempFileName();
var listenerLogFile = Mock.Of <IFileBackedLog> (x => x.FullPath == testResultFilePath);
File.WriteAllLines(testResultFilePath, new [] { "Some result here", "Tests run: 124", "Some result there" });
logs
.Setup(x => x.Create(It.IsAny <string> (), It.IsAny <string>(), It.IsAny <bool?> ()))
.Returns(listenerLogFile);
logs
.Setup(x => x.CreateFile(It.IsAny <string> (), It.IsAny <string> ()))
.Returns($"/path/to/log-{Guid.NewGuid ()}.log");
simpleListener.SetupGet(x => x.ConnectedTask).Returns(Task.FromResult(true));
var captureLog = new Mock <ICaptureLog> ();
captureLog.SetupGet(x => x.FullPath).Returns(simulatorLogPath);
var captureLogFactory = new Mock <ICaptureLogFactory> ();
captureLogFactory
.Setup(x => x.Create(
Path.Combine(logs.Object.Directory, simulator.Object.Name + ".log"),
simulator.Object.SystemLog,
true,
It.IsAny <string> ()))
.Returns(captureLog.Object);
var expectedArgs = $"--sdkroot {StringUtils.FormatArguments (xcodePath)} -v -v " +
$"-argument=-connection-mode -argument=none -argument=-app-arg:-autostart " +
$"-setenv=NUNIT_AUTOSTART=true -argument=-app-arg:-autoexit -setenv=NUNIT_AUTOEXIT=true " +
$"-argument=-app-arg:-enablenetwork -setenv=NUNIT_ENABLE_NETWORK=true " +
$"-setenv=DISABLE_SYSTEM_PERMISSION_TESTS=1 -argument=-app-arg:-hostname:127.0.0.1 " +
$"-setenv=NUNIT_HOSTNAME=127.0.0.1 -argument=-app-arg:-transport:Tcp -setenv=NUNIT_TRANSPORT=TCP " +
$"-argument=-app-arg:-hostport:{listenerPort} -setenv=NUNIT_HOSTPORT={listenerPort} " +
$"-setenv=env1=value1 -setenv=env2=value2 --launchsim {StringUtils.FormatArguments (appPath)} " +
$"--stdout=tty1 --stderr=tty1 --device=:v2:udid={simulator.Object.UDID}";
processManager
.Setup(x => x.ExecuteCommandAsync(
It.Is <MlaunchArguments> (args => args.AsCommandLine() == expectedArgs),
mainLog.Object,
TimeSpan.FromMinutes(harness.Timeout * 2),
null,
It.IsAny <CancellationToken> ()))
.ReturnsAsync(new ProcessExecutionResult()
{
ExitCode = 0
});
var xmlResultFile = Path.ChangeExtension(testResultFilePath, "xml");
var testReporterFactory = new Mock <ITestReporterFactory> ();
var testReporter = new Mock <ITestReporter> ();
testReporterFactory.SetReturnsDefault(testReporter.Object);
testReporter.Setup(r => r.ParseResult()).Returns(() => {
return(Task.FromResult <(TestExecutingResult, string)> ((TestExecutingResult.Succeeded, null)));
});
testReporter.Setup(r => r.Success).Returns(true);
// Act
var appRunner = new AppRunner(processManager.Object,
appBundleInformationParser,
simulatorsFactory,
listenerFactory,
devicesFactory,
crashReporterFactory.Object,
captureLogFactory.Object,
Mock.Of <IDeviceLogCapturerFactory> (), // Use for devices only
testReporterFactory.Object,
TestTarget.Simulator_iOS64,
harness,
mainLog.Object,
logs.Object,
projectFilePath: projectFilePath,
buildConfiguration: "Debug",
timeoutMultiplier: 2,
ensureCleanSimulatorState: true);
appRunner.InitializeAsync().Wait();
var result = await appRunner.RunAsync();
// Verify
Assert.AreEqual(0, result);
simpleListener.Verify(x => x.InitializeAndGetPort(), Times.AtLeastOnce);
simpleListener.Verify(x => x.StartAsync(), Times.AtLeastOnce);
simpleListener.Verify(x => x.Cancel(), Times.AtLeastOnce);
simpleListener.Verify(x => x.Dispose(), Times.AtLeastOnce);
simulators.VerifyAll();
captureLog.Verify(x => x.StartCapture(), Times.AtLeastOnce);
captureLog.Verify(x => x.StopCapture(), Times.AtLeastOnce);
// When ensureCleanSimulatorState == true
simulator.Verify(x => x.PrepareSimulator(It.IsAny <ILog> (), appName));
simulator.Verify(x => x.KillEverything(mainLog.Object));
}
public async Task RunOnSimulatorWithNoAvailableSimulatorTest()
{
devices.Setup(x => x.ConnectedDevices).Returns(mockDevices.Reverse());
// Crash reporter
var crashReporterFactory = new Mock <ICrashSnapshotReporterFactory> ();
crashReporterFactory
.Setup(x => x.Create(mainLog.Object, It.IsAny <ILogs> (), false, null))
.Returns(snapshotReporter.Object);
// Mock finding simulators
simulators
.Setup(x => x.LoadDevices(It.IsAny <ILog> (), false, false, false))
.Returns(Task.FromResult(true));
string simulatorLogPath = Path.Combine(Path.GetTempPath(), "simulator-logs");
simulators
.Setup(x => x.FindSimulators(TestTarget.Simulator_tvOS, mainLog.Object, true, false))
.ReturnsAsync((null, null));
var listenerLogFile = new Mock <IFileBackedLog> ();
logs
.Setup(x => x.Create(It.IsAny <string> (), "TestLog", It.IsAny <bool> ()))
.Returns(listenerLogFile.Object);
simpleListener.SetupGet(x => x.ConnectedTask).Returns(Task.FromResult(true));
var captureLog = new Mock <ICaptureLog> ();
captureLog.SetupGet(x => x.FullPath).Returns(simulatorLogPath);
var captureLogFactory = new Mock <ICaptureLogFactory> ();
captureLogFactory
.Setup(x => x.Create(
Path.Combine(logs.Object.Directory, "tvos.log"),
"/path/to/simulator.log",
true,
It.IsAny <string> ()))
.Returns(captureLog.Object);
var testReporterFactory = new Mock <ITestReporterFactory> ();
var testReporter = new Mock <ITestReporter> ();
testReporterFactory.SetReturnsDefault(testReporter.Object);
testReporter.Setup(r => r.ParseResult()).Returns(() => {
return(Task.FromResult <(TestExecutingResult, string)> ((TestExecutingResult.Succeeded, null)));
});
testReporter.Setup(r => r.Success).Returns(true);
// Act
var appRunner = new AppRunner(processManager.Object,
appBundleInformationParser,
simulatorsFactory,
listenerFactory,
devicesFactory,
crashReporterFactory.Object,
captureLogFactory.Object,
Mock.Of <IDeviceLogCapturerFactory> (),
testReporterFactory.Object,
TestTarget.Simulator_tvOS,
GetMockedHarness(),
mainLog.Object,
logs.Object,
projectFilePath: projectFilePath,
buildConfiguration: "Debug",
timeoutMultiplier: 2);
appRunner.InitializeAsync().Wait();
var result = await appRunner.RunAsync();
// Verify
Assert.AreEqual(1, result);
mainLog.Verify(x => x.WriteLine("Test run completed"), Times.Never);
simpleListener.Verify(x => x.InitializeAndGetPort(), Times.AtLeastOnce);
simpleListener.Verify(x => x.StartAsync(), Times.AtLeastOnce);
simulators.VerifyAll();
}
public async Task RunTestAsync()
{
mainLog.WriteLine("Running '{0}' on device (candidates: '{1}')", testTask.ProjectFile, string.Join("', '", testTask.Candidates.Select((v) => v.Name).ToArray()));
var uninstall_log = testTask.Logs.Create($"uninstall-{Helpers.Timestamp}.log", "Uninstall log");
using (var device_resource = await testTask.NotifyBlockingWaitAsync(resourceManager.GetDeviceResources(testTask.Candidates).AcquireAnyConcurrentAsync())) {
try {
// Set the device we acquired.
testTask.Device = testTask.Candidates.First((d) => d.UDID == device_resource.Resource.Name);
if (testTask.Device.DevicePlatform == DevicePlatform.watchOS)
{
testTask.CompanionDevice = await devices.FindCompanionDevice(deviceLoadLog, testTask.Device);
}
mainLog.WriteLine("Acquired device '{0}' for '{1}'", testTask.Device.Name, testTask.ProjectFile);
ITunnelBore tunnelBore = null;
if (useTcpTunnel && testTask.Device.DevicePlatform != DevicePlatform.iOS &&
testTask.Device.DevicePlatform != DevicePlatform.tvOS)
{
mainLog.WriteLine("Ignoring request to use a tunnel because it is not supported by the specified platform");
}
else if (useTcpTunnel && (testTask.Device.DevicePlatform == DevicePlatform.iOS ||
testTask.Device.DevicePlatform == DevicePlatform.tvOS))
{
tunnelBore = testTask.TunnelBore;
mainLog.WriteLine("Using tunnel to communicate with device.");
}
testTask.Runner = new AppRunner(testTask.ProcessManager,
new AppBundleInformationParser(),
new SimulatorLoaderFactory(testTask.ProcessManager),
new SimpleListenerFactory(tunnelBore),
new DeviceLoaderFactory(testTask.ProcessManager),
new CrashSnapshotReporterFactory(testTask.ProcessManager),
new CaptureLogFactory(),
new DeviceLogCapturerFactory(testTask.ProcessManager),
new TestReporterFactory(testTask.ProcessManager),
testTask.AppRunnerTarget,
testTask.Harness,
projectFilePath: testTask.ProjectFile,
mainLog: uninstall_log,
logs: new Logs(testTask.LogDirectory ?? defaultLogDirectory),
buildConfiguration: testTask.ProjectConfiguration,
deviceName: testTask.Device.Name,
companionDeviceName: testTask.CompanionDevice?.Name,
timeoutMultiplier: testTask.TimeoutMultiplier,
variation: testTask.Variation,
buildTask: testTask.BuildTask);
await testTask.Runner.InitializeAsync();
// Sometimes devices can't upgrade (depending on what has changed), so make sure to uninstall any existing apps first.
if (uninstallTestApp)
{
testTask.Runner.MainLog = uninstall_log;
var uninstall_result = await testTask.Runner.UninstallAsync();
if (!uninstall_result.Succeeded)
{
mainLog.WriteLine($"Pre-run uninstall failed, exit code: {uninstall_result.ExitCode} (this hopefully won't affect the test result)");
}
}
else
{
uninstall_log.WriteLine($"Pre-run uninstall skipped.");
}
if (!testTask.Failed)
{
// Install the app
InstallLog = new AppInstallMonitorLog(testTask.Logs.Create($"install-{Helpers.Timestamp}.log", "Install log"));
try {
testTask.Runner.MainLog = this.InstallLog;
var install_result = await testTask.Runner.InstallAsync(InstallLog.CancellationToken);
if (!install_result.Succeeded)
{
testTask.FailureMessage = $"Install failed, exit code: {install_result.ExitCode}.";
testTask.ExecutionResult = TestExecutingResult.Failed;
if (generateXmlFailures)
{
resultParser.GenerateFailure(
testTask.Logs,
"install",
testTask.Runner.AppInformation.AppName,
testTask.Variation,
$"AppInstallation on {testTask.Device.Name}",
$"Install failed on {testTask.Device.Name}, exit code: {install_result.ExitCode}",
InstallLog.FullPath,
xmlResultJargon);
}
}
} finally {
InstallLog.Dispose();
InstallLog = null;
}
}
if (!testTask.Failed)
{
// Run the app
testTask.Runner.MainLog = testTask.Logs.Create($"run-{testTask.Device.UDID}-{Helpers.Timestamp}.log", "Run log");
await testTask.Runner.RunAsync();
if (!string.IsNullOrEmpty(testTask.Runner.FailureMessage))
{
testTask.FailureMessage = testTask.Runner.FailureMessage;
}
else if (testTask.Runner.Result != TestExecutingResult.Succeeded)
{
testTask.FailureMessage = testTask.GuessFailureReason(testTask.Runner.MainLog);
}
if (string.IsNullOrEmpty(testTask.FailureMessage) && errorKnowledgeBase.IsKnownTestIssue(testTask.Runner.MainLog, out var failure))
{
testTask.KnownFailure = failure;
mainLog.WriteLine($"Test run has a known failure: '{testTask.KnownFailure}'");
}
if (testTask.Runner.Result == TestExecutingResult.Succeeded && testTask.Platform == TestPlatform.iOS_TodayExtension64)
{
// For the today extension, the main app is just a single test.
// This is because running the today extension will not wake up the device,
// nor will it close & reopen the today app (but launching the main app
// will do both of these things, preparing the device for launching the today extension).
AppRunner todayRunner = new AppRunner(testTask.ProcessManager,
new AppBundleInformationParser(),
new SimulatorLoaderFactory(testTask.ProcessManager),
new SimpleListenerFactory(tunnelBore),
new DeviceLoaderFactory(testTask.ProcessManager),
new CrashSnapshotReporterFactory(testTask.ProcessManager),
new CaptureLogFactory(),
new DeviceLogCapturerFactory(testTask.ProcessManager),
new TestReporterFactory(testTask.ProcessManager),
testTask.AppRunnerTarget,
testTask.Harness,
projectFilePath: testTask.ProjectFile,
mainLog: testTask.Logs.Create($"extension-run-{testTask.Device.UDID}-{Helpers.Timestamp}.log", "Extension run log"),
logs: new Logs(testTask.LogDirectory ?? defaultLogDirectory),
buildConfiguration: testTask.ProjectConfiguration,
deviceName: testTask.Device.Name,
companionDeviceName: testTask.CompanionDevice?.Name,
timeoutMultiplier: testTask.TimeoutMultiplier,
variation: testTask.Variation,
buildTask: testTask.BuildTask);
await todayRunner.InitializeAsync();
testTask.AdditionalRunner = todayRunner;
await todayRunner.RunAsync();
foreach (var log in todayRunner.Logs.Where((v) => !v.Description.StartsWith("Extension ", StringComparison.Ordinal)))
{
log.Description = "Extension " + log.Description [0].ToString().ToLower() + log.Description.Substring(1);
}
testTask.ExecutionResult = todayRunner.Result;
if (!string.IsNullOrEmpty(todayRunner.FailureMessage))
{
testTask.FailureMessage = todayRunner.FailureMessage;
}
}
else
{
testTask.ExecutionResult = testTask.Runner.Result;
}
}
} finally {
// Uninstall again, so that we don't leave junk behind and fill up the device.
if (uninstallTestApp)
{
testTask.Runner.MainLog = uninstall_log;
var uninstall_result = await testTask.Runner.UninstallAsync();
if (!uninstall_result.Succeeded)
{
mainLog.WriteLine($"Post-run uninstall failed, exit code: {uninstall_result.ExitCode} (this won't affect the test result)");
}
}
else
{
uninstall_log.WriteLine($"Post-run uninstall skipped.");
}
// Also clean up after us locally.
if (inCI || cleanSuccessfulTestRuns && testTask.Succeeded)
{
await testTask.BuildTask.CleanAsync();
}
}
}
}
protected override async Task RunTestAsync()
{
Jenkins.MainLog.WriteLine("Running '{0}' on device (candidates: '{1}')", ProjectFile, string.Join("', '", Candidates.Select((v) => v.Name).ToArray()));
var uninstall_log = Logs.Create($"uninstall-{Timestamp}.log", "Uninstall log");
using (var device_resource = await NotifyBlockingWaitAsync(Jenkins.GetDeviceResources(Candidates).AcquireAnyConcurrentAsync())) {
try {
// Set the device we acquired.
Device = Candidates.First((d) => d.UDID == device_resource.Resource.Name);
if (Device.DevicePlatform == DevicePlatform.watchOS)
{
CompanionDevice = Jenkins.Devices.FindCompanionDevice(Jenkins.DeviceLoadLog, Device);
}
Jenkins.MainLog.WriteLine("Acquired device '{0}' for '{1}'", Device.Name, ProjectFile);
runner = new AppRunner {
Harness = Harness,
ProjectFile = ProjectFile,
Target = AppRunnerTarget,
LogDirectory = LogDirectory,
MainLog = uninstall_log,
DeviceName = Device.Name,
CompanionDeviceName = CompanionDevice?.Name,
Configuration = ProjectConfiguration,
TimeoutMultiplier = TimeoutMultiplier,
Variation = Variation,
BuildTask = BuildTask,
};
// Sometimes devices can't upgrade (depending on what has changed), so make sure to uninstall any existing apps first.
if (Jenkins.UninstallTestApp)
{
runner.MainLog = uninstall_log;
var uninstall_result = await runner.UninstallAsync();
if (!uninstall_result.Succeeded)
{
MainLog.WriteLine($"Pre-run uninstall failed, exit code: {uninstall_result.ExitCode} (this hopefully won't affect the test result)");
}
}
else
{
uninstall_log.WriteLine($"Pre-run uninstall skipped.");
}
if (!Failed)
{
// Install the app
this.install_log = new AppInstallMonitorLog(Logs.Create($"install-{Timestamp}.log", "Install log"));
try {
runner.MainLog = this.install_log;
var install_result = await runner.InstallAsync(install_log.CancellationToken);
if (!install_result.Succeeded)
{
FailureMessage = $"Install failed, exit code: {install_result.ExitCode}.";
ExecutionResult = TestExecutingResult.Failed;
if (Harness.InCI)
{
XmlResultParser.GenerateFailure(Logs, "install", runner.AppName, runner.Variation,
$"AppInstallation on {runner.DeviceName}", $"Install failed on {runner.DeviceName}, exit code: {install_result.ExitCode}",
install_log.FullPath, Harness.XmlJargon);
}
}
} finally {
this.install_log.Dispose();
this.install_log = null;
}
}
if (!Failed)
{
// Run the app
runner.MainLog = Logs.Create($"run-{Device.UDID}-{Timestamp}.log", "Run log");
await runner.RunAsync();
if (!string.IsNullOrEmpty(runner.FailureMessage))
{
FailureMessage = runner.FailureMessage;
}
else if (runner.Result != TestExecutingResult.Succeeded)
{
FailureMessage = GuessFailureReason(runner.MainLog);
}
if (runner.Result == TestExecutingResult.Succeeded && Platform == TestPlatform.iOS_TodayExtension64)
{
// For the today extension, the main app is just a single test.
// This is because running the today extension will not wake up the device,
// nor will it close & reopen the today app (but launching the main app
// will do both of these things, preparing the device for launching the today extension).
AppRunner todayRunner = new AppRunner {
Harness = Harness,
ProjectFile = TestProject.GetTodayExtension().Path,
Target = AppRunnerTarget,
LogDirectory = LogDirectory,
MainLog = Logs.Create($"extension-run-{Device.UDID}-{Timestamp}.log", "Extension run log"),
DeviceName = Device.Name,
CompanionDeviceName = CompanionDevice?.Name,
Configuration = ProjectConfiguration,
Variation = Variation,
BuildTask = BuildTask,
};
additional_runner = todayRunner;
await todayRunner.RunAsync();
foreach (var log in todayRunner.Logs.Where((v) => !v.Description.StartsWith("Extension ", StringComparison.Ordinal)))
{
log.Description = "Extension " + log.Description [0].ToString().ToLower() + log.Description.Substring(1);
}
ExecutionResult = todayRunner.Result;
if (!string.IsNullOrEmpty(todayRunner.FailureMessage))
{
FailureMessage = todayRunner.FailureMessage;
}
}
else
{
ExecutionResult = runner.Result;
}
}
} finally {
// Uninstall again, so that we don't leave junk behind and fill up the device.
if (Jenkins.UninstallTestApp)
{
runner.MainLog = uninstall_log;
var uninstall_result = await runner.UninstallAsync();
if (!uninstall_result.Succeeded)
{
MainLog.WriteLine($"Post-run uninstall failed, exit code: {uninstall_result.ExitCode} (this won't affect the test result)");
}
}
else
{
uninstall_log.WriteLine($"Post-run uninstall skipped.");
}
// Also clean up after us locally.
if (Harness.InCI || Jenkins.CleanSuccessfulTestRuns && Succeeded)
{
await BuildTask.CleanAsync();
}
}
}
}