public async Task <bool> BuildAsync()
{
if (Finished)
{
return(true);
}
await VerifyBuildAsync();
if (Finished)
{
return(BuildTask.Succeeded);
}
ExecutionResult = TestExecutingResult.Building;
await BuildTask.RunAsync();
if (!BuildTask.Succeeded)
{
if (BuildTask.TimedOut)
{
ExecutionResult = TestExecutingResult.TimedOut;
}
else
{
ExecutionResult = TestExecutingResult.BuildFailure;
}
FailureMessage = BuildTask.FailureMessage;
if (Harness.InCI && BuildTask is MSBuildTask projectTask)
{
XmlResultParser.GenerateFailure(Logs, "build", projectTask.TestName, projectTask.Variation, "AppBuild", $"App could not be built {FailureMessage}.", projectTask.BuildLog.FullPath, Harness.XmlJargon);
}
}
else
{
ExecutionResult = TestExecutingResult.Built;
}
return(BuildTask.Succeeded);
}
public void GenerateFailureTest(XmlResultJargon jargon)
{
var src = "test-case";
var appName = "MyUnitTest";
var variation = "Debug";
var title = "Testing";
var message = "This is a test";
var stderrMessage = "Something went very wrong";
var stderrPath = Path.GetTempFileName();
// write the message in the stderrParh that should be read
using (var writer = new StreamWriter(stderrPath)) {
writer.WriteLine(stderrMessage);
}
// create a path with data in it
var logs = new Mock <ILogs> ();
var tmpLogMock = new Mock <ILog> ();
var xmlLogMock = new Mock <ILog> ();
var tmpPath = Path.GetTempFileName();
var finalPath = Path.GetTempFileName();
// create a number of fake logs to be added to the failure
var logsDir = Path.GetTempFileName();
File.Delete(logsDir);
Directory.CreateDirectory(logsDir);
var failureLogs = new [] { "first.txt", "second.txt", "last.txt" };
foreach (var file in failureLogs)
{
var path = Path.Combine(logsDir, file);
File.WriteAllText(path, "");
}
// expect the creation of the two diff xml file logs
_ = logs.Setup(l => l.Create(It.IsAny <string> (), "Failure Log tmp", null)).Returns(tmpLogMock.Object);
_ = logs.Setup(l => l.Create(It.IsAny <string> (), LogType.XmlLog.ToString(), null)).Returns(xmlLogMock.Object);
if (jargon == XmlResultJargon.NUnitV3)
{
_ = logs.Setup(l => l.Directory).Returns(logsDir);
_ = tmpLogMock.Setup(tmpLog => tmpLog.FullPath).Returns(tmpPath);
}
// return the two temp files so that we can later validate that everything is present
_ = xmlLogMock.Setup(xmlLog => xmlLog.FullPath).Returns(finalPath);
resultParser.GenerateFailure(logs.Object, src, appName, variation, title, message, stderrPath, jargon);
// actual assertions do happen in the validation functions
ValidationMap [jargon] (src, appName, variation, title, message, stderrMessage, finalPath, failureLogs.Length);
// verify that we are correctly adding the logs
logs.Verify(l => l.Create(It.IsAny <string> (), It.IsAny <string> (), null), jargon == XmlResultJargon.NUnitV3 ? Times.AtMost(2) : Times.AtMostOnce());
if (jargon == XmlResultJargon.NUnitV3)
{
logs.Verify(l => l.Directory, Times.Once);
tmpLogMock.Verify(l => l.FullPath, Times.AtLeastOnce);
}
xmlLogMock.Verify(l => l.FullPath, Times.AtLeastOnce);
// clean files
File.Delete(stderrPath);
File.Delete(tmpPath);
File.Delete(finalPath);
Directory.Delete(logsDir, true);
}
protected override async Task RunTestAsync()
{
Jenkins.MainLog.WriteLine("Running '{0}' on device (candidates: '{1}')", ProjectFile, string.Join("', '", Candidates.Select((v) => v.Name).ToArray()));
var uninstall_log = Logs.Create($"uninstall-{Timestamp}.log", "Uninstall log");
using (var device_resource = await NotifyBlockingWaitAsync(Jenkins.GetDeviceResources(Candidates).AcquireAnyConcurrentAsync())) {
try {
// Set the device we acquired.
Device = Candidates.First((d) => d.UDID == device_resource.Resource.Name);
if (Device.DevicePlatform == DevicePlatform.watchOS)
{
CompanionDevice = Jenkins.Devices.FindCompanionDevice(Jenkins.DeviceLoadLog, Device);
}
Jenkins.MainLog.WriteLine("Acquired device '{0}' for '{1}'", Device.Name, ProjectFile);
runner = new AppRunner {
Harness = Harness,
ProjectFile = ProjectFile,
Target = AppRunnerTarget,
LogDirectory = LogDirectory,
MainLog = uninstall_log,
DeviceName = Device.Name,
CompanionDeviceName = CompanionDevice?.Name,
Configuration = ProjectConfiguration,
TimeoutMultiplier = TimeoutMultiplier,
Variation = Variation,
BuildTask = BuildTask,
};
// Sometimes devices can't upgrade (depending on what has changed), so make sure to uninstall any existing apps first.
if (Jenkins.UninstallTestApp)
{
runner.MainLog = uninstall_log;
var uninstall_result = await runner.UninstallAsync();
if (!uninstall_result.Succeeded)
{
MainLog.WriteLine($"Pre-run uninstall failed, exit code: {uninstall_result.ExitCode} (this hopefully won't affect the test result)");
}
}
else
{
uninstall_log.WriteLine($"Pre-run uninstall skipped.");
}
if (!Failed)
{
// Install the app
this.install_log = new AppInstallMonitorLog(Logs.Create($"install-{Timestamp}.log", "Install log"));
try {
runner.MainLog = this.install_log;
var install_result = await runner.InstallAsync(install_log.CancellationToken);
if (!install_result.Succeeded)
{
FailureMessage = $"Install failed, exit code: {install_result.ExitCode}.";
ExecutionResult = TestExecutingResult.Failed;
if (Harness.InCI)
{
XmlResultParser.GenerateFailure(Logs, "install", runner.AppName, runner.Variation,
$"AppInstallation on {runner.DeviceName}", $"Install failed on {runner.DeviceName}, exit code: {install_result.ExitCode}",
install_log.FullPath, Harness.XmlJargon);
}
}
} finally {
this.install_log.Dispose();
this.install_log = null;
}
}
if (!Failed)
{
// Run the app
runner.MainLog = Logs.Create($"run-{Device.UDID}-{Timestamp}.log", "Run log");
await runner.RunAsync();
if (!string.IsNullOrEmpty(runner.FailureMessage))
{
FailureMessage = runner.FailureMessage;
}
else if (runner.Result != TestExecutingResult.Succeeded)
{
FailureMessage = GuessFailureReason(runner.MainLog);
}
if (runner.Result == TestExecutingResult.Succeeded && Platform == TestPlatform.iOS_TodayExtension64)
{
// For the today extension, the main app is just a single test.
// This is because running the today extension will not wake up the device,
// nor will it close & reopen the today app (but launching the main app
// will do both of these things, preparing the device for launching the today extension).
AppRunner todayRunner = new AppRunner {
Harness = Harness,
ProjectFile = TestProject.GetTodayExtension().Path,
Target = AppRunnerTarget,
LogDirectory = LogDirectory,
MainLog = Logs.Create($"extension-run-{Device.UDID}-{Timestamp}.log", "Extension run log"),
DeviceName = Device.Name,
CompanionDeviceName = CompanionDevice?.Name,
Configuration = ProjectConfiguration,
Variation = Variation,
BuildTask = BuildTask,
};
additional_runner = todayRunner;
await todayRunner.RunAsync();
foreach (var log in todayRunner.Logs.Where((v) => !v.Description.StartsWith("Extension ", StringComparison.Ordinal)))
{
log.Description = "Extension " + log.Description [0].ToString().ToLower() + log.Description.Substring(1);
}
ExecutionResult = todayRunner.Result;
if (!string.IsNullOrEmpty(todayRunner.FailureMessage))
{
FailureMessage = todayRunner.FailureMessage;
}
}
else
{
ExecutionResult = runner.Result;
}
}
} finally {
// Uninstall again, so that we don't leave junk behind and fill up the device.
if (Jenkins.UninstallTestApp)
{
runner.MainLog = uninstall_log;
var uninstall_result = await runner.UninstallAsync();
if (!uninstall_result.Succeeded)
{
MainLog.WriteLine($"Post-run uninstall failed, exit code: {uninstall_result.ExitCode} (this won't affect the test result)");
}
}
else
{
uninstall_log.WriteLine($"Post-run uninstall skipped.");
}
// Also clean up after us locally.
if (Harness.InCI || Jenkins.CleanSuccessfulTestRuns && Succeeded)
{
await BuildTask.CleanAsync();
}
}
}
}