public async Task <bool> BuildAsync() { if (Finished) { return(true); } await VerifyBuildAsync(); if (Finished) { return(BuildTask.Succeeded); } ExecutionResult = TestExecutingResult.Building; await BuildTask.RunAsync(); if (!BuildTask.Succeeded) { if (BuildTask.TimedOut) { ExecutionResult = TestExecutingResult.TimedOut; } else { ExecutionResult = TestExecutingResult.BuildFailure; } FailureMessage = BuildTask.FailureMessage; if (Harness.InCI && BuildTask is MSBuildTask projectTask) { XmlResultParser.GenerateFailure(Logs, "build", projectTask.TestName, projectTask.Variation, "AppBuild", $"App could not be built {FailureMessage}.", projectTask.BuildLog.FullPath, Harness.XmlJargon); } } else { ExecutionResult = TestExecutingResult.Built; } return(BuildTask.Succeeded); }
public void GenerateFailureTest(XmlResultJargon jargon) { var src = "test-case"; var appName = "MyUnitTest"; var variation = "Debug"; var title = "Testing"; var message = "This is a test"; var stderrMessage = "Something went very wrong"; var stderrPath = Path.GetTempFileName(); // write the message in the stderrParh that should be read using (var writer = new StreamWriter(stderrPath)) { writer.WriteLine(stderrMessage); } // create a path with data in it var logs = new Mock <ILogs> (); var tmpLogMock = new Mock <ILog> (); var xmlLogMock = new Mock <ILog> (); var tmpPath = Path.GetTempFileName(); var finalPath = Path.GetTempFileName(); // create a number of fake logs to be added to the failure var logsDir = Path.GetTempFileName(); File.Delete(logsDir); Directory.CreateDirectory(logsDir); var failureLogs = new [] { "first.txt", "second.txt", "last.txt" }; foreach (var file in failureLogs) { var path = Path.Combine(logsDir, file); File.WriteAllText(path, ""); } // expect the creation of the two diff xml file logs _ = logs.Setup(l => l.Create(It.IsAny <string> (), "Failure Log tmp", null)).Returns(tmpLogMock.Object); _ = logs.Setup(l => l.Create(It.IsAny <string> (), LogType.XmlLog.ToString(), null)).Returns(xmlLogMock.Object); if (jargon == XmlResultJargon.NUnitV3) { _ = logs.Setup(l => l.Directory).Returns(logsDir); _ = tmpLogMock.Setup(tmpLog => tmpLog.FullPath).Returns(tmpPath); } // return the two temp files so that we can later validate that everything is present _ = xmlLogMock.Setup(xmlLog => xmlLog.FullPath).Returns(finalPath); resultParser.GenerateFailure(logs.Object, src, appName, variation, title, message, stderrPath, jargon); // actual assertions do happen in the validation functions ValidationMap [jargon] (src, appName, variation, title, message, stderrMessage, finalPath, failureLogs.Length); // verify that we are correctly adding the logs logs.Verify(l => l.Create(It.IsAny <string> (), It.IsAny <string> (), null), jargon == XmlResultJargon.NUnitV3 ? Times.AtMost(2) : Times.AtMostOnce()); if (jargon == XmlResultJargon.NUnitV3) { logs.Verify(l => l.Directory, Times.Once); tmpLogMock.Verify(l => l.FullPath, Times.AtLeastOnce); } xmlLogMock.Verify(l => l.FullPath, Times.AtLeastOnce); // clean files File.Delete(stderrPath); File.Delete(tmpPath); File.Delete(finalPath); Directory.Delete(logsDir, true); }
protected override async Task RunTestAsync() { Jenkins.MainLog.WriteLine("Running '{0}' on device (candidates: '{1}')", ProjectFile, string.Join("', '", Candidates.Select((v) => v.Name).ToArray())); var uninstall_log = Logs.Create($"uninstall-{Timestamp}.log", "Uninstall log"); using (var device_resource = await NotifyBlockingWaitAsync(Jenkins.GetDeviceResources(Candidates).AcquireAnyConcurrentAsync())) { try { // Set the device we acquired. Device = Candidates.First((d) => d.UDID == device_resource.Resource.Name); if (Device.DevicePlatform == DevicePlatform.watchOS) { CompanionDevice = Jenkins.Devices.FindCompanionDevice(Jenkins.DeviceLoadLog, Device); } Jenkins.MainLog.WriteLine("Acquired device '{0}' for '{1}'", Device.Name, ProjectFile); runner = new AppRunner { Harness = Harness, ProjectFile = ProjectFile, Target = AppRunnerTarget, LogDirectory = LogDirectory, MainLog = uninstall_log, DeviceName = Device.Name, CompanionDeviceName = CompanionDevice?.Name, Configuration = ProjectConfiguration, TimeoutMultiplier = TimeoutMultiplier, Variation = Variation, BuildTask = BuildTask, }; // Sometimes devices can't upgrade (depending on what has changed), so make sure to uninstall any existing apps first. if (Jenkins.UninstallTestApp) { runner.MainLog = uninstall_log; var uninstall_result = await runner.UninstallAsync(); if (!uninstall_result.Succeeded) { MainLog.WriteLine($"Pre-run uninstall failed, exit code: {uninstall_result.ExitCode} (this hopefully won't affect the test result)"); } } else { uninstall_log.WriteLine($"Pre-run uninstall skipped."); } if (!Failed) { // Install the app this.install_log = new AppInstallMonitorLog(Logs.Create($"install-{Timestamp}.log", "Install log")); try { runner.MainLog = this.install_log; var install_result = await runner.InstallAsync(install_log.CancellationToken); if (!install_result.Succeeded) { FailureMessage = $"Install failed, exit code: {install_result.ExitCode}."; ExecutionResult = TestExecutingResult.Failed; if (Harness.InCI) { XmlResultParser.GenerateFailure(Logs, "install", runner.AppName, runner.Variation, $"AppInstallation on {runner.DeviceName}", $"Install failed on {runner.DeviceName}, exit code: {install_result.ExitCode}", install_log.FullPath, Harness.XmlJargon); } } } finally { this.install_log.Dispose(); this.install_log = null; } } if (!Failed) { // Run the app runner.MainLog = Logs.Create($"run-{Device.UDID}-{Timestamp}.log", "Run log"); await runner.RunAsync(); if (!string.IsNullOrEmpty(runner.FailureMessage)) { FailureMessage = runner.FailureMessage; } else if (runner.Result != TestExecutingResult.Succeeded) { FailureMessage = GuessFailureReason(runner.MainLog); } if (runner.Result == TestExecutingResult.Succeeded && Platform == TestPlatform.iOS_TodayExtension64) { // For the today extension, the main app is just a single test. // This is because running the today extension will not wake up the device, // nor will it close & reopen the today app (but launching the main app // will do both of these things, preparing the device for launching the today extension). AppRunner todayRunner = new AppRunner { Harness = Harness, ProjectFile = TestProject.GetTodayExtension().Path, Target = AppRunnerTarget, LogDirectory = LogDirectory, MainLog = Logs.Create($"extension-run-{Device.UDID}-{Timestamp}.log", "Extension run log"), DeviceName = Device.Name, CompanionDeviceName = CompanionDevice?.Name, Configuration = ProjectConfiguration, Variation = Variation, BuildTask = BuildTask, }; additional_runner = todayRunner; await todayRunner.RunAsync(); foreach (var log in todayRunner.Logs.Where((v) => !v.Description.StartsWith("Extension ", StringComparison.Ordinal))) { log.Description = "Extension " + log.Description [0].ToString().ToLower() + log.Description.Substring(1); } ExecutionResult = todayRunner.Result; if (!string.IsNullOrEmpty(todayRunner.FailureMessage)) { FailureMessage = todayRunner.FailureMessage; } } else { ExecutionResult = runner.Result; } } } finally { // Uninstall again, so that we don't leave junk behind and fill up the device. if (Jenkins.UninstallTestApp) { runner.MainLog = uninstall_log; var uninstall_result = await runner.UninstallAsync(); if (!uninstall_result.Succeeded) { MainLog.WriteLine($"Post-run uninstall failed, exit code: {uninstall_result.ExitCode} (this won't affect the test result)"); } } else { uninstall_log.WriteLine($"Post-run uninstall skipped."); } // Also clean up after us locally. if (Harness.InCI || Jenkins.CleanSuccessfulTestRuns && Succeeded) { await BuildTask.CleanAsync(); } } } }