public bool TestsSucceeded(Log listener_log, bool timed_out, bool crashed) { string log; using (var reader = listener_log.GetReader()) log = reader.ReadToEnd(); // parsing the result is different if we are in jenkins or not. // When in Jenkins, Touch.Unit produces an xml file instead of a console log (so that we can get better test reporting). // However, for our own reporting, we still want the console-based log. This log is embedded inside the xml produced // by Touch.Unit, so we need to extract it and write it to disk. We also need to re-save the xml output, since Touch.Unit // wraps the NUnit xml output with additional information, which we need to unwrap so that Jenkins understands it. if (Harness.InJenkins) { // we have to parse the xml result crashed = false; var xmldoc = new XmlDocument(); try { xmldoc.LoadXml(log); var nunit_output = xmldoc.SelectSingleNode("/TouchUnitTestRun/NUnitOutput"); var xmllog = nunit_output.InnerXml; var extra_output = xmldoc.SelectSingleNode("/TouchUnitTestRun/TouchUnitExtraData"); log = extra_output.InnerText; File.WriteAllText(listener_log.FullPath, log); var testsResults = new XmlDocument(); testsResults.LoadXml(xmllog); var mainResultNode = testsResults.SelectSingleNode("test-results"); if (mainResultNode == null) { Harness.LogWrench($"Node is null."); } else { // update the information of the main node to add information about the mode and the test that is excuted. This will later create // nicer reports in jenkins mainResultNode.Attributes ["name"].Value = Target.AsString(); // store a clean version of the logs, later this will be used by the bots to show results in github/web var path = listener_log.FullPath; path = Path.ChangeExtension(path, "xml"); testsResults.Save(path); Logs.AddFile(path, "Test xml"); } } catch (Exception e) { main_log.WriteLine("Could not parse xml result file: {0}", e); if (timed_out) { Harness.LogWrench($"@MonkeyWrench: AddSummary: <b><i>{mode} timed out</i></b><br/>"); return(false); } else { Harness.LogWrench($"@MonkeyWrench: AddSummary: <b><i>{mode} crashed</i></b><br/>"); main_log.WriteLine("Test run crashed"); crashed = true; return(false); } } } // parsing the human readable results if (log.Contains("Tests run")) { var tests_run = string.Empty; var log_lines = log.Split('\n'); var failed = false; foreach (var line in log_lines) { if (line.Contains("Tests run:")) { Console.WriteLine(line); tests_run = line.Replace("Tests run: ", ""); break; } else if (line.Contains("FAIL")) { Console.WriteLine(line); failed = true; } } if (failed) { Harness.LogWrench("@MonkeyWrench: AddSummary: <b>{0} failed: {1}</b><br/>", mode, tests_run); main_log.WriteLine("Test run failed"); return(false); } else { Harness.LogWrench("@MonkeyWrench: AddSummary: {0} succeeded: {1}<br/>", mode, tests_run); main_log.WriteLine("Test run succeeded"); return(true); } } else if (timed_out) { Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} timed out</i></b><br/>", mode); return(false); } else { Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} crashed</i></b><br/>", mode); main_log.WriteLine("Test run crashed"); crashed = true; return(false); } }
public async Task EndCaptureAsync(TimeSpan timeout) { // Check for crash reports var crash_report_search_done = false; var crash_report_search_timeout = timeout.TotalSeconds; var watch = new Stopwatch(); watch.Start(); do { var end_crashes = await Harness.CreateCrashReportsSnapshotAsync(Log, !Device, DeviceName); end_crashes.ExceptWith(InitialSet); Reports = end_crashes; if (end_crashes.Count > 0) { Log.WriteLine("Found {0} new crash report(s)", end_crashes.Count); List <LogFile> crash_reports; if (!Device) { crash_reports = new List <LogFile> (end_crashes.Count); foreach (var path in end_crashes) { Logs.AddFile(path, $"Crash report: {Path.GetFileName (path)}"); } } else { // Download crash reports from the device. We put them in the project directory so that they're automatically deleted on wrench // (if we put them in /tmp, they'd never be deleted). var downloaded_crash_reports = new List <LogFile> (); foreach (var file in end_crashes) { var name = Path.GetFileName(file); var crash_report_target = Logs.Create(name, $"Crash report: {name}"); var sb = new StringBuilder(); sb.Append(" --download-crash-report=").Append(StringUtils.Quote(file)); sb.Append(" --download-crash-report-to=").Append(StringUtils.Quote(crash_report_target.Path)); sb.Append(" --sdkroot ").Append(StringUtils.Quote(Harness.XcodeRoot)); if (!string.IsNullOrEmpty(DeviceName)) { sb.Append(" --devname ").Append(StringUtils.Quote(DeviceName)); } var result = await ProcessHelper.ExecuteCommandAsync(Harness.MlaunchPath, sb.ToString(), Log, TimeSpan.FromMinutes(1)); if (result.Succeeded) { Log.WriteLine("Downloaded crash report {0} to {1}", file, crash_report_target.Path); crash_report_target = await Harness.SymbolicateCrashReportAsync(Logs, Log, crash_report_target); downloaded_crash_reports.Add(crash_report_target); } else { Log.WriteLine("Could not download crash report {0}", file); } } crash_reports = downloaded_crash_reports; } foreach (var cp in crash_reports) { Harness.LogWrench("@MonkeyWrench: AddFile: {0}", cp.Path); Log.WriteLine(" {0}", cp.Path); } crash_report_search_done = true; } else { if (watch.Elapsed.TotalSeconds > crash_report_search_timeout) { crash_report_search_done = true; } else { Log.WriteLine("No crash reports, waiting a second to see if the crash report service just didn't complete in time ({0})", (int)(crash_report_search_timeout - watch.Elapsed.TotalSeconds)); Thread.Sleep(TimeSpan.FromSeconds(1)); } } } while (!crash_report_search_done); }
public bool TestsSucceeded(Log listener_log, bool timed_out, bool crashed) { // parsing the result is different if we are in jenkins or not. // When in Jenkins, Touch.Unit produces an xml file instead of a console log (so that we can get better test reporting). // However, for our own reporting, we still want the console-based log. This log is embedded inside the xml produced // by Touch.Unit, so we need to extract it and write it to disk. We also need to re-save the xml output, since Touch.Unit // wraps the NUnit xml output with additional information, which we need to unwrap so that Jenkins understands it. if (Harness.InJenkins) { // use a tmp file so that we can use the reader in xml and write the humman version var tmpFile = Path.GetTempFileName(); // we have to parse the xml result crashed = false; var xmldoc = new XmlDocument(); XmlNode mainResultNode; try { using (var reader = listener_log.GetReader()) { xmldoc.Load(reader); var testsResults = new XmlDocument(); using (var writer = new StreamWriter(tmpFile)) { if (IsTouchUnitResult(xmldoc)) { testsResults = ParseTouchUnitXml(xmldoc, writer); } else { testsResults = ParseNUnitXml(xmldoc, writer); } } mainResultNode = testsResults.SelectSingleNode("test-results"); } if (mainResultNode == null) { Harness.LogWrench($"Node is null."); } else { // update the information of the main node to add information about the mode and the test that is excuted. This will later create // nicer reports in jenkins mainResultNode.Attributes ["name"].Value = Target.AsString(); // store a clean version of the logs, later this will be used by the bots to show results in github/web var path = listener_log.FullPath; path = Path.ChangeExtension(path, "xml"); // we already have all the data needed in the listerner, rather than saving from the doc, copy File.Copy(listener_log.FullPath, path, true); Logs.AddFile(path, "Test xml"); } // write on the log File.Copy(tmpFile, listener_log.FullPath, true); File.Delete(tmpFile); } catch (Exception e) { main_log.WriteLine("Could not parse xml result file: {0}", e); if (timed_out) { Harness.LogWrench($"@MonkeyWrench: AddSummary: <b><i>{mode} timed out</i></b><br/>"); return(false); } else { Harness.LogWrench($"@MonkeyWrench: AddSummary: <b><i>{mode} crashed</i></b><br/>"); main_log.WriteLine("Test run crashed"); crashed = true; return(false); } } } // read until the end and decide if we got results, do not store them // we do not want to use too much memory string resultLine = null; using (var reader = new StreamReader(listener_log.FullPath)) { string line; while ((line = reader.ReadLine()) != null) { if (line.Contains("Tests run")) { resultLine = line; } } } // read the parsed logs in a human readable way if (resultLine != null) { var tests_run = string.Empty; var failed = false; using (var reader = listener_log.GetReader()) { string line; while ((line = reader.ReadLine()) != null) { if (line.Contains("Tests run:")) { Console.WriteLine(line); tests_run = line.Replace("Tests run: ", ""); break; } else if (line.Contains("[FAIL]")) { Console.WriteLine(line); failed = true; } } } if (failed) { Harness.LogWrench("@MonkeyWrench: AddSummary: <b>{0} failed: {1}</b><br/>", mode, tests_run); main_log.WriteLine("Test run failed"); return(false); } else { Harness.LogWrench("@MonkeyWrench: AddSummary: {0} succeeded: {1}<br/>", mode, tests_run); main_log.WriteLine("Test run succeeded"); return(true); } } else if (timed_out) { Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} timed out</i></b><br/>", mode); return(false); } else { Harness.LogWrench("@MonkeyWrench: AddSummary: <b><i>{0} crashed</i></b><br/>", mode); main_log.WriteLine("Test run crashed"); crashed = true; return(false); } }