private bool EnsurePathIsCreated(string path) { try { Directory.CreateDirectory(path); return(true); } catch (Exception ex) { _console.WriteLine($"Error: Could not create logging path at {path}. {ex.GetBaseException().Message}"); } return(false); }
private void FinalizeTestRun() { IsRunning = false; EndTime = DateTime.Now; _console?.WriteLine($"Finalizing test run..."); var anyFailures = RunReports.SelectMany(x => x.Report.TestReports).Any(x => x.TestStatus == TestStatus.Fail); if (anyFailures) { TestStatus = TestStatus.Fail; } else { TestStatus = TestStatus.Pass; } if (_console?.IsOutputRedirected != true) { _console?.ClearAtRange(0, BeginY, 0, BeginY + 1 + _lastNumberOfLinesDrawn); _console?.SetCursorPosition(0, BeginY); } }
/// <summary> /// Write the final report to the console output /// </summary> /// <param name="allReports"></param> /// <param name="eventLog"></param> public TestStatus WriteFinalReport() { var overallTestStatus = TestStatus.Fail; var uniqueRunIds = _runContext.Runs.Select(x => x.Key.CommanderRunId).Distinct(); var commanderIdMap = new Dictionary <Guid, ICollection <Guid> >(); foreach (var commanderRunId in uniqueRunIds) { commanderIdMap.Add(commanderRunId, _runContext.Runs.SelectMany(x => x.Value.Select(y => y.TestRunId)).Distinct().ToList()); } var allReports = _runContext.Runs.SelectMany(x => x.Value); var startTime = _runContext.Runs.Select(x => x.Key.StartTime).OrderBy(x => x).FirstOrDefault(); var endTime = _runContext.Runs.Select(x => x.Key.EndTime).OrderByDescending(x => x).FirstOrDefault(); var totalDuration = TimeSpan.FromTicks(allReports.Sum(x => x.Duration.Ticks)); var isPassed = allReports .SelectMany(x => x.Report.TestReports) .Count(x => x.TestStatus == TestStatus.Fail) == 0; if (isPassed) { overallTestStatus = TestStatus.Pass; } // *********************** // Total Run Summary // *********************** var passFail = _reportFactory.Create <ReportSummary>(allReports); // *********************** // Multiple Run Summary // *********************** var passFailByRun = _reportFactory.Create <ReportRunsSummary>(allReports); // *********************** // Slowest Test Summary // *********************** var performance = new ColorTextBuilder(); if (_configuration.GenerateReportType.HasFlag(GenerateReportType.Performance)) { performance.Append(_reportFactory.Create <SlowestTestSummary>()); } // *********************** // Slowest Assemblies Summary // *********************** if (_configuration.GenerateReportType.HasFlag(GenerateReportType.Performance)) { performance.Append(_reportFactory.Create <SlowestAssembliesSummary>()); } // *********************** // Charts // *********************** if (_configuration.GenerateReportType.HasFlag(GenerateReportType.Charts)) { performance.Append(_reportFactory.Create <StackedCharts>()); } // *********************** // Failed Tests Output // *********************** var testOutput = _reportFactory.Create <FailedTestsReport>(allReports); // *********************** // Total Run Overview // *********************** var overview = _reportFactory.Create <Overview>(allReports); // *********************** // Historical analysis // *********************** var historicalAnalysis = _reportFactory.Create <Overview>(allReports); // *********************** // Print all reports to console // *********************** _console.WriteLine(overview); if (isPassed) { _console.WriteAscii(ColorTextBuilder.Create.Append("PASSED", _colorScheme.Success)); } else { _console.WriteAscii(ColorTextBuilder.Create.Append("FAILED", _colorScheme.Error)); } if (performance.Length > 0) { _console.WriteLine(performance); } if (testOutput.Length > 0) { _console.WriteLine(testOutput); } if (historicalAnalysis.Length > 0) { _console.WriteLine(historicalAnalysis); } if (passFailByRun.Length > 0) { _console.WriteLine(passFailByRun); } if (passFail.Length > 0) { _console.WriteLine(passFail); } if (_allowFileOperations) { EnsurePathIsCreated(_configuration.LogPath); if (_configuration.EnableReportLog) { var builder = new StringBuilder(); builder.AppendLine(overview.ToString()); builder.AppendLine(performance.ToString()); builder.AppendLine(testOutput.ToString()); builder.AppendLine(historicalAnalysis.ToString()); builder.AppendLine(passFailByRun); builder.AppendLine(passFail); var reportFilename = Path.GetFullPath(Path.Combine(_configuration.LogPath, $"{uniqueRunIds.FirstOrDefault()}-report.log")); File.WriteAllText(reportFilename, builder.ToString()); _console.Write(ColorTextBuilder.Create.AppendLine($"Wrote summary report to {reportFilename}", _colorScheme.DarkDefault)); } if (_configuration.EnableTestLog) { // write out test logs for each run foreach (var run in _runContext.Runs) { var builder = new StringBuilder(); builder.AppendLine($"FullName,Duration,TestStatus,StartTime,EndTime,RuntimeVersion"); foreach (var test in run.Value.SelectMany(x => x.Report.TestReports).Where(x => x.TestStatus != TestStatus.Skipped).OrderBy(x => x.StartTime)) { // encode quotes in the test name with double quotes var testName = test.FullName.Replace("\"", "\"\""); builder.AppendLine($"\"{testName}\",\"{test.Duration.TotalMilliseconds:N2}ms\",\"{test.TestStatus}\",\"{test.StartTime.ToString(Constants.TimeFormat)}\",\"{test.EndTime.ToString(Constants.TimeFormat)}\",\"{test.RuntimeVersion}\""); } var reportFilename = Path.GetFullPath(Path.Combine(_configuration.LogPath, $"{run.Key.CommanderRunId}-tests.log")); File.WriteAllText(reportFilename, builder.ToString()); _console.Write(ColorTextBuilder.Create.AppendLine($"Wrote tests report to {reportFilename}", _colorScheme.DarkDefault)); } } } return(overallTestStatus); }
private void ReceiveMessage(object sender, MessageEventArgs e) { _allowDrawActiveTests = true; // a new message has been received from the IpcServer _lock?.Wait(); // cancel any pending shutdown events _finalReportGenerationCancelTokenSource?.Cancel(); if (StartTime == DateTime.MinValue) { StartTime = DateTime.Now; } if (RunContext?.StartTime == DateTime.MinValue) { RunContext.StartTime = DateTime.Now; } try { //System.Diagnostics.Debug.WriteLine($"Received Message {e.EventEntry.Event.Event} - {e.EventEntry.Event.TestName}, {e.EventEntry.Event.TestStatus} {e.EventEntry.Event.TestSuite}"); // inject data e.EventEntry.Event.RunNumber = RunNumber; _eventLog.Add(e.EventEntry); if (!IsTestRunIdReceived(e.EventEntry.Event.TestRunId)) { AddTestRunId(e.EventEntry.Event.TestRunId); } if (!IsFrameworkReceived(e.EventEntry.Event.Runtime)) { AddFramework(e.EventEntry.Event.Runtime); } if (!IsFrameworkVersionReceived(e.EventEntry.Event.RuntimeVersion)) { AddFrameworkVersion(e.EventEntry.Event.RuntimeVersion); } ProcessActiveTests(e.EventEntry); if (e.EventEntry.Event.TestStatus == TestStatus.Fail && e.EventEntry.Event.Event == EventNames.EndTest) { // if we are logging to a file, and a test has failed write it immediately to the output if (_console.IsOutputRedirected) { _console.WriteLine($"{Environment.NewLine}Failed test: {e.EventEntry.Event.FullName} {UTF8Constants.LeftBracket}{DateTime.Now}{UTF8Constants.RightBracket}"); if (!string.IsNullOrEmpty(e.EventEntry.Event.ErrorMessage)) { _console.WriteLine($" Test Error: {e.EventEntry.Event.ErrorMessage}"); } if (!string.IsNullOrEmpty(e.EventEntry.Event.StackTrace)) { _console.WriteLine($" Stack Trace: {e.EventEntry.Event.StackTrace}"); } if (!string.IsNullOrEmpty(e.EventEntry.Event.TestOutput)) { _console.WriteLine($" Test Output: {e.EventEntry.Event.TestOutput}"); } } if (_configuration.ExitOnFirstTestFailure) { // close commander immediately on test failure RunContext.EndTime = DateTime.Now; EndTime = DateTime.Now; Close(); } } } finally { _lock?.Release(); } if (e.EventEntry.Event.Event == EventNames.Report) { System.Diagnostics.Debug.WriteLine($"Report received for {e.EventEntry.Event.TestName}!"); // System.Diagnostics.Debug.WriteLine($"Active Test Suites: ({string.Join(",", _activeTestSuites.Where(x => !x.IsQueuedForRemoval))}), Active Assemblies: ({string.Join(",", _activeAssemblies.Where(x => !x.IsQueuedForRemoval))})"); if (_activeAssemblies.Count(x => !x.IsQueuedForRemoval) == 0) { //System.Diagnostics.Debug.WriteLine($"No active tests running, generating report!"); RunContext.EndTime = DateTime.Now; EndTime = DateTime.Now; ScheduleFinalizeWithDelay(); } } }