private bool EnsurePathIsCreated(string path)
{
try
{
Directory.CreateDirectory(path);
return(true);
}
catch (Exception ex)
{
_console.WriteLine($"Error: Could not create logging path at {path}. {ex.GetBaseException().Message}");
}
return(false);
}
private void FinalizeTestRun()
{
IsRunning = false;
EndTime = DateTime.Now;
_console?.WriteLine($"Finalizing test run...");
var anyFailures = RunReports.SelectMany(x => x.Report.TestReports).Any(x => x.TestStatus == TestStatus.Fail);
if (anyFailures)
{
TestStatus = TestStatus.Fail;
}
else
{
TestStatus = TestStatus.Pass;
}
if (_console?.IsOutputRedirected != true)
{
_console?.ClearAtRange(0, BeginY, 0, BeginY + 1 + _lastNumberOfLinesDrawn);
_console?.SetCursorPosition(0, BeginY);
}
}
/// <summary>
/// Write the final report to the console output
/// </summary>
/// <param name="allReports"></param>
/// <param name="eventLog"></param>
public TestStatus WriteFinalReport()
{
var overallTestStatus = TestStatus.Fail;
var uniqueRunIds = _runContext.Runs.Select(x => x.Key.CommanderRunId).Distinct();
var commanderIdMap = new Dictionary <Guid, ICollection <Guid> >();
foreach (var commanderRunId in uniqueRunIds)
{
commanderIdMap.Add(commanderRunId, _runContext.Runs.SelectMany(x => x.Value.Select(y => y.TestRunId)).Distinct().ToList());
}
var allReports = _runContext.Runs.SelectMany(x => x.Value);
var startTime = _runContext.Runs.Select(x => x.Key.StartTime).OrderBy(x => x).FirstOrDefault();
var endTime = _runContext.Runs.Select(x => x.Key.EndTime).OrderByDescending(x => x).FirstOrDefault();
var totalDuration = TimeSpan.FromTicks(allReports.Sum(x => x.Duration.Ticks));
var isPassed = allReports
.SelectMany(x => x.Report.TestReports)
.Count(x => x.TestStatus == TestStatus.Fail) == 0;
if (isPassed)
{
overallTestStatus = TestStatus.Pass;
}
// ***********************
// Total Run Summary
// ***********************
var passFail = _reportFactory.Create <ReportSummary>(allReports);
// ***********************
// Multiple Run Summary
// ***********************
var passFailByRun = _reportFactory.Create <ReportRunsSummary>(allReports);
// ***********************
// Slowest Test Summary
// ***********************
var performance = new ColorTextBuilder();
if (_configuration.GenerateReportType.HasFlag(GenerateReportType.Performance))
{
performance.Append(_reportFactory.Create <SlowestTestSummary>());
}
// ***********************
// Slowest Assemblies Summary
// ***********************
if (_configuration.GenerateReportType.HasFlag(GenerateReportType.Performance))
{
performance.Append(_reportFactory.Create <SlowestAssembliesSummary>());
}
// ***********************
// Charts
// ***********************
if (_configuration.GenerateReportType.HasFlag(GenerateReportType.Charts))
{
performance.Append(_reportFactory.Create <StackedCharts>());
}
// ***********************
// Failed Tests Output
// ***********************
var testOutput = _reportFactory.Create <FailedTestsReport>(allReports);
// ***********************
// Total Run Overview
// ***********************
var overview = _reportFactory.Create <Overview>(allReports);
// ***********************
// Historical analysis
// ***********************
var historicalAnalysis = _reportFactory.Create <Overview>(allReports);
// ***********************
// Print all reports to console
// ***********************
_console.WriteLine(overview);
if (isPassed)
{
_console.WriteAscii(ColorTextBuilder.Create.Append("PASSED", _colorScheme.Success));
}
else
{
_console.WriteAscii(ColorTextBuilder.Create.Append("FAILED", _colorScheme.Error));
}
if (performance.Length > 0)
{
_console.WriteLine(performance);
}
if (testOutput.Length > 0)
{
_console.WriteLine(testOutput);
}
if (historicalAnalysis.Length > 0)
{
_console.WriteLine(historicalAnalysis);
}
if (passFailByRun.Length > 0)
{
_console.WriteLine(passFailByRun);
}
if (passFail.Length > 0)
{
_console.WriteLine(passFail);
}
if (_allowFileOperations)
{
EnsurePathIsCreated(_configuration.LogPath);
if (_configuration.EnableReportLog)
{
var builder = new StringBuilder();
builder.AppendLine(overview.ToString());
builder.AppendLine(performance.ToString());
builder.AppendLine(testOutput.ToString());
builder.AppendLine(historicalAnalysis.ToString());
builder.AppendLine(passFailByRun);
builder.AppendLine(passFail);
var reportFilename = Path.GetFullPath(Path.Combine(_configuration.LogPath, $"{uniqueRunIds.FirstOrDefault()}-report.log"));
File.WriteAllText(reportFilename, builder.ToString());
_console.Write(ColorTextBuilder.Create.AppendLine($"Wrote summary report to {reportFilename}", _colorScheme.DarkDefault));
}
if (_configuration.EnableTestLog)
{
// write out test logs for each run
foreach (var run in _runContext.Runs)
{
var builder = new StringBuilder();
builder.AppendLine($"FullName,Duration,TestStatus,StartTime,EndTime,RuntimeVersion");
foreach (var test in run.Value.SelectMany(x => x.Report.TestReports).Where(x => x.TestStatus != TestStatus.Skipped).OrderBy(x => x.StartTime))
{
// encode quotes in the test name with double quotes
var testName = test.FullName.Replace("\"", "\"\"");
builder.AppendLine($"\"{testName}\",\"{test.Duration.TotalMilliseconds:N2}ms\",\"{test.TestStatus}\",\"{test.StartTime.ToString(Constants.TimeFormat)}\",\"{test.EndTime.ToString(Constants.TimeFormat)}\",\"{test.RuntimeVersion}\"");
}
var reportFilename = Path.GetFullPath(Path.Combine(_configuration.LogPath, $"{run.Key.CommanderRunId}-tests.log"));
File.WriteAllText(reportFilename, builder.ToString());
_console.Write(ColorTextBuilder.Create.AppendLine($"Wrote tests report to {reportFilename}", _colorScheme.DarkDefault));
}
}
}
return(overallTestStatus);
}
private void ReceiveMessage(object sender, MessageEventArgs e)
{
_allowDrawActiveTests = true;
// a new message has been received from the IpcServer
_lock?.Wait();
// cancel any pending shutdown events
_finalReportGenerationCancelTokenSource?.Cancel();
if (StartTime == DateTime.MinValue)
{
StartTime = DateTime.Now;
}
if (RunContext?.StartTime == DateTime.MinValue)
{
RunContext.StartTime = DateTime.Now;
}
try
{
//System.Diagnostics.Debug.WriteLine($"Received Message {e.EventEntry.Event.Event} - {e.EventEntry.Event.TestName}, {e.EventEntry.Event.TestStatus} {e.EventEntry.Event.TestSuite}");
// inject data
e.EventEntry.Event.RunNumber = RunNumber;
_eventLog.Add(e.EventEntry);
if (!IsTestRunIdReceived(e.EventEntry.Event.TestRunId))
{
AddTestRunId(e.EventEntry.Event.TestRunId);
}
if (!IsFrameworkReceived(e.EventEntry.Event.Runtime))
{
AddFramework(e.EventEntry.Event.Runtime);
}
if (!IsFrameworkVersionReceived(e.EventEntry.Event.RuntimeVersion))
{
AddFrameworkVersion(e.EventEntry.Event.RuntimeVersion);
}
ProcessActiveTests(e.EventEntry);
if (e.EventEntry.Event.TestStatus == TestStatus.Fail && e.EventEntry.Event.Event == EventNames.EndTest)
{
// if we are logging to a file, and a test has failed write it immediately to the output
if (_console.IsOutputRedirected)
{
_console.WriteLine($"{Environment.NewLine}Failed test: {e.EventEntry.Event.FullName} {UTF8Constants.LeftBracket}{DateTime.Now}{UTF8Constants.RightBracket}");
if (!string.IsNullOrEmpty(e.EventEntry.Event.ErrorMessage))
{
_console.WriteLine($" Test Error: {e.EventEntry.Event.ErrorMessage}");
}
if (!string.IsNullOrEmpty(e.EventEntry.Event.StackTrace))
{
_console.WriteLine($" Stack Trace: {e.EventEntry.Event.StackTrace}");
}
if (!string.IsNullOrEmpty(e.EventEntry.Event.TestOutput))
{
_console.WriteLine($" Test Output: {e.EventEntry.Event.TestOutput}");
}
}
if (_configuration.ExitOnFirstTestFailure)
{
// close commander immediately on test failure
RunContext.EndTime = DateTime.Now;
EndTime = DateTime.Now;
Close();
}
}
}
finally
{
_lock?.Release();
}
if (e.EventEntry.Event.Event == EventNames.Report)
{
System.Diagnostics.Debug.WriteLine($"Report received for {e.EventEntry.Event.TestName}!");
// System.Diagnostics.Debug.WriteLine($"Active Test Suites: ({string.Join(",", _activeTestSuites.Where(x => !x.IsQueuedForRemoval))}), Active Assemblies: ({string.Join(",", _activeAssemblies.Where(x => !x.IsQueuedForRemoval))})");
if (_activeAssemblies.Count(x => !x.IsQueuedForRemoval) == 0)
{
//System.Diagnostics.Debug.WriteLine($"No active tests running, generating report!");
RunContext.EndTime = DateTime.Now;
EndTime = DateTime.Now;
ScheduleFinalizeWithDelay();
}
}
}
