internal TestResult(AssemblyInfo assemblyInfo, TestResultInfo testResultInfo, string commandLine, bool isFromCache, string diagnostics = null)
{
AssemblyInfo = assemblyInfo;
TestResultInfo = testResultInfo;
CommandLine = commandLine;
IsFromCache = isFromCache;
Diagnostics = diagnostics;
}
internal TestResult(AssemblyInfo assemblyInfo, TestResultInfo testResultInfo, string commandLine, ImmutableArray <ProcessResult> processResults = default, string?diagnostics = null)
{
AssemblyInfo = assemblyInfo;
TestResultInfo = testResultInfo;
CommandLine = commandLine;
ProcessResults = processResults.IsDefault ? ImmutableArray <ProcessResult> .Empty : processResults;
Diagnostics = diagnostics;
}
private async Task <TestResult> RunTestAsyncInternal(AssemblyInfo assemblyInfo, bool retry, CancellationToken cancellationToken)
{
try
{
var commandLineArguments = GetCommandLineArguments(assemblyInfo);
var resultsFilePath = GetResultsFilePath(assemblyInfo);
var resultsDir = Path.GetDirectoryName(resultsFilePath);
var processResultList = new List <ProcessResult>();
ProcessInfo?procDumpProcessInfo = null;
// NOTE: xUnit doesn't always create the log directory
Directory.CreateDirectory(resultsDir);
// Define environment variables for processes started via ProcessRunner.
var environmentVariables = new Dictionary <string, string>();
Options.ProcDumpInfo?.WriteEnvironmentVariables(environmentVariables);
if (retry && File.Exists(resultsFilePath))
{
// Copy the results file path, since the new xunit run will overwrite it
var backupResultsFilePath = Path.ChangeExtension(resultsFilePath, ".old");
File.Copy(resultsFilePath, backupResultsFilePath, overwrite: true);
ConsoleUtil.WriteLine("Starting a retry. It will run once again tests failed.");
// If running the process with this varialbe added, we assume that this file contains
// xml logs from the first attempt.
environmentVariables.Add("OutputXmlFilePath", backupResultsFilePath);
}
// NOTE: xUnit seems to have an occasional issue creating logs create
// an empty log just in case, so our runner will still fail.
File.Create(resultsFilePath).Close();
var start = DateTime.UtcNow;
var xunitProcessInfo = ProcessRunner.CreateProcess(
ProcessRunner.CreateProcessStartInfo(
Options.XunitPath,
commandLineArguments,
displayWindow: false,
captureOutput: true,
environmentVariables: environmentVariables),
lowPriority: false,
cancellationToken: cancellationToken);
Logger.Log($"Create xunit process with id {xunitProcessInfo.Id} for test {assemblyInfo.DisplayName}");
// Now that xunit is running we should kick off a procDump process if it was specified
if (Options.ProcDumpInfo != null)
{
var procDumpInfo = Options.ProcDumpInfo.Value;
var procDumpStartInfo = ProcessRunner.CreateProcessStartInfo(
procDumpInfo.ProcDumpFilePath,
ProcDumpUtil.GetProcDumpCommandLine(xunitProcessInfo.Id, procDumpInfo.DumpDirectory),
captureOutput: true,
displayWindow: false);
Directory.CreateDirectory(procDumpInfo.DumpDirectory);
procDumpProcessInfo = ProcessRunner.CreateProcess(procDumpStartInfo, cancellationToken: cancellationToken);
Logger.Log($"Create procdump process with id {procDumpProcessInfo.Value.Id} for xunit {xunitProcessInfo.Id} for test {assemblyInfo.DisplayName}");
}
var xunitProcessResult = await xunitProcessInfo.Result;
var span = DateTime.UtcNow - start;
Logger.Log($"Exit xunit process with id {xunitProcessInfo.Id} for test {assemblyInfo.DisplayName} with code {xunitProcessResult.ExitCode}");
processResultList.Add(xunitProcessResult);
if (procDumpProcessInfo != null)
{
var procDumpProcessResult = await procDumpProcessInfo.Value.Result;
Logger.Log($"Exit procdump process with id {procDumpProcessInfo.Value.Id} for {xunitProcessInfo.Id} for test {assemblyInfo.DisplayName} with code {procDumpProcessResult.ExitCode}");
processResultList.Add(procDumpProcessResult);
}
if (xunitProcessResult.ExitCode != 0)
{
// On occasion we get a non-0 output but no actual data in the result file. The could happen
// if xunit manages to crash when running a unit test (a stack overflow could cause this, for instance).
// To avoid losing information, write the process output to the console. In addition, delete the results
// file to avoid issues with any tool attempting to interpret the (potentially malformed) text.
var resultData = string.Empty;
try
{
resultData = File.ReadAllText(resultsFilePath).Trim();
}
catch
{
// Happens if xunit didn't produce a log file
}
if (resultData.Length == 0)
{
// Delete the output file.
File.Delete(resultsFilePath);
resultsFilePath = null;
}
}
var commandLine = GetCommandLine(assemblyInfo);
Logger.Log($"Command line {assemblyInfo.DisplayName}: {commandLine}");
var standardOutput = string.Join(Environment.NewLine, xunitProcessResult.OutputLines) ?? "";
var errorOutput = string.Join(Environment.NewLine, xunitProcessResult.ErrorLines) ?? "";
var testResultInfo = new TestResultInfo(
exitCode: xunitProcessResult.ExitCode,
resultsFilePath: resultsFilePath,
elapsed: span,
standardOutput: standardOutput,
errorOutput: errorOutput);
return(new TestResult(
assemblyInfo,
testResultInfo,
commandLine,
isFromCache: false,
processResults: ImmutableArray.CreateRange(processResultList)));
}
catch (Exception ex)
{
throw new Exception($"Unable to run {assemblyInfo.AssemblyPath} with {Options.XunitPath}. {ex}");
}
}
public async Task <TestResult> RunTestAsync(AssemblyInfo assemblyInfo, CancellationToken cancellationToken)
{
try
{
var commandLineArguments = GetCommandLineArguments(assemblyInfo);
var resultsFilePath = GetResultsFilePath(assemblyInfo);
var resultsDir = Path.GetDirectoryName(resultsFilePath);
// NOTE: xUnit doesn't always create the log directory
Directory.CreateDirectory(resultsDir);
// NOTE: xUnit seems to have an occasional issue creating logs create
// an empty log just in case, so our runner will still fail.
File.Create(resultsFilePath).Close();
// Define environment variables for processes started via ProcessRunner.
var environmentVariables = new Dictionary <string, string>();
_options.ProcDumpInfo?.WriteEnvironmentVariables(environmentVariables);
// Attach procDump to processes when the are started so we can watch for
// unexepected crashes.
void onProcessStart(Process process)
{
if (_options.ProcDumpInfo != null)
{
ProcDumpUtil.AttachProcDump(_options.ProcDumpInfo.Value, process.Id);
}
}
var start = DateTime.UtcNow;
var xunitPath = _options.XunitPath;
var processOutput = await ProcessRunner.RunProcessAsync(
xunitPath,
commandLineArguments,
lowPriority : false,
displayWindow : false,
captureOutput : true,
cancellationToken : cancellationToken,
environmentVariables : environmentVariables,
onProcessStartHandler : onProcessStart);
var span = DateTime.UtcNow - start;
if (processOutput.ExitCode != 0)
{
// On occasion we get a non-0 output but no actual data in the result file. The could happen
// if xunit manages to crash when running a unit test (a stack overflow could cause this, for instance).
// To avoid losing information, write the process output to the console. In addition, delete the results
// file to avoid issues with any tool attempting to interpret the (potentially malformed) text.
var resultData = string.Empty;
try
{
resultData = File.ReadAllText(resultsFilePath).Trim();
}
catch
{
// Happens if xunit didn't produce a log file
}
if (resultData.Length == 0)
{
// Delete the output file.
File.Delete(resultsFilePath);
resultsFilePath = null;
}
}
var commandLine = GetCommandLine(assemblyInfo);
Logger.Log($"Command line {assemblyInfo.DisplayName}: {commandLine}");
var standardOutput = string.Join(Environment.NewLine, processOutput.OutputLines) ?? "";
var errorOutput = string.Join(Environment.NewLine, processOutput.ErrorLines) ?? "";
var testResultInfo = new TestResultInfo(
exitCode: processOutput.ExitCode,
resultsDirectory: resultsDir,
resultsFilePath: resultsFilePath,
elapsed: span,
standardOutput: standardOutput,
errorOutput: errorOutput);
return(new TestResult(
assemblyInfo,
testResultInfo,
commandLine,
isFromCache: false));
}
catch (Exception ex)
{
throw new Exception($"Unable to run {assemblyInfo.AssemblyPath} with {_options.XunitPath}. {ex}");
}
}
private async Task <TestResult> RunTestAsyncInternal(AssemblyInfo assemblyInfo, bool retry, CancellationToken cancellationToken)
{
try
{
var commandLineArguments = GetCommandLineArguments(assemblyInfo, useSingleQuotes: false);
var resultsFilePath = GetResultsFilePath(assemblyInfo);
var resultsDir = Path.GetDirectoryName(resultsFilePath);
var htmlResultsFilePath = Options.IncludeHtml ? GetResultsFilePath(assemblyInfo, "html") : null;
var processResultList = new List <ProcessResult>();
ProcessInfo?procDumpProcessInfo = null;
// NOTE: xUnit doesn't always create the log directory
Directory.CreateDirectory(resultsDir !);
// Define environment variables for processes started via ProcessRunner.
var environmentVariables = new Dictionary <string, string>();
Options.ProcDumpInfo?.WriteEnvironmentVariables(environmentVariables);
if (retry && File.Exists(resultsFilePath))
{
ConsoleUtil.WriteLine("Starting a retry. Tests which failed will run a second time to reduce flakiness.");
try
{
var doc = XDocument.Load(resultsFilePath);
foreach (var test in doc.XPathSelectElements("/assemblies/assembly/collection/test[@result='Fail']"))
{
ConsoleUtil.WriteLine($" {test.Attribute("name")!.Value}: {test.Attribute("result")!.Value}");
}
}
catch
{
ConsoleUtil.WriteLine(" ...Failed to identify the list of specific failures.");
}
// Copy the results file path, since the new xunit run will overwrite it
var backupResultsFilePath = Path.ChangeExtension(resultsFilePath, ".old");
File.Copy(resultsFilePath, backupResultsFilePath, overwrite: true);
// If running the process with this varialbe added, we assume that this file contains
// xml logs from the first attempt.
environmentVariables.Add("OutputXmlFilePath", backupResultsFilePath);
}
// NOTE: xUnit seems to have an occasional issue creating logs create
// an empty log just in case, so our runner will still fail.
File.Create(resultsFilePath).Close();
var start = DateTime.UtcNow;
var dotnetProcessInfo = ProcessRunner.CreateProcess(
ProcessRunner.CreateProcessStartInfo(
Options.DotnetFilePath,
commandLineArguments,
workingDirectory: Path.GetDirectoryName(assemblyInfo.AssemblyPath),
displayWindow: false,
captureOutput: true,
environmentVariables: environmentVariables),
lowPriority: false,
cancellationToken: cancellationToken);
Logger.Log($"Create xunit process with id {dotnetProcessInfo.Id} for test {assemblyInfo.DisplayName}");
var xunitProcessResult = await dotnetProcessInfo.Result;
var span = DateTime.UtcNow - start;
Logger.Log($"Exit xunit process with id {dotnetProcessInfo.Id} for test {assemblyInfo.DisplayName} with code {xunitProcessResult.ExitCode}");
processResultList.Add(xunitProcessResult);
if (procDumpProcessInfo != null)
{
var procDumpProcessResult = await procDumpProcessInfo.Value.Result;
Logger.Log($"Exit procdump process with id {procDumpProcessInfo.Value.Id} for {dotnetProcessInfo.Id} for test {assemblyInfo.DisplayName} with code {procDumpProcessResult.ExitCode}");
processResultList.Add(procDumpProcessResult);
}
if (xunitProcessResult.ExitCode != 0)
{
// On occasion we get a non-0 output but no actual data in the result file. The could happen
// if xunit manages to crash when running a unit test (a stack overflow could cause this, for instance).
// To avoid losing information, write the process output to the console. In addition, delete the results
// file to avoid issues with any tool attempting to interpret the (potentially malformed) text.
var resultData = string.Empty;
try
{
resultData = File.ReadAllText(resultsFilePath).Trim();
}
catch
{
// Happens if xunit didn't produce a log file
}
if (resultData.Length == 0)
{
// Delete the output file.
File.Delete(resultsFilePath);
resultsFilePath = null;
htmlResultsFilePath = null;
}
}
Logger.Log($"Command line {assemblyInfo.DisplayName}: {Options.DotnetFilePath} {commandLineArguments}");
var standardOutput = string.Join(Environment.NewLine, xunitProcessResult.OutputLines) ?? "";
var errorOutput = string.Join(Environment.NewLine, xunitProcessResult.ErrorLines) ?? "";
var testResultInfo = new TestResultInfo(
exitCode: xunitProcessResult.ExitCode,
resultsFilePath: resultsFilePath,
htmlResultsFilePath: htmlResultsFilePath,
elapsed: span,
standardOutput: standardOutput,
errorOutput: errorOutput);
return(new TestResult(
assemblyInfo,
testResultInfo,
commandLineArguments,
processResults: ImmutableArray.CreateRange(processResultList)));
}
catch (Exception ex)
{
throw new Exception($"Unable to run {assemblyInfo.AssemblyPath} with {Options.DotnetFilePath}. {ex}");
}
}