private async Task <TestResult> RunTestAsyncInternal(AssemblyInfo assemblyInfo, bool retry, CancellationToken cancellationToken)
{
try
{
var commandLineArguments = GetCommandLineArguments(assemblyInfo, useSingleQuotes: false);
var resultsFilePath = GetResultsFilePath(assemblyInfo);
var resultsDir = Path.GetDirectoryName(resultsFilePath);
var htmlResultsFilePath = Options.IncludeHtml ? GetResultsFilePath(assemblyInfo, "html") : null;
var processResultList = new List <ProcessResult>();
ProcessInfo?procDumpProcessInfo = null;
// NOTE: xUnit doesn't always create the log directory
Directory.CreateDirectory(resultsDir);
// Define environment variables for processes started via ProcessRunner.
var environmentVariables = new Dictionary <string, string>();
Options.ProcDumpInfo?.WriteEnvironmentVariables(environmentVariables);
if (retry && File.Exists(resultsFilePath))
{
ConsoleUtil.WriteLine("Starting a retry. Tests which failed will run a second time to reduce flakiness.");
try
{
var doc = XDocument.Load(resultsFilePath);
foreach (var test in doc.XPathSelectElements("/assemblies/assembly/collection/test[@result='Fail']"))
{
ConsoleUtil.WriteLine($" {test.Attribute("name").Value}: {test.Attribute("result").Value}");
}
}
catch
{
ConsoleUtil.WriteLine(" ...Failed to identify the list of specific failures.");
}
// Copy the results file path, since the new xunit run will overwrite it
var backupResultsFilePath = Path.ChangeExtension(resultsFilePath, ".old");
File.Copy(resultsFilePath, backupResultsFilePath, overwrite: true);
// If running the process with this varialbe added, we assume that this file contains
// xml logs from the first attempt.
environmentVariables.Add("OutputXmlFilePath", backupResultsFilePath);
}
// NOTE: xUnit seems to have an occasional issue creating logs create
// an empty log just in case, so our runner will still fail.
File.Create(resultsFilePath).Close();
var start = DateTime.UtcNow;
var dotnetProcessInfo = ProcessRunner.CreateProcess(
ProcessRunner.CreateProcessStartInfo(
Options.DotnetFilePath,
commandLineArguments,
workingDirectory: Path.GetDirectoryName(assemblyInfo.AssemblyPath),
displayWindow: false,
captureOutput: true,
environmentVariables: environmentVariables),
lowPriority: false,
cancellationToken: cancellationToken);
Logger.Log($"Create xunit process with id {dotnetProcessInfo.Id} for test {assemblyInfo.DisplayName}");
var xunitProcessResult = await dotnetProcessInfo.Result;
var span = DateTime.UtcNow - start;
Logger.Log($"Exit xunit process with id {dotnetProcessInfo.Id} for test {assemblyInfo.DisplayName} with code {xunitProcessResult.ExitCode}");
processResultList.Add(xunitProcessResult);
if (procDumpProcessInfo != null)
{
var procDumpProcessResult = await procDumpProcessInfo.Value.Result;
Logger.Log($"Exit procdump process with id {procDumpProcessInfo.Value.Id} for {dotnetProcessInfo.Id} for test {assemblyInfo.DisplayName} with code {procDumpProcessResult.ExitCode}");
processResultList.Add(procDumpProcessResult);
}
if (xunitProcessResult.ExitCode != 0)
{
// On occasion we get a non-0 output but no actual data in the result file. The could happen
// if xunit manages to crash when running a unit test (a stack overflow could cause this, for instance).
// To avoid losing information, write the process output to the console. In addition, delete the results
// file to avoid issues with any tool attempting to interpret the (potentially malformed) text.
var resultData = string.Empty;
try
{
resultData = File.ReadAllText(resultsFilePath).Trim();
}
catch
{
// Happens if xunit didn't produce a log file
}
if (resultData.Length == 0)
{
// Delete the output file.
File.Delete(resultsFilePath);
resultsFilePath = null;
htmlResultsFilePath = null;
}
}
Logger.Log($"Command line {assemblyInfo.DisplayName}: {Options.DotnetFilePath} {commandLineArguments}");
var standardOutput = string.Join(Environment.NewLine, xunitProcessResult.OutputLines) ?? "";
var errorOutput = string.Join(Environment.NewLine, xunitProcessResult.ErrorLines) ?? "";
var testResultInfo = new TestResultInfo(
exitCode: xunitProcessResult.ExitCode,
resultsFilePath: resultsFilePath,
htmlResultsFilePath: htmlResultsFilePath,
elapsed: span,
standardOutput: standardOutput,
errorOutput: errorOutput);
return(new TestResult(
assemblyInfo,
testResultInfo,
commandLineArguments,
processResults: ImmutableArray.CreateRange(processResultList)));
}
catch (Exception ex)
{
throw new Exception($"Unable to run {assemblyInfo.AssemblyPath} with {Options.DotnetFilePath}. {ex}");
}
}
internal async Task <RunAllResult> RunAllOnHelixAsync(
IEnumerable <AssemblyInfo> assemblyInfoList,
CancellationToken cancellationToken
)
{
var sourceBranch = Environment.GetEnvironmentVariable("BUILD_SOURCEBRANCH");
if (sourceBranch is null)
{
sourceBranch = "local";
ConsoleUtil.WriteLine(
$@"BUILD_SOURCEBRANCH environment variable was not set. Using source branch ""{sourceBranch}"" instead"
);
Environment.SetEnvironmentVariable("BUILD_SOURCEBRANCH", sourceBranch);
}
var msbuildTestPayloadRoot = Path.GetDirectoryName(_options.ArtifactsDirectory);
if (msbuildTestPayloadRoot is null)
{
throw new IOException(
$@"Malformed ArtifactsDirectory in options: ""{_options.ArtifactsDirectory}"""
);
}
var isAzureDevOpsRun =
Environment.GetEnvironmentVariable("SYSTEM_ACCESSTOKEN") is not null;
if (!isAzureDevOpsRun)
{
ConsoleUtil.WriteLine(
"SYSTEM_ACCESSTOKEN environment variable was not set, so test results will not be published."
);
// in a local run we assume the user runs using the root test.sh and that the test payload is nested in the artifacts directory.
msbuildTestPayloadRoot = Path.Combine(
msbuildTestPayloadRoot,
"artifacts/testPayload"
);
}
var duplicateDir = Path.Combine(msbuildTestPayloadRoot, ".duplicate");
var correlationPayload = $@"<HelixCorrelationPayload Include=""{duplicateDir}"" />";
// https://github.com/dotnet/roslyn/issues/50661
// it's possible we should be using the BUILD_SOURCEVERSIONAUTHOR instead here a la https://github.com/dotnet/arcade/blob/main/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/Readme.md#how-to-use
// however that variable isn't documented at https://docs.microsoft.com/en-us/azure/devops/pipelines/build/variables?view=azure-devops&tabs=yaml
var queuedBy = Environment.GetEnvironmentVariable("BUILD_QUEUEDBY");
if (queuedBy is null)
{
queuedBy = "roslyn";
ConsoleUtil.WriteLine(
$@"BUILD_QUEUEDBY environment variable was not set. Using value ""{queuedBy}"" instead"
);
}
var jobName = Environment.GetEnvironmentVariable("SYSTEM_JOBDISPLAYNAME");
if (jobName is null)
{
ConsoleUtil.WriteLine(
$"SYSTEM_JOBDISPLAYNAME environment variable was not set. Using a blank TestRunNamePrefix for Helix job."
);
}
if (Environment.GetEnvironmentVariable("BUILD_REPOSITORY_NAME") is null)
{
Environment.SetEnvironmentVariable("BUILD_REPOSITORY_NAME", "dotnet/roslyn");
}
if (Environment.GetEnvironmentVariable("SYSTEM_TEAMPROJECT") is null)
{
Environment.SetEnvironmentVariable("SYSTEM_TEAMPROJECT", "dnceng");
}
if (Environment.GetEnvironmentVariable("BUILD_REASON") is null)
{
Environment.SetEnvironmentVariable("BUILD_REASON", "pr");
}
var buildNumber = Environment.GetEnvironmentVariable("BUILD_BUILDNUMBER") ?? "0";
var workItems = assemblyInfoList.Select(ai => makeHelixWorkItemProject(ai));
var globalJson = JsonConvert.DeserializeAnonymousType(
File.ReadAllText(getGlobalJsonPath()),
new { sdk = new { version = "" } }
);
var project =
@"
<Project Sdk=""Microsoft.DotNet.Helix.Sdk"" DefaultTargets=""Test"">
<PropertyGroup>
<TestRunNamePrefix>"
+ jobName
+ @"_</TestRunNamePrefix>
<HelixSource>pr/"
+ sourceBranch
+ @"</HelixSource>
<HelixType>test</HelixType>
<HelixBuild>"
+ buildNumber
+ @"</HelixBuild>
<HelixTargetQueues>"
+ _options.HelixQueueName
+ @"</HelixTargetQueues>
<Creator>"
+ queuedBy
+ @"</Creator>
<IncludeDotNetCli>true</IncludeDotNetCli>
<DotNetCliVersion>"
+ globalJson.sdk.version
+ @"</DotNetCliVersion>
<DotNetCliPackageType>sdk</DotNetCliPackageType>
<EnableAzurePipelinesReporter>"
+ (isAzureDevOpsRun ? "true" : "false")
+ @"</EnableAzurePipelinesReporter>
</PropertyGroup>
<ItemGroup>
"
+ correlationPayload
+ string.Join("", workItems)
+ @"
</ItemGroup>
</Project>
";
File.WriteAllText("helix-tmp.csproj", project);
var process = ProcessRunner.CreateProcess(
executable: _options.DotnetFilePath,
arguments: "build helix-tmp.csproj",
captureOutput: true,
onOutputDataReceived: (e) => ConsoleUtil.WriteLine(e.Data),
cancellationToken: cancellationToken
);
var result = await process.Result;
return(new RunAllResult(
result.ExitCode == 0,
ImmutableArray <TestResult> .Empty,
ImmutableArray.Create(result)
));
/// <summary>
/// Invoked when a timeout occurs and we need to dump all of the test processes and shut down
/// the runnner.
/// </summary>
private static async Task HandleTimeout(Options options, CancellationToken cancellationToken)
{
async Task DumpProcess(Process targetProcess, string procDumpExeFilePath, string dumpFilePath)
{
var name = targetProcess.ProcessName;
// Our space for saving dump files is limited. Skip dumping for processes that won't contribute
// to bug investigations.
if (name == "procdump" || name == "conhost")
{
return;
}
ConsoleUtil.Write($"Dumping {name} {targetProcess.Id} to {dumpFilePath} ... ");
try
{
var args = $"-accepteula -ma {targetProcess.Id} {dumpFilePath}";
var processInfo = ProcessRunner.CreateProcess(procDumpExeFilePath, args, cancellationToken: cancellationToken);
var processOutput = await processInfo.Result;
// The exit code for procdump doesn't obey standard windows rules. It will return non-zero
// for succesful cases (possibly returning the count of dumps that were written). Best
// backup is to test for the dump file being present.
if (File.Exists(dumpFilePath))
{
ConsoleUtil.WriteLine($"succeeded ({new FileInfo(dumpFilePath).Length} bytes)");
}
else
{
ConsoleUtil.WriteLine($"FAILED with {processOutput.ExitCode}");
ConsoleUtil.WriteLine($"{procDumpExeFilePath} {args}");
ConsoleUtil.WriteLine(string.Join(Environment.NewLine, processOutput.OutputLines));
}
}
catch (Exception ex) when(!cancellationToken.IsCancellationRequested)
{
ConsoleUtil.WriteLine("FAILED");
ConsoleUtil.WriteLine(ex.Message);
Logger.Log("Failed to dump process", ex);
}
}
ConsoleUtil.WriteLine("Roslyn Error: test timeout exceeded, dumping remaining processes");
var procDumpInfo = GetProcDumpInfo(options);
if (procDumpInfo != null)
{
var counter = 0;
foreach (var proc in ProcessUtil.GetProcessTree(Process.GetCurrentProcess()).OrderBy(x => x.ProcessName))
{
var dumpDir = PrimaryProcessNames.Contains(proc.ProcessName)
? procDumpInfo.Value.DumpDirectory
: procDumpInfo.Value.SecondaryDumpDirectory;
var dumpFilePath = Path.Combine(dumpDir, $"{proc.ProcessName}-{counter}.dmp");
await DumpProcess(proc, procDumpInfo.Value.ProcDumpFilePath, dumpFilePath);
counter++;
}
}
else
{
ConsoleUtil.WriteLine("Could not locate procdump");
}
WriteLogFile(options);
}
internal static async Task <int> Main(string[] args)
{
Logger.Log("RunTest command line");
Logger.Log(string.Join(" ", args));
var options = Options.Parse(args);
if (options == null)
{
return(ExitFailure);
}
ConsoleUtil.WriteLine($"Running '{options.DotnetFilePath} --version'..");
var dotnetResult = await ProcessRunner.CreateProcess(
options.DotnetFilePath,
arguments : "--version",
captureOutput : true
).Result;
ConsoleUtil.WriteLine(string.Join(Environment.NewLine, dotnetResult.OutputLines));
ConsoleUtil.WriteLine(
ConsoleColor.Red,
string.Join(Environment.NewLine, dotnetResult.ErrorLines)
);
if (options.CollectDumps)
{
if (!DumpUtil.IsAdministrator())
{
ConsoleUtil.WriteLine(
ConsoleColor.Yellow,
"Dump collection specified but user is not administrator so cannot modify registry"
);
}
else
{
DumpUtil.EnableRegistryDumpCollection(options.LogFilesDirectory);
}
}
try
{
// Setup cancellation for ctrl-c key presses
using var cts = new CancellationTokenSource();
Console.CancelKeyPress += delegate
{
cts.Cancel();
DisableRegistryDumpCollection();
};
int result;
if (options.Timeout is { } timeout)
{
result = await RunAsync(options, timeout, cts.Token);
}
else
{
result = await RunAsync(options, cts.Token);
}
CheckTotalDumpFilesSize();
return(result);
}
internal async Task <RunAllResult> RunAllAsync(IEnumerable <AssemblyInfo> assemblyInfoList, CancellationToken cancellationToken)
{
// Use 1.5 times the number of processors for unit tests, but only 1 processor for the open integration tests
// since they perform actual UI operations (such as mouse clicks and sending keystrokes) and we don't want two
// tests to conflict with one-another.
var max = (_options.TestVsi || _options.Sequential) ? 1 : (int)(Environment.ProcessorCount * 1.5);
var waiting = new Stack <AssemblyInfo>(assemblyInfoList);
var running = new List <Task <TestResult> >();
var completed = new List <TestResult>();
var failures = 0;
do
{
cancellationToken.ThrowIfCancellationRequested();
var i = 0;
while (i < running.Count)
{
var task = running[i];
if (task.IsCompleted)
{
try
{
var testResult = await task.ConfigureAwait(false);
if (!testResult.Succeeded)
{
failures++;
ConsoleUtil.WriteLine(ConsoleColor.Red, "Test failure log: " + testResult.ResultsFilePath);
}
completed.Add(testResult);
}
catch (Exception ex)
{
ConsoleUtil.WriteLine(ConsoleColor.Red, $"Error: {ex.Message}");
failures++;
}
running.RemoveAt(i);
}
else
{
i++;
}
}
while (running.Count < max && waiting.Count > 0)
{
var task = _testExecutor.RunTestAsync(waiting.Pop(), cancellationToken);
running.Add(task);
}
// Display the current status of the TestRunner.
// Note: The { ... , 2 } is to right align the values, thus aligns sections into columns.
ConsoleUtil.Write($" {running.Count,2} running, {waiting.Count,2} queued, {completed.Count,2} completed");
if (failures > 0)
{
ConsoleUtil.Write($", {failures,2} failures");
}
ConsoleUtil.WriteLine();
if (running.Count > 0)
{
await Task.WhenAny(running.ToArray());
}
} while (running.Count > 0);
Print(completed);
var processResults = ImmutableArray.CreateBuilder <ProcessResult>();
foreach (var c in completed)
{
processResults.AddRange(c.ProcessResults);
}
return(new RunAllResult((failures == 0), completed.ToImmutableArray(), processResults.ToImmutable()));
}
private async Task <TestResult> RunTestAsyncInternal(AssemblyInfo assemblyInfo, bool retry, CancellationToken cancellationToken)
{
try
{
var commandLineArguments = GetCommandLineArguments(assemblyInfo);
var resultsFilePath = GetResultsFilePath(assemblyInfo);
var resultsDir = Path.GetDirectoryName(resultsFilePath);
var processResultList = new List <ProcessResult>();
ProcessInfo?procDumpProcessInfo = null;
// NOTE: xUnit doesn't always create the log directory
Directory.CreateDirectory(resultsDir);
// Define environment variables for processes started via ProcessRunner.
var environmentVariables = new Dictionary <string, string>();
Options.ProcDumpInfo?.WriteEnvironmentVariables(environmentVariables);
if (retry && File.Exists(resultsFilePath))
{
// Copy the results file path, since the new xunit run will overwrite it
var backupResultsFilePath = Path.ChangeExtension(resultsFilePath, ".old");
File.Copy(resultsFilePath, backupResultsFilePath, overwrite: true);
ConsoleUtil.WriteLine("Starting a retry. It will run once again tests failed.");
// If running the process with this varialbe added, we assume that this file contains
// xml logs from the first attempt.
environmentVariables.Add("OutputXmlFilePath", backupResultsFilePath);
}
// NOTE: xUnit seems to have an occasional issue creating logs create
// an empty log just in case, so our runner will still fail.
File.Create(resultsFilePath).Close();
var start = DateTime.UtcNow;
var xunitProcessInfo = ProcessRunner.CreateProcess(
ProcessRunner.CreateProcessStartInfo(
Options.XunitPath,
commandLineArguments,
displayWindow: false,
captureOutput: true,
environmentVariables: environmentVariables),
lowPriority: false,
cancellationToken: cancellationToken);
Logger.Log($"Create xunit process with id {xunitProcessInfo.Id} for test {assemblyInfo.DisplayName}");
// Now that xunit is running we should kick off a procDump process if it was specified
if (Options.ProcDumpInfo != null)
{
var procDumpInfo = Options.ProcDumpInfo.Value;
var procDumpStartInfo = ProcessRunner.CreateProcessStartInfo(
procDumpInfo.ProcDumpFilePath,
ProcDumpUtil.GetProcDumpCommandLine(xunitProcessInfo.Id, procDumpInfo.DumpDirectory),
captureOutput: true,
displayWindow: false);
Directory.CreateDirectory(procDumpInfo.DumpDirectory);
procDumpProcessInfo = ProcessRunner.CreateProcess(procDumpStartInfo, cancellationToken: cancellationToken);
Logger.Log($"Create procdump process with id {procDumpProcessInfo.Value.Id} for xunit {xunitProcessInfo.Id} for test {assemblyInfo.DisplayName}");
}
var xunitProcessResult = await xunitProcessInfo.Result;
var span = DateTime.UtcNow - start;
Logger.Log($"Exit xunit process with id {xunitProcessInfo.Id} for test {assemblyInfo.DisplayName} with code {xunitProcessResult.ExitCode}");
processResultList.Add(xunitProcessResult);
if (procDumpProcessInfo != null)
{
var procDumpProcessResult = await procDumpProcessInfo.Value.Result;
Logger.Log($"Exit procdump process with id {procDumpProcessInfo.Value.Id} for {xunitProcessInfo.Id} for test {assemblyInfo.DisplayName} with code {procDumpProcessResult.ExitCode}");
processResultList.Add(procDumpProcessResult);
}
if (xunitProcessResult.ExitCode != 0)
{
// On occasion we get a non-0 output but no actual data in the result file. The could happen
// if xunit manages to crash when running a unit test (a stack overflow could cause this, for instance).
// To avoid losing information, write the process output to the console. In addition, delete the results
// file to avoid issues with any tool attempting to interpret the (potentially malformed) text.
var resultData = string.Empty;
try
{
resultData = File.ReadAllText(resultsFilePath).Trim();
}
catch
{
// Happens if xunit didn't produce a log file
}
if (resultData.Length == 0)
{
// Delete the output file.
File.Delete(resultsFilePath);
resultsFilePath = null;
}
}
var commandLine = GetCommandLine(assemblyInfo);
Logger.Log($"Command line {assemblyInfo.DisplayName}: {commandLine}");
var standardOutput = string.Join(Environment.NewLine, xunitProcessResult.OutputLines) ?? "";
var errorOutput = string.Join(Environment.NewLine, xunitProcessResult.ErrorLines) ?? "";
var testResultInfo = new TestResultInfo(
exitCode: xunitProcessResult.ExitCode,
resultsFilePath: resultsFilePath,
elapsed: span,
standardOutput: standardOutput,
errorOutput: errorOutput);
return(new TestResult(
assemblyInfo,
testResultInfo,
commandLine,
isFromCache: false,
processResults: ImmutableArray.CreateRange(processResultList)));
}
catch (Exception ex)
{
throw new Exception($"Unable to run {assemblyInfo.AssemblyPath} with {Options.XunitPath}. {ex}");
}
}
