internal static async Task <int> Main(string[] args)
{
Logger.Log("RunTest command line");
Logger.Log(string.Join(" ", args));
var options = Options.Parse(args);
if (options == null)
{
return(ExitFailure);
}
ConsoleUtil.WriteLine($"Running '{options.DotnetFilePath} --version'..");
var dotnetResult = await ProcessRunner.CreateProcess(options.DotnetFilePath, arguments : "--version", captureOutput : true).Result;
ConsoleUtil.WriteLine(string.Join(Environment.NewLine, dotnetResult.OutputLines));
ConsoleUtil.WriteLine(ConsoleColor.Red, string.Join(Environment.NewLine, dotnetResult.ErrorLines));
if (options.CollectDumps)
{
if (!DumpUtil.IsAdministrator())
{
ConsoleUtil.WriteLine(ConsoleColor.Yellow, "Dump collection specified but user is not administrator so cannot modify registry");
}
else
{
DumpUtil.EnableRegistryDumpCollection(options.LogFilesDirectory);
}
}
try
{
// Setup cancellation for ctrl-c key presses
using var cts = new CancellationTokenSource();
Console.CancelKeyPress += delegate
{
cts.Cancel();
DisableRegistryDumpCollection();
};
int result;
if (options.Timeout is { } timeout)
{
result = await RunAsync(options, timeout, cts.Token);
}
else
{
result = await RunAsync(options, cts.Token);
}
CheckTotalDumpFilesSize();
return(result);
}
/// <summary>
/// Invoked when a timeout occurs and we need to dump all of the test processes and shut down
/// the runnner.
/// </summary>
private static async Task HandleTimeout(Options options, CancellationToken cancellationToken)
{
async Task DumpProcess(Process targetProcess, string procDumpExeFilePath, string dumpFilePath)
{
var name = targetProcess.ProcessName;
// Our space for saving dump files is limited. Skip dumping for processes that won't contribute
// to bug investigations.
if (name == "procdump" || name == "conhost")
{
return;
}
ConsoleUtil.Write($"Dumping {name} {targetProcess.Id} to {dumpFilePath} ... ");
try
{
var args = $"-accepteula -ma {targetProcess.Id} {dumpFilePath}";
var processInfo = ProcessRunner.CreateProcess(procDumpExeFilePath, args, cancellationToken: cancellationToken);
var processOutput = await processInfo.Result;
// The exit code for procdump doesn't obey standard windows rules. It will return non-zero
// for successful cases (possibly returning the count of dumps that were written). Best
// backup is to test for the dump file being present.
if (File.Exists(dumpFilePath))
{
ConsoleUtil.WriteLine($"succeeded ({new FileInfo(dumpFilePath).Length} bytes)");
}
else
{
ConsoleUtil.WriteLine($"FAILED with {processOutput.ExitCode}");
ConsoleUtil.WriteLine($"{procDumpExeFilePath} {args}");
ConsoleUtil.WriteLine(string.Join(Environment.NewLine, processOutput.OutputLines));
}
}
catch (Exception ex) when(!cancellationToken.IsCancellationRequested)
{
ConsoleUtil.WriteLine("FAILED");
ConsoleUtil.WriteLine(ex.Message);
Logger.Log("Failed to dump process", ex);
}
}
ConsoleUtil.WriteLine("Roslyn Error: test timeout exceeded, dumping remaining processes");
var procDumpInfo = GetProcDumpInfo(options);
if (procDumpInfo != null)
{
var counter = 0;
foreach (var proc in ProcessUtil.GetProcessTree(Process.GetCurrentProcess()).OrderBy(x => x.ProcessName))
{
var dumpDir = PrimaryProcessNames.Contains(proc.ProcessName)
? procDumpInfo.Value.DumpDirectory
: procDumpInfo.Value.SecondaryDumpDirectory;
var dumpFilePath = Path.Combine(dumpDir, $"{proc.ProcessName}-{counter}.dmp");
await DumpProcess(proc, procDumpInfo.Value.ProcDumpFilePath, dumpFilePath);
counter++;
}
}
else
{
ConsoleUtil.WriteLine("Could not locate procdump");
}
WriteLogFile(options);
}
internal async Task <RunAllResult> RunAllOnHelixAsync(IEnumerable <AssemblyInfo> assemblyInfoList, CancellationToken cancellationToken)
{
var sourceBranch = Environment.GetEnvironmentVariable("BUILD_SOURCEBRANCH");
if (sourceBranch is null)
{
sourceBranch = "local";
ConsoleUtil.WriteLine($@"BUILD_SOURCEBRANCH environment variable was not set. Using source branch ""{sourceBranch}"" instead");
Environment.SetEnvironmentVariable("BUILD_SOURCEBRANCH", sourceBranch);
}
var msbuildTestPayloadRoot = Path.GetDirectoryName(_options.ArtifactsDirectory);
if (msbuildTestPayloadRoot is null)
{
throw new IOException($@"Malformed ArtifactsDirectory in options: ""{_options.ArtifactsDirectory}""");
}
var isAzureDevOpsRun = Environment.GetEnvironmentVariable("SYSTEM_ACCESSTOKEN") is not null;
if (!isAzureDevOpsRun)
{
ConsoleUtil.WriteLine("SYSTEM_ACCESSTOKEN environment variable was not set, so test results will not be published.");
// in a local run we assume the user runs using the root test.sh and that the test payload is nested in the artifacts directory.
msbuildTestPayloadRoot = Path.Combine(msbuildTestPayloadRoot, "artifacts/testPayload");
}
var duplicateDir = Path.Combine(msbuildTestPayloadRoot, ".duplicate");
var correlationPayload = $@"<HelixCorrelationPayload Include=""{duplicateDir}"" />";
// https://github.com/dotnet/roslyn/issues/50661
// it's possible we should be using the BUILD_SOURCEVERSIONAUTHOR instead here a la https://github.com/dotnet/arcade/blob/master/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/Readme.md#how-to-use
// however that variable isn't documented at https://docs.microsoft.com/en-us/azure/devops/pipelines/build/variables?view=azure-devops&tabs=yaml
var queuedBy = Environment.GetEnvironmentVariable("BUILD_QUEUEDBY");
if (queuedBy is null)
{
queuedBy = "roslyn";
ConsoleUtil.WriteLine($@"BUILD_QUEUEDBY environment variable was not set. Using value ""{queuedBy}"" instead");
}
var jobName = Environment.GetEnvironmentVariable("SYSTEM_JOBDISPLAYNAME");
if (jobName is null)
{
ConsoleUtil.WriteLine($"SYSTEM_JOBDISPLAYNAME environment variable was not set. Using a blank TestRunNamePrefix for Helix job.");
}
if (Environment.GetEnvironmentVariable("BUILD_REPOSITORY_NAME") is null)
{
Environment.SetEnvironmentVariable("BUILD_REPOSITORY_NAME", "dotnet/roslyn");
}
if (Environment.GetEnvironmentVariable("SYSTEM_TEAMPROJECT") is null)
{
Environment.SetEnvironmentVariable("SYSTEM_TEAMPROJECT", "dnceng");
}
if (Environment.GetEnvironmentVariable("BUILD_REASON") is null)
{
Environment.SetEnvironmentVariable("BUILD_REASON", "pr");
}
var buildNumber = Environment.GetEnvironmentVariable("BUILD_BUILDNUMBER") ?? "0";
var workItems = assemblyInfoList.Select(ai => makeHelixWorkItemProject(ai));
var globalJson = JsonConvert.DeserializeAnonymousType(File.ReadAllText(getGlobalJsonPath()), new { sdk = new { version = "" } });
var project = @"
<Project Sdk=""Microsoft.DotNet.Helix.Sdk"" DefaultTargets=""Test"">
<PropertyGroup>
<TestRunNamePrefix>" + jobName + @"_</TestRunNamePrefix>
<HelixSource>pr/" + sourceBranch + @"</HelixSource>
<HelixType>test</HelixType>
<HelixBuild>" + buildNumber + @"</HelixBuild>
<HelixTargetQueues>" + _options.HelixQueueName + @"</HelixTargetQueues>
<Creator>" + queuedBy + @"</Creator>
<IncludeDotNetCli>true</IncludeDotNetCli>
<DotNetCliVersion>" + globalJson.sdk.version + @"</DotNetCliVersion>
<DotNetCliPackageType>sdk</DotNetCliPackageType>
<EnableAzurePipelinesReporter>" + (isAzureDevOpsRun ? "true" : "false") + @"</EnableAzurePipelinesReporter>
</PropertyGroup>
<ItemGroup>
" + correlationPayload + string.Join("", workItems) + @"
</ItemGroup>
</Project>
";
File.WriteAllText("helix-tmp.csproj", project);
var process = ProcessRunner.CreateProcess(
executable: _options.DotnetFilePath,
arguments: "build helix-tmp.csproj",
captureOutput: true,
onOutputDataReceived: (e) => ConsoleUtil.WriteLine(e.Data),
cancellationToken: cancellationToken);
var result = await process.Result;
return(new RunAllResult(result.ExitCode == 0, ImmutableArray <TestResult> .Empty, ImmutableArray.Create(result)));
private async Task <TestResult> RunTestAsyncInternal(AssemblyInfo assemblyInfo, bool retry, CancellationToken cancellationToken)
{
try
{
var commandLineArguments = GetCommandLineArguments(assemblyInfo);
var resultsFilePath = GetResultsFilePath(assemblyInfo);
var resultsDir = Path.GetDirectoryName(resultsFilePath);
var processResultList = new List <ProcessResult>();
ProcessInfo?procDumpProcessInfo = null;
// NOTE: xUnit doesn't always create the log directory
Directory.CreateDirectory(resultsDir);
// Define environment variables for processes started via ProcessRunner.
var environmentVariables = new Dictionary <string, string>();
Options.ProcDumpInfo?.WriteEnvironmentVariables(environmentVariables);
if (retry && File.Exists(resultsFilePath))
{
// Copy the results file path, since the new xunit run will overwrite it
var backupResultsFilePath = Path.ChangeExtension(resultsFilePath, ".old");
File.Copy(resultsFilePath, backupResultsFilePath, overwrite: true);
ConsoleUtil.WriteLine("Starting a retry. It will run once again tests failed.");
// If running the process with this varialbe added, we assume that this file contains
// xml logs from the first attempt.
environmentVariables.Add("OutputXmlFilePath", backupResultsFilePath);
}
// NOTE: xUnit seems to have an occasional issue creating logs create
// an empty log just in case, so our runner will still fail.
File.Create(resultsFilePath).Close();
var start = DateTime.UtcNow;
var xunitProcessInfo = ProcessRunner.CreateProcess(
ProcessRunner.CreateProcessStartInfo(
Options.XunitPath,
commandLineArguments,
displayWindow: false,
captureOutput: true,
environmentVariables: environmentVariables),
lowPriority: false,
cancellationToken: cancellationToken);
Logger.Log($"Create xunit process with id {xunitProcessInfo.Id} for test {assemblyInfo.DisplayName}");
// Now that xunit is running we should kick off a procDump process if it was specified
if (Options.ProcDumpInfo != null)
{
var procDumpInfo = Options.ProcDumpInfo.Value;
var procDumpStartInfo = ProcessRunner.CreateProcessStartInfo(
procDumpInfo.ProcDumpFilePath,
ProcDumpUtil.GetProcDumpCommandLine(xunitProcessInfo.Id, procDumpInfo.DumpDirectory),
captureOutput: true,
displayWindow: false);
Directory.CreateDirectory(procDumpInfo.DumpDirectory);
procDumpProcessInfo = ProcessRunner.CreateProcess(procDumpStartInfo, cancellationToken: cancellationToken);
Logger.Log($"Create procdump process with id {procDumpProcessInfo.Value.Id} for xunit {xunitProcessInfo.Id} for test {assemblyInfo.DisplayName}");
}
var xunitProcessResult = await xunitProcessInfo.Result;
var span = DateTime.UtcNow - start;
Logger.Log($"Exit xunit process with id {xunitProcessInfo.Id} for test {assemblyInfo.DisplayName} with code {xunitProcessResult.ExitCode}");
processResultList.Add(xunitProcessResult);
if (procDumpProcessInfo != null)
{
var procDumpProcessResult = await procDumpProcessInfo.Value.Result;
Logger.Log($"Exit procdump process with id {procDumpProcessInfo.Value.Id} for {xunitProcessInfo.Id} for test {assemblyInfo.DisplayName} with code {procDumpProcessResult.ExitCode}");
processResultList.Add(procDumpProcessResult);
}
if (xunitProcessResult.ExitCode != 0)
{
// On occasion we get a non-0 output but no actual data in the result file. The could happen
// if xunit manages to crash when running a unit test (a stack overflow could cause this, for instance).
// To avoid losing information, write the process output to the console. In addition, delete the results
// file to avoid issues with any tool attempting to interpret the (potentially malformed) text.
var resultData = string.Empty;
try
{
resultData = File.ReadAllText(resultsFilePath).Trim();
}
catch
{
// Happens if xunit didn't produce a log file
}
if (resultData.Length == 0)
{
// Delete the output file.
File.Delete(resultsFilePath);
resultsFilePath = null;
}
}
var commandLine = GetCommandLine(assemblyInfo);
Logger.Log($"Command line {assemblyInfo.DisplayName}: {commandLine}");
var standardOutput = string.Join(Environment.NewLine, xunitProcessResult.OutputLines) ?? "";
var errorOutput = string.Join(Environment.NewLine, xunitProcessResult.ErrorLines) ?? "";
var testResultInfo = new TestResultInfo(
exitCode: xunitProcessResult.ExitCode,
resultsFilePath: resultsFilePath,
elapsed: span,
standardOutput: standardOutput,
errorOutput: errorOutput);
return(new TestResult(
assemblyInfo,
testResultInfo,
commandLine,
isFromCache: false,
processResults: ImmutableArray.CreateRange(processResultList)));
}
catch (Exception ex)
{
throw new Exception($"Unable to run {assemblyInfo.AssemblyPath} with {Options.XunitPath}. {ex}");
}
}
private async Task <TestResult> RunTestAsyncInternal(AssemblyInfo assemblyInfo, bool retry, CancellationToken cancellationToken)
{
try
{
var commandLineArguments = GetCommandLineArguments(assemblyInfo, useSingleQuotes: false);
var resultsFilePath = GetResultsFilePath(assemblyInfo);
var resultsDir = Path.GetDirectoryName(resultsFilePath);
var htmlResultsFilePath = Options.IncludeHtml ? GetResultsFilePath(assemblyInfo, "html") : null;
var processResultList = new List <ProcessResult>();
ProcessInfo?procDumpProcessInfo = null;
// NOTE: xUnit doesn't always create the log directory
Directory.CreateDirectory(resultsDir !);
// Define environment variables for processes started via ProcessRunner.
var environmentVariables = new Dictionary <string, string>();
Options.ProcDumpInfo?.WriteEnvironmentVariables(environmentVariables);
if (retry && File.Exists(resultsFilePath))
{
ConsoleUtil.WriteLine("Starting a retry. Tests which failed will run a second time to reduce flakiness.");
try
{
var doc = XDocument.Load(resultsFilePath);
foreach (var test in doc.XPathSelectElements("/assemblies/assembly/collection/test[@result='Fail']"))
{
ConsoleUtil.WriteLine($" {test.Attribute("name")!.Value}: {test.Attribute("result")!.Value}");
}
}
catch
{
ConsoleUtil.WriteLine(" ...Failed to identify the list of specific failures.");
}
// Copy the results file path, since the new xunit run will overwrite it
var backupResultsFilePath = Path.ChangeExtension(resultsFilePath, ".old");
File.Copy(resultsFilePath, backupResultsFilePath, overwrite: true);
// If running the process with this varialbe added, we assume that this file contains
// xml logs from the first attempt.
environmentVariables.Add("OutputXmlFilePath", backupResultsFilePath);
}
// NOTE: xUnit seems to have an occasional issue creating logs create
// an empty log just in case, so our runner will still fail.
File.Create(resultsFilePath).Close();
var start = DateTime.UtcNow;
var dotnetProcessInfo = ProcessRunner.CreateProcess(
ProcessRunner.CreateProcessStartInfo(
Options.DotnetFilePath,
commandLineArguments,
workingDirectory: Path.GetDirectoryName(assemblyInfo.AssemblyPath),
displayWindow: false,
captureOutput: true,
environmentVariables: environmentVariables),
lowPriority: false,
cancellationToken: cancellationToken);
Logger.Log($"Create xunit process with id {dotnetProcessInfo.Id} for test {assemblyInfo.DisplayName}");
var xunitProcessResult = await dotnetProcessInfo.Result;
var span = DateTime.UtcNow - start;
Logger.Log($"Exit xunit process with id {dotnetProcessInfo.Id} for test {assemblyInfo.DisplayName} with code {xunitProcessResult.ExitCode}");
processResultList.Add(xunitProcessResult);
if (procDumpProcessInfo != null)
{
var procDumpProcessResult = await procDumpProcessInfo.Value.Result;
Logger.Log($"Exit procdump process with id {procDumpProcessInfo.Value.Id} for {dotnetProcessInfo.Id} for test {assemblyInfo.DisplayName} with code {procDumpProcessResult.ExitCode}");
processResultList.Add(procDumpProcessResult);
}
if (xunitProcessResult.ExitCode != 0)
{
// On occasion we get a non-0 output but no actual data in the result file. The could happen
// if xunit manages to crash when running a unit test (a stack overflow could cause this, for instance).
// To avoid losing information, write the process output to the console. In addition, delete the results
// file to avoid issues with any tool attempting to interpret the (potentially malformed) text.
var resultData = string.Empty;
try
{
resultData = File.ReadAllText(resultsFilePath).Trim();
}
catch
{
// Happens if xunit didn't produce a log file
}
if (resultData.Length == 0)
{
// Delete the output file.
File.Delete(resultsFilePath);
resultsFilePath = null;
htmlResultsFilePath = null;
}
}
Logger.Log($"Command line {assemblyInfo.DisplayName}: {Options.DotnetFilePath} {commandLineArguments}");
var standardOutput = string.Join(Environment.NewLine, xunitProcessResult.OutputLines) ?? "";
var errorOutput = string.Join(Environment.NewLine, xunitProcessResult.ErrorLines) ?? "";
var testResultInfo = new TestResultInfo(
exitCode: xunitProcessResult.ExitCode,
resultsFilePath: resultsFilePath,
htmlResultsFilePath: htmlResultsFilePath,
elapsed: span,
standardOutput: standardOutput,
errorOutput: errorOutput);
return(new TestResult(
assemblyInfo,
testResultInfo,
commandLineArguments,
processResults: ImmutableArray.CreateRange(processResultList)));
}
catch (Exception ex)
{
throw new Exception($"Unable to run {assemblyInfo.AssemblyPath} with {Options.DotnetFilePath}. {ex}");
}
}