Esempio n. 1
0
        public async Task <TestResult> RunTestAsync(string assemblyPath, CancellationToken cancellationToken)
        {
            try
            {
                var commandLineArguments = GetCommandLineArguments(assemblyPath);
                var resultsFilePath      = GetResultsFilePath(assemblyPath);
                var resultsDir           = Path.GetDirectoryName(resultsFilePath);

                // NOTE: xUnit doesn't always create the log directory
                Directory.CreateDirectory(resultsDir);

                // NOTE: xUnit seems to have an occasional issue creating logs create
                // an empty log just in case, so our runner will still fail.
                File.Create(resultsFilePath).Close();

                var start         = DateTime.UtcNow;
                var xunitPath     = _options.XunitPath;
                var processOutput = await ProcessRunner.RunProcessAsync(
                    xunitPath,
                    commandLineArguments,
                    lowPriority : false,
                    displayWindow : false,
                    captureOutput : true,
                    cancellationToken : cancellationToken).ConfigureAwait(false);

                var span = DateTime.UtcNow - start;

                if (processOutput.ExitCode != 0)
                {
                    // On occasion we get a non-0 output but no actual data in the result file.  The could happen
                    // if xunit manages to crash when running a unit test (a stack overflow could cause this, for instance).
                    // To avoid losing information, write the process output to the console.  In addition, delete the results
                    // file to avoid issues with any tool attempting to interpret the (potentially malformed) text.
                    var resultData = string.Empty;
                    try
                    {
                        resultData = File.ReadAllText(resultsFilePath).Trim();
                    }
                    catch
                    {
                        // Happens if xunit didn't produce a log file
                    }

                    if (resultData.Length == 0)
                    {
                        // Delete the output file.
                        File.Delete(resultsFilePath);
                        resultsFilePath = null;
                    }
                }

                var commandLine    = GetCommandLine(assemblyPath);
                var standardOutput = string.Join(Environment.NewLine, processOutput.OutputLines);
                var errorOutput    = string.Join(Environment.NewLine, processOutput.ErrorLines);

                return(new TestResult(
                           exitCode: processOutput.ExitCode,
                           assemblyPath: assemblyPath,
                           resultDir: resultsDir,
                           resultsFilePath: resultsFilePath,
                           commandLine: commandLine,
                           elapsed: span,
                           standardOutput: standardOutput,
                           errorOutput: errorOutput));
            }
            catch (Exception ex)
            {
                throw new Exception($"Unable to run {assemblyPath} with {_options.XunitPath}. {ex}");
            }
        }
Esempio n. 2
0
        private async Task <TestResult> RunTestAsyncInternal(AssemblyInfo assemblyInfo, bool retry, CancellationToken cancellationToken)
        {
            try
            {
                var         commandLineArguments = GetCommandLineArguments(assemblyInfo);
                var         resultsFilePath      = GetResultsFilePath(assemblyInfo);
                var         resultsDir           = Path.GetDirectoryName(resultsFilePath);
                var         processResultList    = new List <ProcessResult>();
                ProcessInfo?procDumpProcessInfo  = null;

                // NOTE: xUnit doesn't always create the log directory
                Directory.CreateDirectory(resultsDir);

                // Define environment variables for processes started via ProcessRunner.
                var environmentVariables = new Dictionary <string, string>();
                Options.ProcDumpInfo?.WriteEnvironmentVariables(environmentVariables);

                if (retry && File.Exists(resultsFilePath))
                {
                    // Copy the results file path, since the new xunit run will overwrite it
                    var backupResultsFilePath = Path.ChangeExtension(resultsFilePath, ".old");
                    File.Copy(resultsFilePath, backupResultsFilePath, overwrite: true);

                    ConsoleUtil.WriteLine("Starting a retry. It will run once again tests failed.");
                    // If running the process with this varialbe added, we assume that this file contains
                    // xml logs from the first attempt.
                    environmentVariables.Add("OutputXmlFilePath", backupResultsFilePath);
                }

                // NOTE: xUnit seems to have an occasional issue creating logs create
                // an empty log just in case, so our runner will still fail.
                File.Create(resultsFilePath).Close();

                var start            = DateTime.UtcNow;
                var xunitProcessInfo = ProcessRunner.CreateProcess(
                    ProcessRunner.CreateProcessStartInfo(
                        Options.XunitPath,
                        commandLineArguments,
                        displayWindow: false,
                        captureOutput: true,
                        environmentVariables: environmentVariables),
                    lowPriority: false,
                    cancellationToken: cancellationToken);
                Logger.Log($"Create xunit process with id {xunitProcessInfo.Id} for test {assemblyInfo.DisplayName}");

                // Now that xunit is running we should kick off a procDump process if it was specified
                if (Options.ProcDumpInfo != null)
                {
                    var procDumpInfo      = Options.ProcDumpInfo.Value;
                    var procDumpStartInfo = ProcessRunner.CreateProcessStartInfo(
                        procDumpInfo.ProcDumpFilePath,
                        ProcDumpUtil.GetProcDumpCommandLine(xunitProcessInfo.Id, procDumpInfo.DumpDirectory),
                        captureOutput: true,
                        displayWindow: false);
                    Directory.CreateDirectory(procDumpInfo.DumpDirectory);
                    procDumpProcessInfo = ProcessRunner.CreateProcess(procDumpStartInfo, cancellationToken: cancellationToken);
                    Logger.Log($"Create procdump process with id {procDumpProcessInfo.Value.Id} for xunit {xunitProcessInfo.Id} for test {assemblyInfo.DisplayName}");
                }

                var xunitProcessResult = await xunitProcessInfo.Result;
                var span = DateTime.UtcNow - start;

                Logger.Log($"Exit xunit process with id {xunitProcessInfo.Id} for test {assemblyInfo.DisplayName} with code {xunitProcessResult.ExitCode}");
                processResultList.Add(xunitProcessResult);
                if (procDumpProcessInfo != null)
                {
                    var procDumpProcessResult = await procDumpProcessInfo.Value.Result;
                    Logger.Log($"Exit procdump process with id {procDumpProcessInfo.Value.Id} for {xunitProcessInfo.Id} for test {assemblyInfo.DisplayName} with code {procDumpProcessResult.ExitCode}");
                    processResultList.Add(procDumpProcessResult);
                }

                if (xunitProcessResult.ExitCode != 0)
                {
                    // On occasion we get a non-0 output but no actual data in the result file.  The could happen
                    // if xunit manages to crash when running a unit test (a stack overflow could cause this, for instance).
                    // To avoid losing information, write the process output to the console.  In addition, delete the results
                    // file to avoid issues with any tool attempting to interpret the (potentially malformed) text.
                    var resultData = string.Empty;
                    try
                    {
                        resultData = File.ReadAllText(resultsFilePath).Trim();
                    }
                    catch
                    {
                        // Happens if xunit didn't produce a log file
                    }

                    if (resultData.Length == 0)
                    {
                        // Delete the output file.
                        File.Delete(resultsFilePath);
                        resultsFilePath = null;
                    }
                }

                var commandLine = GetCommandLine(assemblyInfo);
                Logger.Log($"Command line {assemblyInfo.DisplayName}: {commandLine}");
                var standardOutput = string.Join(Environment.NewLine, xunitProcessResult.OutputLines) ?? "";
                var errorOutput    = string.Join(Environment.NewLine, xunitProcessResult.ErrorLines) ?? "";
                var testResultInfo = new TestResultInfo(
                    exitCode: xunitProcessResult.ExitCode,
                    resultsFilePath: resultsFilePath,
                    elapsed: span,
                    standardOutput: standardOutput,
                    errorOutput: errorOutput);

                return(new TestResult(
                           assemblyInfo,
                           testResultInfo,
                           commandLine,
                           isFromCache: false,
                           processResults: ImmutableArray.CreateRange(processResultList)));
            }
            catch (Exception ex)
            {
                throw new Exception($"Unable to run {assemblyInfo.AssemblyPath} with {Options.XunitPath}. {ex}");
            }
        }
Esempio n. 3
0
        /// <summary>
        /// Invoked when a timeout occurs and we need to dump all of the test processes and shut down
        /// the runnner.
        /// </summary>
        private static async Task HandleTimeout(Options options, CancellationToken cancellationToken)
        {
            async Task DumpProcess(Process targetProcess, string procDumpExeFilePath, string dumpFilePath)
            {
                var name = targetProcess.ProcessName;

                // Our space for saving dump files is limited. Skip dumping for processes that won't contribute
                // to bug investigations.
                if (name == "procdump" || name == "conhost")
                {
                    return;
                }

                ConsoleUtil.Write($"Dumping {name} {targetProcess.Id} to {dumpFilePath} ... ");
                try
                {
                    var args          = $"-accepteula -ma {targetProcess.Id} {dumpFilePath}";
                    var processInfo   = ProcessRunner.CreateProcess(procDumpExeFilePath, args, cancellationToken: cancellationToken);
                    var processOutput = await processInfo.Result;

                    // The exit code for procdump doesn't obey standard windows rules.  It will return non-zero
                    // for successful cases (possibly returning the count of dumps that were written).  Best
                    // backup is to test for the dump file being present.
                    if (File.Exists(dumpFilePath))
                    {
                        ConsoleUtil.WriteLine($"succeeded ({new FileInfo(dumpFilePath).Length} bytes)");
                    }
                    else
                    {
                        ConsoleUtil.WriteLine($"FAILED with {processOutput.ExitCode}");
                        ConsoleUtil.WriteLine($"{procDumpExeFilePath} {args}");
                        ConsoleUtil.WriteLine(string.Join(Environment.NewLine, processOutput.OutputLines));
                    }
                }
                catch (Exception ex) when(!cancellationToken.IsCancellationRequested)
                {
                    ConsoleUtil.WriteLine("FAILED");
                    ConsoleUtil.WriteLine(ex.Message);
                    Logger.Log("Failed to dump process", ex);
                }
            }

            ConsoleUtil.WriteLine("Roslyn Error: test timeout exceeded, dumping remaining processes");
            var procDumpInfo = GetProcDumpInfo(options);

            if (procDumpInfo != null)
            {
                var counter = 0;
                foreach (var proc in ProcessUtil.GetProcessTree(Process.GetCurrentProcess()).OrderBy(x => x.ProcessName))
                {
                    var dumpDir = PrimaryProcessNames.Contains(proc.ProcessName)
                        ? procDumpInfo.Value.DumpDirectory
                        : procDumpInfo.Value.SecondaryDumpDirectory;
                    var dumpFilePath = Path.Combine(dumpDir, $"{proc.ProcessName}-{counter}.dmp");
                    await DumpProcess(proc, procDumpInfo.Value.ProcDumpFilePath, dumpFilePath);

                    counter++;
                }
            }
            else
            {
                ConsoleUtil.WriteLine("Could not locate procdump");
            }

            WriteLogFile(options);
        }
Esempio n. 4
0
        internal async Task <RunAllResult> RunAllOnHelixAsync(IEnumerable <AssemblyInfo> assemblyInfoList, CancellationToken cancellationToken)
        {
            var sourceBranch = Environment.GetEnvironmentVariable("BUILD_SOURCEBRANCH");

            if (sourceBranch is null)
            {
                sourceBranch = "local";
                ConsoleUtil.WriteLine($@"BUILD_SOURCEBRANCH environment variable was not set. Using source branch ""{sourceBranch}"" instead");
                Environment.SetEnvironmentVariable("BUILD_SOURCEBRANCH", sourceBranch);
            }

            var msbuildTestPayloadRoot = Path.GetDirectoryName(_options.ArtifactsDirectory);

            if (msbuildTestPayloadRoot is null)
            {
                throw new IOException($@"Malformed ArtifactsDirectory in options: ""{_options.ArtifactsDirectory}""");
            }

            var isAzureDevOpsRun = Environment.GetEnvironmentVariable("SYSTEM_ACCESSTOKEN") is not null;

            if (!isAzureDevOpsRun)
            {
                ConsoleUtil.WriteLine("SYSTEM_ACCESSTOKEN environment variable was not set, so test results will not be published.");
                // in a local run we assume the user runs using the root test.sh and that the test payload is nested in the artifacts directory.
                msbuildTestPayloadRoot = Path.Combine(msbuildTestPayloadRoot, "artifacts/testPayload");
            }
            var duplicateDir       = Path.Combine(msbuildTestPayloadRoot, ".duplicate");
            var correlationPayload = $@"<HelixCorrelationPayload Include=""{duplicateDir}"" />";

            // https://github.com/dotnet/roslyn/issues/50661
            // it's possible we should be using the BUILD_SOURCEVERSIONAUTHOR instead here a la https://github.com/dotnet/arcade/blob/master/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/Readme.md#how-to-use
            // however that variable isn't documented at https://docs.microsoft.com/en-us/azure/devops/pipelines/build/variables?view=azure-devops&tabs=yaml
            var queuedBy = Environment.GetEnvironmentVariable("BUILD_QUEUEDBY");

            if (queuedBy is null)
            {
                queuedBy = "roslyn";
                ConsoleUtil.WriteLine($@"BUILD_QUEUEDBY environment variable was not set. Using value ""{queuedBy}"" instead");
            }

            var jobName = Environment.GetEnvironmentVariable("SYSTEM_JOBDISPLAYNAME");

            if (jobName is null)
            {
                ConsoleUtil.WriteLine($"SYSTEM_JOBDISPLAYNAME environment variable was not set. Using a blank TestRunNamePrefix for Helix job.");
            }

            if (Environment.GetEnvironmentVariable("BUILD_REPOSITORY_NAME") is null)
            {
                Environment.SetEnvironmentVariable("BUILD_REPOSITORY_NAME", "dotnet/roslyn");
            }

            if (Environment.GetEnvironmentVariable("SYSTEM_TEAMPROJECT") is null)
            {
                Environment.SetEnvironmentVariable("SYSTEM_TEAMPROJECT", "dnceng");
            }

            if (Environment.GetEnvironmentVariable("BUILD_REASON") is null)
            {
                Environment.SetEnvironmentVariable("BUILD_REASON", "pr");
            }

            var buildNumber = Environment.GetEnvironmentVariable("BUILD_BUILDNUMBER") ?? "0";
            var workItems   = assemblyInfoList.Select(ai => makeHelixWorkItemProject(ai));

            var globalJson = JsonConvert.DeserializeAnonymousType(File.ReadAllText(getGlobalJsonPath()), new { sdk = new { version = "" } });
            var project    = @"
<Project Sdk=""Microsoft.DotNet.Helix.Sdk"" DefaultTargets=""Test"">
    <PropertyGroup>
        <TestRunNamePrefix>" + jobName + @"_</TestRunNamePrefix>
        <HelixSource>pr/" + sourceBranch + @"</HelixSource>
        <HelixType>test</HelixType>
        <HelixBuild>" + buildNumber + @"</HelixBuild>
        <HelixTargetQueues>" + _options.HelixQueueName + @"</HelixTargetQueues>
        <Creator>" + queuedBy + @"</Creator>
        <IncludeDotNetCli>true</IncludeDotNetCli>
        <DotNetCliVersion>" + globalJson.sdk.version + @"</DotNetCliVersion>
        <DotNetCliPackageType>sdk</DotNetCliPackageType>
        <EnableAzurePipelinesReporter>" + (isAzureDevOpsRun ? "true" : "false") + @"</EnableAzurePipelinesReporter>
    </PropertyGroup>

    <ItemGroup>
        " + correlationPayload + string.Join("", workItems) + @"
    </ItemGroup>
</Project>
";

            File.WriteAllText("helix-tmp.csproj", project);
            var process = ProcessRunner.CreateProcess(
                executable: _options.DotnetFilePath,
                arguments: "build helix-tmp.csproj",
                captureOutput: true,
                onOutputDataReceived: (e) => ConsoleUtil.WriteLine(e.Data),
                cancellationToken: cancellationToken);
            var result = await process.Result;

            return(new RunAllResult(result.ExitCode == 0, ImmutableArray <TestResult> .Empty, ImmutableArray.Create(result)));
Esempio n. 5
0
        public async Task <TestResult> RunTestAsync(AssemblyInfo assemblyInfo, CancellationToken cancellationToken)
        {
            try
            {
                var commandLineArguments = GetCommandLineArguments(assemblyInfo);
                var resultsFilePath      = GetResultsFilePath(assemblyInfo);
                var resultsDir           = Path.GetDirectoryName(resultsFilePath);

                // NOTE: xUnit doesn't always create the log directory
                Directory.CreateDirectory(resultsDir);

                // NOTE: xUnit seems to have an occasional issue creating logs create
                // an empty log just in case, so our runner will still fail.
                File.Create(resultsFilePath).Close();

                // Define environment variables for processes started via ProcessRunner.
                var environmentVariables = new Dictionary <string, string>();
                _options.ProcDumpInfo?.WriteEnvironmentVariables(environmentVariables);

                // Attach procDump to processes when the are started so we can watch for
                // unexepected crashes.
                void onProcessStart(Process process)
                {
                    if (_options.ProcDumpInfo != null)
                    {
                        ProcDumpUtil.AttachProcDump(_options.ProcDumpInfo.Value, process.Id);
                    }
                }

                var start         = DateTime.UtcNow;
                var xunitPath     = _options.XunitPath;
                var processOutput = await ProcessRunner.RunProcessAsync(
                    xunitPath,
                    commandLineArguments,
                    lowPriority : false,
                    displayWindow : false,
                    captureOutput : true,
                    cancellationToken : cancellationToken,
                    environmentVariables : environmentVariables,
                    onProcessStartHandler : onProcessStart);

                var span = DateTime.UtcNow - start;

                if (processOutput.ExitCode != 0)
                {
                    // On occasion we get a non-0 output but no actual data in the result file.  The could happen
                    // if xunit manages to crash when running a unit test (a stack overflow could cause this, for instance).
                    // To avoid losing information, write the process output to the console.  In addition, delete the results
                    // file to avoid issues with any tool attempting to interpret the (potentially malformed) text.
                    var resultData = string.Empty;
                    try
                    {
                        resultData = File.ReadAllText(resultsFilePath).Trim();
                    }
                    catch
                    {
                        // Happens if xunit didn't produce a log file
                    }

                    if (resultData.Length == 0)
                    {
                        // Delete the output file.
                        File.Delete(resultsFilePath);
                        resultsFilePath = null;
                    }
                }

                var commandLine = GetCommandLine(assemblyInfo);
                Logger.Log($"Command line {assemblyInfo.DisplayName}: {commandLine}");
                var standardOutput = string.Join(Environment.NewLine, processOutput.OutputLines) ?? "";
                var errorOutput    = string.Join(Environment.NewLine, processOutput.ErrorLines) ?? "";
                var testResultInfo = new TestResultInfo(
                    exitCode: processOutput.ExitCode,
                    resultsDirectory: resultsDir,
                    resultsFilePath: resultsFilePath,
                    elapsed: span,
                    standardOutput: standardOutput,
                    errorOutput: errorOutput);

                return(new TestResult(
                           assemblyInfo,
                           testResultInfo,
                           commandLine,
                           isFromCache: false));
            }
            catch (Exception ex)
            {
                throw new Exception($"Unable to run {assemblyInfo.AssemblyPath} with {_options.XunitPath}. {ex}");
            }
        }
Esempio n. 6
0
        private async Task <TestResult> RunTestAsyncInternal(AssemblyInfo assemblyInfo, bool retry, CancellationToken cancellationToken)
        {
            try
            {
                var         commandLineArguments = GetCommandLineArguments(assemblyInfo, useSingleQuotes: false);
                var         resultsFilePath      = GetResultsFilePath(assemblyInfo);
                var         resultsDir           = Path.GetDirectoryName(resultsFilePath);
                var         htmlResultsFilePath  = Options.IncludeHtml ? GetResultsFilePath(assemblyInfo, "html") : null;
                var         processResultList    = new List <ProcessResult>();
                ProcessInfo?procDumpProcessInfo  = null;

                // NOTE: xUnit doesn't always create the log directory
                Directory.CreateDirectory(resultsDir !);

                // Define environment variables for processes started via ProcessRunner.
                var environmentVariables = new Dictionary <string, string>();
                Options.ProcDumpInfo?.WriteEnvironmentVariables(environmentVariables);

                if (retry && File.Exists(resultsFilePath))
                {
                    ConsoleUtil.WriteLine("Starting a retry. Tests which failed will run a second time to reduce flakiness.");
                    try
                    {
                        var doc = XDocument.Load(resultsFilePath);
                        foreach (var test in doc.XPathSelectElements("/assemblies/assembly/collection/test[@result='Fail']"))
                        {
                            ConsoleUtil.WriteLine($"  {test.Attribute("name")!.Value}: {test.Attribute("result")!.Value}");
                        }
                    }
                    catch
                    {
                        ConsoleUtil.WriteLine("  ...Failed to identify the list of specific failures.");
                    }

                    // Copy the results file path, since the new xunit run will overwrite it
                    var backupResultsFilePath = Path.ChangeExtension(resultsFilePath, ".old");
                    File.Copy(resultsFilePath, backupResultsFilePath, overwrite: true);

                    // If running the process with this varialbe added, we assume that this file contains
                    // xml logs from the first attempt.
                    environmentVariables.Add("OutputXmlFilePath", backupResultsFilePath);
                }

                // NOTE: xUnit seems to have an occasional issue creating logs create
                // an empty log just in case, so our runner will still fail.
                File.Create(resultsFilePath).Close();

                var start             = DateTime.UtcNow;
                var dotnetProcessInfo = ProcessRunner.CreateProcess(
                    ProcessRunner.CreateProcessStartInfo(
                        Options.DotnetFilePath,
                        commandLineArguments,
                        workingDirectory: Path.GetDirectoryName(assemblyInfo.AssemblyPath),
                        displayWindow: false,
                        captureOutput: true,
                        environmentVariables: environmentVariables),
                    lowPriority: false,
                    cancellationToken: cancellationToken);
                Logger.Log($"Create xunit process with id {dotnetProcessInfo.Id} for test {assemblyInfo.DisplayName}");

                var xunitProcessResult = await dotnetProcessInfo.Result;
                var span = DateTime.UtcNow - start;

                Logger.Log($"Exit xunit process with id {dotnetProcessInfo.Id} for test {assemblyInfo.DisplayName} with code {xunitProcessResult.ExitCode}");
                processResultList.Add(xunitProcessResult);
                if (procDumpProcessInfo != null)
                {
                    var procDumpProcessResult = await procDumpProcessInfo.Value.Result;
                    Logger.Log($"Exit procdump process with id {procDumpProcessInfo.Value.Id} for {dotnetProcessInfo.Id} for test {assemblyInfo.DisplayName} with code {procDumpProcessResult.ExitCode}");
                    processResultList.Add(procDumpProcessResult);
                }

                if (xunitProcessResult.ExitCode != 0)
                {
                    // On occasion we get a non-0 output but no actual data in the result file.  The could happen
                    // if xunit manages to crash when running a unit test (a stack overflow could cause this, for instance).
                    // To avoid losing information, write the process output to the console.  In addition, delete the results
                    // file to avoid issues with any tool attempting to interpret the (potentially malformed) text.
                    var resultData = string.Empty;
                    try
                    {
                        resultData = File.ReadAllText(resultsFilePath).Trim();
                    }
                    catch
                    {
                        // Happens if xunit didn't produce a log file
                    }

                    if (resultData.Length == 0)
                    {
                        // Delete the output file.
                        File.Delete(resultsFilePath);
                        resultsFilePath     = null;
                        htmlResultsFilePath = null;
                    }
                }

                Logger.Log($"Command line {assemblyInfo.DisplayName}: {Options.DotnetFilePath} {commandLineArguments}");
                var standardOutput = string.Join(Environment.NewLine, xunitProcessResult.OutputLines) ?? "";
                var errorOutput    = string.Join(Environment.NewLine, xunitProcessResult.ErrorLines) ?? "";

                var testResultInfo = new TestResultInfo(
                    exitCode: xunitProcessResult.ExitCode,
                    resultsFilePath: resultsFilePath,
                    htmlResultsFilePath: htmlResultsFilePath,
                    elapsed: span,
                    standardOutput: standardOutput,
                    errorOutput: errorOutput);

                return(new TestResult(
                           assemblyInfo,
                           testResultInfo,
                           commandLineArguments,
                           processResults: ImmutableArray.CreateRange(processResultList)));
            }
            catch (Exception ex)
            {
                throw new Exception($"Unable to run {assemblyInfo.AssemblyPath} with {Options.DotnetFilePath}. {ex}");
            }
        }
Esempio n. 7
0
        public async Task <TestResult> RunTestAsync(string assemblyPath, CancellationToken cancellationToken)
        {
            try
            {
                var assemblyName    = Path.GetFileName(assemblyPath);
                var resultsDir      = Path.Combine(Path.GetDirectoryName(assemblyPath), Constants.ResultsDirectoryName);
                var resultsFilePath = Path.Combine(resultsDir, $"{assemblyName}.{(_options.UseHtml ? "html" : "xml")}");

                // NOTE: xUnit doesn't always create the log directory
                Directory.CreateDirectory(resultsDir);

                // NOTE: xUnit seems to have an occasional issue creating logs create
                // an empty log just in case, so our runner will still fail.
                File.Create(resultsFilePath).Close();

                var builder = new StringBuilder();
                builder.AppendFormat(@"""{0}""", assemblyPath);
                builder.AppendFormat(@" -{0} ""{1}""", _options.UseHtml ? "html" : "xml", resultsFilePath);
                builder.Append(" -noshadow -verbose");

                if (!string.IsNullOrWhiteSpace(_options.Trait))
                {
                    var traits = _options.Trait.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
                    foreach (var trait in traits)
                    {
                        builder.AppendFormat(" -trait {0}", trait);
                    }
                }

                if (!string.IsNullOrWhiteSpace(_options.NoTrait))
                {
                    var traits = _options.NoTrait.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
                    foreach (var trait in traits)
                    {
                        builder.AppendFormat(" -notrait {0}", trait);
                    }
                }

                var start = DateTime.UtcNow;

                var xunitPath     = _options.XunitPath;
                var processOutput = await ProcessRunner.RunProcessAsync(
                    xunitPath,
                    builder.ToString(),
                    lowPriority : false,
                    displayWindow : false,
                    captureOutput : true,
                    cancellationToken : cancellationToken).ConfigureAwait(false);

                var span = DateTime.UtcNow - start;

                if (processOutput.ExitCode != 0)
                {
                    // On occasion we get a non-0 output but no actual data in the result file.  The could happen
                    // if xunit manages to crash when running a unit test (a stack overflow could cause this, for instance).
                    // To avoid losing information, write the process output to the console.  In addition, delete the results
                    // file to avoid issues with any tool attempting to interpret the (potentially malformed) text.
                    var resultData = string.Empty;
                    try
                    {
                        resultData = File.ReadAllText(resultsFilePath).Trim();
                    }
                    catch
                    {
                        // Happens if xunit didn't produce a log file
                    }

                    if (resultData.Length == 0)
                    {
                        // Delete the output file.
                        File.Delete(resultsFilePath);
                        resultsFilePath = null;
                    }
                }

                var commandLine    = $"{xunitPath} {builder.ToString()}";
                var standardOutput = string.Join(Environment.NewLine, processOutput.OutputLines);
                var errorOutput    = string.Join(Environment.NewLine, processOutput.ErrorLines);

                return(new TestResult(
                           exitCode: processOutput.ExitCode,
                           assemblyPath: assemblyPath,
                           resultDir: resultsDir,
                           resultsFilePath: resultsFilePath,
                           commandLine: commandLine,
                           elapsed: span,
                           standardOutput: standardOutput,
                           errorOutput: errorOutput));
            }
            catch (Exception ex)
            {
                throw new Exception($"Unable to run {assemblyPath} with {_options.XunitPath}. {ex}");
            }
        }
Esempio n. 8
0
        private async Task <TestResult> RunTest(string assemblyPath, CancellationToken cancellationToken)
        {
            try
            {
                var assemblyName  = Path.GetFileName(assemblyPath);
                var resultsFile   = Path.Combine(Path.GetDirectoryName(assemblyPath), "xUnitResults", $"{assemblyName}.{(_options.UseHtml ? "html" : "xml")}");
                var resultsDir    = Path.GetDirectoryName(resultsFile);
                var outputLogPath = Path.Combine(resultsDir, $"{assemblyName}.out.log");

                // NOTE: xUnit doesn't always create the log directory
                Directory.CreateDirectory(resultsDir);

                // NOTE: xUnit seems to have an occasional issue creating logs create
                // an empty log just in case, so our runner will still fail.
                File.Create(resultsFile).Close();

                var builder = new StringBuilder();
                builder.AppendFormat(@"""{0}""", assemblyPath);
                builder.AppendFormat(@" -{0} ""{1}""", _options.UseHtml ? "html" : "xml", resultsFile);
                builder.Append(" -noshadow -verbose");

                if (!string.IsNullOrWhiteSpace(_options.Trait))
                {
                    var traits = _options.Trait.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
                    foreach (var trait in traits)
                    {
                        builder.AppendFormat(" -trait {0}", trait);
                    }
                }

                if (!string.IsNullOrWhiteSpace(_options.NoTrait))
                {
                    var traits = _options.NoTrait.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
                    foreach (var trait in traits)
                    {
                        builder.AppendFormat(" -notrait {0}", trait);
                    }
                }

                var errorOutput = new StringBuilder();
                var start       = DateTime.UtcNow;

                var xunitPath     = _options.XunitPath;
                var processOutput = await ProcessRunner.RunProcessAsync(
                    xunitPath,
                    builder.ToString(),
                    lowPriority : false,
                    displayWindow : false,
                    captureOutput : true,
                    cancellationToken : cancellationToken).ConfigureAwait(false);

                var span = DateTime.UtcNow - start;

                if (processOutput.ExitCode != 0)
                {
                    File.WriteAllLines(outputLogPath, processOutput.OutputLines);

                    // On occasion we get a non-0 output but no actual data in the result file.  The could happen
                    // if xunit manages to crash when running a unit test (a stack overflow could cause this, for instance).
                    // To avoid losing information, write the process output to the console.  In addition, delete the results
                    // file to avoid issues with any tool attempting to interpret the (potentially malformed) text.
                    var resultData = string.Empty;
                    try
                    {
                        resultData = File.ReadAllText(resultsFile).Trim();
                    }
                    catch
                    {
                        // Happens if xunit didn't produce a log file
                    }

                    if (resultData.Length == 0)
                    {
                        // Delete the output file.
                        File.Delete(resultsFile);
                    }

                    errorOutput.AppendLine($"Command: {_options.XunitPath} {builder}");
                    errorOutput.AppendLine($"xUnit output: {outputLogPath}");

                    if (processOutput.ErrorLines.Any())
                    {
                        foreach (var line in processOutput.ErrorLines)
                        {
                            errorOutput.AppendLine(line);
                        }
                    }
                    else
                    {
                        errorOutput.AppendLine($"xunit produced no error output but had exit code {processOutput.ExitCode}");
                    }

                    // If the results are html, use Process.Start to open in the browser.

                    if (_options.UseHtml && resultData.Length > 0)
                    {
                        Process.Start(resultsFile);
                    }
                }

                return(new TestResult(processOutput.ExitCode == 0, assemblyName, span, errorOutput.ToString()));
            }
            catch (Exception ex)
            {
                throw new Exception($"Unable to run {assemblyPath} with {_options.XunitPath}. {ex}");
            }
        }
Esempio n. 9
0
        private async Task <TestResult> RunTest(string assemblyPath, CancellationToken cancellationToken)
        {
            var assemblyName = Path.GetFileName(assemblyPath);
            var extension    = _useHtml ? ".TestResults.html" : ".TestResults.xml";
            var resultsPath  = Path.Combine(Path.GetDirectoryName(assemblyPath), Path.ChangeExtension(assemblyName, extension));

            DeleteFile(resultsPath);

            var builder = new StringBuilder();

            builder.AppendFormat(@"""{0}""", assemblyPath);
            builder.AppendFormat(@" -{0} ""{1}""", _useHtml ? "html" : "xml", resultsPath);
            builder.Append(" -noshadow");

            var errorOutput   = string.Empty;
            var start         = DateTime.UtcNow;
            var processOutput = await ProcessRunner.RunProcessAsync(
                _xunitConsolePath,
                builder.ToString(),
                lowPriority : false,
                displayWindow : false,
                captureOutput : true,
                cancellationToken : cancellationToken).ConfigureAwait(false);

            var span = DateTime.UtcNow - start;

            if (processOutput.ExitCode != 0)
            {
                // On occasion we get a non-0 output but no actual data in the result file.  The could happen
                // if xunit manages to crash when running a unit test (a stack overflow could cause this, for instance).
                // To avoid losing information, write the process output to the console.  In addition, delete the results
                // file to avoid issues with any tool attempting to interpret the (potentially malformed) text.
                var all = string.Empty;
                try
                {
                    all = File.ReadAllText(resultsPath).Trim();
                }
                catch
                {
                    // Happens if xunit didn't produce a log file
                }

                bool noResultsData = (all.Length == 0);
                if (noResultsData)
                {
                    var output = processOutput.OutputLines.Concat(processOutput.ErrorLines).ToArray();
                    Console.Write(output);

                    // Delete the output file.
                    File.Delete(resultsPath);
                }

                errorOutput = processOutput.ErrorLines.Any()
                    ? processOutput.ErrorLines.Aggregate((x, y) => x + Environment.NewLine + y)
                    : string.Format("xunit produced no error output but had exit code {0}", processOutput.ExitCode);

                errorOutput = string.Format("Command: {0} {1}", _xunitConsolePath, builder.ToString())
                              + Environment.NewLine
                              + errorOutput;

                // If the results are html, use Process.Start to open in the browser.

                if (_useHtml && !noResultsData)
                {
                    Process.Start(resultsPath);
                }
            }

            return(new TestResult(processOutput.ExitCode == 0, assemblyName, span, errorOutput));
        }
Esempio n. 10
0
        internal static async Task <int> Main(string[] args)
        {
            Logger.Log("RunTest command line");
            Logger.Log(string.Join(" ", args));
            var options = Options.Parse(args);

            if (options == null)
            {
                return(ExitFailure);
            }

            ConsoleUtil.WriteLine($"Running '{options.DotnetFilePath} --version'..");
            var dotnetResult = await ProcessRunner.CreateProcess(
                options.DotnetFilePath,
                arguments : "--version",
                captureOutput : true
                ).Result;

            ConsoleUtil.WriteLine(string.Join(Environment.NewLine, dotnetResult.OutputLines));
            ConsoleUtil.WriteLine(
                ConsoleColor.Red,
                string.Join(Environment.NewLine, dotnetResult.ErrorLines)
                );

            if (options.CollectDumps)
            {
                if (!DumpUtil.IsAdministrator())
                {
                    ConsoleUtil.WriteLine(
                        ConsoleColor.Yellow,
                        "Dump collection specified but user is not administrator so cannot modify registry"
                        );
                }
                else
                {
                    DumpUtil.EnableRegistryDumpCollection(options.LogFilesDirectory);
                }
            }

            try
            {
                // Setup cancellation for ctrl-c key presses
                using var cts           = new CancellationTokenSource();
                Console.CancelKeyPress += delegate
                {
                    cts.Cancel();
                    DisableRegistryDumpCollection();
                };

                int result;
                if (options.Timeout is { } timeout)
                {
                    result = await RunAsync(options, timeout, cts.Token);
                }
                else
                {
                    result = await RunAsync(options, cts.Token);
                }

                CheckTotalDumpFilesSize();
                return(result);
            }