public string GetCommandLineArguments(AssemblyInfo assemblyInfo)
        {
            var assemblyName = Path.GetFileName(assemblyInfo.AssemblyPath);
            var resultsFilePath = GetResultsFilePath(assemblyInfo);

            var builder = new StringBuilder();
            builder.AppendFormat(@"""{0}""", assemblyInfo.AssemblyPath);
            builder.AppendFormat(@" {0}", assemblyInfo.ExtraArguments);
            builder.AppendFormat(@" -{0} ""{1}""", _options.UseHtml ? "html" : "xml", resultsFilePath);
            builder.Append(" -noshadow -verbose");

            if (!string.IsNullOrWhiteSpace(_options.Trait))
            {
                var traits = _options.Trait.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
                foreach (var trait in traits)
                {
                    builder.AppendFormat(" -trait {0}", trait);
                }
            }

            if (!string.IsNullOrWhiteSpace(_options.NoTrait))
            {
                var traits = _options.NoTrait.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
                foreach (var trait in traits)
                {
                    builder.AppendFormat(" -notrait {0}", trait);
                }
            }

            return builder.ToString();
        }
Exemple #2
0
 internal TestResult(int exitCode, AssemblyInfo assemblyInfo, string resultDir, string resultsFilePath, string commandLine, TimeSpan elapsed, string standardOutput, string errorOutput, bool isResultFromCache)
 {
     ExitCode = exitCode;
     AssemblyInfo = assemblyInfo;
     CommandLine = commandLine;
     ResultDir = resultDir;
     ResultsFilePath = resultsFilePath;
     Elapsed = elapsed;
     StandardOutput = standardOutput;
     ErrorOutput = errorOutput;
     IsResultFromCache = isResultFromCache;
 }
Exemple #3
0
            private void CreateChunk()
            {
                var assemblyName = Path.GetFileName(_assemblyPath);
                var displayName = $"{assemblyName}.{_currentId}";
                var suffix = _useHtml ? "html" : "xml";
                var resultsFileName = $"{assemblyName}.{_currentId}.{suffix}";
                var assemblyInfo = new AssemblyInfo(
                    _assemblyPath,
                    displayName,
                    resultsFileName,
                    _builder.ToString());

                _chunkList.Add(new Chunk(_assemblyPath, _currentId, _currentTypeInfoList));
                _assemblyInfoList.Add(assemblyInfo);

                _currentId++;
                _currentTypeInfoList = new List<TypeInfo>();
                _builder.Length = 0;
            }
Exemple #4
0
            private void FinishPartition()
            {
                var assemblyName = Path.GetFileName(_assemblyPath);
                var displayName = $"{assemblyName}.{_currentId}";
                var suffix = _useHtml ? "html" : "xml";
                var resultsFileName = $"{assemblyName}.{_currentId}.{suffix}";
                var assemblyInfo = new AssemblyInfo(
                    _assemblyPath,
                    displayName,
                    resultsFileName,
                    _builder.ToString());

                _partitionList.Add(new Partition(_assemblyPath, _currentId, _currentTypeInfoList));
                _assemblyInfoList.Add(assemblyInfo);
            }
        public async Task<TestResult> RunTestAsync(AssemblyInfo assemblyInfo, CancellationToken cancellationToken)
        {
            try
            {
                var commandLineArguments = GetCommandLineArguments(assemblyInfo);
                var resultsFilePath = GetResultsFilePath(assemblyInfo);
                var resultsDir = Path.GetDirectoryName(resultsFilePath);

                // NOTE: xUnit doesn't always create the log directory
                Directory.CreateDirectory(resultsDir);

                // NOTE: xUnit seems to have an occasional issue creating logs create
                // an empty log just in case, so our runner will still fail.
                File.Create(resultsFilePath).Close();

                var dumpOutputFilePath = Path.Combine(resultsDir, $"{assemblyInfo.DisplayName}.dmp");

                var start = DateTime.UtcNow;
                var xunitPath = _options.XunitPath;
                var processOutput = await ProcessRunner.RunProcessAsync(
                    xunitPath,
                    commandLineArguments,
                    lowPriority: false,
                    displayWindow: false,
                    captureOutput: true,
                    cancellationToken: cancellationToken,
                    processMonitor: p => CrashDumps.TryMonitorProcess(p, dumpOutputFilePath)).ConfigureAwait(false);
                var span = DateTime.UtcNow - start;

                if (processOutput.ExitCode != 0)
                {
                    // On occasion we get a non-0 output but no actual data in the result file.  The could happen
                    // if xunit manages to crash when running a unit test (a stack overflow could cause this, for instance).
                    // To avoid losing information, write the process output to the console.  In addition, delete the results
                    // file to avoid issues with any tool attempting to interpret the (potentially malformed) text.
                    var resultData = string.Empty;
                    try
                    {
                        resultData = File.ReadAllText(resultsFilePath).Trim();
                    }
                    catch
                    {
                        // Happens if xunit didn't produce a log file
                    }

                    if (resultData.Length == 0)
                    {
                        // Delete the output file.
                        File.Delete(resultsFilePath);
                        resultsFilePath = null;
                    }
                }

                var commandLine = GetCommandLine(assemblyInfo);
                Logger.Log($"Command line {assemblyInfo.DisplayName}: {commandLine}");
                var standardOutput = string.Join(Environment.NewLine, processOutput.OutputLines);
                var errorOutput = string.Join(Environment.NewLine, processOutput.ErrorLines);

                return new TestResult(
                    exitCode: processOutput.ExitCode,
                    assemblyInfo: assemblyInfo,
                    resultDir: resultsDir,
                    resultsFilePath: resultsFilePath,
                    commandLine: commandLine,
                    elapsed: span,
                    standardOutput: standardOutput,
                    errorOutput: errorOutput,
                    isResultFromCache: false);
            }
            catch (Exception ex)
            {
                throw new Exception($"Unable to run {assemblyInfo.AssemblyPath} with {_options.XunitPath}. {ex}");
            }
        }
 private string GetResultsFilePath(AssemblyInfo assemblyInfo)
 {
     var resultsDir = Path.Combine(Path.GetDirectoryName(assemblyInfo.AssemblyPath), Constants.ResultsDirectoryName);
     return Path.Combine(resultsDir, assemblyInfo.ResultsFileName);
 }
 public string GetCommandLine(AssemblyInfo assemblyInfo)
 {
     return $"{_options.XunitPath} {GetCommandLineArguments(assemblyInfo)}";
 }
        public string GetCommandLineArguments(AssemblyInfo assemblyInfo, bool useSingleQuotes, bool isHelix)
        {
            // http://www.gnu.org/software/bash/manual/html_node/Single-Quotes.html
            // Single quotes are needed in bash to avoid the need to escape characters such as backtick (`) which are found in metadata names.
            // Batch scripts don't need to worry about escaping backticks, but they don't support single quoted strings, so we have to use double quotes.
            // We also need double quotes when building an arguments string for Process.Start in .NET Core so that splitting/unquoting works as expected.
            var sep = useSingleQuotes ? "'" : @"""";

            var builder = new StringBuilder();

            builder.Append($@"test");
            builder.Append($@" {sep}{assemblyInfo.AssemblyName}{sep}");
            var typeInfoList = assemblyInfo.PartitionInfo.TypeInfoList;

            if (typeInfoList.Length > 0 || !string.IsNullOrWhiteSpace(Options.TestFilter))
            {
                builder.Append($@" --filter {sep}");
                var any = false;
                foreach (var typeInfo in typeInfoList)
                {
                    MaybeAddSeparator();
                    // https://docs.microsoft.com/en-us/dotnet/core/testing/selective-unit-tests?pivots=mstest#syntax
                    // We want to avoid matching other test classes whose names are prefixed with this test class's name.
                    // For example, avoid running 'AttributeTests_WellKnownMember', when the request here is to run 'AttributeTests'.
                    // We append a '.', assuming that all test methods in the class *will* match it, but not methods in other classes.
                    builder.Append(typeInfo.FullName);
                    builder.Append('.');
                }
                builder.Append(sep);

                if (Options.TestFilter is object)
                {
                    MaybeAddSeparator();
                    builder.Append(Options.TestFilter);
                }

                void MaybeAddSeparator(char separator = '|')
                {
                    if (any)
                    {
                        builder.Append(separator);
                    }

                    any = true;
                }
            }

            builder.Append($@" --arch {assemblyInfo.Architecture}");
            builder.Append($@" --framework {assemblyInfo.TargetFramework}");
            builder.Append($@" --logger {sep}xunit;LogFilePath={GetResultsFilePath(assemblyInfo, "xml")}{sep}");

            if (Options.IncludeHtml)
            {
                builder.AppendFormat($@" --logger {sep}html;LogFileName={GetResultsFilePath(assemblyInfo, "html")}{sep}");
            }

            if (!Options.CollectDumps)
            {
                // The 'CollectDumps' option uses operating system features to collect dumps when a process crashes. We
                // only enable the test executor blame feature in remaining cases, as the latter relies on ProcDump and
                // interferes with automatic crash dump collection on Windows.
                builder.Append(" --blame-crash");
            }

            // The 25 minute timeout in integration tests accounts for the fact that VSIX deployment and/or experimental hive reset and
            // configuration can take significant time (seems to vary from ~10 seconds to ~15 minutes), and the blame
            // functionality cannot separate this configuration overhead from the first test which will eventually run.
            // https://github.com/dotnet/roslyn/issues/59851
            //
            // Helix timeout is 15 minutes as helix jobs fully timeout in 30minutes.  So in order to capture dumps we need the timeout
            // to be 2x shorter than the expected test run time (15min) in case only the last test hangs.
            var timeout = isHelix ? "15minutes" : "25minutes";

            builder.Append($" --blame-hang-dump-type full --blame-hang-timeout {timeout}");

            return(builder.ToString());
        }
Exemple #9
0
        private async Task <TestResult> RunTestAsyncInternal(AssemblyInfo assemblyInfo, bool retry, CancellationToken cancellationToken)
        {
            try
            {
                var         commandLineArguments = GetCommandLineArguments(assemblyInfo, useSingleQuotes: false);
                var         resultsFilePath      = GetResultsFilePath(assemblyInfo);
                var         resultsDir           = Path.GetDirectoryName(resultsFilePath);
                var         htmlResultsFilePath  = Options.IncludeHtml ? GetResultsFilePath(assemblyInfo, "html") : null;
                var         processResultList    = new List <ProcessResult>();
                ProcessInfo?procDumpProcessInfo  = null;

                // NOTE: xUnit doesn't always create the log directory
                Directory.CreateDirectory(resultsDir);

                // Define environment variables for processes started via ProcessRunner.
                var environmentVariables = new Dictionary <string, string>();
                Options.ProcDumpInfo?.WriteEnvironmentVariables(environmentVariables);

                if (retry && File.Exists(resultsFilePath))
                {
                    ConsoleUtil.WriteLine("Starting a retry. Tests which failed will run a second time to reduce flakiness.");
                    try
                    {
                        var doc = XDocument.Load(resultsFilePath);
                        foreach (var test in doc.XPathSelectElements("/assemblies/assembly/collection/test[@result='Fail']"))
                        {
                            ConsoleUtil.WriteLine($"  {test.Attribute("name").Value}: {test.Attribute("result").Value}");
                        }
                    }
                    catch
                    {
                        ConsoleUtil.WriteLine("  ...Failed to identify the list of specific failures.");
                    }

                    // Copy the results file path, since the new xunit run will overwrite it
                    var backupResultsFilePath = Path.ChangeExtension(resultsFilePath, ".old");
                    File.Copy(resultsFilePath, backupResultsFilePath, overwrite: true);

                    // If running the process with this varialbe added, we assume that this file contains
                    // xml logs from the first attempt.
                    environmentVariables.Add("OutputXmlFilePath", backupResultsFilePath);
                }

                // NOTE: xUnit seems to have an occasional issue creating logs create
                // an empty log just in case, so our runner will still fail.
                File.Create(resultsFilePath).Close();

                var start             = DateTime.UtcNow;
                var dotnetProcessInfo = ProcessRunner.CreateProcess(
                    ProcessRunner.CreateProcessStartInfo(
                        Options.DotnetFilePath,
                        commandLineArguments,
                        workingDirectory: Path.GetDirectoryName(assemblyInfo.AssemblyPath),
                        displayWindow: false,
                        captureOutput: true,
                        environmentVariables: environmentVariables),
                    lowPriority: false,
                    cancellationToken: cancellationToken);
                Logger.Log($"Create xunit process with id {dotnetProcessInfo.Id} for test {assemblyInfo.DisplayName}");

                var xunitProcessResult = await dotnetProcessInfo.Result;
                var span = DateTime.UtcNow - start;

                Logger.Log($"Exit xunit process with id {dotnetProcessInfo.Id} for test {assemblyInfo.DisplayName} with code {xunitProcessResult.ExitCode}");
                processResultList.Add(xunitProcessResult);
                if (procDumpProcessInfo != null)
                {
                    var procDumpProcessResult = await procDumpProcessInfo.Value.Result;
                    Logger.Log($"Exit procdump process with id {procDumpProcessInfo.Value.Id} for {dotnetProcessInfo.Id} for test {assemblyInfo.DisplayName} with code {procDumpProcessResult.ExitCode}");
                    processResultList.Add(procDumpProcessResult);
                }

                if (xunitProcessResult.ExitCode != 0)
                {
                    // On occasion we get a non-0 output but no actual data in the result file.  The could happen
                    // if xunit manages to crash when running a unit test (a stack overflow could cause this, for instance).
                    // To avoid losing information, write the process output to the console.  In addition, delete the results
                    // file to avoid issues with any tool attempting to interpret the (potentially malformed) text.
                    var resultData = string.Empty;
                    try
                    {
                        resultData = File.ReadAllText(resultsFilePath).Trim();
                    }
                    catch
                    {
                        // Happens if xunit didn't produce a log file
                    }

                    if (resultData.Length == 0)
                    {
                        // Delete the output file.
                        File.Delete(resultsFilePath);
                        resultsFilePath     = null;
                        htmlResultsFilePath = null;
                    }
                }

                Logger.Log($"Command line {assemblyInfo.DisplayName}: {Options.DotnetFilePath} {commandLineArguments}");
                var standardOutput = string.Join(Environment.NewLine, xunitProcessResult.OutputLines) ?? "";
                var errorOutput    = string.Join(Environment.NewLine, xunitProcessResult.ErrorLines) ?? "";

                var testResultInfo = new TestResultInfo(
                    exitCode: xunitProcessResult.ExitCode,
                    resultsFilePath: resultsFilePath,
                    htmlResultsFilePath: htmlResultsFilePath,
                    elapsed: span,
                    standardOutput: standardOutput,
                    errorOutput: errorOutput);

                return(new TestResult(
                           assemblyInfo,
                           testResultInfo,
                           commandLineArguments,
                           processResults: ImmutableArray.CreateRange(processResultList)));
            }
            catch (Exception ex)
            {
                throw new Exception($"Unable to run {assemblyInfo.AssemblyPath} with {Options.DotnetFilePath}. {ex}");
            }
        }
        private string GetResultsFilePath(AssemblyInfo assemblyInfo, string suffix = "xml")
        {
            var fileName = $"{assemblyInfo.DisplayName}_{assemblyInfo.TargetFramework}_{assemblyInfo.Architecture}_test_results.{suffix}";

            return(Path.Combine(Options.TestResultsDirectory, fileName));
        }
 static bool HasBuiltInRetry(AssemblyInfo assemblyInfo)
 {
     // vs-extension-testing handles test retry internally.
     return(assemblyInfo.AssemblyName == "Microsoft.VisualStudio.LanguageServices.New.IntegrationTests.dll");
 }
Exemple #12
0
        public async Task <TestResult> RunTestAsync(AssemblyInfo assemblyInfo, CancellationToken cancellationToken)
        {
            try
            {
                var commandLineArguments = GetCommandLineArguments(assemblyInfo);
                var resultsFilePath      = GetResultsFilePath(assemblyInfo);
                var resultsDir           = Path.GetDirectoryName(resultsFilePath);

                // NOTE: xUnit doesn't always create the log directory
                Directory.CreateDirectory(resultsDir);

                // NOTE: xUnit seems to have an occasional issue creating logs create
                // an empty log just in case, so our runner will still fail.
                File.Create(resultsFilePath).Close();

                var dumpOutputFilePath = Path.Combine(resultsDir, $"{assemblyInfo.DisplayName}.dmp");

                var start         = DateTime.UtcNow;
                var xunitPath     = _options.XunitPath;
                var processOutput = await ProcessRunner.RunProcessAsync(
                    xunitPath,
                    commandLineArguments,
                    lowPriority : false,
                    displayWindow : false,
                    captureOutput : true,
                    cancellationToken : cancellationToken);

                var span = DateTime.UtcNow - start;

                if (processOutput.ExitCode != 0)
                {
                    // On occasion we get a non-0 output but no actual data in the result file.  The could happen
                    // if xunit manages to crash when running a unit test (a stack overflow could cause this, for instance).
                    // To avoid losing information, write the process output to the console.  In addition, delete the results
                    // file to avoid issues with any tool attempting to interpret the (potentially malformed) text.
                    var resultData = string.Empty;
                    try
                    {
                        resultData = File.ReadAllText(resultsFilePath).Trim();
                    }
                    catch
                    {
                        // Happens if xunit didn't produce a log file
                    }

                    if (resultData.Length == 0)
                    {
                        // Delete the output file.
                        File.Delete(resultsFilePath);
                        resultsFilePath = null;
                    }
                }

                var commandLine = GetCommandLine(assemblyInfo);
                Logger.Log($"Command line {assemblyInfo.DisplayName}: {commandLine}");
                var standardOutput = string.Join(Environment.NewLine, processOutput.OutputLines) ?? "";
                var errorOutput    = string.Join(Environment.NewLine, processOutput.ErrorLines) ?? "";
                var testResultInfo = new TestResultInfo(
                    exitCode: processOutput.ExitCode,
                    resultsDirectory: resultsDir,
                    resultsFilePath: resultsFilePath,
                    elapsed: span,
                    standardOutput: standardOutput,
                    errorOutput: errorOutput);

                return(new TestResult(
                           assemblyInfo,
                           testResultInfo,
                           commandLine,
                           isFromCache: false));
            }
            catch (Exception ex)
            {
                throw new Exception($"Unable to run {assemblyInfo.AssemblyPath} with {_options.XunitPath}. {ex}");
            }
        }
Exemple #13
0
        private string GetResultsFilePath(AssemblyInfo assemblyInfo)
        {
            var resultsDir = Path.Combine(Path.GetDirectoryName(assemblyInfo.AssemblyPath), Constants.ResultsDirectoryName);

            return(Path.Combine(resultsDir, assemblyInfo.ResultsFileName));
        }
Exemple #14
0
 public string GetCommandLine(AssemblyInfo assemblyInfo)
 {
     return($"{_options.XunitPath} {GetCommandLineArguments(assemblyInfo)}");
 }
Exemple #15
0
        private async Task <TestResult> RunTestAsyncInternal(AssemblyInfo assemblyInfo, bool retry, CancellationToken cancellationToken)
        {
            try
            {
                var         commandLineArguments = GetCommandLineArguments(assemblyInfo);
                var         resultsFilePath      = GetResultsFilePath(assemblyInfo);
                var         resultsDir           = Path.GetDirectoryName(resultsFilePath);
                var         processResultList    = new List <ProcessResult>();
                ProcessInfo?procDumpProcessInfo  = null;

                // NOTE: xUnit doesn't always create the log directory
                Directory.CreateDirectory(resultsDir);

                // Define environment variables for processes started via ProcessRunner.
                var environmentVariables = new Dictionary <string, string>();
                Options.ProcDumpInfo?.WriteEnvironmentVariables(environmentVariables);

                if (retry && File.Exists(resultsFilePath))
                {
                    // Copy the results file path, since the new xunit run will overwrite it
                    var backupResultsFilePath = Path.ChangeExtension(resultsFilePath, ".old");
                    File.Copy(resultsFilePath, backupResultsFilePath, overwrite: true);

                    ConsoleUtil.WriteLine("Starting a retry. It will run once again tests failed.");
                    // If running the process with this varialbe added, we assume that this file contains
                    // xml logs from the first attempt.
                    environmentVariables.Add("OutputXmlFilePath", backupResultsFilePath);
                }

                // NOTE: xUnit seems to have an occasional issue creating logs create
                // an empty log just in case, so our runner will still fail.
                File.Create(resultsFilePath).Close();

                var start            = DateTime.UtcNow;
                var xunitProcessInfo = ProcessRunner.CreateProcess(
                    ProcessRunner.CreateProcessStartInfo(
                        Options.XunitPath,
                        commandLineArguments,
                        displayWindow: false,
                        captureOutput: true,
                        environmentVariables: environmentVariables),
                    lowPriority: false,
                    cancellationToken: cancellationToken);
                Logger.Log($"Create xunit process with id {xunitProcessInfo.Id} for test {assemblyInfo.DisplayName}");

                // Now that xunit is running we should kick off a procDump process if it was specified
                if (Options.ProcDumpInfo != null)
                {
                    var procDumpInfo      = Options.ProcDumpInfo.Value;
                    var procDumpStartInfo = ProcessRunner.CreateProcessStartInfo(
                        procDumpInfo.ProcDumpFilePath,
                        ProcDumpUtil.GetProcDumpCommandLine(xunitProcessInfo.Id, procDumpInfo.DumpDirectory),
                        captureOutput: true,
                        displayWindow: false);
                    Directory.CreateDirectory(procDumpInfo.DumpDirectory);
                    procDumpProcessInfo = ProcessRunner.CreateProcess(procDumpStartInfo, cancellationToken: cancellationToken);
                    Logger.Log($"Create procdump process with id {procDumpProcessInfo.Value.Id} for xunit {xunitProcessInfo.Id} for test {assemblyInfo.DisplayName}");
                }

                var xunitProcessResult = await xunitProcessInfo.Result;
                var span = DateTime.UtcNow - start;

                Logger.Log($"Exit xunit process with id {xunitProcessInfo.Id} for test {assemblyInfo.DisplayName} with code {xunitProcessResult.ExitCode}");
                processResultList.Add(xunitProcessResult);
                if (procDumpProcessInfo != null)
                {
                    var procDumpProcessResult = await procDumpProcessInfo.Value.Result;
                    Logger.Log($"Exit procdump process with id {procDumpProcessInfo.Value.Id} for {xunitProcessInfo.Id} for test {assemblyInfo.DisplayName} with code {procDumpProcessResult.ExitCode}");
                    processResultList.Add(procDumpProcessResult);
                }

                if (xunitProcessResult.ExitCode != 0)
                {
                    // On occasion we get a non-0 output but no actual data in the result file.  The could happen
                    // if xunit manages to crash when running a unit test (a stack overflow could cause this, for instance).
                    // To avoid losing information, write the process output to the console.  In addition, delete the results
                    // file to avoid issues with any tool attempting to interpret the (potentially malformed) text.
                    var resultData = string.Empty;
                    try
                    {
                        resultData = File.ReadAllText(resultsFilePath).Trim();
                    }
                    catch
                    {
                        // Happens if xunit didn't produce a log file
                    }

                    if (resultData.Length == 0)
                    {
                        // Delete the output file.
                        File.Delete(resultsFilePath);
                        resultsFilePath = null;
                    }
                }

                var commandLine = GetCommandLine(assemblyInfo);
                Logger.Log($"Command line {assemblyInfo.DisplayName}: {commandLine}");
                var standardOutput = string.Join(Environment.NewLine, xunitProcessResult.OutputLines) ?? "";
                var errorOutput    = string.Join(Environment.NewLine, xunitProcessResult.ErrorLines) ?? "";
                var testResultInfo = new TestResultInfo(
                    exitCode: xunitProcessResult.ExitCode,
                    resultsFilePath: resultsFilePath,
                    elapsed: span,
                    standardOutput: standardOutput,
                    errorOutput: errorOutput);

                return(new TestResult(
                           assemblyInfo,
                           testResultInfo,
                           commandLine,
                           isFromCache: false,
                           processResults: ImmutableArray.CreateRange(processResultList)));
            }
            catch (Exception ex)
            {
                throw new Exception($"Unable to run {assemblyInfo.AssemblyPath} with {Options.XunitPath}. {ex}");
            }
        }
Exemple #16
0
 private string GetResultsFilePath(AssemblyInfo assemblyInfo)
 {
     return(Path.Combine(Options.OutputDirectory, assemblyInfo.ResultsFileName));
 }