private async Task <ExitCode> RunVsTestAsync(
ILogger logger,
string vsTestReportDirectoryPath,
string vsTestExePath,
bool runTestsInReleaseConfiguration,
string assemblyFilePrefix)
{
Type testClassAttribute = typeof(TestClassAttribute);
Type testMethodAttribute = typeof(TestMethodAttribute);
var directory = new DirectoryInfo(_sourceRoot);
var typesToFind = new List <Type> {
testClassAttribute, testMethodAttribute
};
List <string> vsTestConsoleArguments =
new UnitTestFinder(typesToFind).GetUnitTestFixtureDlls(directory, runTestsInReleaseConfiguration, assemblyFilePrefix, FrameworkConstants.NetFramework)
.ToList();
if (!vsTestConsoleArguments.Any())
{
logger.WriteWarning(
$"Could not find any VSTest tests in directory '{directory.FullName}' or any sub-directory");
return(ExitCode.Success);
}
IEnumerable <string> options = GetVsTestConsoleOptions();
vsTestConsoleArguments.AddRange(options);
EnsureTestReportDirectoryExists(vsTestReportDirectoryPath);
string oldCurrentDirectory = SaveCurrentDirectory();
SetCurrentDirectory(vsTestReportDirectoryPath);
LogExecution(logger, vsTestConsoleArguments, vsTestExePath);
try
{
Task <ExitCode> execute = ProcessRunner.ExecuteAsync(
vsTestExePath,
arguments: vsTestConsoleArguments,
standardOutLog: logger.Write,
standardErrorAction: logger.WriteError,
toolAction: logger.Write);
ExitCode result = await execute;
return(result);
}
finally
{
RestoreCurrentDirectory(oldCurrentDirectory);
}
}
private async Task <ExitCode> RunNUnitAsync(
IVariable externalTools,
ILogger logger,
IVariable reportPath,
bool runTestsInReleaseConfiguration,
string assemblyFilePrefix)
{
Type fixtureAttribute = typeof(TestFixtureAttribute);
Type testMethodAttribute = typeof(TestAttribute);
var directory = new DirectoryInfo(_sourceRoot);
var typesToFind = new List <Type> {
fixtureAttribute, testMethodAttribute
};
Stopwatch stopwatch = Stopwatch.StartNew();
List <string> testDlls = new UnitTestFinder(typesToFind)
.GetUnitTestFixtureDlls(directory, runTestsInReleaseConfiguration, assemblyFilePrefix, FrameworkConstants.NetFramework)
.ToList();
stopwatch.Stop();
logger.Write($"NUnit test assembly lookup took {stopwatch.ElapsedMilliseconds:F2} milliseconds");
if (!testDlls.Any())
{
logger.WriteWarning(
$"Could not find any NUnit tests in directory '{directory.FullName}' or any sub-directory");
return(ExitCode.Success);
}
string nunitExePath = GetNunitExePath(externalTools);
var results = new List <Tuple <string, ExitCode> >();
foreach (string testDll in testDlls)
{
var nunitConsoleArguments = new List <string> {
testDll
};
string reportFilePath = GetNUnitXmlReportFilePath(reportPath, testDll);
EnsureNUnitReportDirectoryExists(reportFilePath);
IEnumerable <string> options = GetNUnitConsoleOptions(externalTools, reportFilePath);
nunitConsoleArguments.AddRange(options);
LogExecution(logger, nunitConsoleArguments, nunitExePath);
Stopwatch executionStopwatch = Stopwatch.StartNew();
ExitCode result = await ProcessRunner.ExecuteAsync(
nunitExePath,
arguments : nunitConsoleArguments,
standardOutLog : logger.Write,
standardErrorAction : logger.WriteError,
toolAction : logger.Write);
executionStopwatch.Stop();
logger.Write($"NUnit execution took {executionStopwatch.ElapsedMilliseconds:F2} milliseconds");
results.Add(Tuple.Create(testDll, result));
}
if (results.All(result => result.Item2.IsSuccess))
{
return(ExitCode.Success);
}
var failedTestsBuilder = new StringBuilder();
failedTestsBuilder.AppendLine("The following DLL files were not tested successfully:");
foreach (Tuple <string, ExitCode> result in results.Where(r => !r.Item2.IsSuccess))
{
failedTestsBuilder.AppendLine(result.Item1);
}
logger.WriteError(failedTestsBuilder.ToString());
return(ExitCode.Failure);
}
public async Task <ExitCode> ExecuteAsync([NotNull] ILogger logger, [NotNull] IReadOnlyCollection <IVariable> buildVariables, CancellationToken cancellationToken)
{
if (logger == null)
{
throw new ArgumentNullException(nameof(logger));
}
if (buildVariables == null)
{
throw new ArgumentNullException(nameof(buildVariables));
}
bool enabled = buildVariables.GetBooleanByKey(WellKnownVariables.XUnitNetCoreAppEnabled, false);
if (!enabled)
{
logger.WriteDebug("Xunit .NET Core App test runner is not enabled");
return(ExitCode.Success);
}
_sourceRoot = buildVariables.Require(WellKnownVariables.SourceRoot).ThrowIfEmptyValue().Value;
IVariable reportPath = buildVariables.Require(WellKnownVariables.ReportPath).ThrowIfEmptyValue();
string xunitDllPath = buildVariables.GetVariableValueOrDefault(WellKnownVariables.XUnitNetCoreAppDllPath, null) ?? Path.Combine(buildVariables.Require(WellKnownVariables.ExternalTools).Value, "xunit", "netcoreapp2.0", "xunit.console.dll");
logger.WriteDebug($"Using XUnit dll path '{xunitDllPath}'");
Type theoryType = typeof(TheoryAttribute);
Type factAttribute = typeof(FactAttribute);
var directory = new DirectoryInfo(_sourceRoot);
var typesToFind = new List <Type> {
theoryType, factAttribute
};
bool runTestsInReleaseConfiguration =
buildVariables.GetBooleanByKey(
WellKnownVariables.RunTestsInReleaseConfigurationEnabled,
true);
string configuration = runTestsInReleaseConfiguration ? "release" : "debug";
string assemblyFilePrefix = buildVariables.GetVariableValueOrDefault(WellKnownVariables.TestsAssemblyStartsWith, string.Empty);
logger.Write($"Finding Xunit test DLL files built with {configuration} in directory '{_sourceRoot}'");
logger.Write($"Looking for types {string.Join(", ", typesToFind.Select(t => t.FullName))} in directory '{_sourceRoot}'");
List <string> testDlls = new UnitTestFinder(typesToFind, logger: logger, debugLogEnabled: true)
.GetUnitTestFixtureDlls(directory, runTestsInReleaseConfiguration, assemblyFilePrefix: assemblyFilePrefix, targetFrameworkPrefix: FrameworkConstants.NetCoreApp)
.ToList();
if (!testDlls.Any())
{
logger.Write("Found no .NETCoreApp Assemblies with Xunit tests");
return(ExitCode.Success);
}
string dotNetExePath =
buildVariables.GetVariableValueOrDefault(WellKnownVariables.DotNetExePath, string.Empty);
if (string.IsNullOrWhiteSpace(dotNetExePath))
{
logger.Write(
$"Path to 'dotnet.exe' has not been specified, set variable '{WellKnownVariables.DotNetExePath}' or ensure the dotnet.exe is installed in its standard location");
return(ExitCode.Failure);
}
logger.WriteDebug($"Using dotnet.exe in path '{dotNetExePath}'");
string xmlReportName = $"{Guid.NewGuid()}.xml";
var arguments = new List <string>();
string reportFile = Path.Combine(reportPath.Value, "xunit", xmlReportName);
var reportFileInfo = new FileInfo(reportFile);
reportFileInfo.Directory.EnsureExists();
arguments.Add(xunitDllPath);
arguments.AddRange(testDlls);
arguments.Add("-nunit");
arguments.Add(reportFileInfo.FullName);
ExitCode result = await ProcessRunner.ExecuteAsync(
dotNetExePath,
arguments : arguments,
standardOutLog : logger.Write,
standardErrorAction : logger.WriteError,
toolAction : logger.Write,
cancellationToken : cancellationToken);
return(result);
}
public async Task <ExitCode> ExecuteAsync(
ILogger logger,
IReadOnlyCollection <IVariable> buildVariables,
CancellationToken cancellationToken)
{
bool enabled = buildVariables.GetBooleanByKey(WellKnownVariables.MSpecEnabled, true);
if (!enabled)
{
logger.WriteWarning($"{MachineSpecificationsConstants.MachineSpecificationsName} not enabled");
return(ExitCode.Success);
}
string externalToolsPath =
buildVariables.Require(WellKnownVariables.ExternalTools).ThrowIfEmptyValue().Value;
string sourceRoot =
buildVariables.Require(WellKnownVariables.SourceRoot).ThrowIfEmptyValue().Value;
string testReportDirectoryPath =
buildVariables.Require(WellKnownVariables.ExternalTools_MSpec_ReportPath).ThrowIfEmptyValue().Value;
string sourceRootOverride =
buildVariables.GetVariableValueOrDefault(WellKnownVariables.SourceRootOverride, string.Empty);
string sourceDirectoryPath;
if (string.IsNullOrWhiteSpace(sourceRootOverride) || !Directory.Exists(sourceRootOverride))
{
if (sourceRoot == null)
{
throw new InvalidOperationException("Source root cannot be null");
}
sourceDirectoryPath = sourceRoot;
}
else
{
sourceDirectoryPath = sourceRootOverride;
}
var directory = new DirectoryInfo(sourceDirectoryPath);
string mspecExePath = Path.Combine(
externalToolsPath,
MachineSpecificationsConstants.MachineSpecificationsName,
"mspec-clr4.exe");
bool runTestsInReleaseConfiguration =
buildVariables.GetBooleanByKey(
WellKnownVariables.RunTestsInReleaseConfigurationEnabled,
true);
IEnumerable <Type> typesToFind = new List <Type>
{
typeof(It),
typeof(BehaviorsAttribute),
typeof(SubjectAttribute),
typeof(Behaves_like <>)
};
logger.WriteVerbose(
$"Scanning directory '{directory.FullName}' for assemblies containing Machine.Specifications tests");
string assemblyFilePrefix = buildVariables.GetVariableValueOrDefault(WellKnownVariables.TestsAssemblyStartsWith, string.Empty);
List <string> testDlls =
new UnitTestFinder(typesToFind, logger: logger)
.GetUnitTestFixtureDlls(directory, runTestsInReleaseConfiguration, assemblyFilePrefix, FrameworkConstants.NetFramework)
.ToList();
if (!testDlls.Any())
{
logger.WriteWarning(
$"No DLL files with {MachineSpecificationsConstants.MachineSpecificationsName} specifications was found");
return(ExitCode.Success);
}
var arguments = new List <string>();
arguments.AddRange(testDlls);
arguments.Add("--xml");
string timestamp = DateTime.UtcNow.ToString("O").Replace(":", ".");
string fileName = "MSpec_" + timestamp + ".xml";
string xmlReportPath = Path.Combine(testReportDirectoryPath, "Xml", fileName);
new FileInfo(xmlReportPath).Directory.EnsureExists();
arguments.Add(xmlReportPath);
string htmlPath = Path.Combine(testReportDirectoryPath, "Html", "MSpec_" + timestamp);
new DirectoryInfo(htmlPath).EnsureExists();
IReadOnlyCollection <string> excludedTags = buildVariables
.GetVariableValueOrDefault(
WellKnownVariables.IgnoredTestCategories,
string.Empty)
.Split(new[] { "," }, StringSplitOptions.RemoveEmptyEntries)
.Select(item => item.Trim())
.Where(item => !string.IsNullOrWhiteSpace(item))
.ToReadOnlyCollection();
arguments.Add("--html");
arguments.Add(htmlPath);
bool hasArborTestDll =
testDlls.Any(dll => dll.IndexOf("arbor", StringComparison.InvariantCultureIgnoreCase) >= 0);
if (hasArborTestDll || excludedTags.Any())
{
var allExcludedTags = new List <string>();
arguments.Add("--exclude");
if (hasArborTestDll)
{
allExcludedTags.Add(MSpecInternalConstants.RecursiveArborXTest);
}
if (excludedTags.Any())
{
allExcludedTags.AddRange(excludedTags);
}
string excludedTagsParameter = string.Join(",", allExcludedTags);
logger.Write($"Running MSpec with excluded tags: {excludedTagsParameter}");
arguments.Add(excludedTagsParameter);
}
// ReSharper disable once CollectionNeverUpdated.Local
var environmentVariables = new Dictionary <string, string>();
ExitCode exitCode = await
ProcessRunner.ExecuteAsync(
mspecExePath,
arguments : arguments,
cancellationToken : cancellationToken,
standardOutLog : logger.Write,
standardErrorAction : logger.WriteError,
toolAction : logger.Write,
verboseAction : logger.WriteVerbose,
environmentVariables : environmentVariables,
debugAction : logger.WriteDebug);
if (buildVariables.GetBooleanByKey(
WellKnownVariables.MSpecJUnitXslTransformationEnabled,
false))
{
logger.WriteVerbose(
$"Transforming {MachineSpecificationsConstants.MachineSpecificationsName} test reports to JUnit format");
const string junitSuffix = "_junit.xml";
DirectoryInfo xmlReportDirectory = new FileInfo(xmlReportPath).Directory;
// ReSharper disable once PossibleNullReferenceException
IReadOnlyCollection <FileInfo> xmlReports = xmlReportDirectory
.GetFiles("*.xml")
.Where(report => !report.Name.EndsWith(junitSuffix, StringComparison.Ordinal))
.ToReadOnlyCollection();
if (xmlReports.Any())
{
Encoding encoding = Encoding.UTF8;
using (Stream stream = new MemoryStream(encoding.GetBytes(MSpecJUnitXsl.Xml)))
{
using (XmlReader xmlReader = new XmlTextReader(stream))
{
var myXslTransform = new XslCompiledTransform();
myXslTransform.Load(xmlReader);
foreach (FileInfo xmlReport in xmlReports)
{
logger.WriteDebug($"Transforming '{xmlReport.FullName}' to JUnit XML format");
try
{
TransformReport(xmlReport, junitSuffix, encoding, myXslTransform, logger);
}
catch (Exception ex)
{
logger.WriteError($"Could not transform '{xmlReport.FullName}', {ex}");
return(ExitCode.Failure);
}
logger.WriteDebug(
$"Successfully transformed '{xmlReport.FullName}' to JUnit XML format");
}
}
}
}
}
return(exitCode);
}
public async Task <ExitCode> ExecuteAsync([NotNull] ILogger logger, [NotNull] IReadOnlyCollection <IVariable> buildVariables, CancellationToken cancellationToken)
{
if (logger == null)
{
throw new ArgumentNullException(nameof(logger));
}
if (buildVariables == null)
{
throw new ArgumentNullException(nameof(buildVariables));
}
bool enabled = buildVariables.GetBooleanByKey(WellKnownVariables.XUnitNetFrameworkEnabled, false);
if (!enabled)
{
logger.WriteDebug("Xunit .NET Framework test runner is not enabled");
return(ExitCode.Success);
}
_sourceRoot = buildVariables.Require(WellKnownVariables.SourceRoot).ThrowIfEmptyValue().Value;
IVariable reportPath = buildVariables.Require(WellKnownVariables.ReportPath).ThrowIfEmptyValue();
string xunitExePath = buildVariables.GetVariableValueOrDefault(WellKnownVariables.XUnitNetFrameworkExePath, Path.Combine(buildVariables.Require(WellKnownVariables.ExternalTools).Value, "xunit", "net452", "xunit.console.exe"));
Type theoryType = typeof(TheoryAttribute);
Type factAttribute = typeof(FactAttribute);
var directory = new DirectoryInfo(_sourceRoot);
var typesToFind = new List <Type> {
theoryType, factAttribute
};
bool runTestsInReleaseConfiguration =
buildVariables.GetBooleanByKey(
WellKnownVariables.RunTestsInReleaseConfigurationEnabled,
true);
string assemblyFilePrefix = buildVariables.GetVariableValueOrDefault(WellKnownVariables.TestsAssemblyStartsWith, string.Empty);
List <string> testDlls = new UnitTestFinder(typesToFind)
.GetUnitTestFixtureDlls(directory, runTestsInReleaseConfiguration, assemblyFilePrefix, FrameworkConstants.NetFramework)
.ToList();
if (!testDlls.Any())
{
logger.Write("Could not find any DLL files with Xunit test and target framework .NETFramework, skipping Xunit Net Framework tests");
return(ExitCode.Success);
}
string xmlReportName = $"{Guid.NewGuid()}.xml";
var arguments = new List <string>();
string reportFile = Path.Combine(reportPath.Value, "xunit", xmlReportName);
var fileInfo = new FileInfo(reportFile);
fileInfo.Directory.EnsureExists();
arguments.AddRange(testDlls);
arguments.Add("-nunit");
arguments.Add(fileInfo.FullName);
ExitCode result = await ProcessRunner.ExecuteAsync(
xunitExePath,
arguments : arguments,
standardOutLog : logger.Write,
standardErrorAction : logger.WriteError,
toolAction : logger.Write,
cancellationToken : cancellationToken);
return(result);
}
