Assembly LoadAssembly(AssemblyLoadContext context, AssemblyName assemblyName)
{
if (lookupCache.TryGetValue(assemblyName.Name, out var result))
{
return(result);
}
if (internalDiagnosticsMessageSink != null)
{
internalDiagnosticsMessageSink.OnMessage(new DiagnosticMessage($"[AssemblyHelper_NetCoreApp.LoadAssembly] Resolving '{assemblyName.Name}'"));
}
var path = Path.Combine(directory, assemblyName.Name);
result = ResolveAndLoadAssembly(context, path, out var resolvedAssemblyPath);
if (internalDiagnosticsMessageSink != null)
{
if (result == null)
{
internalDiagnosticsMessageSink.OnMessage(new DiagnosticMessage($"[AssemblyHelper_NetCoreApp.LoadAssembly] Resolution failed, passed down to next resolver"));
}
else
{
internalDiagnosticsMessageSink.OnMessage(new DiagnosticMessage($"[AssemblyHelper_NetCoreApp.LoadAssembly] Successful: '{resolvedAssemblyPath}'"));
}
}
lookupCache[assemblyName.Name] = result;
return(result);
}
public async Task InstallingSamePackageFromRemoteUpdatesLocal(string installCommandName, string uninstallCommandName)
{
_messageSink.OnMessage(new DiagnosticMessage($"{nameof(InstallingSamePackageFromRemoteUpdatesLocal)} started."));
var home = TestUtils.CreateTemporaryFolder("Home");
using var packageManager = new PackageManager();
string packageLocation = await packageManager.GetNuGetPackage(
"Microsoft.DotNet.Common.ProjectTemplates.5.0",
minimumVersion : new NuGet.Versioning.NuGetVersion(6, 0, 0),
logger : new XunitNuGetLogger(_messageSink)).ConfigureAwait(false);
_messageSink.OnMessage(new DiagnosticMessage($"{nameof(InstallingSamePackageFromRemoteUpdatesLocal)}: Microsoft.DotNet.Common.ProjectTemplates.5.0 is downloaded to {packageLocation}.)"));
new DotnetNewCommand(_log, installCommandName, packageLocation)
.WithCustomHive(home)
.WithWorkingDirectory(TestUtils.CreateTemporaryFolder())
.Execute()
.Should().ExitWith(0)
.And.NotHaveStdErr()
.And.HaveStdOutMatching($"Success: Microsoft\\.DotNet\\.Common\\.ProjectTemplates\\.5\\.0::([\\d\\.a-z-])+ installed the following templates:")
.And.HaveStdOutContaining("console")
.And.HaveStdOutContaining("classlib");
new DotnetNewCommand(_log, uninstallCommandName)
.WithCustomHive(home)
.WithWorkingDirectory(TestUtils.CreateTemporaryFolder())
.Execute()
.Should().ExitWith(0)
.And.NotHaveStdErr()
.And.HaveStdOutContaining("Microsoft.DotNet.Common.ProjectTemplates.5.0")
.And.HaveStdOutContaining("Author: Microsoft")
.And.HaveStdOutContaining("Version:")
.And.NotHaveStdOutContaining("Version: 5.0.0");
new DotnetNewCommand(_log, installCommandName, "Microsoft.DotNet.Common.ProjectTemplates.5.0::5.0.0")
.WithCustomHive(home)
.WithWorkingDirectory(TestUtils.CreateTemporaryFolder())
.Execute()
.Should().ExitWith(0)
.And.NotHaveStdErr()
.And.HaveStdOutContaining("The following template packages will be installed:")
.And.HaveStdOutContaining("Microsoft.DotNet.Common.ProjectTemplates.5.0::5.0.0")
.And.HaveStdOutMatching("Microsoft\\.DotNet\\.Common\\.ProjectTemplates\\.5\\.0 \\(version ([\\d\\.a-z-])+\\) is already installed, it will be replaced with version 5\\.0\\.0")
.And.HaveStdOutMatching("Microsoft\\.DotNet\\.Common\\.ProjectTemplates\\.5\\.0::([\\d\\.a-z-])+ was successfully uninstalled")
.And.HaveStdOutContaining($"Success: Microsoft.DotNet.Common.ProjectTemplates.5.0::5.0.0 installed the following templates:")
.And.HaveStdOutContaining("console")
.And.HaveStdOutContaining("classlib");
new DotnetNewCommand(_log, uninstallCommandName)
.WithCustomHive(home)
.WithWorkingDirectory(TestUtils.CreateTemporaryFolder())
.Execute()
.Should().ExitWith(0)
.And.NotHaveStdErr()
.And.HaveStdOutContaining("Microsoft.DotNet.Common.ProjectTemplates.5.0")
.And.HaveStdOutContaining("Author: Microsoft")
.And.HaveStdOutContaining("Version: 5.0.0");
_messageSink.OnMessage(new DiagnosticMessage($"{nameof(InstallingSamePackageFromRemoteUpdatesLocal)} finished."));
}
RunSummary RunTestClass(ITestCollection testCollection, string className, IList <Xunit1TestCase> testCases, IMessageSink messageSink)
{
var handler = new TestClassCallbackHandler(testCases, messageSink);
var results = handler.TestClassResults;
results.Continue = messageSink.OnMessage(new TestClassStarting(testCollection, className));
if (results.Continue)
{
try
{
var methodNames = testCases.Select(tc => tc.Method.Name).ToList();
executor.RunTests(className, methodNames, handler);
handler.LastNodeArrived.WaitOne();
}
catch (Exception ex)
{
var failureInformation = Xunit1ExceptionUtility.ConvertToFailureInformation(ex);
results.Continue = messageSink.OnMessage(new ErrorMessage(failureInformation.ExceptionTypes, failureInformation.Messages, failureInformation.StackTraces, failureInformation.ExceptionParentIndices)) && results.Continue;
}
}
results.Continue = messageSink.OnMessage(new TestClassFinished(testCollection, className, results.Time, results.Total, results.Failed, results.Skipped)) && results.Continue;
return(results);
}
bool OnStart(XmlNode xml)
{
var testCase = FindTestCase(xml.Attributes["type"].Value, xml.Attributes["method"].Value);
SendTestCaseMessagesWhenAppropriate(testCase);
return(messageSink.OnMessage(new TestStarting(testCase, xml.Attributes["name"].Value)) && TestClassResults.Continue);
}
bool OnClass(XmlNode xml)
{
SendTestCaseMessagesWhenAppropriate(null);
var @continue = true;
XmlNode failureNode;
if ((failureNode = xml.SelectSingleNode("failure")) != null)
{
var failureInformation = Xunit1ExceptionUtility.ConvertToFailureInformation(failureNode);
var errorMessage = new ErrorMessage(testCases,
failureInformation.ExceptionTypes,
failureInformation.Messages,
failureInformation.StackTraces,
failureInformation.ExceptionParentIndices);
@continue = messageSink.OnMessage(errorMessage);
}
TestClassResults.Time = Decimal.Parse(xml.Attributes["time"].Value, CultureInfo.InvariantCulture);
TestClassResults.Total = Int32.Parse(xml.Attributes["total"].Value, CultureInfo.InvariantCulture);
TestClassResults.Failed = Int32.Parse(xml.Attributes["failed"].Value, CultureInfo.InvariantCulture);
TestClassResults.Skipped = Int32.Parse(xml.Attributes["skipped"].Value, CultureInfo.InvariantCulture);
return(@continue && TestClassResults.Continue);
}
public OutputController(IReportConfiguration configuration, IMessageSink diagnosticMessageSink)
{
_configuration = configuration;
_diagnosticMessageSink = diagnosticMessageSink;
IReportWriter writer;
if (!configuration.WriteOutput)
{
_diagnosticMessageSink.OnMessage(new DiagnosticMessage("Output is disabled"));
writer = new NullWriter();
}
else
{
_diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Creating report at {configuration.XmlReportFile}"));
_fileStream = new FileStream(configuration.XmlReportFile, FileMode.Create, FileAccess.Write,
FileShare.None, 4096, true);
_sw = new StreamWriter(_fileStream);
XmlWriterSettings xws = new XmlWriterSettings();
xws.Async = true;
_xw = XmlWriter.Create(_sw, xws);
writer = new ReportWriter(_xw);
}
Report = new ScenarioReport(configuration.AssemblyName, writer, _diagnosticMessageSink);
}
/// <inheritdoc/>
public void Run(IEnumerable<ITestCase> testMethods, IMessageSink messageSink)
{
bool cancelled = false;
var totalSummary = new RunSummary();
string currentDirectory = Directory.GetCurrentDirectory();
try
{
Directory.SetCurrentDirectory(Path.GetDirectoryName(assemblyInfo.AssemblyPath));
if (messageSink.OnMessage(new TestAssemblyStarting()))
{
var classGroups = testMethods.Cast<XunitTestCase>().GroupBy(tc => tc.Class).ToList();
if (classGroups.Count > 0)
{
var collectionSummary = new RunSummary();
if (messageSink.OnMessage(new TestCollectionStarting()))
{
foreach (var group in classGroups)
{
var classSummary = new RunSummary();
if (!messageSink.OnMessage(new TestClassStarting { ClassName = group.Key.Name }))
cancelled = true;
else
{
cancelled = RunTestClass(messageSink, group, classSummary);
collectionSummary.Aggregate(classSummary);
}
if (!messageSink.OnMessage(new TestClassFinished { Assembly = assemblyInfo, ClassName = group.Key.Name, TestsRun = classSummary.Total }))
cancelled = true;
if (cancelled)
break;
}
}
messageSink.OnMessage(new TestCollectionFinished { Assembly = assemblyInfo, TestsRun = collectionSummary.Total });
totalSummary.Aggregate(collectionSummary);
}
}
messageSink.OnMessage(new TestAssemblyFinished
{
Assembly = assemblyInfo,
TestsRun = totalSummary.Total,
TestsFailed = totalSummary.Failed,
TestsSkipped = totalSummary.Skipped,
ExecutionTime = totalSummary.Time
});
}
finally
{
Directory.SetCurrentDirectory(currentDirectory);
}
}
public Assembly LoadManagedDll([NotNull] string assemblyName, [NotNull] Func <string, Assembly> managedAssemblyLoader)
{
if (_managedAssemblyCache.TryGetValue(assemblyName, out var result))
{
return(result);
}
_internalDiagnosticsMessageSink?.OnMessage(new DiagnosticMessage($"[DependencyContextAssemblyCache.LoadManagedDll] Resolving '{assemblyName}'"));
var tupleResult = ResolveManagedAssembly(assemblyName, managedAssemblyLoader);
var resolvedAssemblyPath = tupleResult.Item1;
result = tupleResult.Item2;
_managedAssemblyCache[assemblyName] = result;
if (_internalDiagnosticsMessageSink == null)
{
return(result);
}
_internalDiagnosticsMessageSink.OnMessage(
result == null
? new DiagnosticMessage("[DependencyContextAssemblyCache.LoadManagedDll] Resolution failed, passed down to next resolver")
: new DiagnosticMessage($"[DependencyContextAssemblyCache.LoadManagedDll] Successful: '{resolvedAssemblyPath}'"));
return(result);
}
/// <inheritdoc/>
protected override IntPtr LoadUnmanagedDll(string unmanagedDllName)
{
var result = base.LoadUnmanagedDll(unmanagedDllName);
if (result != default)
{
return(result);
}
if (!unmanagedAssemblyCache.TryGetValue(unmanagedDllName, out var resolvedAssemblyPath))
{
if (internalDiagnosticsMessageSink != null)
{
internalDiagnosticsMessageSink.OnMessage(new DiagnosticMessage($"[NetCoreAssemblyDependencyResolver.LoadUnmanagedDll] Resolving '{unmanagedDllName}'"));
}
resolvedAssemblyPath = ResolveUnmanagedAssembly(unmanagedDllName);
unmanagedAssemblyCache[unmanagedDllName] = resolvedAssemblyPath;
if (internalDiagnosticsMessageSink != null)
{
if (resolvedAssemblyPath == null)
{
internalDiagnosticsMessageSink.OnMessage(new DiagnosticMessage($"[NetCoreAssemblyDependencyResolver.LoadUnmanagedDll] Resolution failed, passed down to next resolver"));
}
else
{
internalDiagnosticsMessageSink.OnMessage(new DiagnosticMessage($"[NetCoreAssemblyDependencyResolver.LoadUnmanagedDll] Successful: '{resolvedAssemblyPath}'"));
}
}
}
return(resolvedAssemblyPath != null?LoadUnmanagedDllFromPath(resolvedAssemblyPath) : default);
void Find(Predicate <ITestCaseDiscoveryMessage> filter, bool includeSourceInformation, IMessageSink messageSink)
{
try
{
XmlNode assemblyXml = null;
var handler = new XmlNodeCallbackHandler(xml => { assemblyXml = xml; return(true); });
executor.EnumerateTests(handler);
foreach (XmlNode method in assemblyXml.SelectNodes("//method"))
{
var testCase = method.ToTestCase(assemblyFileName, configFileName);
if (testCase != null)
{
if (includeSourceInformation)
{
testCase.SourceInformation = sourceInformationProvider.GetSourceInformation(testCase);
}
var message = new TestCaseDiscoveryMessage(testCase);
if (filter(message))
{
messageSink.OnMessage(message);
}
}
}
}
finally
{
messageSink.OnMessage(new DiscoveryCompleteMessage());
}
}
protected override async Task <RunSummary> RunTestCaseAsync(IXunitTestCase testCase)
{
var parameters = string.Empty;
if (testCase.TestMethodArguments != null)
{
parameters = string.Join(", ", testCase.TestMethodArguments.Select(a => a?.ToString() ?? "null"));
}
var test = $"{TestMethod.TestClass.Class.Name}.{TestMethod.Method.Name}({parameters})";
_diagnosticMessageSink.OnMessage(new DiagnosticMessage($"STARTED: {test}"));
using var timer = new Timer(
_ => _diagnosticMessageSink.OnMessage(new DiagnosticMessage($"WARNING: {test} has been running for more than 15 minutes")),
null,
TimeSpan.FromMinutes(15),
Timeout.InfiniteTimeSpan);
try
{
var result = await base.RunTestCaseAsync(testCase);
var status = result.Failed > 0 ? "FAILURE" : "SUCCESS";
_diagnosticMessageSink.OnMessage(new DiagnosticMessage($"{status}: {test} ({result.Time}s)"));
return(result);
}
catch (Exception ex)
{
_diagnosticMessageSink.OnMessage(new DiagnosticMessage($"ERROR: {test} ({ex.Message})"));
throw;
}
}
Assembly LoadAssembly(AssemblyName assemblyName)
{
if (lookupCache.TryGetValue(assemblyName.Name, out var result))
{
return(result);
}
var path = Path.Combine(directory, assemblyName.Name);
result = ResolveAndLoadAssembly(path, out var resolvedAssemblyPath);
if (internalDiagnosticsMessageSink != null)
{
if (result == null)
{
internalDiagnosticsMessageSink.OnMessage(new _DiagnosticMessage($"[AssemblyHelper_Desktop.LoadAssembly] Resolution for '{assemblyName.Name}' failed, passed down to next resolver"));
}
else
{
internalDiagnosticsMessageSink.OnMessage(new _DiagnosticMessage($"[AssemblyHelper_Desktop.LoadAssembly] Resolved '{assemblyName.Name}' to '{resolvedAssemblyPath}'"));
}
}
lookupCache[assemblyName.Name] = result;
return(result);
}
/// <summary>
/// Starts the process of running all the xUnit.net v1 tests.
/// </summary>
/// <param name="testCases">The test cases to run; if null, all tests in the assembly are run.</param>
/// <param name="messageSink">The message sink to report results back to.</param>
public void Run(IEnumerable <ITestCase> testCases, IMessageSink messageSink)
{
var results = new Xunit1RunSummary();
var environment = String.Format("{0}-bit .NET {1}", IntPtr.Size * 8, Environment.Version);
var firstTestCase = testCases.FirstOrDefault();
var testCollection = firstTestCase == null ? null : firstTestCase.TestMethod.TestClass.TestCollection;
if (testCollection != null)
{
try
{
if (messageSink.OnMessage(new TestAssemblyStarting(testCases, testCollection.TestAssembly, DateTime.Now, environment, TestFrameworkDisplayName)))
{
results = RunTestCollection(testCollection, testCases, messageSink);
}
}
catch (Exception ex)
{
var failureInformation = Xunit1ExceptionUtility.ConvertToFailureInformation(ex);
messageSink.OnMessage(new ErrorMessage(testCases, failureInformation.ExceptionTypes,
failureInformation.Messages, failureInformation.StackTraces,
failureInformation.ExceptionParentIndices));
}
finally
{
messageSink.OnMessage(new TestAssemblyFinished(testCases, testCollection.TestAssembly, results.Time, results.Total, results.Failed, results.Skipped));
}
}
}
/// <summary>
/// Gives an opportunity to override test case orderer. By default, this method gets the
/// orderer from the collection definition. If this function returns <c>null</c>, the
/// test case orderer passed into the constructor will be used.
/// </summary>
protected virtual ITestCaseOrderer GetTestCaseOrderer()
{
if (TestCollection.CollectionDefinition != null)
{
var ordererAttribute = TestCollection.CollectionDefinition.GetCustomAttributes(typeof(TestCaseOrdererAttribute)).SingleOrDefault();
if (ordererAttribute != null)
{
try
{
var testCaseOrderer = ExtensibilityPointFactory.GetTestCaseOrderer(diagnosticMessageSink, ordererAttribute);
if (testCaseOrderer != null)
{
return(testCaseOrderer);
}
var args = ordererAttribute.GetConstructorArguments().Cast <string>().ToList();
diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Could not find type '{args[0]}' in {args[1]} for collection-level test case orderer on test collection '{TestCollection.DisplayName}'"));
}
catch (Exception ex)
{
var innerEx = ex.Unwrap();
var args = ordererAttribute.GetConstructorArguments().Cast <string>().ToList();
diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Collection-level test case orderer '{args[0]}' for test collection '{TestCollection.DisplayName}' threw '{innerEx.GetType().FullName}' during construction: {innerEx.Message}{Environment.NewLine}{innerEx.StackTrace}"));
}
}
}
return(null);
}
Xunit1RunSummary RunTestCollection(ITestCollection testCollection, IEnumerable <ITestCase> testCases, IMessageSink messageSink)
{
var results = new Xunit1RunSummary();
results.Continue = messageSink.OnMessage(new TestCollectionStarting(testCases, testCollection));
try
{
if (results.Continue)
{
foreach (var testClassGroup in testCases.GroupBy(tc => tc.TestMethod.TestClass, Comparer.Instance))
{
var classResults = RunTestClass(testClassGroup.Key, testClassGroup.ToList(), messageSink);
results.Aggregate(classResults);
if (!classResults.Continue)
{
break;
}
}
}
}
finally
{
results.Continue = messageSink.OnMessage(new TestCollectionFinished(testCases, testCollection, results.Time, results.Total, results.Failed, results.Skipped)) && results.Continue;
}
return(results);
}
/// <inheritdoc/>
protected void SetTestCaseOrderer(IReflectionTypeInfo Class)
{
var ordererAttribute = Class.GetCustomAttributes(typeof(TestCaseOrdererAttribute)).SingleOrDefault();
if (ordererAttribute != null)
{
try
{
var testCaseOrderer = ExtensibilityPointFactory.GetTestCaseOrderer(diagnosticMessageSink, ordererAttribute);
if (testCaseOrderer != null)
{
TestCaseOrderer = testCaseOrderer;
}
else
{
var args = ordererAttribute.GetConstructorArguments().Cast <string>().ToList();
diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Could not find type '{args[0]}' in {args[1]} for class-level test case orderer on test class '{Class.Name}'"));
}
}
catch (Exception ex)
{
var innerEx = ex.InnerException ?? ex; //.Unwrap();
var args = ordererAttribute.GetConstructorArguments().Cast <string>().ToList();
diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Class-level test case orderer '{args[0]}' for test class '{Class.Name}' threw '{innerEx.GetType().FullName}' " +
$"during construction: {innerEx.Message}{Environment.NewLine}{innerEx.StackTrace}"));
}
}
}
/// <inheritdoc/>
protected override bool RunTests(IMessageSink messageSink, object[] constructorArguments, ExceptionAggregator aggregator)
{
bool cancelled = false;
if (!messageSink.OnMessage(new TestStarting { TestCase = this, TestDisplayName = DisplayName }))
cancelled = true;
else
{
try
{
lambda();
if (!messageSink.OnMessage(new TestPassed { TestCase = this, TestDisplayName = DisplayName }))
cancelled = true;
}
catch (Exception ex)
{
if (!messageSink.OnMessage(new TestFailed(ex) { TestCase = this, TestDisplayName = DisplayName }))
cancelled = true;
}
}
if (!messageSink.OnMessage(new TestFinished { TestCase = this, TestDisplayName = DisplayName }))
cancelled = true;
return cancelled;
}
public static IDisposable SubscribeResolveForAssembly([NotNull] string assemblyFileName, [CanBeNull] IMessageSink internalDiagnosticsMessageSink)
{
var assemblyFolder = Path.GetDirectoryName(assemblyFileName).ThrowIfNull();
var depsJsonFile = Path.Combine(assemblyFolder, Path.GetFileNameWithoutExtension(assemblyFileName) + ".deps.json");
if (!File.Exists(depsJsonFile))
{
internalDiagnosticsMessageSink?.OnMessage(
new DiagnosticMessage(
$"[DependencyContextAssemblyHelper.SubscribeResolveForAssembly] Skipping resolution for '{depsJsonFile}': File not found"));
return(null);
}
using (var stream = File.OpenRead(depsJsonFile))
{
var context = JsonReader.Read(stream);
if (context != null)
{
return(new DependencyContextAssemblyHelper(assemblyFolder, context, internalDiagnosticsMessageSink));
}
internalDiagnosticsMessageSink?.OnMessage(
new DiagnosticMessage(
$"[DependencyContextAssemblyHelper.SubscribeResolveForAssembly] Skipping resolution for '{depsJsonFile}': File appears to be malformed"));
return(null);
}
}
protected override async Task <RunSummary> RunTestCaseAsync(IXunitTestCase testCase)
{
var parameters = string.Empty;
if (testCase.TestMethodArguments != null)
{
parameters = string.Join(", ", testCase.TestMethodArguments.Select(a => a?.ToString() ?? "null"));
}
var test = $"{TestMethod.TestClass.Class.Name}.{TestMethod.Method.Name}({parameters})";
_diagnosticMessageSink.OnMessage(new DiagnosticMessage($"STARTED: {test}"));
try
{
var result = await base.RunTestCaseAsync(testCase);
var status = result.Failed > 0 ? "FAILURE" : "SUCCESS";
_diagnosticMessageSink.OnMessage(new DiagnosticMessage($"{status}: {test} ({result.Time}s)"));
return(result);
}
catch (Exception ex)
{
_diagnosticMessageSink.OnMessage(new DiagnosticMessage($"ERROR: {test} ({ex.Message})"));
throw;
}
}
static Type GetTestFrameworkType(IAssemblyInfo testAssembly, IMessageSink diagnosticMessageSink)
{
try
{
var testFrameworkAttr = testAssembly.GetCustomAttributes(typeof(ITestFrameworkAttribute)).FirstOrDefault();
if (testFrameworkAttr != null)
{
var discovererAttr = testFrameworkAttr.GetCustomAttributes(typeof(TestFrameworkDiscovererAttribute)).FirstOrDefault();
if (discovererAttr != null)
{
var discoverer = ExtensibilityPointFactory.GetTestFrameworkTypeDiscoverer(diagnosticMessageSink, discovererAttr);
if (discoverer != null)
{
return(discoverer.GetTestFrameworkType(testFrameworkAttr));
}
var ctorArgs = discovererAttr.GetConstructorArguments().ToArray();
diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Unable to create custom test framework discoverer type '{ctorArgs[1]}, {ctorArgs[0]}'"));
}
else
{
diagnosticMessageSink.OnMessage(new DiagnosticMessage("Assembly-level test framework attribute was not decorated with [TestFrameworkDiscoverer]"));
}
}
}
catch (Exception ex)
{
diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Exception thrown during test framework discoverer construction: {ex.Unwrap()}"));
}
return(typeof(XunitTestFramework));
}
protected override async Task <RunSummary> RunTestAsync()
{
var runSummary = new BenchmarkRunSummary
{
TestClassFullName = TestCase.TestMethod.TestClass.Class.Name,
TestClass = TestCase.TestMethod.TestClass.Class.Name.Split('.').Last(),
TestMethod = TestCase.TestMethodName,
Variation = TestCase.Variation,
ProductReportingVersion = BenchmarkConfig.Instance.ProductReportingVersion,
RunStarted = DateTime.UtcNow,
MachineName = _machineName,
Framework = _framework,
Architecture = IntPtr.Size > 4 ? "x64" : "x86",
WarmupIterations = TestCase.WarmupIterations,
Iterations = TestCase.Iterations,
CustomData = BenchmarkConfig.Instance.CustomData
};
for (var i = 0; i < TestCase.WarmupIterations; i++)
{
var runner = CreateRunner(i + 1, TestCase.WarmupIterations, TestCase.Variation, warmup: true);
runSummary.Aggregate(await runner.RunAsync());
}
for (var i = 0; i < TestCase.Iterations; i++)
{
TestCase.MetricCollector.Reset();
var runner = CreateRunner(i + 1, TestCase.Iterations, TestCase.Variation, warmup: false);
var iterationSummary = new BenchmarkIterationSummary();
iterationSummary.Aggregate(await runner.RunAsync(), TestCase.MetricCollector);
runSummary.Aggregate(iterationSummary);
}
if (runSummary.Failed != 0)
{
_diagnosticMessageSink.OnMessage(new XunitDiagnosticMessage($"No valid results for {TestCase.DisplayName}. {runSummary.Failed} of {TestCase.Iterations + TestCase.WarmupIterations} iterations failed."));
}
else
{
runSummary.PopulateMetrics();
_diagnosticMessageSink.OnMessage(new XunitDiagnosticMessage(runSummary.ToString()));
foreach (var database in BenchmarkConfig.Instance.ResultDatabases)
{
try
{
new SqlServerBenchmarkResultProcessor(database).SaveSummary(runSummary);
}
catch (Exception ex)
{
_diagnosticMessageSink.OnMessage(new XunitDiagnosticMessage($"Failed to save results to {database}{Environment.NewLine} {ex}"));
throw;
}
}
}
return(runSummary);
}
/// <summary>
/// Initializes a new instance of the <see cref="DependencyContextAssemblyCache" /> class.
/// </summary>
/// <param name="assemblyFolder">The assembly folder</param>
/// <param name="dependencyContext">The dependency context</param>
/// <param name="internalDiagnosticsMessageSink">The message sink</param>
public DependencyContextAssemblyCache(
[NotNull] string assemblyFolder,
[NotNull] DependencyContext dependencyContext,
[CanBeNull] IMessageSink internalDiagnosticsMessageSink)
{
_assemblyFolder = assemblyFolder;
_internalDiagnosticsMessageSink = internalDiagnosticsMessageSink;
_assemblyResolver = new XunitPackageCompilationAssemblyResolver(internalDiagnosticsMessageSink);
internalDiagnosticsMessageSink?.OnMessage(
new DiagnosticMessage(
$"[DependencyContextAssemblyCache..ctor] Runtime graph: [{string.Join(",", dependencyContext.RuntimeGraph.Select(x => $"'{x.Runtime}'"))}]"));
var currentRuntime = RuntimeEnvironment.GetRuntimeIdentifier();
var fallbacks = dependencyContext.RuntimeGraph.FirstOrDefault(x => string.Equals(x.Runtime, currentRuntime, StringComparison.OrdinalIgnoreCase));
HashSet <string> compatibleRuntimes = fallbacks != null
? new HashSet <string>(fallbacks.Fallbacks, StringComparer.OrdinalIgnoreCase)
: new HashSet <string>(StringComparer.OrdinalIgnoreCase);
compatibleRuntimes.Add(currentRuntime);
compatibleRuntimes.Add(string.Empty);
internalDiagnosticsMessageSink?.OnMessage(
new DiagnosticMessage(
$"[DependencyContextAssemblyCache..ctor] Compatible runtimes: [{string.Join(",", compatibleRuntimes.Select(x => $"'{x}'"))}]"));
_managedAssemblyCache = new Dictionary <string, Assembly>(StringComparer.OrdinalIgnoreCase);
_managedAssemblyMap =
dependencyContext.RuntimeLibraries
.Where(lib => lib.RuntimeAssemblyGroups?.Count > 0)
.Select(lib => Tuple.Create(lib, lib.RuntimeAssemblyGroups.FirstOrDefault(libGroup => compatibleRuntimes.Contains(libGroup.Runtime))))
.Where(tuple => tuple.Item2?.AssetPaths != null)
.SelectMany(
tuple => tuple.Item2.AssetPaths.Where(x => x != null)
.Select(path => Tuple.Create(Path.GetFileNameWithoutExtension(path), Tuple.Create(tuple.Item1, tuple.Item2))))
.ToDictionaryIgnoringDuplicateKeys(tuple => tuple.Item1, tuple => tuple.Item2, StringComparer.OrdinalIgnoreCase);
internalDiagnosticsMessageSink?.OnMessage(
new DiagnosticMessage(
$"[DependencyContextAssemblyCache..ctor] Managed assembly map includes: {string.Join(",", _managedAssemblyMap.Keys.Select(k => $"'{k}'").OrderBy(k => k, StringComparer.OrdinalIgnoreCase))}"));
#if NETCOREAPP1_0
unmanagedAssemblyCache = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
unmanagedAssemblyMap =
dependencyContext.RuntimeLibraries
.Where(lib => lib.NativeLibraryGroups?.Count > 0)
.Select(lib => Tuple.Create(lib, lib.NativeLibraryGroups.FirstOrDefault(libGroup => compatibleRuntimes.Contains(libGroup.Runtime))))
.Where(tuple => tuple.Item2?.AssetPaths != null)
.SelectMany(tuple => tuple.Item2.AssetPaths.Where(x => x != null).Select(path => Tuple.Create(Path.GetFileName(path), Tuple.Create(tuple.Item1, tuple.Item2))))
.ToDictionaryIgnoringDuplicateKeys(tuple => tuple.Item1, tuple => tuple.Item2, StringComparer.OrdinalIgnoreCase);
if (internalDiagnosticsMessageSink != null)
{
internalDiagnosticsMessageSink.OnMessage(new DiagnosticMessage($"[DependencyContextAssemblyCache..ctor] Unmanaged assembly map includes: {string.Join(",", unmanagedAssemblyMap.Keys.Select(k => $"'{k}'").OrderBy(k => k, StringComparer.OrdinalIgnoreCase))}"));
}
#endif
}
public DriverFixture(IMessageSink messageSink)
{
try
{
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
var testWebBrowser = Environment.GetEnvironmentVariable("TestWebBrowser") ?? "Chrome";
messageSink.OnMessage(new DiagnosticMessage($"TestWebBrowser = {testWebBrowser}"));
if (testWebBrowser == "Firefox")
{
var profiles = new FirefoxProfileManager();
var profile = profiles.GetProfile("selenium");
messageSink.OnMessage(new DiagnosticMessage("Profiles:"));
foreach (var existingProfile in profiles.ExistingProfiles)
{
messageSink.OnMessage(new DiagnosticMessage($"Profile = {existingProfile}"));
}
var options = new FirefoxOptions {
Profile = profile
};
options.SetLoggingPreference(LogType.Browser, LogLevel.All);
Driver = new FirefoxDriver(FirefoxDriverService.CreateDefaultService(Environment.CurrentDirectory),
new FirefoxOptions {
Profile = profile
},
TimeSpan.FromSeconds(30));
}
else if (testWebBrowser == "IE")
{
Driver = new InternetExplorerDriver(Environment.CurrentDirectory);
}
else if (testWebBrowser == "Chrome")
{
Driver = new ChromeDriver(Environment.CurrentDirectory);
}
Driver.Manage().Window.Position = new Point(0, 0);
Driver.Manage().Window.Size = new Size(1920, 1080);
Driver.Url = Environment.GetEnvironmentVariable("WebUrl") ?? "http://localhost:5000/";
messageSink.OnMessage(new DiagnosticMessage($"WebUrl = {Driver?.Url ?? "NULL"}"));
Driver.Manage().Timeouts().ImplicitWait = TimeSpan.FromSeconds(10);
Driver.Manage().Timeouts().AsynchronousJavaScript = TimeSpan.FromSeconds(10);
Driver.Manage().Timeouts().PageLoad = TimeSpan.FromSeconds(10);
Driver.FindElement(By.TagName("app-root"));
}
catch (Exception ex)
{
messageSink.OnMessage(new DiagnosticMessage($"Exception while initializing driver: {ex.Message}"));
Driver?.Dispose();
}
}
void RunTestsInAssembly(IRunContext runContext,
IFrameworkHandle frameworkHandle,
LoggerHelper logger,
IMessageSink reporterMessageHandler,
AssemblyRunInfo runInfo)
{
if (cancelled)
{
return;
}
var assembly = new XunitProjectAssembly {
AssemblyFilename = runInfo.AssemblyFileName
};
var assemblyFileName = runInfo.AssemblyFileName;
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName);
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var appDomain = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior;
if (DisableAppDomainRequestedInRunContext(runContext.RunSettings.SettingsXml))
{
appDomain = AppDomainSupport.Denied;
}
try
{
#if PLATFORM_DOTNET
// For AppX Apps, use the package location
assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName));
#endif
var diagnosticSink = new DiagnosticMessageSink(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault);
using (var controller = new XunitFrontController(appDomain, assemblyFileName: assemblyFileName, configFileName: null, shadowCopy: shadowCopy, diagnosticMessageSink: MessageSinkAdapter.Wrap(diagnosticSink)))
{
var xunitTestCases = runInfo.TestCases.Select(tc => new { vs = tc, xunit = Deserialize(logger, controller, tc) })
.Where(tc => tc.xunit != null)
.ToDictionary(tc => tc.xunit, tc => tc.vs);
var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration);
executionOptions.SetSynchronousMessageReporting(true);
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
using (var executionSink = new VsExecutionSink(frameworkHandle, logger, xunitTestCases, executionOptions, () => cancelled))
{
controller.RunTests(xunitTestCases.Keys.ToList(), executionSink, executionOptions);
executionSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, executionSink.ExecutionSummary));
}
}
}
catch (Exception ex)
{
logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex);
}
}
/// <summary>
/// Initializes a new instance of the <see cref="NetCoreAssemblyDependencyResolver"/> class.
/// </summary>
/// <param name="assemblyFilePath">The path to the assembly</param>
/// <param name="internalDiagnosticsMessageSink">An optional message sink for use with internal diagnostics messages;
/// may pass <c>null</c> for no internal diagnostics messages</param>
public NetCoreAssemblyDependencyResolver(string assemblyFilePath, IMessageSink internalDiagnosticsMessageSink)
{
this.internalDiagnosticsMessageSink = internalDiagnosticsMessageSink;
var assembly = LoadFromAssemblyPath(assemblyFilePath);
assemblyFolder = Path.GetDirectoryName(assemblyFilePath);
assemblyResolver = new XunitPackageCompilationAssemblyResolver();
dependencyContext = DependencyContext.Load(assembly);
var compatibleRuntimes = default(HashSet <string>);
var currentRuntime = RuntimeEnvironment.GetRuntimeIdentifier();
var fallbacks = dependencyContext.RuntimeGraph.FirstOrDefault(x => string.Equals(x.Runtime, currentRuntime, StringComparison.OrdinalIgnoreCase));
if (fallbacks != null)
{
compatibleRuntimes = new HashSet <string>(fallbacks.Fallbacks, StringComparer.OrdinalIgnoreCase);
}
else
{
compatibleRuntimes = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
}
compatibleRuntimes.Add(currentRuntime);
compatibleRuntimes.Add(string.Empty);
managedAssemblyCache = new Dictionary <string, Assembly>(StringComparer.OrdinalIgnoreCase);
managedAssemblyMap =
dependencyContext.RuntimeLibraries
.Where(lib => lib.RuntimeAssemblyGroups?.Count > 0)
.Select(lib => Tuple.Create(lib, lib.RuntimeAssemblyGroups.FirstOrDefault(libGroup => compatibleRuntimes.Contains(libGroup.Runtime))))
.Where(tuple => tuple.Item2?.AssetPaths != null)
.SelectMany(tuple => tuple.Item2.AssetPaths.Where(x => x != null).Select(path => Tuple.Create(Path.GetFileNameWithoutExtension(path), Tuple.Create(tuple.Item1, tuple.Item2))))
.ToDictionaryIgnoringDuplicateKeys(tuple => tuple.Item1, tuple => tuple.Item2, StringComparer.OrdinalIgnoreCase);
if (internalDiagnosticsMessageSink != null)
{
internalDiagnosticsMessageSink.OnMessage(new DiagnosticMessage($"[NetCoreAssemblyDependencyResolver..ctor] Managed assembly map includes: {string.Join(",", managedAssemblyMap.Keys.Select(k => $"'{k}'").OrderBy(k => k, StringComparer.OrdinalIgnoreCase))}"));
}
unmanagedAssemblyCache = new Dictionary <string, string>(StringComparer.OrdinalIgnoreCase);
unmanagedAssemblyMap =
dependencyContext.RuntimeLibraries
.Where(lib => lib.NativeLibraryGroups?.Count > 0)
.Select(lib => Tuple.Create(lib, lib.NativeLibraryGroups.FirstOrDefault(libGroup => compatibleRuntimes.Contains(libGroup.Runtime))))
.Where(tuple => tuple.Item2?.AssetPaths != null)
.SelectMany(tuple => tuple.Item2.AssetPaths.Where(x => x != null).Select(path => Tuple.Create(Path.GetFileName(path), Tuple.Create(tuple.Item1, tuple.Item2))))
.ToDictionaryIgnoringDuplicateKeys(tuple => tuple.Item1, tuple => tuple.Item2, StringComparer.OrdinalIgnoreCase);
if (internalDiagnosticsMessageSink != null)
{
internalDiagnosticsMessageSink.OnMessage(new DiagnosticMessage($"[NetCoreAssemblyDependencyResolver..ctor] Unmanaged assembly map includes: {string.Join(",", unmanagedAssemblyMap.Keys.Select(k => $"'{k}'").OrderBy(k => k, StringComparer.OrdinalIgnoreCase))}"));
}
Default.Resolving += OnResolving;
}
/// <summary>
/// Runs a retryable test case, handling any wait and retry logic between test runs, reporting statuses out to xunit etc...
/// </summary>
/// <param name="testCase">The test case to be retried</param>
/// <param name="diagnosticMessageSink">The diagnostic message sink to write messages to about retries, waits etc...</param>
/// <param name="messageBus">The message bus xunit is listening for statuses to report on</param>
/// <param name="cancellationTokenSource">The cancellation token source from xunit</param>
/// <param name="fnRunSingle">(async) Lambda to run this test case once (without retries) - takes the blocking message bus and returns the test run result</param>
/// <returns>Resulting run summary</returns>
public static async Task <RunSummary> RunAsync(
IRetryableTestCase testCase,
IMessageSink diagnosticMessageSink,
IMessageBus messageBus,
CancellationTokenSource cancellationTokenSource,
Func <IMessageBus, Task <RunSummary> > fnRunSingle)
{
for (var i = 1; ; i++)
{
// Prevent messages from the test run from being passed through, as we don't want
// a message to mark the test as failed when we're going to retry it
using BlockingMessageBus blockingMessageBus = new BlockingMessageBus(messageBus);
diagnosticMessageSink.OnMessage(new DiagnosticMessage("Running test \"{0}\" attempt ({1}/{2})",
testCase.DisplayName, i, testCase.MaxRetries));
blockingMessageBus.QueueMessage(new DiagnosticMessage("Running test \"{0}\" attempt ({1}/{2})",
testCase.DisplayName, i, testCase.MaxRetries));
RunSummary summary = await fnRunSingle(blockingMessageBus);
// If we succeeded, or we've reached the max retries return the result
if (summary.Failed == 0 || i == testCase.MaxRetries)
{
// If we have failed (after all retries, log that)
if (summary.Failed != 0)
{
diagnosticMessageSink.OnMessage(new DiagnosticMessage(
"Test \"{0}\" has failed and been retried the maximum number of times ({1})",
testCase.DisplayName, testCase.MaxRetries));
}
blockingMessageBus.Flush();
return(summary);
}
// Otherwise log that we've had a failed run and will retry
diagnosticMessageSink.OnMessage(new DiagnosticMessage(
"Test \"{0}\" failed but is set to retry ({1}/{2}) . . .", testCase.DisplayName, i,
testCase.MaxRetries));
// If there is a delay between test attempts, apply it now
if (testCase.DelayBetweenRetriesMs > 0)
{
diagnosticMessageSink.OnMessage(new DiagnosticMessage(
"Test \"{0}\" attempt ({1}/{2}) delayed by {3}ms. Waiting . . .", testCase.DisplayName, i,
testCase.MaxRetries, testCase.DelayBetweenRetriesMs));
// Don't await to prevent thread hopping.
// If all of a users test cases in a collection/class are synchronous and expecting to not thread-hop
// (because they're making use of thread static/thread local/managed thread ID to share data between tests rather than
// a more modern async-friendly mechanism) then if a thread-hop were to happen here we'd get flickering tests.
// SpecFlow relies on this as they use the managed thread ID to separate instances of some of their internal classes, which caused
// a this problem for xRetry.SpecFlow: https://github.com/JoshKeegan/xRetry/issues/18
Task.Delay(testCase.DelayBetweenRetriesMs, cancellationTokenSource.Token).Wait();
}
}
}
private void ReturnDiscoveryMessages(IMessageSink sink)
{
foreach (var testCase in DiscoveryTestCases)
{
sink.OnMessage(new TestCaseDiscoveryMessage {
TestCase = testCase
});
}
sink.OnMessage(new DiscoveryCompleteMessage());
}
public WinSqlite3TestFramework(IMessageSink messageSink)
: base(messageSink)
{
messageSink.OnMessage(new DiagnosticMessage {
Message = "Using custom test framework"
});
if (Environment.GetEnvironmentVariable("TEST_WINSQLITE3") != null)
{
messageSink.OnMessage(new DiagnosticMessage {
Message = "Using winsqlite3"
});
SqliteEngine.UseWinSqlite3();
}
}
/// <inheritdoc/>
public override bool OnMessage(IMessageSinkMessage message)
{
var result = base.OnMessage(message);
result = innerMessageSink.OnMessage(message) || result;
return(result);
}
public ServiceProviderFixture(IMessageSink diagnosticMessageSink)
{
this.TestId = Guid.NewGuid();
this.Configuration = new ConfigurationBuilder().
AddJsonFile("appsettings.test.json").
AddJsonFile("appsettings.test.Development.json", true).
// https://www.jerriepelser.com/blog/aspnet-core-no-more-worries-about-checking-in-secrets/
AddUserSecrets <ServiceProviderFixture>().
AddEnvironmentVariables("CTP_").
Build();
var containerType = Enum.Parse <ContainerType>(Configuration.GetValue("Container", "BuiltIn"));
diagnosticMessageSink.OnMessage(new DiagnosticMessage("Use container {0}", containerType.ToString()));
switch (containerType)
{
case ContainerType.BuiltIn:
var services = new ServiceCollection();
services.UseCommercetools(Configuration, "Client", TokenFlow.ClientCredentials);
services.AddLogging(c => c.AddProvider(new InMemoryLoggerProvider()));
this.ServiceProvider = services.BuildServiceProvider();
break;
case ContainerType.SimpleInjector:
var container = new Container();
container.UseCommercetools(Configuration, "Client", TokenFlow.ClientCredentials);
container.Verify();
this.ServiceProvider = container;
break;
}
}
// This method is called by the xUnit test framework classes to run the test case. We will do the
// loop here, forwarding on to the implementation in XunitTestCase to do the heavy lifting. We will
// continue to re-run the test until the aggregator has an error (meaning that some internal error
// condition happened), or the test runs without failure, or we've hit the maximum number of tries.
public override async Task <RunSummary> RunAsync(IMessageSink diagnosticMessageSink,
IMessageBus messageBus,
object[] constructorArguments,
ExceptionAggregator aggregator,
CancellationTokenSource cancellationTokenSource)
{
var runCount = 0;
while (true)
{
// This is really the only tricky bit: we need to capture and delay messages (since those will
// contain run status) until we know we've decided to accept the final result;
var delayedMessageBus = new DelayedMessageBus(messageBus);
var summary = await base.RunAsync(diagnosticMessageSink, delayedMessageBus, constructorArguments, aggregator, cancellationTokenSource);
if (aggregator.HasExceptions || summary.Failed == 0 || ++runCount >= maxRetries)
{
delayedMessageBus.Dispose(); // Sends all the delayed messages
return(summary);
}
diagnosticMessageSink.OnMessage(new DiagnosticMessage("Execution of '{0}' failed (attempt #{1}), retrying...", DisplayName, runCount));
var delayInMs = Math.Min((int)Math.Pow(2, runCount - 1) * this.exponentialBackoffMs, MaxDelayInMs);
await Task.Delay(delayInMs);
}
}
/// <inheritdoc/>
protected override void Initialize()
{
base.Initialize();
var factAttribute = TestMethod.Method.GetCustomAttributes(typeof(FactAttribute)).Single();
var baseDisplayName = factAttribute.GetNamedArgument <string>("DisplayName") ?? BaseDisplayName;
DisplayName = TypeUtility.GetDisplayNameWithArguments(TestMethod.Method, baseDisplayName, TestMethodArguments, MethodGenericTypes);
SkipReason = factAttribute.GetNamedArgument <string>("Skip");
foreach (var traitAttribute in TestMethod.Method.GetCustomAttributes(typeof(ITraitAttribute))
.Concat(TestMethod.TestClass.Class.GetCustomAttributes(typeof(ITraitAttribute))))
{
var discovererAttribute = traitAttribute.GetCustomAttributes(typeof(TraitDiscovererAttribute)).FirstOrDefault();
if (discovererAttribute != null)
{
var discoverer = ExtensibilityPointFactory.GetTraitDiscoverer(diagnosticMessageSink, discovererAttribute);
if (discoverer != null)
{
foreach (var keyValuePair in discoverer.GetTraits(traitAttribute))
{
Traits.Add(keyValuePair.Key, keyValuePair.Value);
}
}
}
else
{
diagnosticMessageSink.OnMessage(new DiagnosticMessage("Trait attribute on '{0}' did not have [TraitDiscoverer]", DisplayName));
}
}
}
/// <summary>
/// Gets the test collection definitions for the given assembly.
/// </summary>
/// <param name="assemblyInfo">The assembly.</param>
/// <param name="diagnosticMessageSink">The message sink used to send diagnostic messages</param>
/// <returns>A list of mappings from test collection name to test collection definitions (as <see cref="ITypeInfo"/></returns>
public static Dictionary<string, ITypeInfo> GetTestCollectionDefinitions(IAssemblyInfo assemblyInfo, IMessageSink diagnosticMessageSink)
{
var attributeTypesByName =
assemblyInfo.GetTypes(false)
.Select(type => new { Type = type, Attribute = type.GetCustomAttributes(typeof(CollectionDefinitionAttribute).AssemblyQualifiedName).FirstOrDefault() })
.Where(list => list.Attribute != null)
.GroupBy(list => (string)list.Attribute.GetConstructorArguments().Single(),
list => list.Type,
StringComparer.OrdinalIgnoreCase);
var result = new Dictionary<string, ITypeInfo>();
foreach (var grouping in attributeTypesByName)
{
var types = grouping.ToList();
result[grouping.Key] = types[0];
if (types.Count > 1)
diagnosticMessageSink.OnMessage(new DiagnosticMessage("Multiple test collections declared with name '{0}': {1}",
grouping.Key,
string.Join(", ", types.Select(type => type.Name))));
}
return result;
}
public ConditionalTestFramework(IMessageSink messageSink)
: base(messageSink)
{
messageSink.OnMessage(new DiagnosticMessage
{
Message = "Using " + nameof(ConditionalTestFramework)
});
}
/// <inheritdoc />
protected override bool RunTestsOnMethod(IMessageSink messageSink,
Type classUnderTest,
object[] constructorArguments,
MethodInfo methodUnderTest,
List<BeforeAfterTestAttribute> beforeAfterAttributes,
ExceptionAggregator aggregator,
ref decimal executionTime)
{
try
{
var testMethod = Reflector.Wrap(methodUnderTest);
var dataAttributes = testMethod.GetCustomAttributes(typeof(DataAttribute));
foreach (var dataAttribute in dataAttributes)
{
var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First();
var args = discovererAttribute.GetConstructorArguments().Cast<string>().ToList();
var discovererType = Reflector.GetType(args[0], args[1]);
IDataDiscoverer discoverer = (IDataDiscoverer)Activator.CreateInstance(discovererType);
foreach (object[] dataRow in discoverer.GetData(dataAttribute, testMethod))
if (RunTestWithArguments(messageSink, classUnderTest, constructorArguments, methodUnderTest, dataRow, GetDisplayNameWithArguments(DisplayName, dataRow), beforeAfterAttributes, aggregator, ref executionTime))
return true;
}
return false;
}
catch (Exception ex)
{
var cancelled = false;
if (!messageSink.OnMessage(new TestStarting { TestCase = this, TestDisplayName = DisplayName }))
cancelled = true;
else
{
if (!messageSink.OnMessage(new TestFailed(ex.Unwrap()) { TestCase = this, TestDisplayName = DisplayName }))
cancelled = true;
}
if (!messageSink.OnMessage(new TestFinished { TestCase = this, TestDisplayName = DisplayName }))
cancelled = true;
return cancelled;
}
}
/// <inheritdoc />
protected override void RunTestsOnMethod(IMessageSink messageSink,
Type classUnderTest,
object[] constructorArguments,
MethodInfo methodUnderTest,
List<BeforeAfterTestAttribute> beforeAfterAttributes,
ExceptionAggregator aggregator,
CancellationTokenSource cancellationTokenSource,
ref decimal executionTime)
{
try
{
var testMethod = Reflector.Wrap(methodUnderTest);
var dataAttributes = testMethod.GetCustomAttributes(typeof(DataAttribute));
foreach (var dataAttribute in dataAttributes)
{
var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First();
var args = discovererAttribute.GetConstructorArguments().Cast<string>().ToList();
var discovererType = Reflector.GetType(args[1], args[0]);
IDataDiscoverer discoverer = (IDataDiscoverer)Activator.CreateInstance(discovererType);
foreach (object[] dataRow in discoverer.GetData(dataAttribute, testMethod))
{
RunTestWithArguments(messageSink, classUnderTest, constructorArguments, methodUnderTest, dataRow, GetDisplayNameWithArguments(DisplayName, dataRow), beforeAfterAttributes, aggregator, cancellationTokenSource, ref executionTime);
if (cancellationTokenSource.IsCancellationRequested)
return;
}
}
}
catch (Exception ex)
{
if (!messageSink.OnMessage(new TestStarting(this, DisplayName)))
cancellationTokenSource.Cancel();
else
{
if (!messageSink.OnMessage(new TestFailed(this, DisplayName, executionTime, ex.Unwrap())))
cancellationTokenSource.Cancel();
}
if (!messageSink.OnMessage(new TestFinished(this, DisplayName, executionTime)))
cancellationTokenSource.Cancel();
}
}
/// <inheritdoc/>
public void Find(bool includeSourceInformation, IMessageSink messageSink)
{
Guard.ArgumentNotNull("messageSink", messageSink);
ThreadPool.QueueUserWorkItem(_ =>
{
foreach (var type in assemblyInfo.GetTypes(includePrivateTypes: false))
if (!FindImpl(type, includeSourceInformation, messageSink))
break;
messageSink.OnMessage(new DiscoveryCompleteMessage());
});
}
/// <inheritdoc/>
protected override void RunTests(IMessageSink messageSink, object[] constructorArguments, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource)
{
if (!messageSink.OnMessage(new TestStarting(this, DisplayName)))
cancellationTokenSource.Cancel();
else
{
try
{
lambda();
if (!messageSink.OnMessage(new TestPassed(this, DisplayName, 0)))
cancellationTokenSource.Cancel();
}
catch (Exception ex)
{
if (!messageSink.OnMessage(new TestFailed(this, DisplayName, 0, ex)))
cancellationTokenSource.Cancel();
}
}
if (!messageSink.OnMessage(new TestFinished(this, DisplayName, 0)))
cancellationTokenSource.Cancel();
}
/// <inheritdoc/>
public void Find(string typeName, bool includeSourceInformation, IMessageSink messageSink)
{
Guard.ArgumentNotNullOrEmpty("typeName", typeName);
Guard.ArgumentNotNull("messageSink", messageSink);
ThreadPool.QueueUserWorkItem(_ =>
{
ITypeInfo typeInfo = assemblyInfo.GetType(typeName);
if (typeInfo != null)
FindImpl(typeInfo, includeSourceInformation, messageSink);
messageSink.OnMessage(new DiscoveryCompleteMessage());
});
}
/// <inheritdoc/>
public async void Run(IEnumerable<ITestCase> testCases, IMessageSink messageSink)
{
var cancellationTokenSource = new CancellationTokenSource();
var totalSummary = new RunSummary();
string currentDirectory = Directory.GetCurrentDirectory();
try
{
Directory.SetCurrentDirectory(Path.GetDirectoryName(assemblyInfo.AssemblyPath));
if (messageSink.OnMessage(new TestAssemblyStarting(assemblyFileName, AppDomain.CurrentDomain.SetupInformation.ConfigurationFile, DateTime.Now,
String.Format("{0}-bit .NET {1}", IntPtr.Size * 8, Environment.Version),
XunitTestFrameworkDiscoverer.DisplayName)))
{
// TODO: Contract for Run() states that null "testCases" means "run everything".
var tasks =
testCases.Cast<XunitTestCase>()
.GroupBy(tc => tc.TestCollection)
.Select(collectionGroup => Task.Run(() => RunTestCollection(messageSink, collectionGroup.Key, collectionGroup, cancellationTokenSource)))
.ToArray();
var summaries = await Task.WhenAll(tasks);
totalSummary.Time = summaries.Sum(s => s.Time);
totalSummary.Total = summaries.Sum(s => s.Total);
totalSummary.Failed = summaries.Sum(s => s.Failed);
totalSummary.Skipped = summaries.Sum(s => s.Skipped);
}
messageSink.OnMessage(new TestAssemblyFinished(assemblyInfo, totalSummary.Time, totalSummary.Total, totalSummary.Failed, totalSummary.Skipped));
}
finally
{
Directory.SetCurrentDirectory(currentDirectory);
}
}
static ITestFramework CreateInnerTestFramework(Type testFrameworkType, IMessageSink diagnosticMessageSink)
{
try
{
var ctorWithSink = testFrameworkType.GetTypeInfo().DeclaredConstructors
.FirstOrDefault(ctor =>
{
var paramInfos = ctor.GetParameters();
return paramInfos.Length == 1 && paramInfos[0].ParameterType == typeof(IMessageSink);
});
if (ctorWithSink != null)
return (ITestFramework)ctorWithSink.Invoke(new object[] { diagnosticMessageSink });
return (ITestFramework)Activator.CreateInstance(testFrameworkType);
}
catch (Exception ex)
{
diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Exception thrown during test framework construction: {ex.Unwrap()}"));
return new XunitTestFramework(diagnosticMessageSink);
}
}
// This method is called by the xUnit test framework classes to run the test case. We will do the
// loop here, forwarding on to the implementation in XunitTestCase to do the heavy lifting. We will
// continue to re-run the test until the aggregator has an error (meaning that some internal error
// condition happened), or the test runs without failure, or we've hit the maximum number of tries.
public override async Task<RunSummary> RunAsync(IMessageSink diagnosticMessageSink,
IMessageBus messageBus,
object[] constructorArguments,
ExceptionAggregator aggregator,
CancellationTokenSource cancellationTokenSource)
{
var runCount = 0;
while (true)
{
// This is really the only tricky bit: we need to capture and delay messages (since those will
// contain run status) until we know we've decided to accept the final result;
var delayedMessageBus = new DelayedMessageBus(messageBus);
var summary = await base.RunAsync(diagnosticMessageSink, delayedMessageBus, constructorArguments, aggregator, cancellationTokenSource);
if (aggregator.HasExceptions || summary.Failed == 0 || ++runCount >= maxRetries)
{
delayedMessageBus.Dispose(); // Sends all the delayed messages
return summary;
}
diagnosticMessageSink.OnMessage(new DiagnosticMessage("Execution of '{0}' failed (attempt #{1}), retrying...", DisplayName, runCount));
}
}
RunSummary RunTestClass(ITestCollection testCollection, string className, IList<Xunit1TestCase> testCases, IMessageSink messageSink)
{
var handler = new TestClassCallbackHandler(testCases, messageSink);
var results = handler.TestClassResults;
results.Continue = messageSink.OnMessage(new TestClassStarting(testCollection, className));
if (results.Continue)
{
try
{
var methodNames = testCases.Select(tc => tc.Method.Name).ToList();
executor.RunTests(className, methodNames, handler);
handler.LastNodeArrived.WaitOne();
}
catch (Exception ex)
{
var failureInformation = Xunit1ExceptionUtility.ConvertToFailureInformation(ex);
results.Continue = messageSink.OnMessage(new ErrorMessage(failureInformation.ExceptionTypes, failureInformation.Messages, failureInformation.StackTraces, failureInformation.ExceptionParentIndices)) && results.Continue;
}
}
results.Continue = messageSink.OnMessage(new TestClassFinished(testCollection, className, results.Time, results.Total, results.Failed, results.Skipped)) && results.Continue;
return results;
}
public override bool Execute()
{
RemotingUtility.CleanUpRegisteredChannels();
if (TeamCity)
{
Log.LogError("The 'TeamCity' property is deprecated. Please set the 'Reporter' property to 'teamcity' instead.");
return false;
}
if (Verbose)
{
Log.LogError("The 'Verbose' property is deprecated. Please set the 'Reporter' property to 'verbose' instead.");
return false;
}
XElement assembliesElement = null;
var environment = string.Format("{0}-bit .NET {1}", IntPtr.Size * 8, Environment.Version);
if (NeedsXml)
assembliesElement = new XElement("assemblies");
switch (MaxParallelThreads)
{
case null:
case "default":
break;
case "unlimited":
maxThreadCount = 0;
break;
default:
int threadValue;
if (!int.TryParse(MaxParallelThreads, out threadValue) || threadValue < 0)
{
Log.LogError("MaxParallelThreads value '{0}' is invalid: must be 'default', 'unlimited', or a positive number", MaxParallelThreads);
return false;
}
maxThreadCount = threadValue;
break;
}
var originalWorkingFolder = Directory.GetCurrentDirectory();
using (AssemblyHelper.SubscribeResolve())
{
var reporters = GetAvailableRunnerReporters();
var reporter = reporters.FirstOrDefault(r => r.IsEnvironmentallyEnabled);
if (reporter == null && !string.IsNullOrWhiteSpace(Reporter))
{
reporter = reporters.FirstOrDefault(r => string.Equals(r.RunnerSwitch, Reporter, StringComparison.OrdinalIgnoreCase));
if (reporter == null)
{
var switchableReporters = reporters.Where(r => !string.IsNullOrWhiteSpace(r.RunnerSwitch)).Select(r => r.RunnerSwitch.ToLowerInvariant()).OrderBy(x => x).ToList();
if (switchableReporters.Count == 0)
Log.LogError("Reporter value '{0}' is invalid. There are no available reporters.", Reporter);
else
Log.LogError("Reporter value '{0}' is invalid. Available reporters: {1}", Reporter, string.Join(", ", switchableReporters));
return false;
}
}
if (reporter == null)
reporter = new DefaultRunnerReporter();
logger = new MSBuildLogger(Log);
reporterMessageHandler = reporter.CreateMessageHandler(logger);
if (!NoLogo)
Log.LogMessage(MessageImportance.High, "xUnit.net MSBuild Runner ({0})", environment);
var project = new XunitProject();
foreach (var assembly in Assemblies)
{
var assemblyFileName = assembly.GetMetadata("FullPath");
var configFileName = assembly.GetMetadata("ConfigFile");
if (configFileName != null && configFileName.Length == 0)
configFileName = null;
project.Add(new XunitProjectAssembly { AssemblyFilename = assemblyFileName, ConfigFilename = configFileName, ShadowCopy = ShadowCopy });
}
if (WorkingFolder != null)
Directory.SetCurrentDirectory(WorkingFolder);
var clockTime = Stopwatch.StartNew();
if (!parallelizeAssemblies.HasValue)
parallelizeAssemblies = project.All(assembly => assembly.Configuration.ParallelizeAssemblyOrDefault);
if (parallelizeAssemblies.GetValueOrDefault())
{
var tasks = project.Assemblies.Select(assembly => Task.Run(() => ExecuteAssembly(assembly)));
var results = Task.WhenAll(tasks).GetAwaiter().GetResult();
foreach (var assemblyElement in results.Where(result => result != null))
assembliesElement.Add(assemblyElement);
}
else
{
foreach (var assembly in project.Assemblies)
{
var assemblyElement = ExecuteAssembly(assembly);
if (assemblyElement != null)
assembliesElement.Add(assemblyElement);
}
}
clockTime.Stop();
if (completionMessages.Count > 0)
reporterMessageHandler.OnMessage(new TestExecutionSummary(clockTime.Elapsed, completionMessages.OrderBy(kvp => kvp.Key).ToList()));
}
Directory.SetCurrentDirectory(WorkingFolder ?? originalWorkingFolder);
if (NeedsXml)
{
if (Xml != null)
assembliesElement.Save(Xml.GetMetadata("FullPath"));
if (XmlV1 != null)
Transform("xUnit1.xslt", assembliesElement, XmlV1);
if (Html != null)
Transform("HTML.xslt", assembliesElement, Html);
if (NUnit != null)
Transform("NUnitXml.xslt", assembliesElement, NUnit);
}
return ExitCode == 0;
}
/// <inheritdoc/>
public void Find(bool includeSourceInformation, IMessageSink messageSink)
{
Guard.ArgumentNotNull("messageSink", messageSink);
ThreadPool.QueueUserWorkItem(_ =>
{
foreach (var type in assemblyInfo.GetTypes(includePrivateTypes: false))
if (!FindImpl(type, includeSourceInformation, messageSink))
break;
var warnings = messageAggregator.GetAndClear<EnvironmentalWarning>().Select(w => w.Message).ToList();
messageSink.OnMessage(new DiscoveryCompleteMessage(warnings));
});
}
/// <summary>
/// Runs a single test for a given test method.
/// </summary>
/// <param name="messageSink">The message sink to send results to.</param>
/// <param name="classUnderTest">The class under test.</param>
/// <param name="constructorArguments">The arguments to pass to the constructor.</param>
/// <param name="methodUnderTest">The method under test.</param>
/// <param name="testMethodArguments">The arguments to pass to the test method.</param>
/// <param name="displayName">The display name for the test.</param>
/// <param name="beforeAfterAttributes">The <see cref="BeforeAfterTestAttribute"/> instances attached to the test.</param>
/// <param name="aggregator">The error aggregator to use for catching exception.</param>
/// <param name="executionTime">The time spent executing the tests.</param>
protected bool RunTestWithArguments(IMessageSink messageSink,
Type classUnderTest,
object[] constructorArguments,
MethodInfo methodUnderTest,
object[] testMethodArguments,
string displayName,
List<BeforeAfterTestAttribute> beforeAfterAttributes,
ExceptionAggregator aggregator,
ref decimal executionTime)
{
bool cancelled = false;
if (!messageSink.OnMessage(new TestStarting { TestCase = this, TestDisplayName = displayName }))
cancelled = true;
else
{
if (!String.IsNullOrEmpty(SkipReason))
{
if (!messageSink.OnMessage(new TestSkipped { TestCase = this, TestDisplayName = DisplayName, Reason = SkipReason }))
cancelled = true;
}
else
{
var beforeAttributesRun = new Stack<BeforeAfterTestAttribute>();
var stopwatch = Stopwatch.StartNew();
if (!aggregator.HasExceptions)
{
aggregator.Run(() =>
{
object testClass = null;
if (!methodUnderTest.IsStatic)
{
if (!messageSink.OnMessage(new TestClassConstructionStarting { TestCase = this, TestDisplayName = displayName }))
cancelled = true;
try
{
if (!cancelled)
testClass = Activator.CreateInstance(classUnderTest, constructorArguments);
}
finally
{
if (!messageSink.OnMessage(new TestClassConstructionFinished { TestCase = this, TestDisplayName = displayName }))
cancelled = true;
}
}
if (!cancelled)
{
aggregator.Run(() =>
{
foreach (var beforeAfterAttribute in beforeAfterAttributes)
{
if (!messageSink.OnMessage(new BeforeTestStarting { TestCase = this, TestDisplayName = displayName, AttributeName = beforeAfterAttribute.GetType().Name }))
cancelled = true;
else
{
try
{
beforeAfterAttribute.Before(methodUnderTest);
beforeAttributesRun.Push(beforeAfterAttribute);
}
finally
{
if (!messageSink.OnMessage(new BeforeTestFinished { TestCase = this, TestDisplayName = displayName, AttributeName = beforeAfterAttribute.GetType().Name }))
cancelled = true;
}
}
if (cancelled)
return;
}
if (!cancelled)
{
var parameterTypes = methodUnderTest.GetParameters().Select(p => p.ParameterType).ToArray();
aggregator.Run(() =>
{
var result = methodUnderTest.Invoke(testClass, ConvertArguments(testMethodArguments ?? EmptyArray, parameterTypes));
var task = result as Task;
if (task != null)
task.GetAwaiter().GetResult();
});
}
});
foreach (var beforeAfterAttribute in beforeAttributesRun)
{
if (!messageSink.OnMessage(new AfterTestStarting { TestCase = this, TestDisplayName = displayName, AttributeName = beforeAfterAttribute.GetType().Name }))
cancelled = true;
aggregator.Run(() => beforeAfterAttribute.After(methodUnderTest));
if (!messageSink.OnMessage(new AfterTestFinished { TestCase = this, TestDisplayName = displayName, AttributeName = beforeAfterAttribute.GetType().Name }))
cancelled = true;
}
}
aggregator.Run(() =>
{
IDisposable disposable = testClass as IDisposable;
if (disposable != null)
{
if (!messageSink.OnMessage(new TestClassDisposeStarting { TestCase = this, TestDisplayName = displayName }))
cancelled = true;
try
{
disposable.Dispose();
}
finally
{
if (!messageSink.OnMessage(new TestClassDisposeFinished { TestCase = this, TestDisplayName = displayName }))
cancelled = true;
}
}
});
});
}
stopwatch.Stop();
if (!cancelled)
{
executionTime = (decimal)stopwatch.Elapsed.TotalSeconds;
var exception = aggregator.ToException();
var testResult = exception == null ? (TestResultMessage)new TestPassed() : new TestFailed(exception);
testResult.TestCase = this;
testResult.TestDisplayName = displayName;
testResult.ExecutionTime = executionTime;
if (!messageSink.OnMessage(testResult))
cancelled = true;
}
}
}
if (!messageSink.OnMessage(new TestFinished { TestCase = this, TestDisplayName = displayName, ExecutionTime = executionTime }))
cancelled = true;
return cancelled;
}
private static bool RunTestClass(IMessageSink messageSink, IGrouping<ITypeInfo, XunitTestCase> group, RunSummary classSummary)
{
bool cancelled = false;
var aggregator = new ExceptionAggregator();
Type testClassType = ((IReflectionTypeInfo)group.Key).Type;
Dictionary<Type, object> fixtureMappings = new Dictionary<Type, object>();
List<object> constructorArguments = new List<object>();
// TODO: Read class fixtures from test collection
foreach (var iface in testClassType.GetInterfaces().Where(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IClassFixture<>)))
{
Type fixtureType = iface.GetGenericArguments().Single();
object fixture = null;
aggregator.Run(() => fixture = Activator.CreateInstance(fixtureType));
fixtureMappings.Add(fixtureType, fixture);
}
var ctors = testClassType.GetConstructors();
if (ctors.Length != 1)
{
aggregator.Add(new TestClassException("A test class may only define a single public constructor."));
}
else
{
var ctor = ctors.Single();
List<string> unusedArguments = new List<string>();
foreach (var paramInfo in ctor.GetParameters())
{
object fixture;
if (fixtureMappings.TryGetValue(paramInfo.ParameterType, out fixture))
constructorArguments.Add(fixture);
else
unusedArguments.Add(paramInfo.ParameterType.Name + " " + paramInfo.Name);
}
if (unusedArguments.Count > 0)
aggregator.Add(new TestClassException("The following constructor arguments did not have matching fixture data: " + String.Join(", ", unusedArguments)));
}
var methodGroups = group.GroupBy(tc => tc.Method);
foreach (var method in methodGroups)
{
if (!messageSink.OnMessage(new TestMethodStarting { ClassName = group.Key.Name, MethodName = method.Key.Name }))
cancelled = true;
else
cancelled = RunTestMethod(messageSink, constructorArguments.ToArray(), method, classSummary, aggregator);
if (!messageSink.OnMessage(new TestMethodFinished { ClassName = group.Key.Name, MethodName = method.Key.Name }))
cancelled = true;
if (cancelled)
break;
}
foreach (var fixture in fixtureMappings.Values.OfType<IDisposable>())
{
try
{
fixture.Dispose();
}
catch (Exception ex)
{
if (!messageSink.OnMessage(new ErrorMessage(ex.Unwrap())))
cancelled = true;
}
}
return cancelled;
}
private void ReturnDiscoveryMessages(IMessageSink sink)
{
foreach (var testCase in DiscoveryTestCases)
sink.OnMessage(new TestCaseDiscoveryMessage { TestCase = testCase });
sink.OnMessage(new DiscoveryCompleteMessage());
}
/// <summary>
/// Core implementation to discover unit tests in a given test class.
/// </summary>
/// <param name="type">The test class.</param>
/// <param name="includeSourceInformation">Set to <c>true</c> to attempt to include source information.</param>
/// <param name="messageSink">The message sink to send discovery messages to.</param>
/// <returns>Returns <c>true</c> if discovery should continue; <c>false</c> otherwise.</returns>
protected virtual bool FindImpl(ITypeInfo type, bool includeSourceInformation, IMessageSink messageSink)
{
string currentDirectory = Directory.GetCurrentDirectory();
var testCollection = TestCollectionFactory.Get(type);
try
{
if (!String.IsNullOrEmpty(assemblyInfo.AssemblyPath))
Directory.SetCurrentDirectory(Path.GetDirectoryName(assemblyInfo.AssemblyPath));
foreach (IMethodInfo method in type.GetMethods(includePrivateMethods: true))
{
IAttributeInfo factAttribute = method.GetCustomAttributes(typeof(FactAttribute)).FirstOrDefault();
if (factAttribute != null)
{
IAttributeInfo discovererAttribute = factAttribute.GetCustomAttributes(typeof(XunitDiscovererAttribute)).FirstOrDefault();
if (discovererAttribute != null)
{
var args = discovererAttribute.GetConstructorArguments().Cast<string>().ToList();
var discovererType = Reflector.GetType(args[1], args[0]);
if (discovererType != null)
{
IXunitDiscoverer discoverer = (IXunitDiscoverer)Activator.CreateInstance(discovererType);
foreach (XunitTestCase testCase in discoverer.Discover(testCollection, assemblyInfo, type, method, factAttribute))
if (!messageSink.OnMessage(new TestCaseDiscoveryMessage(UpdateTestCaseWithSourceInfo(testCase, includeSourceInformation))))
return false;
}
// TODO: Figure out a way to report back an error when discovererType is not available
// TODO: What if the discovererType can't be created or cast to IXunitDiscoverer?
// TODO: Performance optimization: cache instances of the discoverer type
}
}
}
return true;
}
finally
{
Directory.SetCurrentDirectory(currentDirectory);
}
}
void Find(Predicate<ITestCaseDiscoveryMessage> filter, bool includeSourceInformation, IMessageSink messageSink)
{
try
{
XmlNode assemblyXml = null;
using (var handler = new XmlNodeCallbackHandler(xml => { assemblyXml = xml; return true; }))
executor.EnumerateTests(handler);
foreach (XmlNode method in assemblyXml.SelectNodes("//method"))
{
var testCase = method.ToTestCase(assemblyFileName);
if (testCase != null)
{
if (includeSourceInformation)
testCase.SourceInformation = sourceInformationProvider.GetSourceInformation(testCase);
testCase.TestCollection = testCollection;
var message = new TestCaseDiscoveryMessage(testCase);
if (filter(message))
messageSink.OnMessage(message);
}
}
}
finally
{
messageSink.OnMessage(new DiscoveryCompleteMessage(new string[0]));
}
}
private RunSummary RunTestCollection(IMessageSink messageSink, ITestCollection collection, IEnumerable<XunitTestCase> testCases, CancellationTokenSource cancellationTokenSource)
{
var collectionSummary = new RunSummary();
var collectionFixtureMappings = new Dictionary<Type, object>();
var aggregator = new ExceptionAggregator();
if (collection.CollectionDefinition != null)
{
var declarationType = ((IReflectionTypeInfo)collection.CollectionDefinition).Type;
foreach (var interfaceType in declarationType.GetInterfaces().Where(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(ICollectionFixture<>)))
CreateFixture(interfaceType, aggregator, collectionFixtureMappings);
}
if (messageSink.OnMessage(new TestCollectionStarting(collection)))
{
foreach (var testCasesByClass in testCases.GroupBy(tc => tc.Class))
{
var classSummary = new RunSummary();
if (!messageSink.OnMessage(new TestClassStarting(collection, testCasesByClass.Key.Name)))
cancellationTokenSource.Cancel();
else
{
RunTestClass(messageSink, collection, collectionFixtureMappings, (IReflectionTypeInfo)testCasesByClass.Key, testCasesByClass, classSummary, aggregator, cancellationTokenSource);
collectionSummary.Aggregate(classSummary);
}
if (!messageSink.OnMessage(new TestClassFinished(collection, testCasesByClass.Key.Name, classSummary.Time, classSummary.Total, classSummary.Failed, classSummary.Skipped)))
cancellationTokenSource.Cancel();
if (cancellationTokenSource.IsCancellationRequested)
break;
}
}
foreach (var fixture in collectionFixtureMappings.Values.OfType<IDisposable>())
{
try
{
fixture.Dispose();
}
catch (Exception ex)
{
if (!messageSink.OnMessage(new ErrorMessage(ex.Unwrap())))
cancellationTokenSource.Cancel();
}
}
messageSink.OnMessage(new TestCollectionFinished(collection, collectionSummary.Time, collectionSummary.Total, collectionSummary.Failed, collectionSummary.Skipped));
return collectionSummary;
}
protected static bool OnMessage(IMessageSink messageSink, IMessageSinkMessage message)
{
var result = messageSink.OnMessage(message);
RemotingServices.Disconnect((MarshalByRefObject)message);
return result;
}
/// <summary>
/// Runs a single test for a given test method.
/// </summary>
/// <param name="messageSink">The message sink to send results to.</param>
/// <param name="classUnderTest">The class under test.</param>
/// <param name="constructorArguments">The arguments to pass to the constructor.</param>
/// <param name="methodUnderTest">The method under test.</param>
/// <param name="testMethodArguments">The arguments to pass to the test method.</param>
/// <param name="displayName">The display name for the test.</param>
/// <param name="beforeAfterAttributes">The <see cref="BeforeAfterTestAttribute"/> instances attached to the test.</param>
/// <param name="parentAggregator">The parent aggregator that contains the exceptions up to this point.</param>
/// <param name="cancellationTokenSource">The cancellation token source that indicates whether cancellation has been requested.</param>
/// <param name="executionTime">The time spent executing the tests.</param>
protected void RunTestWithArguments(IMessageSink messageSink,
Type classUnderTest,
object[] constructorArguments,
MethodInfo methodUnderTest,
object[] testMethodArguments,
string displayName,
List<BeforeAfterTestAttribute> beforeAfterAttributes,
ExceptionAggregator parentAggregator,
CancellationTokenSource cancellationTokenSource,
ref decimal executionTime)
{
var aggregator = new ExceptionAggregator(parentAggregator);
if (!messageSink.OnMessage(new TestStarting(this, displayName)))
cancellationTokenSource.Cancel();
else
{
if (!String.IsNullOrEmpty(SkipReason))
{
if (!messageSink.OnMessage(new TestSkipped(this, displayName, SkipReason)))
cancellationTokenSource.Cancel();
}
else
{
var beforeAttributesRun = new Stack<BeforeAfterTestAttribute>();
var stopwatch = Stopwatch.StartNew();
if (!aggregator.HasExceptions)
aggregator.Run(() =>
{
object testClass = null;
if (!methodUnderTest.IsStatic)
{
if (!messageSink.OnMessage(new TestClassConstructionStarting(this, displayName)))
cancellationTokenSource.Cancel();
try
{
if (!cancellationTokenSource.IsCancellationRequested)
testClass = Activator.CreateInstance(classUnderTest, constructorArguments);
}
finally
{
if (!messageSink.OnMessage(new TestClassConstructionFinished(this, displayName)))
cancellationTokenSource.Cancel();
}
}
if (!cancellationTokenSource.IsCancellationRequested)
{
aggregator.Run(() =>
{
foreach (var beforeAfterAttribute in beforeAfterAttributes)
{
var attributeName = beforeAfterAttribute.GetType().Name;
if (!messageSink.OnMessage(new BeforeTestStarting(this, displayName, attributeName)))
cancellationTokenSource.Cancel();
else
{
try
{
beforeAfterAttribute.Before(methodUnderTest);
beforeAttributesRun.Push(beforeAfterAttribute);
}
finally
{
if (!messageSink.OnMessage(new BeforeTestFinished(this, displayName, attributeName)))
cancellationTokenSource.Cancel();
}
}
if (cancellationTokenSource.IsCancellationRequested)
return;
}
if (!cancellationTokenSource.IsCancellationRequested)
{
var parameterTypes = methodUnderTest.GetParameters().Select(p => p.ParameterType).ToArray();
var oldSyncContext = SynchronizationContext.Current;
try
{
var asyncSyncContext = new AsyncTestSyncContext();
SetSynchronizationContext(asyncSyncContext);
aggregator.Run(() =>
{
var result = methodUnderTest.Invoke(testClass, ConvertArguments(testMethodArguments ?? EmptyArray, parameterTypes));
var task = result as Task;
if (task != null)
task.GetAwaiter().GetResult();
else
{
var ex = asyncSyncContext.WaitForCompletion();
if (ex != null)
aggregator.Add(ex);
}
});
}
finally
{
SetSynchronizationContext(oldSyncContext);
}
}
});
foreach (var beforeAfterAttribute in beforeAttributesRun)
{
var attributeName = beforeAfterAttribute.GetType().Name;
if (!messageSink.OnMessage(new AfterTestStarting(this, displayName, attributeName)))
cancellationTokenSource.Cancel();
aggregator.Run(() => beforeAfterAttribute.After(methodUnderTest));
if (!messageSink.OnMessage(new AfterTestFinished(this, displayName, attributeName)))
cancellationTokenSource.Cancel();
}
}
aggregator.Run(() =>
{
IDisposable disposable = testClass as IDisposable;
if (disposable != null)
{
if (!messageSink.OnMessage(new TestClassDisposeStarting(this, displayName)))
cancellationTokenSource.Cancel();
try
{
disposable.Dispose();
}
finally
{
if (!messageSink.OnMessage(new TestClassDisposeFinished(this, displayName)))
cancellationTokenSource.Cancel();
}
}
});
});
stopwatch.Stop();
if (!cancellationTokenSource.IsCancellationRequested)
{
executionTime = (decimal)stopwatch.Elapsed.TotalSeconds;
var exception = aggregator.ToException();
var testResult = exception == null ? (TestResultMessage)new TestPassed(this, displayName, executionTime) : new TestFailed(this, displayName, executionTime, exception);
if (!messageSink.OnMessage(testResult))
cancellationTokenSource.Cancel();
}
}
}
if (!messageSink.OnMessage(new TestFinished(this, displayName, executionTime)))
cancellationTokenSource.Cancel();
}
/// <summary>
/// Executes the test case, returning 0 or more result messages through the message sink.
/// </summary>
/// <param name="messageSink">The message sink to report results to.</param>
/// <param name="constructorArguments">The arguments to pass to the constructor.</param>
/// <param name="aggregator">The error aggregator to use for catching exception.</param>
/// <param name="cancellationTokenSource">The cancellation token source that indicates whether cancellation has been requested.</param>
public virtual void Run(IMessageSink messageSink, object[] constructorArguments, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource)
{
int totalFailed = 0;
int totalRun = 0;
int totalSkipped = 0;
decimal executionTime = 0M;
if (!messageSink.OnMessage(new TestCaseStarting(this)))
cancellationTokenSource.Cancel();
else
{
var delegatingSink = new DelegatingMessageSink(messageSink, msg =>
{
if (msg is ITestResultMessage)
{
totalRun++;
executionTime += ((ITestResultMessage)msg).ExecutionTime;
}
if (msg is ITestFailed)
totalFailed++;
if (msg is ITestSkipped)
totalSkipped++;
});
RunTests(delegatingSink, constructorArguments, aggregator, cancellationTokenSource);
}
if (!messageSink.OnMessage(new TestCaseFinished(this, executionTime, totalRun, totalFailed, totalSkipped)))
cancellationTokenSource.Cancel();
}
static Type GetTestFrameworkType(IAssemblyInfo testAssembly, IMessageSink diagnosticMessageSink)
{
try
{
var testFrameworkAttr = testAssembly.GetCustomAttributes(typeof(ITestFrameworkAttribute)).FirstOrDefault();
if (testFrameworkAttr != null)
{
var discovererAttr = testFrameworkAttr.GetCustomAttributes(typeof(TestFrameworkDiscovererAttribute)).FirstOrDefault();
if (discovererAttr != null)
{
var discoverer = ExtensibilityPointFactory.GetTestFrameworkTypeDiscoverer(diagnosticMessageSink, discovererAttr);
if (discoverer != null)
return discoverer.GetTestFrameworkType(testFrameworkAttr);
var ctorArgs = discovererAttr.GetConstructorArguments().ToArray();
diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Unable to create custom test framework discoverer type '{ctorArgs[1]}, {ctorArgs[0]}'"));
}
else
{
diagnosticMessageSink.OnMessage(new DiagnosticMessage("Assembly-level test framework attribute was not decorated with [TestFrameworkDiscoverer]"));
}
}
}
catch (Exception ex)
{
diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Exception thrown during test framework discoverer construction: {ex.Unwrap()}"));
}
return typeof(XunitTestFramework);
}
RunSummary RunTestCollection(ITestCollection testCollection, IEnumerable<Xunit1TestCase> testCases, IMessageSink messageSink)
{
var results = new RunSummary();
results.Continue = messageSink.OnMessage(new TestCollectionStarting(testCollection));
if (results.Continue)
foreach (var testClassGroup in testCases.GroupBy(tc => tc.Class.Name))
{
var classResults = RunTestClass(testCollection, testClassGroup.Key, testClassGroup.ToList(), messageSink);
results.Aggregate(classResults);
if (!classResults.Continue)
break;
}
results.Continue = messageSink.OnMessage(new TestCollectionFinished(testCollection, results.Time, results.Total, results.Failed, results.Skipped)) && results.Continue;
return results;
}
RunSummary RunTestClass(ITestCollection testCollection, string className, IList<Xunit1TestCase> testCases, IMessageSink messageSink)
{
var handler = new TestClassCallbackHandler(testCases, messageSink);
var results = handler.TestClassResults;
results.Continue = messageSink.OnMessage(new TestClassStarting(testCollection, className));
if (results.Continue)
{
try
{
var methodNames = testCases.Select(tc => tc.Method.Name).ToList();
executor.RunTests(className, methodNames, handler);
handler.LastNodeArrived.WaitOne();
}
catch (Exception ex)
{
var stackTrace = ex.StackTrace;
var rethrowIndex = stackTrace.IndexOf("$$RethrowMarker$$");
if (rethrowIndex > -1)
stackTrace = stackTrace.Substring(0, rethrowIndex);
results.Continue = messageSink.OnMessage(new ErrorMessage(ex.GetType().FullName, ex.Message, stackTrace)) && results.Continue;
}
}
results.Continue = messageSink.OnMessage(new TestClassFinished(testCollection, className, results.Time, results.Total, results.Failed, results.Skipped)) && results.Continue;
return results;
}
/// <inheritdoc/>
public void Find(string typeName, bool includeSourceInformation, IMessageSink messageSink)
{
Guard.ArgumentNotNullOrEmpty("typeName", typeName);
Guard.ArgumentNotNull("messageSink", messageSink);
ThreadPool.QueueUserWorkItem(_ =>
{
ITypeInfo typeInfo = assemblyInfo.GetType(typeName);
if (typeInfo != null)
FindImpl(typeInfo, includeSourceInformation, messageSink);
var warnings = messageAggregator.GetAndClear<EnvironmentalWarning>().Select(w => w.Message).ToList();
messageSink.OnMessage(new DiscoveryCompleteMessage(warnings));
});
}
/// <inheritdoc/>
public void Run(IEnumerable<ITestCase> testCases, IMessageSink messageSink)
{
var results = new RunSummary();
var environment = String.Format("{0}-bit .NET {1}", IntPtr.Size * 8, Environment.Version);
// TODO: Contract for Run() states that null "testCases" means "run everything".
if (messageSink.OnMessage(new TestAssemblyStarting(assemblyFileName, configFileName, DateTime.Now, environment, TestFrameworkDisplayName)))
foreach (var testCollectionGroup in testCases.Cast<Xunit1TestCase>().GroupBy(tc => tc.TestCollection))
{
var collectionResults = RunTestCollection(testCollectionGroup.Key, testCollectionGroup, messageSink);
results.Aggregate(collectionResults);
if (!collectionResults.Continue)
break;
}
messageSink.OnMessage(new TestAssemblyFinished(new Xunit1AssemblyInfo(assemblyFileName), results.Time, results.Total, results.Failed, results.Skipped));
}
private static void RunTestClass(IMessageSink messageSink,
ITestCollection collection,
Dictionary<Type, object> collectionFixtureMappings,
IReflectionTypeInfo testClass,
IEnumerable<XunitTestCase> testCases,
RunSummary classSummary,
ExceptionAggregator aggregator,
CancellationTokenSource cancellationTokenSource)
{
var testClassType = testClass.Type;
var fixtureMappings = new Dictionary<Type, object>();
var constructorArguments = new List<object>();
if (testClassType.GetInterfaces().Any(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(ICollectionFixture<>)))
aggregator.Add(new TestClassException("A test class may not be decorated with ICollectionFixture<> (decorate the test collection class instead)."));
foreach (var interfaceType in testClassType.GetInterfaces().Where(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IClassFixture<>)))
CreateFixture(interfaceType, aggregator, fixtureMappings);
if (collection.CollectionDefinition != null)
{
var declarationType = ((IReflectionTypeInfo)collection.CollectionDefinition).Type;
foreach (var interfaceType in declarationType.GetInterfaces().Where(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IClassFixture<>)))
CreateFixture(interfaceType, aggregator, fixtureMappings);
}
var isStaticClass = testClassType.IsAbstract && testClassType.IsSealed;
if (!isStaticClass)
{
var ctors = testClassType.GetConstructors();
if (ctors.Length != 1)
{
aggregator.Add(new TestClassException("A test class may only define a single public constructor."));
}
else
{
var ctor = ctors.Single();
var unusedArguments = new List<string>();
foreach (var paramInfo in ctor.GetParameters())
{
object fixture;
if (fixtureMappings.TryGetValue(paramInfo.ParameterType, out fixture) || collectionFixtureMappings.TryGetValue(paramInfo.ParameterType, out fixture))
constructorArguments.Add(fixture);
else
unusedArguments.Add(paramInfo.ParameterType.Name + " " + paramInfo.Name);
}
if (unusedArguments.Count > 0)
aggregator.Add(new TestClassException("The following constructor arguments did not have matching fixture data: " + String.Join(", ", unusedArguments)));
}
}
var methodGroups = testCases.GroupBy(tc => tc.Method);
foreach (var method in methodGroups)
{
if (!messageSink.OnMessage(new TestMethodStarting(collection, testClass.Name, method.Key.Name)))
cancellationTokenSource.Cancel();
else
RunTestMethod(messageSink, constructorArguments.ToArray(), method, classSummary, aggregator, cancellationTokenSource);
if (!messageSink.OnMessage(new TestMethodFinished(collection, testClass.Name, method.Key.Name)))
cancellationTokenSource.Cancel();
if (cancellationTokenSource.IsCancellationRequested)
break;
}
foreach (var fixture in fixtureMappings.Values.OfType<IDisposable>())
{
try
{
fixture.Dispose();
}
catch (Exception ex)
{
if (!messageSink.OnMessage(new ErrorMessage(ex.Unwrap())))
cancellationTokenSource.Cancel();
}
}
}