public CefSharpXunitTestFramework(IMessageSink messageSink)
: base(messageSink)
{
//Create a new BindingRedirectAssemblyResolver to resolve the assemblies
//that xUnit is unable to Load
DisposalTracker.Add(new BindingRedirectAssemblyResolver());
}
public async void CanRunTests()
{
Assert.SkipWhen(EnvironmentHelper.IsMono, "Mono does not fully support dynamic assemblies");
var assemblyInfo = new ReflectionAssemblyInfo(fixture.Assembly);
await using var disposalTracker = new DisposalTracker();
var testFramework = ExtensibilityPointFactory.GetTestFramework(assemblyInfo);
disposalTracker.Add(testFramework);
var messages = new List <_MessageSinkMessage>();
var testExecutor = testFramework.GetExecutor(assemblyInfo);
await testExecutor.RunAll(SpyMessageSink.Create(messages: messages), _TestFrameworkOptions.ForDiscovery(), _TestFrameworkOptions.ForExecution());
var assemblyStarting = Assert.Single(messages.OfType <_TestAssemblyStarting>());
Assert.Equal("DynamicAssembly, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null", assemblyStarting.AssemblyName);
Assert.Null(assemblyStarting.AssemblyPath);
Assert.Null(assemblyStarting.ConfigFilePath);
Assert.Single(messages.OfType <_TestFailed>());
Assert.Single(messages.OfType <_TestPassed>());
Assert.Empty(messages.OfType <_TestSkipped>());
}
public TestFrameworkDiscoverer(IAssemblyInfo assemblyInfo, ISourceInformationProvider sourceProvider, IMessageSink diagnosticMessageSink, IList <AssemblyLoaderAttribute> assemblyLoaders, IXunitTestCollectionFactory collectionFactory = null)
: base(assemblyInfo, sourceProvider, diagnosticMessageSink, collectionFactory)
{
foreach (var assemblyLoader in assemblyLoaders)
{
DisposalTracker.Add(assemblyLoader);
}
}
public async ValueTask NoExceptions_DoesNotThrow()
{
var classUnderTest = new DisposalTracker();
var obj = Substitute.For <IDisposable, InterfaceProxy <IDisposable> >();
classUnderTest.Add(obj);
var ex = await Record.ExceptionAsync(() => classUnderTest.DisposeAsync());
Assert.Null(ex);
obj.Received().Dispose();
}
public async ValueTask MultipleExceptions_ThrowsAggregateException()
{
var classUnderTest = new DisposalTracker();
var obj1 = Substitute.For <IDisposable, InterfaceProxy <IDisposable> >();
obj1.When(x => x.Dispose()).Throw <DivideByZeroException>();
classUnderTest.Add(obj1);
var obj2 = Substitute.For <IDisposable, InterfaceProxy <IDisposable> >();
obj2.When(x => x.Dispose()).Throw <InvalidOperationException>();
classUnderTest.Add(obj2);
var ex = await Record.ExceptionAsync(() => classUnderTest.DisposeAsync());
var aggEx = Assert.IsType <AggregateException>(ex);
Assert.Collection(
aggEx.InnerExceptions,
ex => Assert.IsType <InvalidOperationException>(ex),
ex => Assert.IsType <DivideByZeroException>(ex)
);
}
public async ValueTask SingleException_CleansUpAllObjects_ThrowsTheSingleException()
{
var classUnderTest = new DisposalTracker();
var obj1 = Substitute.For <IDisposable, InterfaceProxy <IDisposable> >();
classUnderTest.Add(obj1);
var thrown = new DivideByZeroException();
var obj2 = Substitute.For <IDisposable, InterfaceProxy <IDisposable> >();
obj2.When(x => x.Dispose()).Throw(thrown);
classUnderTest.Add(obj2);
var obj3 = Substitute.For <IDisposable, InterfaceProxy <IDisposable> >();
classUnderTest.Add(obj3);
var ex = await Record.ExceptionAsync(() => classUnderTest.DisposeAsync());
Assert.Same(thrown, ex);
obj1.Received().Dispose();
obj2.Received().Dispose();
obj3.Received().Dispose();
}
public TdNetRunnerHelper(
Assembly assembly,
ITestListener testListener)
{
this.testListener = testListener;
var assemblyFileName = assembly.GetLocalCodeBase();
var project = new XunitProject();
projectAssembly = new XunitProjectAssembly(project)
{
Assembly = assembly,
AssemblyFilename = assemblyFileName,
TargetFramework = AssemblyUtility.GetTargetFramework(assemblyFileName)
};
projectAssembly.Configuration.ShadowCopy = false;
ConfigReader.Load(projectAssembly.Configuration, assemblyFileName);
var diagnosticMessageSink = new DiagnosticMessageSink(testListener, Path.GetFileNameWithoutExtension(assemblyFileName), projectAssembly.Configuration.DiagnosticMessagesOrDefault);
frontController = Xunit2.ForDiscoveryAndExecution(projectAssembly, diagnosticMessageSink: diagnosticMessageSink);
disposalTracker.Add(frontController);
}
public DependencyInjectionTestFrameworkExecutor(IServiceProvider provider,
AssemblyName assemblyName,
ISourceInformationProvider sourceInformationProvider,
IMessageSink diagnosticMessageSink)
: base(assemblyName, sourceInformationProvider, diagnosticMessageSink)
{
_provider = provider;
var scope = _provider.GetService <IServiceScope>();
if (scope != null)
{
DisposalTracker.Add(scope);
}
}
public DependencyInjectionTestFrameworkExecutor(IHost?host,
Exception?exception,
AssemblyName assemblyName,
ISourceInformationProvider sourceInformationProvider,
IMessageSink diagnosticMessageSink)
: base(assemblyName, sourceInformationProvider, diagnosticMessageSink)
{
_host = host;
_exception = exception;
if (_host != null)
{
DisposalTracker.Add(_host);
}
}
public AutofacTestFramework(IMessageSink messageSink) : base(messageSink)
{
var cb = new ContainerBuilder();
cb
.RegisterInstance(this)
.AsSelf()
.As <TestFramework>()
.ExternallyOwned();
cb
.RegisterInstance(SourceInformationProvider)
.ExternallyOwned();
cb
.RegisterInstance(messageSink)
.ExternallyOwned();
cb
.RegisterType <AutofacTestFrameworkExecutor>()
.As <ITestFrameworkExecutor>()
.ExternallyOwned();
cb
.RegisterType <AutofacTestAssemblyRunner>()
.As <TestAssemblyRunner <IXunitTestCase> >();
cb
.RegisterType <AutofacTestCollectionRunner>()
.As <TestCollectionRunner <IXunitTestCase> >();
cb
.RegisterType <AutofacTestClassRunner>()
.As <TestClassRunner <IXunitTestCase> >();
cb
.RegisterType <AutofacTestMethodRunner>()
.As <TestMethodRunner <IXunitTestCase> >();
cb
.RegisterType <XunitTestFrameworkDiscoverer>()
.As <ITestFrameworkDiscoverer>()
.ExternallyOwned();
_rootContainer = cb.Build();
DisposalTracker.Add(_rootContainer);
}
public async void CanDiscoverTests()
{
Assert.SkipWhen(EnvironmentHelper.IsMono, "Mono does not fully support dynamic assemblies");
var assemblyInfo = new ReflectionAssemblyInfo(fixture.Assembly);
await using var disposalTracker = new DisposalTracker();
var testFramework = ExtensibilityPointFactory.GetTestFramework(assemblyInfo);
disposalTracker.Add(testFramework);
var testCases = new List <_ITestCase>();
var testDiscoverer = testFramework.GetDiscoverer(assemblyInfo);
await testDiscoverer.Find(testCase => { testCases.Add(testCase); return(new(true)); }, _TestFrameworkOptions.ForDiscovery());
Assert.Collection(
testCases.OrderBy(tc => tc.TestCaseDisplayName),
testCase => Assert.Equal("UnitTests.Failing", testCase.TestCaseDisplayName),
testCase => Assert.Equal("UnitTests.Passing", testCase.TestCaseDisplayName)
);
}
/// <summary>
/// Initializes a new instance of the <see cref="Xunit2SourceInformationProvider"/> class.
/// </summary>
/// <param name="v3Provider"></param>
public Xunit2SourceInformationProvider(_ISourceInformationProvider v3Provider)
{
disposalTracker.Add(v3Provider);
this.v3Provider = Guard.ArgumentNotNull(nameof(v3Provider), v3Provider);
}
public ConnectionUtils(IMessageSink messageSink) : base(messageSink)
{
_runner = MongoDbRunner.Start(singleNodeReplSet: true);
DisposalTracker.Add(_runner);
}
/// <inheritdoc/>
protected override async Task AfterTestCaseStartingAsync()
{
await base.AfterTestCaseStartingAsync();
try
{
var dataAttributes = TestCase.TestMethod.Method.GetCustomAttributes(typeof(DataAttribute));
var testIndex = 0;
foreach (var dataAttribute in dataAttributes)
{
var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).FirstOrDefault();
if (discovererAttribute == null)
{
if (dataAttribute is _IReflectionAttributeInfo reflectionAttribute)
{
Aggregator.Add(new InvalidOperationException($"Data attribute {reflectionAttribute.Attribute.GetType().FullName} on {TestCase.TestMethod.TestClass.Class.Name}.{TestCase.TestMethod.Method.Name} does not have a discoverer attribute attached."));
}
else
{
Aggregator.Add(new InvalidOperationException($"A data attribute specified on {TestCase.TestMethod.TestClass.Class.Name}.{TestCase.TestMethod.Method.Name} does not have a discoverer attribute attached."));
}
continue;
}
IDataDiscoverer?discoverer;
try
{
discoverer = ExtensibilityPointFactory.GetDataDiscoverer(DiagnosticMessageSink, discovererAttribute);
if (discoverer == null)
{
if (dataAttribute is _IReflectionAttributeInfo reflectionAttribute)
{
Aggregator.Add(new InvalidOperationException($"Data discoverer specified for {reflectionAttribute.Attribute.GetType()} on {TestCase.TestMethod.TestClass.Class.Name}.{TestCase.TestMethod.Method.Name} does not exist or could not be constructed."));
}
else
{
Aggregator.Add(new InvalidOperationException($"A data discoverer specified on {TestCase.TestMethod.TestClass.Class.Name}.{TestCase.TestMethod.Method.Name} does not exist or could not be constructed."));
}
continue;
}
}
catch (InvalidCastException)
{
if (dataAttribute is _IReflectionAttributeInfo reflectionAttribute)
{
Aggregator.Add(new InvalidOperationException($"Data discoverer specified for {reflectionAttribute.Attribute.GetType()} on {TestCase.TestMethod.TestClass.Class.Name}.{TestCase.TestMethod.Method.Name} does not implement IDataDiscoverer."));
}
else
{
Aggregator.Add(new InvalidOperationException($"A data discoverer specified on {TestCase.TestMethod.TestClass.Class.Name}.{TestCase.TestMethod.Method.Name} does not implement IDataDiscoverer."));
}
continue;
}
var data = discoverer.GetData(dataAttribute, TestCase.TestMethod.Method);
if (data == null)
{
Aggregator.Add(new InvalidOperationException($"Test data returned null for {TestCase.TestMethod.TestClass.Class.Name}.{TestCase.TestMethod.Method.Name}. Make sure it is statically initialized before this test method is called."));
continue;
}
foreach (var dataRow in data)
{
foreach (var dataRowItem in dataRow)
{
disposalTracker.Add(dataRowItem);
}
_ITypeInfo[]? resolvedTypes = null;
var methodToRun = TestMethod;
var convertedDataRow = methodToRun.ResolveMethodArguments(dataRow);
if (methodToRun.IsGenericMethodDefinition)
{
resolvedTypes = TestCase.TestMethod.Method.ResolveGenericTypes(convertedDataRow);
methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((_IReflectionTypeInfo)t).Type).ToArray());
}
var parameterTypes = methodToRun.GetParameters().Select(p => p.ParameterType).ToArray();
convertedDataRow = Reflector.ConvertArguments(convertedDataRow, parameterTypes);
var theoryDisplayName = TestCase.TestMethod.Method.GetDisplayNameWithArguments(DisplayName, convertedDataRow, resolvedTypes);
var test = CreateTest(TestCase, theoryDisplayName, testIndex++);
var skipReason = SkipReason ?? dataAttribute.GetNamedArgument <string>("Skip");
testRunners.Add(CreateTestRunner(test, MessageBus, TestClass, ConstructorArguments, methodToRun, convertedDataRow, skipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource));
}
}
}
catch (Exception ex)
{
// Stash the exception so we can surface it during RunTestAsync
dataDiscoveryException = ex;
}
}
async ValueTask <XElement?> ExecuteAssembly(
XunitProjectAssembly assembly,
bool needsXml,
_IMessageSink reporterMessageHandler)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
// Setup discovery and execution options with command-line overrides
var discoveryOptions = _TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = _TestFrameworkOptions.ForExecution(assembly.Configuration);
// The normal default is true here, but we want it to be false for us by default
if (!assembly.Configuration.PreEnumerateTheories.HasValue)
{
discoveryOptions.SetPreEnumerateTheories(false);
}
var assemblyDisplayName = assembly.AssemblyDisplayName;
var noColor = assembly.Project.Configuration.NoColorOrDefault;
var diagnosticMessageSink = ConsoleDiagnosticMessageSink.ForDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
var internalDiagnosticsMessageSink = ConsoleDiagnosticMessageSink.ForInternalDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.InternalDiagnosticMessagesOrDefault, noColor);
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
var assemblyInfo = new ReflectionAssemblyInfo(testAssembly);
await using var disposalTracker = new DisposalTracker();
var testFramework = ExtensibilityPointFactory.GetTestFramework(diagnosticMessageSink, assemblyInfo);
disposalTracker.Add(testFramework);
var discoverySink = new TestDiscoverySink(() => cancel);
// Discover & filter the tests
var testDiscoverer = testFramework.GetDiscoverer(assemblyInfo);
var discoveryStarting = new TestAssemblyDiscoveryStarting
{
AppDomain = AppDomainOption.NotAvailable,
Assembly = assembly,
DiscoveryOptions = discoveryOptions,
ShadowCopy = false
};
reporterMessageHandler.OnMessage(discoveryStarting);
testDiscoverer.Find(discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(assembly.Configuration.Filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
var discoveryFinished = new TestAssemblyDiscoveryFinished
{
Assembly = assembly,
DiscoveryOptions = discoveryOptions,
TestCasesDiscovered = testCasesDiscovered,
TestCasesToRun = testCasesToRun
};
reporterMessageHandler.OnMessage(discoveryFinished);
// Run the filtered tests
if (testCasesToRun == 0)
{
testExecutionSummaries.Add(testDiscoverer.TestAssemblyUniqueID, new ExecutionSummary());
}
else
{
var executionStarting = new TestAssemblyExecutionStarting
{
Assembly = assembly,
ExecutionOptions = executionOptions
};
reporterMessageHandler.OnMessage(executionStarting);
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel);
if (assemblyElement != null)
{
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
}
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
}
if (assembly.Configuration.FailSkipsOrDefault)
{
resultsSink = new DelegatingFailSkipSink(resultsSink);
}
using (resultsSink)
{
var executor = testFramework.GetExecutor(assemblyInfo);
executor.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
testExecutionSummaries.Add(testDiscoverer.TestAssemblyUniqueID, resultsSink.ExecutionSummary);
var executionFinished = new TestAssemblyExecutionFinished
{
Assembly = assembly,
ExecutionOptions = executionOptions,
ExecutionSummary = resultsSink.ExecutionSummary
};
reporterMessageHandler.OnMessage(executionFinished);
if (assembly.Configuration.StopOnFailOrDefault && resultsSink.ExecutionSummary.Failed != 0)
{
Console.WriteLine("Canceling due to test failure...");
cancel = true;
}
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine($"{e.GetType().FullName}: {e.Message}");
if (assembly.Configuration.InternalDiagnosticMessagesOrDefault)
{
Console.WriteLine(e.StackTrace);
}
e = e.InnerException;
}
}
return(assemblyElement);
}
Xunit2(
_IMessageSink diagnosticMessageSink,
AppDomainSupport appDomainSupport,
_ISourceInformationProvider sourceInformationProvider,
_IAssemblyInfo?assemblyInfo,
string?assemblyFileName,
string xunitExecutionAssemblyPath,
string?configFileName,
bool shadowCopy,
string?shadowCopyFolder,
bool verifyAssembliesOnDisk)
{
#if NETFRAMEWORK
// Only safe to assume the execution reference is copied in a desktop project
if (verifyAssembliesOnDisk)
{
Guard.FileExists(xunitExecutionAssemblyPath);
}
CanUseAppDomains = !IsDotNet(xunitExecutionAssemblyPath);
#else
CanUseAppDomains = false;
#endif
DiagnosticMessageSink = diagnosticMessageSink;
var appDomainAssembly = assemblyFileName ?? xunitExecutionAssemblyPath;
AppDomain = AppDomainManagerFactory.Create(appDomainSupport != AppDomainSupport.Denied && CanUseAppDomains, appDomainAssembly, configFileName, shadowCopy, shadowCopyFolder, diagnosticMessageSink);
DisposalTracker.Add(AppDomain);
#if NETFRAMEWORK
var runnerUtilityAssemblyLocation = Path.GetDirectoryName(typeof(AssemblyHelper).Assembly.GetLocalCodeBase());
assemblyHelper = AppDomain.CreateObjectFrom <AssemblyHelper>(typeof(AssemblyHelper).Assembly.Location, typeof(AssemblyHelper).FullName !, runnerUtilityAssemblyLocation);
DisposalTracker.Add(assemblyHelper);
#endif
TestFrameworkAssemblyName = GetTestFrameworkAssemblyName(xunitExecutionAssemblyPath);
// We need both a v2 and v3 assembly info, so manufacture the things we're missing
IAssemblyInfo remoteAssemblyInfo;
if (assemblyInfo != null)
{
remoteAssemblyInfo = new Xunit2AssemblyInfo(assemblyInfo);
}
else
{
remoteAssemblyInfo = Guard.NotNull(
"Could not create Xunit.Sdk.TestFrameworkProxy for v2 unit test",
AppDomain.CreateObject <IAssemblyInfo>(TestFrameworkAssemblyName, "Xunit.Sdk.ReflectionAssemblyInfo", assemblyFileName)
);
assemblyInfo = new Xunit3AssemblyInfo(remoteAssemblyInfo);
}
this.assemblyInfo = assemblyInfo;
this.configFileName = configFileName;
TestAssemblyUniqueID = UniqueIDGenerator.ForAssembly(this.assemblyInfo.Name, this.assemblyInfo.AssemblyPath, configFileName);
var v2SourceInformationProvider = Xunit2SourceInformationProviderAdapter.Adapt(sourceInformationProvider);
var v2DiagnosticMessageSink = new Xunit2MessageSink(DiagnosticMessageSink);
remoteFramework = Guard.NotNull(
"Could not create Xunit.Sdk.TestFrameworkProxy for v2 unit test",
AppDomain.CreateObject <ITestFramework>(
TestFrameworkAssemblyName,
"Xunit.Sdk.TestFrameworkProxy",
remoteAssemblyInfo,
v2SourceInformationProvider,
v2DiagnosticMessageSink
)
);
DisposalTracker.Add(remoteFramework);
remoteDiscoverer = Guard.NotNull("Could not get discoverer from test framework for v2 unit test", remoteFramework.GetDiscoverer(remoteAssemblyInfo));
DisposalTracker.Add(remoteDiscoverer);
// If we got an assembly file name, that means we can do execution as well as discovery.
if (assemblyFileName != null)
{
#if NETFRAMEWORK
var assemblyName = AssemblyName.GetAssemblyName(assemblyFileName);
#else
var an = Assembly.Load(new AssemblyName {
Name = Path.GetFileNameWithoutExtension(assemblyFileName)
}).GetName();
var assemblyName = new AssemblyName {
Name = an.Name, Version = an.Version
};
#endif
remoteExecutor = remoteFramework.GetExecutor(assemblyName);
DisposalTracker.Add(remoteExecutor);
}
}