public async ValueTask TheoryAcceptanceTest()
{
var code = @"
module FSharpTests
open Xunit
[<Theory>]
[<InlineData(42)>]
[<InlineData(2112)>]
let TestMethod(x : int) =
Assert.Equal(2112, x)
";
using var assembly = await FSharpAcceptanceTestV2Assembly.Create(code.Replace("\t", " "));
var controller = TestableXunit2.Create(assembly.FileName, null, true);
var settings = new FrontControllerFindAndRunSettings(_TestFrameworkOptions.ForDiscovery(), _TestFrameworkOptions.ForExecution());
using var sink = SpyMessageSink <_TestAssemblyFinished> .Create();
controller.FindAndRun(sink, settings);
sink.Finished.WaitOne();
Assert.Empty(sink.Messages.OfType <_TestSkipped>());
var passedTest = Assert.Single(sink.Messages.OfType <_TestPassed>());
var passedMetadata = sink.Messages.OfType <_TestStarting>().Single(ts => ts.TestUniqueID == passedTest.TestUniqueID);
Assert.Equal("FSharpTests.TestMethod(x: 2112)", passedMetadata.TestDisplayName);
var failedTest = Assert.Single(sink.Messages.OfType <_TestFailed>());
var failedMetadata = sink.Messages.OfType <_TestStarting>().Single(ts => ts.TestUniqueID == failedTest.TestUniqueID);
Assert.Equal("FSharpTests.TestMethod(x: 42)", failedMetadata.TestDisplayName);
}
public async ValueTask AsyncAcceptanceTest()
{
var code = @"
module FSharpTests
open Xunit
[<Fact>]
let AsyncFailing() =
async {
do! Async.Sleep(10)
Assert.True(false)
}
";
using var assembly = await FSharpAcceptanceTestV2Assembly.Create(code.Replace("\t", " "));
var controller = TestableXunit2.Create(assembly.FileName, null, true);
using var sink = SpyMessageSink <_TestAssemblyFinished> .Create();
var settings = new FrontControllerFindAndRunSettings(_TestFrameworkOptions.ForDiscovery(), _TestFrameworkOptions.ForExecution());
controller.FindAndRun(sink, settings);
sink.Finished.WaitOne();
var failures = sink.Messages.OfType <_TestFailed>();
var failure = Assert.Single(failures);
var failureStarting = sink.Messages.OfType <_TestStarting>().Single(s => s.TestUniqueID == failure.TestUniqueID);
Assert.Equal("FSharpTests.AsyncFailing", failureStarting.TestDisplayName);
}
public async ValueTask TheoryAcceptanceTest()
{
var code = @"
using System;
using Xunit;
public class TestClass
{
[Theory]
[InlineData(42)]
[InlineData(2112)]
public void TestMethod(int x) { Assert.Equal(2112, x); }
}";
using var assembly = await CSharpAcceptanceTestV2Assembly.Create(code);
var controller = TestableXunit2.Create(assembly.FileName, null, true);
var settings = new FrontControllerFindAndRunSettings(_TestFrameworkOptions.ForDiscovery(), _TestFrameworkOptions.ForExecution());
using var sink = SpyMessageSink <_TestAssemblyFinished> .Create();
controller.FindAndRun(sink, settings);
sink.Finished.WaitOne();
Assert.Empty(sink.Messages.OfType <_TestSkipped>());
var passedTest = Assert.Single(sink.Messages.OfType <_TestPassed>());
var passedMetadata = sink.Messages.OfType <_TestStarting>().Single(ts => ts.TestUniqueID == passedTest.TestUniqueID);
Assert.Equal("TestClass.TestMethod(x: 2112)", passedMetadata.TestDisplayName);
var failedTest = Assert.Single(sink.Messages.OfType <_TestFailed>());
var failedMetadata = sink.Messages.OfType <_TestStarting>().Single(ts => ts.TestUniqueID == failedTest.TestUniqueID);
Assert.Equal("TestClass.TestMethod(x: 42)", failedMetadata.TestDisplayName);
}
/// <inheritdoc/>
public void FindAndRun(
_IMessageSink messageSink,
FrontControllerFindAndRunSettings settings)
{
Guard.NotNull($"This instance of {typeof(Xunit2).FullName} was created for discovery only; execution-related operations cannot be performed.", remoteExecutor);
Guard.ArgumentNotNull(messageSink);
Guard.ArgumentNotNull(settings);
if (settings.Filters.Empty)
{
remoteExecutor.RunAll(
CreateOptimizedRemoteMessageSink(messageSink),
Xunit2OptionsAdapter.Adapt(settings.DiscoveryOptions),
Xunit2OptionsAdapter.Adapt(settings.ExecutionOptions)
);
return;
}
using var discoverySink = new Xunit2DiscoverySink(settings.Filters);
remoteDiscoverer.Find(
includeSourceInformation: false,
discoverySink,
Xunit2OptionsAdapter.Adapt(settings.DiscoveryOptions)
);
discoverySink.Finished.WaitOne();
remoteExecutor.RunTests(
discoverySink.TestCases,
CreateOptimizedRemoteMessageSink(messageSink),
Xunit2OptionsAdapter.Adapt(settings.ExecutionOptions)
);
}
/// <inheritdoc/>
public void FindAndRun(
_IMessageSink messageSink,
FrontControllerFindAndRunSettings settings)
{
Guard.ArgumentNotNull(nameof(messageSink), messageSink);
Guard.ArgumentNotNull(nameof(settings), settings);
// Pass null for empty filter list, since it bypasses _TestCaseDiscovered creation
FindAndRun(
messageSink,
settings.DiscoveryOptions.GetIncludeSourceInformationOrDefault(),
settings.Filters.Empty ? null : settings.Filters.Filter
);
}
public async ValueTask AsyncAcceptanceTest()
{
var code = @"
using System;
using System.Threading.Tasks;
using Xunit;
public class TestClass
{
[Fact]
public async void AsyncVoid()
{
await Task.Delay(10);
Assert.True(false);
}
[Fact]
public async Task AsyncTask()
{
await Task.Delay(10);
Assert.True(false);
}
}";
using var assembly = await CSharpAcceptanceTestV2Assembly.Create(code);
var controller = TestableXunit2.Create(assembly.FileName, null, true);
var discoveryOptions = _TestFrameworkOptions.ForDiscovery();
var executionOptions = _TestFrameworkOptions.ForExecution();
var settings = new FrontControllerFindAndRunSettings(discoveryOptions, executionOptions);
using var sink = SpyMessageSink <_TestAssemblyFinished> .Create();
controller.FindAndRun(sink, settings);
sink.Finished.WaitOne();
Assert.Empty(sink.Messages.OfType <_TestPassed>());
Assert.Empty(sink.Messages.OfType <_TestSkipped>());
var failedTests =
sink.Messages
.OfType <_TestFailed>()
.Select(f => sink.Messages.OfType <_TestStarting>().Single(ts => ts.TestUniqueID == f.TestUniqueID).TestDisplayName);
Assert.Collection(
failedTests.OrderBy(name => name),
name => Assert.Equal("TestClass.AsyncTask", name),
name => Assert.Equal("TestClass.AsyncVoid", name)
);
}
public async ValueTask NoTestMethods()
{
using var assembly = await CSharpAcceptanceTestV2Assembly.Create(code : "");
var controller = TestableXunit2.Create(assembly.FileName, null, true);
var settings = new FrontControllerFindAndRunSettings(_TestFrameworkOptions.ForDiscovery(), _TestFrameworkOptions.ForExecution());
using var sink = SpyMessageSink <_TestAssemblyFinished> .Create();
controller.FindAndRun(sink, settings);
sink.Finished.WaitOne();
Assert.Empty(sink.Messages.OfType <_TestPassed>());
Assert.Empty(sink.Messages.OfType <_TestFailed>());
Assert.Empty(sink.Messages.OfType <_TestSkipped>());
}
async ValueTask <XElement?> ExecuteAssembly(
XunitProjectAssembly assembly,
bool needsXml,
_IMessageSink reporterMessageHandler)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
var assemblyFileName = Guard.ArgumentNotNull("assembly.AssemblyFilename", assembly.AssemblyFilename);
// Setup discovery and execution options with command-line overrides
var discoveryOptions = _TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = _TestFrameworkOptions.ForExecution(assembly.Configuration);
// The normal default is true here, but we want it to be false for us by default
if (!assembly.Configuration.PreEnumerateTheories.HasValue)
{
discoveryOptions.SetPreEnumerateTheories(false);
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName);
var noColor = assembly.Project.Configuration.NoColorOrDefault;
var diagnosticMessageSink = ConsoleDiagnosticMessageSink.ForDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
var internalDiagnosticsMessageSink = ConsoleDiagnosticMessageSink.ForInternalDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.InternalDiagnosticMessagesOrDefault, noColor);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
using var _ = AssemblyHelper.SubscribeResolveForAssembly(assemblyFileName, internalDiagnosticsMessageSink);
await using var controller = XunitFrontController.ForDiscoveryAndExecution(assembly, diagnosticMessageSink: diagnosticMessageSink);
var executionStarting = new TestAssemblyExecutionStarting
{
Assembly = assembly,
ExecutionOptions = executionOptions
};
reporterMessageHandler.OnMessage(executionStarting);
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (summary, _) => completionMessages.TryAdd(controller.TestAssemblyUniqueID, summary));
if (assemblyElement != null)
{
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
}
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
}
if (assembly.Configuration.FailSkipsOrDefault)
{
resultsSink = new DelegatingFailSkipSink(resultsSink);
}
using (resultsSink)
{
var settings = new FrontControllerFindAndRunSettings(discoveryOptions, executionOptions, assembly.Configuration.Filters);
controller.FindAndRun(resultsSink, settings);
resultsSink.Finished.WaitOne();
var executionFinished = new TestAssemblyExecutionFinished
{
Assembly = assembly,
ExecutionOptions = executionOptions,
ExecutionSummary = resultsSink.ExecutionSummary
};
reporterMessageHandler.OnMessage(executionFinished);
if (assembly.Configuration.StopOnFailOrDefault && resultsSink.ExecutionSummary.Failed != 0)
{
Console.WriteLine("Canceling due to test failure...");
cancel = true;
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine($"{e.GetType().FullName}: {e.Message}");
if (assembly.Configuration.InternalDiagnosticMessagesOrDefault)
{
Console.WriteLine(e.StackTrace);
}
e = e.InnerException;
}
}
return(assemblyElement);
}
protected virtual async ValueTask <XElement?> ExecuteAssembly(
XunitProjectAssembly assembly,
AppDomainSupport?appDomains)
{
if (cancel)
{
return(null);
}
var assemblyElement = NeedsXml ? new XElement("assembly") : null;
try
{
assembly.Configuration.PreEnumerateTheories = PreEnumerateTheories;
assembly.Configuration.DiagnosticMessages |= DiagnosticMessages;
assembly.Configuration.InternalDiagnosticMessages |= InternalDiagnosticMessages;
if (appDomains.HasValue)
{
assembly.Configuration.AppDomain = appDomains;
}
// Setup discovery and execution options with command-line overrides
var discoveryOptions = _TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = _TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue && maxThreadCount.Value > -1)
{
executionOptions.SetMaxParallelThreads(maxThreadCount);
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections);
}
if (stopOnFail.HasValue)
{
executionOptions.SetStopOnTestFail(stopOnFail);
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename) !;
var diagnosticMessageSink = DiagnosticMessageSink.ForDiagnostics(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
await using var controller = XunitFrontController.ForDiscoveryAndExecution(assembly, diagnosticMessageSink: diagnosticMessageSink);
var executionStarting = new TestAssemblyExecutionStarting
{
Assembly = assembly,
ExecutionOptions = executionOptions
};
reporterMessageHandler !.OnMessage(executionStarting);
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler !, () => cancel, (summary, _) => completionMessages.TryAdd(controller.TestAssemblyUniqueID, summary));
if (assemblyElement != null)
{
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
}
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
}
if (FailSkips)
{
resultsSink = new DelegatingFailSkipSink(resultsSink);
}
using (resultsSink)
{
var settings = new FrontControllerFindAndRunSettings(discoveryOptions, executionOptions, assembly.Configuration.Filters);
controller.FindAndRun(resultsSink, settings);
resultsSink.Finished.WaitOne();
var executionFinished = new TestAssemblyExecutionFinished
{
Assembly = assembly,
ExecutionOptions = executionOptions,
ExecutionSummary = resultsSink.ExecutionSummary
};
reporterMessageHandler !.OnMessage(executionFinished);
if (resultsSink.ExecutionSummary.Failed != 0 || resultsSink.ExecutionSummary.Errors != 0)
{
ExitCode = 1;
if (stopOnFail == true)
{
Log.LogMessage(MessageImportance.High, "Canceling due to test failure...");
Cancel();
}
}
}
}
catch (Exception ex)
{
var e = ex;
while (e != null)
{
Log.LogError("{0}: {1}", e.GetType().FullName, e.Message);
if (e.StackTrace != null)
{
foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
{
Log.LogError(stackLine);
}
}
e = e.InnerException;
}
ExitCode = -1;
}
return(assemblyElement);
}
