/// <summary>
/// Creates the fallback test driver.
/// </summary>
/// <param name="testFrameworkOptions">The test framework options.</param>
/// <exception cref="ArgumentNullException">Thrown if <paramref name="testFrameworkOptions"/> is null.</exception>
public FallbackTestDriver(TestFrameworkOptions testFrameworkOptions)
{
if (testFrameworkOptions == null)
throw new ArgumentNullException("testFrameworkOptions");
this.testFrameworkOptions = testFrameworkOptions;
}
private static ITestDriver CreateTestDriver(
IList<ComponentHandle<ITestFramework, TestFrameworkTraits>> testFrameworkHandles,
TestFrameworkOptions testFrameworkOptions,
ILogger logger)
{
return new RSpecTestDriver(logger, testFrameworkHandles[0].Descriptor.Plugin.BaseDirectory);
}
private static ITestDriver CreateTestDriver(
IList<ComponentHandle<ITestFramework, TestFrameworkTraits>> testFrameworkHandles,
TestFrameworkOptions testFrameworkOptions,
ILogger logger)
{
return new ConcordionTestDriver();
}
static ITestDriver CreateDriver(
IList<ComponentHandle<ITestFramework, TestFrameworkTraits>> testFrameworkHandles,
TestFrameworkOptions testFrameworkOptions,
ILogger logger)
{
return new Services.MachineSpecificationsDriver();
}
private ITestDriver CreateTestDriver(
IList<ComponentHandle<ITestFramework, TestFrameworkTraits>> testFrameworkHandles,
TestFrameworkOptions testFrameworkOptions,
ILogger logger
)
{
return new BoostTestDriver(logger);
}
/// <summary>
/// Creates a test framework selection.
/// </summary>
/// <param name="testFrameworkHandle">The selected test framework handle.</param>
/// <param name="testFrameworkOptions">The test framework options.</param>
/// <param name="isFallback">True if the selection includes the fallback test
/// framework because a test file is not supported by any other registered
/// test framework. <seealso cref="TestFrameworkSelector.FallbackMode"/></param>
/// <exception cref="ArgumentNullException">Thrown if <paramref name="testFrameworkHandle"/>
/// or <paramref name="testFrameworkOptions"/> is null.</exception>
public TestFrameworkSelection(
ComponentHandle<ITestFramework, TestFrameworkTraits> testFrameworkHandle,
TestFrameworkOptions testFrameworkOptions,
bool isFallback)
{
if (testFrameworkHandle == null)
throw new ArgumentNullException("testFrameworkHandle");
if (testFrameworkOptions == null)
throw new ArgumentNullException("testFrameworkOptions");
this.testFrameworkHandle = testFrameworkHandle;
this.testFrameworkOptions = testFrameworkOptions;
this.isFallback = isFallback;
}
/// <summary>
/// Resets <see cref="TestFrameworkOptions"/> to its default value and sets <see cref="IsTestFrameworkOptionsSpecified" /> to false.
/// </summary>
public void ResetTestFrameworkOptions()
{
testFrameworkOptions = null;
isTestFrameworkOptionsSpecified = false;
}
private static ITestDriver CreateTestDriver(
IList<ComponentHandle<ITestFramework, TestFrameworkTraits>> testFrameworkHandles,
TestFrameworkOptions testFrameworkOptions,
ILogger logger)
{
string[] testFrameworkIds = GenericCollectionUtils.ConvertAllToArray(testFrameworkHandles, x => x.Id);
StringBuilder testFrameworkName = new StringBuilder();
foreach (var testFrameworkHandle in testFrameworkHandles)
{
if (testFrameworkName.Length != 0)
testFrameworkName.Append(" and ");
testFrameworkName.Append(testFrameworkHandle.GetTraits().Name);
}
return new PatternTestDriver(testFrameworkIds, testFrameworkName.ToString());
}
public List <IMessageSinkMessage> Run(Type[] types)
{
Xunit2 = new Xunit2(AppDomainSupport.Required, new NullSourceInformationProvider(), types[0].Assembly.GetLocalCodeBase(), configFileName: null, shadowCopy: true);
var discoverySink = new SpyMessageSink <IDiscoveryCompleteMessage>();
foreach (var type in types)
{
Xunit2.Find(type.FullName, includeSourceInformation: false, messageSink: discoverySink, discoveryOptions: TestFrameworkOptions.ForDiscovery());
discoverySink.Finished.WaitOne();
discoverySink.Finished.Reset();
}
var testCases = discoverySink.Messages.OfType <ITestCaseDiscoveryMessage>().Select(msg => msg.TestCase).ToArray();
var runSink = new SpyMessageSink <ITestAssemblyFinished>();
Xunit2.RunTests(testCases, runSink, TestFrameworkOptions.ForExecution());
runSink.Finished.WaitOne();
return(runSink.Messages.ToList());
}
public void RoundTrip()
{
string code = @"
using System;
using Xunit;
namespace Namespace1
{
public class Class1
{
[Fact]
public void FactMethod() { }
[Theory]
[InlineData(42)]
public void TheoryMethod(int x) { }
}
}";
using (var assembly = CSharpAcceptanceTestV2Assembly.Create(code))
{
var serializations = default(List <string>);
var testCollectionId = default(Guid);
using (var serializationController = new TestableXunitFrontController(assembly.FileName))
{
var sink = new SpyMessageSink <IDiscoveryCompleteMessage>();
serializationController.Find(includeSourceInformation: false, messageSink: sink, discoveryOptions: TestFrameworkOptions.ForDiscovery());
sink.Finished.WaitOne();
var testCases = sink.Messages.OfType <ITestCaseDiscoveryMessage>().OrderBy(tcdm => tcdm.TestCase.TestMethod.Method.Name).Select(tcdm => tcdm.TestCase).ToList();
testCollectionId = testCases[0].TestMethod.TestClass.TestCollection.UniqueID;
var descriptors = serializationController.GetTestCaseDescriptors(testCases, true);
serializations = descriptors.Select(d => d.Serialization).ToList();
}
Assert.Collection(serializations,
s => Assert.Equal($":F:Namespace1.Class1:FactMethod:1:0:{testCollectionId.ToString("N")}", s),
s => Assert.StartsWith("Xunit.Sdk.XunitTestCase, xunit.execution.{Platform}:", s)
);
using (var deserializationController = new TestableXunitFrontController(assembly.FileName))
{
var deserializations = deserializationController.BulkDeserialize(serializations);
Assert.Collection(deserializations.Select(kvp => kvp.Value),
testCase => Assert.Equal("Namespace1.Class1.FactMethod", testCase.DisplayName),
testCase => Assert.Equal("Namespace1.Class1.TheoryMethod(x: 42)", testCase.DisplayName)
);
}
}
}
private bool DiscoverTestsInSource <TVisitor>(XunitFrontController framework,
LoggerHelper logger,
TestPlatformContext testPlatformContext,
RunSettings runSettings,
Func <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitorFactory,
Action <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitComplete,
string assemblyFileName,
bool shadowCopy,
TestAssemblyConfiguration configuration)
where TVisitor : IVsDiscoverySink, IDisposable
{
if (cancelled)
{
return(false);
}
var fileName = "(unknown assembly)";
try
{
var reporterMessageHandler = GetRunnerReporter(logger, runSettings, new[] { assemblyFileName }).CreateMessageHandler(new VisualStudioRunnerLogger(logger));
var assembly = new XunitProjectAssembly {
AssemblyFilename = assemblyFileName
};
fileName = Path.GetFileNameWithoutExtension(assemblyFileName);
if (!IsXunitTestAssembly(assemblyFileName))
{
if (configuration.DiagnosticMessagesOrDefault)
{
logger.Log("Skipping: {0} (no reference to xUnit.net)", fileName);
}
}
else
{
var targetFramework = framework.TargetFramework;
if (targetFramework.StartsWith("MonoTouch", StringComparison.OrdinalIgnoreCase) ||
targetFramework.StartsWith("MonoAndroid", StringComparison.OrdinalIgnoreCase) ||
targetFramework.StartsWith("Xamarin.iOS", StringComparison.OrdinalIgnoreCase))
{
if (configuration.DiagnosticMessagesOrDefault)
{
logger.Log("Skipping: {0} (unsupported target framework '{1}')", fileName, targetFramework);
}
}
else
{
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
using (var visitor = visitorFactory(assemblyFileName, framework, discoveryOptions))
{
var usingAppDomains = framework.CanUseAppDomains && AppDomainDefaultBehavior != AppDomainSupport.Denied;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, usingAppDomains, shadowCopy, discoveryOptions));
framework.Find(testPlatformContext.RequireSourceInformation, visitor, discoveryOptions);
var totalTests = visitor.Finish();
visitComplete?.Invoke(assemblyFileName, framework, discoveryOptions, visitor);
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, totalTests, totalTests));
}
}
}
}
catch (Exception e)
{
var ex = e.Unwrap();
if (ex is InvalidOperationException)
{
logger.LogWarning("Skipping: {0} ({1})", fileName, ex.Message);
}
else if (ex is FileNotFoundException fileNotFound)
{
logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileNotFound.FileName));
}
#if !WINDOWS_UAP
else if (ex is FileLoadException fileLoad)
{
logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileLoad.FileName));
}
#endif
else
{
logger.LogWarning("Exception discovering tests from {0}: {1}", fileName, ex);
}
}
return(true);
}
public static void RunTests(Options o)
{
var nullMessage = new Xunit.NullMessageSink();
var discoveryOptions = TestFrameworkOptions.ForDiscovery();
using (var c = new XunitFrontController(AppDomainSupport.Denied, o.Assembly, null, false))
{
var tv = new TestDiscoverySink();
var excludeTestCaseSet = new TestDiscoverySink();
c.Find(true, tv, discoveryOptions);
tv.Finished.WaitOne();
foreach (var tc in tv.TestCases)
{
var method = tc.TestMethod.Method;
var attributes = method.GetCustomAttributes(typeof(FactAttribute));
foreach (ReflectionAttributeInfo at in attributes)
{
var result = at.GetNamedArgument<string>("Skip");
if (result != null)
{
Console.WriteLine("SKIPPY! {0} because {1}", method, result);
}
if (o.TestType != TestType.All)
{
if (!at.ToString().EndsWith(o.TestType.ToString()))
{
excludeTestCaseSet.TestCases.Add(tc);
}
}
}
}
foreach (var tc in excludeTestCaseSet.TestCases)
{
tv.TestCases.Remove(tc);
}
Console.WriteLine("TEST COUNT: {0}", tv.TestCases.Count);
//core execution Sink
int testCaseCount = tv.TestCases.Count;
Stream file = new FileStream(".\\result.xml", FileMode.Create);
int totalResult = 0;
int totalErrors = 0;
int totalFailed = 0;
int totalSkipped = 0;
for (int i = 0; i < testCaseCount; i++)
{
IExecutionSink resultsSink;
ConcurrentDictionary<string, ExecutionSummary> completionMessages = new ConcurrentDictionary<string, ExecutionSummary>();
IMessageSinkWithTypes reporterMessageHandler;
var reporters = GetAvailableRunnerReporters();
var commandLine = CommandLine.Parse(reporters, @"CoreXunit.dll");
IRunnerLogger logger = new ConsoleRunnerLogger(!commandLine.NoColor);
reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(commandLine.Reporter.CreateMessageHandler(logger));
var xmlElement = new XElement("TestResult");
resultsSink = new XmlAggregateSink(reporterMessageHandler, completionMessages, xmlElement, () => true);
var message = new Xunit.NullMessageSink();
var executionOptions = TestFrameworkOptions.ForExecution();
c.RunTests(tv.TestCases.Take<Xunit.Abstractions.ITestCase>(1), resultsSink, executionOptions);
resultsSink.Finished.WaitOne(o.TimeOut);
tv.TestCases.RemoveAt(0);
totalResult++;
totalErrors = totalErrors + resultsSink.ExecutionSummary.Errors;
totalFailed = totalFailed + resultsSink.ExecutionSummary.Failed;
totalSkipped = totalSkipped + resultsSink.ExecutionSummary.Skipped;
xmlElement.Save(file);
file.Flush();
}
file.Dispose();
Console.WriteLine("Total tests: " + totalResult);
Console.ForegroundColor = ConsoleColor.DarkRed;
Console.WriteLine("Error tests: " + totalErrors);
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("Failed tests: " + totalFailed);
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine("Skipped tests: " + totalSkipped);
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("Passed tests: " + (totalResult - totalErrors - totalFailed - totalSkipped));
Console.ResetColor();
}
}
public virtual IReadOnlyList <ITestCase> Discover() => Discover(sink => xunit.Find(false, sink, TestFrameworkOptions.ForDiscovery(configuration)));
public static void GuardClauses()
{
var framework = TestableXunitTestFrameworkDiscoverer.Create();
Assert.Throws <ArgumentNullException>("messageSink", () => framework.Find(includeSourceInformation: false, messageSink: null, discoveryOptions: TestFrameworkOptions.ForDiscovery()));
Assert.Throws <ArgumentNullException>("discoveryOptions", () => framework.Find(includeSourceInformation: false, messageSink: Substitute.For <IMessageSink>(), discoveryOptions: null));
}
static XElement ExecuteAssembly(object consoleLock,
XunitProjectAssembly assembly,
bool serialize,
bool needsXml,
bool?parallelizeTestCollections,
int?maxThreadCount,
bool diagnosticMessages,
bool noColor,
bool noAppDomain,
bool failSkips,
XunitFilters filters)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
{
return(null);
}
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= diagnosticMessages;
if (noAppDomain)
{
assembly.Configuration.AppDomain = AppDomainSupport.Denied;
}
// Setup discovery and execution options with command-line overrides
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue)
{
executionOptions.SetMaxParallelThreads(maxThreadCount);
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageSink = new DiagnosticMessageSink(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
using (var discoverySink = new TestDiscoverySink(() => cancel))
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions));
controller.Find(false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
}
else
{
if (serialize)
{
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary));
if (assemblyElement != null)
{
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
}
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
}
if (failSkips)
{
resultsSink = new DelegatingFailSkipSink(resultsSink);
}
controller.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine($"{e.GetType().FullName}: {e.Message}");
e = e.InnerException;
}
}
return(assemblyElement);
}
void DiscoverTests <TVisitor>(IEnumerable <string> sources,
LoggerHelper logger,
Func <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitorFactory,
Action <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitComplete = null)
where TVisitor : IVsDiscoveryVisitor, IDisposable
{
try
{
RemotingUtility.CleanUpRegisteredChannels();
using (AssemblyHelper.SubscribeResolve())
{
var reporterMessageHandler = new DefaultRunnerReporter().CreateMessageHandler(new VisualStudioRunnerLogger(logger));
foreach (var assemblyFileName in sources)
{
var assembly = new XunitProjectAssembly {
AssemblyFilename = assemblyFileName
};
var configuration = LoadConfiguration(assemblyFileName);
var fileName = Path.GetFileNameWithoutExtension(assemblyFileName);
var shadowCopy = configuration.ShadowCopyOrDefault;
try
{
if (cancelled)
{
break;
}
if (!IsXunitTestAssembly(assemblyFileName))
{
if (configuration.DiagnosticMessagesOrDefault)
{
logger.Log("Skipping: {0} (no reference to xUnit.net)", fileName);
}
}
else
{
var diagnosticMessageVisitor = new DiagnosticMessageVisitor(logger, fileName, configuration.DiagnosticMessagesOrDefault);
using (var framework = new XunitFrontController(AppDomain, assemblyFileName: assemblyFileName, configFileName: null, shadowCopy: shadowCopy, diagnosticMessageSink: diagnosticMessageVisitor))
{
var targetFramework = framework.TargetFramework;
if (targetFramework.StartsWith("MonoTouch", StringComparison.OrdinalIgnoreCase) ||
targetFramework.StartsWith("MonoAndroid", StringComparison.OrdinalIgnoreCase) ||
targetFramework.StartsWith("Xamarin.iOS", StringComparison.OrdinalIgnoreCase))
{
if (configuration.DiagnosticMessagesOrDefault)
{
logger.Log("Skipping: {0} (unsupported target framework '{1}')", fileName, targetFramework);
}
}
else
{
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
using (var visitor = visitorFactory(assemblyFileName, framework, discoveryOptions))
{
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, framework.CanUseAppDomains && AppDomain != AppDomainSupport.Denied, shadowCopy, discoveryOptions));
framework.Find(includeSourceInformation: true, messageSink: visitor, discoveryOptions: discoveryOptions);
var totalTests = visitor.Finish();
if (visitComplete != null)
{
visitComplete(assemblyFileName, framework, discoveryOptions, visitor);
}
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, totalTests, totalTests));
}
}
}
}
}
catch (Exception e)
{
var ex = e.Unwrap();
var fileNotFound = ex as FileNotFoundException;
#if !PLATFORM_DOTNET
var fileLoad = ex as FileLoadException;
#endif
if (ex is InvalidOperationException)
{
logger.LogWarning("Skipping: {0} ({1})", fileName, ex.Message);
}
else if (fileNotFound != null)
{
logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileNotFound.FileName));
}
#if !PLATFORM_DOTNET
else if (fileLoad != null)
{
logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileLoad.FileName));
}
#endif
else
{
logger.LogWarning("Exception discovering tests from {0}: {1}", fileName, ex);
}
}
}
}
}
catch (Exception e)
{
logger.LogWarning("Exception discovering tests: {0}", e.Unwrap());
}
}
protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly)
{
if (cancel)
{
return(null);
}
var assemblyElement = NeedsXml ? new XElement("assembly") : null;
try
{
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= DiagnosticMessages;
if (appDomains.HasValue)
{
assembly.Configuration.AppDomain = appDomains.GetValueOrDefault() ? AppDomainSupport.Required : AppDomainSupport.Denied;
}
// Setup discovery and execution options with command-line overrides
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue && maxThreadCount.Value > -1)
{
executionOptions.SetMaxParallelThreads(maxThreadCount);
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections);
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageVisitor = new DiagnosticMessageVisitor(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy, diagnosticMessageSink: diagnosticMessageVisitor))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, discoveryOptions));
controller.Find(false, discoveryVisitor, discoveryOptions);
discoveryVisitor.Finished.WaitOne();
var testCasesDiscovered = discoveryVisitor.TestCases.Count;
var filteredTestCases = discoveryVisitor.TestCases.Where(Filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
}
else
{
if (SerializeTestCases)
{
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
}
var resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel);
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary));
if (resultsVisitor.Failed != 0)
{
ExitCode = 1;
}
}
}
}
catch (Exception ex)
{
var e = ex;
while (e != null)
{
Log.LogError("{0}: {1}", e.GetType().FullName, e.Message);
foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
{
Log.LogError(stackLine);
}
e = e.InnerException;
}
ExitCode = -1;
}
return(assemblyElement);
}
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool appVeyor, bool showProgress, bool?parallelizeTestCollections, int?maxThreadCount, XunitFilters filters)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
{
return(null);
}
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue)
{
executionOptions.SetMaxParallelThreads(maxThreadCount.GetValueOrDefault());
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
}
lock (consoleLock)
{
if (assembly.Configuration.DiagnosticMessages ?? false)
{
Console.WriteLine("Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})",
Path.GetFileNameWithoutExtension(assembly.AssemblyFilename),
discoveryOptions.GetMethodDisplay(),
!executionOptions.GetDisableParallelization(),
executionOptions.GetMaxParallelThreads());
}
else
{
Console.WriteLine("Discovering: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
}
}
using (var controller = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.Configuration.ShadowCopyOrDefault))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions);
discoveryVisitor.Finished.WaitOne();
lock (consoleLock)
Console.WriteLine("Discovered: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, appVeyor, showProgress);
var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList();
if (filteredTestCases.Count == 0)
{
lock (consoleLock)
{
Console.ForegroundColor = ConsoleColor.DarkYellow;
Console.WriteLine("Info: {0} has no tests to run", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
Console.ForegroundColor = ConsoleColor.Gray;
}
}
else
{
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
}
}
}
catch (Exception ex)
{
Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message);
failed = true;
}
return(assemblyElement);
}
public void SupportsAsyncReturningMethods()
{
string code = @"
module FSharpTests
open Xunit
[<Fact>]
let AsyncFailing() =
async {
do! Async.Sleep(10)
Assert.True(false)
}
";
using (var assembly = FSharpAcceptanceTestV2Assembly.Create(code))
using (var controller = new TestableXunit2(assembly.FileName, null, true))
{
var sink = new SpyMessageSink <ITestAssemblyFinished>();
controller.RunAll(sink, discoveryOptions: TestFrameworkOptions.ForDiscovery(), executionOptions: TestFrameworkOptions.ForExecution());
sink.Finished.WaitOne();
var failures = sink.Messages.OfType <ITestFailed>();
var failure = Assert.Single(failures);
Assert.Equal("FSharpTests.AsyncFailing", failure.TestCase.DisplayName);
}
}
public void TheoryWithInlineData()
{
string code = @"
module FSharpTests
open Xunit
[<Theory>]
[<InlineData>]
[<InlineData(42)>]
[<InlineData(42, 21.12)>]
let TestMethod (x:int) =
Assert.True(true)
";
using (var assembly = FSharpAcceptanceTestV2Assembly.Create(code))
using (var controller = new TestableXunit2(assembly.FileName, null, true))
{
var sink = new TestDiscoverySink();
controller.Find(includeSourceInformation: false, messageSink: sink, discoveryOptions: TestFrameworkOptions.ForDiscovery());
sink.Finished.WaitOne();
Assert.Collection(sink.TestCases.OrderBy(tc => tc.DisplayName),
testCase => Assert.Equal("FSharpTests.TestMethod(x: ???)", testCase.DisplayName),
testCase => Assert.Equal("FSharpTests.TestMethod(x: 42)", testCase.DisplayName),
testCase => Assert.Equal("FSharpTests.TestMethod(x: 42, ???: 21.12)", testCase.DisplayName)
);
}
}
public static void TheoryWithNonSerializableData_ReturnsAsASingleTestCase()
{
var sourceProvider = new NullSourceInformationProvider();
var assemblyInfo = Reflector.Wrap(Assembly.GetExecutingAssembly());
var discoverer = new XunitTestFrameworkDiscoverer(assemblyInfo, sourceProvider, SpyMessageSink.Create());
var sink = new TestDiscoverySink();
discoverer.Find(typeof(ClassWithNonSerializableTheoryData).FullName, false, sink, TestFrameworkOptions.ForDiscovery());
sink.Finished.WaitOne();
var testCase = Assert.Single(sink.TestCases);
Assert.IsType <XunitTheoryTestCase>(testCase);
var deserialized = SerializationHelper.Deserialize <ITestCase>(SerializationHelper.Serialize(testCase));
Assert.IsType <XunitTheoryTestCase>(deserialized);
}
public void FactAcceptanceTest()
{
string code = @"
module FSharpTests
open Xunit
[<Fact>]
[<Trait(""Name!"", ""Value!"")>]
let Trait() =
Assert.True(true)
[<Fact(Skip = ""Skipping"")>]
let Skipped() =
Assert.True(false)
[<Fact(DisplayName=""Custom Test Name"")>]
let CustomName() =
Assert.True(true)
";
using (var assembly = FSharpAcceptanceTestV2Assembly.Create(code))
using (var controller = new TestableXunit2(assembly.FileName, null, true))
{
var sink = new TestDiscoverySink();
controller.Find(includeSourceInformation: false, messageSink: sink, discoveryOptions: TestFrameworkOptions.ForDiscovery());
sink.Finished.WaitOne();
Assert.Collection(sink.TestCases.OrderBy(tc => tc.DisplayName),
testCase => Assert.Equal("Custom Test Name", testCase.DisplayName),
testCase =>
{
Assert.Equal("FSharpTests.Skipped", testCase.DisplayName);
Assert.Equal("Skipping", testCase.SkipReason);
},
testCase =>
{
Assert.Equal("FSharpTests.Trait", testCase.DisplayName);
Assert.Collection(testCase.Traits,
kvp =>
{
Assert.Equal("Name!", kvp.Key);
Assert.Equal("Value!", kvp.Value.Single());
}
);
}
);
}
}
public void Find(string typeName, bool includeSourceInformation = false)
{
base.Find(typeName, includeSourceInformation, Visitor, TestFrameworkOptions.ForDiscovery());
Visitor.Finished.WaitOne();
}
IReadOnlyList <ITestCase> Discover(Type type) => Discover(sink => xunit.Find(type.FullName, false, sink, TestFrameworkOptions.ForDiscovery(configuration)));
public virtual bool FindTestsForClass(ITestClass testClass, bool includeSourceInformation = false)
{
using (var messageBus = new MessageBus(Visitor))
return(base.FindTestsForType(testClass, includeSourceInformation, messageBus, TestFrameworkOptions.ForDiscovery()));
}
static int Main(string[] args)
{
var nodeIndex = CommandLine.GetInt32("multinode.index");
var nodeRole = CommandLine.GetProperty("multinode.role");
var assemblyFileName = CommandLine.GetProperty("multinode.test-assembly");
var typeName = CommandLine.GetProperty("multinode.test-class");
var testName = CommandLine.GetProperty("multinode.test-method");
var displayName = testName;
var listenAddress = IPAddress.Parse(CommandLine.GetProperty("multinode.listen-address"));
var listenPort = CommandLine.GetInt32("multinode.listen-port");
var listenEndpoint = new IPEndPoint(listenAddress, listenPort);
var system = ActorSystem.Create("NoteTestRunner-" + nodeIndex);
var tcpClient = _logger = system.ActorOf <RunnerTcpClient>();
system.Tcp().Tell(new Tcp.Connect(listenEndpoint), tcpClient);
#if CORECLR
// In NetCore, if the assembly file hasn't been touched,
// XunitFrontController would fail loading external assemblies and its dependencies.
AssemblyLoadContext.Default.Resolving += (assemblyLoadContext, assemblyName) => DefaultOnResolving(assemblyLoadContext, assemblyName, assemblyFileName);
var assembly = AssemblyLoadContext.Default.LoadFromAssemblyPath(assemblyFileName);
DependencyContext.Load(assembly)
.CompileLibraries
.Where(dep => dep.Name.ToLower()
.Contains(assembly.FullName.Split(new[] { ',' })[0].ToLower()))
.Select(dependency => AssemblyLoadContext.Default.LoadFromAssemblyName(new AssemblyName(dependency.Name)));
#endif
Thread.Sleep(TimeSpan.FromSeconds(10));
using (var controller = new XunitFrontController(AppDomainSupport.IfAvailable, assemblyFileName))
{
/* need to pass in just the assembly name to Discovery, not the full path
* i.e. "Akka.Cluster.Tests.MultiNode.dll"
* not "bin/Release/Akka.Cluster.Tests.MultiNode.dll" - this will cause
* the Discovery class to actually not find any individual specs to run
*/
var assemblyName = Path.GetFileName(assemblyFileName);
Console.WriteLine("Running specs for {0} [{1}]", assemblyName, assemblyFileName);
using (var discovery = new Discovery(assemblyName, typeName))
{
using (var sink = new Sink(nodeIndex, nodeRole, tcpClient))
{
try
{
controller.Find(true, discovery, TestFrameworkOptions.ForDiscovery());
discovery.Finished.WaitOne();
controller.RunTests(discovery.TestCases, sink, TestFrameworkOptions.ForExecution());
}
catch (AggregateException ex)
{
var specFail = new SpecFail(nodeIndex, nodeRole, displayName);
specFail.FailureExceptionTypes.Add(ex.GetType().ToString());
specFail.FailureMessages.Add(ex.Message);
specFail.FailureStackTraces.Add(ex.StackTrace);
foreach (var innerEx in ex.Flatten().InnerExceptions)
{
specFail.FailureExceptionTypes.Add(innerEx.GetType().ToString());
specFail.FailureMessages.Add(innerEx.Message);
specFail.FailureStackTraces.Add(innerEx.StackTrace);
}
_logger.Tell(specFail.ToString());
Console.WriteLine(specFail);
//make sure message is send over the wire
FlushLogMessages();
Environment.Exit(1); //signal failure
return(1);
}
catch (Exception ex)
{
var specFail = new SpecFail(nodeIndex, nodeRole, displayName);
specFail.FailureExceptionTypes.Add(ex.GetType().ToString());
specFail.FailureMessages.Add(ex.Message);
specFail.FailureStackTraces.Add(ex.StackTrace);
var innerEx = ex.InnerException;
while (innerEx != null)
{
specFail.FailureExceptionTypes.Add(innerEx.GetType().ToString());
specFail.FailureMessages.Add(innerEx.Message);
specFail.FailureStackTraces.Add(innerEx.StackTrace);
innerEx = innerEx.InnerException;
}
_logger.Tell(specFail.ToString());
Console.WriteLine(specFail);
//make sure message is send over the wire
FlushLogMessages();
Environment.Exit(1); //signal failure
return(1);
}
var timedOut = false;
if (!sink.Finished.WaitOne(MaxProcessWaitTimeout)) //timed out
{
var line = string.Format("Timed out while waiting for test to complete after {0} ms",
MaxProcessWaitTimeout);
_logger.Tell(line);
Console.WriteLine(line);
timedOut = true;
}
FlushLogMessages();
system.Terminate().Wait();
Environment.Exit(sink.Passed && !timedOut ? 0 : 1);
return(sink.Passed ? 0 : 1);
}
}
}
}
/// <summary>
/// MultiNodeTestRunner takes the following <see cref="args"/>:
///
/// C:\> Akka.MultiNodeTestRunner.exe [assembly name] [-Dmultinode.enable-filesink=on] [-Dmultinode.output-directory={dir path}] [-Dmultinode.spec={spec name}]
///
/// <list type="number">
/// <listheader>
/// <term>Argument</term>
/// <description>The name and possible value of a given Akka.MultiNodeTestRunner.exe argument.</description>
/// </listheader>
/// <item>
/// <term>AssemblyName</term>
/// <description>
/// The full path or name of an assembly containing as least one MultiNodeSpec in the current working directory.
///
/// i.e. "Akka.Cluster.Tests.MultiNode.dll"
/// "C:\akka.net\src\Akka.Cluster.Tests\bin\Debug\Akka.Cluster.Tests.MultiNode.dll"
/// </description>
/// </item>
/// <item>
/// <term>-Dmultinode.enable-filesink</term>
/// <description>Having this flag set means that the contents of this test run will be saved in the
/// current working directory as a .JSON file.
/// </description>
/// </item>
/// <item>
/// <term>-Dmultinode.multinode.output-directory</term>
/// <description>Setting this flag means that any persistent multi-node test runner output files
/// will be written to this directory instead of the default, which is the same folder
/// as the test binary.
/// </description>
/// </item>
/// <item>
/// <term>-Dmultinode.listen-address={ip}</term>
/// <description>
/// Determines the address that this multi-node test runner will use to listen for log messages from
/// individual NodeTestRunner.exe processes.
///
/// Defaults to 127.0.0.1
/// </description>
/// </item>
/// <item>
/// <term>-Dmultinode.listen-port={port}</term>
/// <description>
/// Determines the port number that this multi-node test runner will use to listen for log messages from
/// individual NodeTestRunner.exe processes.
///
/// Defaults to 6577
/// </description>
/// </item>
/// <item>
/// <term>-Dmultinode.spec={spec name}</term>
/// <description>
/// Setting this flag means that only tests which contains the spec name will be executed
/// otherwise all tests will be executed
/// </description>
/// </item>
/// </list>
/// </summary>
static void Main(string[] args)
{
OutputDirectory = CommandLine.GetPropertyOrDefault("multinode.output-directory", string.Empty);
FailedSpecsDirectory = CommandLine.GetPropertyOrDefault("multinode.failed-specs-directory", "FAILED_SPECS_LOGS");
string logLevel = CommandLine.GetPropertyOrDefault("multinode.loglevel", "WARNING");
TestRunSystem = ActorSystem.Create("TestRunnerLogging", $"akka.loglevel={logLevel}");
var suiteName = Path.GetFileNameWithoutExtension(Path.GetFullPath(args[0].Trim('"')));
var teamCityFormattingOn = CommandLine.GetPropertyOrDefault("multinode.teamcity", "false");
if (!Boolean.TryParse(teamCityFormattingOn, out TeamCityFormattingOn))
{
throw new ArgumentException("Invalid argument provided for -Dteamcity");
}
var listenAddress = IPAddress.Parse(CommandLine.GetPropertyOrDefault("multinode.listen-address", "127.0.0.1"));
var listenPort = CommandLine.GetInt32OrDefault("multinode.listen-port", 6577);
var listenEndpoint = new IPEndPoint(listenAddress, listenPort);
var specName = CommandLine.GetPropertyOrDefault("multinode.spec", "");
var platform = CommandLine.GetPropertyOrDefault("multinode.platform", "net");
var reporter = CommandLine.GetPropertyOrDefault("multinode.reporter", "console");
var clearOutputDirectory = CommandLine.GetInt32OrDefault("multinode.clear-output", 0);
if (clearOutputDirectory > 0 && Directory.Exists(OutputDirectory))
{
Directory.Delete(OutputDirectory, true);
}
Props coordinatorProps;
switch (reporter.ToLowerInvariant())
{
case "trx":
coordinatorProps = Props.Create(() => new SinkCoordinator(new[] { new TrxMessageSink(suiteName) }));
break;
case "teamcity":
coordinatorProps = Props.Create(() => new SinkCoordinator(new[] { new TeamCityMessageSink(Console.WriteLine, suiteName) }));
break;
case "console":
coordinatorProps = Props.Create(() => new SinkCoordinator(new[] { new ConsoleMessageSink() }));
break;
default:
throw new ArgumentException($"Given reporter name '{reporter}' is not understood, valid reporters are: trx and teamcity");
}
SinkCoordinator = TestRunSystem.ActorOf(coordinatorProps, "sinkCoordinator");
#if CORECLR
if (!_validNetCorePlatform.Contains(platform))
{
throw new Exception($"Target platform not supported: {platform}. Supported platforms are net and netcore");
}
#else
if (platform != "net")
{
throw new Exception($"Target platform not supported: {platform}. Supported platforms are net");
}
#endif
var tcpLogger = TestRunSystem.ActorOf(Props.Create(() => new TcpLoggingServer(SinkCoordinator)), "TcpLogger");
TestRunSystem.Tcp().Tell(new Tcp.Bind(tcpLogger, listenEndpoint), sender: tcpLogger);
var assemblyPath = Path.GetFullPath(args[0].Trim('"')); //unquote the string first
EnableAllSinks(assemblyPath, platform);
PublishRunnerMessage($"Running MultiNodeTests for {assemblyPath}");
#if CORECLR
// In NetCore, if the assembly file hasn't been touched,
// XunitFrontController would fail loading external assemblies and its dependencies.
var assembly = AssemblyLoadContext.Default.LoadFromAssemblyPath(assemblyPath);
var asms = assembly.GetReferencedAssemblies();
var basePath = Path.GetDirectoryName(assemblyPath);
foreach (var asm in asms)
{
try
{
Assembly.Load(new AssemblyName(asm.FullName));
}
catch (Exception)
{
var path = Path.Combine(basePath, asm.Name + ".dll");
try
{
AssemblyLoadContext.Default.LoadFromAssemblyPath(path);
}
catch (Exception e)
{
Console.Out.WriteLine($"Failed to load dll: {path}");
}
}
}
#endif
using (var controller = new XunitFrontController(AppDomainSupport.IfAvailable, assemblyPath))
{
using (var discovery = new Discovery())
{
controller.Find(false, discovery, TestFrameworkOptions.ForDiscovery());
discovery.Finished.WaitOne();
if (discovery.WasSuccessful)
{
foreach (var test in discovery.Tests.Reverse())
{
if (!string.IsNullOrEmpty(test.Value.First().SkipReason))
{
PublishRunnerMessage($"Skipping test {test.Value.First().MethodName}. Reason - {test.Value.First().SkipReason}");
continue;
}
if (!string.IsNullOrWhiteSpace(specName) &&
CultureInfo.InvariantCulture.CompareInfo.IndexOf(test.Value.First().TestName,
specName,
CompareOptions.IgnoreCase) < 0)
{
PublishRunnerMessage($"Skipping [{test.Value.First().MethodName}] (Filtering)");
continue;
}
var processes = new List <Process>();
PublishRunnerMessage($"Starting test {test.Value.First().MethodName}");
Console.Out.WriteLine($"Starting test {test.Value.First().MethodName}");
StartNewSpec(test.Value);
#if CORECLR
var ntrNetPath = Path.Combine(AppContext.BaseDirectory, "Akka.NodeTestRunner.exe");
var ntrNetCorePath = Path.Combine(AppContext.BaseDirectory, "Akka.NodeTestRunner.dll");
var alternateIndex = 0;
#endif
var timelineCollector = TestRunSystem.ActorOf(Props.Create(() => new TimelineLogCollectorActor()));
string testOutputDir = null;
string runningSpecName = null;
foreach (var nodeTest in test.Value)
{
//Loop through each test, work out number of nodes to run on and kick off process
var sbArguments = new StringBuilder()
//.Append($@"-Dmultinode.test-assembly=""{assemblyPath}"" ")
.Append($@"-Dmultinode.test-class=""{nodeTest.TypeName}"" ")
.Append($@"-Dmultinode.test-method=""{nodeTest.MethodName}"" ")
.Append($@"-Dmultinode.max-nodes={test.Value.Count} ")
.Append($@"-Dmultinode.server-host=""{"localhost"}"" ")
.Append($@"-Dmultinode.host=""{"localhost"}"" ")
.Append($@"-Dmultinode.index={nodeTest.Node - 1} ")
.Append($@"-Dmultinode.role=""{nodeTest.Role}"" ")
.Append($@"-Dmultinode.listen-address={listenAddress} ")
.Append($@"-Dmultinode.listen-port={listenPort} ");
#if CORECLR
string fileName = null;
switch (platform)
{
case "net":
fileName = ntrNetPath;
sbArguments.Insert(0, $@" -Dmultinode.test-assembly=""{assemblyPath}"" ");
break;
case "netcore":
case "net5":
fileName = "dotnet";
sbArguments.Insert(0, $@" -Dmultinode.test-assembly=""{assemblyPath}"" ");
sbArguments.Insert(0, ntrNetCorePath);
break;
}
var process = new Process
{
StartInfo = new ProcessStartInfo
{
FileName = fileName,
UseShellExecute = false,
RedirectStandardOutput = true,
Arguments = sbArguments.ToString(),
WorkingDirectory = Path.GetDirectoryName(assemblyPath)
}
};
#else
sbArguments.Insert(0, $@"-Dmultinode.test-assembly=""{assemblyPath}"" ");
var process = new Process
{
StartInfo = new ProcessStartInfo
{
FileName = "Akka.NodeTestRunner.exe",
UseShellExecute = false,
RedirectStandardOutput = true,
Arguments = sbArguments.ToString()
}
};
#endif
processes.Add(process);
var nodeIndex = nodeTest.Node;
var nodeRole = nodeTest.Role;
#if CORECLR
if (platform == "netcore" || platform == "net5")
{
process.StartInfo.FileName = "dotnet";
process.StartInfo.Arguments = ntrNetCorePath + " " + process.StartInfo.Arguments;
process.StartInfo.WorkingDirectory = Path.GetDirectoryName(assemblyPath);
}
#endif
//TODO: might need to do some validation here to avoid the 260 character max path error on Windows
var folder = Directory.CreateDirectory(Path.Combine(OutputDirectory, nodeTest.TestName));
testOutputDir = testOutputDir ?? folder.FullName;
var logFilePath = Path.Combine(folder.FullName, $"node{nodeIndex}__{nodeRole}__{platform}.txt");
runningSpecName = nodeTest.TestName;
var nodeInfo = new TimelineLogCollectorActor.NodeInfo(nodeIndex, nodeRole, platform, nodeTest.TestName);
var fileActor = TestRunSystem.ActorOf(Props.Create(() => new FileSystemAppenderActor(logFilePath)));
process.OutputDataReceived += (sender, eventArgs) =>
{
if (eventArgs?.Data != null)
{
fileActor.Tell(eventArgs.Data);
timelineCollector.Tell(new TimelineLogCollectorActor.LogMessage(nodeInfo, eventArgs.Data));
if (TeamCityFormattingOn)
{
// teamCityTest.WriteStdOutput(eventArgs.Data); TODO: open flood gates
}
}
};
var closureTest = nodeTest;
process.Exited += (sender, eventArgs) =>
{
if (process.ExitCode == 0)
{
ReportSpecPassFromExitCode(nodeIndex, nodeRole, closureTest.TestName);
}
};
process.Start();
process.BeginOutputReadLine();
PublishRunnerMessage($"Started node {nodeIndex} : {nodeRole} on pid {process.Id}");
}
var specFailed = false;
foreach (var process in processes)
{
process.WaitForExit();
specFailed = specFailed || process.ExitCode > 0;
process.Dispose();
}
PublishRunnerMessage("Waiting 3 seconds for all messages from all processes to be collected.");
Thread.Sleep(TimeSpan.FromSeconds(3));
if (testOutputDir != null)
{
var dumpTasks = new List <Task>()
{
// Dump aggregated timeline to file for this test
timelineCollector.Ask <Done>(new TimelineLogCollectorActor.DumpToFile(Path.Combine(testOutputDir, "aggregated.txt"))),
// Print aggregated timeline into the console
timelineCollector.Ask <Done>(new TimelineLogCollectorActor.PrintToConsole(logLevel))
};
if (specFailed)
{
var dumpFailureArtifactTask = timelineCollector.Ask <Done>(
new TimelineLogCollectorActor.DumpToFile(Path.Combine(Path.GetFullPath(OutputDirectory), FailedSpecsDirectory, $"{runningSpecName}.txt")));
dumpTasks.Add(dumpFailureArtifactTask);
}
Task.WaitAll(dumpTasks.ToArray());
}
FinishSpec(test.Value, timelineCollector);
}
Console.WriteLine("Complete");
PublishRunnerMessage("Waiting 5 seconds for all messages from all processes to be collected.");
Thread.Sleep(TimeSpan.FromSeconds(5));
}
else
{
var sb = new StringBuilder();
sb.AppendLine("One or more exception was thrown while discovering test cases. Test Aborted.");
foreach (var err in discovery.Errors)
{
for (int i = 0; i < err.ExceptionTypes.Length; ++i)
{
sb.AppendLine();
sb.Append($"{err.ExceptionTypes[i]}: {err.Messages[i]}");
sb.Append(err.StackTraces[i]);
}
}
PublishRunnerMessage(sb.ToString());
Console.Out.WriteLine(sb.ToString());
}
}
}
AbortTcpLoggingServer(tcpLogger);
CloseAllSinks();
//Block until all Sinks have been terminated.
TestRunSystem.WhenTerminated.Wait(TimeSpan.FromMinutes(1));
if (Debugger.IsAttached)
{
Console.ReadLine(); //block when debugging
}
//Return the proper exit code
Environment.Exit(ExitCodeContainer.ExitCode);
}
void RunTestsInAssembly(IRunContext runContext,
IFrameworkHandle frameworkHandle,
LoggerHelper logger,
TestPlatformContext testPlatformContext,
RunSettings runSettings,
IMessageSinkWithTypes reporterMessageHandler,
AssemblyRunInfo runInfo)
{
if (cancelled)
{
return;
}
var assemblyDisplayName = "(unknown assembly)";
try
{
var assembly = new XunitProjectAssembly {
AssemblyFilename = runInfo.AssemblyFileName
};
var assemblyFileName = runInfo.AssemblyFileName;
assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName);
var configuration = runInfo.Configuration;
var shadowCopy = configuration.ShadowCopyOrDefault;
var appDomain = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
if (runSettings.DisableAppDomain)
{
appDomain = AppDomainSupport.Denied;
}
#if WINDOWS_UAP
// For AppX Apps, use the package location
assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName));
#endif
var diagnosticSink = DiagnosticMessageSink.ForDiagnostics(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault);
var diagnosticMessageSink = MessageSinkAdapter.Wrap(diagnosticSink);
using (var controller = new XunitFrontController(appDomain, assemblyFileName, shadowCopy: shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
{
var testCasesMap = new Dictionary <string, TestCase>();
var testCases = new List <ITestCase>();
if (runInfo.TestCases == null || !runInfo.TestCases.Any())
{
// Discover tests
AssemblyDiscoveredInfo assemblyDiscoveredInfo = new AssemblyDiscoveredInfo();
DiscoverTestsInSource(controller, logger, testPlatformContext, runSettings,
(source, discoverer, discoveryOptions) => new VsExecutionDiscoverySink(() => cancelled),
(source, discoverer, discoveryOptions, visitor) =>
{
if (discoveryOptions.GetInternalDiagnosticMessagesOrDefault())
{
foreach (var testCase in visitor.TestCases)
{
logger.Log(testCase, "Discovered [execution] test case '{0}' (ID = '{1}')",
testCase.DisplayName, testCase.UniqueID);
}
}
assemblyDiscoveredInfo = new AssemblyDiscoveredInfo
{
AssemblyFileName = source,
DiscoveredTestCases = GetVsTestCases(source, discoverer, visitor, logger, testPlatformContext)
};
},
assemblyFileName,
shadowCopy,
configuration
);
if (assemblyDiscoveredInfo.DiscoveredTestCases == null || !assemblyDiscoveredInfo.DiscoveredTestCases.Any())
{
if (configuration.InternalDiagnosticMessagesOrDefault)
{
logger.LogWarning("Skipping '{0}' since no tests were found during discovery [execution].", assemblyDiscoveredInfo.AssemblyFileName);
}
return;
}
// Filter tests
var traitNames = new HashSet <string>(assemblyDiscoveredInfo.DiscoveredTestCases.SelectMany(testCase => testCase.TraitNames));
var filter = new TestCaseFilter(runContext, logger, assemblyDiscoveredInfo.AssemblyFileName, traitNames);
var filteredTestCases = assemblyDiscoveredInfo.DiscoveredTestCases.Where(dtc => filter.MatchTestCase(dtc.VSTestCase)).ToList();
foreach (var filteredTestCase in filteredTestCases)
{
var uniqueID = filteredTestCase.UniqueID;
if (testCasesMap.ContainsKey(uniqueID))
{
logger.LogWarning(filteredTestCase.TestCase, "Skipping test case with duplicate ID '{0}' ('{1}' and '{2}')", uniqueID, testCasesMap[uniqueID].DisplayName, filteredTestCase.VSTestCase.DisplayName);
}
else
{
testCasesMap.Add(uniqueID, filteredTestCase.VSTestCase);
testCases.Add(filteredTestCase.TestCase);
}
}
}
else
{
try
{
var serializations = runInfo.TestCases
.Select(tc => tc.GetPropertyValue <string>(SerializedTestCaseProperty, null))
.ToList();
if (configuration.InternalDiagnosticMessagesOrDefault)
{
logger.LogWithSource(runInfo.AssemblyFileName, "Deserializing {0} test case(s):{1}{2}",
serializations.Count,
Environment.NewLine,
string.Join(Environment.NewLine, serializations.Select(x => $" {x}")));
}
var deserializedTestCasesByUniqueId = controller.BulkDeserialize(serializations);
if (deserializedTestCasesByUniqueId == null)
{
logger.LogErrorWithSource(assemblyFileName, "Received null response from BulkDeserialize");
}
else
{
for (int idx = 0; idx < runInfo.TestCases.Count; ++idx)
{
try
{
var kvp = deserializedTestCasesByUniqueId[idx];
var vsTestCase = runInfo.TestCases[idx];
if (kvp.Value == null)
{
logger.LogErrorWithSource(assemblyFileName, "Test case {0} failed to deserialize: {1}", vsTestCase.DisplayName, kvp.Key);
}
else
{
testCasesMap.Add(kvp.Key, vsTestCase);
testCases.Add(kvp.Value);
}
}
catch (Exception ex)
{
logger.LogErrorWithSource(assemblyFileName, "Catastrophic error deserializing item #{0}: {1}", idx, ex);
}
}
}
}
catch (Exception ex)
{
logger.LogWarningWithSource(assemblyFileName, "Catastrophic error during deserialization: {0}", ex);
}
}
// Execute tests
var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration);
if (runSettings.DisableParallelization)
{
executionOptions.SetSynchronousMessageReporting(true);
executionOptions.SetDisableParallelization(true);
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, testCasesMap, executionOptions, () => cancelled))
{
IExecutionSink resultsSink = vsExecutionSink;
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink);
}
controller.RunTests(testCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
}
}
}
catch (Exception ex)
{
logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex);
}
}
public async void RunTests(TestNodeViewModel testNodeViewModel)
{
var testCases = testNodeViewModel.EnumerateTestCases();
var testCaseViewModels = new Dictionary <string, TestCaseViewModel>();
foreach (var testCase in testCases)
{
testCaseViewModels.Add(testCase.TestCase.UniqueID, testCase);
}
int testCasesFinished = 0;
await Task.Run(() =>
{
// Reset progress
Dispatcher.UIThread.Post(() =>
{
TestCompletion = 0.0;
RunningTests = true;
});
var sink = new XSink
{
HandleTestCaseStarting = args =>
{
if (testCaseViewModels.TryGetValue(args.Message.TestCase.UniqueID, out var testCaseViewModel))
{
Dispatcher.UIThread.Post(() =>
{
// Test status
testCaseViewModel.Running = true;
});
}
},
HandleTestCaseFinished = args =>
{
if (testCaseViewModels.TryGetValue(args.Message.TestCase.UniqueID, out var testCaseViewModel))
{
Dispatcher.UIThread.Post(() =>
{
// Test status
testCaseViewModel.Failed = args.Message.TestsFailed > 0;
testCaseViewModel.Succeeded = args.Message.TestsFailed == 0;
testCaseViewModel.Running = false;
// Update progress
TestCompletion = ((double)Interlocked.Increment(ref testCasesFinished) / (double)testCaseViewModels.Count) * 100.0;
});
}
},
};
Controller.RunTests(testCaseViewModels.Select(x => x.Value.TestCase).ToArray(), sink, TestFrameworkOptions.ForExecution());
sink.Finished.WaitOne();
Dispatcher.UIThread.Post(() =>
{
RunningTests = false;
});
});
}
public void TheoryWithInlineData()
{
string code = @"
using System;
using Xunit;
public class TestClass
{
[Theory]
[InlineData]
[InlineData(42)]
[InlineData(42, 21.12)]
public void TestMethod(int x) { }
}
";
using (var assembly = new AcceptanceTestAssembly(code))
using (var controller = new TestableXunit2(assembly.FileName, null, true))
{
var sink = new SpyMessageSink <IDiscoveryCompleteMessage>();
controller.Find(includeSourceInformation: false, messageSink: sink, discoveryOptions: TestFrameworkOptions.ForDiscovery());
sink.Finished.WaitOne();
string[] testCaseNames = sink.Messages.OfType <ITestCaseDiscoveryMessage>().Select(tcdm => tcdm.TestCase.DisplayName).ToArray();
Assert.Equal(3, testCaseNames.Length);
Assert.Contains("TestClass.TestMethod(x: ???)", testCaseNames);
Assert.Contains("TestClass.TestMethod(x: 42)", testCaseNames);
Assert.Contains(String.Format("TestClass.TestMethod(x: 42, ???: {0})", 21.12), testCaseNames);
}
}
private IEnumerable <ITestCase> Discover(Type type)
{
return(Discover(sink => xunit.Find(type.FullName, false, sink, TestFrameworkOptions.ForDiscovery(configuration))));
}
XElement ExecuteAssembly(object consoleLock,
XunitProjectAssembly assembly,
bool needsXml,
bool?parallelizeTestCollections,
int?maxThreadCount,
bool diagnosticMessages,
bool noColor,
XunitFilters filters,
bool designTime,
bool listTestCases,
IReadOnlyList <string> designTimeFullyQualifiedNames)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
// Turn off pre-enumeration of theories when we're not running in Visual Studio
if (!designTime)
{
assembly.Configuration.PreEnumerateTheories = false;
}
if (diagnosticMessages)
{
assembly.Configuration.DiagnosticMessages = true;
}
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue)
{
executionOptions.SetMaxParallelThreads(maxThreadCount);
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageVisitor = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
var sourceInformationProvider = new SourceInformationProviderAdapater(services);
using (var controller = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, false, diagnosticMessageSink: diagnosticMessageVisitor, sourceInformationProvider: sourceInformationProvider))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
var includeSourceInformation = designTime && listTestCases;
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, false, false, discoveryOptions));
controller.Find(includeSourceInformation: includeSourceInformation, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions);
discoveryVisitor.Finished.WaitOne();
IDictionary <ITestCase, VsTestCase> vsTestCases = null;
if (designTime)
{
vsTestCases = DesignTimeTestConverter.Convert(discoveryVisitor.TestCases);
}
if (listTestCases)
{
lock (consoleLock)
{
if (designTime)
{
var sink = (ITestDiscoverySink)services.GetService(typeof(ITestDiscoverySink));
foreach (var testcase in vsTestCases.Values)
{
if (sink != null)
{
sink.SendTest(testcase);
}
Console.WriteLine(testcase.FullyQualifiedName);
}
}
else
{
foreach (var testcase in discoveryVisitor.TestCases)
{
Console.WriteLine(testcase.DisplayName);
}
}
}
return(assemblyElement);
}
IExecutionVisitor resultsVisitor;
if (designTime)
{
var sink = (ITestExecutionSink)services.GetService(typeof(ITestExecutionSink));
resultsVisitor = new DesignTimeExecutionVisitor(sink, vsTestCases, reporterMessageHandler);
}
else
{
resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel);
}
IList <ITestCase> filteredTestCases;
var testCasesDiscovered = discoveryVisitor.TestCases.Count;
if (!designTime || designTimeFullyQualifiedNames.Count == 0)
{
filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList();
}
else
{
filteredTestCases = vsTestCases.Where(t => designTimeFullyQualifiedNames.Contains(t.Value.FullyQualifiedName)).Select(t => t.Key).ToList();
}
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
if (filteredTestCases.Count == 0)
{
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
}
else
{
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary));
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine("{0}: {1}", e.GetType().FullName, e.Message);
e = e.InnerException;
}
}
return(assemblyElement);
}
public void SingleTestMethod()
{
string code = @"
using Xunit;
public class Foo
{
[Fact]
public void Bar() { }
}
";
using (var assm = new AcceptanceTestAssembly(code))
using (var controller = new TestableXunit2(assm.FileName, null, true))
{
var sink = new SpyMessageSink <IDiscoveryCompleteMessage>();
controller.Find(includeSourceInformation: false, messageSink: sink, discoveryOptions: TestFrameworkOptions.ForDiscovery());
sink.Finished.WaitOne();
ITestCase testCase = sink.Messages.OfType <ITestCaseDiscoveryMessage>().Single().TestCase;
Assert.Equal("Foo.Bar", testCase.DisplayName);
}
}
public virtual IEnumerable <ITestCase> Discover()
{
return(Discover(sink => xunit.Find(false, sink, TestFrameworkOptions.ForDiscovery(configuration))));
}
public void FactAcceptanceTest()
{
string code = @"
using System;
using Xunit;
namespace Namespace1
{
public class Class1
{
[Fact]
[Trait(""Name!"", ""Value!"")]
public void Trait() { }
[Fact(Skip=""Skipping"")]
public void Skipped() { }
[Fact(DisplayName=""Custom Test Name"")]
public void CustomName() { }
}
}
namespace Namespace2
{
public class OuterClass
{
public class Class2
{
[Fact]
public void TestMethod() { }
}
}
}
";
using (var assembly = new AcceptanceTestAssembly(code))
using (var controller = new TestableXunit2(assembly.FileName, null, true))
{
var sink = new SpyMessageSink <IDiscoveryCompleteMessage>();
controller.Find(includeSourceInformation: false, messageSink: sink, discoveryOptions: TestFrameworkOptions.ForDiscovery());
sink.Finished.WaitOne();
ITestCase[] testCases = sink.Messages.OfType <ITestCaseDiscoveryMessage>().Select(tcdm => tcdm.TestCase).ToArray();
Assert.Equal(4, testCases.Length);
ITestCase traitTest = Assert.Single(testCases, tc => tc.DisplayName == "Namespace1.Class1.Trait");
string key = Assert.Single(traitTest.Traits.Keys);
Assert.Equal("Name!", key);
string value = Assert.Single(traitTest.Traits[key]);
Assert.Equal("Value!", value);
ITestCase skipped = Assert.Single(testCases, tc => tc.DisplayName == "Namespace1.Class1.Skipped");
Assert.Equal("Skipping", skipped.SkipReason);
Assert.Single(testCases, tc => tc.DisplayName == "Custom Test Name");
Assert.Single(testCases, tc => tc.DisplayName == "Namespace2.OuterClass+Class2.TestMethod");
}
}
public static void GuardClauses()
{
var framework = TestableXunitTestFrameworkDiscoverer.Create();
var typeName = typeof(Object).FullName;
var sink = Substitute.For<IMessageSink>();
var options = new TestFrameworkOptions();
Assert.Throws<ArgumentNullException>("typeName", () => framework.Find(typeName: null, includeSourceInformation: false, messageSink: sink, options: options));
Assert.Throws<ArgumentException>("typeName", () => framework.Find(typeName: "", includeSourceInformation: false, messageSink: sink, options: options));
Assert.Throws<ArgumentNullException>("messageSink", () => framework.Find(typeName, includeSourceInformation: false, messageSink: null, options: options));
Assert.Throws<ArgumentNullException>("options", () => framework.Find(typeName, includeSourceInformation: false, messageSink: sink, options: null));
}
public int Run(string assemblyFileName, bool printXml, XunitFilters filters)
{
var configuration = new TestAssemblyConfiguration()
{
ShadowCopy = false, ParallelizeAssembly = false, ParallelizeTestCollections = false, MaxParallelThreads = 1, PreEnumerateTheories = false
};
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
var discoverySink = new TestDiscoverySink();
var diagnosticSink = new ConsoleDiagnosticMessageSink();
var testOptions = TestFrameworkOptions.ForExecution(configuration);
var testSink = new TestMessageSink();
var controller = new Xunit2(AppDomainSupport.Denied, new NullSourceInformationProvider(), assemblyFileName, configFileName: null, shadowCopy: false, shadowCopyFolder: null, diagnosticMessageSink: diagnosticSink, verifyTestAssemblyExists: false);
discoveryOptions.SetSynchronousMessageReporting(true);
testOptions.SetSynchronousMessageReporting(true);
Console.WriteLine($"Discovering tests for {assemblyFileName}");
var assembly = Assembly.LoadFrom(assemblyFileName);
var assemblyInfo = new global::Xunit.Sdk.ReflectionAssemblyInfo(assembly);
var discoverer = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink);
discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList();
Console.WriteLine($"Discovery finished.");
var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => { Console.WriteLine($"Tests run: {summary.Total}, Errors: 0, Failures: {summary.Failed}, Skipped: {summary.Skipped}. Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s"); });
var resultsXmlAssembly = new XElement("assembly");
var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly);
testSink.Execution.TestPassedEvent += args => { Console.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); };
testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); };
testSink.Execution.TestFailedEvent += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}"); };
testSink.Execution.TestAssemblyStartingEvent += args => { Console.WriteLine($"Running tests for {args.Message.TestAssembly.Assembly}"); };
testSink.Execution.TestAssemblyFinishedEvent += args => { Console.WriteLine($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); };
controller.RunTests(testCasesToRun, resultsSink, testOptions);
var threadpoolPump = typeof(ThreadPool).GetMethod("PumpThreadPool", BindingFlags.NonPublic | BindingFlags.Static);
var timerPump = Type.GetType("System.Threading.TimerQueue")?.GetMethod("PumpTimerQueue", BindingFlags.NonPublic | BindingFlags.Static);
if (threadpoolPump != null && timerPump != null)
{
while (!resultsSink.Finished.WaitOne(0))
{
threadpoolPump.Invoke(this, null);
timerPump.Invoke(this, null);
}
}
else
{
resultsSink.Finished.WaitOne();
}
if (printXml)
{
Console.WriteLine($"STARTRESULTXML");
var resultsXml = new XElement("assemblies");
resultsXml.Add(resultsXmlAssembly);
resultsXml.Save(Console.OpenStandardOutput());
Console.WriteLine();
Console.WriteLine($"ENDRESULTXML");
}
var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0;
return(failed ? 1 : 0);
}
public void NoTestMethods()
{
using (var assm = new AcceptanceTestAssembly(code: ""))
using (var controller = new TestableXunit2(assm.FileName, null, true))
{
var sink = new SpyMessageSink <IDiscoveryCompleteMessage>();
controller.Find(includeSourceInformation: false, messageSink: sink, discoveryOptions: TestFrameworkOptions.ForDiscovery());
sink.Finished.WaitOne();
Assert.False(sink.Messages.Any(msg => msg is ITestCaseDiscoveryMessage));
}
}
static int Main(string[] args)
{
if (args.Length < 3)
{
Console.WriteLine("Usage: <out-dir> <corefx dir> <test assembly filename> <xunit console options>");
return(1);
}
var testAssemblyName = Path.GetFileNameWithoutExtension(args [2]);
var testAssemblyFull = Path.GetFullPath(args[2]);
var outdir_name = Path.Combine(args [0], testAssemblyName);
var sdkdir = args [1] + "/artifacts/bin/runtime/netcoreapp-OSX-Debug-x64";
args = args.Skip(2).ToArray();
// Response file support
var extra_args = new List <string> ();
for (int i = 0; i < args.Length; ++i)
{
var arg = args [i];
if (arg [0] == '@')
{
foreach (var line in File.ReadAllLines(arg.Substring(1)))
{
if (line.Length == 0 || line [0] == '#')
{
continue;
}
extra_args.AddRange(line.Split(' '));
}
args [i] = "";
}
}
args = args.Where(s => s != String.Empty).Concat(extra_args).ToArray();
// Despite a lot of effort, couldn't get dotnet to load these assemblies from the sdk dir, so copy them to our binary dir
File.Copy($"{sdkdir}/TestUtilities.dll", AppContext.BaseDirectory, true);
File.Copy($"{sdkdir}/Microsoft.DotNet.XUnitExtensions.dll", AppContext.BaseDirectory, true);
var cmdline = CommandLine.Parse(args);
// Ditto
File.Copy(cmdline.Project.Assemblies.First().AssemblyFilename, AppContext.BaseDirectory, true);
var assembly = Assembly.LoadFrom(Path.Combine(AppContext.BaseDirectory, Path.GetFileName(cmdline.Project.Assemblies.First().AssemblyFilename)));
var msg_sink = new MsgSink();
var xunit2 = new Xunit2Discoverer(AppDomainSupport.Denied, new NullSourceInformationProvider(), new ReflectionAssemblyInfo(assembly), null, null, msg_sink);
var sink = new TestDiscoverySink();
var config = new TestAssemblyConfiguration()
{
DiagnosticMessages = true, InternalDiagnosticMessages = true, PreEnumerateTheories = false
};
xunit2.Find(false, sink, TestFrameworkOptions.ForDiscovery(config));
sink.Finished.WaitOne();
foreach (XunitTestCase tc in sink.TestCases)
{
ComputeTraits(assembly, tc);
}
// Compute testcase data
var tc_data = new Dictionary <XunitTestCase, List <TcCase> > ();
foreach (XunitTestCase tc in sink.TestCases)
{
var m = ((ReflectionMethodInfo)tc.Method).MethodInfo;
var t = m.ReflectedType;
var cases = new List <TcCase> ();
if (m.GetParameters().Length > 0)
{
foreach (var cattr in m.GetCustomAttributes(true))
{
if (cattr is InlineDataAttribute)
{
var data = ((InlineDataAttribute)cattr).GetData(null).First();
if (data == null)
{
data = new object [m.GetParameters().Length];
}
if (data.Length != m.GetParameters().Length)
{
throw new Exception();
}
bool unhandled = false;
foreach (var val in data)
{
if (val is float || val is double)
{
unhandled = true;
}
if (val is Type)
{
var type = val as Type;
if (!type.IsVisible)
{
unhandled = true;
}
}
if (val is Enum)
{
if (!val.GetType().IsPublic)
{
unhandled = true;
}
}
}
if (!unhandled)
{
cases.Add(new TcCase()
{
Values = data
});
}
}
else if (cattr is MemberDataAttribute memberData && memberData.Parameters.Length == 0)
{
MethodInfo testDataMethod = m.DeclaringType.GetMethod(memberData.MemberName);
if (testDataMethod == null)
{
continue;
}
cases.Add(new TcCase {
MemberDataMethod = testDataMethod
});
}
}
}
else
{
cases.Add(new TcCase());
}
tc_data [tc] = cases;
}
var filters = cmdline.Project.Filters;
//
// Generate code using the roslyn syntax apis
// Creating syntax nodes one-by-one is very cumbersome, so use
// CSharpSyntaxTree.ParseText () to create them and ReplaceNode () to insert them into syntax trees.
//
var blocks = new List <BlockSyntax> ();
foreach (XunitTestCase tc in sink.TestCases)
{
var m = ((ReflectionMethodInfo)tc.Method).MethodInfo;
//Console.WriteLine ("" + m.ReflectedType + " " + m + " " + (tc.TestMethodArguments == null));
var t = m.ReflectedType;
if (t.IsGenericType)
{
nskipped++;
continue;
}
if (!filters.Filter(tc))
{
nskipped++;
continue;
}
var cases = tc_data [tc];
int caseindex = 0;
foreach (var test in cases)
{
string typename = GetTypeName(t);
string msg;
if (cases.Count > 1)
{
msg = $"{typename}.{m.Name}[{caseindex}] ...";
}
else
{
msg = $"{typename}.{m.Name} ...";
}
caseindex++;
var block = ParseText <BlockSyntax> (case_template
.Replace("#MSG#", msg)
.Replace("#NRUN#", test.MemberDataMethod == null ? "1" : ((IEnumerable)test.MemberDataMethod.Invoke(null, null)).Cast <object> ().Count().ToString()));
// Obtain the node for the CALL () line
var try_body_node = block.DescendantNodes().OfType <ExpressionStatementSyntax> ().Skip(2).First().Parent;
// Replace with the generated call code
var stmts = GenerateTcCall(t, test, m);
blocks.Add(block.ReplaceNode(try_body_node, Block(stmts.ToArray())));
}
}
var cu = CSharpSyntaxTree.ParseText(runner_template.Replace("#NSKIPPED#", nskipped.ToString())).GetRoot();
// Replace the body of the Run () method with the generated body
var run_body = cu.DescendantNodes().OfType <MethodDeclarationSyntax> ().First().DescendantNodes().OfType <BlockSyntax> ().First();
cu = cu.ReplaceNode(run_body, Block(blocks.ToArray()));
// Create runner.cs
Directory.CreateDirectory(outdir_name);
var outfile_name = Path.Combine(outdir_name, "runner.cs");
File.WriteAllText(outfile_name, cu.NormalizeWhitespace().ToString());
// Generate csproj file
var csproj_template = File.ReadAllText("gen-test.csproj.template");
csproj_template = csproj_template.Replace("#XUNIT_LOCATION#", sdkdir);
csproj_template = csproj_template.Replace("#TEST_ASSEMBLY#", testAssemblyName);
csproj_template = csproj_template.Replace("#TEST_ASSEMBLY_LOCATION#", testAssemblyFull);
File.WriteAllText(Path.Combine(outdir_name, testAssemblyName + "-runner.csproj"), csproj_template);
return(0);
}