void DiscoverTests <TVisitor>(IEnumerable <string> sources,
LoggerHelper logger,
Func <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitorFactory,
Action <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitComplete = null)
where TVisitor : IVsDiscoverySink, IDisposable
{
try
{
RemotingUtility.CleanUpRegisteredChannels();
using (AssemblyHelper.SubscribeResolve())
{
var reporterMessageHandler = new DefaultRunnerReporterWithTypes().CreateMessageHandler(new VisualStudioRunnerLogger(logger));
foreach (var assemblyFileNameCanBeWithoutAbsolutePath in sources)
{
var assemblyFileName = assemblyFileNameCanBeWithoutAbsolutePath;
#if !PLATFORM_DOTNET
assemblyFileName = Path.GetFullPath(assemblyFileNameCanBeWithoutAbsolutePath);
#endif
var assembly = new XunitProjectAssembly {
AssemblyFilename = assemblyFileName
};
var configuration = LoadConfiguration(assemblyFileName);
var fileName = Path.GetFileNameWithoutExtension(assemblyFileName);
var shadowCopy = configuration.ShadowCopyOrDefault;
try
{
if (cancelled)
{
break;
}
if (!IsXunitTestAssembly(assemblyFileName))
{
if (configuration.DiagnosticMessagesOrDefault)
{
logger.Log("Skipping: {0} (no reference to xUnit.net)", fileName);
}
}
else
{
var diagnosticSink = new DiagnosticMessageSink(logger, fileName, configuration.DiagnosticMessagesOrDefault);
using (var framework = new XunitFrontController(AppDomainDefaultBehavior, assemblyFileName: assemblyFileName, configFileName: null, shadowCopy: shadowCopy, diagnosticMessageSink: MessageSinkAdapter.Wrap(diagnosticSink)))
{
var targetFramework = framework.TargetFramework;
if (targetFramework.StartsWith("MonoTouch", StringComparison.OrdinalIgnoreCase) ||
targetFramework.StartsWith("MonoAndroid", StringComparison.OrdinalIgnoreCase) ||
targetFramework.StartsWith("Xamarin.iOS", StringComparison.OrdinalIgnoreCase))
{
if (configuration.DiagnosticMessagesOrDefault)
{
logger.Log("Skipping: {0} (unsupported target framework '{1}')", fileName, targetFramework);
}
}
else
{
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
using (var visitor = visitorFactory(assemblyFileName, framework, discoveryOptions))
{
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, framework.CanUseAppDomains && AppDomainDefaultBehavior != AppDomainSupport.Denied, shadowCopy, discoveryOptions));
framework.Find(includeSourceInformation: true, discoveryMessageSink: visitor, discoveryOptions: discoveryOptions);
var totalTests = visitor.Finish();
visitComplete?.Invoke(assemblyFileName, framework, discoveryOptions, visitor);
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, totalTests, totalTests));
}
}
}
}
}
catch (Exception e)
{
var ex = e.Unwrap();
var fileNotFound = ex as FileNotFoundException;
#if !PLATFORM_DOTNET
var fileLoad = ex as FileLoadException;
#endif
if (ex is InvalidOperationException)
{
logger.LogWarning("Skipping: {0} ({1})", fileName, ex.Message);
}
else if (fileNotFound != null)
{
logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileNotFound.FileName));
}
#if !PLATFORM_DOTNET
else if (fileLoad != null)
{
logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileLoad.FileName));
}
#endif
else
{
logger.LogWarning("Exception discovering tests from {0}: {1}", fileName, ex);
}
}
}
}
}
catch (Exception e)
{
logger.LogWarning("Exception discovering tests: {0}", e.Unwrap());
}
}
public void TheoryWithInlineData()
{
string code = @"
module FSharpTests
open Xunit
[<Theory>]
[<InlineData>]
[<InlineData(42)>]
[<InlineData(42, 21.12)>]
let TestMethod (x:int) =
Assert.True(true)
";
using (var assembly = FSharpAcceptanceTestV2Assembly.Create(code))
using (var controller = new TestableXunit2(assembly.FileName, null, true))
{
var sink = new TestDiscoverySink();
controller.Find(includeSourceInformation: false, messageSink: sink, discoveryOptions: TestFrameworkOptions.ForDiscovery());
sink.Finished.WaitOne();
Assert.Collection(sink.TestCases.OrderBy(tc => tc.DisplayName),
testCase => Assert.Equal("FSharpTests.TestMethod(x: ???)", testCase.DisplayName),
testCase => Assert.Equal("FSharpTests.TestMethod(x: 42)", testCase.DisplayName),
testCase => Assert.Equal("FSharpTests.TestMethod(x: 42, ???: 21.12)", testCase.DisplayName)
);
}
}
public void FactAcceptanceTest()
{
string code = @"
using System;
using Xunit;
namespace Namespace1
{
public class Class1
{
[Fact]
[Trait(""Name!"", ""Value!"")]
public void Trait() { }
[Fact(Skip=""Skipping"")]
public void Skipped() { }
[Fact(DisplayName=""Custom Test Name"")]
public void CustomName() { }
}
}
namespace Namespace2
{
public class OuterClass
{
public class Class2
{
[Fact]
public void TestMethod() { }
}
}
}";
using (var assembly = CSharpAcceptanceTestV2Assembly.Create(code))
using (var controller = new TestableXunit2(assembly.FileName, null, true))
{
var sink = new SpyMessageSink <IDiscoveryCompleteMessage>();
controller.Find(includeSourceInformation: false, messageSink: sink, discoveryOptions: TestFrameworkOptions.ForDiscovery());
sink.Finished.WaitOne();
ITestCase[] testCases = sink.Messages.OfType <ITestCaseDiscoveryMessage>().Select(tcdm => tcdm.TestCase).ToArray();
Assert.Equal(4, testCases.Length);
ITestCase traitTest = Assert.Single(testCases, tc => tc.DisplayName == "Namespace1.Class1.Trait");
string key = Assert.Single(traitTest.Traits.Keys);
Assert.Equal("Name!", key);
string value = Assert.Single(traitTest.Traits[key]);
Assert.Equal("Value!", value);
ITestCase skipped = Assert.Single(testCases, tc => tc.DisplayName == "Namespace1.Class1.Skipped");
Assert.Equal("Skipping", skipped.SkipReason);
Assert.Single(testCases, tc => tc.DisplayName == "Custom Test Name");
Assert.Single(testCases, tc => tc.DisplayName == "Namespace2.OuterClass+Class2.TestMethod");
}
}
public void RoundTrip()
{
string code = @"
using System;
using Xunit;
namespace Namespace1
{
public class Class1
{
[Fact]
public void FactMethod() { }
[Theory]
[InlineData(42)]
public void TheoryMethod(int x) { }
}
}";
using (var assembly = CSharpAcceptanceTestV2Assembly.Create(code))
{
var serializations = default(List <string>);
var testCollectionId = default(Guid);
using (var serializationController = new TestableXunitFrontController(assembly.FileName))
{
var sink = new SpyMessageSink <IDiscoveryCompleteMessage>();
serializationController.Find(includeSourceInformation: false, messageSink: sink, discoveryOptions: TestFrameworkOptions.ForDiscovery());
sink.Finished.WaitOne();
var testCases = sink.Messages.OfType <ITestCaseDiscoveryMessage>().OrderBy(tcdm => tcdm.TestCase.TestMethod.Method.Name).Select(tcdm => tcdm.TestCase).ToList();
testCollectionId = testCases[0].TestMethod.TestClass.TestCollection.UniqueID;
var descriptors = serializationController.GetTestCaseDescriptors(testCases, true);
serializations = descriptors.Select(d => d.Serialization).ToList();
}
Assert.Collection(serializations,
s => Assert.Equal($":F:Namespace1.Class1:FactMethod:1:0:{testCollectionId.ToString("N")}", s),
s => Assert.StartsWith("Xunit.Sdk.XunitTestCase, xunit.execution.{Platform}:", s)
);
using (var deserializationController = new TestableXunitFrontController(assembly.FileName))
{
var deserializations = deserializationController.BulkDeserialize(serializations);
Assert.Collection(deserializations.Select(kvp => kvp.Value),
testCase => Assert.Equal("Namespace1.Class1.FactMethod", testCase.DisplayName),
testCase => Assert.Equal("Namespace1.Class1.TheoryMethod(x: 42)", testCase.DisplayName)
);
}
}
}
public void TheoryWithInlineData()
{
string code = @"
using System;
using Xunit;
public class TestClass
{
[Theory]
[InlineData]
[InlineData(42)]
[InlineData(42, 21.12)]
public void TestMethod(int x) { }
}";
using (var assembly = CSharpAcceptanceTestV2Assembly.Create(code))
using (var controller = new TestableXunit2(assembly.FileName, null, true))
{
var sink = new SpyMessageSink <IDiscoveryCompleteMessage>();
controller.Find(includeSourceInformation: false, messageSink: sink, discoveryOptions: TestFrameworkOptions.ForDiscovery());
sink.Finished.WaitOne();
string[] testCaseNames = sink.Messages.OfType <ITestCaseDiscoveryMessage>().Select(tcdm => tcdm.TestCase.DisplayName).ToArray();
Assert.Equal(3, testCaseNames.Length);
Assert.Contains("TestClass.TestMethod(x: ???)", testCaseNames);
Assert.Contains("TestClass.TestMethod(x: 42)", testCaseNames);
Assert.Contains($"TestClass.TestMethod(x: 42, ???: {21.12})", testCaseNames);
}
}
XElement ExecuteAssembly(object consoleLock,
XunitProjectAssembly assembly,
bool needsXml,
bool?parallelizeTestCollections,
int?maxThreadCount,
bool diagnosticMessages,
bool noColor,
XunitFilters filters,
bool designTime,
bool listTestCases,
IReadOnlyList <string> designTimeFullyQualifiedNames)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
// Turn off pre-enumeration of theories when we're not running in Visual Studio
if (!designTime)
{
assembly.Configuration.PreEnumerateTheories = false;
}
if (diagnosticMessages)
{
assembly.Configuration.DiagnosticMessages = true;
}
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue)
{
executionOptions.SetMaxParallelThreads(maxThreadCount);
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageVisitor = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
var sourceInformationProvider = new SourceInformationProviderAdapater(services);
using (var controller = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, false, diagnosticMessageSink: diagnosticMessageVisitor, sourceInformationProvider: sourceInformationProvider))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
var includeSourceInformation = designTime && listTestCases;
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, false, false, discoveryOptions));
controller.Find(includeSourceInformation: includeSourceInformation, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions);
discoveryVisitor.Finished.WaitOne();
IDictionary <ITestCase, VsTestCase> vsTestCases = null;
if (designTime)
{
vsTestCases = DesignTimeTestConverter.Convert(discoveryVisitor.TestCases);
}
if (listTestCases)
{
lock (consoleLock)
{
if (designTime)
{
var sink = (ITestDiscoverySink)services.GetService(typeof(ITestDiscoverySink));
foreach (var testcase in vsTestCases.Values)
{
if (sink != null)
{
sink.SendTest(testcase);
}
Console.WriteLine(testcase.FullyQualifiedName);
}
}
else
{
foreach (var testcase in discoveryVisitor.TestCases)
{
Console.WriteLine(testcase.DisplayName);
}
}
}
return(assemblyElement);
}
IExecutionVisitor resultsVisitor;
if (designTime)
{
var sink = (ITestExecutionSink)services.GetService(typeof(ITestExecutionSink));
resultsVisitor = new DesignTimeExecutionVisitor(sink, vsTestCases, reporterMessageHandler);
}
else
{
resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel);
}
IList <ITestCase> filteredTestCases;
var testCasesDiscovered = discoveryVisitor.TestCases.Count;
if (!designTime || designTimeFullyQualifiedNames.Count == 0)
{
filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList();
}
else
{
filteredTestCases = vsTestCases.Where(t => designTimeFullyQualifiedNames.Contains(t.Value.FullyQualifiedName)).Select(t => t.Key).ToList();
}
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
if (filteredTestCases.Count == 0)
{
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
}
else
{
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary));
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine("{0}: {1}", e.GetType().FullName, e.Message);
e = e.InnerException;
}
}
return(assemblyElement);
}
XElement ExecuteAssembly(object consoleLock,
XunitProjectAssembly assembly,
bool serialize,
bool needsXml,
bool?parallelizeTestCollections,
int?maxThreadCount,
bool diagnosticMessages,
bool noColor,
bool noAppDomain,
bool failSkips,
bool stopOnFail,
XunitFilters filters,
bool internalDiagnosticMessages)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
{
return(null);
}
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= diagnosticMessages;
assembly.Configuration.InternalDiagnosticMessages |= internalDiagnosticMessages;
if (noAppDomain)
{
assembly.Configuration.AppDomain = AppDomainSupport.Denied;
}
// Setup discovery and execution options with command-line overrides
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
executionOptions.SetStopOnTestFail(stopOnFail);
if (maxThreadCount.HasValue)
{
executionOptions.SetMaxParallelThreads(maxThreadCount);
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageSink = new DiagnosticMessageSink(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
var internalDiagnosticsMessageSink = new DiagnosticMessageSink(consoleLock, assemblyDisplayName, assembly.Configuration.InternalDiagnosticMessagesOrDefault, noColor, ConsoleColor.DarkGray);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
#if NETCOREAPP1_0 || NETCOREAPP2_0
using (new NetCoreAssemblyDependencyResolver(assembly.AssemblyFilename, internalDiagnosticsMessageSink))
#endif
using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
using (var discoverySink = new TestDiscoverySink(() => cancel))
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions));
controller.Find(false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
}
else
{
if (serialize)
{
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary));
if (assemblyElement != null)
{
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
}
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
}
if (failSkips)
{
resultsSink = new DelegatingFailSkipSink(resultsSink);
}
controller.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
if (stopOnFail && resultsSink.ExecutionSummary.Failed != 0)
{
Console.WriteLine("Canceling due to test failure...");
cancel = true;
}
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine($"{e.GetType().FullName}: {e.Message}");
if (internalDiagnosticMessages)
{
Console.WriteLine(e.StackTrace);
}
e = e.InnerException;
}
}
return(assemblyElement);
}
/// <summary>
/// MultiNodeTestRunner takes the following <see cref="args"/>:
///
/// C:\> Akka.MultiNodeTestRunner.exe [assembly name] [-Dmultinode.enable-filesink=on] [-Dmultinode.output-directory={dir path}]
///
/// <list type="number">
/// <listheader>
/// <term>Argument</term>
/// <description>The name and possible value of a given Akka.MultiNodeTestRunner.exe argument.</description>
/// </listheader>
/// <item>
/// <term>AssemblyName</term>
/// <description>
/// The full path or name of an assembly containing as least one MultiNodeSpec in the current working directory.
///
/// i.e. "Akka.Cluster.Tests.MultiNode.dll"
/// "C:\akka.net\src\Akka.Cluster.Tests\bin\Debug\Akka.Cluster.Tests.MultiNode.dll"
/// </description>
/// </item>
/// <item>
/// <term>-Dmultinode.enable-filesink</term>
/// <description>Having this flag set means that the contents of this test run will be saved in the
/// current working directory as a .JSON file.
/// </description>
/// </item>
/// <item>
/// <term>-Dmultinode.multinode.output-directory</term>
/// <description>Setting this flag means that any persistent multi-node test runner output files
/// will be written to this directory instead of the default, which is the same folder
/// as the test binary.
/// </description>
/// </item>
/// </list>
/// </summary>
static void Main(string[] args)
{
TestRunSystem = ActorSystem.Create("TestRunnerLogging");
SinkCoordinator = TestRunSystem.ActorOf(Props.Create <SinkCoordinator>(), "sinkCoordinator");
var assemblyName = Path.GetFullPath(args[0]);
EnableAllSinks(assemblyName);
PublishRunnerMessage(String.Format("Running MultiNodeTests for {0}", assemblyName));
using (var controller = new XunitFrontController(assemblyName))
{
using (var discovery = new Discovery())
{
controller.Find(false, discovery, TestFrameworkOptions.ForDiscovery());
discovery.Finished.WaitOne();
foreach (var test in discovery.Tests.Reverse())
{
if (!string.IsNullOrEmpty(test.Value.First().SkipReason))
{
PublishRunnerMessage(string.Format("Skipping test {0}. Reason - {1}", test.Value.First().MethodName, test.Value.First().SkipReason));
continue;
}
PublishRunnerMessage(string.Format("Starting test {0}", test.Value.First().MethodName));
var processes = new List <Process>();
StartNewSpec(test.Value);
foreach (var nodeTest in test.Value)
{
//Loop through each test, work out number of nodes to run on and kick off process
var process = new Process();
processes.Add(process);
process.StartInfo.UseShellExecute = false;
process.StartInfo.RedirectStandardOutput = true;
process.StartInfo.FileName = "Akka.NodeTestRunner.exe";
process.StartInfo.Arguments = String.Format(@"-Dmultinode.test-assembly=""{0}"" -Dmultinode.test-class=""{1}"" -Dmultinode.test-method=""{2}"" -Dmultinode.max-nodes={3} -Dmultinode.server-host=""{4}"" -Dmultinode.host=""{5}"" -Dmultinode.index={6}",
assemblyName, nodeTest.TypeName, nodeTest.MethodName, test.Value.Count, "localhost", "localhost", nodeTest.Node - 1);
var nodeIndex = nodeTest.Node;
process.OutputDataReceived +=
(sender, line) =>
{
//ignore any trailing whitespace
if (string.IsNullOrEmpty(line.Data) || string.IsNullOrWhiteSpace(line.Data))
{
return;
}
string message = line.Data;
if (!message.StartsWith("[NODE", true, CultureInfo.InvariantCulture))
{
message = "[NODE" + nodeIndex + "]" + message;
}
PublishToAllSinks(message);
};
var closureTest = nodeTest;
process.Exited += (sender, eventArgs) =>
{
if (process.ExitCode == 0)
{
ReportSpecPassFromExitCode(nodeIndex, closureTest.TestName);
}
};
process.Start();
process.BeginOutputReadLine();
PublishRunnerMessage(string.Format("Started node {0} on pid {1}", nodeTest.Node, process.Id));
}
foreach (var process in processes)
{
process.WaitForExit();
var exitCode = process.ExitCode;
process.Close();
}
PublishRunnerMessage("Waiting 3 seconds for all messages from all processes to be collected.");
Thread.Sleep(TimeSpan.FromSeconds(3));
FinishSpec();
}
}
}
Console.WriteLine("Complete");
PublishRunnerMessage("Waiting 5 seconds for all messages from all processes to be collected.");
Thread.Sleep(TimeSpan.FromSeconds(5));
CloseAllSinks();
//Block until all Sinks have been terminated.
TestRunSystem.AwaitTermination(TimeSpan.FromMinutes(1));
//Return the proper exit code
Environment.Exit(ExitCodeContainer.ExitCode);
}
static XElement ExecuteAssembly(object consoleLock,
XunitProjectAssembly assembly,
bool serialize,
bool needsXml,
bool?parallelizeTestCollections,
int?maxThreadCount,
bool diagnosticMessages,
bool noColor,
bool?noAppDomain,
XunitFilters filters)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
{
return(null);
}
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= diagnosticMessages;
if (noAppDomain.HasValue)
{
assembly.Configuration.UseAppDomain = !noAppDomain.GetValueOrDefault();
}
// Setup discovery and execution options with command-line overrides
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue)
{
executionOptions.SetMaxParallelThreads(maxThreadCount);
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageVisitor = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
var useAppDomain = assembly.Configuration.UseAppDomainOrDefault;
using (var controller = new XunitFrontController(useAppDomain, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy, diagnosticMessageSink: diagnosticMessageVisitor))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, useAppDomain, discoveryOptions));
controller.Find(false, discoveryVisitor, discoveryOptions);
discoveryVisitor.Finished.WaitOne();
var testCasesDiscovered = discoveryVisitor.TestCases.Count;
var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
}
else
{
if (serialize)
{
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
var resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel);
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary));
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine($"{e.GetType().FullName}: {e.Message}");
e = e.InnerException;
}
}
return(assemblyElement);
}
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool appVeyor, bool showProgress, bool?parallelizeTestCollections, int?maxThreadCount, ExtendedXunitFilters filters)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
{
return(null);
}
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue)
{
executionOptions.SetMaxParallelThreads(maxThreadCount.GetValueOrDefault());
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
}
lock (consoleLock)
{
if (assembly.Configuration.DiagnosticMessages ?? false)
{
Console.WriteLine("Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})",
Path.GetFileNameWithoutExtension(assembly.AssemblyFilename),
discoveryOptions.GetMethodDisplay(),
!executionOptions.GetDisableParallelization(),
executionOptions.GetMaxParallelThreads());
}
else
{
Console.WriteLine("Discovering: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
}
}
using (var controller = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.Configuration.ShadowCopyOrDefault))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions);
discoveryVisitor.Finished.WaitOne();
lock (consoleLock)
{
Console.WriteLine("Discovered: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
}
var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, appVeyor, showProgress);
var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList();
if (filteredTestCases.Count == 0)
{
lock (consoleLock)
{
Console.ForegroundColor = ConsoleColor.DarkYellow;
Console.WriteLine("Info: {0} has no tests to run", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
Console.ForegroundColor = ConsoleColor.Gray;
}
}
else
{
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
}
}
}
catch (Exception ex)
{
Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message);
failed = true;
}
return(assemblyElement);
}
public static void Main(string[] args)
{
OutputDirectory = CommandLine.GetPropertyOrDefault("multinode.output-directory", string.Empty);
TestRunSystem = ActorSystem.Create("TestRunnerLogging");
var teamCityFormattingOn = CommandLine.GetPropertyOrDefault("multinode.teamcity", "false");
if (!Boolean.TryParse(teamCityFormattingOn, out TeamCityFormattingOn))
{
throw new ArgumentException("Invalid argument provided for -Dteamcity");
}
SinkCoordinator = TestRunSystem.ActorOf( // mutes ConsoleMessageSinkActor
Props.Create <SinkCoordinator>(), "sinkCoordinator");
var listenAddress = IPAddress.Parse(CommandLine.GetPropertyOrDefault("multinode.listen-address", "127.0.0.1"));
var listenPort = CommandLine.GetInt32OrDefault("multinode.listen-port", 6577);
var listenEndpoint = new IPEndPoint(listenAddress, listenPort);
var specName = CommandLine.GetPropertyOrDefault("multinode.spec", "");
var platform = CommandLine.GetPropertyOrDefault("multinode.platform", "netcore");
if (!_validNetCorePlatform.Contains(platform))
{
throw new Exception($"Target platform not supported: {platform}. Supported platforms are net and netcore");
}
var tcpLogger = TestRunSystem.ActorOf(Props.Create(() => new TcpLoggingServer(SinkCoordinator)), "TcpLogger");
TestRunSystem.Tcp().Tell(new Tcp.Bind(tcpLogger, listenEndpoint));
var assemblyPath = Path.GetFullPath(args[0].Trim('"')); //unquote the string first
EnableAllSinks(assemblyPath, platform);
PublishRunnerMessage($"Running MultiNodeTests for {assemblyPath}");
// In NetCore, if the assembly file hasn't been touched,
// XunitFrontController would fail loading external assemblies and its dependencies.
var assembly = AssemblyLoadContext.Default.LoadFromAssemblyPath(assemblyPath);
var asms = assembly.GetReferencedAssemblies();
var basePath = Path.GetDirectoryName(assemblyPath);
foreach (var asm in asms)
{
try
{
Assembly.Load(new AssemblyName(asm.FullName));
}
catch (Exception)
{
var path = Path.Combine(basePath, asm.Name + ".dll");
try
{
AssemblyLoadContext.Default.LoadFromAssemblyPath(path);
}
catch (Exception e)
{
Console.Out.WriteLine($"Failed to load dll: {path}, {e}");
}
}
}
using (var controller = new XunitFrontController(AppDomainSupport.IfAvailable, assemblyPath))
{
using (var discovery = new Discovery())
{
controller.Find(false, discovery, TestFrameworkOptions.ForDiscovery());
discovery.Finished.WaitOne();
if (discovery.WasSuccessful)
{
foreach (var test in discovery.Tests.Reverse())
{
if (!string.IsNullOrEmpty(test.Value.First().SkipReason))
{
PublishRunnerMessage($"Skipping test {test.Value.First().MethodName}. Reason - {test.Value.First().SkipReason}");
continue;
}
if (!string.IsNullOrWhiteSpace(specName) &&
CultureInfo.InvariantCulture.CompareInfo.IndexOf(test.Value.First().TestName,
specName,
CompareOptions.IgnoreCase) < 0)
{
PublishRunnerMessage($"Skipping [{test.Value.First().MethodName}] (Filtering)");
continue;
}
var processes = new List <Process>();
PublishRunnerMessage($"Starting test {test.Value.First().MethodName}");
Console.Out.WriteLine($"Starting test {test.Value.First().MethodName}");
StartNewSpec(test.Value);
var ntrNetPath = Path.Combine(AppContext.BaseDirectory, "Akka.NodeTestRunner.exe");
var ntrNetCorePath = Path.Combine(AppContext.BaseDirectory, "Akka.NodeTestRunner.dll");
foreach (var nodeTest in test.Value)
{
//Loop through each test, work out number of nodes to run on and kick off process
var sbArguments = new StringBuilder()
//.Append($@"-Dmultinode.test-assembly=""{assemblyPath}"" ")
.Append($@"-Dmultinode.test-class=""{nodeTest.TypeName}"" ")
.Append($@"-Dmultinode.test-method=""{nodeTest.MethodName}"" ")
.Append($@"-Dmultinode.max-nodes={test.Value.Count} ")
.Append($@"-Dmultinode.server-host=""{"localhost"}"" ")
.Append($@"-Dmultinode.host=""{"localhost"}"" ")
.Append($@"-Dmultinode.index={nodeTest.Node - 1} ")
.Append($@"-Dmultinode.role=""{nodeTest.Role}"" ")
.Append($@"-Dmultinode.listen-address={listenAddress} ")
.Append($@"-Dmultinode.listen-port={listenPort} ");
string fileName = null;
switch (platform)
{
case "net":
fileName = ntrNetPath;
sbArguments.Insert(0, $@" -Dmultinode.test-assembly=""{assemblyPath}"" ");
break;
case "netcore":
fileName = "dotnet";
sbArguments.Insert(0, $@" -Dmultinode.test-assembly=""{assemblyPath}"" ");
sbArguments.Insert(0, ntrNetCorePath);
break;
}
var process = new Process
{
StartInfo = new ProcessStartInfo
{
FileName = fileName ?? throw new NullException(nameof(fileName)),
UseShellExecute = false,
RedirectStandardOutput = true,
Arguments = sbArguments.ToString(),
WorkingDirectory = Path.GetDirectoryName(assemblyPath)
}
};
processes.Add(process);
var nodeIndex = nodeTest.Node;
var nodeRole = nodeTest.Role;
if (platform == "netcore")
{
process.StartInfo.FileName = "dotnet";
process.StartInfo.Arguments = ntrNetCorePath + " " + process.StartInfo.Arguments;
process.StartInfo.WorkingDirectory = Path.GetDirectoryName(assemblyPath);
}
//TODO: might need to do some validation here to avoid the 260 character max path error on Windows
var folder = Directory.CreateDirectory(Path.Combine(OutputDirectory, nodeTest.TestName));
var logFilePath = Path.Combine(folder.FullName, $"node{nodeIndex}__{nodeRole}__{platform}.txt");
var fileActor = TestRunSystem.ActorOf(Props.Create(() => new FileSystemAppenderActor(logFilePath)));
process.OutputDataReceived += (sender, eventArgs) =>
{
if (eventArgs?.Data != null)
{
fileActor.Tell(eventArgs.Data);
if (TeamCityFormattingOn)
{
// teamCityTest.WriteStdOutput(eventArgs.Data); TODO: open flood gates
}
}
};
var closureTest = nodeTest;
process.Exited += (sender, eventArgs) =>
{
if (process.ExitCode == 0)
{
ReportSpecPassFromExitCode(nodeIndex, nodeRole,
closureTest.TestName);
}
};
process.Start();
process.BeginOutputReadLine();
PublishRunnerMessage($"Started node {nodeIndex} : {nodeRole} on pid {process.Id}");
}
foreach (var process in processes)
{
process.WaitForExit();
process.Dispose();
}
PublishRunnerMessage("Waiting 3 seconds for all messages from all processes to be collected.");
Thread.Sleep(TimeSpan.FromSeconds(3));
FinishSpec(test.Value);
}
Console.WriteLine("Complete");
PublishRunnerMessage("Waiting 5 seconds for all messages from all processes to be collected.");
Thread.Sleep(TimeSpan.FromSeconds(5));
}
else
{
var sb = new StringBuilder();
sb.AppendLine("One or more exception was thrown while discovering test cases. Test Aborted.");
foreach (var err in discovery.Errors)
{
for (int i = 0; i < err.ExceptionTypes.Length; ++i)
{
sb.AppendLine();
sb.Append($"{err.ExceptionTypes[i]}: {err.Messages[i]}");
sb.Append(err.StackTraces[i]);
}
}
PublishRunnerMessage(sb.ToString());
Console.Out.WriteLine(sb.ToString());
}
}
}
CloseAllSinks();
//Block until all Sinks have been terminated.
TestRunSystem.WhenTerminated.Wait(TimeSpan.FromMinutes(1));
if (Debugger.IsAttached)
{
Console.ReadLine(); //block when debugging
}
//Return the proper exit code
Environment.Exit(ExitCodeContainer.ExitCode);
}
Task ITestDiscoverer.AnalyzeDocumentAsync(DocumentTestDiscoveryContext context)
{
if (context.CancellationToken.IsCancellationRequested)
{
return(Task.FromCanceled(context.CancellationToken));
}
using (var discoverySink = new TestDiscoveryVisitor())
using (var xunit2 = new Xunit2Discoverer(
AppDomainSupport.Denied,
sourceInformationProvider: DummySourceInformationProvider.Instance,
assemblyInfo: new SourceAssemblyInfo(context),
xunitExecutionAssemblyPath: XunitExecutionAssemblyPath.Value,
verifyAssembliesOnDisk: false))
{
xunit2.Find(includeSourceInformation: false, messageSink: discoverySink, discoveryOptions: TestFrameworkOptions.ForDiscovery());
while (!discoverySink.Finished.WaitOne(50))
{
if (context.CancellationToken.IsCancellationRequested)
{
break;
}
TrySendDiscoveredTestCases(context, discoverySink);
}
TrySendDiscoveredTestCases(context, discoverySink);
}
return(context.CancellationToken.IsCancellationRequested ?
Task.FromCanceled(context.CancellationToken) :
Task.CompletedTask);
}
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool quiet, bool serialize, bool needsXml, bool teamCity, bool appVeyor, bool?parallelizeTestCollections, int?maxThreadCount, XunitFilters filters)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
{
return(null);
}
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue)
{
executionOptions.SetMaxParallelThreads(maxThreadCount.GetValueOrDefault());
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
lock (consoleLock)
{
if (assembly.Configuration.DiagnosticMessagesOrDefault)
{
Console.WriteLine("Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})",
assemblyDisplayName,
discoveryOptions.GetMethodDisplayOrDefault(),
!executionOptions.GetDisableParallelizationOrDefault(),
executionOptions.GetMaxParallelThreadsOrDefault());
}
else if (!quiet)
{
Console.WriteLine("Discovering: {0}", assemblyDisplayName);
}
}
var diagnosticMessageVisitor = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault);
using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy, diagnosticMessageSink: diagnosticMessageVisitor))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions);
discoveryVisitor.Finished.WaitOne();
if (!quiet)
{
lock (consoleLock)
Console.WriteLine("Discovered: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
}
var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList();
if (filteredTestCases.Count == 0)
{
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
}
else
{
if (serialize)
{
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
}
var resultsVisitor = CreateVisitor(consoleLock, quiet, defaultDirectory, assemblyElement, teamCity, appVeyor);
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine("{0}: {1}", e.GetType().FullName, e.Message);
e = e.InnerException;
}
}
return(assemblyElement);
}
private void ExecuteTests(
[NotNull] XunitProjectAssembly assembly,
[NotNull] TestRunOptions options,
[NotNull] XunitFilters filters,
[NotNull] XunitFrontController controller,
[NotNull] TestDiscoverySink discoverySink,
[NotNull] ITestFrameworkExecutionOptions executionOptions,
[CanBeNull] XElement assemblyElement,
[NotNull] DiagnosticMessageSink diagnosticMessageSink)
{
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var serialize = options.Serialize;
var failSkips = options.FailSkips;
var stopOnFail = options.StopOnFail;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
// Discover & filter the tests
_reporterMessageHandler.OnMessage(
new TestAssemblyDiscoveryStarting(
assembly,
controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied,
shadowCopy,
discoveryOptions));
controller.Find(false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
_reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
_completionMessages.TryAdd(assembly.GetFileName().ThrowIfNull(), new ExecutionSummary());
}
else
{
if (serialize)
{
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
}
_reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(
_reporterMessageHandler,
() => _cancel,
(path, summary) => _completionMessages.TryAdd(path, summary));
if (assemblyElement != null)
{
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
}
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
}
if (failSkips)
{
resultsSink = new DelegatingFailSkipSink(resultsSink);
}
controller.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
_reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
if (!stopOnFail || resultsSink.ExecutionSummary.Failed == 0)
{
return;
}
Console.WriteLine("Canceling due to test failure...");
_cancel = true;
}
}
public List <IMessageSinkMessage> Run(Type[] types)
{
Xunit2 = new Xunit2(AppDomainSupport.Required, new NullSourceInformationProvider(), types[0].Assembly.GetLocalCodeBase(), configFileName: null, shadowCopy: true);
var discoverySink = new SpyMessageSink <IDiscoveryCompleteMessage>();
foreach (var type in types)
{
Xunit2.Find(type.FullName, includeSourceInformation: false, messageSink: discoverySink, discoveryOptions: TestFrameworkOptions.ForDiscovery());
discoverySink.Finished.WaitOne();
discoverySink.Finished.Reset();
}
var testCases = discoverySink.Messages.OfType <ITestCaseDiscoveryMessage>().Select(msg => msg.TestCase).ToArray();
var runSink = new SpyMessageSink <ITestAssemblyFinished>();
Xunit2.RunTests(testCases, runSink, TestFrameworkOptions.ForExecution());
runSink.Finished.WaitOne();
return(runSink.Messages.ToList());
}
public EtwPerformanceMetricLogger(XunitPerformanceProject project, Program program)
{
_etlPath = Path.Combine(project.OutputDir, project.OutputBaseFileName + ".etl");
_program = program;
_project = project;
var diagnosticMessageSink = new ConsoleReporter();
foreach (var assembly in project.Assemblies)
{
program.PrintIfVerbose($"Discovering tests for {assembly.AssemblyFilename}.");
// Note: We do not use shadowCopy because that creates a new AppDomain which can cause
// assembly load failures with delay-signed or "fake signed" assemblies.
using (var controller = new XunitFrontController(
assemblyFileName: assembly.AssemblyFilename,
shadowCopy: false,
appDomainSupport: AppDomainSupport.Denied,
diagnosticMessageSink: new ConsoleDiagnosticsMessageVisitor())
)
using (var discoveryVisitor = new PerformanceTestDiscoveryVisitor(assembly, project.Filters, diagnosticMessageSink))
{
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: TestFrameworkOptions.ForDiscovery());
discoveryVisitor.Finished.WaitOne();
_tests.AddRange(discoveryVisitor.Tests);
}
}
program.PrintIfVerbose($"Discovered a total of {_tests.Count} tests.");
}
public static int Main(string[] args)
{
var asm = typeof(SingleFileTestRunner).Assembly;
Console.WriteLine("Running assembly:" + asm.FullName);
var diagnosticSink = new ConsoleDiagnosticMessageSink();
var testsFinished = new TaskCompletionSource();
var testSink = new TestMessageSink();
var summarySink = new DelegatingExecutionSummarySink(testSink,
() => false,
(completed, summary) => Console.WriteLine($"Tests run: {summary.Total}, Errors: {summary.Errors}, Failures: {summary.Failed}, Skipped: {summary.Skipped}. Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s"));
var resultsXmlAssembly = new XElement("assembly");
var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly);
testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); };
testSink.Execution.TestFailedEvent += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{Xunit.ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{Xunit.ExceptionUtility.CombineStackTraces(args.Message)}"); };
testSink.Execution.TestAssemblyFinishedEvent += args =>
{
Console.WriteLine($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}");
testsFinished.SetResult();
};
var assemblyConfig = new TestAssemblyConfiguration()
{
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
PreEnumerateTheories = false,
};
var xunitTestFx = new SingleFileTestRunner(diagnosticSink);
var asmInfo = Reflector.Wrap(asm);
var asmName = asm.GetName();
var discoverySink = new TestDiscoverySink();
var discoverer = xunitTestFx.CreateDiscoverer(asmInfo);
discoverer.Find(false, discoverySink, TestFrameworkOptions.ForDiscovery(assemblyConfig));
discoverySink.Finished.WaitOne();
string xmlResultFileName = null;
XunitFilters filters = new XunitFilters();
// Quick hack wo much validation to get args that are passed (notrait, xml)
Dictionary <string, List <string> > noTraits = new Dictionary <string, List <string> >();
for (int i = 0; i < args.Length; i++)
{
if (args[i].Equals("-notrait", StringComparison.OrdinalIgnoreCase))
{
var traitKeyValue = args[i + 1].Split("=", StringSplitOptions.TrimEntries);
if (!noTraits.TryGetValue(traitKeyValue[0], out List <string> values))
{
noTraits.Add(traitKeyValue[0], values = new List <string>());
}
values.Add(traitKeyValue[1]);
}
if (args[i].Equals("-xml", StringComparison.OrdinalIgnoreCase))
{
xmlResultFileName = args[i + 1].Trim();
}
}
foreach (KeyValuePair <string, List <string> > kvp in noTraits)
{
filters.ExcludedTraits.Add(kvp.Key, kvp.Value);
}
var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList();
var executor = xunitTestFx.CreateExecutor(asmName);
executor.RunTests(filteredTestCases, resultsSink, TestFrameworkOptions.ForExecution(assemblyConfig));
resultsSink.Finished.WaitOne();
// Helix need to see results file in the drive to detect if the test has failed or not
if (xmlResultFileName != null)
{
resultsXmlAssembly.Save(xmlResultFileName);
}
var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0;
return(failed ? 1 : 0);
}
public RealDiscovery()
{
var sourceInformationProvider = new NullSourceInformationProvider();
var diagnosticMessageSink = new Xunit.NullMessageSink();
var assembly = typeof(TestCaseBulkDeserializerTests).Assembly;
var assemblyInfo = Reflector.Wrap(assembly);
discoverer = new XunitTestFrameworkDiscoverer(assemblyInfo, sourceInformationProvider, diagnosticMessageSink);
var discoverySink = new SpyMessageSink <IDiscoveryCompleteMessage>();
discoverer.Find("TestCaseDescriptorFactoryTests+TestClass", false, discoverySink, TestFrameworkOptions.ForDiscovery());
discoverySink.Finished.WaitOne();
testCases = discoverySink.Messages.OfType <ITestCaseDiscoveryMessage>().Select(m => m.TestCase).ToList();
}
private bool DiscoverTestsInSource <TVisitor>(XunitFrontController framework,
LoggerHelper logger,
TestPlatformContext testPlatformContext,
RunSettings runSettings,
Func <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitorFactory,
Action <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitComplete,
string assemblyFileName,
bool shadowCopy,
TestAssemblyConfiguration configuration)
where TVisitor : IVsDiscoverySink, IDisposable
{
if (cancelled)
{
return(false);
}
var fileName = "(unknown assembly)";
try
{
var reporterMessageHandler = GetRunnerReporter(logger, runSettings, new[] { assemblyFileName }).CreateMessageHandler(new VisualStudioRunnerLogger(logger));
var assembly = new XunitProjectAssembly {
AssemblyFilename = assemblyFileName
};
fileName = Path.GetFileNameWithoutExtension(assemblyFileName);
if (!IsXunitTestAssembly(assemblyFileName))
{
if (configuration.DiagnosticMessagesOrDefault)
{
logger.Log("Skipping: {0} (no reference to xUnit.net)", fileName);
}
}
else
{
var targetFramework = framework.TargetFramework;
if (targetFramework.StartsWith("MonoTouch", StringComparison.OrdinalIgnoreCase) ||
targetFramework.StartsWith("MonoAndroid", StringComparison.OrdinalIgnoreCase) ||
targetFramework.StartsWith("Xamarin.iOS", StringComparison.OrdinalIgnoreCase))
{
if (configuration.DiagnosticMessagesOrDefault)
{
logger.Log("Skipping: {0} (unsupported target framework '{1}')", fileName, targetFramework);
}
}
else
{
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
using (var visitor = visitorFactory(assemblyFileName, framework, discoveryOptions))
{
var totalTests = 0;
var usingAppDomains = framework.CanUseAppDomains && AppDomainDefaultBehavior != AppDomainSupport.Denied;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, usingAppDomains, shadowCopy, discoveryOptions));
try
{
framework.Find(testPlatformContext.RequireSourceInformation, visitor, discoveryOptions);
totalTests = visitor.Finish();
visitComplete?.Invoke(assemblyFileName, framework, discoveryOptions, visitor);
}
finally
{
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, totalTests, totalTests));
}
}
}
}
}
catch (Exception e)
{
var ex = e.Unwrap();
if (ex is InvalidOperationException)
{
logger.LogWarning("Skipping: {0} ({1})", fileName, ex.Message);
}
else if (ex is FileNotFoundException fileNotFound)
{
logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileNotFound.FileName));
}
#if !WINDOWS_UAP
else if (ex is FileLoadException fileLoad)
{
logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileLoad.FileName));
}
#endif
else
{
logger.LogWarning("Exception discovering tests from {0}: {1}", fileName, ex);
}
}
return(true);
}
public static void GuardClauses()
{
var framework = TestableXunitTestFrameworkDiscoverer.Create();
Assert.Throws <ArgumentNullException>("discoveryMessageSink", () => framework.Find(includeSourceInformation: false, discoveryMessageSink: null, discoveryOptions: TestFrameworkOptions.ForDiscovery()));
Assert.Throws <ArgumentNullException>("discoveryOptions", () => framework.Find(includeSourceInformation: false, discoveryMessageSink: Substitute.For <IMessageSink>(), discoveryOptions: null));
}
static int Main(string[] args)
{
CommandLine.Initialize(args);
var nodeIndex = CommandLine.GetInt32("multinode.index");
var nodeRole = CommandLine.GetProperty("multinode.role");
var assemblyFileName = CommandLine.GetProperty("multinode.test-assembly");
var typeName = CommandLine.GetProperty("multinode.test-class");
var testName = CommandLine.GetProperty("multinode.test-method");
var displayName = testName;
var listenAddress = IPAddress.Parse(CommandLine.GetProperty("multinode.listen-address"));
var listenPort = CommandLine.GetInt32("multinode.listen-port");
var listenEndpoint = new IPEndPoint(listenAddress, listenPort);
var system = ActorSystem.Create("NoteTestRunner-" + nodeIndex);
var tcpClient = _logger = system.ActorOf <RunnerTcpClient>();
system.Tcp().Tell(new Tcp.Connect(listenEndpoint), tcpClient);
#if CORECLR
// In NetCore, if the assembly file hasn't been touched,
// XunitFrontController would fail loading external assemblies and its dependencies.
AssemblyLoadContext.Default.Resolving += (assemblyLoadContext, assemblyName) => DefaultOnResolving(assemblyLoadContext, assemblyName, assemblyFileName);
var assembly = AssemblyLoadContext.Default.LoadFromAssemblyPath(assemblyFileName);
DependencyContext.Load(assembly)
.CompileLibraries
.Where(dep => dep.Name.ToLower()
.Contains(assembly.FullName.Split(new[] { ',' })[0].ToLower()))
.Select(dependency => AssemblyLoadContext.Default.LoadFromAssemblyName(new AssemblyName(dependency.Name)));
#endif
Thread.Sleep(TimeSpan.FromSeconds(10));
Environment.SetEnvironmentVariable(MultiNodeFactAttribute.MultiNodeTestEnvironmentName, "1");
try
{
using (var controller = new XunitFrontController(AppDomainSupport.IfAvailable, assemblyFileName))
{
/* need to pass in just the assembly name to Discovery, not the full path
* i.e. "Akka.Cluster.Tests.MultiNode.dll"
* not "bin/Release/Akka.Cluster.Tests.MultiNode.dll" - this will cause
* the Discovery class to actually not find any individual specs to run
*/
var assemblyName = Path.GetFileName(assemblyFileName);
Console.WriteLine("Running specs for {0} [{1}]", assemblyName, assemblyFileName);
using (var discovery = new Discovery(assemblyName, typeName))
{
using (var sink = new Sink(nodeIndex, nodeRole, tcpClient))
{
try
{
controller.Find(true, discovery, TestFrameworkOptions.ForDiscovery());
discovery.Finished.WaitOne();
controller.RunTests(discovery.TestCases, sink, TestFrameworkOptions.ForExecution());
}
catch (AggregateException ex)
{
var specFail = new SpecFail(nodeIndex, nodeRole, displayName);
specFail.FailureExceptionTypes.Add(ex.GetType().ToString());
specFail.FailureMessages.Add(ex.Message);
specFail.FailureStackTraces.Add(ex.StackTrace);
foreach (var innerEx in ex.Flatten().InnerExceptions)
{
specFail.FailureExceptionTypes.Add(innerEx.GetType().ToString());
specFail.FailureMessages.Add(innerEx.Message);
specFail.FailureStackTraces.Add(innerEx.StackTrace);
}
_logger.Tell(specFail.ToString());
Console.WriteLine(specFail);
//make sure message is send over the wire
FlushLogMessages();
Environment.Exit(1); //signal failure
return(1);
}
catch (Exception ex)
{
var specFail = new SpecFail(nodeIndex, nodeRole, displayName);
specFail.FailureExceptionTypes.Add(ex.GetType().ToString());
specFail.FailureMessages.Add(ex.Message);
specFail.FailureStackTraces.Add(ex.StackTrace);
var innerEx = ex.InnerException;
while (innerEx != null)
{
specFail.FailureExceptionTypes.Add(innerEx.GetType().ToString());
specFail.FailureMessages.Add(innerEx.Message);
specFail.FailureStackTraces.Add(innerEx.StackTrace);
innerEx = innerEx.InnerException;
}
_logger.Tell(specFail.ToString());
Console.WriteLine(specFail);
//make sure message is send over the wire
FlushLogMessages();
Environment.Exit(1); //signal failure
return(1);
}
var timedOut = false;
if (!sink.Finished.WaitOne(MaxProcessWaitTimeout)) //timed out
{
var line = string.Format("Timed out while waiting for test to complete after {0} ms",
MaxProcessWaitTimeout);
_logger.Tell(line);
Console.WriteLine(line);
timedOut = true;
}
FlushLogMessages();
system.Terminate().Wait();
Environment.Exit(sink.Passed && !timedOut ? 0 : 1);
return(sink.Passed ? 0 : 1);
}
}
}
}
finally
{
Environment.SetEnvironmentVariable(MultiNodeFactAttribute.MultiNodeTestEnvironmentName, null);
}
}
public void Find(string typeName, bool includeSourceInformation = false)
{
base.Find(typeName, includeSourceInformation, Visitor, TestFrameworkOptions.ForDiscovery());
Visitor.Finished.WaitOne();
}
public void NoTestMethods()
{
using (var assm = CSharpAcceptanceTestV2Assembly.Create(code: ""))
using (var controller = new TestableXunit2(assm.FileName, null, true))
{
var sink = new SpyMessageSink <IDiscoveryCompleteMessage>();
controller.Find(includeSourceInformation: false, messageSink: sink, discoveryOptions: TestFrameworkOptions.ForDiscovery());
sink.Finished.WaitOne();
Assert.False(sink.Messages.Any(msg => msg is ITestCaseDiscoveryMessage));
}
}
public virtual bool FindTestsForClass(ITestClass testClass, bool includeSourceInformation = false)
{
using (var messageBus = new MessageBus(Visitor))
return(base.FindTestsForType(testClass, includeSourceInformation, messageBus, TestFrameworkOptions.ForDiscovery()));
}
public void FactAcceptanceTest()
{
string code = @"
module FSharpTests
open Xunit
[<Fact>]
[<Trait(""Name!"", ""Value!"")>]
let Trait() =
Assert.True(true)
[<Fact(Skip = ""Skipping"")>]
let Skipped() =
Assert.True(false)
[<Fact(DisplayName=""Custom Test Name"")>]
let CustomName() =
Assert.True(true)
";
using (var assembly = FSharpAcceptanceTestV2Assembly.Create(code))
using (var controller = new TestableXunit2(assembly.FileName, null, true))
{
var sink = new TestDiscoverySink();
controller.Find(includeSourceInformation: false, messageSink: sink, discoveryOptions: TestFrameworkOptions.ForDiscovery());
sink.Finished.WaitOne();
Assert.Collection(sink.TestCases.OrderBy(tc => tc.DisplayName),
testCase => Assert.Equal("Custom Test Name", testCase.DisplayName),
testCase =>
{
Assert.Equal("FSharpTests.Skipped", testCase.DisplayName);
Assert.Equal("Skipping", testCase.SkipReason);
},
testCase =>
{
Assert.Equal("FSharpTests.Trait", testCase.DisplayName);
Assert.Collection(testCase.Traits,
kvp =>
{
Assert.Equal("Name!", kvp.Key);
Assert.Equal("Value!", kvp.Value.Single());
}
);
}
);
}
}
protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly)
{
if (cancel)
{
return(null);
}
var assemblyElement = NeedsXml ? new XElement("assembly") : null;
try
{
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= DiagnosticMessages;
if (appDomains.HasValue)
{
assembly.Configuration.AppDomain = appDomains.GetValueOrDefault() ? AppDomainSupport.Required : AppDomainSupport.Denied;
}
// Setup discovery and execution options with command-line overrides
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue && maxThreadCount.Value > -1)
{
executionOptions.SetMaxParallelThreads(maxThreadCount);
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections);
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageVisitor = new DiagnosticMessageVisitor(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageVisitor))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions));
controller.Find(false, discoveryVisitor, discoveryOptions);
discoveryVisitor.Finished.WaitOne();
var testCasesDiscovered = discoveryVisitor.TestCases.Count;
var filteredTestCases = discoveryVisitor.TestCases.Where(Filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
}
else
{
if (SerializeTestCases)
{
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
}
IExecutionVisitor resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel);
if (FailSkips)
{
resultsVisitor = new FailSkipVisitor(resultsVisitor);
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary));
if (resultsVisitor.ExecutionSummary.Failed != 0)
{
ExitCode = 1;
}
}
}
}
catch (Exception ex)
{
var e = ex;
while (e != null)
{
Log.LogError("{0}: {1}", e.GetType().FullName, e.Message);
foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
{
Log.LogError(stackLine);
}
e = e.InnerException;
}
ExitCode = -1;
}
return(assemblyElement);
}
public void SingleTestMethod()
{
string code = @"
using Xunit;
public class Foo
{
[Fact]
public void Bar() { }
}";
using (var assm = CSharpAcceptanceTestV2Assembly.Create(code))
using (var controller = new TestableXunit2(assm.FileName, null, true))
{
var sink = new SpyMessageSink <IDiscoveryCompleteMessage>();
controller.Find(includeSourceInformation: false, messageSink: sink, discoveryOptions: TestFrameworkOptions.ForDiscovery());
sink.Finished.WaitOne();
ITestCase testCase = sink.Messages.OfType <ITestCaseDiscoveryMessage>().Single().TestCase;
Assert.Equal("Foo.Bar", testCase.DisplayName);
}
}
public async void RunTests(string arguments)
{
log = await Helpers.GetFileStreamWriterInLocalStorageAsync("stdout.txt");
string[] args = arguments.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
log.WriteLine(arguments);
var commandLine = CommandLine.Parse(args);
if (commandLine.Debug)
{
Debugger.Launch();
}
var completionMessages = new ConcurrentDictionary <string, ExecutionSummary>();
var assembliesElement = new XElement("assemblies");
foreach (var assembly in commandLine.Project.Assemblies)
{
if (cancel)
{
return;
}
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= commandLine.DiagnosticMessages;
assembly.Configuration.AppDomain = AppDomainSupport.Denied;
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
executionOptions.SetDisableParallelization(true);
try
{
using (var xunit = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.Configuration.ShadowCopyOrDefault))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
string assemblyName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
// Discover & filter the tests
log.WriteLine($"Discovering: {assemblyName}");
xunit.Find(false, discoveryVisitor, discoveryOptions);
discoveryVisitor.Finished.WaitOne();
var testCasesDiscovered = discoveryVisitor.TestCases.Count;
var filteredTestCases = discoveryVisitor.TestCases.Where(commandLine.Project.Filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
log.WriteLine($"Discovered: {assemblyName}");
// Run the filtered tests
if (testCasesToRun == 0)
{
log.WriteLine($"Info: {assemblyName} has no tests to run");
}
else
{
if (commandLine.Serialize)
{
filteredTestCases = filteredTestCases.Select(xunit.Serialize).Select(xunit.Deserialize).ToList();
}
var assemblyElement = new XElement("assembly");
StandardUapVisitor resultsVisitor = new StandardUapVisitor(assemblyElement, () => cancel, log, completionMessages, commandLine.ShowProgress, commandLine.FailSkips);
xunit.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
assembliesElement.Add(assemblyElement);
log.WriteLine($"{Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)} Total: {resultsVisitor.ExecutionSummary.Total}, Errors: {resultsVisitor.ExecutionSummary.Errors}, Failed: {resultsVisitor.ExecutionSummary.Failed}, Skipped: {resultsVisitor.ExecutionSummary.Skipped}, Time: {resultsVisitor.ExecutionSummary.Time}");
}
}
}
catch (Exception e)
{
assembliesElement = new XElement("error");
assembliesElement.Add(e);
log.WriteLine(e);
}
finally
{
await WriteResults(Path.GetFileName(assembly.AssemblyFilename), assembliesElement);
log.Dispose();
}
}
Application.Current.Exit();
}
private async void RunTests(string arguments)
{
var reporters = new List <IRunnerReporter>();
string[] args = arguments.Split(new[] { '\x1F' }, StringSplitOptions.RemoveEmptyEntries);
log = string.Empty;
log += "Args: " + string.Join(" ", args) + "\n";
var commandLine = CommandLine.Parse(args);
if (commandLine.Debug)
{
Debugger.Launch();
}
var reporterMessageHandler = commandLine.Reporter.CreateMessageHandler(new RunLogger());
var completionMessages = new ConcurrentDictionary <string, ExecutionSummary>();
var assembliesElement = new XElement("assemblies");
foreach (var assembly in commandLine.Project.Assemblies)
{
if (cancel)
{
return;
}
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= commandLine.DiagnosticMessages;
assembly.Configuration.AppDomain = AppDomainSupport.Denied;
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
executionOptions.SetDisableParallelization(true);
try
{
using (var xunit = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.Configuration.ShadowCopyOrDefault))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, false, false, discoveryOptions));
xunit.Find(false, discoveryVisitor, discoveryOptions);
discoveryVisitor.Finished.WaitOne();
var testCasesDiscovered = discoveryVisitor.TestCases.Count;
var filteredTestCases = discoveryVisitor.TestCases.Where(commandLine.Project.Filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
log += "testCasesToRun: " + testCasesToRun + "\n";
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
}
else
{
if (commandLine.Serialize)
{
filteredTestCases = filteredTestCases.Select(xunit.Serialize).Select(xunit.Deserialize).ToList();
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
var assemblyElement = new XElement("assembly");
IExecutionVisitor resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel);
if (commandLine.FailSkips)
{
resultsVisitor = new FailSkipVisitor(resultsVisitor);
}
xunit.RunTests(filteredTestCases, resultsVisitor, executionOptions);
log += "finished running tests \n";
resultsVisitor.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary));
assembliesElement.Add(assemblyElement);
}
}
}
catch (Exception e)
{
assembliesElement = new XElement("error");
assembliesElement.Add(e);
log += "logged exec errors: " + e + "\n";
}
}
await WriteResults(assembliesElement);
await WriteLogs(log);
Application.Current.Exit();
}
public bool Run(string assemblyFileName, IEnumerable <string> excludedTraits)
{
WebAssembly.Runtime.InvokeJS($"if (document) document.body.innerHTML = ''");
Log("Starting tests...");
var filters = new XunitFilters();
foreach (var trait in excludedTraits ?? Array.Empty <string>())
{
ParseEqualSeparatedArgument(filters.ExcludedTraits, trait);
}
var configuration = new TestAssemblyConfiguration
{
ShadowCopy = false,
ParallelizeAssembly = false,
ParallelizeTestCollections = false,
MaxParallelThreads = 1,
PreEnumerateTheories = false
};
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
var discoverySink = new TestDiscoverySink();
var diagnosticSink = new ConsoleDiagnosticMessageSink();
var testOptions = TestFrameworkOptions.ForExecution(configuration);
var testSink = new TestMessageSink();
var controller = new Xunit2(
AppDomainSupport.Denied,
new NullSourceInformationProvider(),
assemblyFileName,
configFileName: null,
shadowCopy: false,
shadowCopyFolder: null,
diagnosticMessageSink: diagnosticSink,
verifyTestAssemblyExists: false);
discoveryOptions.SetSynchronousMessageReporting(true);
testOptions.SetSynchronousMessageReporting(true);
Log($"Discovering tests for {assemblyFileName}...");
var assembly = Assembly.LoadFrom(assemblyFileName);
var assemblyInfo = new Xunit.Sdk.ReflectionAssemblyInfo(assembly);
var discoverer = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink);
discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList();
Log($"Discovery finished.");
Log("");
var summarySink = new DelegatingExecutionSummarySink(
testSink,
() => false,
(completed, summary) => { Log($"Tests run: {summary.Total}, Errors: 0, Failures: {summary.Failed}, Skipped: {summary.Skipped}. Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s"); });
var resultsXmlAssembly = new XElement("assembly");
var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly);
testSink.Execution.TestPassedEvent += args => { Log($"[PASS] {args.Message.Test.DisplayName}", color: "green"); };
testSink.Execution.TestSkippedEvent += args => { Log($"[SKIP] {args.Message.Test.DisplayName}", color: "orange"); };
testSink.Execution.TestFailedEvent += args => { Log($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}", color: "red"); };
testSink.Execution.TestAssemblyStartingEvent += args => { Log($"Running tests for {args.Message.TestAssembly.Assembly}"); };
testSink.Execution.TestAssemblyFinishedEvent += args => { Log($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); };
controller.RunTests(testCasesToRun, resultsSink, testOptions);
resultsSink.Finished.WaitOne();
var resultsXml = new XElement("assemblies");
resultsXml.Add(resultsXmlAssembly);
Console.WriteLine(resultsXml.ToString());
Log("");
Log("Test results (Base64 encoded):");
var base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(resultsXml.ToString()));
Log(base64, id: "results");
return(resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0);
}