public static int Main(string[] args)
{
var asm = typeof(SingleFileTestRunner).Assembly;
Console.WriteLine("Running assembly:" + asm.FullName);
var diagnosticSink = new ConsoleDiagnosticMessageSink();
var testsFinished = new TaskCompletionSource();
var testSink = new TestMessageSink();
var summarySink = new DelegatingExecutionSummarySink(testSink,
() => false,
(completed, summary) => Console.WriteLine($"Tests run: {summary.Total}, Errors: {summary.Errors}, Failures: {summary.Failed}, Skipped: {summary.Skipped}. Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s"));
var resultsXmlAssembly = new XElement("assembly");
var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly);
testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); };
testSink.Execution.TestFailedEvent += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{Xunit.ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{Xunit.ExceptionUtility.CombineStackTraces(args.Message)}"); };
testSink.Execution.TestAssemblyFinishedEvent += args =>
{
Console.WriteLine($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}");
testsFinished.SetResult();
};
var xunitTestFx = new SingleFileTestRunner(diagnosticSink);
var asmInfo = Reflector.Wrap(asm);
var asmName = asm.GetName();
var discoverySink = new TestDiscoverySink();
var discoverer = xunitTestFx.CreateDiscoverer(asmInfo);
discoverer.Find(false, discoverySink, TestFrameworkOptions.ForDiscovery());
discoverySink.Finished.WaitOne();
XunitFilters filters = new XunitFilters();
filters.ExcludedTraits.Add("category", new List <string> {
"failing"
});
var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList();
var executor = xunitTestFx.CreateExecutor(asmName);
executor.RunTests(filteredTestCases, resultsSink, TestFrameworkOptions.ForExecution());
resultsSink.Finished.WaitOne();
var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0;
return(failed ? 1 : 0);
}
async ValueTask <XElement?> ExecuteAssembly(
XunitProjectAssembly assembly,
bool needsXml,
_IMessageSink reporterMessageHandler)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
var assemblyFileName = Guard.ArgumentNotNull("assembly.AssemblyFilename", assembly.AssemblyFilename);
// Setup discovery and execution options with command-line overrides
var discoveryOptions = _TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = _TestFrameworkOptions.ForExecution(assembly.Configuration);
// The normal default is true here, but we want it to be false for us by default
if (!assembly.Configuration.PreEnumerateTheories.HasValue)
{
discoveryOptions.SetPreEnumerateTheories(false);
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName);
var noColor = assembly.Project.Configuration.NoColorOrDefault;
var diagnosticMessageSink = ConsoleDiagnosticMessageSink.ForDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
var internalDiagnosticsMessageSink = ConsoleDiagnosticMessageSink.ForInternalDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.InternalDiagnosticMessagesOrDefault, noColor);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
using var _ = AssemblyHelper.SubscribeResolveForAssembly(assemblyFileName, internalDiagnosticsMessageSink);
await using var controller = XunitFrontController.ForDiscoveryAndExecution(assembly, diagnosticMessageSink: diagnosticMessageSink);
var executionStarting = new TestAssemblyExecutionStarting
{
Assembly = assembly,
ExecutionOptions = executionOptions
};
reporterMessageHandler.OnMessage(executionStarting);
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (summary, _) => completionMessages.TryAdd(controller.TestAssemblyUniqueID, summary));
if (assemblyElement != null)
{
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
}
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
}
if (assembly.Configuration.FailSkipsOrDefault)
{
resultsSink = new DelegatingFailSkipSink(resultsSink);
}
using (resultsSink)
{
var settings = new FrontControllerFindAndRunSettings(discoveryOptions, executionOptions, assembly.Configuration.Filters);
controller.FindAndRun(resultsSink, settings);
resultsSink.Finished.WaitOne();
var executionFinished = new TestAssemblyExecutionFinished
{
Assembly = assembly,
ExecutionOptions = executionOptions,
ExecutionSummary = resultsSink.ExecutionSummary
};
reporterMessageHandler.OnMessage(executionFinished);
if (assembly.Configuration.StopOnFailOrDefault && resultsSink.ExecutionSummary.Failed != 0)
{
Console.WriteLine("Canceling due to test failure...");
cancel = true;
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine($"{e.GetType().FullName}: {e.Message}");
if (assembly.Configuration.InternalDiagnosticMessagesOrDefault)
{
Console.WriteLine(e.StackTrace);
}
e = e.InnerException;
}
}
return(assemblyElement);
}
public async ValueTask <int> EntryPoint()
{
var globalInternalDiagnosticMessages = false;
var noColor = false;
try
{
var reporters = GetAvailableRunnerReporters();
if (args.Length == 0 || args[0] == "-?" || args[0] == "/?" || args[0] == "-h" || args[0] == "--help")
{
PrintHeader();
PrintUsage(reporters);
return(2);
}
if (commandLine.ParseFault != null)
{
ExceptionDispatchInfo.Capture(commandLine.ParseFault).Throw();
}
if (commandLine.Project.Assemblies.Count == 0)
{
throw new ArgumentException("must specify at least one assembly");
}
AppDomain.CurrentDomain.UnhandledException += OnUnhandledException;
Console.CancelKeyPress += (sender, e) =>
{
if (!cancel)
{
Console.WriteLine("Canceling... (Press Ctrl+C again to terminate)");
cancel = true;
e.Cancel = true;
}
};
await using var reporter = commandLine.ChooseReporter(reporters);
if (commandLine.Project.Configuration.PauseOrDefault)
{
Console.Write("Press any key to start execution...");
Console.ReadKey(true);
Console.WriteLine();
}
if (commandLine.Project.Configuration.DebugOrDefault)
{
Debugger.Launch();
}
// We will enable "global" internal diagnostic messages if any test assembly wanted them
globalInternalDiagnosticMessages = commandLine.Project.Assemblies.Any(a => a.Configuration.InternalDiagnosticMessagesOrDefault);
noColor = commandLine.Project.Configuration.NoColorOrDefault;
logger = new ConsoleRunnerLogger(!noColor, consoleLock);
var diagnosticMessageSink = ConsoleDiagnosticMessageSink.ForInternalDiagnostics(consoleLock, globalInternalDiagnosticMessages, noColor);
var reporterMessageHandler = await reporter.CreateMessageHandler(logger, diagnosticMessageSink);
if (!reporter.ForceNoLogo && !commandLine.Project.Configuration.NoLogoOrDefault)
{
PrintHeader();
}
var failCount = await RunProject(
commandLine.Project,
reporterMessageHandler
);
if (cancel)
{
return(-1073741510); // 0xC000013A: The application terminated as a result of a CTRL+C
}
if (commandLine.Project.Configuration.WaitOrDefault)
{
Console.WriteLine();
Console.Write("Press any key to continue...");
Console.ReadKey();
Console.WriteLine();
}
return(failCount > 0 ? 1 : 0);
}
catch (Exception ex)
{
if (!noColor)
{
ConsoleHelper.SetForegroundColor(ConsoleColor.Red);
}
Console.WriteLine($"error: {ex.Message}");
if (globalInternalDiagnosticMessages)
{
if (!noColor)
{
ConsoleHelper.SetForegroundColor(ConsoleColor.DarkGray);
}
Console.WriteLine(ex.StackTrace);
}
return(ex is ArgumentException ? 3 : 4);
}
finally
{
if (!noColor)
{
ConsoleHelper.ResetColor();
}
}
}
public static int Main(string[] args)
{
var asm = typeof(SingleFileTestRunner).Assembly;
Console.WriteLine("Running assembly:" + asm.FullName);
var diagnosticSink = new ConsoleDiagnosticMessageSink();
var testsFinished = new TaskCompletionSource();
var testSink = new TestMessageSink();
var summarySink = new DelegatingExecutionSummarySink(testSink,
() => false,
(completed, summary) => Console.WriteLine($"Tests run: {summary.Total}, Errors: {summary.Errors}, Failures: {summary.Failed}, Skipped: {summary.Skipped}. Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s"));
var resultsXmlAssembly = new XElement("assembly");
var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly);
testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); };
testSink.Execution.TestFailedEvent += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{Xunit.ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{Xunit.ExceptionUtility.CombineStackTraces(args.Message)}"); };
testSink.Execution.TestAssemblyFinishedEvent += args =>
{
Console.WriteLine($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}");
testsFinished.SetResult();
};
var assemblyConfig = new TestAssemblyConfiguration()
{
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
PreEnumerateTheories = false,
};
var xunitTestFx = new SingleFileTestRunner(diagnosticSink);
var asmInfo = Reflector.Wrap(asm);
var asmName = asm.GetName();
var discoverySink = new TestDiscoverySink();
var discoverer = xunitTestFx.CreateDiscoverer(asmInfo);
discoverer.Find(false, discoverySink, TestFrameworkOptions.ForDiscovery(assemblyConfig));
discoverySink.Finished.WaitOne();
string xmlResultFileName = null;
XunitFilters filters = new XunitFilters();
// Quick hack wo much validation to get args that are passed (notrait, xml)
Dictionary <string, List <string> > noTraits = new Dictionary <string, List <string> >();
for (int i = 0; i < args.Length; i++)
{
if (args[i].Equals("-notrait", StringComparison.OrdinalIgnoreCase))
{
var traitKeyValue = args[i + 1].Split("=", StringSplitOptions.TrimEntries);
if (!noTraits.TryGetValue(traitKeyValue[0], out List <string> values))
{
noTraits.Add(traitKeyValue[0], values = new List <string>());
}
values.Add(traitKeyValue[1]);
}
if (args[i].Equals("-xml", StringComparison.OrdinalIgnoreCase))
{
xmlResultFileName = args[i + 1].Trim();
}
}
foreach (KeyValuePair <string, List <string> > kvp in noTraits)
{
filters.ExcludedTraits.Add(kvp.Key, kvp.Value);
}
var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList();
var executor = xunitTestFx.CreateExecutor(asmName);
executor.RunTests(filteredTestCases, resultsSink, TestFrameworkOptions.ForExecution(assemblyConfig));
resultsSink.Finished.WaitOne();
// Helix need to see results file in the drive to detect if the test has failed or not
if (xmlResultFileName != null)
{
resultsXmlAssembly.Save(xmlResultFileName);
}
var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0;
return(failed ? 1 : 0);
}
async ValueTask <XElement?> ExecuteAssembly(
XunitProjectAssembly assembly,
bool needsXml,
_IMessageSink reporterMessageHandler)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
// Setup discovery and execution options with command-line overrides
var discoveryOptions = _TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = _TestFrameworkOptions.ForExecution(assembly.Configuration);
// The normal default is true here, but we want it to be false for us by default
if (!assembly.Configuration.PreEnumerateTheories.HasValue)
{
discoveryOptions.SetPreEnumerateTheories(false);
}
var assemblyDisplayName = assembly.AssemblyDisplayName;
var noColor = assembly.Project.Configuration.NoColorOrDefault;
var diagnosticMessageSink = ConsoleDiagnosticMessageSink.ForDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
var internalDiagnosticsMessageSink = ConsoleDiagnosticMessageSink.ForInternalDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.InternalDiagnosticMessagesOrDefault, noColor);
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
var assemblyInfo = new ReflectionAssemblyInfo(testAssembly);
await using var disposalTracker = new DisposalTracker();
var testFramework = ExtensibilityPointFactory.GetTestFramework(diagnosticMessageSink, assemblyInfo);
disposalTracker.Add(testFramework);
var discoverySink = new TestDiscoverySink(() => cancel);
// Discover & filter the tests
var testDiscoverer = testFramework.GetDiscoverer(assemblyInfo);
var discoveryStarting = new TestAssemblyDiscoveryStarting
{
AppDomain = AppDomainOption.NotAvailable,
Assembly = assembly,
DiscoveryOptions = discoveryOptions,
ShadowCopy = false
};
reporterMessageHandler.OnMessage(discoveryStarting);
testDiscoverer.Find(discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(assembly.Configuration.Filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
var discoveryFinished = new TestAssemblyDiscoveryFinished
{
Assembly = assembly,
DiscoveryOptions = discoveryOptions,
TestCasesDiscovered = testCasesDiscovered,
TestCasesToRun = testCasesToRun
};
reporterMessageHandler.OnMessage(discoveryFinished);
// Run the filtered tests
if (testCasesToRun == 0)
{
testExecutionSummaries.Add(testDiscoverer.TestAssemblyUniqueID, new ExecutionSummary());
}
else
{
var executionStarting = new TestAssemblyExecutionStarting
{
Assembly = assembly,
ExecutionOptions = executionOptions
};
reporterMessageHandler.OnMessage(executionStarting);
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel);
if (assemblyElement != null)
{
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
}
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
}
if (assembly.Configuration.FailSkipsOrDefault)
{
resultsSink = new DelegatingFailSkipSink(resultsSink);
}
using (resultsSink)
{
var executor = testFramework.GetExecutor(assemblyInfo);
executor.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
testExecutionSummaries.Add(testDiscoverer.TestAssemblyUniqueID, resultsSink.ExecutionSummary);
var executionFinished = new TestAssemblyExecutionFinished
{
Assembly = assembly,
ExecutionOptions = executionOptions,
ExecutionSummary = resultsSink.ExecutionSummary
};
reporterMessageHandler.OnMessage(executionFinished);
if (assembly.Configuration.StopOnFailOrDefault && resultsSink.ExecutionSummary.Failed != 0)
{
Console.WriteLine("Canceling due to test failure...");
cancel = true;
}
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine($"{e.GetType().FullName}: {e.Message}");
if (assembly.Configuration.InternalDiagnosticMessagesOrDefault)
{
Console.WriteLine(e.StackTrace);
}
e = e.InnerException;
}
}
return(assemblyElement);
}
public bool Run(string assemblyFileName, IEnumerable <string> excludedTraits)
{
WebAssembly.Runtime.InvokeJS($"if (document) document.body.innerHTML = ''");
Log("Starting tests...");
var filters = new XunitFilters();
foreach (var trait in excludedTraits ?? Array.Empty <string>())
{
ParseEqualSeparatedArgument(filters.ExcludedTraits, trait);
}
var configuration = new TestAssemblyConfiguration
{
ShadowCopy = false,
ParallelizeAssembly = false,
ParallelizeTestCollections = false,
MaxParallelThreads = 1,
PreEnumerateTheories = false
};
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
var discoverySink = new TestDiscoverySink();
var diagnosticSink = new ConsoleDiagnosticMessageSink();
var testOptions = TestFrameworkOptions.ForExecution(configuration);
var testSink = new TestMessageSink();
var controller = new Xunit2(
AppDomainSupport.Denied,
new NullSourceInformationProvider(),
assemblyFileName,
configFileName: null,
shadowCopy: false,
shadowCopyFolder: null,
diagnosticMessageSink: diagnosticSink,
verifyTestAssemblyExists: false);
discoveryOptions.SetSynchronousMessageReporting(true);
testOptions.SetSynchronousMessageReporting(true);
Log($"Discovering tests for {assemblyFileName}...");
var assembly = Assembly.LoadFrom(assemblyFileName);
var assemblyInfo = new Xunit.Sdk.ReflectionAssemblyInfo(assembly);
var discoverer = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink);
discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList();
Log($"Discovery finished.");
Log("");
var summarySink = new DelegatingExecutionSummarySink(
testSink,
() => false,
(completed, summary) => { Log($"Tests run: {summary.Total}, Errors: 0, Failures: {summary.Failed}, Skipped: {summary.Skipped}. Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s"); });
var resultsXmlAssembly = new XElement("assembly");
var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly);
testSink.Execution.TestPassedEvent += args => { Log($"[PASS] {args.Message.Test.DisplayName}", color: "green"); };
testSink.Execution.TestSkippedEvent += args => { Log($"[SKIP] {args.Message.Test.DisplayName}", color: "orange"); };
testSink.Execution.TestFailedEvent += args => { Log($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}", color: "red"); };
testSink.Execution.TestAssemblyStartingEvent += args => { Log($"Running tests for {args.Message.TestAssembly.Assembly}"); };
testSink.Execution.TestAssemblyFinishedEvent += args => { Log($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); };
controller.RunTests(testCasesToRun, resultsSink, testOptions);
resultsSink.Finished.WaitOne();
var resultsXml = new XElement("assemblies");
resultsXml.Add(resultsXmlAssembly);
Console.WriteLine(resultsXml.ToString());
Log("");
Log("Test results (Base64 encoded):");
var base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(resultsXml.ToString()));
Log(base64, id: "results");
return(resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0);
}
