internal static void Go(string assemblyFileName, Stream stream)
{
Go(assemblyFileName, stream, AppDomainSupport.Denied,
(xunit, configuration, writer) =>
{
using (var sink = new TestDiscoverySink(writer))
{
xunit.Find(includeSourceInformation: false, messageSink: sink,
discoveryOptions: TestFrameworkOptions.ForDiscovery(configuration));
sink.Finished.WaitOne();
writer.Write(TestDataKind.EndOfData);
}
});
}
IReadOnlyList<ITestCase> Discover(Action<IMessageSinkWithTypes> discoveryAction)
{
try
{
var sink = new TestDiscoverySink();
toDispose.Push(sink);
discoveryAction(sink);
sink.Finished.WaitOne();
return sink.TestCases.ToList();
}
catch (Exception ex)
{
testListener.WriteLine("Error during test discovery:\r\n" + ex, Category.Error);
return new ITestCase[0];
}
}
public static int Main(string[] args)
{
var arguments = new Stack <string> ();
string host = null;
int port = 0;
bool closeAfterTestRun;
bool failed;
for (var i = args.Length - 1; i >= 0; i--)
{
arguments.Push(args[i]);
}
// First argument is the connection string if we're driven by harness.exe, otherwise we're driven by UITests
if (arguments.Count > 0 && arguments.Peek().StartsWith("tcp:", StringComparison.Ordinal))
{
var parts = arguments.Pop().Split(':');
if (parts.Length != 3)
{
throw new Exception();
}
host = parts [1];
port = Int32.Parse(parts [2]);
closeAfterTestRun = true;
}
else
{
closeAfterTestRun = false;
}
// Make sure the TLS subsystem including the DependencyInjector is initialized.
// This would normally happen on system startup in
// `xamarin-macios/src/ObjcRuntime/Runtime.cs`.
MonoTlsProviderFactory.Initialize();
// some tests assert having a SynchronizationContext for MONOTOUCH, provide a default one
SynchronizationContext.SetSynchronizationContext(new SynchronizationContext());
#if XUNIT_RUNNER
var writer = new TcpWriter(host, port);
var assemblyFileName = arguments.Pop();
var filters = XunitArgumentsParser.ParseArgumentsToFilter(arguments);
var configuration = new TestAssemblyConfiguration()
{
ShadowCopy = false
};
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
var discoverySink = new TestDiscoverySink();
var diagnosticSink = new DiagnosticTextWriterMessageSink(writer);
var testOptions = TestFrameworkOptions.ForExecution(configuration);
var testSink = new TestMessageSink();
var controller = new XunitFrontController(AppDomainSupport.Denied, assemblyFileName, configFileName: null, shadowCopy: false, diagnosticMessageSink: diagnosticSink);
Interop.mono_sdks_ui_set_test_summary_message($"Running {assemblyFileName}...");
writer.WriteLine($"Discovering tests for {assemblyFileName}");
controller.Find(includeSourceInformation: false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList();
writer.WriteLine($"Discovery finished.");
var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => { writer.WriteLine($"Tests run: {summary.Total}, Errors: 0, Failures: {summary.Failed}, Skipped: {summary.Skipped}{Environment.NewLine}Time: {TimeSpan.FromSeconds ((double)summary.Time).TotalSeconds}s"); });
var resultsXmlAssembly = new XElement("assembly");
var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly);
testSink.Execution.TestPassedEvent += args => { writer.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); Interop.mono_sdks_ui_increment_testcase_result(0); };
testSink.Execution.TestSkippedEvent += args => { writer.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); Interop.mono_sdks_ui_increment_testcase_result(1); };
testSink.Execution.TestFailedEvent += args => { writer.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages (args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces (args.Message)}"); Interop.mono_sdks_ui_increment_testcase_result(2); };
testSink.Execution.TestAssemblyStartingEvent += args => { writer.WriteLine($"Running tests for {args.Message.TestAssembly.Assembly}"); };
testSink.Execution.TestAssemblyFinishedEvent += args => { writer.WriteLine($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); };
controller.RunTests(testCasesToRun, resultsSink, testOptions);
resultsSink.Finished.WaitOne();
var resultsXml = new XElement("assemblies");
resultsXml.Add(resultsXmlAssembly);
resultsXml.Save(resultsXmlPath);
if (host != null)
{
writer.WriteLine($"STARTRESULTXML");
resultsXml.Save(((TcpWriter)writer).RawStream);
writer.WriteLine();
writer.WriteLine($"ENDRESULTXML");
}
failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0;
#else
MonoSdksTextUI runner;
TextWriter writer = null;
string resultsXmlPath = Path.GetTempFileName();
string assemblyFileName = arguments.Peek();
if (File.Exists("nunit-excludes.txt"))
{
var excludes = File.ReadAllLines("nunit-excludes.txt");
arguments.Push("-exclude:" + String.Join(",", excludes));
}
arguments.Push("-labels");
arguments.Push("-format:xunit");
arguments.Push($"-result:{resultsXmlPath}");
if (host != null)
{
Console.WriteLine($"Connecting to harness at {host}:{port}.");
writer = new TcpWriter(host, port);
}
else
{
writer = ConsoleWriter.Out;
}
Interop.mono_sdks_ui_set_test_summary_message($"Running {assemblyFileName}...");
runner = new MonoSdksTextUI(writer);
runner.Execute(arguments.ToArray());
if (host != null)
{
writer.WriteLine($"STARTRESULTXML");
using (var resultsXmlStream = File.OpenRead(resultsXmlPath)) resultsXmlStream.CopyTo(((TcpWriter)writer).RawStream);
writer.WriteLine();
writer.WriteLine($"ENDRESULTXML");
}
failed = runner.Failure;
#endif
Interop.mono_sdks_ui_set_test_summary_message($"Summary: {(failed ? "Failed" : "Succeeded")} for {assemblyFileName}.");
if (!closeAfterTestRun)
{
Thread.Sleep(Int32.MaxValue);
}
return(failed ? 1 : 0);
}
XElement ExecuteAssembly(object consoleLock,
XunitProjectAssembly assembly,
bool needsXml,
bool?parallelizeTestCollections,
int?maxThreadCount,
bool diagnosticMessages,
bool noColor,
bool failSkips,
bool stopOnFail,
XunitFilters filters,
bool internalDiagnosticMessages)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.ConfigFilename))
{
return(null);
}
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= diagnosticMessages;
assembly.Configuration.InternalDiagnosticMessages |= internalDiagnosticMessages;
// Setup discovery and execution options with command-line overrides
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
executionOptions.SetStopOnTestFail(stopOnFail);
if (maxThreadCount.HasValue)
{
executionOptions.SetMaxParallelThreads(maxThreadCount);
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageSink = DiagnosticMessageSink.ForDiagnostics(consoleLock, assemblyDisplayName, diagnosticMessages, noColor);
var internalDiagnosticsMessageSink = DiagnosticMessageSink.ForInternalDiagnostics(consoleLock, assemblyDisplayName, internalDiagnosticMessages, noColor);
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
using var testFramework = new XunitTestFramework(diagnosticMessageSink, assembly.ConfigFilename);
var entryAssembly = Assembly.GetEntryAssembly();
var assemblyInfo = new ReflectionAssemblyInfo(entryAssembly);
var discoverySink = new TestDiscoverySink(() => cancel);
using (var testDiscoverer = testFramework.GetDiscoverer(assemblyInfo))
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, appDomain: false, shadowCopy: false, discoveryOptions));
testDiscoverer.Find(includeSourceInformation: false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
}
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
executionSummary = new ExecutionSummary();
}
else
{
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel);
if (assemblyElement != null)
{
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
}
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), MessageSinkWithTypesAdapter.Wrap(diagnosticMessageSink));
}
if (failSkips)
{
resultsSink = new DelegatingFailSkipSink(resultsSink);
}
using var executor = testFramework.GetExecutor(entryAssembly.GetName());
executor.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
executionSummary = resultsSink.ExecutionSummary;
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
if (stopOnFail && resultsSink.ExecutionSummary.Failed != 0)
{
Console.WriteLine("Canceling due to test failure...");
cancel = true;
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine($"{e.GetType().FullName}: {e.Message}");
if (internalDiagnosticMessages)
{
Console.WriteLine(e.StackTrace);
}
e = e.InnerException;
}
}
return(assemblyElement);
}
public static int Main(string[] args)
{
var asm = typeof(SingleFileTestRunner).Assembly;
Console.WriteLine("Running assembly:" + asm.FullName);
var diagnosticSink = new ConsoleDiagnosticMessageSink();
var testsFinished = new TaskCompletionSource();
var testSink = new TestMessageSink();
var summarySink = new DelegatingExecutionSummarySink(testSink,
() => false,
(completed, summary) => Console.WriteLine($"Tests run: {summary.Total}, Errors: {summary.Errors}, Failures: {summary.Failed}, Skipped: {summary.Skipped}. Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s"));
var resultsXmlAssembly = new XElement("assembly");
var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly);
testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); };
testSink.Execution.TestFailedEvent += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{Xunit.ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{Xunit.ExceptionUtility.CombineStackTraces(args.Message)}"); };
testSink.Execution.TestAssemblyFinishedEvent += args =>
{
Console.WriteLine($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}");
testsFinished.SetResult();
};
var assemblyConfig = new TestAssemblyConfiguration()
{
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
PreEnumerateTheories = false,
};
var xunitTestFx = new SingleFileTestRunner(diagnosticSink);
var asmInfo = Reflector.Wrap(asm);
var asmName = asm.GetName();
var discoverySink = new TestDiscoverySink();
var discoverer = xunitTestFx.CreateDiscoverer(asmInfo);
discoverer.Find(false, discoverySink, TestFrameworkOptions.ForDiscovery(assemblyConfig));
discoverySink.Finished.WaitOne();
string xmlResultFileName = null;
XunitFilters filters = new XunitFilters();
// Quick hack wo much validation to get args that are passed (notrait, xml)
Dictionary <string, List <string> > noTraits = new Dictionary <string, List <string> >();
for (int i = 0; i < args.Length; i++)
{
if (args[i].Equals("-notrait", StringComparison.OrdinalIgnoreCase))
{
var traitKeyValue = args[i + 1].Split("=", StringSplitOptions.TrimEntries);
if (!noTraits.TryGetValue(traitKeyValue[0], out List <string> values))
{
noTraits.Add(traitKeyValue[0], values = new List <string>());
}
values.Add(traitKeyValue[1]);
}
if (args[i].Equals("-xml", StringComparison.OrdinalIgnoreCase))
{
xmlResultFileName = args[i + 1].Trim();
}
}
foreach (KeyValuePair <string, List <string> > kvp in noTraits)
{
filters.ExcludedTraits.Add(kvp.Key, kvp.Value);
}
var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList();
var executor = xunitTestFx.CreateExecutor(asmName);
executor.RunTests(filteredTestCases, resultsSink, TestFrameworkOptions.ForExecution(assemblyConfig));
resultsSink.Finished.WaitOne();
// Helix need to see results file in the drive to detect if the test has failed or not
if (xmlResultFileName != null)
{
resultsXmlAssembly.Save(xmlResultFileName);
}
var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0;
return(failed ? 1 : 0);
}
public static void RunTests(Options o)
{
var nullMessage = new Xunit.NullMessageSink();
var discoveryOptions = TestFrameworkOptions.ForDiscovery();
using (var c = new XunitFrontController(AppDomainSupport.Denied, o.Assembly, null, false))
{
var tv = new TestDiscoverySink();
var excludeTestCaseSet = new TestDiscoverySink();
c.Find(true, tv, discoveryOptions);
tv.Finished.WaitOne();
foreach (var tc in tv.TestCases)
{
var method = tc.TestMethod.Method;
var attributes = method.GetCustomAttributes(typeof(FactAttribute));
foreach (ReflectionAttributeInfo at in attributes)
{
var result = at.GetNamedArgument<string>("Skip");
if (result != null)
{
Console.WriteLine("SKIPPY! {0} because {1}", method, result);
}
if (o.TestType != TestType.All)
{
if (!at.ToString().EndsWith(o.TestType.ToString()))
{
excludeTestCaseSet.TestCases.Add(tc);
}
}
}
}
foreach (var tc in excludeTestCaseSet.TestCases)
{
tv.TestCases.Remove(tc);
}
Console.WriteLine("TEST COUNT: {0}", tv.TestCases.Count);
//core execution Sink
int testCaseCount = tv.TestCases.Count;
Stream file = new FileStream(".\\result.xml", FileMode.Create);
int totalResult = 0;
int totalErrors = 0;
int totalFailed = 0;
int totalSkipped = 0;
for (int i = 0; i < testCaseCount; i++)
{
IExecutionSink resultsSink;
ConcurrentDictionary<string, ExecutionSummary> completionMessages = new ConcurrentDictionary<string, ExecutionSummary>();
IMessageSinkWithTypes reporterMessageHandler;
var reporters = GetAvailableRunnerReporters();
var commandLine = CommandLine.Parse(reporters, @"CoreXunit.dll");
IRunnerLogger logger = new ConsoleRunnerLogger(!commandLine.NoColor);
reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(commandLine.Reporter.CreateMessageHandler(logger));
var xmlElement = new XElement("TestResult");
resultsSink = new XmlAggregateSink(reporterMessageHandler, completionMessages, xmlElement, () => true);
var message = new Xunit.NullMessageSink();
var executionOptions = TestFrameworkOptions.ForExecution();
c.RunTests(tv.TestCases.Take<Xunit.Abstractions.ITestCase>(1), resultsSink, executionOptions);
resultsSink.Finished.WaitOne(o.TimeOut);
tv.TestCases.RemoveAt(0);
totalResult++;
totalErrors = totalErrors + resultsSink.ExecutionSummary.Errors;
totalFailed = totalFailed + resultsSink.ExecutionSummary.Failed;
totalSkipped = totalSkipped + resultsSink.ExecutionSummary.Skipped;
xmlElement.Save(file);
file.Flush();
}
file.Dispose();
Console.WriteLine("Total tests: " + totalResult);
Console.ForegroundColor = ConsoleColor.DarkRed;
Console.WriteLine("Error tests: " + totalErrors);
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("Failed tests: " + totalFailed);
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine("Skipped tests: " + totalSkipped);
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("Passed tests: " + (totalResult - totalErrors - totalFailed - totalSkipped));
Console.ResetColor();
}
}
XElement ExecuteAssembly(object consoleLock,
XunitProjectAssembly2 assembly,
bool needsXml,
bool?parallelizeTestCollections,
int?maxThreadCount,
bool diagnosticMessages,
bool noColor,
AppDomainSupport?appDomain,
bool failSkips,
XunitFilters filters,
bool designTime,
bool listTestCases,
IReadOnlyList <string> designTimeFullyQualifiedNames)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
// if we had a config file use it
var config = assembly.ConfigFilename != null ? assembly.Configuration : assembly.ConfigurationStream;
// Turn off pre-enumeration of theories when we're not running in Visual Studio
if (!designTime)
{
config.PreEnumerateTheories = false;
}
if (appDomain.HasValue)
{
config.AppDomain = appDomain.GetValueOrDefault();
}
if (diagnosticMessages)
{
config.DiagnosticMessages = true;
}
var discoveryOptions = TestFrameworkOptions.ForDiscovery(config);
var executionOptions = TestFrameworkOptions.ForExecution(config);
if (maxThreadCount.HasValue)
{
executionOptions.SetMaxParallelThreads(maxThreadCount);
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageSink = new DiagnosticMessageSink(consoleLock, assemblyDisplayName, config.DiagnosticMessagesOrDefault, noColor);
var appDomainSupport = config.AppDomainOrDefault;
var shadowCopy = config.ShadowCopyOrDefault;
var longRunningSeconds = config.LongRunningTestSecondsOrDefault;
var sourceInformationProvider = GetSourceInformationProviderAdapater(assembly);
using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink, sourceInformationProvider: sourceInformationProvider))
using (var discoverySink = new TestDiscoverySink())
{
var includeSourceInformation = designTime && listTestCases;
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions));
controller.Find(includeSourceInformation: includeSourceInformation, messageSink: discoverySink, discoveryOptions: discoveryOptions);
discoverySink.Finished.WaitOne();
IDictionary <ITestCase, VsTestCase> vsTestCases = null;
if (designTime)
{
vsTestCases = DesignTimeTestConverter.Convert(discoverySink.TestCases);
}
if (listTestCases)
{
lock (consoleLock)
{
if (designTime)
{
foreach (var testcase in vsTestCases.Values)
{
testDiscoverySink?.SendTestFound(testcase);
Console.WriteLine(testcase.FullyQualifiedName);
}
}
else
{
foreach (var testcase in discoverySink.TestCases)
{
Console.WriteLine(testcase.DisplayName);
}
}
}
return(assemblyElement);
}
IExecutionSink resultsSink;
if (designTime)
{
resultsSink = new DesignTimeExecutionSink(testExecutionSink, vsTestCases, reporterMessageHandler);
}
else
{
resultsSink = new XmlAggregateSink(reporterMessageHandler, assemblyElement, diagnosticMessageSink, completionMessages, () => cancel, longRunningSeconds);
}
if (failSkips)
{
resultsSink = new FailSkipSink(resultsSink);
}
IList <ITestCase> filteredTestCases;
var testCasesDiscovered = discoverySink.TestCases.Count;
if (!designTime || designTimeFullyQualifiedNames.Count == 0)
{
filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList();
}
else
{
filteredTestCases = vsTestCases.Where(t => designTimeFullyQualifiedNames.Contains(t.Value.FullyQualifiedName)).Select(t => t.Key).ToList();
}
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
if (filteredTestCases.Count == 0)
{
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
}
else
{
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
controller.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine("{0}: {1}", e.GetType().FullName, e.Message);
e = e.InnerException;
}
}
return(assemblyElement);
}
protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly)
{
if (cancel)
{
return(null);
}
var assemblyElement = NeedsXml ? new XElement("assembly") : null;
try
{
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= DiagnosticMessages;
if (appDomains.HasValue)
{
assembly.Configuration.AppDomain = appDomains.GetValueOrDefault() ? AppDomainSupport.Required : AppDomainSupport.Denied;
}
// Setup discovery and execution options with command-line overrides
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue && maxThreadCount.Value > -1)
{
executionOptions.SetMaxParallelThreads(maxThreadCount);
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections);
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageSink = new DiagnosticMessageSink(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
using (var discoverySink = new TestDiscoverySink(() => cancel))
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions));
controller.Find(false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(Filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
}
else
{
if (SerializeTestCases)
{
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
}
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary));
if (assemblyElement != null)
{
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
}
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
}
if (FailSkips)
{
resultsSink = new DelegatingFailSkipSink(resultsSink);
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
controller.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
if (resultsSink.ExecutionSummary.Failed != 0)
{
ExitCode = 1;
}
}
}
}
catch (Exception ex)
{
var e = ex;
while (e != null)
{
Log.LogError("{0}: {1}", e.GetType().FullName, e.Message);
foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
{
Log.LogError(stackLine);
}
e = e.InnerException;
}
ExitCode = -1;
}
return(assemblyElement);
}
protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly)
{
if (cancel)
return null;
var assemblyElement = NeedsXml ? new XElement("assembly") : null;
try
{
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= DiagnosticMessages;
if (appDomains.HasValue)
assembly.Configuration.AppDomain = appDomains.GetValueOrDefault() ? AppDomainSupport.Required : AppDomainSupport.Denied;
// Setup discovery and execution options with command-line overrides
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue && maxThreadCount.Value > -1)
executionOptions.SetMaxParallelThreads(maxThreadCount);
if (parallelizeTestCollections.HasValue)
executionOptions.SetDisableParallelization(!parallelizeTestCollections);
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageSink = new DiagnosticMessageSink(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
using (var discoverySink = new TestDiscoverySink(() => cancel))
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions));
controller.Find(false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(Filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
else
{
if (SerializeTestCases)
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary));
if (assemblyElement != null)
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
if (longRunningSeconds > 0)
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
if (FailSkips)
resultsSink = new DelegatingFailSkipSink(resultsSink);
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
controller.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
if (resultsSink.ExecutionSummary.Failed != 0)
ExitCode = 1;
}
}
}
catch (Exception ex)
{
var e = ex;
while (e != null)
{
Log.LogError("{0}: {1}", e.GetType().FullName, e.Message);
foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
Log.LogError(stackLine);
e = e.InnerException;
}
ExitCode = -1;
}
return assemblyElement;
}
private XElement ExecuteAssembly(
[NotNull] XunitProjectAssembly assembly,
[NotNull] TestRunOptions options,
bool needsXml,
[NotNull] XunitFilters filters)
{
if (assembly == null)
{
throw new ArgumentNullException(nameof(assembly));
}
if (options == null)
{
throw new ArgumentNullException(nameof(options));
}
if (filters == null)
{
throw new ArgumentNullException(nameof(filters));
}
if (_cancel)
{
return(null);
}
var internalDiagnosticMessages = options.InternalDiagnosticMessages;
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(assembly.AssemblyFilename) || !ValidateFileExists(assembly.ConfigFilename))
{
return(null);
}
ConfigureAssembly(assembly, options);
// Setup discovery and execution options with command-line overrides
var executionOptions = ConfiguExecutionOptions(assembly, options);
var assemblyDisplayName = assembly.GetFileNameWithoutExtension().EmptyIfNull();
void LogAction(MessageHandlerArgs <IDiagnosticMessage> args, string assemblyName) => Console.WriteLine($"{assemblyDisplayName}: {args.Message.Message}");
var diagnosticMessageSink = new DiagnosticMessageSink(LogAction, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault);
var internalDiagnosticsMessageSink = new DiagnosticMessageSink(LogAction, assemblyDisplayName, assembly.Configuration.InternalDiagnosticMessagesOrDefault);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
using (AssemblyHelper.SubscribeResolveForAssembly(assembly.AssemblyFilename, internalDiagnosticsMessageSink))
using (var controller = new XunitFrontController(
appDomainSupport,
assembly.AssemblyFilename,
assembly.ConfigFilename,
shadowCopy,
diagnosticMessageSink: diagnosticMessageSink))
using (var discoverySink = new TestDiscoverySink(() => _cancel))
{
ExecuteTests(assembly, options, filters, controller, discoverySink, executionOptions, assemblyElement, diagnosticMessageSink);
}
}
catch (Exception ex)
{
_failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine($"{e.GetType().FullName}: {e.Message}");
if (internalDiagnosticMessages)
{
Console.WriteLine(e.StackTrace);
}
e = e.InnerException;
}
}
return(assemblyElement);
}
public bool Run(string assemblyFileName, IEnumerable <string> excludedTraits)
{
WebAssembly.Runtime.InvokeJS($"if (document) document.body.innerHTML = ''");
Log("Starting tests...");
var filters = new XunitFilters();
foreach (var trait in excludedTraits ?? Array.Empty <string>())
{
ParseEqualSeparatedArgument(filters.ExcludedTraits, trait);
}
var configuration = new TestAssemblyConfiguration
{
ShadowCopy = false,
ParallelizeAssembly = false,
ParallelizeTestCollections = false,
MaxParallelThreads = 1,
PreEnumerateTheories = false
};
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
var discoverySink = new TestDiscoverySink();
var diagnosticSink = new ConsoleDiagnosticMessageSink();
var testOptions = TestFrameworkOptions.ForExecution(configuration);
var testSink = new TestMessageSink();
var controller = new Xunit2(
AppDomainSupport.Denied,
new NullSourceInformationProvider(),
assemblyFileName,
configFileName: null,
shadowCopy: false,
shadowCopyFolder: null,
diagnosticMessageSink: diagnosticSink,
verifyTestAssemblyExists: false);
discoveryOptions.SetSynchronousMessageReporting(true);
testOptions.SetSynchronousMessageReporting(true);
Log($"Discovering tests for {assemblyFileName}...");
var assembly = Assembly.LoadFrom(assemblyFileName);
var assemblyInfo = new Xunit.Sdk.ReflectionAssemblyInfo(assembly);
var discoverer = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink);
discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList();
Log($"Discovery finished.");
Log("");
var summarySink = new DelegatingExecutionSummarySink(
testSink,
() => false,
(completed, summary) => { Log($"Tests run: {summary.Total}, Errors: 0, Failures: {summary.Failed}, Skipped: {summary.Skipped}. Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s"); });
var resultsXmlAssembly = new XElement("assembly");
var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly);
testSink.Execution.TestPassedEvent += args => { Log($"[PASS] {args.Message.Test.DisplayName}", color: "green"); };
testSink.Execution.TestSkippedEvent += args => { Log($"[SKIP] {args.Message.Test.DisplayName}", color: "orange"); };
testSink.Execution.TestFailedEvent += args => { Log($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}", color: "red"); };
testSink.Execution.TestAssemblyStartingEvent += args => { Log($"Running tests for {args.Message.TestAssembly.Assembly}"); };
testSink.Execution.TestAssemblyFinishedEvent += args => { Log($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); };
controller.RunTests(testCasesToRun, resultsSink, testOptions);
resultsSink.Finished.WaitOne();
var resultsXml = new XElement("assemblies");
resultsXml.Add(resultsXmlAssembly);
Console.WriteLine(resultsXml.ToString());
Log("");
Log("Test results (Base64 encoded):");
var base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(resultsXml.ToString()));
Log(base64, id: "results");
return(resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0);
}
/// <summary>
/// <see cref="StarcounterXunitRunner.Start(string, string)"/> for description.
/// </summary>
/// <param name="typeName"></param>
/// <param name="testReportName"></param>
private void ExecuteTests(string typeName = null, string testReportName = null)
{
ExecutionSummary executionSummary = null;
XunitProjectAssembly assembly = new XunitProjectAssembly
{
AssemblyFilename = assemblyLocation,
ConfigFilename = null
};
XElement assembliesElement = new XElement("assemblies");
XElement assemblyElement = new XElement("assembly");
// Logger
var verboserReporter = new XunitReporters.VerboseReporter();
IRunnerLogger logger = new ConsoleRunnerLogger(useColors: true);
IMessageSinkWithTypes reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(verboserReporter.CreateMessageHandler(logger));
// Option setup
ITestFrameworkDiscoveryOptions discoveryOptions = TestFrameworkOptions.ForDiscovery(null);
ITestFrameworkExecutionOptions executionOptions = TestFrameworkOptions.ForExecution(null);
executionOptions.SetSynchronousMessageReporting(true);
executionOptions.SetDisableParallelization(DeveloperMode || !RunTestsInParallel);
executionOptions.SetDiagnosticMessages(true);
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var clockTime = Stopwatch.StartNew();
bool cancel = false;
using (var controller = new XunitFrontController(
appDomainSupport: AppDomainSupport.Denied,
assemblyFileName: assembly.AssemblyFilename,
configFileName: null,
shadowCopy: shadowCopy,
shadowCopyFolder: null,
sourceInformationProvider: null,
diagnosticMessageSink: null))
using (var discoverySink = new TestDiscoverySink(() => cancel))
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(
assembly: assembly,
appDomain: controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied,
shadowCopy: shadowCopy,
discoveryOptions: discoveryOptions));
if (typeName != null)
{
controller.Find(typeName, false, discoverySink, discoveryOptions);
}
else
{
controller.Find(false, discoverySink, discoveryOptions);
}
discoverySink.Finished.WaitOne();
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(TestCaseFilter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
executionSummary = new ExecutionSummary();
}
else
{
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => { executionSummary = summary; });
if (assemblyElement != null)
{
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
}
controller.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
assembliesElement.Add(assemblyElement);
}
}
clockTime.Stop();
assembliesElement.Add(new XAttribute("timestamp", DateTime.Now.ToString(CultureInfo.InvariantCulture)));
if (executionSummary != null)
{
Console.WriteLine();
KeyValuePair <string, ExecutionSummary> kvpExecutionSummary = new KeyValuePair <string, ExecutionSummary>(this.assebmlyName, executionSummary);
reporterMessageHandler.OnMessage(new TestExecutionSummary(clockTime.Elapsed, new List <KeyValuePair <string, ExecutionSummary> > {
kvpExecutionSummary
}));
if (testReportName != null)
{
// Create folder if it does not exist
FileInfo fi = new FileInfo(testReportName);
DirectoryInfo directory = fi.Directory;
if (!directory.Exists)
{
Directory.CreateDirectory(directory.FullName);
}
CreateXmlTestReport(assembliesElement, fi);
CreateHtmlTestReport(assembliesElement, fi);
}
Console.WriteLine();
Console.WriteLine();
}
}
private static List<ITestCase> GetTestCaseList(XunitFrontController xunit, TestAssemblyConfiguration configuration, HashSet<string> testCaseNameSet)
{
var testCaseList = new List<ITestCase>();
using (var sink = new TestDiscoverySink(testCaseNameSet, testCaseList))
{
xunit.Find(includeSourceInformation: false, messageSink: sink,
discoveryOptions: TestFrameworkOptions.ForDiscovery(configuration));
sink.Finished.WaitOne();
}
return testCaseList;
}
public static int Main(string[] args)
{
string host = null;
int port = 0;
// First argument is the connection string
if (args [0].StartsWith("tcp:"))
{
var parts = args [0].Split(':');
if (parts.Length != 3)
{
throw new Exception();
}
host = parts [1];
port = Int32.Parse(parts [2]);
args = args.Skip(1).ToArray();
}
// Make sure the TLS subsystem including the DependencyInjector is initialized.
// This would normally happen on system startup in
// `xamarin-macios/src/ObjcRuntime/Runtime.cs`.
MonoTlsProviderFactory.Initialize();
#if XUNIT_RUNNER
var writer = new TcpWriter(host, port);
var assemblyFileName = args[0];
var configuration = new TestAssemblyConfiguration()
{
ShadowCopy = false
};
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
var discoverySink = new TestDiscoverySink();
var diagnosticSink = new DiagnosticTextWriterMessageSink(writer);
var testOptions = TestFrameworkOptions.ForExecution(configuration);
var testSink = new TestMessageSink();
var controller = new XunitFrontController(AppDomainSupport.Denied, assemblyFileName, configFileName: null, shadowCopy: false, diagnosticMessageSink: diagnosticSink);
writer.WriteLine($"Discovering tests for {assemblyFileName}");
controller.Find(includeSourceInformation: false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
writer.WriteLine($"Discovery finished.");
var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => { writer.WriteLine($"Tests run: {summary.Total}, Errors: 0, Failures: {summary.Failed}, Skipped: {summary.Skipped}{Environment.NewLine}Time: {TimeSpan.FromSeconds ((double)summary.Time).TotalSeconds}s"); });
var resultsXmlAssembly = new XElement("assembly");
var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly);
testSink.Execution.TestPassedEvent += args => { writer.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); Interop.mono_sdks_ui_increment_testcase_result(0); };
testSink.Execution.TestSkippedEvent += args => { writer.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); Interop.mono_sdks_ui_increment_testcase_result(1); };
testSink.Execution.TestFailedEvent += args => { writer.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages (args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces (args.Message)}"); Interop.mono_sdks_ui_increment_testcase_result(2); };
testSink.Execution.TestAssemblyStartingEvent += args => { writer.WriteLine($"Running tests for {args.Message.TestAssembly.Assembly}"); };
testSink.Execution.TestAssemblyFinishedEvent += args => { writer.WriteLine($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); };
controller.RunTests(discoverySink.TestCases, resultsSink, testOptions);
resultsSink.Finished.WaitOne();
writer.WriteLine($"STARTRESULTXML");
var resultsXml = new XElement("assemblies");
resultsXml.Add(resultsXmlAssembly);
resultsXml.Save(writer.RawStream);
writer.WriteLine();
writer.WriteLine($"ENDRESULTXML");
var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0;
return(failed ? 1 : 0);
#else
MonoSdksTextUI runner;
TcpWriter writer = null;
string resultsXml = null;
if (host != null)
{
Console.WriteLine($"Connecting to harness at {host}:{port}.");
resultsXml = Path.GetTempFileName();
args = args.Concat(new string[] { "-format:xunit", $"-result:{resultsXml}" }).ToArray();
writer = new TcpWriter(host, port);
runner = new MonoSdksTextUI(writer);
}
else
{
runner = new MonoSdksTextUI();
}
runner.Execute(args);
if (resultsXml != null)
{
writer.WriteLine($"STARTRESULTXML");
using (var resultsXmlStream = File.OpenRead(resultsXml)) resultsXmlStream.CopyTo(writer.RawStream);
writer.WriteLine();
writer.WriteLine($"ENDRESULTXML");
}
return(runner.Failure ? 1 : 0);
#endif
}
static int Main(string[] args)
{
if (args.Length < 3)
{
Console.WriteLine("Usage: <out-dir> <corefx dir> <test assembly filename> <xunit console options>");
return(1);
}
var testAssemblyName = Path.GetFileNameWithoutExtension(args [2]);
var testAssemblyFull = Path.GetFullPath(args[2]);
var outdir_name = Path.Combine(args [0], testAssemblyName);
var sdkdir = args [1] + "/artifacts/bin/runtime/netcoreapp-OSX-Debug-x64";
args = args.Skip(2).ToArray();
// Response file support
var extra_args = new List <string> ();
for (int i = 0; i < args.Length; ++i)
{
var arg = args [i];
if (arg [0] == '@')
{
foreach (var line in File.ReadAllLines(arg.Substring(1)))
{
if (line.Length == 0 || line [0] == '#')
{
continue;
}
extra_args.AddRange(line.Split(' '));
}
args [i] = "";
}
}
args = args.Where(s => s != String.Empty).Concat(extra_args).ToArray();
// Despite a lot of effort, couldn't get dotnet to load these assemblies from the sdk dir, so copy them to our binary dir
// File.Copy ($"{sdkdir}/Microsoft.DotNet.PlatformAbstractions.dll", AppContext.BaseDirectory, true);
File.Copy($"{sdkdir}/CoreFx.Private.TestUtilities.dll", AppContext.BaseDirectory, true);
File.Copy($"{sdkdir}/Microsoft.DotNet.XUnitExtensions.dll", AppContext.BaseDirectory, true);
var cmdline = CommandLine.Parse(args);
// Ditto
File.Copy(cmdline.Project.Assemblies.First().AssemblyFilename, AppContext.BaseDirectory, true);
var assembly = Assembly.LoadFrom(Path.Combine(AppContext.BaseDirectory, Path.GetFileName(cmdline.Project.Assemblies.First().AssemblyFilename)));
var msg_sink = new MsgSink();
var xunit2 = new Xunit2Discoverer(AppDomainSupport.Denied, new NullSourceInformationProvider(), new ReflectionAssemblyInfo(assembly), null, null, msg_sink);
var sink = new TestDiscoverySink();
var config = new TestAssemblyConfiguration()
{
DiagnosticMessages = true, InternalDiagnosticMessages = true, PreEnumerateTheories = false
};
xunit2.Find(false, sink, TestFrameworkOptions.ForDiscovery(config));
sink.Finished.WaitOne();
foreach (XunitTestCase tc in sink.TestCases)
{
ComputeTraits(assembly, tc);
}
// Compute testcase data
var tc_data = new Dictionary <XunitTestCase, List <TcCase> > ();
foreach (XunitTestCase tc in sink.TestCases)
{
var m = ((ReflectionMethodInfo)tc.Method).MethodInfo;
var t = m.ReflectedType;
var cases = new List <TcCase> ();
if (m.GetParameters().Length > 0)
{
foreach (var cattr in m.GetCustomAttributes(true))
{
if (cattr is InlineDataAttribute)
{
var data = ((InlineDataAttribute)cattr).GetData(null).First();
if (data == null)
{
data = new object [m.GetParameters().Length];
}
if (data.Length != m.GetParameters().Length)
{
throw new Exception();
}
bool unhandled = false;
foreach (var val in data)
{
if (val is float || val is double)
{
unhandled = true;
}
if (val is Type)
{
var type = val as Type;
if (!type.IsVisible)
{
unhandled = true;
}
}
if (val is Enum)
{
if (!val.GetType().IsPublic)
{
unhandled = true;
}
}
}
if (!unhandled)
{
cases.Add(new TcCase()
{
Values = data
});
}
}
else if (cattr is MemberDataAttribute memberData && memberData.Parameters.Length == 0)
{
MethodInfo testDataMethod = m.DeclaringType.GetMethod(memberData.MemberName);
if (testDataMethod == null)
{
continue;
}
cases.Add(new TcCase {
MemberDataMethod = testDataMethod
});
}
}
}
else
{
cases.Add(new TcCase());
}
tc_data [tc] = cases;
}
var filters = cmdline.Project.Filters;
//
// Generate code using the roslyn syntax apis
// Creating syntax nodes one-by-one is very cumbersome, so use
// CSharpSyntaxTree.ParseText () to create them and ReplaceNode () to insert them into syntax trees.
//
var blocks = new List <BlockSyntax> ();
foreach (XunitTestCase tc in sink.TestCases)
{
var m = ((ReflectionMethodInfo)tc.Method).MethodInfo;
//Console.WriteLine ("" + m.ReflectedType + " " + m + " " + (tc.TestMethodArguments == null));
var t = m.ReflectedType;
if (t.IsGenericType)
{
nskipped++;
continue;
}
if (!filters.Filter(tc))
{
nskipped++;
continue;
}
var cases = tc_data [tc];
int caseindex = 0;
foreach (var test in cases)
{
string typename = GetTypeName(t);
string msg;
if (cases.Count > 1)
{
msg = $"{typename}.{m.Name}[{caseindex}] ...";
}
else
{
msg = $"{typename}.{m.Name} ...";
}
caseindex++;
var block = ParseText <BlockSyntax> (case_template
.Replace("#MSG#", msg)
.Replace("#NRUN#", test.MemberDataMethod == null ? "1" : ((IEnumerable)test.MemberDataMethod.Invoke(null, null)).Cast <object> ().Count().ToString()));
// Obtain the node for the CALL () line
var try_body_node = block.DescendantNodes().OfType <ExpressionStatementSyntax> ().Skip(2).First().Parent;
// Replace with the generated call code
var stmts = GenerateTcCall(t, test, m);
blocks.Add(block.ReplaceNode(try_body_node, Block(stmts.ToArray())));
}
}
var cu = CSharpSyntaxTree.ParseText(runner_template.Replace("#NSKIPPED#", nskipped.ToString())).GetRoot();
// Replace the body of the Run () method with the generated body
var run_body = cu.DescendantNodes().OfType <MethodDeclarationSyntax> ().First().DescendantNodes().OfType <BlockSyntax> ().First();
cu = cu.ReplaceNode(run_body, Block(blocks.ToArray()));
// Create runner.cs
Directory.CreateDirectory(outdir_name);
var outfile_name = Path.Combine(outdir_name, "runner.cs");
File.WriteAllText(outfile_name, cu.NormalizeWhitespace().ToString());
// Generate csproj file
var csproj_template = File.ReadAllText("gen-test.csproj.template");
csproj_template = csproj_template.Replace("#XUNIT_LOCATION#", sdkdir);
csproj_template = csproj_template.Replace("#TEST_ASSEMBLY#", testAssemblyName);
csproj_template = csproj_template.Replace("#TEST_ASSEMBLY_LOCATION#", testAssemblyFull);
File.WriteAllText(Path.Combine(outdir_name, testAssemblyName + "-runner.csproj"), csproj_template);
return(0);
}
static int Main(string[] args)
{
if (args.Length < 3)
{
Console.WriteLine("Usage: <out-dir> <corefx dir> <test assembly filename> <xunit console options>");
return(1);
}
var testAssemblyName = Path.GetFileNameWithoutExtension(args [2]);
var testAssemblyFull = Path.GetFullPath(args[2]);
var outdir_name = Path.Combine(args [0], testAssemblyName);
var sdkdir = args [1] + "/artifacts/bin/runtime/netcoreapp-OSX-Debug-x64";
args = args.Skip(2).ToArray();
// Response file support
var extra_args = new List <string> ();
for (int i = 0; i < args.Length; ++i)
{
var arg = args [i];
if (arg [0] == '@')
{
foreach (var line in File.ReadAllLines(arg.Substring(1)))
{
if (line.Length == 0 || line [0] == '#')
{
continue;
}
extra_args.AddRange(line.Split(' '));
}
args [i] = "";
}
}
args = args.Where(s => s != String.Empty).Concat(extra_args).ToArray();
// Despite a lot of effort, couldn't get dotnet to load these assemblies from the sdk dir, so copy them to our binary dir
// File.Copy ($"{sdkdir}/Microsoft.DotNet.PlatformAbstractions.dll", AppContext.BaseDirectory, true);
File.Copy($"{sdkdir}/CoreFx.Private.TestUtilities.dll", AppContext.BaseDirectory, true);
File.Copy($"{sdkdir}/Microsoft.DotNet.XUnitExtensions.dll", AppContext.BaseDirectory, true);
var cmdline = CommandLine.Parse(args);
// Ditto
File.Copy(cmdline.Project.Assemblies.First().AssemblyFilename, AppContext.BaseDirectory, true);
var assembly = Assembly.LoadFrom(Path.Combine(AppContext.BaseDirectory, Path.GetFileName(cmdline.Project.Assemblies.First().AssemblyFilename)));
var msg_sink = new MsgSink();
var xunit2 = new Xunit2Discoverer(AppDomainSupport.Denied, new NullSourceInformationProvider(), new ReflectionAssemblyInfo(assembly), null, null, msg_sink);
var sink = new TestDiscoverySink();
var config = new TestAssemblyConfiguration()
{
DiagnosticMessages = true, InternalDiagnosticMessages = true, PreEnumerateTheories = false
};
xunit2.Find(false, sink, TestFrameworkOptions.ForDiscovery(config));
sink.Finished.WaitOne();
foreach (XunitTestCase tc in sink.TestCases)
{
ComputeTraits(assembly, tc);
}
// Compute testcase data
var tc_data = new Dictionary <XunitTestCase, List <CaseData> > ();
foreach (XunitTestCase tc in sink.TestCases)
{
var m = ((ReflectionMethodInfo)tc.Method).MethodInfo;
var t = m.ReflectedType;
var cases = new List <CaseData> ();
if (m.GetParameters().Length > 0)
{
foreach (var cattr in m.GetCustomAttributes(true))
{
if (cattr is InlineDataAttribute)
{
var data = ((InlineDataAttribute)cattr).GetData(null).First();
if (data == null)
{
data = new object [m.GetParameters().Length];
}
if (data.Length != m.GetParameters().Length)
{
throw new Exception();
}
bool unhandled = false;
foreach (var val in data)
{
if (val is float || val is double)
{
unhandled = true;
}
if (val is Type)
{
var type = val as Type;
if (!type.IsVisible)
{
unhandled = true;
}
}
if (val is Enum)
{
if (!val.GetType().IsPublic)
{
unhandled = true;
}
}
}
if (!unhandled)
{
cases.Add(new CaseData()
{
Values = data
});
}
}
}
}
else
{
cases.Add(new CaseData());
}
tc_data [tc] = cases;
}
#if FALSE
//w.WriteLine ($"\t\ttypeof({typename}).GetMethod (\"{m.Name}\", BindingFlags.Static|BindingFlags.NonPublic).Invoke(null, null);");
//w.WriteLine ($"\t\tusing (var o = new {typename} ()) {{");
//if (cmdline.StopOnFail)
//w.WriteLine ("\t\tif (nfailed > 0) return 1;");
//w.WriteLine ("\t\tConsole.WriteLine (\"RUN: \" + nrun + \", FAILED: \" + nfailed);");
#endif
var cu = new CodeCompileUnit();
var ns = new CodeNamespace("");
cu.Namespaces.Add(ns);
ns.Imports.Add(new CodeNamespaceImport("System"));
ns.Imports.Add(new CodeNamespaceImport("System.Reflection"));
var code_class = new CodeTypeDeclaration("RunTests");
ns.Types.Add(code_class);
var code_main = new CodeEntryPointMethod();
code_main.ReturnType = new CodeTypeReference("System.Int32");
code_class.Members.Add(code_main);
var statements = code_main.Statements;
statements.Add(new CodeVariableDeclarationStatement(typeof(int), "nrun", new CodePrimitiveExpression(0)));
statements.Add(new CodeVariableDeclarationStatement(typeof(int), "nfailed", new CodePrimitiveExpression(0)));
int nskipped = 0;
var filters = cmdline.Project.Filters;
foreach (XunitTestCase tc in sink.TestCases)
{
var m = ((ReflectionMethodInfo)tc.Method).MethodInfo;
//Console.WriteLine ("" + m.ReflectedType + " " + m + " " + (tc.TestMethodArguments == null));
var t = m.ReflectedType;
if (t.IsGenericType)
{
nskipped++;
continue;
}
if (!filters.Filter(tc))
{
nskipped++;
continue;
}
var cases = tc_data [tc];
int caseindex = 0;
foreach (var test in cases)
{
string typename = GetTypeName(t);
string msg;
if (cases.Count > 1)
{
msg = $"{typename}.{m.Name}[{caseindex}] ...";
}
else
{
msg = $"{typename}.{m.Name} ...";
}
caseindex++;
statements.Add(new CodeMethodInvokeExpression(new CodeTypeReferenceExpression("Console"), "WriteLine", new CodeExpression [] { new CodePrimitiveExpression(msg) }));
statements.Add(new CodeAssignStatement(new CodeVariableReferenceExpression("nrun"), new CodeBinaryOperatorExpression(new CodeVariableReferenceExpression("nrun"), CodeBinaryOperatorType.Add, new CodePrimitiveExpression(1))));
var try1 = new CodeTryCatchFinallyStatement();
statements.Add(try1);
if (!m.IsStatic)
{
// FIXME: Disposable
try1.TryStatements.Add(new CodeVariableDeclarationStatement("var", "o", new CodeObjectCreateExpression(t, new CodeExpression[] {})));
}
if (!m.IsPublic)
{
// FIXME:
nskipped++;
}
else
{
CodeMethodInvokeExpression call;
if (m.IsStatic)
{
call = new CodeMethodInvokeExpression(new CodeTypeReferenceExpression(t), m.Name, new CodeExpression [] {});
}
else
{
call = new CodeMethodInvokeExpression(new CodeVariableReferenceExpression("o"), m.Name, new CodeExpression [] {});
}
if (test.Values != null)
{
var parameters = m.GetParameters();
for (var index = 0; index < test.Values.Length; index++)
{
var val = test.Values [index];
call.Parameters.Add(EncodeValue(val, parameters [index].ParameterType));
}
}
try1.TryStatements.Add(call);
}
var catch1 = new CodeCatchClause("ex", new CodeTypeReference("System.Exception"));
catch1.Statements.Add(new CodeAssignStatement(new CodeVariableReferenceExpression("nfailed"), new CodeBinaryOperatorExpression(new CodeVariableReferenceExpression("nfailed"), CodeBinaryOperatorType.Add, new CodePrimitiveExpression(1))));
catch1.Statements.Add(new CodeMethodInvokeExpression(new CodeTypeReferenceExpression("Console"), "WriteLine", new CodeExpression [] {
new CodeBinaryOperatorExpression(
new CodePrimitiveExpression("FAILED: "),
CodeBinaryOperatorType.Add,
new CodeVariableReferenceExpression("ex"))
}));
try1.CatchClauses.Add(catch1);
}
}
statements.Add(new CodeMethodInvokeExpression(new CodeTypeReferenceExpression("Console"), "WriteLine", new CodeExpression [] {
new CodeBinaryOperatorExpression(
new CodePrimitiveExpression("RUN: "),
CodeBinaryOperatorType.Add,
new CodeVariableReferenceExpression("nrun"))
}));
statements.Add(new CodeMethodInvokeExpression(new CodeTypeReferenceExpression("Console"), "WriteLine", new CodeExpression [] {
new CodeBinaryOperatorExpression(
new CodePrimitiveExpression("FAILURES: "),
CodeBinaryOperatorType.Add,
new CodeVariableReferenceExpression("nfailed"))
}));
statements.Add(new CodeMethodInvokeExpression(new CodeTypeReferenceExpression("Console"), "WriteLine", new CodeExpression [] {
new CodeBinaryOperatorExpression(
new CodePrimitiveExpression("SKIPPED: "),
CodeBinaryOperatorType.Add,
new CodePrimitiveExpression(nskipped))
}));
statements.Add(new CodeMethodReturnStatement(new CodePrimitiveExpression(0)));
Directory.CreateDirectory(outdir_name);
var outfile_name = Path.Combine(outdir_name, "runner.cs");
var provider = new CSharpCodeProvider();
using (var w2 = File.CreateText(outfile_name)) {
provider.GenerateCodeFromCompileUnit(cu, w2, new CodeGeneratorOptions());
}
var csproj_template = File.ReadAllText("gen-test.csproj.template");
csproj_template = csproj_template.Replace("#XUNIT_LOCATION#", sdkdir);
csproj_template = csproj_template.Replace("#TEST_ASSEMBLY#", testAssemblyName);
csproj_template = csproj_template.Replace("#TEST_ASSEMBLY_LOCATION#", testAssemblyFull);
File.WriteAllText(Path.Combine(outdir_name, testAssemblyName + "-runner.csproj"), csproj_template);
return(0);
}
private void ExecuteTests(
[NotNull] XunitProjectAssembly assembly,
[NotNull] TestRunOptions options,
[NotNull] XunitFilters filters,
[NotNull] XunitFrontController controller,
[NotNull] TestDiscoverySink discoverySink,
[NotNull] ITestFrameworkExecutionOptions executionOptions,
[CanBeNull] XElement assemblyElement,
[NotNull] DiagnosticMessageSink diagnosticMessageSink)
{
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var serialize = options.Serialize;
var failSkips = options.FailSkips;
var stopOnFail = options.StopOnFail;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
// Discover & filter the tests
_reporterMessageHandler.OnMessage(
new TestAssemblyDiscoveryStarting(
assembly,
controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied,
shadowCopy,
discoveryOptions));
controller.Find(false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
_reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
_completionMessages.TryAdd(assembly.GetFileName().ThrowIfNull(), new ExecutionSummary());
}
else
{
if (serialize)
{
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
}
_reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(
_reporterMessageHandler,
() => _cancel,
(path, summary) => _completionMessages.TryAdd(path, summary));
if (assemblyElement != null)
{
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
}
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
}
if (failSkips)
{
resultsSink = new DelegatingFailSkipSink(resultsSink);
}
controller.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
_reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
if (!stopOnFail || resultsSink.ExecutionSummary.Failed == 0)
{
return;
}
Console.WriteLine("Canceling due to test failure...");
_cancel = true;
}
}
static int Main(string[] args)
{
if (args.Length < 3)
{
Console.WriteLine("Usage: <outfile> <corefx dir> <test assembly filename> <xunit console options>");
return(1);
}
var outfile_name = args [0];
var sdkdir = args [1] + "/artifacts/bin/runtime/netcoreapp-OSX-Debug-x64";
args = args.Skip(2).ToArray();
// Despite a lot of effort, couldn't get dotnet to load these assemblies from the sdk dir, so copy them to our binary dir
File.Copy($"{sdkdir}/Microsoft.DotNet.PlatformAbstractions.dll", AppContext.BaseDirectory, true);
File.Copy($"{sdkdir}/CoreFx.Private.TestUtilities.dll", AppContext.BaseDirectory, true);
File.Copy($"{sdkdir}/Microsoft.DotNet.XUnitExtensions.dll", AppContext.BaseDirectory, true);
var cmdline = CommandLine.Parse(args);
// Ditto
File.Copy(cmdline.Project.Assemblies.First().AssemblyFilename, AppContext.BaseDirectory, true);
var assembly = Assembly.LoadFrom(Path.Combine(AppContext.BaseDirectory, Path.GetFileName(cmdline.Project.Assemblies.First().AssemblyFilename)));
var msg_sink = new MsgSink();
var xunit2 = new Xunit2Discoverer(AppDomainSupport.Denied, new NullSourceInformationProvider(), new ReflectionAssemblyInfo(assembly), null, null, msg_sink);
var sink = new TestDiscoverySink();
var config = new TestAssemblyConfiguration()
{
DiagnosticMessages = true, InternalDiagnosticMessages = true, PreEnumerateTheories = false
};
xunit2.Find(false, sink, TestFrameworkOptions.ForDiscovery(config));
sink.Finished.WaitOne();
foreach (XunitTestCase tc in sink.TestCases)
{
ComputeTraits(assembly, tc);
}
var w = File.CreateText(outfile_name);
w.WriteLine("using System;");
w.WriteLine("using System.Reflection;");
w.WriteLine("public class RunTests {");
w.WriteLine("\tpublic static int Main () {");
w.WriteLine("\t\tint nrun = 0;");
w.WriteLine("\t\tint nfailed = 0;");
var filters = cmdline.Project.Filters;
foreach (XunitTestCase tc in sink.TestCases)
{
var m = ((ReflectionMethodInfo)tc.Method).MethodInfo;
//Console.WriteLine ("" + m.ReflectedType + " " + m + " " + (tc.TestMethodArguments == null));
if (tc.TestMethodArguments != null || m.GetParameters().Length > 0)
{
continue;
}
var t = m.ReflectedType;
if (t.IsGenericType)
{
continue;
}
/*
* foreach (var trait in tc.Traits) {
* foreach (var s in trait.Value)
* Console.WriteLine (m.Name + " " + trait.Key + " " + s);
* }
*/
if (!filters.Filter(tc))
{
//Console.WriteLine ("FILTERED: " + m);
continue;
}
string typename = GetTypeName(t);
w.WriteLine($"Console.WriteLine (\"{typename}:{m.Name}...\");");
w.WriteLine("\t\tnrun ++;");
w.WriteLine("\t\ttry {");
if (m.IsStatic)
{
if (!m.IsPublic)
{
w.WriteLine($"\t\ttypeof({typename}).GetMethod (\"{m.Name}\", BindingFlags.Static|BindingFlags.NonPublic).Invoke(null, null);");
}
else
{
w.WriteLine($"\t\t{typename}.{m.Name} ();");
}
}
else
{
if (typeof(IDisposable).IsAssignableFrom(t))
{
w.WriteLine($"\t\tusing (var o = new {typename} ()) {{");
}
else
{
w.WriteLine("\t\t{");
w.WriteLine($"\t\t\tvar o = new {typename} ();");
}
w.WriteLine($"\t\t\to.{m.Name} ();");
w.WriteLine("\t\t}");
}
w.WriteLine("\t\t} catch (Exception ex) { nfailed ++; Console.WriteLine (\"FAILED: \" + ex); }");
if (cmdline.StopOnFail)
{
w.WriteLine("\t\tif (nfailed > 0) return 1;");
}
}
w.WriteLine("\t\tConsole.WriteLine (\"RUN: \" + nrun + \", FAILED: \" + nfailed);");
w.WriteLine("\t\treturn 0;");
w.WriteLine("\t}");
w.WriteLine("}");
w.Close();
return(0);
}
async ValueTask <XElement?> ExecuteAssembly(
XunitProjectAssembly assembly,
bool needsXml,
_IMessageSink reporterMessageHandler)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
// Setup discovery and execution options with command-line overrides
var discoveryOptions = _TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = _TestFrameworkOptions.ForExecution(assembly.Configuration);
// The normal default is true here, but we want it to be false for us by default
if (!assembly.Configuration.PreEnumerateTheories.HasValue)
{
discoveryOptions.SetPreEnumerateTheories(false);
}
var assemblyDisplayName = assembly.AssemblyDisplayName;
var noColor = assembly.Project.Configuration.NoColorOrDefault;
var diagnosticMessageSink = ConsoleDiagnosticMessageSink.ForDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
var internalDiagnosticsMessageSink = ConsoleDiagnosticMessageSink.ForInternalDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.InternalDiagnosticMessagesOrDefault, noColor);
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
var assemblyInfo = new ReflectionAssemblyInfo(testAssembly);
await using var disposalTracker = new DisposalTracker();
var testFramework = ExtensibilityPointFactory.GetTestFramework(diagnosticMessageSink, assemblyInfo);
disposalTracker.Add(testFramework);
var discoverySink = new TestDiscoverySink(() => cancel);
// Discover & filter the tests
var testDiscoverer = testFramework.GetDiscoverer(assemblyInfo);
var discoveryStarting = new TestAssemblyDiscoveryStarting
{
AppDomain = AppDomainOption.NotAvailable,
Assembly = assembly,
DiscoveryOptions = discoveryOptions,
ShadowCopy = false
};
reporterMessageHandler.OnMessage(discoveryStarting);
testDiscoverer.Find(discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(assembly.Configuration.Filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
var discoveryFinished = new TestAssemblyDiscoveryFinished
{
Assembly = assembly,
DiscoveryOptions = discoveryOptions,
TestCasesDiscovered = testCasesDiscovered,
TestCasesToRun = testCasesToRun
};
reporterMessageHandler.OnMessage(discoveryFinished);
// Run the filtered tests
if (testCasesToRun == 0)
{
testExecutionSummaries.Add(testDiscoverer.TestAssemblyUniqueID, new ExecutionSummary());
}
else
{
var executionStarting = new TestAssemblyExecutionStarting
{
Assembly = assembly,
ExecutionOptions = executionOptions
};
reporterMessageHandler.OnMessage(executionStarting);
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel);
if (assemblyElement != null)
{
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
}
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
}
if (assembly.Configuration.FailSkipsOrDefault)
{
resultsSink = new DelegatingFailSkipSink(resultsSink);
}
using (resultsSink)
{
var executor = testFramework.GetExecutor(assemblyInfo);
executor.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
testExecutionSummaries.Add(testDiscoverer.TestAssemblyUniqueID, resultsSink.ExecutionSummary);
var executionFinished = new TestAssemblyExecutionFinished
{
Assembly = assembly,
ExecutionOptions = executionOptions,
ExecutionSummary = resultsSink.ExecutionSummary
};
reporterMessageHandler.OnMessage(executionFinished);
if (assembly.Configuration.StopOnFailOrDefault && resultsSink.ExecutionSummary.Failed != 0)
{
Console.WriteLine("Canceling due to test failure...");
cancel = true;
}
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine($"{e.GetType().FullName}: {e.Message}");
if (assembly.Configuration.InternalDiagnosticMessagesOrDefault)
{
Console.WriteLine(e.StackTrace);
}
e = e.InnerException;
}
}
return(assemblyElement);
}
public void FindByAssemblyReturnsAllTestMethodsFromExecutorXml()
{
var xml = @"
<assembly>
<class name='Type1'>
<method name='Method1 Display Name' type='Type1' method='Method1'/>
</class>
<class name='SpecialType'>
<method name='SpecialType.SkippedMethod' type='SpecialType' method='SkippedMethod' skip='I am not run'/>
<method name='SpecialType.MethodWithTraits' type='SpecialType' method='MethodWithTraits'>
<traits>
<trait name='Trait1' value='Value1'/>
<trait name='Trait2' value='Value2'/>
</traits>
</method>
</class>
</assembly>";
using (var xunit1 = new TestableXunit1())
{
xunit1.Executor
.WhenForAnyArgs(x => x.EnumerateTests(null))
.Do(callInfo => callInfo.Arg <ICallbackEventHandler>().RaiseCallbackEvent(xml));
var sink = new TestDiscoverySink();
xunit1.Find(false, sink);
sink.Finished.WaitOne();
Assert.Collection(sink.TestCases,
testCase =>
{
Assert.Equal("Type1", testCase.TestMethod.TestClass.Class.Name);
Assert.Equal("Method1", testCase.TestMethod.Method.Name);
Assert.Equal("Method1 Display Name", testCase.DisplayName);
Assert.Null(testCase.SkipReason);
Assert.Empty(testCase.Traits);
},
testCase =>
{
Assert.Equal("SpecialType", testCase.TestMethod.TestClass.Class.Name);
Assert.Equal("SkippedMethod", testCase.TestMethod.Method.Name);
Assert.Equal("SpecialType.SkippedMethod", testCase.DisplayName);
Assert.Equal("I am not run", testCase.SkipReason);
},
testCase =>
{
Assert.Equal("SpecialType", testCase.TestMethod.TestClass.Class.Name);
Assert.Equal("MethodWithTraits", testCase.TestMethod.Method.Name);
Assert.Equal("SpecialType.MethodWithTraits", testCase.DisplayName);
Assert.Collection(testCase.Traits.Keys,
key =>
{
Assert.Equal("Trait1", key);
Assert.Collection(testCase.Traits[key],
value => Assert.Equal("Value1", value)
);
},
key =>
{
Assert.Equal("Trait2", key);
Assert.Collection(testCase.Traits[key],
value => Assert.Equal("Value2", value)
);
}
);
}
);
}
}
static XElement ExecuteAssembly(object consoleLock,
XunitProjectAssembly assembly,
bool serialize,
bool needsXml,
bool? parallelizeTestCollections,
int? maxThreadCount,
bool diagnosticMessages,
bool noColor,
bool noAppDomain,
bool failSkips,
XunitFilters filters)
{
if (cancel)
return null;
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
return null;
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= diagnosticMessages;
if (noAppDomain)
assembly.Configuration.AppDomain = AppDomainSupport.Denied;
// Setup discovery and execution options with command-line overrides
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue)
executionOptions.SetMaxParallelThreads(maxThreadCount);
if (parallelizeTestCollections.HasValue)
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageSink = new DiagnosticMessageSink(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
using (var discoverySink = new TestDiscoverySink(() => cancel))
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions));
controller.Find(false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
else
{
if (serialize)
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary));
if (assemblyElement != null)
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
if (longRunningSeconds > 0)
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
if (failSkips)
resultsSink = new DelegatingFailSkipSink(resultsSink);
controller.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine($"{e.GetType().FullName}: {e.Message}");
e = e.InnerException;
}
}
return assemblyElement;
}
XElement ExecuteAssembly(object consoleLock,
XunitProjectAssembly assembly,
bool serialize,
bool needsXml,
bool?parallelizeTestCollections,
int?maxThreadCount,
bool diagnosticMessages,
bool noColor,
bool noAppDomain,
bool failSkips,
XunitFilters filters,
bool internalDiagnosticMessages)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
{
return(null);
}
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= diagnosticMessages;
assembly.Configuration.InternalDiagnosticMessages |= internalDiagnosticMessages;
if (noAppDomain)
{
assembly.Configuration.AppDomain = AppDomainSupport.Denied;
}
// Setup discovery and execution options with command-line overrides
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue)
{
executionOptions.SetMaxParallelThreads(maxThreadCount);
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageSink = new DiagnosticMessageSink(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
#if NETCOREAPP1_0 || NETCOREAPP2_0
using (new NetCoreAssemblyDependencyResolver(assembly.AssemblyFilename))
#endif
using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
using (var discoverySink = new TestDiscoverySink(() => cancel))
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions));
controller.Find(false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
}
else
{
if (serialize)
{
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary));
if (assemblyElement != null)
{
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
}
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
}
if (failSkips)
{
resultsSink = new DelegatingFailSkipSink(resultsSink);
}
controller.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine($"{e.GetType().FullName}: {e.Message}");
if (internalDiagnosticMessages)
{
Console.WriteLine(e.StackTrace);
}
e = e.InnerException;
}
}
return(assemblyElement);
}
public int Run(string assemblyFileName, bool printXml, XunitFilters filters)
{
var configuration = new TestAssemblyConfiguration()
{
ShadowCopy = false, ParallelizeAssembly = false, ParallelizeTestCollections = false, MaxParallelThreads = 1, PreEnumerateTheories = false
};
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
var discoverySink = new TestDiscoverySink();
var diagnosticSink = new ConsoleDiagnosticMessageSink();
var testOptions = TestFrameworkOptions.ForExecution(configuration);
var testSink = new TestMessageSink();
var controller = new Xunit2(AppDomainSupport.Denied, new NullSourceInformationProvider(), assemblyFileName, configFileName: null, shadowCopy: false, shadowCopyFolder: null, diagnosticMessageSink: diagnosticSink, verifyTestAssemblyExists: false);
discoveryOptions.SetSynchronousMessageReporting(true);
testOptions.SetSynchronousMessageReporting(true);
Console.WriteLine($"Discovering tests for {assemblyFileName}");
var assembly = Assembly.LoadFrom(assemblyFileName);
var assemblyInfo = new global::Xunit.Sdk.ReflectionAssemblyInfo(assembly);
var discoverer = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink);
discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList();
Console.WriteLine($"Discovery finished.");
var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => { Console.WriteLine($"Tests run: {summary.Total}, Errors: 0, Failures: {summary.Failed}, Skipped: {summary.Skipped}. Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s"); });
var resultsXmlAssembly = new XElement("assembly");
var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly);
testSink.Execution.TestPassedEvent += args => { Console.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); };
testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); };
testSink.Execution.TestFailedEvent += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}"); };
testSink.Execution.TestAssemblyStartingEvent += args => { Console.WriteLine($"Running tests for {args.Message.TestAssembly.Assembly}"); };
testSink.Execution.TestAssemblyFinishedEvent += args => { Console.WriteLine($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); };
controller.RunTests(testCasesToRun, resultsSink, testOptions);
var threadpoolPump = typeof(ThreadPool).GetMethod("PumpThreadPool", BindingFlags.NonPublic | BindingFlags.Static);
var timerPump = Type.GetType("System.Threading.TimerQueue")?.GetMethod("PumpTimerQueue", BindingFlags.NonPublic | BindingFlags.Static);
if (threadpoolPump != null && timerPump != null)
{
while (!resultsSink.Finished.WaitOne(0))
{
threadpoolPump.Invoke(this, null);
timerPump.Invoke(this, null);
}
}
else
{
resultsSink.Finished.WaitOne();
}
if (printXml)
{
Console.WriteLine($"STARTRESULTXML");
var resultsXml = new XElement("assemblies");
resultsXml.Add(resultsXmlAssembly);
resultsXml.Save(Console.OpenStandardOutput());
Console.WriteLine();
Console.WriteLine($"ENDRESULTXML");
}
var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0;
return(failed ? 1 : 0);
}
