protected static void Go(string assemblyFileName, Stream stream, AppDomainSupport appDomainSupport,
Action<XunitFrontController, TestAssemblyConfiguration, ClientWriter> action)
{
using (AssemblyHelper.SubscribeResolve())
using (var xunit = new XunitFrontController(appDomainSupport, assemblyFileName, shadowCopy: false))
using (var writer = new ClientWriter(stream))
{
var configuration = ConfigReader.Load(assemblyFileName);
action(xunit, configuration, writer);
}
}
private async void RunTests(string arguments)
{
var reporters = new List <IRunnerReporter>();
string[] args = arguments.Split(new[] { '\x1F' }, StringSplitOptions.RemoveEmptyEntries);
log = string.Empty;
log += "Args: " + string.Join(" ", args) + "\n";
var commandLine = CommandLine.Parse(args);
if (commandLine.Debug)
{
Debugger.Launch();
}
var reporterMessageHandler = commandLine.Reporter.CreateMessageHandler(new RunLogger());
var completionMessages = new ConcurrentDictionary <string, ExecutionSummary>();
var assembliesElement = new XElement("assemblies");
foreach (var assembly in commandLine.Project.Assemblies)
{
if (cancel)
{
return;
}
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= commandLine.DiagnosticMessages;
assembly.Configuration.AppDomain = AppDomainSupport.Denied;
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
executionOptions.SetDisableParallelization(true);
try
{
using (var xunit = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.Configuration.ShadowCopyOrDefault))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, false, false, discoveryOptions));
xunit.Find(false, discoveryVisitor, discoveryOptions);
discoveryVisitor.Finished.WaitOne();
var testCasesDiscovered = discoveryVisitor.TestCases.Count;
var filteredTestCases = discoveryVisitor.TestCases.Where(commandLine.Project.Filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
log += "testCasesToRun: " + testCasesToRun + "\n";
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
}
else
{
if (commandLine.Serialize)
{
filteredTestCases = filteredTestCases.Select(xunit.Serialize).Select(xunit.Deserialize).ToList();
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
var assemblyElement = new XElement("assembly");
IExecutionVisitor resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel);
if (commandLine.FailSkips)
{
resultsVisitor = new FailSkipVisitor(resultsVisitor);
}
xunit.RunTests(filteredTestCases, resultsVisitor, executionOptions);
log += "finished running tests \n";
resultsVisitor.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary));
assembliesElement.Add(assemblyElement);
}
}
}
catch (Exception e)
{
assembliesElement = new XElement("error");
assembliesElement.Add(e);
log += "logged exec errors: " + e + "\n";
}
}
await WriteResults(assembliesElement);
await WriteLogs(log);
Application.Current.Exit();
}
void DiscoverTests <TVisitor>(IEnumerable <string> sources,
IMessageLogger logger,
XunitVisualStudioSettings settings,
Func <string, ITestFrameworkDiscoverer, TVisitor> visitorFactory,
Action <string, ITestFrameworkDiscoverer, TVisitor> visitComplete = null,
Stopwatch stopwatch = null)
where TVisitor : IVsDiscoveryVisitor
{
if (stopwatch == null)
{
stopwatch = Stopwatch.StartNew();
}
try
{
RemotingUtility.CleanUpRegisteredChannels();
if (settings.MessageDisplay == MessageDisplay.Diagnostic)
{
logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery started", stopwatch.Elapsed));
}
using (AssemblyHelper.SubscribeResolve())
{
foreach (string assemblyFileName in sources)
{
var fileName = Path.GetFileName(assemblyFileName);
try
{
if (cancelled)
{
break;
}
if (!IsXunitTestAssembly(assemblyFileName))
{
if (settings.MessageDisplay == MessageDisplay.Diagnostic)
{
logger.SendMessage(TestMessageLevel.Informational,
String.Format("[xUnit.net {0}] Skipping: {1} (no reference to xUnit.net)", stopwatch.Elapsed, fileName));
}
}
else
{
using (var framework = new XunitFrontController(assemblyFileName, configFileName: null, shadowCopy: true))
using (var visitor = visitorFactory(assemblyFileName, framework))
{
var targetFramework = framework.TargetFramework;
if (targetFramework.StartsWith("MonoTouch", StringComparison.OrdinalIgnoreCase) ||
targetFramework.StartsWith("MonoAndroid", StringComparison.OrdinalIgnoreCase))
{
if (settings.MessageDisplay == MessageDisplay.Diagnostic)
{
logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Skipping: {1} (unsupported target framework '{2}')", stopwatch.Elapsed, fileName, targetFramework));
}
}
else
{
if (settings.MessageDisplay == MessageDisplay.Diagnostic)
{
logger.SendMessage(TestMessageLevel.Informational,
String.Format("[xUnit.net {0}] Discovery starting: {1}", stopwatch.Elapsed, fileName));
}
framework.Find(includeSourceInformation: true, messageSink: visitor, options: new TestFrameworkOptions());
var totalTests = visitor.Finish();
if (visitComplete != null)
{
visitComplete(assemblyFileName, framework, visitor);
}
if (settings.MessageDisplay == MessageDisplay.Diagnostic)
{
logger.SendMessage(TestMessageLevel.Informational,
String.Format("[xUnit.net {0}] Discovery finished: {1} ({2} tests)", stopwatch.Elapsed, fileName, totalTests));
}
}
}
}
}
catch (Exception e)
{
var ex = e.Unwrap();
var fileNotFound = ex as FileNotFoundException;
var fileLoad = ex as FileLoadException;
if (fileNotFound != null)
{
logger.SendMessage(TestMessageLevel.Informational,
String.Format("[xUnit.net {0}] Skipping: {1} (could not find dependent assembly '{2}')", stopwatch.Elapsed, fileName, Path.GetFileNameWithoutExtension(fileNotFound.FileName)));
}
else if (fileLoad != null)
{
logger.SendMessage(TestMessageLevel.Informational,
String.Format("[xUnit.net {0}] Skipping: {1} (could not find dependent assembly '{2}')", stopwatch.Elapsed, fileName, Path.GetFileNameWithoutExtension(fileLoad.FileName)));
}
else
{
logger.SendMessage(TestMessageLevel.Error,
String.Format("[xUnit.net {0}] Exception discovering tests from {1}: {2}", stopwatch.Elapsed, fileName, ex));
}
}
}
}
}
catch (Exception e)
{
logger.SendMessage(TestMessageLevel.Error,
String.Format("[xUnit.net {0}] Exception discovering tests: {1}", stopwatch.Elapsed, e.Unwrap()));
}
stopwatch.Stop();
if (settings.MessageDisplay == MessageDisplay.Diagnostic)
{
logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery complete", stopwatch.Elapsed));
}
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
{
Guard.ArgumentNotNull("sources", sources);
Guard.ArgumentNotNull("logger", logger);
Guard.ArgumentNotNull("discoverySink", discoverySink);
var settings = SettingsProvider.Load();
var stopwatch = Stopwatch.StartNew();
try
{
RemotingUtility.CleanUpRegisteredChannels();
if (settings.MessageDisplay == MessageDisplay.Diagnostic)
{
logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery started", stopwatch.Elapsed));
}
using (AssemblyHelper.SubscribeResolve())
{
foreach (string assemblyFileName in sources)
{
var fileName = Path.GetFileName(assemblyFileName);
try
{
if (cancelled)
{
break;
}
if (!IsXunitTestAssembly(assemblyFileName))
{
if (settings.MessageDisplay == MessageDisplay.Diagnostic)
{
logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Skipping: {1}", stopwatch.Elapsed, fileName));
}
}
else
{
if (settings.MessageDisplay == MessageDisplay.Diagnostic)
{
logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery starting: {1}", stopwatch.Elapsed, fileName));
}
using (var framework = new XunitFrontController(assemblyFileName, configFileName: null, shadowCopy: true))
using (var sink = new VsDiscoveryVisitor(assemblyFileName, framework, logger, discoveryContext, discoverySink, () => cancelled))
{
framework.Find(includeSourceInformation: true, messageSink: sink, options: new TestFrameworkOptions());
sink.Finished.WaitOne();
if (settings.MessageDisplay == MessageDisplay.Diagnostic)
{
logger.SendMessage(TestMessageLevel.Informational,
String.Format("[xUnit.net {0}] Discovery finished: {1} ({2} tests)", stopwatch.Elapsed, fileName, sink.TotalTests));
}
}
}
}
catch (Exception e)
{
logger.SendMessage(TestMessageLevel.Error,
String.Format("[xUnit.net {0}] Exception discovering tests from {1}: {2}", stopwatch.Elapsed, fileName, e));
}
}
}
}
catch (Exception e)
{
logger.SendMessage(TestMessageLevel.Error,
String.Format("[xUnit.net {0}] Exception discovering tests: {1}", stopwatch.Elapsed, e));
}
stopwatch.Stop();
if (settings.MessageDisplay == MessageDisplay.Diagnostic)
{
logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery complete", stopwatch.Elapsed));
}
}
void RunTestsInAssembly(IRunContext runContext,
IFrameworkHandle frameworkHandle,
LoggerHelper logger,
TestPlatformContext testPlatformContext,
RunSettings runSettings,
IMessageSinkWithTypes reporterMessageHandler,
AssemblyRunInfo runInfo)
{
if (cancelled)
{
return;
}
var assemblyDisplayName = "(unknown assembly)";
try
{
var assembly = new XunitProjectAssembly {
AssemblyFilename = runInfo.AssemblyFileName
};
var assemblyFileName = runInfo.AssemblyFileName;
assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName);
var configuration = runInfo.Configuration;
var shadowCopy = configuration.ShadowCopyOrDefault;
var appDomain = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
if (runSettings.DisableAppDomain)
{
appDomain = AppDomainSupport.Denied;
}
#if WINDOWS_UAP
// For AppX Apps, use the package location
assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName));
#endif
var diagnosticSink = DiagnosticMessageSink.ForDiagnostics(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault);
var diagnosticMessageSink = MessageSinkAdapter.Wrap(diagnosticSink);
using (var controller = new XunitFrontController(appDomain, assemblyFileName, shadowCopy: shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
{
var testCasesMap = new Dictionary <string, TestCase>();
var testCases = new List <ITestCase>();
if (runInfo.TestCases == null || !runInfo.TestCases.Any())
{
// Discover tests
AssemblyDiscoveredInfo assemblyDiscoveredInfo = new AssemblyDiscoveredInfo();
DiscoverTestsInSource(controller, logger, testPlatformContext, runSettings,
(source, discoverer, discoveryOptions) => new VsExecutionDiscoverySink(() => cancelled),
(source, discoverer, discoveryOptions, visitor) =>
{
if (discoveryOptions.GetInternalDiagnosticMessagesOrDefault())
{
foreach (var testCase in visitor.TestCases)
{
logger.Log(testCase, "Discovered [execution] test case '{0}' (ID = '{1}')",
testCase.DisplayName, testCase.UniqueID);
}
}
assemblyDiscoveredInfo = new AssemblyDiscoveredInfo
{
AssemblyFileName = source,
DiscoveredTestCases = GetVsTestCases(source, discoverer, visitor, logger, testPlatformContext)
};
},
assemblyFileName,
shadowCopy,
configuration
);
if (assemblyDiscoveredInfo.DiscoveredTestCases == null || !assemblyDiscoveredInfo.DiscoveredTestCases.Any())
{
if (configuration.InternalDiagnosticMessagesOrDefault)
{
logger.LogWarning("Skipping '{0}' since no tests were found during discovery [execution].", assemblyDiscoveredInfo.AssemblyFileName);
}
return;
}
// Filter tests
var traitNames = new HashSet <string>(assemblyDiscoveredInfo.DiscoveredTestCases.SelectMany(testCase => testCase.TraitNames));
var filter = new TestCaseFilter(runContext, logger, assemblyDiscoveredInfo.AssemblyFileName, traitNames);
var filteredTestCases = assemblyDiscoveredInfo.DiscoveredTestCases.Where(dtc => filter.MatchTestCase(dtc.VSTestCase)).ToList();
foreach (var filteredTestCase in filteredTestCases)
{
var uniqueID = filteredTestCase.UniqueID;
if (testCasesMap.ContainsKey(uniqueID))
{
logger.LogWarning(filteredTestCase.TestCase, "Skipping test case with duplicate ID '{0}' ('{1}' and '{2}')", uniqueID, testCasesMap[uniqueID].DisplayName, filteredTestCase.VSTestCase.DisplayName);
}
else
{
testCasesMap.Add(uniqueID, filteredTestCase.VSTestCase);
testCases.Add(filteredTestCase.TestCase);
}
}
}
else
{
try
{
var serializations = runInfo.TestCases
.Select(tc => tc.GetPropertyValue <string>(SerializedTestCaseProperty, null))
.ToList();
if (configuration.InternalDiagnosticMessagesOrDefault)
{
logger.LogWithSource(runInfo.AssemblyFileName, "Deserializing {0} test case(s):{1}{2}",
serializations.Count,
Environment.NewLine,
string.Join(Environment.NewLine, serializations.Select(x => $" {x}")));
}
var deserializedTestCasesByUniqueId = controller.BulkDeserialize(serializations);
if (deserializedTestCasesByUniqueId == null)
{
logger.LogErrorWithSource(assemblyFileName, "Received null response from BulkDeserialize");
}
else
{
for (int idx = 0; idx < runInfo.TestCases.Count; ++idx)
{
try
{
var kvp = deserializedTestCasesByUniqueId[idx];
var vsTestCase = runInfo.TestCases[idx];
if (kvp.Value == null)
{
logger.LogErrorWithSource(assemblyFileName, "Test case {0} failed to deserialize: {1}", vsTestCase.DisplayName, kvp.Key);
}
else
{
testCasesMap.Add(kvp.Key, vsTestCase);
testCases.Add(kvp.Value);
}
}
catch (Exception ex)
{
logger.LogErrorWithSource(assemblyFileName, "Catastrophic error deserializing item #{0}: {1}", idx, ex);
}
}
}
}
catch (Exception ex)
{
logger.LogWarningWithSource(assemblyFileName, "Catastrophic error during deserialization: {0}", ex);
}
}
// Execute tests
var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration);
if (runSettings.DisableParallelization)
{
executionOptions.SetSynchronousMessageReporting(true);
executionOptions.SetDisableParallelization(true);
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, testCasesMap, executionOptions, () => cancelled))
{
IExecutionSink resultsSink = vsExecutionSink;
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink);
}
controller.RunTests(testCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
}
}
}
catch (Exception ex)
{
logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex);
}
}
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool appVeyor, bool showProgress, bool? parallelizeTestCollections, int? maxThreadCount, XunitFilters filters)
{
if (cancel)
return null;
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
return null;
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue)
executionOptions.SetMaxParallelThreads(maxThreadCount.GetValueOrDefault());
if (parallelizeTestCollections.HasValue)
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
lock (consoleLock)
{
if (assembly.Configuration.DiagnosticMessages ?? false)
Console.WriteLine("Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})",
Path.GetFileNameWithoutExtension(assembly.AssemblyFilename),
discoveryOptions.GetMethodDisplay(),
!executionOptions.GetDisableParallelization(),
executionOptions.GetMaxParallelThreads());
else
Console.WriteLine("Discovering: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
}
using (var controller = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.Configuration.ShadowCopyOrDefault))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions);
discoveryVisitor.Finished.WaitOne();
lock (consoleLock)
{
Console.WriteLine("Discovered: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
}
var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, appVeyor, showProgress);
var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList();
if (filteredTestCases.Count == 0)
{
lock (consoleLock)
{
Console.ForegroundColor = ConsoleColor.DarkYellow;
Console.WriteLine("Info: {0} has no tests to run", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
Console.ForegroundColor = ConsoleColor.Gray;
}
}
else
{
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
}
}
}
catch (Exception ex)
{
Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message);
failed = true;
}
return assemblyElement;
}
private static List<ITestCase> GetTestCaseList(XunitFrontController xunit, TestAssemblyConfiguration configuration, HashSet<string> testCaseNameSet)
{
var testCaseList = new List<ITestCase>();
using (var sink = new TestDiscoverySink(testCaseNameSet, testCaseList))
{
xunit.Find(includeSourceInformation: false, messageSink: sink,
discoveryOptions: TestFrameworkOptions.ForDiscovery(configuration));
sink.Finished.WaitOne();
}
return testCaseList;
}
static int Main(string[] args)
{
var nodeIndex = CommandLine.GetInt32("multinode.index");
var nodeRole = CommandLine.GetProperty("multinode.role");
var assemblyFileName = CommandLine.GetProperty("multinode.test-assembly");
var typeName = CommandLine.GetProperty("multinode.test-class");
var testName = CommandLine.GetProperty("multinode.test-method");
var displayName = testName;
var listenAddress = IPAddress.Parse(CommandLine.GetProperty("multinode.listen-address"));
var listenPort = CommandLine.GetInt32("multinode.listen-port");
var listenEndpoint = new IPEndPoint(listenAddress, listenPort);
var system = ActorSystem.Create("NoteTestRunner-" + nodeIndex);
var tcpClient = _logger = system.ActorOf <RunnerTcpClient>();
system.Tcp().Tell(new Tcp.Connect(listenEndpoint), tcpClient);
#if CORECLR
// In NetCore, if the assembly file hasn't been touched,
// XunitFrontController would fail loading external assemblies and its dependencies.
AssemblyLoadContext.Default.Resolving += (assemblyLoadContext, assemblyName) => DefaultOnResolving(assemblyLoadContext, assemblyName, assemblyFileName);
var assembly = AssemblyLoadContext.Default.LoadFromAssemblyPath(assemblyFileName);
DependencyContext.Load(assembly)
.CompileLibraries
.Where(dep => dep.Name.ToLower()
.Contains(assembly.FullName.Split(new[] { ',' })[0].ToLower()))
.Select(dependency => AssemblyLoadContext.Default.LoadFromAssemblyName(new AssemblyName(dependency.Name)));
#endif
Thread.Sleep(TimeSpan.FromSeconds(10));
using (var controller = new XunitFrontController(AppDomainSupport.IfAvailable, assemblyFileName))
{
/* need to pass in just the assembly name to Discovery, not the full path
* i.e. "Akka.Cluster.Tests.MultiNode.dll"
* not "bin/Release/Akka.Cluster.Tests.MultiNode.dll" - this will cause
* the Discovery class to actually not find any individual specs to run
*/
var assemblyName = Path.GetFileName(assemblyFileName);
Console.WriteLine("Running specs for {0} [{1}]", assemblyName, assemblyFileName);
using (var discovery = new Discovery(assemblyName, typeName))
{
using (var sink = new Sink(nodeIndex, nodeRole, tcpClient))
{
try
{
controller.Find(true, discovery, TestFrameworkOptions.ForDiscovery());
discovery.Finished.WaitOne();
controller.RunTests(discovery.TestCases, sink, TestFrameworkOptions.ForExecution());
}
catch (AggregateException ex)
{
var specFail = new SpecFail(nodeIndex, nodeRole, displayName);
specFail.FailureExceptionTypes.Add(ex.GetType().ToString());
specFail.FailureMessages.Add(ex.Message);
specFail.FailureStackTraces.Add(ex.StackTrace);
foreach (var innerEx in ex.Flatten().InnerExceptions)
{
specFail.FailureExceptionTypes.Add(innerEx.GetType().ToString());
specFail.FailureMessages.Add(innerEx.Message);
specFail.FailureStackTraces.Add(innerEx.StackTrace);
}
_logger.Tell(specFail.ToString());
Console.WriteLine(specFail);
//make sure message is send over the wire
FlushLogMessages();
Environment.Exit(1); //signal failure
return(1);
}
catch (Exception ex)
{
var specFail = new SpecFail(nodeIndex, nodeRole, displayName);
specFail.FailureExceptionTypes.Add(ex.GetType().ToString());
specFail.FailureMessages.Add(ex.Message);
specFail.FailureStackTraces.Add(ex.StackTrace);
var innerEx = ex.InnerException;
while (innerEx != null)
{
specFail.FailureExceptionTypes.Add(innerEx.GetType().ToString());
specFail.FailureMessages.Add(innerEx.Message);
specFail.FailureStackTraces.Add(innerEx.StackTrace);
innerEx = innerEx.InnerException;
}
_logger.Tell(specFail.ToString());
Console.WriteLine(specFail);
//make sure message is send over the wire
FlushLogMessages();
Environment.Exit(1); //signal failure
return(1);
}
var timedOut = false;
if (!sink.Finished.WaitOne(MaxProcessWaitTimeout)) //timed out
{
var line = string.Format("Timed out while waiting for test to complete after {0} ms",
MaxProcessWaitTimeout);
_logger.Tell(line);
Console.WriteLine(line);
timedOut = true;
}
FlushLogMessages();
system.Terminate().Wait();
Environment.Exit(sink.Passed && !timedOut ? 0 : 1);
return(sink.Passed ? 0 : 1);
}
}
}
}
void RunTestsInAssembly(List<IDisposable> toDispose,
string assemblyFileName,
IEnumerable<TestCaseViewModel> testCases,
Stopwatch stopwatch)
{
if (cancelled)
return;
var controller = new XunitFrontController(assemblyFileName, configFileName: null, shadowCopy: true);
lock (toDispose)
toDispose.Add(controller);
var xunitTestCases = testCases.ToDictionary(tc => tc.TestCase);
using (var executionVisitor = new TestExecutionVisitor(xunitTestCases, this, () => cancelled, context))
{
var executionOptions = TestFrameworkOptions.ForExecution();
controller.RunTests(xunitTestCases.Keys.ToList(), executionVisitor, executionOptions);
executionVisitor.Finished.WaitOne();
}
}
public static int Main(string[] args)
{
string host = null;
int port = 0;
// First argument is the connection string
if (args [0].StartsWith("tcp:"))
{
var parts = args [0].Split(':');
if (parts.Length != 3)
{
throw new Exception();
}
host = parts [1];
port = Int32.Parse(parts [2]);
args = args.Skip(1).ToArray();
}
// Make sure the TLS subsystem including the DependencyInjector is initialized.
// This would normally happen on system startup in
// `xamarin-macios/src/ObjcRuntime/Runtime.cs`.
MonoTlsProviderFactory.Initialize();
#if XUNIT_RUNNER
var writer = new TcpWriter(host, port);
var assemblyFileName = args[0];
var configuration = new TestAssemblyConfiguration()
{
ShadowCopy = false
};
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
var discoverySink = new TestDiscoverySink();
var diagnosticSink = new DiagnosticTextWriterMessageSink(writer);
var testOptions = TestFrameworkOptions.ForExecution(configuration);
var testSink = new TestMessageSink();
var controller = new XunitFrontController(AppDomainSupport.Denied, assemblyFileName, configFileName: null, shadowCopy: false, diagnosticMessageSink: diagnosticSink);
writer.WriteLine($"Discovering tests for {assemblyFileName}");
controller.Find(includeSourceInformation: false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
writer.WriteLine($"Discovery finished.");
var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => { writer.WriteLine($"Tests run: {summary.Total}, Errors: 0, Failures: {summary.Failed}, Skipped: {summary.Skipped}{Environment.NewLine}Time: {TimeSpan.FromSeconds ((double)summary.Time).TotalSeconds}s"); });
var resultsXmlAssembly = new XElement("assembly");
var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly);
testSink.Execution.TestPassedEvent += args => { writer.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); Interop.mono_sdks_ui_increment_testcase_result(0); };
testSink.Execution.TestSkippedEvent += args => { writer.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); Interop.mono_sdks_ui_increment_testcase_result(1); };
testSink.Execution.TestFailedEvent += args => { writer.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages (args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces (args.Message)}"); Interop.mono_sdks_ui_increment_testcase_result(2); };
testSink.Execution.TestAssemblyStartingEvent += args => { writer.WriteLine($"Running tests for {args.Message.TestAssembly.Assembly}"); };
testSink.Execution.TestAssemblyFinishedEvent += args => { writer.WriteLine($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); };
controller.RunTests(discoverySink.TestCases, resultsSink, testOptions);
resultsSink.Finished.WaitOne();
writer.WriteLine($"STARTRESULTXML");
var resultsXml = new XElement("assemblies");
resultsXml.Add(resultsXmlAssembly);
resultsXml.Save(writer.RawStream);
writer.WriteLine();
writer.WriteLine($"ENDRESULTXML");
var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0;
return(failed ? 1 : 0);
#else
MonoSdksTextUI runner;
TcpWriter writer = null;
string resultsXml = null;
if (host != null)
{
Console.WriteLine($"Connecting to harness at {host}:{port}.");
resultsXml = Path.GetTempFileName();
args = args.Concat(new string[] { "-format:xunit", $"-result:{resultsXml}" }).ToArray();
writer = new TcpWriter(host, port);
runner = new MonoSdksTextUI(writer);
}
else
{
runner = new MonoSdksTextUI();
}
runner.Execute(args);
if (resultsXml != null)
{
writer.WriteLine($"STARTRESULTXML");
using (var resultsXmlStream = File.OpenRead(resultsXml)) resultsXmlStream.CopyTo(writer.RawStream);
writer.WriteLine();
writer.WriteLine($"ENDRESULTXML");
}
return(runner.Failure ? 1 : 0);
#endif
}
IEnumerable <IGrouping <string, TestCase> > GetTests(IEnumerable <string> sources, IMessageLogger logger, XunitVisualStudioSettings settings, Stopwatch stopwatch)
{
var sourceSinks = new List <SourceSink <TestDiscoveryVisitor> >();
var result = new List <IGrouping <string, TestCase> >();
try
{
if (settings.MessageDisplay == MessageDisplay.Diagnostic)
{
lock (stopwatch)
logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery started", stopwatch.Elapsed));
}
foreach (string assemblyFileName in sources)
{
try
{
if (cancelled)
{
break;
}
if (!IsXunitTestAssembly(assemblyFileName))
{
if (settings.MessageDisplay == MessageDisplay.Diagnostic)
{
lock (stopwatch)
logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Skipping: {1}", stopwatch.Elapsed, Path.GetFileName(assemblyFileName)));
}
}
else
{
if (settings.MessageDisplay == MessageDisplay.Diagnostic)
{
lock (stopwatch)
logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery starting: {1}", stopwatch.Elapsed, Path.GetFileName(assemblyFileName)));
}
var framework = new XunitFrontController(assemblyFileName, configFileName: null, shadowCopy: true);
var sink = new TestDiscoveryVisitor();
sourceSinks.Add(new SourceSink <TestDiscoveryVisitor> {
Framework = framework, Sink = sink, AssemblyFileName = assemblyFileName
});
framework.Find(includeSourceInformation: true, messageSink: sink);
}
}
catch (Exception e)
{
lock (stopwatch)
logger.SendMessage(TestMessageLevel.Error,
String.Format("[xUnit.net {0}] Exception discovering tests from {1}: {2}", stopwatch.Elapsed, assemblyFileName, e));
}
}
var toFinish = new List <SourceSink <TestDiscoveryVisitor> >(sourceSinks);
while (toFinish.Count > 0)
{
var finishedIdx = WaitHandle.WaitAny(sourceSinks.Select(sink => sink.Sink.Finished).ToArray());
var sourceSink = toFinish[finishedIdx];
result.Add(
new Grouping <string, TestCase>(
sourceSink.AssemblyFileName,
sourceSink.Sink.TestCases
.GroupBy(tc => String.Format("{0}.{1}", tc.Class.Name, tc.Method.Name))
.SelectMany(group => group.Select(testCase => VsDiscoveryVisitor.CreateVsTestCase(sourceSink.AssemblyFileName, sourceSink.Framework, testCase, settings, forceUniqueNames: group.Count() > 1)))
.ToList()
)
);
if (settings.MessageDisplay != MessageDisplay.None)
{
lock (stopwatch)
logger.SendMessage(TestMessageLevel.Informational,
String.Format("[xUnit.net {0}] Discovery finished: {1} ({2} tests)", stopwatch.Elapsed, Path.GetFileName(sourceSink.AssemblyFileName), sourceSink.Sink.TestCases.Count));
}
toFinish.RemoveAt(finishedIdx);
}
}
finally
{
foreach (var sourceSink in sourceSinks)
{
sourceSink.Sink.Dispose();
sourceSink.Framework.Dispose();
}
}
if (settings.MessageDisplay == MessageDisplay.Diagnostic)
{
lock (stopwatch)
logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery complete", stopwatch.Elapsed));
}
return(result);
}
public static void RunTests(Options o)
{
var nullMessage = new Xunit.NullMessageSink();
var discoveryOptions = TestFrameworkOptions.ForDiscovery();
using (var c = new XunitFrontController(AppDomainSupport.Denied, o.Assembly, null, false))
{
var tv = new TestDiscoverySink();
var excludeTestCaseSet = new TestDiscoverySink();
c.Find(true, tv, discoveryOptions);
tv.Finished.WaitOne();
foreach (var tc in tv.TestCases)
{
var method = tc.TestMethod.Method;
var attributes = method.GetCustomAttributes(typeof(FactAttribute));
foreach (ReflectionAttributeInfo at in attributes)
{
var result = at.GetNamedArgument<string>("Skip");
if (result != null)
{
Console.WriteLine("SKIPPY! {0} because {1}", method, result);
}
if (o.TestType != TestType.All)
{
if (!at.ToString().EndsWith(o.TestType.ToString()))
{
excludeTestCaseSet.TestCases.Add(tc);
}
}
}
}
foreach (var tc in excludeTestCaseSet.TestCases)
{
tv.TestCases.Remove(tc);
}
Console.WriteLine("TEST COUNT: {0}", tv.TestCases.Count);
//core execution Sink
int testCaseCount = tv.TestCases.Count;
Stream file = new FileStream(".\\result.xml", FileMode.Create);
int totalResult = 0;
int totalErrors = 0;
int totalFailed = 0;
int totalSkipped = 0;
for (int i = 0; i < testCaseCount; i++)
{
IExecutionSink resultsSink;
ConcurrentDictionary<string, ExecutionSummary> completionMessages = new ConcurrentDictionary<string, ExecutionSummary>();
IMessageSinkWithTypes reporterMessageHandler;
var reporters = GetAvailableRunnerReporters();
var commandLine = CommandLine.Parse(reporters, @"CoreXunit.dll");
IRunnerLogger logger = new ConsoleRunnerLogger(!commandLine.NoColor);
reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(commandLine.Reporter.CreateMessageHandler(logger));
var xmlElement = new XElement("TestResult");
resultsSink = new XmlAggregateSink(reporterMessageHandler, completionMessages, xmlElement, () => true);
var message = new Xunit.NullMessageSink();
var executionOptions = TestFrameworkOptions.ForExecution();
c.RunTests(tv.TestCases.Take<Xunit.Abstractions.ITestCase>(1), resultsSink, executionOptions);
resultsSink.Finished.WaitOne(o.TimeOut);
tv.TestCases.RemoveAt(0);
totalResult++;
totalErrors = totalErrors + resultsSink.ExecutionSummary.Errors;
totalFailed = totalFailed + resultsSink.ExecutionSummary.Failed;
totalSkipped = totalSkipped + resultsSink.ExecutionSummary.Skipped;
xmlElement.Save(file);
file.Flush();
}
file.Dispose();
Console.WriteLine("Total tests: " + totalResult);
Console.ForegroundColor = ConsoleColor.DarkRed;
Console.WriteLine("Error tests: " + totalErrors);
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("Failed tests: " + totalFailed);
Console.ForegroundColor = ConsoleColor.Yellow;
Console.WriteLine("Skipped tests: " + totalSkipped);
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("Passed tests: " + (totalResult - totalErrors - totalFailed - totalSkipped));
Console.ResetColor();
}
}
XElement ExecuteAssembly(object consoleLock,
XunitProjectAssembly assembly,
bool serialize,
bool needsXml,
bool?parallelizeTestCollections,
int?maxThreadCount,
bool diagnosticMessages,
bool noColor,
AppDomainSupport?appDomains,
bool failSkips,
bool stopOnFail,
XunitFilters filters,
bool internalDiagnosticMessages)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
{
return(null);
}
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= diagnosticMessages;
assembly.Configuration.InternalDiagnosticMessages |= internalDiagnosticMessages;
if (appDomains.HasValue)
{
assembly.Configuration.AppDomain = appDomains;
}
// Setup discovery and execution options with command-line overrides
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
executionOptions.SetStopOnTestFail(stopOnFail);
if (maxThreadCount.HasValue)
{
executionOptions.SetMaxParallelThreads(maxThreadCount);
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageSink = DiagnosticMessageSink.ForDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
var internalDiagnosticsMessageSink = DiagnosticMessageSink.ForInternalDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.InternalDiagnosticMessagesOrDefault, noColor);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
#if !WINDOWS_UWP
using (AssemblyHelper.SubscribeResolveForAssembly(assembly.AssemblyFilename, internalDiagnosticsMessageSink))
#endif
using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
using (var discoverySink = new TestDiscoverySink(() => cancel))
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions));
controller.Find(false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
}
else
{
if (serialize)
{
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary));
if (assemblyElement != null)
{
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
}
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), MessageSinkWithTypesAdapter.Wrap(diagnosticMessageSink));
}
if (failSkips)
{
resultsSink = new DelegatingFailSkipSink(resultsSink);
}
controller.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
if (stopOnFail && resultsSink.ExecutionSummary.Failed != 0)
{
Console.WriteLine("Canceling due to test failure...");
cancel = true;
}
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine($"{e.GetType().FullName}: {e.Message}");
if (internalDiagnosticMessages)
{
Console.WriteLine(e.StackTrace);
}
e = e.InnerException;
}
}
return(assemblyElement);
}
void RunTestsInAssembly(IRunContext runContext,
IFrameworkHandle frameworkHandle,
LoggerHelper logger,
TestPlatformContext testPlatformContext,
IMessageSinkWithTypes reporterMessageHandler,
AssemblyRunInfo runInfo)
{
if (cancelled)
{
return;
}
var assembly = new XunitProjectAssembly {
AssemblyFilename = runInfo.AssemblyFileName
};
var assemblyFileName = runInfo.AssemblyFileName;
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName);
var configuration = runInfo.Configuration;
var shadowCopy = configuration.ShadowCopyOrDefault;
var appDomain = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
if (RunSettingsHelper.DisableAppDomain)
{
appDomain = AppDomainSupport.Denied;
}
try
{
#if WINDOWS_UAP
// For AppX Apps, use the package location
assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName));
#endif
var diagnosticSink = new DiagnosticMessageSink(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault);
var diagnosticMessageSink = MessageSinkAdapter.Wrap(diagnosticSink);
using (var controller = new XunitFrontController(appDomain, assemblyFileName, shadowCopy: shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
{
var testCasesMap = new Dictionary <string, TestCase>();
var testCases = new List <ITestCase>();
if (runInfo.TestCases == null || !runInfo.TestCases.Any())
{
// Discover tests
AssemblyDiscoveredInfo assemblyDiscoveredInfo = new AssemblyDiscoveredInfo();
DiscoverTestsInSource(controller, logger, testPlatformContext,
(source, discoverer, discoveryOptions) => new VsExecutionDiscoverySink(() => cancelled),
(source, discoverer, discoveryOptions, visitor) =>
{
if (discoveryOptions.GetInternalDiagnosticMessagesOrDefault())
{
foreach (var testCase in visitor.TestCases)
{
logger.Log(testCase, "Discovered [execution] test case '{0}' (ID = '{1}')",
testCase.DisplayName, testCase.UniqueID);
}
}
assemblyDiscoveredInfo = new AssemblyDiscoveredInfo
{
AssemblyFileName = source,
DiscoveredTestCases = visitor.TestCases.Select(testCase => new DiscoveredTestCase(source, discoverer, testCase, logger, testPlatformContext)).ToList()
};
},
assemblyFileName,
shadowCopy,
configuration
);
if (assemblyDiscoveredInfo.DiscoveredTestCases == null || !assemblyDiscoveredInfo.DiscoveredTestCases.Any())
{
if (configuration.InternalDiagnosticMessagesOrDefault)
{
logger.LogWarning("Skipping '{0}' since no tests were found during discovery [execution].", assemblyDiscoveredInfo.AssemblyFileName);
}
return;
}
// Filter tests
var traitNames = new HashSet <string>(assemblyDiscoveredInfo.DiscoveredTestCases.SelectMany(testCase => testCase.TraitNames));
// Apply any filtering
var filter = new TestCaseFilter(runContext, logger, assemblyDiscoveredInfo.AssemblyFileName, traitNames);
var filteredTestCases = assemblyDiscoveredInfo.DiscoveredTestCases.Where(dtc => filter.MatchTestCase(dtc.VSTestCase)).ToList();
// Force unique names if there is more than 1 testcase with the same name
foreach (var groupWithDuplicateNames in filteredTestCases.GroupBy(dtc => dtc.Name).Where(group => group.Count() > 1))
{
foreach (var discoveredTestCaseWithDuplicateName in groupWithDuplicateNames)
{
discoveredTestCaseWithDuplicateName.ForceUniqueName();
}
}
foreach (var filteredTestCase in filteredTestCases)
{
var uniqueID = filteredTestCase.UniqueID;
if (testCasesMap.ContainsKey(uniqueID))
{
logger.LogWarning(filteredTestCase.TestCase, "Skipping test case with duplicate ID '{0}' ('{1}' and '{2}')", uniqueID, testCasesMap[uniqueID].DisplayName, filteredTestCase.VSTestCase.DisplayName);
}
else
{
testCasesMap.Add(uniqueID, filteredTestCase.VSTestCase);
testCases.Add(filteredTestCase.TestCase);
}
}
}
else
{
// We are in Run Specific tests scenario, the `TestCase` objects are available.
// Query the `TestCase` objects to find XunitTestCase objects.
foreach (var vstestCase in runInfo.TestCases)
{
var xunitTestCase = Deserialize(logger, controller, vstestCase);
if (xunitTestCase != null)
{
testCasesMap.Add(xunitTestCase.UniqueID, vstestCase);
testCases.Add(xunitTestCase);
}
}
}
// Execute tests
var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration);
if (RunSettingsHelper.DisableParallelization)
{
executionOptions.SetSynchronousMessageReporting(true);
executionOptions.SetDisableParallelization(true);
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
using (var vsExecutionSink = new VsExecutionSink(reporterMessageHandler, frameworkHandle, logger, testCasesMap, executionOptions, () => cancelled))
{
IExecutionSink resultsSink = vsExecutionSink;
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticSink);
}
controller.RunTests(testCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
}
}
}
catch (Exception ex)
{
logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex);
}
}
static XElement ExecuteAssembly(object consoleLock,
string defaultDirectory,
XunitProjectAssembly assembly,
bool quiet,
bool serialize,
bool needsXml,
bool teamCity,
bool appVeyor,
bool? parallelizeTestCollections,
int? maxThreadCount,
bool diagnosticMessages,
XunitFilters filters)
{
if (cancel)
return null;
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
return null;
if (diagnosticMessages)
assembly.Configuration.DiagnosticMessages = true;
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue && maxThreadCount.Value > -1)
executionOptions.SetMaxParallelThreads(maxThreadCount.GetValueOrDefault());
if (parallelizeTestCollections.HasValue)
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
if (!quiet)
lock (consoleLock)
{
if (assembly.Configuration.DiagnosticMessagesOrDefault)
Console.WriteLine("Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})",
assemblyDisplayName,
discoveryOptions.GetMethodDisplayOrDefault(),
!executionOptions.GetDisableParallelizationOrDefault(),
executionOptions.GetMaxParallelThreadsOrDefault());
else
Console.WriteLine("Discovering: {0}", assemblyDisplayName);
}
var diagnosticMessageVisitor = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault);
using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy, diagnosticMessageSink: diagnosticMessageVisitor))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions);
discoveryVisitor.Finished.WaitOne();
if (!quiet)
lock (consoleLock)
Console.WriteLine("Discovered: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList();
if (filteredTestCases.Count == 0)
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
else
{
if (serialize)
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
var resultsVisitor = CreateVisitor(consoleLock, quiet, defaultDirectory, assemblyElement, teamCity, appVeyor);
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine("{0}: {1}", e.GetType().FullName, e.Message);
e = e.InnerException;
}
}
return assemblyElement;
}
XElement ExecuteAssembly(object consoleLock,
XunitProjectAssembly assembly,
bool needsXml,
bool?parallelizeTestCollections,
int?maxThreadCount,
bool diagnosticMessages,
bool noColor,
XunitFilters filters,
bool designTime,
bool listTestCases,
IReadOnlyList <string> designTimeFullyQualifiedNames)
{
if (_cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
// Turn off pre-enumeration of theories when we're not running in Visual Studio
if (!designTime)
{
assembly.Configuration.PreEnumerateTheories = false;
}
if (diagnosticMessages)
{
assembly.Configuration.DiagnosticMessages = true;
}
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue)
{
executionOptions.SetMaxParallelThreads(maxThreadCount);
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageVisitor = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
var sourceInformationProvider = GetSourceInformationProviderAdapater(assembly);
using (var controller = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, false, diagnosticMessageSink: diagnosticMessageVisitor, sourceInformationProvider: sourceInformationProvider))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
var includeSourceInformation = designTime && listTestCases;
// Discover & filter the tests
_reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, false, false, discoveryOptions));
controller.Find(includeSourceInformation: includeSourceInformation, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions);
discoveryVisitor.Finished.WaitOne();
IDictionary <ITestCase, VsTestCase> vsTestCases = null;
if (designTime)
{
vsTestCases = DesignTimeTestConverter.Convert(discoveryVisitor.TestCases);
}
if (listTestCases)
{
lock (consoleLock)
{
if (designTime)
{
foreach (var testcase in vsTestCases.Values)
{
_testDiscoverySink?.SendTestFound(testcase);
Console.WriteLine(testcase.FullyQualifiedName);
}
}
else
{
foreach (var testcase in discoveryVisitor.TestCases)
{
Console.WriteLine(testcase.DisplayName);
}
}
}
return(assemblyElement);
}
IExecutionVisitor resultsVisitor;
if (designTime)
{
resultsVisitor = new DesignTimeExecutionVisitor(_testExecutionSink, vsTestCases, _reporterMessageHandler);
}
else
{
resultsVisitor = new XmlAggregateVisitor(_reporterMessageHandler, _completionMessages, assemblyElement, () => _cancel);
}
IList <ITestCase> filteredTestCases;
var testCasesDiscovered = discoveryVisitor.TestCases.Count;
if (!designTime || designTimeFullyQualifiedNames.Count == 0)
{
filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList();
}
else
{
filteredTestCases = vsTestCases.Where(t => designTimeFullyQualifiedNames.Contains(t.Value.FullyQualifiedName)).Select(t => t.Key).ToList();
}
var testCasesToRun = filteredTestCases.Count;
_reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
if (filteredTestCases.Count == 0)
{
_completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
}
else
{
_reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
_reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary));
}
}
}
catch (Exception ex)
{
_failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine("{0}: {1}", e.GetType().FullName, e.Message);
e = e.InnerException;
}
}
return(assemblyElement);
}
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool silent)
{
if (cancel)
return null;
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
return null;
using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, options: new TestFrameworkOptions());
discoveryVisitor.Finished.WaitOne();
var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, silent);
controller.Run(discoveryVisitor.TestCases, resultsVisitor, new TestFrameworkOptions());
resultsVisitor.Finished.WaitOne();
}
}
catch (Exception ex)
{
var e = ex;
while (e != null)
{
Console.WriteLine("{0}: {1}", e.GetType().FullName, e.Message);
foreach (string stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
Console.WriteLine(stackLine);
e = e.InnerException;
}
}
return assemblyElement;
}
/// <summary>
/// MultiNodeTestRunner takes the following <see cref="args"/>:
///
/// C:\> Akka.MultiNodeTestRunner.exe [assembly name] [-Dmultinode.enable-filesink=on] [-Dmultinode.output-directory={dir path}] [-Dmultinode.spec={spec name}]
///
/// <list type="number">
/// <listheader>
/// <term>Argument</term>
/// <description>The name and possible value of a given Akka.MultiNodeTestRunner.exe argument.</description>
/// </listheader>
/// <item>
/// <term>AssemblyName</term>
/// <description>
/// The full path or name of an assembly containing as least one MultiNodeSpec in the current working directory.
///
/// i.e. "Akka.Cluster.Tests.MultiNode.dll"
/// "C:\akka.net\src\Akka.Cluster.Tests\bin\Debug\Akka.Cluster.Tests.MultiNode.dll"
/// </description>
/// </item>
/// <item>
/// <term>-Dmultinode.enable-filesink</term>
/// <description>Having this flag set means that the contents of this test run will be saved in the
/// current working directory as a .JSON file.
/// </description>
/// </item>
/// <item>
/// <term>-Dmultinode.multinode.output-directory</term>
/// <description>Setting this flag means that any persistent multi-node test runner output files
/// will be written to this directory instead of the default, which is the same folder
/// as the test binary.
/// </description>
/// </item>
/// <item>
/// <term>-Dmultinode.listen-address={ip}</term>
/// <description>
/// Determines the address that this multi-node test runner will use to listen for log messages from
/// individual NodeTestRunner.exe processes.
///
/// Defaults to 127.0.0.1
/// </description>
/// </item>
/// <item>
/// <term>-Dmultinode.listen-port={port}</term>
/// <description>
/// Determines the port number that this multi-node test runner will use to listen for log messages from
/// individual NodeTestRunner.exe processes.
///
/// Defaults to 6577
/// </description>
/// </item>
/// <item>
/// <term>-Dmultinode.spec={spec name}</term>
/// <description>
/// Setting this flag means that only tests which contains the spec name will be executed
/// otherwise all tests will be executed
/// </description>
/// </item>
/// <item>
/// <term>-Dmultinode.include={include filter}</term>
/// <description>
/// Setting this flag means that only tests which matches the comma separated wildcard
/// filter will be executed, otherwise all tests will be executed
/// </description>
/// </item>
/// <item>
/// <term>-Dmultinode.exclude={exclude filter}</term>
/// <description>
/// Setting this flag means that only tests which does not match the comma separated
/// wildcard filter will be executed, otherwise all tests will be executed
/// </description>
/// </item>
/// </list>
/// </summary>
static void Main(string[] args)
{
// Force load the args
CommandLine.GetPropertyOrDefault("force load", null);
if (CommandLine.ShowHelp)
{
PrintHelp();
return;
}
if (CommandLine.ShowVersion)
{
var version = Assembly.GetExecutingAssembly().GetName().Version.ToString();
Console.WriteLine($"Version: {version}");
return;
}
OutputDirectory = CommandLine.GetPropertyOrDefault("multinode.output-directory", string.Empty);
FailedSpecsDirectory = CommandLine.GetPropertyOrDefault("multinode.failed-specs-directory", "FAILED_SPECS_LOGS");
string logLevel = CommandLine.GetPropertyOrDefault("multinode.loglevel", "WARNING");
TestRunSystem = ActorSystem.Create("TestRunnerLogging", $"akka.loglevel={logLevel}");
var suiteName = Path.GetFileNameWithoutExtension(Path.GetFullPath(args[0].Trim('"')));
var teamCityFormattingOn = CommandLine.GetPropertyOrDefault("multinode.teamcity", "false");
if (!Boolean.TryParse(teamCityFormattingOn, out TeamCityFormattingOn))
{
throw new ArgumentException("Invalid argument provided for -Dmultinode.teamcity");
}
var listenAddress = IPAddress.Parse(CommandLine.GetPropertyOrDefault("multinode.listen-address", "127.0.0.1"));
var listenPort = CommandLine.GetInt32OrDefault("multinode.listen-port", 6577);
var listenEndpoint = new IPEndPoint(listenAddress, listenPort);
var specName = CommandLine.GetPropertyOrDefault("multinode.spec", "");
var platform = CommandLine.GetPropertyOrDefault("multinode.platform", "net");
var reporter = CommandLine.GetPropertyOrDefault("multinode.reporter", "console");
var clearOutputDirectory = CommandLine.GetInt32OrDefault("multinode.clear-output", 0);
if (clearOutputDirectory > 0 && Directory.Exists(OutputDirectory))
{
Directory.Delete(OutputDirectory, true);
}
var include = new WildcardPatterns(CommandLine.GetPropertyOrDefault("multinode.include", null), true);
var exclude = new WildcardPatterns(CommandLine.GetPropertyOrDefault("multinode.exclude", null), false);
Props coordinatorProps;
switch (reporter.ToLowerInvariant())
{
case "trx":
coordinatorProps = Props.Create(() => new SinkCoordinator(new[] { new TrxMessageSink(suiteName) }));
break;
case "teamcity":
coordinatorProps = Props.Create(() => new SinkCoordinator(new[] { new TeamCityMessageSink(Console.WriteLine, suiteName) }));
break;
case "console":
coordinatorProps = Props.Create(() => new SinkCoordinator(new[] { new ConsoleMessageSink() }));
break;
default:
throw new ArgumentException($"Given reporter name '{reporter}' is not understood, valid reporters are: trx and teamcity");
}
SinkCoordinator = TestRunSystem.ActorOf(coordinatorProps, "sinkCoordinator");
#if CORECLR
if (!_validNetCorePlatform.Contains(platform))
{
throw new Exception($"Target platform not supported: {platform}. Supported platforms are net and netcore");
}
#else
if (platform != "net")
{
throw new Exception($"Target platform not supported: {platform}. Supported platforms are net");
}
#endif
var tcpLogger = TestRunSystem.ActorOf(Props.Create(() => new TcpLoggingServer(SinkCoordinator)), "TcpLogger");
TestRunSystem.Tcp().Tell(new Tcp.Bind(tcpLogger, listenEndpoint), sender: tcpLogger);
var assemblyPath = Path.GetFullPath(args[0].Trim('"')); //unquote the string first
EnableAllSinks(assemblyPath, platform);
PublishRunnerMessage($"Running MultiNodeTests for {assemblyPath}");
#if CORECLR
// In NetCore, if the assembly file hasn't been touched,
// XunitFrontController would fail loading external assemblies and its dependencies.
var assembly = AssemblyLoadContext.Default.LoadFromAssemblyPath(assemblyPath);
var asms = assembly.GetReferencedAssemblies();
var basePath = Path.GetDirectoryName(assemblyPath);
foreach (var asm in asms)
{
try
{
Assembly.Load(new AssemblyName(asm.FullName));
}
catch (Exception)
{
var path = Path.Combine(basePath, asm.Name + ".dll");
try
{
AssemblyLoadContext.Default.LoadFromAssemblyPath(path);
}
catch (Exception e)
{
Console.Out.WriteLine($"Failed to load dll: {path}");
}
}
}
#endif
using (var controller = new XunitFrontController(AppDomainSupport.IfAvailable, assemblyPath))
{
using (var discovery = new Discovery())
{
controller.Find(false, discovery, TestFrameworkOptions.ForDiscovery());
discovery.Finished.WaitOne();
if (discovery.WasSuccessful)
{
foreach (var test in discovery.Tests.Reverse())
{
var node = test.Value.First();
if (!string.IsNullOrEmpty(node.SkipReason))
{
PublishRunnerMessage($"Skipping test {node.MethodName}. Reason - {node.SkipReason}");
continue;
}
if (!string.IsNullOrWhiteSpace(specName) &&
CultureInfo.InvariantCulture.CompareInfo.IndexOf(node.TestName,
specName,
CompareOptions.IgnoreCase) < 0)
{
PublishRunnerMessage($"Skipping [{node.MethodName}] (Filtering)");
continue;
}
// include filter
if (!include.IsMatch(node.MethodName))
{
PublishRunnerMessage($"Skipping [{node.MethodName}] (Include filter)");
continue;
}
if (exclude.IsMatch(node.MethodName))
{
PublishRunnerMessage($"Skipping [{node.MethodName}] (Exclude filter)");
continue;
}
var processes = new List <Process>();
PublishRunnerMessage($"Starting test {node.MethodName}");
Console.Out.WriteLine($"Starting test {node.MethodName}");
StartNewSpec(test.Value);
#if CORECLR
var ntrNetPath = Path.Combine(AppContext.BaseDirectory, "Akka.NodeTestRunner.exe");
var ntrNetCorePath = Path.Combine(AppContext.BaseDirectory, "Akka.NodeTestRunner.dll");
var alternateIndex = 0;
#endif
var timelineCollector = TestRunSystem.ActorOf(Props.Create(() => new TimelineLogCollectorActor()));
string testOutputDir = null;
string runningSpecName = null;
foreach (var nodeTest in test.Value)
{
//Loop through each test, work out number of nodes to run on and kick off process
var sbArguments = new StringBuilder()
//.Append($@"-Dmultinode.test-assembly=""{assemblyPath}"" ")
.Append($@"-Dmultinode.test-class=""{nodeTest.TypeName}"" ")
.Append($@"-Dmultinode.test-method=""{nodeTest.MethodName}"" ")
.Append($@"-Dmultinode.max-nodes={test.Value.Count} ")
.Append($@"-Dmultinode.server-host=""{"localhost"}"" ")
.Append($@"-Dmultinode.host=""{"localhost"}"" ")
.Append($@"-Dmultinode.index={nodeTest.Node - 1} ")
.Append($@"-Dmultinode.role=""{nodeTest.Role}"" ")
.Append($@"-Dmultinode.listen-address={listenAddress} ")
.Append($@"-Dmultinode.listen-port={listenPort} ");
#if CORECLR
string fileName = null;
switch (platform)
{
case "net":
fileName = ntrNetPath;
sbArguments.Insert(0, $@" -Dmultinode.test-assembly=""{assemblyPath}"" ");
break;
case "netcore":
case "net5":
fileName = "dotnet";
sbArguments.Insert(0, $@" -Dmultinode.test-assembly=""{assemblyPath}"" ");
sbArguments.Insert(0, ntrNetCorePath);
break;
}
var process = new Process
{
StartInfo = new ProcessStartInfo
{
FileName = fileName,
UseShellExecute = false,
RedirectStandardOutput = true,
Arguments = sbArguments.ToString(),
WorkingDirectory = Path.GetDirectoryName(assemblyPath)
}
};
#else
sbArguments.Insert(0, $@"-Dmultinode.test-assembly=""{assemblyPath}"" ");
var process = new Process
{
StartInfo = new ProcessStartInfo
{
FileName = "Akka.NodeTestRunner.exe",
UseShellExecute = false,
RedirectStandardOutput = true,
Arguments = sbArguments.ToString()
}
};
#endif
processes.Add(process);
var nodeIndex = nodeTest.Node;
var nodeRole = nodeTest.Role;
#if CORECLR
if (platform == "netcore" || platform == "net5")
{
process.StartInfo.FileName = "dotnet";
process.StartInfo.Arguments = ntrNetCorePath + " " + process.StartInfo.Arguments;
process.StartInfo.WorkingDirectory = Path.GetDirectoryName(assemblyPath);
}
#endif
//TODO: might need to do some validation here to avoid the 260 character max path error on Windows
var folder = Directory.CreateDirectory(Path.Combine(OutputDirectory, nodeTest.TestName));
testOutputDir = testOutputDir ?? folder.FullName;
var logFilePath = Path.Combine(folder.FullName, $"node{nodeIndex}__{nodeRole}__{platform}.txt");
runningSpecName = nodeTest.TestName;
var nodeInfo = new TimelineLogCollectorActor.NodeInfo(nodeIndex, nodeRole, platform, nodeTest.TestName);
var fileActor = TestRunSystem.ActorOf(Props.Create(() => new FileSystemAppenderActor(logFilePath)));
process.OutputDataReceived += (sender, eventArgs) =>
{
if (eventArgs?.Data != null)
{
fileActor.Tell(eventArgs.Data);
timelineCollector.Tell(new TimelineLogCollectorActor.LogMessage(nodeInfo, eventArgs.Data));
if (TeamCityFormattingOn)
{
// teamCityTest.WriteStdOutput(eventArgs.Data); TODO: open flood gates
}
}
};
var closureTest = nodeTest;
process.Exited += (sender, eventArgs) =>
{
if (process.ExitCode == 0)
{
ReportSpecPassFromExitCode(nodeIndex, nodeRole, closureTest.TestName);
}
};
process.Start();
process.BeginOutputReadLine();
PublishRunnerMessage($"Started node {nodeIndex} : {nodeRole} on pid {process.Id}");
}
var specFailed = false;
foreach (var process in processes)
{
process.WaitForExit();
specFailed = specFailed || process.ExitCode > 0;
process.Dispose();
}
PublishRunnerMessage("Waiting 3 seconds for all messages from all processes to be collected.");
Thread.Sleep(TimeSpan.FromSeconds(3));
if (testOutputDir != null)
{
var dumpTasks = new List <Task>()
{
// Dump aggregated timeline to file for this test
timelineCollector.Ask <Done>(new TimelineLogCollectorActor.DumpToFile(Path.Combine(testOutputDir, "aggregated.txt"))),
// Print aggregated timeline into the console
timelineCollector.Ask <Done>(new TimelineLogCollectorActor.PrintToConsole(logLevel))
};
if (specFailed)
{
var dumpFailureArtifactTask = timelineCollector.Ask <Done>(
new TimelineLogCollectorActor.DumpToFile(Path.Combine(Path.GetFullPath(OutputDirectory), FailedSpecsDirectory, $"{runningSpecName}.txt")));
dumpTasks.Add(dumpFailureArtifactTask);
}
Task.WaitAll(dumpTasks.ToArray());
}
FinishSpec(test.Value, timelineCollector);
}
Console.WriteLine("Complete");
PublishRunnerMessage("Waiting 5 seconds for all messages from all processes to be collected.");
Thread.Sleep(TimeSpan.FromSeconds(5));
}
else
{
var sb = new StringBuilder();
sb.AppendLine("One or more exception was thrown while discovering test cases. Test Aborted.");
foreach (var err in discovery.Errors)
{
for (int i = 0; i < err.ExceptionTypes.Length; ++i)
{
sb.AppendLine();
sb.Append($"{err.ExceptionTypes[i]}: {err.Messages[i]}");
sb.Append(err.StackTraces[i]);
}
}
PublishRunnerMessage(sb.ToString());
Console.Out.WriteLine(sb.ToString());
}
}
}
AbortTcpLoggingServer(tcpLogger);
CloseAllSinks();
//Block until all Sinks have been terminated.
TestRunSystem.WhenTerminated.Wait(TimeSpan.FromMinutes(1));
if (Debugger.IsAttached)
{
Console.ReadLine(); //block when debugging
}
//Return the proper exit code
Environment.Exit(ExitCodeContainer.ExitCode);
}
/// <summary>
/// <see cref="StarcounterXunitRunner.Start(string, string)"/> for description.
/// </summary>
/// <param name="typeName"></param>
/// <param name="testReportName"></param>
private void ExecuteTests(string typeName = null, string testReportName = null)
{
ExecutionSummary executionSummary = null;
XunitProjectAssembly assembly = new XunitProjectAssembly
{
AssemblyFilename = assemblyLocation,
ConfigFilename = null
};
XElement assembliesElement = new XElement("assemblies");
XElement assemblyElement = new XElement("assembly");
// Logger
var verboserReporter = new XunitReporters.VerboseReporter();
IRunnerLogger logger = new ConsoleRunnerLogger(useColors: true);
IMessageSinkWithTypes reporterMessageHandler = MessageSinkWithTypesAdapter.Wrap(verboserReporter.CreateMessageHandler(logger));
// Option setup
ITestFrameworkDiscoveryOptions discoveryOptions = TestFrameworkOptions.ForDiscovery(null);
ITestFrameworkExecutionOptions executionOptions = TestFrameworkOptions.ForExecution(null);
executionOptions.SetSynchronousMessageReporting(true);
executionOptions.SetDisableParallelization(DeveloperMode || !RunTestsInParallel);
executionOptions.SetDiagnosticMessages(true);
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var clockTime = Stopwatch.StartNew();
bool cancel = false;
using (var controller = new XunitFrontController(
appDomainSupport: AppDomainSupport.Denied,
assemblyFileName: assembly.AssemblyFilename,
configFileName: null,
shadowCopy: shadowCopy,
shadowCopyFolder: null,
sourceInformationProvider: null,
diagnosticMessageSink: null))
using (var discoverySink = new TestDiscoverySink(() => cancel))
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(
assembly: assembly,
appDomain: controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied,
shadowCopy: shadowCopy,
discoveryOptions: discoveryOptions));
if (typeName != null)
{
controller.Find(typeName, false, discoverySink, discoveryOptions);
}
else
{
controller.Find(false, discoverySink, discoveryOptions);
}
discoverySink.Finished.WaitOne();
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(TestCaseFilter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
executionSummary = new ExecutionSummary();
}
else
{
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => { executionSummary = summary; });
if (assemblyElement != null)
{
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
}
controller.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
assembliesElement.Add(assemblyElement);
}
}
clockTime.Stop();
assembliesElement.Add(new XAttribute("timestamp", DateTime.Now.ToString(CultureInfo.InvariantCulture)));
if (executionSummary != null)
{
Console.WriteLine();
KeyValuePair <string, ExecutionSummary> kvpExecutionSummary = new KeyValuePair <string, ExecutionSummary>(this.assebmlyName, executionSummary);
reporterMessageHandler.OnMessage(new TestExecutionSummary(clockTime.Elapsed, new List <KeyValuePair <string, ExecutionSummary> > {
kvpExecutionSummary
}));
if (testReportName != null)
{
// Create folder if it does not exist
FileInfo fi = new FileInfo(testReportName);
DirectoryInfo directory = fi.Directory;
if (!directory.Exists)
{
Directory.CreateDirectory(directory.FullName);
}
CreateXmlTestReport(assembliesElement, fi);
CreateHtmlTestReport(assembliesElement, fi);
}
Console.WriteLine();
Console.WriteLine();
}
}
protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly)
{
if (cancel)
return null;
var assemblyElement = NeedsXml ? new XElement("assembly") : null;
try
{
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
if (DiagnosticMessages)
assembly.Configuration.DiagnosticMessages = true;
// Setup discovery and execution options with command-line overrides
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue && maxThreadCount.Value > -1)
executionOptions.SetMaxParallelThreads(maxThreadCount);
if (parallelizeTestCollections.HasValue)
executionOptions.SetDisableParallelization(!parallelizeTestCollections);
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageVisitor = new DiagnosticMessageVisitor(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault);
using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy, diagnosticMessageSink: diagnosticMessageVisitor))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, discoveryOptions, executionOptions));
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions);
discoveryVisitor.Finished.WaitOne();
var testCasesDiscovered = discoveryVisitor.TestCases.Count;
var filteredTestCases = discoveryVisitor.TestCases.Where(Filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, executionOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
else
{
if (SerializeTestCases)
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
var resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel);
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
if (resultsVisitor.Failed != 0)
ExitCode = 1;
}
}
}
catch (Exception ex)
{
var e = ex;
while (e != null)
{
Log.LogError("{0}: {1}", e.GetType().FullName, e.Message);
foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
Log.LogError(stackLine);
e = e.InnerException;
}
ExitCode = -1;
}
return assemblyElement;
}
protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly)
{
if (cancel)
{
return(null);
}
var assemblyElement = NeedsXml ? new XElement("assembly") : null;
try
{
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= DiagnosticMessages;
if (appDomains.HasValue)
{
assembly.Configuration.UseAppDomain = appDomains.GetValueOrDefault();
}
// Setup discovery and execution options with command-line overrides
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue && maxThreadCount.Value > -1)
{
executionOptions.SetMaxParallelThreads(maxThreadCount);
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections);
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageVisitor = new DiagnosticMessageVisitor(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault);
var useAppDomain = assembly.Configuration.UseAppDomainOrDefault;
using (var controller = new XunitFrontController(useAppDomain, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy, diagnosticMessageSink: diagnosticMessageVisitor))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, useAppDomain, discoveryOptions));
controller.Find(false, discoveryVisitor, discoveryOptions);
discoveryVisitor.Finished.WaitOne();
var testCasesDiscovered = discoveryVisitor.TestCases.Count;
var filteredTestCases = discoveryVisitor.TestCases.Where(Filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
}
else
{
if (SerializeTestCases)
{
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
}
var resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel);
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary));
if (resultsVisitor.Failed != 0)
{
ExitCode = 1;
}
}
}
}
catch (Exception ex)
{
var e = ex;
while (e != null)
{
Log.LogError("{0}: {1}", e.GetType().FullName, e.Message);
foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
{
Log.LogError(stackLine);
}
e = e.InnerException;
}
ExitCode = -1;
}
return(assemblyElement);
}
public static int Main(string[] args)
{
var arguments = new Stack <string> ();
string host = null;
int port = 0;
bool closeAfterTestRun;
bool failed;
for (var i = args.Length - 1; i >= 0; i--)
{
arguments.Push(args[i]);
}
// First argument is the connection string if we're driven by harness.exe, otherwise we're driven by UITests
if (arguments.Count > 0 && arguments.Peek().StartsWith("tcp:", StringComparison.Ordinal))
{
var parts = arguments.Pop().Split(':');
if (parts.Length != 3)
{
throw new Exception();
}
host = parts [1];
port = Int32.Parse(parts [2]);
closeAfterTestRun = true;
}
else
{
closeAfterTestRun = false;
}
#if !ENABLE_NETCORE
// Make sure the TLS subsystem including the DependencyInjector is initialized.
// This would normally happen on system startup in
// `xamarin-macios/src/ObjcRuntime/Runtime.cs`.
MonoTlsProviderFactory.Initialize();
#endif
// some tests assert having a SynchronizationContext for MONOTOUCH, provide a default one
SynchronizationContext.SetSynchronizationContext(new SynchronizationContext());
#if XUNIT_RUNNER
var writer = new TcpWriter(host, port);
var assemblyFileName = arguments.Pop();
var filters = XunitArgumentsParser.ParseArgumentsToFilter(arguments);
var configuration = new TestAssemblyConfiguration()
{
ShadowCopy = false
};
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
var discoverySink = new TestDiscoverySink();
var diagnosticSink = new DiagnosticTextWriterMessageSink(writer);
var testOptions = TestFrameworkOptions.ForExecution(configuration);
var testSink = new TestMessageSink();
var controller = new XunitFrontController(AppDomainSupport.Denied, assemblyFileName, configFileName: null, shadowCopy: false, diagnosticMessageSink: diagnosticSink);
Interop.mono_sdks_ui_set_test_summary_message($"Running {assemblyFileName}...");
writer.WriteLine($"Discovering tests for {assemblyFileName}");
controller.Find(includeSourceInformation: false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList();
writer.WriteLine($"Discovery finished.");
var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => { writer.WriteLine($"Tests run: {summary.Total}, Errors: 0, Failures: {summary.Failed}, Skipped: {summary.Skipped}{Environment.NewLine}Time: {TimeSpan.FromSeconds ((double)summary.Time).TotalSeconds}s"); });
var resultsXmlAssembly = new XElement("assembly");
var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly);
testSink.Execution.TestPassedEvent += args => { writer.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); Interop.mono_sdks_ui_increment_testcase_result(0); };
testSink.Execution.TestSkippedEvent += args => { writer.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); Interop.mono_sdks_ui_increment_testcase_result(1); };
testSink.Execution.TestFailedEvent += args => { writer.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages (args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces (args.Message)}"); Interop.mono_sdks_ui_increment_testcase_result(2); };
testSink.Execution.TestAssemblyStartingEvent += args => { writer.WriteLine($"Running tests for {args.Message.TestAssembly.Assembly}"); };
testSink.Execution.TestAssemblyFinishedEvent += args => { writer.WriteLine($"Finished {args.Message.TestAssembly.Assembly}{Environment.NewLine}"); };
controller.RunTests(testCasesToRun, resultsSink, testOptions);
resultsSink.Finished.WaitOne();
var resultsXml = new XElement("assemblies");
resultsXml.Add(resultsXmlAssembly);
resultsXml.Save(resultsXmlPath);
if (host != null)
{
writer.WriteLine($"STARTRESULTXML");
resultsXml.Save(((TcpWriter)writer).RawStream);
writer.WriteLine();
writer.WriteLine($"ENDRESULTXML");
}
failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0;
#else
MonoSdksTextUI runner;
TextWriter writer = null;
string resultsXmlPath = Path.GetTempFileName();
string assemblyFileName = arguments.Peek();
if (File.Exists("nunit-excludes.txt"))
{
var excludes = File.ReadAllLines("nunit-excludes.txt");
arguments.Push("-exclude:" + String.Join(",", excludes));
}
arguments.Push("-labels");
arguments.Push("-format:xunit");
arguments.Push($"-result:{resultsXmlPath}");
if (host != null)
{
Console.WriteLine($"Connecting to harness at {host}:{port}.");
writer = new TcpWriter(host, port);
}
else
{
writer = ConsoleWriter.Out;
}
Interop.mono_sdks_ui_set_test_summary_message($"Running {assemblyFileName}...");
runner = new MonoSdksTextUI(writer);
runner.Execute(arguments.ToArray());
if (host != null)
{
writer.WriteLine($"STARTRESULTXML");
using (var resultsXmlStream = File.OpenRead(resultsXmlPath)) resultsXmlStream.CopyTo(((TcpWriter)writer).RawStream);
writer.WriteLine();
writer.WriteLine($"ENDRESULTXML");
}
failed = runner.Failure;
#endif
Interop.mono_sdks_ui_set_test_summary_message($"Summary: {(failed ? "Failed" : "Succeeded")} for {assemblyFileName}.");
if (!closeAfterTestRun)
{
Thread.Sleep(Int32.MaxValue);
}
return(failed ? 1 : 0);
}
void DiscoverTests <TVisitor>(IEnumerable <string> sources,
LoggerHelper logger,
Func <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitorFactory,
Action <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitComplete = null)
where TVisitor : IVsDiscoverySink, IDisposable
{
try
{
RemotingUtility.CleanUpRegisteredChannels();
using (AssemblyHelper.SubscribeResolve())
{
var reporterMessageHandler = new DefaultRunnerReporterWithTypes().CreateMessageHandler(new VisualStudioRunnerLogger(logger));
foreach (var assemblyFileNameCanBeWithoutAbsolutePath in sources)
{
var assemblyFileName = assemblyFileNameCanBeWithoutAbsolutePath;
#if !PLATFORM_DOTNET
assemblyFileName = Path.GetFullPath(assemblyFileNameCanBeWithoutAbsolutePath);
#endif
var assembly = new XunitProjectAssembly {
AssemblyFilename = assemblyFileName
};
var configuration = LoadConfiguration(assemblyFileName);
var fileName = Path.GetFileNameWithoutExtension(assemblyFileName);
var shadowCopy = configuration.ShadowCopyOrDefault;
try
{
if (cancelled)
{
break;
}
if (!IsXunitTestAssembly(assemblyFileName))
{
if (configuration.DiagnosticMessagesOrDefault)
{
logger.Log("Skipping: {0} (no reference to xUnit.net)", fileName);
}
}
else
{
var diagnosticSink = new DiagnosticMessageSink(logger, fileName, configuration.DiagnosticMessagesOrDefault);
using (var framework = new XunitFrontController(AppDomainDefaultBehavior, assemblyFileName: assemblyFileName, configFileName: null, shadowCopy: shadowCopy, diagnosticMessageSink: MessageSinkAdapter.Wrap(diagnosticSink)))
{
var targetFramework = framework.TargetFramework;
if (targetFramework.StartsWith("MonoTouch", StringComparison.OrdinalIgnoreCase) ||
targetFramework.StartsWith("MonoAndroid", StringComparison.OrdinalIgnoreCase) ||
targetFramework.StartsWith("Xamarin.iOS", StringComparison.OrdinalIgnoreCase))
{
if (configuration.DiagnosticMessagesOrDefault)
{
logger.Log("Skipping: {0} (unsupported target framework '{1}')", fileName, targetFramework);
}
}
else
{
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
using (var visitor = visitorFactory(assemblyFileName, framework, discoveryOptions))
{
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, framework.CanUseAppDomains && AppDomainDefaultBehavior != AppDomainSupport.Denied, shadowCopy, discoveryOptions));
framework.Find(includeSourceInformation: true, discoveryMessageSink: visitor, discoveryOptions: discoveryOptions);
var totalTests = visitor.Finish();
visitComplete?.Invoke(assemblyFileName, framework, discoveryOptions, visitor);
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, totalTests, totalTests));
}
}
}
}
}
catch (Exception e)
{
var ex = e.Unwrap();
var fileNotFound = ex as FileNotFoundException;
#if !PLATFORM_DOTNET
var fileLoad = ex as FileLoadException;
#endif
if (ex is InvalidOperationException)
{
logger.LogWarning("Skipping: {0} ({1})", fileName, ex.Message);
}
else if (fileNotFound != null)
{
logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileNotFound.FileName));
}
#if !PLATFORM_DOTNET
else if (fileLoad != null)
{
logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileLoad.FileName));
}
#endif
else
{
logger.LogWarning("Exception discovering tests from {0}: {1}", fileName, ex);
}
}
}
}
}
catch (Exception e)
{
logger.LogWarning("Exception discovering tests: {0}", e.Unwrap());
}
}
IEnumerable <IGrouping <string, TestCase> > GetTests(IEnumerable <string> sources, IMessageLogger logger, XunitVisualStudioSettings settings, Stopwatch stopwatch)
{
var result = new List <IGrouping <string, TestCase> >();
RemotingUtility.CleanUpRegisteredChannels();
using (AssemblyHelper.SubscribeResolve())
{
if (settings.MessageDisplay == MessageDisplay.Diagnostic)
{
logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery started", stopwatch.Elapsed));
}
foreach (string assemblyFileName in sources)
{
var fileName = Path.GetFileName(assemblyFileName);
try
{
if (cancelled)
{
break;
}
if (!IsXunitTestAssembly(assemblyFileName))
{
if (settings.MessageDisplay == MessageDisplay.Diagnostic)
{
logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Skipping: {1}", stopwatch.Elapsed, fileName));
}
}
else
{
if (settings.MessageDisplay == MessageDisplay.Diagnostic)
{
logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery starting: {1}", stopwatch.Elapsed, fileName));
}
using (var framework = new XunitFrontController(assemblyFileName, configFileName: null, shadowCopy: true))
using (var sink = new TestDiscoveryVisitor())
{
framework.Find(includeSourceInformation: true, messageSink: sink, options: new TestFrameworkOptions());
sink.Finished.WaitOne();
result.Add(
new Grouping <string, TestCase>(
assemblyFileName,
sink.TestCases
.GroupBy(tc => String.Format("{0}.{1}", tc.Class.Name, tc.Method.Name))
.SelectMany(group => group.Select(testCase => VsDiscoveryVisitor.CreateVsTestCase(assemblyFileName, framework, testCase, settings, forceUniqueNames: group.Count() > 1)))
.ToList()
)
);
if (settings.MessageDisplay != MessageDisplay.None)
{
logger.SendMessage(TestMessageLevel.Informational,
String.Format("[xUnit.net {0}] Discovery finished: {1} ({2} tests)", stopwatch.Elapsed, Path.GetFileName(assemblyFileName), sink.TestCases.Count));
}
}
}
}
catch (Exception e)
{
logger.SendMessage(TestMessageLevel.Error,
String.Format("[xUnit.net {0}] Exception discovering tests from {1}: {2}", stopwatch.Elapsed, assemblyFileName, e));
}
}
if (settings.MessageDisplay == MessageDisplay.Diagnostic)
{
logger.SendMessage(TestMessageLevel.Informational, String.Format("[xUnit.net {0}] Discovery complete", stopwatch.Elapsed));
}
return(result);
}
}
void RunTestsInAssembly(IRunContext runContext,
IFrameworkHandle frameworkHandle,
LoggerHelper logger,
IMessageSink reporterMessageHandler,
AssemblyRunInfo runInfo)
{
if (cancelled)
{
return;
}
var assembly = new XunitProjectAssembly {
AssemblyFilename = runInfo.AssemblyFileName
};
var assemblyFileName = runInfo.AssemblyFileName;
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName);
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var appDomain = assembly.Configuration.AppDomain ?? AppDomainDefaultBehavior;
if (RunSettingsHelper.DisableAppDomain)
{
appDomain = AppDomainSupport.Denied;
}
try
{
#if PLATFORM_DOTNET
// For AppX Apps, use the package location
assemblyFileName = Path.Combine(Windows.ApplicationModel.Package.Current.InstalledLocation.Path, Path.GetFileName(assemblyFileName));
#endif
var diagnosticSink = new DiagnosticMessageSink(logger, assemblyDisplayName, runInfo.Configuration.DiagnosticMessagesOrDefault);
using (var controller = new XunitFrontController(appDomain, assemblyFileName: assemblyFileName, configFileName: null, shadowCopy: shadowCopy, diagnosticMessageSink: MessageSinkAdapter.Wrap(diagnosticSink)))
{
var xunitTestCases = runInfo.TestCases.Select(tc => new { vs = tc, xunit = Deserialize(logger, controller, tc) })
.Where(tc => tc.xunit != null)
.ToDictionary(tc => tc.xunit, tc => tc.vs);
var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration);
if (RunSettingsHelper.DisableParallelization)
{
executionOptions.SetSynchronousMessageReporting(true);
executionOptions.SetDisableParallelization(true);
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
using (var executionSink = new VsExecutionSink(frameworkHandle, logger, xunitTestCases, executionOptions, () => cancelled))
{
controller.RunTests(xunitTestCases.Keys.ToList(), executionSink, executionOptions);
executionSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, executionSink.ExecutionSummary));
}
}
}
catch (Exception ex)
{
logger.LogError("{0}: Catastrophic failure: {1}", assemblyDisplayName, ex);
}
}
public TdNetRunnerHelper(Assembly assembly, ITestListener testListener)
{
this.testListener = testListener;
frontController = new XunitFrontController(new Uri(assembly.CodeBase).LocalPath);
}
private XElement ExecuteAssembly(
[NotNull] XunitProjectAssembly assembly,
[NotNull] TestRunOptions options,
bool needsXml,
[NotNull] XunitFilters filters)
{
if (assembly == null)
{
throw new ArgumentNullException(nameof(assembly));
}
if (options == null)
{
throw new ArgumentNullException(nameof(options));
}
if (filters == null)
{
throw new ArgumentNullException(nameof(filters));
}
if (_cancel)
{
return(null);
}
var internalDiagnosticMessages = options.InternalDiagnosticMessages;
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(assembly.AssemblyFilename) || !ValidateFileExists(assembly.ConfigFilename))
{
return(null);
}
ConfigureAssembly(assembly, options);
// Setup discovery and execution options with command-line overrides
var executionOptions = ConfiguExecutionOptions(assembly, options);
var assemblyDisplayName = assembly.GetFileNameWithoutExtension().EmptyIfNull();
void LogAction(MessageHandlerArgs <IDiagnosticMessage> args, string assemblyName) => Console.WriteLine($"{assemblyDisplayName}: {args.Message.Message}");
var diagnosticMessageSink = new DiagnosticMessageSink(LogAction, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault);
var internalDiagnosticsMessageSink = new DiagnosticMessageSink(LogAction, assemblyDisplayName, assembly.Configuration.InternalDiagnosticMessagesOrDefault);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
using (AssemblyHelper.SubscribeResolveForAssembly(assembly.AssemblyFilename, internalDiagnosticsMessageSink))
using (var controller = new XunitFrontController(
appDomainSupport,
assembly.AssemblyFilename,
assembly.ConfigFilename,
shadowCopy,
diagnosticMessageSink: diagnosticMessageSink))
using (var discoverySink = new TestDiscoverySink(() => _cancel))
{
ExecuteTests(assembly, options, filters, controller, discoverySink, executionOptions, assemblyElement, diagnosticMessageSink);
}
}
catch (Exception ex)
{
_failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine($"{e.GetType().FullName}: {e.Message}");
if (internalDiagnosticMessages)
{
Console.WriteLine(e.StackTrace);
}
e = e.InnerException;
}
}
return(assemblyElement);
}
static XElement ExecuteAssembly(object consoleLock,
XunitProjectAssembly assembly,
bool serialize,
bool needsXml,
bool?parallelizeTestCollections,
int?maxThreadCount,
bool diagnosticMessages,
bool noColor,
bool noAppDomain,
bool failSkips,
XunitFilters filters)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
{
return(null);
}
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= diagnosticMessages;
if (noAppDomain)
{
assembly.Configuration.AppDomain = AppDomainSupport.Denied;
}
// Setup discovery and execution options with command-line overrides
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue)
{
executionOptions.SetMaxParallelThreads(maxThreadCount);
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageSink = new DiagnosticMessageSink(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
using (var discoverySink = new TestDiscoverySink())
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions));
controller.Find(false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
}
else
{
if (serialize)
{
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
IExecutionSink resultsSink = new XmlAggregateSink(reporterMessageHandler, assemblyElement, diagnosticMessageSink, completionMessages, () => cancel, longRunningSeconds);
if (failSkips)
{
resultsSink = new FailSkipSink(resultsSink);
}
controller.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine($"{e.GetType().FullName}: {e.Message}");
e = e.InnerException;
}
}
return(assemblyElement);
}
private void ExecuteTests(
[NotNull] XunitProjectAssembly assembly,
[NotNull] TestRunOptions options,
[NotNull] XunitFilters filters,
[NotNull] XunitFrontController controller,
[NotNull] TestDiscoverySink discoverySink,
[NotNull] ITestFrameworkExecutionOptions executionOptions,
[CanBeNull] XElement assemblyElement,
[NotNull] DiagnosticMessageSink diagnosticMessageSink)
{
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var serialize = options.Serialize;
var failSkips = options.FailSkips;
var stopOnFail = options.StopOnFail;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
// Discover & filter the tests
_reporterMessageHandler.OnMessage(
new TestAssemblyDiscoveryStarting(
assembly,
controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied,
shadowCopy,
discoveryOptions));
controller.Find(false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
_reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
_completionMessages.TryAdd(assembly.GetFileName().ThrowIfNull(), new ExecutionSummary());
}
else
{
if (serialize)
{
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
}
_reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(
_reporterMessageHandler,
() => _cancel,
(path, summary) => _completionMessages.TryAdd(path, summary));
if (assemblyElement != null)
{
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
}
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
}
if (failSkips)
{
resultsSink = new DelegatingFailSkipSink(resultsSink);
}
controller.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
_reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
if (!stopOnFail || resultsSink.ExecutionSummary.Failed == 0)
{
return;
}
Console.WriteLine("Canceling due to test failure...");
_cancel = true;
}
}
private bool DiscoverTestsInSource <TVisitor>(XunitFrontController framework,
LoggerHelper logger,
TestPlatformContext testPlatformContext,
Func <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitorFactory,
Action <string, ITestFrameworkDiscoverer, ITestFrameworkDiscoveryOptions, TVisitor> visitComplete,
string assemblyFileName,
bool shadowCopy,
TestAssemblyConfiguration configuration)
where TVisitor : IVsDiscoverySink, IDisposable
{
var reporterMessageHandler = GetRunnerReporter(new[] { assemblyFileName }).CreateMessageHandler(new VisualStudioRunnerLogger(logger));
var assembly = new XunitProjectAssembly {
AssemblyFilename = assemblyFileName
};
var fileName = Path.GetFileNameWithoutExtension(assemblyFileName);
try
{
if (cancelled)
{
return(false);
}
if (!IsXunitTestAssembly(assemblyFileName))
{
if (configuration.DiagnosticMessagesOrDefault)
{
logger.Log("Skipping: {0} (no reference to xUnit.net)", fileName);
}
}
else
{
var targetFramework = framework.TargetFramework;
if (targetFramework.StartsWith("MonoTouch", StringComparison.OrdinalIgnoreCase) ||
targetFramework.StartsWith("MonoAndroid", StringComparison.OrdinalIgnoreCase) ||
targetFramework.StartsWith("Xamarin.iOS", StringComparison.OrdinalIgnoreCase))
{
if (configuration.DiagnosticMessagesOrDefault)
{
logger.Log("Skipping: {0} (unsupported target framework '{1}')", fileName, targetFramework);
}
}
else
{
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
using (var visitor = visitorFactory(assemblyFileName, framework, discoveryOptions))
{
var usingAppDomains = framework.CanUseAppDomains && AppDomainDefaultBehavior != AppDomainSupport.Denied;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, usingAppDomains, shadowCopy, discoveryOptions));
framework.Find(testPlatformContext.RequireSourceInformation, visitor, discoveryOptions);
var totalTests = visitor.Finish();
visitComplete?.Invoke(assemblyFileName, framework, discoveryOptions, visitor);
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, totalTests, totalTests));
}
}
}
}
catch (Exception e)
{
var ex = e.Unwrap();
if (ex is InvalidOperationException)
{
logger.LogWarning("Skipping: {0} ({1})", fileName, ex.Message);
}
else if (ex is FileNotFoundException fileNotFound)
{
logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileNotFound.FileName));
}
#if !WINDOWS_UAP
else if (ex is FileLoadException fileLoad)
{
logger.LogWarning("Skipping: {0} (could not find dependent assembly '{1}')", fileName, Path.GetFileNameWithoutExtension(fileLoad.FileName));
}
#endif
else
{
logger.LogWarning("Exception discovering tests from {0}: {1}", fileName, ex);
}
}
return(true);
}
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool quiet, bool serialize, bool needsXml, bool teamCity, bool appVeyor, bool?parallelizeTestCollections, int?maxThreadCount, XunitFilters filters)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
{
return(null);
}
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue)
{
executionOptions.SetMaxParallelThreads(maxThreadCount.GetValueOrDefault());
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
lock (consoleLock)
{
if (assembly.Configuration.DiagnosticMessagesOrDefault)
{
Console.WriteLine("Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})",
assemblyDisplayName,
discoveryOptions.GetMethodDisplayOrDefault(),
!executionOptions.GetDisableParallelizationOrDefault(),
executionOptions.GetMaxParallelThreadsOrDefault());
}
else if (!quiet)
{
Console.WriteLine("Discovering: {0}", assemblyDisplayName);
}
}
var diagnosticMessageVisitor = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault);
using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy, diagnosticMessageSink: diagnosticMessageVisitor))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions);
discoveryVisitor.Finished.WaitOne();
if (!quiet)
{
lock (consoleLock)
Console.WriteLine("Discovered: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
}
var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList();
if (filteredTestCases.Count == 0)
{
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
}
else
{
if (serialize)
{
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
}
var resultsVisitor = CreateVisitor(consoleLock, quiet, defaultDirectory, assemblyElement, teamCity, appVeyor);
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine("{0}: {1}", e.GetType().FullName, e.Message);
e = e.InnerException;
}
}
return(assemblyElement);
}
static int Main(string[] args)
{
var nodeIndex = CommandLine.GetInt32("multinode.index");
var assemblyFileName = CommandLine.GetProperty("multinode.test-assembly");
var typeName = CommandLine.GetProperty("multinode.test-class");
var testName = CommandLine.GetProperty("multinode.test-method");
var displayName = testName;
var listenAddress = IPAddress.Parse(CommandLine.GetProperty("multinode.listen-address"));
var listenPort = CommandLine.GetInt32("multinode.listen-port");
var listenEndpoint = new IPEndPoint(listenAddress, listenPort);
var system = ActorSystem.Create("NoteTestRunner-" + nodeIndex);
var tcpClient = _logger = system.ActorOf <RunnerTcpClient>();
system.Tcp().Tell(new Tcp.Connect(listenEndpoint), tcpClient);
Thread.Sleep(TimeSpan.FromSeconds(10));
using (var controller = new XunitFrontController(assemblyFileName))
{
/* need to pass in just the assembly name to Discovery, not the full path
* i.e. "Akka.Cluster.Tests.MultiNode.dll"
* not "bin/Release/Akka.Cluster.Tests.MultiNode.dll" - this will cause
* the Discovery class to actually not find any indivudal specs to run
*/
var assemblyName = Path.GetFileName(assemblyFileName);
Console.WriteLine("Running specs for {0} [{1}]", assemblyName, assemblyFileName);
using (var discovery = new Discovery(assemblyName, typeName))
{
using (var sink = new Sink(nodeIndex, tcpClient))
{
Thread.Sleep(10000);
try
{
controller.Find(true, discovery, TestFrameworkOptions.ForDiscovery());
discovery.Finished.WaitOne();
controller.RunTests(discovery.TestCases, sink, TestFrameworkOptions.ForExecution());
}
catch (AggregateException ex)
{
var specFail = new SpecFail(nodeIndex, displayName);
specFail.FailureExceptionTypes.Add(ex.GetType().ToString());
specFail.FailureMessages.Add(ex.Message);
specFail.FailureStackTraces.Add(ex.StackTrace);
foreach (var innerEx in ex.Flatten().InnerExceptions)
{
specFail.FailureExceptionTypes.Add(innerEx.GetType().ToString());
specFail.FailureMessages.Add(innerEx.Message);
specFail.FailureStackTraces.Add(innerEx.StackTrace);
}
_logger.Tell(specFail.ToString());
Console.WriteLine(specFail);
//make sure message is send over the wire
FlushLogMessages();
Environment.Exit(1); //signal failure
return(1);
}
catch (Exception ex)
{
var specFail = new SpecFail(nodeIndex, displayName);
specFail.FailureExceptionTypes.Add(ex.GetType().ToString());
specFail.FailureMessages.Add(ex.Message);
specFail.FailureStackTraces.Add(ex.StackTrace);
_logger.Tell(specFail.ToString());
Console.WriteLine(specFail);
//make sure message is send over the wire
FlushLogMessages();
Environment.Exit(1); //signal failure
return(1);
}
var timedOut = false;
if (!sink.Finished.WaitOne(MaxProcessWaitTimeout)) //timed out
{
var line = string.Format("Timed out while waiting for test to complete after {0} ms",
MaxProcessWaitTimeout);
_logger.Tell(line);
Console.WriteLine(line);
timedOut = true;
}
FlushLogMessages();
system.Shutdown();
system.AwaitTermination();
Environment.Exit(sink.Passed && !timedOut ? 0 : 1);
return(sink.Passed ? 0 : 1);
}
}
}
}
public static void Main(string[] args)
{
OutputDirectory = CommandLine.GetPropertyOrDefault("multinode.output-directory", string.Empty);
TestRunSystem = ActorSystem.Create("TestRunnerLogging");
var teamCityFormattingOn = CommandLine.GetPropertyOrDefault("multinode.teamcity", "false");
if (!Boolean.TryParse(teamCityFormattingOn, out TeamCityFormattingOn))
{
throw new ArgumentException("Invalid argument provided for -Dteamcity");
}
SinkCoordinator = TestRunSystem.ActorOf( // mutes ConsoleMessageSinkActor
Props.Create <SinkCoordinator>(), "sinkCoordinator");
var listenAddress = IPAddress.Parse(CommandLine.GetPropertyOrDefault("multinode.listen-address", "127.0.0.1"));
var listenPort = CommandLine.GetInt32OrDefault("multinode.listen-port", 6577);
var listenEndpoint = new IPEndPoint(listenAddress, listenPort);
var specName = CommandLine.GetPropertyOrDefault("multinode.spec", "");
var platform = CommandLine.GetPropertyOrDefault("multinode.platform", "netcore");
if (!_validNetCorePlatform.Contains(platform))
{
throw new Exception($"Target platform not supported: {platform}. Supported platforms are net and netcore");
}
var tcpLogger = TestRunSystem.ActorOf(Props.Create(() => new TcpLoggingServer(SinkCoordinator)), "TcpLogger");
TestRunSystem.Tcp().Tell(new Tcp.Bind(tcpLogger, listenEndpoint));
var assemblyPath = Path.GetFullPath(args[0].Trim('"')); //unquote the string first
EnableAllSinks(assemblyPath, platform);
PublishRunnerMessage($"Running MultiNodeTests for {assemblyPath}");
// In NetCore, if the assembly file hasn't been touched,
// XunitFrontController would fail loading external assemblies and its dependencies.
var assembly = AssemblyLoadContext.Default.LoadFromAssemblyPath(assemblyPath);
var asms = assembly.GetReferencedAssemblies();
var basePath = Path.GetDirectoryName(assemblyPath);
foreach (var asm in asms)
{
try
{
Assembly.Load(new AssemblyName(asm.FullName));
}
catch (Exception)
{
var path = Path.Combine(basePath, asm.Name + ".dll");
try
{
AssemblyLoadContext.Default.LoadFromAssemblyPath(path);
}
catch (Exception e)
{
Console.Out.WriteLine($"Failed to load dll: {path}, {e}");
}
}
}
using (var controller = new XunitFrontController(AppDomainSupport.IfAvailable, assemblyPath))
{
using (var discovery = new Discovery())
{
controller.Find(false, discovery, TestFrameworkOptions.ForDiscovery());
discovery.Finished.WaitOne();
if (discovery.WasSuccessful)
{
foreach (var test in discovery.Tests.Reverse())
{
if (!string.IsNullOrEmpty(test.Value.First().SkipReason))
{
PublishRunnerMessage($"Skipping test {test.Value.First().MethodName}. Reason - {test.Value.First().SkipReason}");
continue;
}
if (!string.IsNullOrWhiteSpace(specName) &&
CultureInfo.InvariantCulture.CompareInfo.IndexOf(test.Value.First().TestName,
specName,
CompareOptions.IgnoreCase) < 0)
{
PublishRunnerMessage($"Skipping [{test.Value.First().MethodName}] (Filtering)");
continue;
}
var processes = new List <Process>();
PublishRunnerMessage($"Starting test {test.Value.First().MethodName}");
Console.Out.WriteLine($"Starting test {test.Value.First().MethodName}");
StartNewSpec(test.Value);
var ntrNetPath = Path.Combine(AppContext.BaseDirectory, "Akka.NodeTestRunner.exe");
var ntrNetCorePath = Path.Combine(AppContext.BaseDirectory, "Akka.NodeTestRunner.dll");
foreach (var nodeTest in test.Value)
{
//Loop through each test, work out number of nodes to run on and kick off process
var sbArguments = new StringBuilder()
//.Append($@"-Dmultinode.test-assembly=""{assemblyPath}"" ")
.Append($@"-Dmultinode.test-class=""{nodeTest.TypeName}"" ")
.Append($@"-Dmultinode.test-method=""{nodeTest.MethodName}"" ")
.Append($@"-Dmultinode.max-nodes={test.Value.Count} ")
.Append($@"-Dmultinode.server-host=""{"localhost"}"" ")
.Append($@"-Dmultinode.host=""{"localhost"}"" ")
.Append($@"-Dmultinode.index={nodeTest.Node - 1} ")
.Append($@"-Dmultinode.role=""{nodeTest.Role}"" ")
.Append($@"-Dmultinode.listen-address={listenAddress} ")
.Append($@"-Dmultinode.listen-port={listenPort} ");
string fileName = null;
switch (platform)
{
case "net":
fileName = ntrNetPath;
sbArguments.Insert(0, $@" -Dmultinode.test-assembly=""{assemblyPath}"" ");
break;
case "netcore":
fileName = "dotnet";
sbArguments.Insert(0, $@" -Dmultinode.test-assembly=""{assemblyPath}"" ");
sbArguments.Insert(0, ntrNetCorePath);
break;
}
var process = new Process
{
StartInfo = new ProcessStartInfo
{
FileName = fileName ?? throw new NullException(nameof(fileName)),
UseShellExecute = false,
RedirectStandardOutput = true,
Arguments = sbArguments.ToString(),
WorkingDirectory = Path.GetDirectoryName(assemblyPath)
}
};
processes.Add(process);
var nodeIndex = nodeTest.Node;
var nodeRole = nodeTest.Role;
if (platform == "netcore")
{
process.StartInfo.FileName = "dotnet";
process.StartInfo.Arguments = ntrNetCorePath + " " + process.StartInfo.Arguments;
process.StartInfo.WorkingDirectory = Path.GetDirectoryName(assemblyPath);
}
//TODO: might need to do some validation here to avoid the 260 character max path error on Windows
var folder = Directory.CreateDirectory(Path.Combine(OutputDirectory, nodeTest.TestName));
var logFilePath = Path.Combine(folder.FullName, $"node{nodeIndex}__{nodeRole}__{platform}.txt");
var fileActor = TestRunSystem.ActorOf(Props.Create(() => new FileSystemAppenderActor(logFilePath)));
process.OutputDataReceived += (sender, eventArgs) =>
{
if (eventArgs?.Data != null)
{
fileActor.Tell(eventArgs.Data);
if (TeamCityFormattingOn)
{
// teamCityTest.WriteStdOutput(eventArgs.Data); TODO: open flood gates
}
}
};
var closureTest = nodeTest;
process.Exited += (sender, eventArgs) =>
{
if (process.ExitCode == 0)
{
ReportSpecPassFromExitCode(nodeIndex, nodeRole,
closureTest.TestName);
}
};
process.Start();
process.BeginOutputReadLine();
PublishRunnerMessage($"Started node {nodeIndex} : {nodeRole} on pid {process.Id}");
}
foreach (var process in processes)
{
process.WaitForExit();
process.Dispose();
}
PublishRunnerMessage("Waiting 3 seconds for all messages from all processes to be collected.");
Thread.Sleep(TimeSpan.FromSeconds(3));
FinishSpec(test.Value);
}
Console.WriteLine("Complete");
PublishRunnerMessage("Waiting 5 seconds for all messages from all processes to be collected.");
Thread.Sleep(TimeSpan.FromSeconds(5));
}
else
{
var sb = new StringBuilder();
sb.AppendLine("One or more exception was thrown while discovering test cases. Test Aborted.");
foreach (var err in discovery.Errors)
{
for (int i = 0; i < err.ExceptionTypes.Length; ++i)
{
sb.AppendLine();
sb.Append($"{err.ExceptionTypes[i]}: {err.Messages[i]}");
sb.Append(err.StackTraces[i]);
}
}
PublishRunnerMessage(sb.ToString());
Console.Out.WriteLine(sb.ToString());
}
}
}
CloseAllSinks();
//Block until all Sinks have been terminated.
TestRunSystem.WhenTerminated.Wait(TimeSpan.FromMinutes(1));
if (Debugger.IsAttached)
{
Console.ReadLine(); //block when debugging
}
//Return the proper exit code
Environment.Exit(ExitCodeContainer.ExitCode);
}
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool appVeyor, bool?parallelizeTestCollections, int?maxThreadCount, XunitFilters filters)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
{
return(null);
}
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue)
{
executionOptions.SetMaxParallelThreads(maxThreadCount.GetValueOrDefault());
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
}
lock (consoleLock)
{
if (assembly.Configuration.DiagnosticMessages)
{
Console.WriteLine("Discovering: {0} (method display = {1}, parallel test collections = {2}, max threads = {3})",
Path.GetFileNameWithoutExtension(assembly.AssemblyFilename),
discoveryOptions.GetMethodDisplay(),
!executionOptions.GetDisableParallelization(),
executionOptions.GetMaxParallelThreads());
}
else
{
Console.WriteLine("Discovering: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
}
}
using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, discoveryOptions: discoveryOptions);
discoveryVisitor.Finished.WaitOne();
lock (consoleLock)
Console.WriteLine("Discovered: {0}", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, appVeyor);
var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList();
if (filteredTestCases.Count == 0)
{
lock (consoleLock)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("ERROR: {0} has no tests to run", Path.GetFileNameWithoutExtension(assembly.AssemblyFilename));
Console.ForegroundColor = ConsoleColor.Gray;
}
}
else
{
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
}
}
}
catch (Exception ex)
{
Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message);
failed = true;
}
return(assemblyElement);
}
/// <summary>
/// MultiNodeTestRunner takes the following <see cref="args"/>:
///
/// C:\> Akka.MultiNodeTestRunner.exe [assembly name] [-Dmultinode.enable-filesink=on] [-Dmultinode.output-directory={dir path}]
///
/// <list type="number">
/// <listheader>
/// <term>Argument</term>
/// <description>The name and possible value of a given Akka.MultiNodeTestRunner.exe argument.</description>
/// </listheader>
/// <item>
/// <term>AssemblyName</term>
/// <description>
/// The full path or name of an assembly containing as least one MultiNodeSpec in the current working directory.
///
/// i.e. "Akka.Cluster.Tests.MultiNode.dll"
/// "C:\akka.net\src\Akka.Cluster.Tests\bin\Debug\Akka.Cluster.Tests.MultiNode.dll"
/// </description>
/// </item>
/// <item>
/// <term>-Dmultinode.enable-filesink</term>
/// <description>Having this flag set means that the contents of this test run will be saved in the
/// current working directory as a .JSON file.
/// </description>
/// </item>
/// <item>
/// <term>-Dmultinode.multinode.output-directory</term>
/// <description>Setting this flag means that any persistent multi-node test runner output files
/// will be written to this directory instead of the default, which is the same folder
/// as the test binary.
/// </description>
/// </item>
/// </list>
/// </summary>
static void Main(string[] args)
{
TestRunSystem = ActorSystem.Create("TestRunnerLogging");
SinkCoordinator = TestRunSystem.ActorOf(Props.Create <SinkCoordinator>(), "sinkCoordinator");
var assemblyName = Path.GetFullPath(args[0]);
EnableAllSinks(assemblyName);
PublishRunnerMessage(String.Format("Running MultiNodeTests for {0}", assemblyName));
using (var controller = new XunitFrontController(assemblyName))
{
using (var discovery = new Discovery())
{
controller.Find(false, discovery, TestFrameworkOptions.ForDiscovery());
discovery.Finished.WaitOne();
foreach (var test in discovery.Tests.Reverse())
{
if (!string.IsNullOrEmpty(test.Value.First().SkipReason))
{
PublishRunnerMessage(string.Format("Skipping test {0}. Reason - {1}", test.Value.First().MethodName, test.Value.First().SkipReason));
continue;
}
PublishRunnerMessage(string.Format("Starting test {0}", test.Value.First().MethodName));
var processes = new List <Process>();
StartNewSpec(test.Value);
foreach (var nodeTest in test.Value)
{
//Loop through each test, work out number of nodes to run on and kick off process
var process = new Process();
processes.Add(process);
process.StartInfo.UseShellExecute = false;
process.StartInfo.RedirectStandardOutput = true;
process.StartInfo.FileName = "Akka.NodeTestRunner.exe";
process.StartInfo.Arguments = String.Format(@"-Dmultinode.test-assembly=""{0}"" -Dmultinode.test-class=""{1}"" -Dmultinode.test-method=""{2}"" -Dmultinode.max-nodes={3} -Dmultinode.server-host=""{4}"" -Dmultinode.host=""{5}"" -Dmultinode.index={6}",
assemblyName, nodeTest.TypeName, nodeTest.MethodName, test.Value.Count, "localhost", "localhost", nodeTest.Node - 1);
var nodeIndex = nodeTest.Node;
process.OutputDataReceived +=
(sender, line) =>
{
//ignore any trailing whitespace
if (string.IsNullOrEmpty(line.Data) || string.IsNullOrWhiteSpace(line.Data))
{
return;
}
string message = line.Data;
if (!message.StartsWith("[NODE", true, CultureInfo.InvariantCulture))
{
message = "[NODE" + nodeIndex + "]" + message;
}
PublishToAllSinks(message);
};
var closureTest = nodeTest;
process.Exited += (sender, eventArgs) =>
{
if (process.ExitCode == 0)
{
ReportSpecPassFromExitCode(nodeIndex, closureTest.TestName);
}
};
process.Start();
process.BeginOutputReadLine();
PublishRunnerMessage(string.Format("Started node {0} on pid {1}", nodeTest.Node, process.Id));
}
foreach (var process in processes)
{
process.WaitForExit();
var exitCode = process.ExitCode;
process.Close();
}
PublishRunnerMessage("Waiting 3 seconds for all messages from all processes to be collected.");
Thread.Sleep(TimeSpan.FromSeconds(3));
FinishSpec();
}
}
}
Console.WriteLine("Complete");
PublishRunnerMessage("Waiting 5 seconds for all messages from all processes to be collected.");
Thread.Sleep(TimeSpan.FromSeconds(5));
CloseAllSinks();
//Block until all Sinks have been terminated.
TestRunSystem.AwaitTermination(TimeSpan.FromMinutes(1));
//Return the proper exit code
Environment.Exit(ExitCodeContainer.ExitCode);
}
protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly, AppDomainSupport?appDomains)
{
if (cancel)
{
return(null);
}
var assemblyElement = NeedsXml ? new XElement("assembly") : null;
try
{
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= DiagnosticMessages;
assembly.Configuration.InternalDiagnosticMessages |= InternalDiagnosticMessages;
if (appDomains.HasValue)
{
assembly.Configuration.AppDomain = appDomains;
}
// Setup discovery and execution options with command-line overrides
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue && maxThreadCount.Value > -1)
{
executionOptions.SetMaxParallelThreads(maxThreadCount);
}
if (parallelizeTestCollections.HasValue)
{
executionOptions.SetDisableParallelization(!parallelizeTestCollections);
}
if (stopOnFail.HasValue)
{
executionOptions.SetStopOnTestFail(stopOnFail);
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageSink = DiagnosticMessageSink.ForDiagnostics(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
using (var discoverySink = new TestDiscoverySink(() => cancel))
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions));
controller.Find(false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(Filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
{
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
}
else
{
if (SerializeTestCases)
{
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
}
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary));
if (assemblyElement != null)
{
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
}
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
}
if (FailSkips)
{
resultsSink = new DelegatingFailSkipSink(resultsSink);
}
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
controller.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
if (resultsSink.ExecutionSummary.Failed != 0)
{
ExitCode = 1;
if (stopOnFail == true)
{
Log.LogMessage(MessageImportance.High, "Canceling due to test failure...");
Cancel();
}
}
}
}
}
catch (Exception ex)
{
var e = ex;
while (e != null)
{
Log.LogError("{0}: {1}", e.GetType().FullName, e.Message);
foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
{
Log.LogError(stackLine);
}
e = e.InnerException;
}
ExitCode = -1;
}
return(assemblyElement);
}
IEnumerable<AssemblyRunInfo> DiscoverTestsInAssemblies()
{
var result = new List<AssemblyRunInfo>();
try
{
using (AssemblyHelper.SubscribeResolve())
{
foreach (var assm in TestAssemblies)
{
// Xunit needs the file name
var assemblyFileName = assm.GetName()
.Name + ".dll";
var configuration = GetConfiguration(assm.GetName()
.Name);
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
try
{
if (cancelled)
break;
using (var framework = new XunitFrontController(AppDomainSupport.Denied, assemblyFileName, null, false))
using (var sink = new TestDiscoveryVisitor())
{
framework.Find(true, sink, discoveryOptions);
sink.Finished.WaitOne();
result.Add(new AssemblyRunInfo
{
AssemblyFileName = assemblyFileName,
Configuration = configuration,
TestCases = sink.TestCases.Select(tc => new TestCaseViewModel(assemblyFileName, tc, navigation, this))
.ToList()
});
}
}
catch (Exception e)
{
Debug.WriteLine(e);
}
}
}
}
catch (Exception e)
{
Debug.WriteLine(e);
}
return result;
}
protected virtual XElement ExecuteAssembly(XunitProjectAssembly assembly)
{
if (cancel)
return null;
var assemblyElement = NeedsXml ? new XElement("assembly") : null;
try
{
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= DiagnosticMessages;
if (appDomains.HasValue)
assembly.Configuration.AppDomain = appDomains.GetValueOrDefault() ? AppDomainSupport.Required : AppDomainSupport.Denied;
// Setup discovery and execution options with command-line overrides
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue && maxThreadCount.Value > -1)
executionOptions.SetMaxParallelThreads(maxThreadCount);
if (parallelizeTestCollections.HasValue)
executionOptions.SetDisableParallelization(!parallelizeTestCollections);
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageSink = new DiagnosticMessageSink(Log, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, shadowCopy, diagnosticMessageSink: diagnosticMessageSink))
using (var discoverySink = new TestDiscoverySink(() => cancel))
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, shadowCopy, discoveryOptions));
controller.Find(false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(Filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
else
{
if (SerializeTestCases)
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (path, summary) => completionMessages.TryAdd(path, summary));
if (assemblyElement != null)
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
if (longRunningSeconds > 0)
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
if (FailSkips)
resultsSink = new DelegatingFailSkipSink(resultsSink);
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
controller.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsSink.ExecutionSummary));
if (resultsSink.ExecutionSummary.Failed != 0)
ExitCode = 1;
}
}
}
catch (Exception ex)
{
var e = ex;
while (e != null)
{
Log.LogError("{0}: {1}", e.GetType().FullName, e.Message);
foreach (var stackLine in e.StackTrace.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries))
Log.LogError(stackLine);
e = e.InnerException;
}
ExitCode = -1;
}
return assemblyElement;
}
void RunTestsInAssembly(List<IDisposable> toDispose, AssemblyRunInfo runInfo)
{
if (cancelled)
return;
var assemblyFileName = runInfo.AssemblyFileName;
var controller = new XunitFrontController(AppDomainSupport.Denied, assemblyFileName);
lock (toDispose)
toDispose.Add(controller);
var xunitTestCases = runInfo.TestCases.Select(tc => new
{
vm = tc,
tc = tc.TestCase
})
.Where(tc => tc.tc.UniqueID != null)
.ToDictionary(tc => tc.tc, tc => tc.vm);
var executionOptions = TestFrameworkOptions.ForExecution(runInfo.Configuration);
using (var executionVisitor = new TestExecutionVisitor(xunitTestCases, this, executionOptions, () => cancelled, context))
{
controller.RunTests(xunitTestCases.Select(tc => tc.Value.TestCase).ToList(), executionVisitor, executionOptions);
executionVisitor.Finished.WaitOne();
}
}
static XElement ExecuteAssembly(object consoleLock,
XunitProjectAssembly assembly,
bool serialize,
bool needsXml,
bool? parallelizeTestCollections,
int? maxThreadCount,
bool diagnosticMessages,
bool noColor,
bool noAppDomain,
bool failSkips,
XunitFilters filters)
{
if (cancel)
return null;
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
return null;
// Turn off pre-enumeration of theories, since there is no theory selection UI in this runner
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= diagnosticMessages;
if (noAppDomain)
assembly.Configuration.AppDomain = AppDomainSupport.Denied;
// Setup discovery and execution options with command-line overrides
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
if (maxThreadCount.HasValue)
executionOptions.SetMaxParallelThreads(maxThreadCount);
if (parallelizeTestCollections.HasValue)
executionOptions.SetDisableParallelization(!parallelizeTestCollections.GetValueOrDefault());
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
var diagnosticMessageVisitor = new DiagnosticMessageVisitor(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
using (var controller = new XunitFrontController(appDomainSupport, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy, diagnosticMessageSink: diagnosticMessageVisitor))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
// Discover & filter the tests
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryStarting(assembly, controller.CanUseAppDomains && appDomainSupport != AppDomainSupport.Denied, discoveryOptions));
controller.Find(false, discoveryVisitor, discoveryOptions);
discoveryVisitor.Finished.WaitOne();
var testCasesDiscovered = discoveryVisitor.TestCases.Count;
var filteredTestCases = discoveryVisitor.TestCases.Where(filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
reporterMessageHandler.OnMessage(new TestAssemblyDiscoveryFinished(assembly, discoveryOptions, testCasesDiscovered, testCasesToRun));
// Run the filtered tests
if (testCasesToRun == 0)
completionMessages.TryAdd(Path.GetFileName(assembly.AssemblyFilename), new ExecutionSummary());
else
{
if (serialize)
filteredTestCases = filteredTestCases.Select(controller.Serialize).Select(controller.Deserialize).ToList();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionStarting(assembly, executionOptions));
IExecutionVisitor resultsVisitor = new XmlAggregateVisitor(reporterMessageHandler, completionMessages, assemblyElement, () => cancel);
if (failSkips)
resultsVisitor = new FailSkipVisitor(resultsVisitor);
controller.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
reporterMessageHandler.OnMessage(new TestAssemblyExecutionFinished(assembly, executionOptions, resultsVisitor.ExecutionSummary));
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine($"{e.GetType().FullName}: {e.Message}");
e = e.InnerException;
}
}
return assemblyElement;
}
public async void RunTests(string arguments)
{
log = await Helpers.GetFileStreamWriterInLocalStorageAsync("stdout.txt");
string[] args = arguments.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
log.WriteLine(arguments);
var commandLine = CommandLine.Parse(args);
if (commandLine.Debug)
{
Debugger.Launch();
}
var completionMessages = new ConcurrentDictionary <string, ExecutionSummary>();
var assembliesElement = new XElement("assemblies");
foreach (var assembly in commandLine.Project.Assemblies)
{
if (cancel)
{
return;
}
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= commandLine.DiagnosticMessages;
assembly.Configuration.AppDomain = AppDomainSupport.Denied;
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
executionOptions.SetDisableParallelization(true);
try
{
using (var xunit = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.Configuration.ShadowCopyOrDefault))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
string assemblyName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
// Discover & filter the tests
log.WriteLine($"Discovering: {assemblyName}");
xunit.Find(false, discoveryVisitor, discoveryOptions);
discoveryVisitor.Finished.WaitOne();
var testCasesDiscovered = discoveryVisitor.TestCases.Count;
var filteredTestCases = discoveryVisitor.TestCases.Where(commandLine.Project.Filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
log.WriteLine($"Discovered: {assemblyName}");
// Run the filtered tests
if (testCasesToRun == 0)
{
log.WriteLine($"Info: {assemblyName} has no tests to run");
}
else
{
if (commandLine.Serialize)
{
filteredTestCases = filteredTestCases.Select(xunit.Serialize).Select(xunit.Deserialize).ToList();
}
var assemblyElement = new XElement("assembly");
StandardUapVisitor resultsVisitor = new StandardUapVisitor(assemblyElement, () => cancel, log, completionMessages, commandLine.ShowProgress, commandLine.FailSkips);
xunit.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
assembliesElement.Add(assemblyElement);
log.WriteLine($"{Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)} Total: {resultsVisitor.ExecutionSummary.Total}, Errors: {resultsVisitor.ExecutionSummary.Errors}, Failed: {resultsVisitor.ExecutionSummary.Failed}, Skipped: {resultsVisitor.ExecutionSummary.Skipped}, Time: {resultsVisitor.ExecutionSummary.Time}");
}
}
}
catch (Exception e)
{
assembliesElement = new XElement("error");
assembliesElement.Add(e);
log.WriteLine(e);
}
finally
{
await WriteResults(Path.GetFileName(assembly.AssemblyFilename), assembliesElement);
log.Dispose();
}
}
Application.Current.Exit();
}
static XElement ExecuteAssembly(object consoleLock, string defaultDirectory, XunitProjectAssembly assembly, bool needsXml, bool teamCity, bool appVeyor, bool silent, bool parallelizeTestCollections, int maxThreadCount, XunitFilters filters)
{
if (cancel)
return null;
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
if (!ValidateFileExists(consoleLock, assembly.AssemblyFilename) || !ValidateFileExists(consoleLock, assembly.ConfigFilename))
return null;
using (var controller = new XunitFrontController(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
controller.Find(includeSourceInformation: false, messageSink: discoveryVisitor, options: new TestFrameworkOptions());
discoveryVisitor.Finished.WaitOne();
var executionOptions = new XunitExecutionOptions { DisableParallelization = !parallelizeTestCollections, MaxParallelThreads = maxThreadCount };
var resultsVisitor = CreateVisitor(consoleLock, defaultDirectory, assemblyElement, teamCity, appVeyor, silent);
controller.RunTests(discoveryVisitor.TestCases.Where(filters.Filter).ToList(), resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
}
}
catch (Exception ex)
{
Console.WriteLine("{0}: {1}", ex.GetType().FullName, ex.Message);
failed = true;
}
return assemblyElement;
}
static int Main(string[] args)
{
// Handle RemoteExecutor
if (args.Length > 0 && args[0] == "remote")
{
return(RemoteExecutor.Execute(args.Skip(1).ToArray()));
}
var commandLine = CommandLine.Parse(args);
if (commandLine.Debug)
{
Debugger.Launch();
}
var completionMessages = new ConcurrentDictionary <string, ExecutionSummary>();
var assembliesElement = new XElement("assemblies");
foreach (var assembly in commandLine.Project.Assemblies)
{
if (cancel)
{
break;
}
assembly.Configuration.PreEnumerateTheories = false;
assembly.Configuration.DiagnosticMessages |= commandLine.DiagnosticMessages;
assembly.Configuration.AppDomain = AppDomainSupport.Denied;
var discoveryOptions = TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = TestFrameworkOptions.ForExecution(assembly.Configuration);
executionOptions.SetDisableParallelization(true);
try
{
using (var xunit = new XunitFrontController(AppDomainSupport.Denied, assembly.AssemblyFilename, assembly.ConfigFilename, assembly.Configuration.ShadowCopyOrDefault))
using (var discoveryVisitor = new TestDiscoveryVisitor())
{
string assemblyName = Path.GetFileNameWithoutExtension(assembly.AssemblyFilename);
// Discover & filter the tests
Console.WriteLine($"Discovering: {assemblyName}");
xunit.Find(false, discoveryVisitor, discoveryOptions);
discoveryVisitor.Finished.WaitOne();
var testCasesDiscovered = discoveryVisitor.TestCases.Count;
var filteredTestCases = discoveryVisitor.TestCases.Where(commandLine.Project.Filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
Console.WriteLine($"Discovered: {assemblyName}");
// Run the filtered tests
if (testCasesToRun == 0)
{
Console.WriteLine($"Info: {assemblyName} has no tests to run");
}
else
{
if (commandLine.Serialize)
{
filteredTestCases = filteredTestCases.Select(xunit.Serialize).Select(xunit.Deserialize).ToList();
}
var assemblyElement = new XElement("assembly");
StandardUapVisitor resultsVisitor = new StandardUapVisitor(assemblyElement, () => cancel, completionMessages, commandLine.ShowProgress, commandLine.FailSkips);
xunit.RunTests(filteredTestCases, resultsVisitor, executionOptions);
resultsVisitor.Finished.WaitOne();
assembliesElement.Add(assemblyElement);
Console.WriteLine($"{Path.GetFileNameWithoutExtension(assembly.AssemblyFilename)} Total: {resultsVisitor.ExecutionSummary.Total}, Errors: {resultsVisitor.ExecutionSummary.Errors}, Failed: {resultsVisitor.ExecutionSummary.Failed}, Skipped: {resultsVisitor.ExecutionSummary.Skipped}, Time: {resultsVisitor.ExecutionSummary.Time}");
}
}
}
catch (Exception e)
{
assembliesElement = new XElement("error");
assembliesElement.Add(e);
Console.WriteLine(e);
}
finally
{
WriteResults(Path.GetFileName(assembly.AssemblyFilename), assembliesElement).GetAwaiter().GetResult();
}
}
return(0);
}
