/// <summary>
/// INTERNAL METHOD, FOR TESTING PURPOSES ONLY. DO NOT CALL.
/// </summary>
protected void Run(
IEnumerable <Xunit1TestCase> testCases,
_IMessageSink messageSink)
{
var results = new Xunit1RunSummary();
var environment = $"{IntPtr.Size * 8}-bit .NET {Environment.Version}";
var testCasesList = testCases.ToList();
var testAssemblyStartingMessage = new _TestAssemblyStarting
{
AssemblyName = testAssemblyName,
AssemblyPath = assemblyFileName,
AssemblyUniqueID = TestAssemblyUniqueID,
ConfigFilePath = configFileName,
StartTime = DateTimeOffset.Now,
TestEnvironment = environment,
TestFrameworkDisplayName = TestFrameworkDisplayName,
};
if (messageSink.OnMessage(testAssemblyStartingMessage))
{
try
{
if (testCasesList.Count != 0)
{
results = RunTestCollection(testCasesList, messageSink);
}
}
catch (Exception ex)
{
var errorMetadata = Xunit1ExceptionUtility.ConvertToErrorMetadata(ex);
var errorMessage = new _ErrorMessage
{
ExceptionParentIndices = errorMetadata.ExceptionParentIndices,
ExceptionTypes = errorMetadata.ExceptionTypes,
Messages = errorMetadata.Messages,
StackTraces = errorMetadata.StackTraces
};
messageSink.OnMessage(errorMessage);
}
finally
{
var assemblyFinished = new _TestAssemblyFinished
{
AssemblyUniqueID = testAssemblyStartingMessage.AssemblyUniqueID,
ExecutionTime = results.Time,
TestsFailed = results.Failed,
TestsRun = results.Total,
TestsSkipped = results.Skipped
};
messageSink.OnMessage(assemblyFinished);
}
}
}
bool OnClass(XmlNode xml)
{
SendTestCaseMessagesWhenAppropriate(null);
var @continue = true;
XmlNode?failureNode;
if ((failureNode = xml.SelectSingleNode("failure")) != null)
{
var errorMetadata = Xunit1ExceptionUtility.ConvertToErrorMetadata(failureNode);
var errorMessage = new _ErrorMessage
{
ExceptionParentIndices = errorMetadata.ExceptionParentIndices,
ExceptionTypes = errorMetadata.ExceptionTypes,
Messages = errorMetadata.Messages,
StackTraces = errorMetadata.StackTraces
};
@continue = messageSink.OnMessage(errorMessage);
}
TestClassResults.Time = decimal.Parse(xml.Attributes?["time"]?.Value ?? "0", CultureInfo.InvariantCulture);
TestClassResults.Total = int.Parse(xml.Attributes?["total"]?.Value ?? "0", CultureInfo.InvariantCulture);
TestClassResults.Failed = int.Parse(xml.Attributes?["failed"]?.Value ?? "0", CultureInfo.InvariantCulture);
TestClassResults.Skipped = int.Parse(xml.Attributes?["skipped"]?.Value ?? "0", CultureInfo.InvariantCulture);
return(@continue && TestClassResults.Continue);
}
Xunit1RunSummary RunTestCollection(
IList <Xunit1TestCase> testCases,
_IMessageSink messageSink)
{
Guard.ArgumentValid(nameof(testCases), "testCases must contain at least one test case", testCases.Count > 0);
var collectionStarting = new _TestCollectionStarting
{
AssemblyUniqueID = testCases[0].AssemblyUniqueID,
TestCollectionClass = null,
TestCollectionDisplayName = $"xUnit.net v1 Tests for {assemblyFileName}",
TestCollectionUniqueID = testCases[0].TestCollectionUniqueID
};
var results = new Xunit1RunSummary
{
Continue = messageSink.OnMessage(collectionStarting)
};
try
{
if (results.Continue)
{
foreach (var testClassGroup in testCases.GroupBy(tc => tc.TestClass))
{
var classResults = RunTestClass(testClassGroup.Key, testClassGroup.ToList(), messageSink);
results.Aggregate(classResults);
if (!classResults.Continue)
{
break;
}
}
}
}
finally
{
var collectionFinished = new _TestCollectionFinished
{
AssemblyUniqueID = collectionStarting.AssemblyUniqueID,
ExecutionTime = results.Time,
TestCollectionUniqueID = collectionStarting.TestCollectionUniqueID,
TestsFailed = results.Failed,
TestsRun = results.Total,
TestsSkipped = results.Skipped
};
results.Continue = messageSink.OnMessage(collectionFinished) && results.Continue;
}
return(results);
}
public ClassFixtureWithMessageSinkDependency(_IMessageSink messageSink)
{
MessageSink = messageSink;
MessageSink.OnMessage(new _DiagnosticMessage {
Message = "ClassFixtureWithMessageSinkDependency constructor message"
});
}
Xunit1RunSummary RunTestClass(
string typeName,
IList <Xunit1TestCase> testCases,
_IMessageSink messageSink)
{
Guard.ArgumentValid(nameof(testCases), "testCases must contain at least one test case", testCases.Count > 0);
var handler = new TestClassCallbackHandler(testCases, messageSink);
var results = handler.TestClassResults;
var testClassStarting = new _TestClassStarting
{
AssemblyUniqueID = testCases[0].AssemblyUniqueID,
TestClass = typeName,
TestClassUniqueID = testCases[0].TestClassUniqueID,
TestCollectionUniqueID = testCases[0].TestCollectionUniqueID
};
results.Continue = messageSink.OnMessage(testClassStarting);
try
{
if (results.Continue)
{
var methodNames = testCases.Select(tc => tc.TestMethod).ToList();
Executor.RunTests(typeName, methodNames, handler);
handler.LastNodeArrived.WaitOne();
}
}
finally
{
var testClassFinished = new _TestClassFinished
{
AssemblyUniqueID = testClassStarting.AssemblyUniqueID,
ExecutionTime = results.Time,
TestClassUniqueID = testClassStarting.TestClassUniqueID,
TestCollectionUniqueID = testClassStarting.TestCollectionUniqueID,
TestsFailed = results.Failed,
TestsRun = results.Total,
TestsSkipped = results.Skipped
};
results.Continue = messageSink.OnMessage(testClassFinished) && results.Continue;
}
return(results);
}
/// <inheritdoc/>
public bool OnMessageWithTypes(
IMessageSinkMessage message,
HashSet <string>?messageTypes)
{
Guard.ArgumentNotNull(nameof(message), message);
var v3Message = adapter.Adapt(message, messageTypes);
return(v3MessageSink.OnMessage(v3Message));
}
public bool OnMessage(_MessageSinkMessage message)
{
if (message is _TestCaseDiscovered discovered)
{
if (!filter(discovered))
{
return(true);
}
}
return(innerMessageSink.OnMessage(message));
}
void DispatchMessages()
{
while (reporterQueue.TryDequeue(out var message))
{
try
{
continueRunning &= messageSink.OnMessage(message);
}
catch (Exception ex)
{
try
{
var errorMessage = _ErrorMessage.FromException(ex);
if (!messageSink.OnMessage(errorMessage))
{
continueRunning = false;
}
}
catch { }
}
}
}
void SendDiscoveryStartingMessage(_IMessageSink messageSink)
{
// There is no v2 equivalent to this, so we manufacture it ourselves
var discoveryStarting = new _DiscoveryStarting
{
AssemblyName = assemblyInfo.Name,
AssemblyPath = assemblyInfo.AssemblyPath,
AssemblyUniqueID = UniqueIDGenerator.ForAssembly(assemblyInfo.Name, assemblyInfo.AssemblyPath, configFileName),
ConfigFilePath = configFileName
};
messageSink.OnMessage(discoveryStarting);
}
/// <summary>
/// INTERNAL METHOD, FOR TESTING PURPOSES ONLY. DO NOT CALL.
/// </summary>
protected void Find(
_IMessageSink messageSink,
bool includeSourceInformation,
Predicate <_TestCaseDiscovered>?filter)
{
Guard.ArgumentNotNull(nameof(messageSink), messageSink);
var discoveryStarting = new _DiscoveryStarting
{
AssemblyName = testAssemblyName,
AssemblyPath = assemblyFileName,
AssemblyUniqueID = TestAssemblyUniqueID,
ConfigFilePath = configFileName
};
messageSink.OnMessage(discoveryStarting);
try
{
Find(includeSourceInformation, testCase =>
{
var msg = testCase.ToTestCaseDiscovered(includeSerialization: true);
if (filter == null || filter(msg))
{
messageSink.OnMessage(msg);
}
});
}
finally
{
var discoveryComplete = new _DiscoveryComplete {
AssemblyUniqueID = TestAssemblyUniqueID
};
messageSink.OnMessage(discoveryComplete);
}
}
async ValueTask <int> RunProject(
XunitProject project,
_IMessageSink reporterMessageHandler)
{
XElement?assembliesElement = null;
var clockTime = Stopwatch.StartNew();
var xmlTransformers = TransformFactory.GetXmlTransformers(project);
var needsXml = xmlTransformers.Count > 0;
if (needsXml)
{
assembliesElement = new XElement("assemblies");
}
var originalWorkingFolder = Directory.GetCurrentDirectory();
var assembly = project.Assemblies.Single();
var assemblyElement = await ExecuteAssembly(
assembly,
needsXml,
reporterMessageHandler
);
if (assemblyElement != null)
{
assembliesElement?.Add(assemblyElement);
}
clockTime.Stop();
if (assembliesElement != null)
{
assembliesElement.Add(new XAttribute("timestamp", DateTime.Now.ToString(CultureInfo.InvariantCulture)));
}
testExecutionSummaries.ElapsedClockTime = clockTime.Elapsed;
reporterMessageHandler.OnMessage(testExecutionSummaries);
Directory.SetCurrentDirectory(originalWorkingFolder);
if (assembliesElement != null)
{
xmlTransformers.ForEach(transformer => transformer(assembliesElement));
}
return(failed ? 1 : testExecutionSummaries.SummariesByAssemblyUniqueID.Sum(s => s.Summary.Failed + s.Summary.Errors));
}
/// <inheritdoc/>
public bool OnMessage(_MessageSinkMessage message)
{
Guard.ArgumentNotNull(nameof(message), message);
var result = innerSink.OnMessage(message);
return
(message.DispatchWhen <_ErrorMessage>(args => Interlocked.Increment(ref errors)) &&
message.DispatchWhen <_TestAssemblyCleanupFailure>(args => Interlocked.Increment(ref errors)) &&
message.DispatchWhen <_TestAssemblyFinished>(HandleTestAssemblyFinished) &&
message.DispatchWhen <_TestAssemblyStarting>(HandleTestAssemblyStarting) &&
message.DispatchWhen <_TestCaseCleanupFailure>(args => Interlocked.Increment(ref errors)) &&
message.DispatchWhen <_TestClassCleanupFailure>(args => Interlocked.Increment(ref errors)) &&
message.DispatchWhen <_TestCleanupFailure>(args => Interlocked.Increment(ref errors)) &&
message.DispatchWhen <_TestCollectionCleanupFailure>(args => Interlocked.Increment(ref errors)) &&
message.DispatchWhen <_TestMethodCleanupFailure>(args => Interlocked.Increment(ref errors)) &&
result &&
!cancelThunk());
}
/// <inheritdoc/>
public IEnumerable <TTestCase> OrderTestCases <TTestCase>(IEnumerable <TTestCase> testCases)
where TTestCase : _ITestCase
{
var result = testCases.ToList();
try
{
result.Sort(Compare);
}
catch (Exception ex)
{
diagnosticMessageSink.OnMessage(new _DiagnosticMessage {
Message = $"Exception thrown in DefaultTestCaseOrderer.OrderTestCases(); falling back to random order.{Environment.NewLine}{ex}"
});
result = Randomize(result);
}
return(result);
}
/// <inheritdoc/>
public virtual bool OnMessage(_MessageSinkMessage message)
{
callback?.Invoke(message);
return(innerSink.OnMessage(message));
}
async ValueTask <int> RunProject(
XunitProject project,
_IMessageSink reporterMessageHandler)
{
XElement?assembliesElement = null;
var clockTime = Stopwatch.StartNew();
var xmlTransformers = TransformFactory.GetXmlTransformers(project);
var needsXml = xmlTransformers.Count > 0;
// TODO: Parallelize the ones that will parallelize, and then run the rest sequentially?
var parallelizeAssemblies = project.All(assembly => assembly.Configuration.ParallelizeAssemblyOrDefault);
if (needsXml)
{
assembliesElement = new XElement("assemblies");
}
var originalWorkingFolder = Directory.GetCurrentDirectory();
if (parallelizeAssemblies)
{
var tasks = project.Assemblies.Select(
assembly => Task.Run(
() => ExecuteAssembly(
assembly,
needsXml,
reporterMessageHandler
).AsTask()
)
);
var results = Task.WhenAll(tasks).GetAwaiter().GetResult();
foreach (var assemblyElement in results.WhereNotNull())
{
assembliesElement?.Add(assemblyElement);
}
}
else
{
foreach (var assembly in project.Assemblies)
{
var assemblyElement = await ExecuteAssembly(
assembly,
needsXml,
reporterMessageHandler
);
if (assemblyElement != null)
{
assembliesElement?.Add(assemblyElement);
}
}
}
clockTime.Stop();
if (assembliesElement != null)
{
assembliesElement.Add(new XAttribute("timestamp", DateTime.Now.ToString(CultureInfo.InvariantCulture)));
}
if (completionMessages.Count > 0)
{
var summaries = new TestExecutionSummaries {
ElapsedClockTime = clockTime.Elapsed
};
foreach (var completionMessage in completionMessages.OrderBy(kvp => kvp.Key))
{
summaries.Add(completionMessage.Key, completionMessage.Value);
}
reporterMessageHandler.OnMessage(summaries);
}
Directory.SetCurrentDirectory(originalWorkingFolder);
if (assembliesElement != null)
{
xmlTransformers.ForEach(transformer => transformer(assembliesElement));
}
return(failed ? 1 : completionMessages.Values.Sum(summary => summary.Failed + summary.Errors));
}
async ValueTask <XElement?> ExecuteAssembly(
XunitProjectAssembly assembly,
bool needsXml,
_IMessageSink reporterMessageHandler)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
// Setup discovery and execution options with command-line overrides
var discoveryOptions = _TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = _TestFrameworkOptions.ForExecution(assembly.Configuration);
// The normal default is true here, but we want it to be false for us by default
if (!assembly.Configuration.PreEnumerateTheories.HasValue)
{
discoveryOptions.SetPreEnumerateTheories(false);
}
var assemblyDisplayName = assembly.AssemblyDisplayName;
var noColor = assembly.Project.Configuration.NoColorOrDefault;
var diagnosticMessageSink = ConsoleDiagnosticMessageSink.ForDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
var internalDiagnosticsMessageSink = ConsoleDiagnosticMessageSink.ForInternalDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.InternalDiagnosticMessagesOrDefault, noColor);
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
var assemblyInfo = new ReflectionAssemblyInfo(testAssembly);
await using var disposalTracker = new DisposalTracker();
var testFramework = ExtensibilityPointFactory.GetTestFramework(diagnosticMessageSink, assemblyInfo);
disposalTracker.Add(testFramework);
var discoverySink = new TestDiscoverySink(() => cancel);
// Discover & filter the tests
var testDiscoverer = testFramework.GetDiscoverer(assemblyInfo);
var discoveryStarting = new TestAssemblyDiscoveryStarting
{
AppDomain = AppDomainOption.NotAvailable,
Assembly = assembly,
DiscoveryOptions = discoveryOptions,
ShadowCopy = false
};
reporterMessageHandler.OnMessage(discoveryStarting);
testDiscoverer.Find(discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesDiscovered = discoverySink.TestCases.Count;
var filteredTestCases = discoverySink.TestCases.Where(assembly.Configuration.Filters.Filter).ToList();
var testCasesToRun = filteredTestCases.Count;
var discoveryFinished = new TestAssemblyDiscoveryFinished
{
Assembly = assembly,
DiscoveryOptions = discoveryOptions,
TestCasesDiscovered = testCasesDiscovered,
TestCasesToRun = testCasesToRun
};
reporterMessageHandler.OnMessage(discoveryFinished);
// Run the filtered tests
if (testCasesToRun == 0)
{
testExecutionSummaries.Add(testDiscoverer.TestAssemblyUniqueID, new ExecutionSummary());
}
else
{
var executionStarting = new TestAssemblyExecutionStarting
{
Assembly = assembly,
ExecutionOptions = executionOptions
};
reporterMessageHandler.OnMessage(executionStarting);
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel);
if (assemblyElement != null)
{
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
}
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
}
if (assembly.Configuration.FailSkipsOrDefault)
{
resultsSink = new DelegatingFailSkipSink(resultsSink);
}
using (resultsSink)
{
var executor = testFramework.GetExecutor(assemblyInfo);
executor.RunTests(filteredTestCases, resultsSink, executionOptions);
resultsSink.Finished.WaitOne();
testExecutionSummaries.Add(testDiscoverer.TestAssemblyUniqueID, resultsSink.ExecutionSummary);
var executionFinished = new TestAssemblyExecutionFinished
{
Assembly = assembly,
ExecutionOptions = executionOptions,
ExecutionSummary = resultsSink.ExecutionSummary
};
reporterMessageHandler.OnMessage(executionFinished);
if (assembly.Configuration.StopOnFailOrDefault && resultsSink.ExecutionSummary.Failed != 0)
{
Console.WriteLine("Canceling due to test failure...");
cancel = true;
}
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine($"{e.GetType().FullName}: {e.Message}");
if (assembly.Configuration.InternalDiagnosticMessagesOrDefault)
{
Console.WriteLine(e.StackTrace);
}
e = e.InnerException;
}
}
return(assemblyElement);
}
async ValueTask <XElement?> ExecuteAssembly(
XunitProjectAssembly assembly,
bool needsXml,
_IMessageSink reporterMessageHandler)
{
if (cancel)
{
return(null);
}
var assemblyElement = needsXml ? new XElement("assembly") : null;
try
{
var assemblyFileName = Guard.ArgumentNotNull("assembly.AssemblyFilename", assembly.AssemblyFilename);
// Setup discovery and execution options with command-line overrides
var discoveryOptions = _TestFrameworkOptions.ForDiscovery(assembly.Configuration);
var executionOptions = _TestFrameworkOptions.ForExecution(assembly.Configuration);
// The normal default is true here, but we want it to be false for us by default
if (!assembly.Configuration.PreEnumerateTheories.HasValue)
{
discoveryOptions.SetPreEnumerateTheories(false);
}
var assemblyDisplayName = Path.GetFileNameWithoutExtension(assemblyFileName);
var noColor = assembly.Project.Configuration.NoColorOrDefault;
var diagnosticMessageSink = ConsoleDiagnosticMessageSink.ForDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.DiagnosticMessagesOrDefault, noColor);
var internalDiagnosticsMessageSink = ConsoleDiagnosticMessageSink.ForInternalDiagnostics(consoleLock, assemblyDisplayName, assembly.Configuration.InternalDiagnosticMessagesOrDefault, noColor);
var appDomainSupport = assembly.Configuration.AppDomainOrDefault;
var shadowCopy = assembly.Configuration.ShadowCopyOrDefault;
var longRunningSeconds = assembly.Configuration.LongRunningTestSecondsOrDefault;
using var _ = AssemblyHelper.SubscribeResolveForAssembly(assemblyFileName, internalDiagnosticsMessageSink);
await using var controller = XunitFrontController.ForDiscoveryAndExecution(assembly, diagnosticMessageSink: diagnosticMessageSink);
var executionStarting = new TestAssemblyExecutionStarting
{
Assembly = assembly,
ExecutionOptions = executionOptions
};
reporterMessageHandler.OnMessage(executionStarting);
IExecutionSink resultsSink = new DelegatingExecutionSummarySink(reporterMessageHandler, () => cancel, (summary, _) => completionMessages.TryAdd(controller.TestAssemblyUniqueID, summary));
if (assemblyElement != null)
{
resultsSink = new DelegatingXmlCreationSink(resultsSink, assemblyElement);
}
if (longRunningSeconds > 0)
{
resultsSink = new DelegatingLongRunningTestDetectionSink(resultsSink, TimeSpan.FromSeconds(longRunningSeconds), diagnosticMessageSink);
}
if (assembly.Configuration.FailSkipsOrDefault)
{
resultsSink = new DelegatingFailSkipSink(resultsSink);
}
using (resultsSink)
{
var settings = new FrontControllerFindAndRunSettings(discoveryOptions, executionOptions, assembly.Configuration.Filters);
controller.FindAndRun(resultsSink, settings);
resultsSink.Finished.WaitOne();
var executionFinished = new TestAssemblyExecutionFinished
{
Assembly = assembly,
ExecutionOptions = executionOptions,
ExecutionSummary = resultsSink.ExecutionSummary
};
reporterMessageHandler.OnMessage(executionFinished);
if (assembly.Configuration.StopOnFailOrDefault && resultsSink.ExecutionSummary.Failed != 0)
{
Console.WriteLine("Canceling due to test failure...");
cancel = true;
}
}
}
catch (Exception ex)
{
failed = true;
var e = ex;
while (e != null)
{
Console.WriteLine($"{e.GetType().FullName}: {e.Message}");
if (assembly.Configuration.InternalDiagnosticMessagesOrDefault)
{
Console.WriteLine(e.StackTrace);
}
e = e.InnerException;
}
}
return(assemblyElement);
}
/// <summary/>
public bool QueueMessage(_MessageSinkMessage message)
{
Guard.ArgumentNotNull(message);
return(messageSink.OnMessage(message));
}
