public async void OnlyIncludesLongRunningTests()
{
var events = new List <LongRunningTestsSummary>();
using var sink = new TestableDelegatingLongRunningTestDetectionSink(longRunningSeconds: 1, callback: summary => events.Add(summary));
var testCaseStarting1 = new _TestCaseStarting {
TestCaseUniqueID = "test-case-id-1"
};
var testCaseStarting2 = new _TestCaseStarting {
TestCaseUniqueID = "test-case-id-2"
};
sink.OnMessage(new _TestAssemblyStarting());
sink.OnMessage(testCaseStarting1);
await sink.AdvanceClockAsync(500);
sink.OnMessage(testCaseStarting2); // Started later, hasn't run long enough
await sink.AdvanceClockAsync(500);
sink.OnMessage(new _TestCaseFinished {
TestCaseUniqueID = "test-case-id-1"
});
sink.OnMessage(new _TestCaseFinished {
TestCaseUniqueID = "test-case-id-2"
});
sink.OnMessage(new _TestAssemblyFinished());
var @event = Assert.Single(events);
Assert.Equal(TimeSpan.FromSeconds(1), @event.ConfiguredLongRunningTime);
var receivedTestCasePair = Assert.Single(@event.TestCases);
Assert.Same(testCaseStarting1, receivedTestCasePair.Key);
Assert.Equal(TimeSpan.FromSeconds(1), receivedTestCasePair.Value);
}
public async ValueTask TestCaseCleanupFailure()
{
var caseStarting = new _TestCaseStarting
{
AssemblyUniqueID = assemblyID,
TestCaseUniqueID = testCaseID,
TestCaseDisplayName = "MyTestCase",
TestClassUniqueID = classID,
TestCollectionUniqueID = collectionID,
TestMethodUniqueID = methodID
};
var caseCleanupFailure = new _TestCaseCleanupFailure
{
AssemblyUniqueID = assemblyID,
ExceptionParentIndices = exceptionParentIndices,
ExceptionTypes = exceptionTypes,
Messages = messages,
StackTraces = stackTraces,
TestCaseUniqueID = testCaseID,
TestCollectionUniqueID = collectionID,
TestClassUniqueID = classID,
TestMethodUniqueID = methodID
};
var listener = Substitute.For <ITestListener>();
await using var sink = new ResultSink(listener, 42)
{
TestRunState = TestRunState.NoTests
};
sink.OnMessage(caseStarting);
sink.OnMessage(caseCleanupFailure);
AssertFailure(listener, sink.TestRunState, "Test Case Cleanup Failure (MyTestCase)");
}
public async void LongRunningTest_Once_WithDiagnosticMessageSink()
{
var events = new List <_DiagnosticMessage>();
var diagSink = Substitute.For <_IMessageSink>();
diagSink
.WhenForAnyArgs(x => x.OnMessage(null !))
.Do(callInfo =>
{
var message = callInfo.Arg <_MessageSinkMessage>();
if (message is _DiagnosticMessage diagnosticMessage)
{
events.Add(diagnosticMessage);
}
});
using var sink = new TestableDelegatingLongRunningTestDetectionSink(longRunningSeconds: 1, diagnosticMessageSink: diagSink);
var testCaseStarting = new _TestCaseStarting {
TestCaseDisplayName = "My test display name", TestCaseUniqueID = "test-case-id"
};
sink.OnMessage(new _TestAssemblyStarting());
sink.OnMessage(testCaseStarting);
await sink.AdvanceClockAsync(1500);
sink.OnMessage(new _TestCaseFinished {
TestCaseUniqueID = "test-case-id"
});
sink.OnMessage(new _TestAssemblyFinished());
var @event = Assert.Single(events);
Assert.Equal("[Long Running Test] 'My test display name', Elapsed: 00:00:01", @event.Message);
}
public async void LongRunningTest_Once_WithCallback()
{
var events = new List <LongRunningTestsSummary>();
using var sink = new TestableDelegatingLongRunningTestDetectionSink(longRunningSeconds: 1, callback: summary => events.Add(summary));
var testCaseStarting = new _TestCaseStarting {
TestCaseUniqueID = "test-case-id"
};
sink.OnMessage(new _TestAssemblyStarting());
sink.OnMessage(testCaseStarting);
await sink.AdvanceClockAsync(1500);
sink.OnMessage(new _TestCaseFinished {
TestCaseUniqueID = "test-case-id"
});
sink.OnMessage(new _TestAssemblyFinished());
var @event = Assert.Single(events);
Assert.Equal(TimeSpan.FromSeconds(1), @event.ConfiguredLongRunningTime);
var receivedTestCasePair = Assert.Single(@event.TestCases);
Assert.Same(testCaseStarting, receivedTestCasePair.Key);
Assert.Equal(TimeSpan.FromMilliseconds(1500), receivedTestCasePair.Value);
}
public void TestCaseCleanupFailure()
{
var caseStarting = new _TestCaseStarting
{
AssemblyUniqueID = assemblyID,
TestCaseUniqueID = testCaseID,
TestCaseDisplayName = "MyTestCase",
TestClassUniqueID = classID,
TestCollectionUniqueID = collectionID,
TestMethodUniqueID = methodID
};
var caseCleanupFailure = new _TestCaseCleanupFailure
{
AssemblyUniqueID = assemblyID,
ExceptionParentIndices = exceptionParentIndices,
ExceptionTypes = exceptionTypes,
Messages = messages,
StackTraces = stackTraces,
TestCaseUniqueID = testCaseID,
TestCollectionUniqueID = collectionID,
TestClassUniqueID = classID,
TestMethodUniqueID = methodID
};
var handler = TestableDefaultRunnerReporterMessageHandler.Create();
handler.OnMessage(caseStarting);
handler.OnMessage(caseCleanupFailure);
AssertFailureMessages(handler.Messages, "Test Case Cleanup Failure (MyTestCase)");
}
public void TestCaseCleanupFailure()
{
var caseStarting = new _TestCaseStarting
{
AssemblyUniqueID = assemblyID,
TestCaseUniqueID = testCaseID,
TestCaseDisplayName = "MyTestCase",
TestClassUniqueID = classID,
TestCollectionUniqueID = collectionID,
TestMethodUniqueID = methodID
};
var caseCleanupFailure = new _TestCaseCleanupFailure
{
AssemblyUniqueID = assemblyID,
ExceptionParentIndices = exceptionParentIndices,
ExceptionTypes = exceptionTypes,
Messages = messages,
StackTraces = stackTraces,
TestCaseUniqueID = testCaseID,
TestCollectionUniqueID = collectionID,
TestClassUniqueID = classID,
TestMethodUniqueID = methodID
};
var assemblyElement = new XElement("assembly");
var sink = new DelegatingXmlCreationSink(innerSink, assemblyElement);
sink.OnMessage(caseStarting);
sink.OnMessage(caseCleanupFailure);
AssertFailureElement(assemblyElement, "test-case-cleanup", "MyTestCase");
}
void SendTestCaseMessagesWhenAppropriate(Xunit1TestCase?current)
{
var results = TestClassResults;
if (current != lastTestCase && lastTestCase != null)
{
var testCaseFinished = new _TestCaseFinished
{
AssemblyUniqueID = lastTestCase.AssemblyUniqueID,
ExecutionTime = testCaseResults.Time,
TestCaseUniqueID = lastTestCase.TestCaseUniqueID,
TestClassUniqueID = lastTestCase.TestClassUniqueID,
TestCollectionUniqueID = lastTestCase.TestCollectionUniqueID,
TestMethodUniqueID = lastTestCase.TestMethodUniqueID,
TestsFailed = testCaseResults.Failed,
TestsRun = testCaseResults.Total,
TestsSkipped = testCaseResults.Skipped
};
results.Continue = messageSink.OnMessage(testCaseFinished) && results.Continue;
testMethodResults.Aggregate(testCaseResults);
testCaseResults.Reset();
if (current == null || lastTestCase.TestMethod != current.TestMethod)
{
var testMethodFinished = new _TestMethodFinished
{
AssemblyUniqueID = lastTestCase.AssemblyUniqueID,
ExecutionTime = testMethodResults.Time,
TestClassUniqueID = lastTestCase.TestClassUniqueID,
TestCollectionUniqueID = lastTestCase.TestCollectionUniqueID,
TestMethodUniqueID = lastTestCase.TestMethodUniqueID,
TestsFailed = testMethodResults.Failed,
TestsRun = testMethodResults.Total,
TestsSkipped = testMethodResults.Skipped
};
results.Continue = messageSink.OnMessage(testMethodFinished) && results.Continue;
testMethodResults.Reset();
}
}
if (current != lastTestCase && current != null)
{
// Dispatch TestMethodStarting if we've moved onto a new method
if (lastTestCase == null || lastTestCase.TestMethod != current.TestMethod)
{
var testMethodStarting = new _TestMethodStarting
{
AssemblyUniqueID = current.AssemblyUniqueID,
TestClassUniqueID = current.TestClassUniqueID,
TestCollectionUniqueID = current.TestCollectionUniqueID,
TestMethod = current.TestMethod,
TestMethodUniqueID = current.TestMethodUniqueID
};
results.Continue = messageSink.OnMessage(testMethodStarting) && results.Continue;
}
// Dispatch TestCaseStarting
var testCaseStarting = new _TestCaseStarting
{
AssemblyUniqueID = current.AssemblyUniqueID,
SkipReason = current.SkipReason,
SourceFilePath = current.SourceFilePath,
SourceLineNumber = current.SourceLineNumber,
TestCaseDisplayName = current.TestCaseDisplayName,
TestCaseUniqueID = current.TestCaseUniqueID,
TestClassUniqueID = current.TestClassUniqueID,
TestCollectionUniqueID = current.TestCollectionUniqueID,
TestMethodUniqueID = current.TestMethodUniqueID,
Traits = current.Traits
};
results.Continue = messageSink.OnMessage(testCaseStarting) && results.Continue;
}
lastTestCase = current;
}
/// <summary/>
public static RunSummary FailTestCases(
IReadOnlyCollection <_ITestCase> testCases,
IMessageBus messageBus,
string messageFormat)
{
var result = new RunSummary();
foreach (var testCase in testCases)
{
var assemblyUniqueID = testCase.TestCollection.TestAssembly.UniqueID;
var testUniqueID = UniqueIDGenerator.ForTest(testCase.UniqueID, -1);
var caseStarting = new _TestCaseStarting
{
AssemblyUniqueID = assemblyUniqueID,
SkipReason = testCase.SkipReason,
SourceFilePath = testCase.SourceFilePath,
SourceLineNumber = testCase.SourceLineNumber,
TestCaseDisplayName = testCase.TestCaseDisplayName,
TestCaseUniqueID = testCase.UniqueID,
TestClassName = testCase.TestClassName,
TestClassNamespace = testCase.TestClassNamespace,
TestClassNameWithNamespace = testCase.TestClassNameWithNamespace,
TestClassUniqueID = testCase.TestClass?.UniqueID,
TestCollectionUniqueID = testCase.TestCollection.UniqueID,
TestMethodName = testCase.TestMethod?.Method.Name,
TestMethodUniqueID = testCase.TestMethod?.UniqueID,
Traits = testCase.Traits
};
messageBus.QueueMessage(caseStarting);
var testStarting = new _TestStarting
{
AssemblyUniqueID = assemblyUniqueID,
TestCaseUniqueID = testCase.UniqueID,
TestClassUniqueID = testCase.TestClass?.UniqueID,
TestCollectionUniqueID = testCase.TestCollection.UniqueID,
TestDisplayName = testCase.TestCaseDisplayName,
TestMethodUniqueID = testCase.TestMethod?.UniqueID,
TestUniqueID = testUniqueID
};
messageBus.QueueMessage(testStarting);
var failed = new _TestFailed
{
AssemblyUniqueID = assemblyUniqueID,
Cause = FailureCause.Exception,
ExceptionParentIndices = new[] { -1 },
ExceptionTypes = new string?[] { null },
ExecutionTime = 0m,
Messages = new[] { string.Format(messageFormat, testCase.TestCaseDisplayName) },
Output = string.Empty,
StackTraces = new string?[] { null },
TestCaseUniqueID = testCase.UniqueID,
TestClassUniqueID = testCase.TestClass?.UniqueID,
TestCollectionUniqueID = testCase.TestCollection.UniqueID,
TestMethodUniqueID = testCase.TestMethod?.UniqueID,
TestUniqueID = testUniqueID
};
messageBus.QueueMessage(failed);
var testFinished = new _TestFinished
{
AssemblyUniqueID = assemblyUniqueID,
ExecutionTime = 0m,
Output = string.Empty,
TestCaseUniqueID = testCase.UniqueID,
TestClassUniqueID = testCase.TestClass?.UniqueID,
TestCollectionUniqueID = testCase.TestCollection.UniqueID,
TestMethodUniqueID = testCase.TestMethod?.UniqueID,
TestUniqueID = testUniqueID
};
messageBus.QueueMessage(testFinished);
var caseFinished = new _TestCaseFinished
{
AssemblyUniqueID = assemblyUniqueID,
ExecutionTime = 0m,
TestCaseUniqueID = testCase.UniqueID,
TestClassUniqueID = testCase.TestClass?.UniqueID,
TestCollectionUniqueID = testCase.TestCollection.UniqueID,
TestMethodUniqueID = testCase.TestMethod?.UniqueID,
TestsFailed = 1,
TestsRun = 1,
TestsSkipped = 0
};
messageBus.QueueMessage(caseFinished);
result.Total++;
result.Failed++;
}
return(result);
}
/// <summary>
/// Runs the tests in the test case.
/// </summary>
/// <returns>Returns summary information about the tests that were run.</returns>
protected async ValueTask <RunSummary> RunAsync(TContext ctxt)
{
await ctxt.InitializeAsync();
try
{
SetTestContext(ctxt, TestEngineStatus.Initializing);
var summary = new RunSummary();
var assemblyUniqueID = ctxt.TestCase.TestCollection.TestAssembly.UniqueID;
var testCollectionUniqueID = ctxt.TestCase.TestCollection.UniqueID;
var testClassUniqueID = ctxt.TestCase.TestClass?.UniqueID;
var testMethodUniqueID = ctxt.TestCase.TestMethod?.UniqueID;
var testCaseUniqueID = ctxt.TestCase.UniqueID;
var testCaseStarting = new _TestCaseStarting
{
AssemblyUniqueID = assemblyUniqueID,
SkipReason = ctxt.TestCase.SkipReason,
SourceFilePath = ctxt.TestCase.SourceFilePath,
SourceLineNumber = ctxt.TestCase.SourceLineNumber,
TestCaseDisplayName = ctxt.TestCase.TestCaseDisplayName,
TestCaseUniqueID = testCaseUniqueID,
TestClassUniqueID = testClassUniqueID,
TestCollectionUniqueID = testCollectionUniqueID,
TestMethodUniqueID = testMethodUniqueID,
Traits = ctxt.TestCase.Traits
};
if (!ctxt.MessageBus.QueueMessage(testCaseStarting))
{
ctxt.CancellationTokenSource.Cancel();
}
else
{
try
{
await AfterTestCaseStartingAsync(ctxt);
SetTestContext(ctxt, TestEngineStatus.Running);
summary = await RunTestsAsync(ctxt);
SetTestContext(ctxt, TestEngineStatus.CleaningUp);
ctxt.Aggregator.Clear();
await BeforeTestCaseFinishedAsync(ctxt);
if (ctxt.Aggregator.HasExceptions)
{
var testCaseCleanupFailure = _TestCaseCleanupFailure.FromException(
ctxt.Aggregator.ToException() !,
assemblyUniqueID,
testCollectionUniqueID,
testClassUniqueID,
testMethodUniqueID,
testCaseUniqueID
);
if (!ctxt.MessageBus.QueueMessage(testCaseCleanupFailure))
{
ctxt.CancellationTokenSource.Cancel();
}
}
}
finally
{
var testCaseFinished = new _TestCaseFinished
{
AssemblyUniqueID = assemblyUniqueID,
ExecutionTime = summary.Time,
TestCaseUniqueID = testCaseUniqueID,
TestClassUniqueID = testClassUniqueID,
TestCollectionUniqueID = testCollectionUniqueID,
TestMethodUniqueID = testMethodUniqueID,
TestsFailed = summary.Failed,
TestsRun = summary.Total,
TestsSkipped = summary.Skipped
};
if (!ctxt.MessageBus.QueueMessage(testCaseFinished))
{
ctxt.CancellationTokenSource.Cancel();
}
}
}
return(summary);
}
finally
{
await ctxt.DisposeAsync();
}
}
