public static async void FailureInQueueOfTestCollectionStarting_DoesNotQueueTestCollectionFinished_DoesNotRunTestClasses()
{
var messages = new List <_MessageSinkMessage>();
var messageBus = Substitute.For <IMessageBus>();
messageBus
.QueueMessage(null !)
.ReturnsForAnyArgs(callInfo =>
{
var msg = callInfo.Arg <_MessageSinkMessage>();
messages.Add(msg);
if (msg is _TestCollectionStarting)
{
throw new InvalidOperationException();
}
return(true);
});
var runner = TestableTestCollectionRunner.Create(messageBus);
var ex = await Record.ExceptionAsync(() => runner.RunAsync());
Assert.IsType <InvalidOperationException>(ex);
var starting = Assert.Single(messages);
Assert.IsAssignableFrom <_TestCollectionStarting>(starting);
Assert.Empty(runner.ClassesRun);
}
public static async void TestsAreGroupedByCollection()
{
var passing1 = Mocks.TestCase <ClassUnderTest>("Passing");
var other1 = Mocks.TestCase <ClassUnderTest>("Other");
var passing2 = Mocks.TestCase <ClassUnderTest2>("Passing");
var other2 = Mocks.TestCase <ClassUnderTest2>("Other");
var runner = TestableTestCollectionRunner.Create(testCases: new[] { passing1, passing2, other2, other1 });
await runner.RunAsync();
Assert.Collection(
runner.ClassesRun,
tuple =>
{
Assert.Equal("TestCollectionRunnerTests+ClassUnderTest", tuple.Item1?.Name);
Assert.Collection(tuple.Item2,
testCase => Assert.Same(passing1, testCase),
testCase => Assert.Same(other1, testCase)
);
},
tuple =>
{
Assert.Equal("TestCollectionRunnerTests+ClassUnderTest2", tuple.Item1?.Name);
Assert.Collection(tuple.Item2,
testCase => Assert.Same(passing2, testCase),
testCase => Assert.Same(other2, testCase)
);
}
);
}
public static async void TestContextInspection()
{
var runner = TestableTestCollectionRunner.Create();
await runner.RunAsync();
Assert.NotNull(runner.AfterTestCollectionStarting_Context);
Assert.Equal(TestEngineStatus.Running, runner.AfterTestCollectionStarting_Context.TestAssemblyStatus);
Assert.Equal(TestEngineStatus.Initializing, runner.AfterTestCollectionStarting_Context.TestCollectionStatus);
Assert.Equal(TestPipelineStage.TestCollectionExecution, runner.AfterTestCollectionStarting_Context.PipelineStage);
Assert.Null(runner.AfterTestCollectionStarting_Context.TestClassStatus);
Assert.Null(runner.AfterTestCollectionStarting_Context.TestMethodStatus);
Assert.Null(runner.AfterTestCollectionStarting_Context.TestCaseStatus);
Assert.Null(runner.AfterTestCollectionStarting_Context.TestStatus);
Assert.Same(runner.TestCollection, runner.AfterTestCollectionStarting_Context.TestCollection);
Assert.NotNull(runner.RunTestClassAsync_Context);
Assert.Equal(TestEngineStatus.Running, runner.RunTestClassAsync_Context.TestAssemblyStatus);
Assert.Equal(TestEngineStatus.Running, runner.RunTestClassAsync_Context.TestCollectionStatus);
Assert.Null(runner.RunTestClassAsync_Context.TestClassStatus);
Assert.Null(runner.RunTestClassAsync_Context.TestMethodStatus);
Assert.Null(runner.RunTestClassAsync_Context.TestCaseStatus);
Assert.Null(runner.RunTestClassAsync_Context.TestStatus);
Assert.Same(runner.TestCollection, runner.RunTestClassAsync_Context.TestCollection);
Assert.NotNull(runner.BeforeTestCollectionFinished_Context);
Assert.Equal(TestEngineStatus.Running, runner.BeforeTestCollectionFinished_Context.TestAssemblyStatus);
Assert.Equal(TestEngineStatus.CleaningUp, runner.BeforeTestCollectionFinished_Context.TestCollectionStatus);
Assert.Null(runner.BeforeTestCollectionFinished_Context.TestClassStatus);
Assert.Null(runner.BeforeTestCollectionFinished_Context.TestMethodStatus);
Assert.Null(runner.BeforeTestCollectionFinished_Context.TestCaseStatus);
Assert.Null(runner.BeforeTestCollectionFinished_Context.TestStatus);
Assert.Same(runner.TestCollection, runner.BeforeTestCollectionFinished_Context.TestCollection);
}
public static async void Messages()
{
var summary = new RunSummary {
Total = 4, Failed = 2, Skipped = 1, Time = 21.12m
};
var messageBus = new SpyMessageBus();
var testCase = Mocks.TestCase <ClassUnderTest>("Passing");
var runner = TestableTestCollectionRunner.Create(messageBus, new[] { testCase }, summary);
var result = await runner.RunAsync();
Assert.Equal(result.Total, summary.Total);
Assert.Equal(result.Failed, summary.Failed);
Assert.Equal(result.Skipped, summary.Skipped);
Assert.Equal(result.Time, summary.Time);
Assert.False(runner.TokenSource.IsCancellationRequested);
Assert.Collection(messageBus.Messages,
msg =>
{
var starting = Assert.IsAssignableFrom <ITestCollectionStarting>(msg);
Assert.Same(testCase.TestMethod.TestClass.TestCollection, starting.TestCollection);
},
msg =>
{
var finished = Assert.IsAssignableFrom <ITestCollectionFinished>(msg);
Assert.Same(testCase.TestMethod.TestClass.TestCollection, finished.TestCollection);
Assert.Equal(21.12m, finished.ExecutionTime);
Assert.Equal(4, finished.TestsRun);
Assert.Equal(2, finished.TestsFailed);
Assert.Equal(1, finished.TestsSkipped);
}
);
}
public static async void RunTestClassAsync_AggregatorIncludesPassedInExceptions()
{
var messageBus = new SpyMessageBus();
var ex = new DivideByZeroException();
var runner = TestableTestCollectionRunner.Create(messageBus, aggregatorSeedException: ex);
await runner.RunAsync();
Assert.Same(ex, runner.RunTestClassAsync_AggregatorResult);
Assert.Empty(messageBus.Messages.OfType <_TestCollectionCleanupFailure>());
}
public static async void Cancellation_TestCollectionCleanupFailure_SetsCancellationToken()
{
var messageBus = new SpyMessageBus(msg => !(msg is _TestCollectionCleanupFailure));
var runner = TestableTestCollectionRunner.Create(messageBus);
runner.BeforeTestCollectionFinished_Callback = aggregator => aggregator.Add(new Exception());
await runner.RunAsync();
Assert.True(runner.TokenSource.IsCancellationRequested);
}
public static async void Cancellation_TestCollectionFinished_CallsExtensibilityCallbacks()
{
var messageBus = new SpyMessageBus(msg => !(msg is _TestCollectionFinished));
var runner = TestableTestCollectionRunner.Create(messageBus);
await runner.RunAsync();
Assert.True(runner.TokenSource.IsCancellationRequested);
Assert.True(runner.AfterTestCollectionStarting_Called);
Assert.True(runner.BeforeTestCollectionFinished_Called);
}
public static async void SignalingCancellationStopsRunningClasses()
{
var passing1 = Mocks.TestCase <ClassUnderTest>("Passing");
var passing2 = Mocks.TestCase <ClassUnderTest2>("Passing");
var runner = TestableTestCollectionRunner.Create(testCases: new[] { passing1, passing2 }, cancelInRunTestClassAsync: true);
await runner.RunAsync();
var tuple = Assert.Single(runner.ClassesRun);
Assert.Equal("TestCollectionRunnerTests+ClassUnderTest", tuple.Item1?.Name);
}
public static async void FailureInAfterTestCollectionStarting_GivesErroredAggregatorToTestClassRunner_NoCleanupFailureMessage()
{
var messageBus = new SpyMessageBus();
var runner = TestableTestCollectionRunner.Create(messageBus);
var ex = new DivideByZeroException();
runner.AfterTestCollectionStarting_Callback = aggregator => aggregator.Add(ex);
await runner.RunAsync();
Assert.Same(ex, runner.RunTestClassAsync_AggregatorResult);
Assert.Empty(messageBus.Messages.OfType <_TestCollectionCleanupFailure>());
}
public static async void FailureInBeforeTestCollectionFinished_ReportsCleanupFailure_DoesNotIncludeExceptionsFromAfterTestCollectionStarting()
{
var messageBus = new SpyMessageBus();
var testCases = new[] { Mocks.TestCase <TestAssemblyRunnerTests.RunAsync>("Messages") };
var runner = TestableTestCollectionRunner.Create(messageBus, testCases);
var startingException = new DivideByZeroException();
var finishedException = new InvalidOperationException();
runner.AfterTestCollectionStarting_Callback = aggregator => aggregator.Add(startingException);
runner.BeforeTestCollectionFinished_Callback = aggregator => aggregator.Add(finishedException);
await runner.RunAsync();
var cleanupFailure = Assert.Single(messageBus.Messages.OfType <_TestCollectionCleanupFailure>());
Assert.Equal(typeof(InvalidOperationException).FullName, cleanupFailure.ExceptionTypes.Single());
}
