public static async void FailureInQueueOfTestStarting_DoesNotQueueTestFinished_DoesNotInvokeTest() { var messages = new List <IMessageSinkMessage>(); var messageBus = Substitute.For <IMessageBus>(); messageBus .QueueMessage(null) .ReturnsForAnyArgs(callInfo => { var msg = callInfo.Arg <IMessageSinkMessage>(); messages.Add(msg); if (msg is ITestStarting) { throw new InvalidOperationException(); } return(true); }); var runner = TestableTestRunner.Create(messageBus); await Assert.ThrowsAsync <InvalidOperationException>(() => runner.RunAsync()); var starting = Assert.Single(messages); Assert.IsAssignableFrom <ITestStarting>(starting); Assert.False(runner.InvokeTestAsync_Called); }
public static async void Messages() { var messageBus = new SpyMessageBus(); var runner = TestableTestRunner.Create(messageBus, displayName: "Display Name", runTime: 21.12m); var result = await runner.RunAsync(); Assert.Equal(21.12m, result.Time); Assert.False(runner.TokenSource.IsCancellationRequested); Assert.Collection(messageBus.Messages, msg => { var testStarting = Assert.IsAssignableFrom <ITestStarting>(msg); Assert.Same(runner.TestCase.TestMethod.TestClass.TestCollection, testStarting.TestCollection); Assert.Same(runner.TestCase, testStarting.TestCase); Assert.Equal("Display Name", testStarting.Test.DisplayName); }, msg => { }, // Pass/fail/skip, will be tested elsewhere msg => { var testFinished = Assert.IsAssignableFrom <ITestFinished>(msg); Assert.Same(runner.TestCase.TestMethod.TestClass.TestCollection, testFinished.TestCollection); Assert.Same(runner.TestCase, testFinished.TestCase); Assert.Equal("Display Name", testFinished.Test.DisplayName); Assert.Equal(21.12m, testFinished.ExecutionTime); Assert.Empty(testFinished.Output); } ); }
public static async void Cancellation_TestCleanupFailure_SetsCancellationToken() { var messageBus = new SpyMessageBus(msg => !(msg is ITestCleanupFailure)); var runner = TestableTestRunner.Create(messageBus); runner.BeforeTestFinished_Callback = aggregator => aggregator.Add(new Exception()); await runner.RunAsync(); Assert.True(runner.TokenSource.IsCancellationRequested); }
public static async void Cancellation_AllOthers_CallsExtensibilityMethods(Type messageTypeToCancelOn, bool shouldTestPass, string skipReason = null) { var messageBus = new SpyMessageBus(msg => !(messageTypeToCancelOn.IsAssignableFrom(msg.GetType()))); var runner = TestableTestRunner.Create(messageBus, skipReason: skipReason, lambda: () => Assert.True(shouldTestPass)); await runner.RunAsync(); Assert.True(runner.TokenSource.IsCancellationRequested); Assert.True(runner.AfterTestStarting_Called); Assert.True(runner.BeforeTestFinished_Called); }
public static async void Cancellation_TestStarting_DoesNotCallExtensibilityMethods() { var messageBus = new SpyMessageBus(msg => !(msg is ITestStarting)); var runner = TestableTestRunner.Create(messageBus); await runner.RunAsync(); Assert.True(runner.TokenSource.IsCancellationRequested); Assert.False(runner.AfterTestStarting_Called); Assert.False(runner.BeforeTestFinished_Called); }
public static async void Output() { var messageBus = new SpyMessageBus(); var runner = TestableTestRunner.Create(messageBus, output: "This is my text output"); await runner.RunAsync(); var passed = messageBus.Messages.OfType <ITestPassed>().Single(); Assert.Equal("This is my text output", passed.Output); }
public static async void WithPreSeededException_ReturnsTestFailed_NoCleanupFailureMessage() { var messageBus = new SpyMessageBus(); var ex = new DivideByZeroException(); var runner = TestableTestRunner.Create(messageBus, aggregatorSeedException: ex); await runner.RunAsync(); var failed = Assert.Single(messageBus.Messages.OfType <ITestFailed>()); Assert.Equal(typeof(DivideByZeroException).FullName, failed.ExceptionTypes.Single()); Assert.Empty(messageBus.Messages.OfType <ITestCleanupFailure>()); }
public static async void FailureInAfterTestStarting_ReturnsTestFailed_NoCleanupFailureMessage() { var messageBus = new SpyMessageBus(); var runner = TestableTestRunner.Create(messageBus); var ex = new DivideByZeroException(); runner.AfterTestStarting_Callback = aggregator => aggregator.Add(ex); await runner.RunAsync(); var failed = Assert.Single(messageBus.Messages.OfType <ITestFailed>()); Assert.Equal(typeof(DivideByZeroException).FullName, failed.ExceptionTypes.Single()); Assert.Empty(messageBus.Messages.OfType <ITestCleanupFailure>()); }
public static async void FailureInBeforeTestFinished_ReportsCleanupFailure_DoesNotIncludeExceptionsFromAfterTestStarting() { var messageBus = new SpyMessageBus(); var testCase = Mocks.TestCase <TestAssemblyRunnerTests.RunAsync>("Messages"); var runner = TestableTestRunner.Create(messageBus, testCase); var startingException = new DivideByZeroException(); var finishedException = new InvalidOperationException(); runner.AfterTestStarting_Callback = aggregator => aggregator.Add(startingException); runner.BeforeTestFinished_Callback = aggregator => aggregator.Add(finishedException); await runner.RunAsync(); var cleanupFailure = Assert.Single(messageBus.Messages.OfType <_TestCleanupFailure>()); Assert.Equal(typeof(InvalidOperationException).FullName, cleanupFailure.ExceptionTypes.Single()); }
public static async void Failing() { var messageBus = new SpyMessageBus(); var runner = TestableTestRunner.Create(messageBus, displayName: "Display Name", runTime: 21.12m, lambda: () => Assert.True(false)); var result = await runner.RunAsync(); // Direct run summary Assert.Equal(1, result.Total); Assert.Equal(1, result.Failed); Assert.Equal(0, result.Skipped); Assert.Equal(21.12m, result.Time); // Fail message var failed = messageBus.Messages.OfType <_TestFailed>().Single(); var failedStarting = Assert.Single(messageBus.Messages.OfType <_TestStarting>().Where(s => s.TestUniqueID == failed.TestUniqueID)); Assert.Equal("Display Name", failedStarting.TestDisplayName); Assert.Equal(21.12m, failed.ExecutionTime); Assert.Empty(failed.Output); Assert.Equal("Xunit.Sdk.TrueException", failed.ExceptionTypes.Single()); // Statuses Assert.NotNull(runner.AfterTestStarting_Context); Assert.Equal(TestEngineStatus.Running, runner.AfterTestStarting_Context.TestAssemblyStatus); Assert.Equal(TestEngineStatus.Running, runner.AfterTestStarting_Context.TestCollectionStatus); Assert.Equal(TestEngineStatus.Running, runner.AfterTestStarting_Context.TestClassStatus); Assert.Equal(TestEngineStatus.Running, runner.AfterTestStarting_Context.TestMethodStatus); Assert.Equal(TestEngineStatus.Running, runner.AfterTestStarting_Context.TestCaseStatus); Assert.Equal(TestEngineStatus.Initializing, runner.AfterTestStarting_Context.TestStatus); Assert.NotNull(runner.BeforeTestFinished_Context); Assert.Equal(TestEngineStatus.Running, runner.BeforeTestFinished_Context.TestAssemblyStatus); Assert.Equal(TestEngineStatus.Running, runner.BeforeTestFinished_Context.TestCollectionStatus); Assert.Equal(TestEngineStatus.Running, runner.BeforeTestFinished_Context.TestClassStatus); Assert.Equal(TestEngineStatus.Running, runner.BeforeTestFinished_Context.TestMethodStatus); Assert.Equal(TestEngineStatus.Running, runner.BeforeTestFinished_Context.TestCaseStatus); Assert.Equal(TestEngineStatus.CleaningUp, runner.BeforeTestFinished_Context.TestStatus); Assert.NotNull(runner.BeforeTestFinished_Context.TestState); Assert.Equal(failed.Messages, runner.BeforeTestFinished_Context.TestState.ExceptionMessages); Assert.Equal(failed.ExceptionParentIndices, runner.BeforeTestFinished_Context.TestState.ExceptionParentIndices); Assert.Equal(failed.StackTraces, runner.BeforeTestFinished_Context.TestState.ExceptionStackTraces); Assert.Equal(failed.ExceptionTypes, runner.BeforeTestFinished_Context.TestState.ExceptionTypes); Assert.Equal(21.12m, runner.BeforeTestFinished_Context.TestState.ExecutionTime); Assert.Equal(FailureCause.Assertion, runner.BeforeTestFinished_Context.TestState.FailureCause); Assert.Equal(TestResult.Failed, runner.BeforeTestFinished_Context.TestState.Result); Assert.NotNull(runner.BeforeTestFinished_Context.TestOutputHelper); Assert.Equal(string.Empty, runner.BeforeTestFinished_Context.TestOutputHelper.Output); }
public static async void Passing() { var messageBus = new SpyMessageBus(); var runner = TestableTestRunner.Create(messageBus, displayName: "Display Name", runTime: 21.12m); var result = await runner.RunAsync(); // Direct run summary Assert.Equal(1, result.Total); Assert.Equal(0, result.Failed); Assert.Equal(0, result.Skipped); Assert.Equal(21.12m, result.Time); // Pass message var passed = messageBus.Messages.OfType <_TestPassed>().Single(); Assert.Equal(21.12m, passed.ExecutionTime); Assert.Empty(passed.Output); }
public static async void Passing() { var messageBus = new SpyMessageBus(); var runner = TestableTestRunner.Create(messageBus, displayName: "Display Name", runTime: 21.12m); var result = await runner.RunAsync(); // Direct run summary Assert.Equal(1, result.Total); Assert.Equal(0, result.Failed); Assert.Equal(0, result.Skipped); Assert.Equal(21.12m, result.Time); // Pass message var passed = messageBus.Messages.OfType <_TestPassed>().Single(); Assert.Equal(21.12m, passed.ExecutionTime); Assert.Empty(passed.Output); // Statuses Assert.NotNull(runner.AfterTestStarting_Context); Assert.Equal(TestEngineStatus.Running, runner.AfterTestStarting_Context.TestAssemblyStatus); Assert.Equal(TestEngineStatus.Running, runner.AfterTestStarting_Context.TestCollectionStatus); Assert.Equal(TestEngineStatus.Running, runner.AfterTestStarting_Context.TestClassStatus); Assert.Equal(TestEngineStatus.Running, runner.AfterTestStarting_Context.TestMethodStatus); Assert.Equal(TestEngineStatus.Running, runner.AfterTestStarting_Context.TestCaseStatus); Assert.Equal(TestEngineStatus.Initializing, runner.AfterTestStarting_Context.TestStatus); Assert.Equal(TestPipelineStage.TestExecution, runner.AfterTestStarting_Context.PipelineStage); Assert.NotNull(runner.BeforeTestFinished_Context); Assert.Equal(TestEngineStatus.Running, runner.BeforeTestFinished_Context.TestAssemblyStatus); Assert.Equal(TestEngineStatus.Running, runner.BeforeTestFinished_Context.TestCollectionStatus); Assert.Equal(TestEngineStatus.Running, runner.BeforeTestFinished_Context.TestClassStatus); Assert.Equal(TestEngineStatus.Running, runner.BeforeTestFinished_Context.TestMethodStatus); Assert.Equal(TestEngineStatus.Running, runner.BeforeTestFinished_Context.TestCaseStatus); Assert.Equal(TestEngineStatus.CleaningUp, runner.BeforeTestFinished_Context.TestStatus); Assert.NotNull(runner.BeforeTestFinished_Context.TestState); Assert.Null(runner.BeforeTestFinished_Context.TestState.ExceptionMessages); Assert.Null(runner.BeforeTestFinished_Context.TestState.ExceptionParentIndices); Assert.Null(runner.BeforeTestFinished_Context.TestState.ExceptionStackTraces); Assert.Null(runner.BeforeTestFinished_Context.TestState.ExceptionTypes); Assert.Equal(21.12m, runner.BeforeTestFinished_Context.TestState.ExecutionTime); Assert.Null(runner.BeforeTestFinished_Context.TestState.FailureCause); Assert.Equal(TestResult.Passed, runner.BeforeTestFinished_Context.TestState.Result); Assert.NotNull(runner.BeforeTestFinished_Context.TestOutputHelper); Assert.Equal(string.Empty, runner.BeforeTestFinished_Context.TestOutputHelper.Output); }
public static async void Skipping() { var messageBus = new SpyMessageBus(); var runner = TestableTestRunner.Create(messageBus, displayName: "Display Name", skipReason: "Please don't run me", runTime: 21.12m, lambda: () => Assert.True(false)); var result = await runner.RunAsync(); // Direct run summary Assert.Equal(1, result.Total); Assert.Equal(0, result.Failed); Assert.Equal(1, result.Skipped); Assert.Equal(0m, result.Time); // Skip message var skipped = Assert.Single(messageBus.Messages.OfType <_TestSkipped>()); var skippedStarting = Assert.Single(messageBus.Messages.OfType <_TestStarting>().Where(s => s.TestUniqueID == skipped.TestUniqueID)); Assert.Equal("Display Name", skippedStarting.TestDisplayName); Assert.Equal(0m, skipped.ExecutionTime); Assert.Empty(skipped.Output); Assert.Equal("Please don't run me", skipped.Reason); }
public static async void Failing() { var messageBus = new SpyMessageBus(); var runner = TestableTestRunner.Create(messageBus, displayName: "Display Name", runTime: 21.12m, lambda: () => Assert.True(false)); var result = await runner.RunAsync(); // Direct run summary Assert.Equal(1, result.Total); Assert.Equal(1, result.Failed); Assert.Equal(0, result.Skipped); Assert.Equal(21.12m, result.Time); // Fail message var failed = messageBus.Messages.OfType <_TestFailed>().Single(); var failedStarting = Assert.Single(messageBus.Messages.OfType <_TestStarting>().Where(s => s.TestUniqueID == failed.TestUniqueID)); Assert.Equal("Display Name", failedStarting.TestDisplayName); Assert.Equal(21.12m, failed.ExecutionTime); Assert.Empty(failed.Output); Assert.Equal("Xunit.Sdk.TrueException", failed.ExceptionTypes.Single()); }
public static async void Skipping() { var messageBus = new SpyMessageBus(); var runner = TestableTestRunner.Create(messageBus, displayName: "Display Name", skipReason: "Please don't run me", runTime: 21.12m, lambda: () => Assert.True(false)); var result = await runner.RunAsync(); // Direct run summary Assert.Equal(1, result.Total); Assert.Equal(0, result.Failed); Assert.Equal(1, result.Skipped); Assert.Equal(0m, result.Time); // Skip message var failed = messageBus.Messages.OfType <ITestSkipped>().Single(); Assert.Same(runner.TestCase.TestMethod.TestClass.TestCollection, failed.TestCollection); Assert.Same(runner.TestCase, failed.TestCase); Assert.Equal("Display Name", failed.Test.DisplayName); Assert.Equal(0m, failed.ExecutionTime); Assert.Empty(failed.Output); Assert.Equal("Please don't run me", failed.Reason); }