Пример #1
0
    public static async void FailureInQueueOfTestStarting_DoesNotQueueTestFinished_DoesNotInvokeTest()
    {
        var messages   = new List <IMessageSinkMessage>();
        var messageBus = Substitute.For <IMessageBus>();

        messageBus
        .QueueMessage(null)
        .ReturnsForAnyArgs(callInfo =>
        {
            var msg = callInfo.Arg <IMessageSinkMessage>();
            messages.Add(msg);

            if (msg is ITestStarting)
            {
                throw new InvalidOperationException();
            }

            return(true);
        });
        var runner = TestableTestRunner.Create(messageBus);

        await Assert.ThrowsAsync <InvalidOperationException>(() => runner.RunAsync());

        var starting = Assert.Single(messages);

        Assert.IsAssignableFrom <ITestStarting>(starting);
        Assert.False(runner.InvokeTestAsync_Called);
    }
Пример #2
0
    public static async void Messages()
    {
        var messageBus = new SpyMessageBus();
        var runner     = TestableTestRunner.Create(messageBus, displayName: "Display Name", runTime: 21.12m);

        var result = await runner.RunAsync();

        Assert.Equal(21.12m, result.Time);
        Assert.False(runner.TokenSource.IsCancellationRequested);
        Assert.Collection(messageBus.Messages,
                          msg =>
        {
            var testStarting = Assert.IsAssignableFrom <ITestStarting>(msg);
            Assert.Same(runner.TestCase.TestMethod.TestClass.TestCollection, testStarting.TestCollection);
            Assert.Same(runner.TestCase, testStarting.TestCase);
            Assert.Equal("Display Name", testStarting.Test.DisplayName);
        },
                          msg => { }, // Pass/fail/skip, will be tested elsewhere
                          msg =>
        {
            var testFinished = Assert.IsAssignableFrom <ITestFinished>(msg);
            Assert.Same(runner.TestCase.TestMethod.TestClass.TestCollection, testFinished.TestCollection);
            Assert.Same(runner.TestCase, testFinished.TestCase);
            Assert.Equal("Display Name", testFinished.Test.DisplayName);
            Assert.Equal(21.12m, testFinished.ExecutionTime);
            Assert.Empty(testFinished.Output);
        }
                          );
    }
Пример #3
0
    public static async void Cancellation_TestCleanupFailure_SetsCancellationToken()
    {
        var messageBus = new SpyMessageBus(msg => !(msg is ITestCleanupFailure));
        var runner     = TestableTestRunner.Create(messageBus);

        runner.BeforeTestFinished_Callback = aggregator => aggregator.Add(new Exception());

        await runner.RunAsync();

        Assert.True(runner.TokenSource.IsCancellationRequested);
    }
Пример #4
0
    public static async void Cancellation_AllOthers_CallsExtensibilityMethods(Type messageTypeToCancelOn, bool shouldTestPass, string skipReason = null)
    {
        var messageBus = new SpyMessageBus(msg => !(messageTypeToCancelOn.IsAssignableFrom(msg.GetType())));
        var runner     = TestableTestRunner.Create(messageBus, skipReason: skipReason, lambda: () => Assert.True(shouldTestPass));

        await runner.RunAsync();

        Assert.True(runner.TokenSource.IsCancellationRequested);
        Assert.True(runner.AfterTestStarting_Called);
        Assert.True(runner.BeforeTestFinished_Called);
    }
Пример #5
0
    public static async void Cancellation_TestStarting_DoesNotCallExtensibilityMethods()
    {
        var messageBus = new SpyMessageBus(msg => !(msg is ITestStarting));
        var runner     = TestableTestRunner.Create(messageBus);

        await runner.RunAsync();

        Assert.True(runner.TokenSource.IsCancellationRequested);
        Assert.False(runner.AfterTestStarting_Called);
        Assert.False(runner.BeforeTestFinished_Called);
    }
Пример #6
0
    public static async void Output()
    {
        var messageBus = new SpyMessageBus();
        var runner     = TestableTestRunner.Create(messageBus, output: "This is my text output");

        await runner.RunAsync();

        var passed = messageBus.Messages.OfType <ITestPassed>().Single();

        Assert.Equal("This is my text output", passed.Output);
    }
Пример #7
0
    public static async void WithPreSeededException_ReturnsTestFailed_NoCleanupFailureMessage()
    {
        var messageBus = new SpyMessageBus();
        var ex         = new DivideByZeroException();
        var runner     = TestableTestRunner.Create(messageBus, aggregatorSeedException: ex);

        await runner.RunAsync();

        var failed = Assert.Single(messageBus.Messages.OfType <ITestFailed>());

        Assert.Equal(typeof(DivideByZeroException).FullName, failed.ExceptionTypes.Single());
        Assert.Empty(messageBus.Messages.OfType <ITestCleanupFailure>());
    }
Пример #8
0
    public static async void FailureInAfterTestStarting_ReturnsTestFailed_NoCleanupFailureMessage()
    {
        var messageBus = new SpyMessageBus();
        var runner     = TestableTestRunner.Create(messageBus);
        var ex         = new DivideByZeroException();

        runner.AfterTestStarting_Callback = aggregator => aggregator.Add(ex);

        await runner.RunAsync();

        var failed = Assert.Single(messageBus.Messages.OfType <ITestFailed>());

        Assert.Equal(typeof(DivideByZeroException).FullName, failed.ExceptionTypes.Single());
        Assert.Empty(messageBus.Messages.OfType <ITestCleanupFailure>());
    }
Пример #9
0
    public static async void FailureInBeforeTestFinished_ReportsCleanupFailure_DoesNotIncludeExceptionsFromAfterTestStarting()
    {
        var messageBus        = new SpyMessageBus();
        var testCase          = Mocks.TestCase <TestAssemblyRunnerTests.RunAsync>("Messages");
        var runner            = TestableTestRunner.Create(messageBus, testCase);
        var startingException = new DivideByZeroException();
        var finishedException = new InvalidOperationException();

        runner.AfterTestStarting_Callback  = aggregator => aggregator.Add(startingException);
        runner.BeforeTestFinished_Callback = aggregator => aggregator.Add(finishedException);

        await runner.RunAsync();

        var cleanupFailure = Assert.Single(messageBus.Messages.OfType <_TestCleanupFailure>());

        Assert.Equal(typeof(InvalidOperationException).FullName, cleanupFailure.ExceptionTypes.Single());
    }
Пример #10
0
    public static async void Failing()
    {
        var messageBus = new SpyMessageBus();
        var runner     = TestableTestRunner.Create(messageBus, displayName: "Display Name", runTime: 21.12m, lambda: () => Assert.True(false));

        var result = await runner.RunAsync();

        // Direct run summary
        Assert.Equal(1, result.Total);
        Assert.Equal(1, result.Failed);
        Assert.Equal(0, result.Skipped);
        Assert.Equal(21.12m, result.Time);
        // Fail message
        var failed         = messageBus.Messages.OfType <_TestFailed>().Single();
        var failedStarting = Assert.Single(messageBus.Messages.OfType <_TestStarting>().Where(s => s.TestUniqueID == failed.TestUniqueID));

        Assert.Equal("Display Name", failedStarting.TestDisplayName);
        Assert.Equal(21.12m, failed.ExecutionTime);
        Assert.Empty(failed.Output);
        Assert.Equal("Xunit.Sdk.TrueException", failed.ExceptionTypes.Single());
        // Statuses
        Assert.NotNull(runner.AfterTestStarting_Context);
        Assert.Equal(TestEngineStatus.Running, runner.AfterTestStarting_Context.TestAssemblyStatus);
        Assert.Equal(TestEngineStatus.Running, runner.AfterTestStarting_Context.TestCollectionStatus);
        Assert.Equal(TestEngineStatus.Running, runner.AfterTestStarting_Context.TestClassStatus);
        Assert.Equal(TestEngineStatus.Running, runner.AfterTestStarting_Context.TestMethodStatus);
        Assert.Equal(TestEngineStatus.Running, runner.AfterTestStarting_Context.TestCaseStatus);
        Assert.Equal(TestEngineStatus.Initializing, runner.AfterTestStarting_Context.TestStatus);
        Assert.NotNull(runner.BeforeTestFinished_Context);
        Assert.Equal(TestEngineStatus.Running, runner.BeforeTestFinished_Context.TestAssemblyStatus);
        Assert.Equal(TestEngineStatus.Running, runner.BeforeTestFinished_Context.TestCollectionStatus);
        Assert.Equal(TestEngineStatus.Running, runner.BeforeTestFinished_Context.TestClassStatus);
        Assert.Equal(TestEngineStatus.Running, runner.BeforeTestFinished_Context.TestMethodStatus);
        Assert.Equal(TestEngineStatus.Running, runner.BeforeTestFinished_Context.TestCaseStatus);
        Assert.Equal(TestEngineStatus.CleaningUp, runner.BeforeTestFinished_Context.TestStatus);
        Assert.NotNull(runner.BeforeTestFinished_Context.TestState);
        Assert.Equal(failed.Messages, runner.BeforeTestFinished_Context.TestState.ExceptionMessages);
        Assert.Equal(failed.ExceptionParentIndices, runner.BeforeTestFinished_Context.TestState.ExceptionParentIndices);
        Assert.Equal(failed.StackTraces, runner.BeforeTestFinished_Context.TestState.ExceptionStackTraces);
        Assert.Equal(failed.ExceptionTypes, runner.BeforeTestFinished_Context.TestState.ExceptionTypes);
        Assert.Equal(21.12m, runner.BeforeTestFinished_Context.TestState.ExecutionTime);
        Assert.Equal(FailureCause.Assertion, runner.BeforeTestFinished_Context.TestState.FailureCause);
        Assert.Equal(TestResult.Failed, runner.BeforeTestFinished_Context.TestState.Result);
        Assert.NotNull(runner.BeforeTestFinished_Context.TestOutputHelper);
        Assert.Equal(string.Empty, runner.BeforeTestFinished_Context.TestOutputHelper.Output);
    }
Пример #11
0
    public static async void Passing()
    {
        var messageBus = new SpyMessageBus();
        var runner     = TestableTestRunner.Create(messageBus, displayName: "Display Name", runTime: 21.12m);

        var result = await runner.RunAsync();

        // Direct run summary
        Assert.Equal(1, result.Total);
        Assert.Equal(0, result.Failed);
        Assert.Equal(0, result.Skipped);
        Assert.Equal(21.12m, result.Time);
        // Pass message
        var passed = messageBus.Messages.OfType <_TestPassed>().Single();

        Assert.Equal(21.12m, passed.ExecutionTime);
        Assert.Empty(passed.Output);
    }
Пример #12
0
    public static async void Passing()
    {
        var messageBus = new SpyMessageBus();
        var runner     = TestableTestRunner.Create(messageBus, displayName: "Display Name", runTime: 21.12m);

        var result = await runner.RunAsync();

        // Direct run summary
        Assert.Equal(1, result.Total);
        Assert.Equal(0, result.Failed);
        Assert.Equal(0, result.Skipped);
        Assert.Equal(21.12m, result.Time);
        // Pass message
        var passed = messageBus.Messages.OfType <_TestPassed>().Single();

        Assert.Equal(21.12m, passed.ExecutionTime);
        Assert.Empty(passed.Output);
        // Statuses
        Assert.NotNull(runner.AfterTestStarting_Context);
        Assert.Equal(TestEngineStatus.Running, runner.AfterTestStarting_Context.TestAssemblyStatus);
        Assert.Equal(TestEngineStatus.Running, runner.AfterTestStarting_Context.TestCollectionStatus);
        Assert.Equal(TestEngineStatus.Running, runner.AfterTestStarting_Context.TestClassStatus);
        Assert.Equal(TestEngineStatus.Running, runner.AfterTestStarting_Context.TestMethodStatus);
        Assert.Equal(TestEngineStatus.Running, runner.AfterTestStarting_Context.TestCaseStatus);
        Assert.Equal(TestEngineStatus.Initializing, runner.AfterTestStarting_Context.TestStatus);
        Assert.Equal(TestPipelineStage.TestExecution, runner.AfterTestStarting_Context.PipelineStage);
        Assert.NotNull(runner.BeforeTestFinished_Context);
        Assert.Equal(TestEngineStatus.Running, runner.BeforeTestFinished_Context.TestAssemblyStatus);
        Assert.Equal(TestEngineStatus.Running, runner.BeforeTestFinished_Context.TestCollectionStatus);
        Assert.Equal(TestEngineStatus.Running, runner.BeforeTestFinished_Context.TestClassStatus);
        Assert.Equal(TestEngineStatus.Running, runner.BeforeTestFinished_Context.TestMethodStatus);
        Assert.Equal(TestEngineStatus.Running, runner.BeforeTestFinished_Context.TestCaseStatus);
        Assert.Equal(TestEngineStatus.CleaningUp, runner.BeforeTestFinished_Context.TestStatus);
        Assert.NotNull(runner.BeforeTestFinished_Context.TestState);
        Assert.Null(runner.BeforeTestFinished_Context.TestState.ExceptionMessages);
        Assert.Null(runner.BeforeTestFinished_Context.TestState.ExceptionParentIndices);
        Assert.Null(runner.BeforeTestFinished_Context.TestState.ExceptionStackTraces);
        Assert.Null(runner.BeforeTestFinished_Context.TestState.ExceptionTypes);
        Assert.Equal(21.12m, runner.BeforeTestFinished_Context.TestState.ExecutionTime);
        Assert.Null(runner.BeforeTestFinished_Context.TestState.FailureCause);
        Assert.Equal(TestResult.Passed, runner.BeforeTestFinished_Context.TestState.Result);
        Assert.NotNull(runner.BeforeTestFinished_Context.TestOutputHelper);
        Assert.Equal(string.Empty, runner.BeforeTestFinished_Context.TestOutputHelper.Output);
    }
Пример #13
0
    public static async void Skipping()
    {
        var messageBus = new SpyMessageBus();
        var runner     = TestableTestRunner.Create(messageBus, displayName: "Display Name", skipReason: "Please don't run me", runTime: 21.12m, lambda: () => Assert.True(false));

        var result = await runner.RunAsync();

        // Direct run summary
        Assert.Equal(1, result.Total);
        Assert.Equal(0, result.Failed);
        Assert.Equal(1, result.Skipped);
        Assert.Equal(0m, result.Time);
        // Skip message
        var skipped         = Assert.Single(messageBus.Messages.OfType <_TestSkipped>());
        var skippedStarting = Assert.Single(messageBus.Messages.OfType <_TestStarting>().Where(s => s.TestUniqueID == skipped.TestUniqueID));

        Assert.Equal("Display Name", skippedStarting.TestDisplayName);
        Assert.Equal(0m, skipped.ExecutionTime);
        Assert.Empty(skipped.Output);
        Assert.Equal("Please don't run me", skipped.Reason);
    }
Пример #14
0
    public static async void Failing()
    {
        var messageBus = new SpyMessageBus();
        var runner     = TestableTestRunner.Create(messageBus, displayName: "Display Name", runTime: 21.12m, lambda: () => Assert.True(false));

        var result = await runner.RunAsync();

        // Direct run summary
        Assert.Equal(1, result.Total);
        Assert.Equal(1, result.Failed);
        Assert.Equal(0, result.Skipped);
        Assert.Equal(21.12m, result.Time);
        // Fail message
        var failed         = messageBus.Messages.OfType <_TestFailed>().Single();
        var failedStarting = Assert.Single(messageBus.Messages.OfType <_TestStarting>().Where(s => s.TestUniqueID == failed.TestUniqueID));

        Assert.Equal("Display Name", failedStarting.TestDisplayName);
        Assert.Equal(21.12m, failed.ExecutionTime);
        Assert.Empty(failed.Output);
        Assert.Equal("Xunit.Sdk.TrueException", failed.ExceptionTypes.Single());
    }
Пример #15
0
    public static async void Skipping()
    {
        var messageBus = new SpyMessageBus();
        var runner     = TestableTestRunner.Create(messageBus, displayName: "Display Name", skipReason: "Please don't run me", runTime: 21.12m, lambda: () => Assert.True(false));

        var result = await runner.RunAsync();

        // Direct run summary
        Assert.Equal(1, result.Total);
        Assert.Equal(0, result.Failed);
        Assert.Equal(1, result.Skipped);
        Assert.Equal(0m, result.Time);
        // Skip message
        var failed = messageBus.Messages.OfType <ITestSkipped>().Single();

        Assert.Same(runner.TestCase.TestMethod.TestClass.TestCollection, failed.TestCollection);
        Assert.Same(runner.TestCase, failed.TestCase);
        Assert.Equal("Display Name", failed.Test.DisplayName);
        Assert.Equal(0m, failed.ExecutionTime);
        Assert.Empty(failed.Output);
        Assert.Equal("Please don't run me", failed.Reason);
    }