Esempio n. 1
0
        public void ShouldProvideCaseReturnValuesToCustomBehaviors()
        {
            using (var console = new RedirectedConsole())
            {
                Convention
                    .CaseExecution
                    .Wrap<TreatBoolReturnValuesAsAssertions>();

                Run<SampleTestClass>();

                Listener.Entries.ShouldEqual(
                    "Fixie.Tests.Cases.NonVoidCaseTests+SampleTestClass.BoolFalse failed: Boolean test case returned false!",
                    "Fixie.Tests.Cases.NonVoidCaseTests+SampleTestClass.BoolThrow failed: 'BoolThrow' failed!",
                    "Fixie.Tests.Cases.NonVoidCaseTests+SampleTestClass.BoolTrue passed",
                    "Fixie.Tests.Cases.NonVoidCaseTests+SampleTestClass.Pass passed",
                    "Fixie.Tests.Cases.NonVoidCaseTests+SampleTestClass.Throw failed: 'Throw' failed!"
                    );

                console.Output.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)
                    .ShouldEqual(
                        "BoolFalse False",
                        "BoolThrow null",
                        "BoolTrue True",
                        "Pass null",
                        "Throw null");
            }
        }
Esempio n. 2
0
        public void ShouldUnpackResultValuesFromStronglyTypedTaskObjectsForAsyncCases()
        {
            using (var console = new RedirectedConsole())
            {
                Convention
                    .CaseExecution
                    .Wrap<TreatBoolReturnValuesAsAssertions>();

                Run<SampleAsyncTestClass>();

                Listener.Entries.ShouldEqual(
                    "Fixie.Tests.Cases.NonVoidCaseTests+SampleAsyncTestClass.BoolFalse failed: Boolean test case returned false!",
                    "Fixie.Tests.Cases.NonVoidCaseTests+SampleAsyncTestClass.BoolThrow failed: 'BoolThrow' failed!",
                    "Fixie.Tests.Cases.NonVoidCaseTests+SampleAsyncTestClass.BoolTrue passed.",
                    "Fixie.Tests.Cases.NonVoidCaseTests+SampleAsyncTestClass.Pass passed.",
                    "Fixie.Tests.Cases.NonVoidCaseTests+SampleAsyncTestClass.Throw failed: 'Throw' failed!"
                    );

                console.Output.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)
                    .ShouldEqual(
                        "BoolFalse False",
                        "BoolThrow null",
                        "BoolTrue True",
                        "Pass null",
                        "Throw null");
            }
        }
Esempio n. 3
0
        public void ShouldReportResultsToTheConsole()
        {
            using (var console = new RedirectedConsole())
            {
                var listener = new ConsoleListener();

                typeof(PassFailTestClass).Run(listener, SelfTestConvention.Build());

                var testClass = typeof(PassFailTestClass).FullName;

                console.Lines()
                       .Select(CleanBrittleValues)
                       .ShouldEqual(
                           "------ Testing Assembly Fixie.Tests.dll ------",
                           "Test '" + testClass + ".SkipA' skipped",
                           "Console.Out: FailA",
                           "Console.Error: FailA",
                           "Console.Out: FailB",
                           "Console.Error: FailB",
                           "Console.Out: PassA",
                           "Console.Error: PassA",
                           "Console.Out: PassB",
                           "Console.Error: PassB",
                           "Console.Out: PassC",
                           "Console.Error: PassC",

                           "Test '" + testClass + ".FailA' failed: Fixie.Tests.FailureException",
                           "'FailA' failed!",
                           "   at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #",
                           "Test '" + testClass + ".FailB' failed: Fixie.Tests.FailureException",
                           "'FailB' failed!",
                           "   at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #",
                           "3 passed, 2 failed, 1 skipped, took 1.23 seconds (Fixie 1.2.3.4).");
            }
        }
Esempio n. 4
0
        public void ShouldReportResultsToTestDrivenDotNet()
        {
            var testDriven = new StubTestListener();

            using (var console = new RedirectedConsole())
            {
                var listener = new TestDrivenListener(testDriven);

                var convention = SelfTestConvention.Build();
                convention.CaseExecution.Skip(x => x.Method.Has<SkipAttribute>(), x => x.Method.GetCustomAttribute<SkipAttribute>().Reason);
                convention.Parameters.Add<InputAttributeParameterSource>();

                typeof(PassFailTestClass).Run(listener, convention);

                var testClass = typeof(PassFailTestClass).FullName;

                console.Lines()
                    .ShouldEqual(
                        "Console.Out: Fail",
                        "Console.Error: Fail",
                        "Console.Out: Pass",
                        "Console.Error: Pass");

                var results = testDriven.TestResults;
                results.Count.ShouldEqual(4);

                foreach (var result in results)
                {
                    result.FixtureType.ShouldEqual(null);
                    result.Method.ShouldEqual(null);
                    result.TimeSpan.ShouldEqual(TimeSpan.Zero);
                    result.TotalTests.ShouldEqual(0);
                    result.TestRunnerName.ShouldBeNull();
                }

                results[0].Name.ShouldEqual(testClass + ".SkipWithReason");
                results[0].State.ShouldEqual(TestState.Ignored);
                results[0].Message.ShouldEqual("Skipped with reason.");
                results[0].StackTrace.ShouldBeNull();

                results[1].Name.ShouldEqual(testClass + ".SkipWithoutReason");
                results[1].State.ShouldEqual(TestState.Ignored);
                results[1].Message.ShouldBeNull();
                results[1].StackTrace.ShouldBeNull();

                results[2].Name.ShouldEqual(testClass + ".Fail");
                results[2].State.ShouldEqual(TestState.Failed);
                results[2].Message.ShouldEqual("Fixie.Tests.FailureException");
                results[2].StackTrace.Lines().Select(CleanBrittleValues).ShouldEqual(
                    "'Fail' failed!",
                    "   at Fixie.Tests.TestDriven.TestDrivenListenerTests.PassFailTestClass.Fail() in " + PathToThisFile() + ":line #");
                
                results[3].Name.ShouldEqual(testClass + ".Pass(123)");
                results[3].State.ShouldEqual(TestState.Passed);
                results[3].Message.ShouldBeNull();
                results[3].StackTrace.ShouldBeNull();
            }
        }
Esempio n. 5
0
        protected Output Run()
        {
            using (var console = new RedirectedConsole())
            {
                var listener = new StubListener();

                Convention.ClassExecution.SortCases((x, y) => String.Compare(y.Name, x.Name, StringComparison.Ordinal));

                typeof(SampleTestClass).Run(listener, Convention);

                return new Output(console.Lines().ToArray(), listener.Entries.ToArray());
            }
        }
Esempio n. 6
0
        public void ShouldReportResultsToTheConsoleInTeamCityFormat()
        {
            using (var console = new RedirectedConsole())
            using (var listener = new TeamCityListener())
            {
                var convention = SelfTestConvention.Build();
                convention.CaseExecution.Skip(x => x.Method.Has<SkipAttribute>(), x => x.Method.GetCustomAttribute<SkipAttribute>().Reason);

                typeof(PassFailTestClass).Run(listener, convention);

                var testClass = typeof(PassFailTestClass).FullName;

                console.Lines()
                       .Select(x => Regex.Replace(x, @":line \d+", ":line #")) //Avoid brittle assertion introduced by stack trace line numbers.
                       .Select(x => Regex.Replace(x, @"duration='\d+'", "duration='#'")) //Avoid brittle assertion introduced by durations.
                       .ShouldEqual(
                           "##teamcity[testSuiteStarted name='Fixie.Tests.dll']",
                           "##teamcity[testIgnored name='" + testClass + ".SkipWithReason' message='Skipped with reason.']",
                           "##teamcity[testIgnored name='" + testClass + ".SkipWithoutReason' message='']",

                           "Console.Out: FailA",
                           "Console.Error: FailA",
                           "Console.Out: FailB",
                           "Console.Error: FailB",
                           "Console.Out: PassA",
                           "Console.Error: PassA",
                           "Console.Out: PassB",
                           "Console.Error: PassB",
                           "Console.Out: PassC",
                           "Console.Error: PassC",

                           "##teamcity[testStarted name='"+testClass+".FailA']",
                           "##teamcity[testStdOut name='" + testClass + ".FailA' out='Console.Out: FailA|r|nConsole.Error: FailA|r|n']",
                           "##teamcity[testFailed name='" + testClass + ".FailA' message='|'FailA|' failed!' details='|'FailA|' failed!|r|n   at Fixie.Tests.ConsoleRunner.TeamCityListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #']",
                           "##teamcity[testFinished name='" + testClass + ".FailA' duration='#']",
                           "##teamcity[testStarted name='" + testClass + ".FailB']",
                           "##teamcity[testStdOut name='" + testClass + ".FailB' out='Console.Out: FailB|r|nConsole.Error: FailB|r|n']",
                           "##teamcity[testFailed name='" + testClass + ".FailB' message='|'FailB|' failed!' details='|'FailB|' failed!|r|n   at Fixie.Tests.ConsoleRunner.TeamCityListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #']",
                           "##teamcity[testFinished name='" + testClass + ".FailB' duration='#']",
                           "##teamcity[testStarted name='" + testClass + ".PassA']",
                           "##teamcity[testStdOut name='" + testClass + ".PassA' out='Console.Out: PassA|r|nConsole.Error: PassA|r|n']",
                           "##teamcity[testFinished name='" + testClass + ".PassA' duration='#']",
                           "##teamcity[testStarted name='" + testClass + ".PassB']",
                           "##teamcity[testStdOut name='" + testClass + ".PassB' out='Console.Out: PassB|r|nConsole.Error: PassB|r|n']",
                           "##teamcity[testFinished name='" + testClass + ".PassB' duration='#']",
                           "##teamcity[testStarted name='" + testClass + ".PassC']",
                           "##teamcity[testStdOut name='" + testClass + ".PassC' out='Console.Out: PassC|r|nConsole.Error: PassC|r|n']",
                           "##teamcity[testFinished name='" + testClass + ".PassC' duration='#']",
                           "##teamcity[testSuiteFinished name='Fixie.Tests.dll']");
            }
        }
Esempio n. 7
0
        private static object Execute(Assembly assembly)
        {
            using (var console = new RedirectedConsole())
            {
                var result = assembly.Execute();

                var writerResult = console.Output;

                if ((result == null || result == Core.Void.Value) && writerResult != "")
                    return writerResult;

                return result;
            }
        }
Esempio n. 8
0
        public void ShouldNotReportSkipCountsWhenZeroTestsHaveBeenSkipped()
        {
            using (var console = new RedirectedConsole())
            using (var listener = new ConsoleListener())
            {
                var convention = SelfTestConvention.Build();

                convention
                    .Methods
                    .Where(method => !method.Has<SkipAttribute>());

                typeof(PassFailTestClass).Run(listener, convention);

                var testClass = typeof(PassFailTestClass).FullName;

                console.Lines()
                       .Select(CleanBrittleValues)
                       .ShouldEqual(
                           "------ Testing Assembly Fixie.Tests.dll ------",
                           "",
                           "Console.Out: FailA",
                           "Console.Error: FailA",
                           "Console.Out: FailB",
                           "Console.Error: FailB",
                           "Console.Out: PassA",
                           "Console.Error: PassA",
                           "Console.Out: PassB",
                           "Console.Error: PassB",
                           "Console.Out: PassC",
                           "Console.Error: PassC",

                           "Test '" + testClass + ".FailA' failed: Fixie.Tests.FailureException",
                           "'FailA' failed!",
                           "   at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #",
                           "",
                           "Test '" + testClass + ".FailB' failed: Fixie.Tests.FailureException",
                           "'FailB' failed!",
                           "   at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #",
                           "",
                           "3 passed, 2 failed, took 1.23 seconds (" + Framework.Version + ").");
            }
        }
Esempio n. 9
0
        public void ShouldReportResultsToTheConsole()
        {
            using (var console = new RedirectedConsole())
            using (var listener = new ConsoleListener())
            {
                var convention = SelfTestConvention.Build();
                convention.CaseExecution.Skip(x => x.Method.Has<SkipAttribute>(), x => x.Method.GetCustomAttribute<SkipAttribute>().Reason);

                typeof(PassFailTestClass).Run(listener, convention);

                var testClass = typeof(PassFailTestClass).FullName;

                console.Lines()
                       .Select(CleanBrittleValues)
                       .ShouldEqual(
                           "------ Testing Assembly Fixie.Tests.dll ------",
                           "",
                           "Test '" + testClass + ".SkipWithReason' skipped: Skipped with reason.",
                           "Test '" + testClass + ".SkipWithoutReason' skipped",
                           "Console.Out: FailA",
                           "Console.Error: FailA",
                           "Console.Out: FailB",
                           "Console.Error: FailB",
                           "Console.Out: PassA",
                           "Console.Error: PassA",
                           "Console.Out: PassB",
                           "Console.Error: PassB",
                           "Console.Out: PassC",
                           "Console.Error: PassC",

                           "Test '" + testClass + ".FailA' failed: Fixie.Tests.FailureException",
                           "'FailA' failed!",
                           "   at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailA() in " + PathToThisFile() + ":line #",
                           "",
                           "Test '" + testClass + ".FailB' failed: Fixie.Tests.FailureException",
                           "'FailB' failed!",
                           "   at Fixie.Tests.ConsoleRunner.ConsoleListenerTests.PassFailTestClass.FailB() in " + PathToThisFile() + ":line #",
                           "",
                           "3 passed, 2 failed, 2 skipped, took 1.23 seconds (" + Framework.Version + ").");
            }
        }
Esempio n. 10
0
        public void ShouldIgnoreCaseReturnValuesByDefault()
        {
            using (var console = new RedirectedConsole())
            {
                Run<SampleTestClass>();
                Run<SampleAsyncTestClass>();

                Listener.Entries.ShouldEqual(
                    "Fixie.Tests.Cases.NonVoidCaseTests+SampleTestClass.BoolFalse passed.",
                    "Fixie.Tests.Cases.NonVoidCaseTests+SampleTestClass.BoolThrow failed: 'BoolThrow' failed!",
                    "Fixie.Tests.Cases.NonVoidCaseTests+SampleTestClass.BoolTrue passed.",
                    "Fixie.Tests.Cases.NonVoidCaseTests+SampleTestClass.Pass passed.",
                    "Fixie.Tests.Cases.NonVoidCaseTests+SampleTestClass.Throw failed: 'Throw' failed!",

                    "Fixie.Tests.Cases.NonVoidCaseTests+SampleAsyncTestClass.BoolFalse passed.",
                    "Fixie.Tests.Cases.NonVoidCaseTests+SampleAsyncTestClass.BoolThrow failed: 'BoolThrow' failed!",
                    "Fixie.Tests.Cases.NonVoidCaseTests+SampleAsyncTestClass.BoolTrue passed.",
                    "Fixie.Tests.Cases.NonVoidCaseTests+SampleAsyncTestClass.Pass passed.",
                    "Fixie.Tests.Cases.NonVoidCaseTests+SampleAsyncTestClass.Throw failed: 'Throw' failed!"
                    );

                console.Output.ShouldBeEmpty();
            }
        }
Esempio n. 11
0
        public void ShouldReportResultsToExecutionRecorder()
        {
            const string assemblyPath = "assembly.path.dll";
            var recorder = new StubExecutionRecorder();

            using (var console = new RedirectedConsole())
            using (var listener = new VisualStudioListener(recorder, assemblyPath))
            {
                var convention = SelfTestConvention.Build();
                convention.CaseExecution.Skip(x => x.Method.Has<SkipAttribute>(), x => x.Method.GetCustomAttribute<SkipAttribute>().Reason);
                convention.Parameters.Add<InputAttributeParameterSource>();

                typeof(PassFailTestClass).Run(listener, convention);

                var testClass = typeof(PassFailTestClass).FullName;

                console.Lines()
                    .ShouldEqual(
                        "Console.Out: Fail",
                        "Console.Error: Fail",
                        "Console.Out: Pass",
                        "Console.Error: Pass");

                var results = recorder.TestResults;
                results.Count.ShouldEqual(4);

                foreach (var result in results)
                {
                    result.Traits.ShouldBeEmpty();
                    result.Attachments.ShouldBeEmpty();
                    result.ComputerName.ShouldEqual(Environment.MachineName);
                    result.TestCase.Traits.ShouldBeEmpty();
                    result.TestCase.LocalExtensionData.ShouldBeNull();
                    result.TestCase.Source.ShouldEqual("assembly.path.dll");

                    //Source locations are a discovery-time concern.
                    result.TestCase.CodeFilePath.ShouldBeNull();
                    result.TestCase.LineNumber.ShouldEqual(-1);
                }

                results[0].TestCase.FullyQualifiedName.ShouldEqual(testClass +".SkipWithReason");
                results[0].TestCase.DisplayName.ShouldEqual(testClass +".SkipWithReason");
                results[0].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/");
                results[0].Outcome.ShouldEqual(TestOutcome.Skipped);
                results[0].ErrorMessage.ShouldEqual("Skipped with reason.");
                results[0].ErrorStackTrace.ShouldBeNull();
                results[0].DisplayName.ShouldEqual(testClass + ".SkipWithReason");
                results[0].Messages.ShouldBeEmpty();
                results[0].Duration.ShouldEqual(TimeSpan.Zero);

                results[1].TestCase.FullyQualifiedName.ShouldEqual(testClass +".SkipWithoutReason");
                results[1].TestCase.DisplayName.ShouldEqual(testClass +".SkipWithoutReason");
                results[1].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/");
                results[1].Outcome.ShouldEqual(TestOutcome.Skipped);
                results[1].ErrorMessage.ShouldBeNull();
                results[1].ErrorStackTrace.ShouldBeNull();
                results[1].DisplayName.ShouldEqual(testClass +".SkipWithoutReason");
                results[1].Messages.ShouldBeEmpty();
                results[1].Duration.ShouldEqual(TimeSpan.Zero);

                results[2].TestCase.FullyQualifiedName.ShouldEqual(testClass +".Fail");
                results[2].TestCase.DisplayName.ShouldEqual(testClass +".Fail");
                results[2].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/");
                results[2].Outcome.ShouldEqual(TestOutcome.Failed);
                results[2].ErrorMessage.ShouldEqual("Fixie.Tests.FailureException");
                results[2].ErrorStackTrace.Lines().Select(CleanBrittleValues)
                    .ShouldEqual(
                        "'Fail' failed!",
                        "   at Fixie.Tests.VisualStudio.TestAdapter.VisualStudioListenerTests.PassFailTestClass.Fail() in " + PathToThisFile() + ":line #");
                results[2].DisplayName.ShouldEqual(testClass + ".Fail");
                results[2].Messages.Count.ShouldEqual(1);
                results[2].Messages[0].Category.ShouldEqual(TestResultMessage.StandardOutCategory);
                results[2].Messages[0].Text.Lines().ShouldEqual("Console.Out: Fail", "Console.Error: Fail");
                results[2].Duration.ShouldBeGreaterThanOrEqualTo(TimeSpan.Zero);

                results[3].TestCase.FullyQualifiedName.ShouldEqual(testClass +".Pass");
                results[3].TestCase.DisplayName.ShouldEqual(testClass +".Pass");
                results[3].TestCase.ExecutorUri.ToString().ShouldEqual("executor://fixie.visualstudio/");
                results[3].Outcome.ShouldEqual(TestOutcome.Passed);
                results[3].ErrorMessage.ShouldBeNull();
                results[3].ErrorStackTrace.ShouldBeNull();
                results[3].DisplayName.ShouldEqual(testClass +".Pass(123)");
                results[3].Messages.Count.ShouldEqual(1);
                results[3].Messages[0].Category.ShouldEqual(TestResultMessage.StandardOutCategory);
                results[3].Messages[0].Text.Lines().ShouldEqual("Console.Out: Pass", "Console.Error: Pass");
                results[3].Duration.ShouldBeGreaterThanOrEqualTo(TimeSpan.Zero);
            }
        }
Esempio n. 12
0
        public ExecutionSummary Run(Type testClass, bool isOnlyTestClass)
        {
            var methods = methodDiscoverer.TestMethods(testClass);

            var summary = new ExecutionSummary();

            if (!methods.Any())
            {
                return(summary);
            }

            Start(testClass);

            var classStopwatch = Stopwatch.StartNew();

            var orderedMethods = OrderedMethods(methods, summary);

            bool classLifecycleFailed       = false;
            bool runCasesInvokedByLifecycle = false;

            try
            {
                Action <Action <Case> > runCases = caseLifecycle =>
                {
                    runCasesInvokedByLifecycle = true;

                    foreach (var @case in YieldCases(orderedMethods, summary))
                    {
                        Start(@case);

                        Exception caseLifecycleException = null;

                        string consoleOutput;
                        using (var console = new RedirectedConsole())
                        {
                            var caseStopwatch = Stopwatch.StartNew();

                            try
                            {
                                caseLifecycle(@case);
                            }
                            catch (Exception exception)
                            {
                                caseLifecycleException = exception;
                            }

                            caseStopwatch.Stop();

                            @case.Duration += caseStopwatch.Elapsed;

                            consoleOutput = console.Output;

                            @case.Output += consoleOutput;
                        }

                        Console.Write(consoleOutput);

                        var caseHasNormalResult = @case.State == CaseState.Failed || @case.State == CaseState.Passed;
                        var caseLifecycleFailed = caseLifecycleException != null;

                        if (caseHasNormalResult)
                        {
                            if (@case.State == CaseState.Failed)
                            {
                                Fail(@case, summary);
                            }
                            else if (!caseLifecycleFailed)
                            {
                                Pass(@case, summary);
                            }
                        }

                        if (caseLifecycleFailed)
                        {
                            Fail(new Case(@case, caseLifecycleException), summary);
                        }
                        else if (!caseHasNormalResult)
                        {
                            Skip(@case, summary);
                        }
                    }
                };

                var runContext = isOnlyTestClass && methods.Count == 1
                    ? new TestClass(testClass, runCases, methods.Single())
                    : new TestClass(testClass, runCases);

                execution.Execute(runContext);
            }
            catch (Exception exception)
            {
                classLifecycleFailed = true;
                foreach (var method in orderedMethods)
                {
                    Fail(method, exception, summary);
                }
            }

            if (!runCasesInvokedByLifecycle && !classLifecycleFailed)
            {
                //No cases ran, and we didn't already emit a general
                //failure for each method, so emit a general skip for
                //each method.
                foreach (var method in orderedMethods)
                {
                    var @case = new Case(method);
                    Skip(@case, summary);
                }
            }

            classStopwatch.Stop();
            Complete(testClass, summary, classStopwatch.Elapsed);

            return(summary);
        }
Esempio n. 13
0
        public void ShouldReportResultsToAppVeyorBuildWorkerApi()
        {
            var results = new List<AppVeyorListener.TestResult>();

            var httpClient = new HttpClient(new FakeHandler(request =>
            {
                request.ShouldNotBeNull();
                request.RequestUri.AbsoluteUri.ShouldEqual("http://localhost:4567/api/tests");
                request.Headers.Accept.ShouldContain(new MediaTypeWithQualityHeaderValue("application/json"));
                request.Content.Headers.ContentType.ToString().ShouldEqual("application/json; charset=utf-8");

                var requestContent = request.Content.ReadAsStringAsync().Result;
                results.Add(new JavaScriptSerializer().Deserialize<AppVeyorListener.TestResult>(requestContent));

                return new HttpResponseMessage { StatusCode = HttpStatusCode.Accepted };
            }));

            using (var console = new RedirectedConsole())
            using (var listener = new AppVeyorListener("http://localhost:4567", httpClient))
            {
                var convention = SelfTestConvention.Build();
                convention.CaseExecution.Skip(x => x.Method.Has<SkipAttribute>(), x => x.Method.GetCustomAttribute<SkipAttribute>().Reason);
                convention.Parameters.Add<InputAttributeParameterSource>();

                typeof(PassFailTestClass).Run(listener, convention);

                var testClass = typeof(PassFailTestClass).FullName;

                console.Lines()
                    .ShouldEqual(
                        "Console.Out: Fail",
                        "Console.Error: Fail",
                        "Console.Out: Pass",
                        "Console.Error: Pass");

                results.Count.ShouldEqual(4);

                foreach (var result in results)
                {
                    result.testFramework.ShouldEqual("Fixie");
                    result.fileName.ShouldEqual("Fixie.Tests.dll");
                }

                results[0].testName.ShouldEqual(testClass + ".SkipWithReason");
                results[0].outcome.ShouldEqual("Skipped");
                results[0].durationMilliseconds.ShouldEqual("0");
                results[0].ErrorMessage.ShouldEqual("Skipped with reason.");
                results[0].ErrorStackTrace.ShouldBeNull();
                results[0].StdOut.ShouldBeNull();

                results[1].testName.ShouldEqual(testClass + ".SkipWithoutReason");
                results[1].outcome.ShouldEqual("Skipped");
                results[1].durationMilliseconds.ShouldEqual("0");
                results[1].ErrorMessage.ShouldBeNull();
                results[1].ErrorStackTrace.ShouldBeNull();
                results[1].StdOut.ShouldBeNull();

                results[2].testName.ShouldEqual(testClass + ".Fail");
                results[2].outcome.ShouldEqual("Failed");
                int.Parse(results[2].durationMilliseconds).ShouldBeGreaterThanOrEqualTo(0);
                results[2].ErrorMessage.ShouldEqual("Fixie.Tests.FailureException");
                results[2].ErrorStackTrace.Lines().Select(CleanBrittleValues)
                    .ShouldEqual("'Fail' failed!",
                         "   at Fixie.Tests.ConsoleRunner.AppVeyorListenerTests.PassFailTestClass.Fail() in " + PathToThisFile() + ":line #");
                results[2].StdOut.Lines().ShouldEqual("Console.Out: Fail", "Console.Error: Fail");

                results[3].testName.ShouldEqual(testClass + ".Pass(123)");
                results[3].outcome.ShouldEqual("Passed");
                int.Parse(results[3].durationMilliseconds).ShouldBeGreaterThanOrEqualTo(0);
                results[3].ErrorMessage.ShouldBeNull();
                results[3].ErrorStackTrace.ShouldBeNull();
                results[3].StdOut.Lines().ShouldEqual("Console.Out: Pass", "Console.Error: Pass");
            }
        }