Esempio n. 1
0
        internal static void RunTestProject(this XunitProject project, bool displaySuccess, bool displayFailureStack)
        {
            var mate = new MultiAssemblyTestEnvironment();
            var tests = project.Assemblies.Select(x => mate.Load(x.AssemblyFilename, x.ConfigFilename, x.ShadowCopy));

            var totalAssemblies = 0;
            var totalTests = 0;
            var totalFailures = 0;
            var totalSkips = 0;
            var totalTime = 0.0;

            foreach (var test in tests)
            {
                Console.WriteLine();
                Console.WriteLine("Test assembly: {0}", test.AssemblyFilename);
                Console.WriteLine();

                try
                {
                    var methods = test.EnumerateTestMethods(project.Filters.Filter).ToList();

                    if (!methods.Any())
                    {
                        Console.WriteLine("Skipping assembly (no tests match the specified filter).");
                    }
                    else
                    {
                        var callback = new DefaultRunnerCallback(displaySuccess, displayFailureStack, methods.Count);
                        test.Run(methods, callback);

                        ++totalAssemblies;
                        totalTests += callback.TotalTests;
                        totalFailures += callback.TotalFailures;
                        totalSkips += callback.TotalSkips;
                        totalTime += callback.TotalTime;
                    }
                }
                catch (ArgumentException ex)
                {
                    Console.WriteLine(ex.Message.Red());
                }

                mate.Unload(test);
            }

            if (totalAssemblies <= 1)
            {
                return;
            }

            Console.WriteLine();
            Console.WriteLine("=== {0} total, {1} failed, {2} skipped, took {3} seconds ===",
                totalTests, totalFailures, totalSkips, totalTime.ToString("0.000", CultureInfo.InvariantCulture));
        }
    public void MultiAssemblyAcceptanceTest()
    {
        string code =
            @"
                using System;
                using Xunit;

                public class MockTestClass
                {
                    [Fact]
                    public void SuccessTest()
                    {
                        Assert.Equal(2, 2);
                    }
                }
            ";

        using (MockAssembly mockAssembly1 = new MockAssembly())
        using (MockAssembly mockAssembly2 = new MockAssembly())
        {
            mockAssembly1.Compile(code);
            mockAssembly2.Compile(code);
            Mock<ITestMethodRunnerCallback> callback = new Mock<ITestMethodRunnerCallback>();
            callback.Setup(c => c.TestStart(It.IsAny<TestMethod>())).Returns(true);
            callback.Setup(c => c.TestFinished(It.IsAny<TestMethod>())).Returns(true);
            MultiAssemblyTestEnvironment mate = new MultiAssemblyTestEnvironment();
            mate.Load(mockAssembly1.FileName);
            mate.Load(mockAssembly2.FileName);
            TestAssembly testAssembly1 = mate.EnumerateTestAssemblies().Where(a => a.AssemblyFilename == mockAssembly1.FileName).Single();
            TestAssembly testAssembly2 = mate.EnumerateTestAssemblies().Where(a => a.AssemblyFilename == mockAssembly2.FileName).Single();
            TestMethod assembly1Method = testAssembly1.EnumerateTestMethods().Single();
            TestMethod assembly2Method = testAssembly1.EnumerateTestMethods().Single();

            mate.Run(mate.EnumerateTestMethods(), callback.Object);

            callback.Verify(c => c.AssemblyStart(testAssembly1));
            callback.Verify(c => c.TestStart(assembly1Method));
            callback.Verify(c => c.TestFinished(assembly1Method));
            callback.Verify(c => c.AssemblyFinished(testAssembly1, 1, 0, 0, It.IsAny<double>()));
            callback.Verify(c => c.AssemblyStart(testAssembly2));
            callback.Verify(c => c.TestStart(assembly2Method));
            callback.Verify(c => c.TestFinished(assembly2Method));
            callback.Verify(c => c.AssemblyFinished(testAssembly2, 1, 0, 0, It.IsAny<double>()));
        }
    }
Esempio n. 3
0
        public XmlNode RunWithMate()
        {
            var mate = new MultiAssemblyTestEnvironment();
            mate.Load(FileName);

            string xml = mate.Run(mate.EnumerateTestMethods(), new Callback());

            var doc = new XmlDocument();
            doc.LoadXml(xml);

            // ChildNodes[0] == <assemblies>
            // ChildNodes[0].ChildNodes[0] == first <assembly> node
            return doc.ChildNodes[0].ChildNodes[0];
        }
Esempio n. 4
0
        static int RunProject(XunitProject project, bool teamcity, bool silent, ParallelXunitCommandLine parallelXunitCommandLine)
        {
            if (!parallelXunitCommandLine.Start.HasValue || !parallelXunitCommandLine.End.HasValue)
                return -1;

            int totalAssemblies = 0;
            int totalTests = 0;
            int totalFailures = 0;
            int totalSkips = 0;
            double totalTime = 0;

            var mate = new MultiAssemblyTestEnvironment();

            foreach (XunitProjectAssembly assembly in project.Assemblies)
            {
                parallelXunitCommandLine.SetOutputArguments(assembly.Output);

                TestAssembly testAssembly = mate.Load(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy);
                List<IResultXmlTransform> transforms = TransformFactory.GetAssemblyTransforms(assembly);

                try
                {
                    var start = parallelXunitCommandLine.Start.Value;
                    var end = parallelXunitCommandLine.End.Value;
                    var methods = new List<TestMethod>(testAssembly.EnumerateTestMethods(project.Filters.Filter)).Skip(start).Take(end - start).ToList();
                    if (methods.Count == 0)
                    {
                        Console.WriteLine("Skipping assembly (no tests match the specified filter).");
                        continue;
                    }

                    var callback = CreateCallback(teamcity, silent, methods.Count);
                    var assemblyXml = testAssembly.Run(methods, callback);

                    ++totalAssemblies;
                    totalTests += callback.TotalTests;
                    totalFailures += callback.TotalFailures;
                    totalSkips += callback.TotalSkips;
                    totalTime += callback.TotalTime;

                    ParallelXunitCommandLine.WriteResultFile(assembly.AssemblyFilename, totalTests, totalFailures, totalSkips, totalTime, start, end);

                    foreach (var transform in transforms)
                        transform.Transform(assemblyXml);
                }
                catch (ArgumentException ex)
                {
                    Console.WriteLine(ex.Message);
                }

                mate.Unload(testAssembly);
            }

            if (!teamcity && totalAssemblies > 1)
            {
                Console.WriteLine();
                Console.WriteLine("=== {0} total, {1} failed, {2} skipped, took {3} seconds ===",
                                   totalTests, totalFailures, totalSkips, totalTime.ToString("0.000", CultureInfo.InvariantCulture));
            }

            return totalFailures;
        }
Esempio n. 5
0
        static int RunProject(XunitProject project, bool teamcity, bool silent, string[] args)
        {
            var defaultArgs = args.Aggregate((a, b) => a + " " + b);

            var mate = new MultiAssemblyTestEnvironment();
            foreach (XunitProjectAssembly assembly in project.Assemblies)
            {
                TestAssembly testAssembly = mate.Load(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy);
                var methods = new List<TestMethod>(testAssembly.EnumerateTestMethods(project.Filters.Filter));
                var methodsCount = methods.Count;

                ParallelXunitCommandLine.DeleteResultFiles(assembly.AssemblyFilename);

                using (var waiter = new ProcessWaiter())
                {
                    waiter.DefaultArgs = defaultArgs;
                    waiter.TeamCity = teamcity;
                    waiter.Silent = silent;

                    Console.WriteLine("Running with {0} threads.", waiter.MaximumNumberOfProcesses);
                    Console.WriteLine("Total number of tests: {0}.", methodsCount);

                    IParallelTestMethodRunnerCallback callback = CreateCallback(teamcity, silent, methodsCount);
                    callback.ParentAssemblyStart(testAssembly);

                    TestBatch batch;
                    var batcher = new TestMethodBatcher(methodsCount);
                    while ((batch = batcher.GetNextBatch()).Size > 0)
                    {
                        waiter.StartProcess(batch);
                    }

                    if (!teamcity)
                    {
                        waiter.PrintSummary();
                    }
                    else
                    {
                        waiter.WaitForAllProcesses();
                    }

                    var testResult = ParallelXunitCommandLine.AggregateTestResults(assembly.AssemblyFilename);
                    callback.ParentAssemblyFinished(testAssembly, testResult.Total, testResult.Failed, testResult.Skiped);

                    mate.Unload(testAssembly);
                }
            }

            return 0;
        }
        public void NullFilterThrows()
        {
            var mate = new MultiAssemblyTestEnvironment();

            Exception ex = Record.Exception(() => mate.EnumerateTestMethods(null).ToList());

            Assert.IsType<ArgumentNullException>(ex);
        }
    public void MultiAssemblyGetTraitsAcceptanceTest()
    {
        string code1 =
            @"
                using System;
                using Xunit;

                public class MockTestClass
                {
                    [Fact]
                    [Trait(""Trait1"", ""Value1"")]
                    public void Value1Test()
                    {
                    }

                    [Fact]
                    [Trait(""Trait1"", ""Value2"")]
                    public void Value2Test()
                    {
                    }

                    [Fact]
                    [Trait(""Trait2"", ""Value1"")]
                    public void Trait2Value1Test()
                    {
                    }
                }
            ";

        string code2 =
            @"
                using System;
                using Xunit;

                public class MockTestClass
                {
                    [Fact]
                    [Trait(""Trait1"", ""Value1"")]
                    public void OtherTest1()
                    {
                    }

                    [Fact]
                    [Trait(""Trait3"", ""Value42"")]
                    public void Crazy()
                    {
                    }
                }
            ";

        using (MockAssembly mockAssembly1 = new MockAssembly())
        using (MockAssembly mockAssembly2 = new MockAssembly())
        {
            mockAssembly1.Compile(code1);
            mockAssembly2.Compile(code2);
            MultiAssemblyTestEnvironment mate = new MultiAssemblyTestEnvironment();
            mate.Load(mockAssembly1.FileName);
            mate.Load(mockAssembly2.FileName);

            MultiValueDictionary<string, string> result = mate.EnumerateTraits();

            var trait1 = result["Trait1"];
            Assert.Equal(2, trait1.Count());
            Assert.Contains("Value1", trait1);
            Assert.Contains("Value2", trait1);
            var trait2 = result["Trait2"];
            Assert.Single(trait2);
            Assert.Contains("Value1", trait2);
            var trait3 = result["Trait3"];
            Assert.Single(trait3);
            Assert.Contains("Value42", trait3);
        }
    }
    public void SingleAssemblyAcceptanceTest()
    {
        string code =
            @"
                using System;
                using Xunit;

                public class MockTestClass
                {
                    [Fact]
                    public void SuccessTest()
                    {
                        Assert.Equal(2, 2);
                    }
                    [Fact]
                    public void FailureTest()
                    {
                        Assert.Equal(2, 3);
                    }
                    [Fact(Skip=""I'm too lazy to run today"")]
                    public void SkippingTest()
                    {
                        Assert.Equal(2, 4);
                    }
                }
            ";

        using (MockAssembly mockAssembly = new MockAssembly())
        {
            mockAssembly.Compile(code);
            Mock<ITestMethodRunnerCallback> callback = new Mock<ITestMethodRunnerCallback>();
            callback.Setup(c => c.TestStart(It.IsAny<TestMethod>())).Returns(true);
            callback.Setup(c => c.TestFinished(It.IsAny<TestMethod>())).Returns(true);
            MultiAssemblyTestEnvironment mate = new MultiAssemblyTestEnvironment();
            mate.Load(mockAssembly.FileName);
            TestAssembly testAssembly = mate.EnumerateTestAssemblies().Single();
            TestMethod passingMethod = testAssembly.EnumerateTestMethods().Where(m => m.MethodName == "SuccessTest").Single();
            TestMethod failingMethod = testAssembly.EnumerateTestMethods().Where(m => m.MethodName == "FailureTest").Single();
            TestMethod skippedMethod = testAssembly.EnumerateTestMethods().Where(m => m.MethodName == "SkippingTest").Single();

            mate.Run(mate.EnumerateTestMethods(), callback.Object);

            callback.Verify(c => c.AssemblyStart(testAssembly));
            callback.Verify(c => c.TestStart(passingMethod));
            callback.Verify(c => c.TestFinished(passingMethod));
            callback.Verify(c => c.TestStart(failingMethod));
            callback.Verify(c => c.TestFinished(failingMethod));
            callback.Verify(c => c.TestStart(skippedMethod), Times.Never());
            callback.Verify(c => c.TestFinished(skippedMethod));
            callback.Verify(c => c.AssemblyFinished(testAssembly, 3, 1, 1, It.IsAny<double>()));
            var passingMethodResult = Assert.IsType<TestPassedResult>(passingMethod.RunResults[0]);
            Assert.Null(passingMethodResult.Output);
            var failingMethodResult = Assert.IsType<TestFailedResult>(failingMethod.RunResults[0]);
            Assert.Null(failingMethodResult.Output);
            Assert.Equal("Xunit.Sdk.EqualException", failingMethodResult.ExceptionType);
            var skippedMethodResult = Assert.IsType<TestSkippedResult>(skippedMethod.RunResults[0]);
        }
    }
    public void MultiAssemblyTraitFilterAcceptanceTest()
    {
        string code =
            @"
                using System;
                using Xunit;

                public class MockTestClass
                {
                    [Fact]
                    [Trait(""Trait1"", ""Value1"")]
                    public void Value1Test()
                    {
                    }

                    [Fact]
                    [Trait(""Trait1"", ""Value2"")]
                    public void Value2Test()
                    {
                    }
                }
            ";

        using (MockAssembly mockAssembly1 = new MockAssembly())
        using (MockAssembly mockAssembly2 = new MockAssembly())
        {
            mockAssembly1.Compile(code);
            mockAssembly2.Compile(code);
            Mock<ITestMethodRunnerCallback> callback = new Mock<ITestMethodRunnerCallback>();
            callback.Setup(c => c.TestStart(It.IsAny<TestMethod>())).Returns(true);
            callback.Setup(c => c.TestFinished(It.IsAny<TestMethod>())).Returns(true);
            MultiAssemblyTestEnvironment mate = new MultiAssemblyTestEnvironment();
            mate.Load(mockAssembly1.FileName);
            mate.Load(mockAssembly2.FileName);
            TestAssembly testAssembly1 = mate.EnumerateTestAssemblies().Where(a => a.AssemblyFilename == mockAssembly1.FileName).Single();
            TestAssembly testAssembly2 = mate.EnumerateTestAssemblies().Where(a => a.AssemblyFilename == mockAssembly2.FileName).Single();
            TestMethod assembly1Value1Method = testAssembly1.EnumerateTestMethods().Where(m => m.MethodName == "Value1Test").Single();
            TestMethod assembly1Value2Method = testAssembly1.EnumerateTestMethods().Where(m => m.MethodName == "Value2Test").Single();
            TestMethod assembly2Value1Method = testAssembly2.EnumerateTestMethods().Where(m => m.MethodName == "Value1Test").Single();
            TestMethod assembly2Value2Method = testAssembly2.EnumerateTestMethods().Where(m => m.MethodName == "Value2Test").Single();

            mate.Run(mate.EnumerateTestMethods(m => m.Traits["Trait1"].FirstOrDefault() == "Value1"), callback.Object);

            callback.Verify(c => c.TestStart(assembly1Value1Method));
            callback.Verify(c => c.TestStart(assembly1Value2Method), Times.Never());
            callback.Verify(c => c.TestStart(assembly2Value1Method));
            callback.Verify(c => c.TestStart(assembly2Value2Method), Times.Never());
        }
    }
    public void MultiAssemblySearchFilterAcceptanceTest()
    {
        string code =
            @"
                using System;
                using Xunit;

                public class MockTestClass
                {
                    [Fact]
                    public void Test1()
                    {
                    }

                    [Fact]
                    public void Test2()
                    {
                    }
                }
            ";

        using (MockAssembly mockAssembly1 = new MockAssembly())
        using (MockAssembly mockAssembly2 = new MockAssembly())
        {
            mockAssembly1.Compile(code);
            mockAssembly2.Compile(code);
            Mock<ITestMethodRunnerCallback> callback = new Mock<ITestMethodRunnerCallback>();
            callback.Setup(c => c.TestStart(It.IsAny<TestMethod>())).Returns(true);
            callback.Setup(c => c.TestFinished(It.IsAny<TestMethod>())).Returns(true);
            MultiAssemblyTestEnvironment mate = new MultiAssemblyTestEnvironment();
            mate.Load(mockAssembly1.FileName);
            mate.Load(mockAssembly2.FileName);
            TestAssembly testAssembly1 = mate.EnumerateTestAssemblies().Where(a => a.AssemblyFilename == mockAssembly1.FileName).Single();
            TestAssembly testAssembly2 = mate.EnumerateTestAssemblies().Where(a => a.AssemblyFilename == mockAssembly2.FileName).Single();
            TestMethod assembly1Test1Method = testAssembly1.EnumerateTestMethods().Where(m => m.MethodName == "Test1").Single();
            TestMethod assembly1Test2Method = testAssembly1.EnumerateTestMethods().Where(m => m.MethodName == "Test2").Single();
            TestMethod assembly2Test1Method = testAssembly2.EnumerateTestMethods().Where(m => m.MethodName == "Test1").Single();
            TestMethod assembly2Test2Method = testAssembly2.EnumerateTestMethods().Where(m => m.MethodName == "Test2").Single();

            mate.Run(mate.EnumerateTestMethods(m => m.MethodName.Contains("t2")), callback.Object);

            callback.Verify(c => c.TestStart(assembly1Test1Method), Times.Never());
            callback.Verify(c => c.TestStart(assembly1Test2Method));
            callback.Verify(c => c.TestStart(assembly2Test1Method), Times.Never());
            callback.Verify(c => c.TestStart(assembly2Test2Method));
        }
    }
Esempio n. 11
0
        static int RunProject(XunitProject project, bool teamcity, bool silent, string[] args)
        {
            var defaultArgs = args.Aggregate((a, b) => a + " " + b);

            var mate = new MultiAssemblyTestEnvironment();
            foreach (XunitProjectAssembly assembly in project.Assemblies)
            {
                TestAssembly testAssembly = mate.Load(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy);
                var methods = new List<TestMethod>(testAssembly.EnumerateTestMethods(project.Filters.Filter));

                var batcher = new TestMethodBatcher(methods.Count);

                var waiter = new ProcessWaiter();
                waiter.DefaultArgs = defaultArgs;
                waiter.TeamCity = teamcity;
                waiter.Silent = silent;

                Console.WriteLine("Running with {0} threads.", waiter.MaximumNumberOfProcesses);
                Console.WriteLine("Total number of tests: {0}.", methods.Count);

                TestBatch batch;
                while ((batch = batcher.GetNextBatch()).Size > 0)
                {
                    waiter.StartProcess(batch);
                }

                waiter.PrintSummary();

                mate.Unload(testAssembly);
            }

            return 0;
        }