internal static void RunTestProject(this XunitProject project, bool displaySuccess, bool displayFailureStack)
{
var mate = new MultiAssemblyTestEnvironment();
var tests = project.Assemblies.Select(x => mate.Load(x.AssemblyFilename, x.ConfigFilename, x.ShadowCopy));
var totalAssemblies = 0;
var totalTests = 0;
var totalFailures = 0;
var totalSkips = 0;
var totalTime = 0.0;
foreach (var test in tests)
{
Console.WriteLine();
Console.WriteLine("Test assembly: {0}", test.AssemblyFilename);
Console.WriteLine();
try
{
var methods = test.EnumerateTestMethods(project.Filters.Filter).ToList();
if (!methods.Any())
{
Console.WriteLine("Skipping assembly (no tests match the specified filter).");
}
else
{
var callback = new DefaultRunnerCallback(displaySuccess, displayFailureStack, methods.Count);
test.Run(methods, callback);
++totalAssemblies;
totalTests += callback.TotalTests;
totalFailures += callback.TotalFailures;
totalSkips += callback.TotalSkips;
totalTime += callback.TotalTime;
}
}
catch (ArgumentException ex)
{
Console.WriteLine(ex.Message.Red());
}
mate.Unload(test);
}
if (totalAssemblies <= 1)
{
return;
}
Console.WriteLine();
Console.WriteLine("=== {0} total, {1} failed, {2} skipped, took {3} seconds ===",
totalTests, totalFailures, totalSkips, totalTime.ToString("0.000", CultureInfo.InvariantCulture));
}
public void MultiAssemblyAcceptanceTest()
{
string code =
@"
using System;
using Xunit;
public class MockTestClass
{
[Fact]
public void SuccessTest()
{
Assert.Equal(2, 2);
}
}
";
using (MockAssembly mockAssembly1 = new MockAssembly())
using (MockAssembly mockAssembly2 = new MockAssembly())
{
mockAssembly1.Compile(code);
mockAssembly2.Compile(code);
Mock<ITestMethodRunnerCallback> callback = new Mock<ITestMethodRunnerCallback>();
callback.Setup(c => c.TestStart(It.IsAny<TestMethod>())).Returns(true);
callback.Setup(c => c.TestFinished(It.IsAny<TestMethod>())).Returns(true);
MultiAssemblyTestEnvironment mate = new MultiAssemblyTestEnvironment();
mate.Load(mockAssembly1.FileName);
mate.Load(mockAssembly2.FileName);
TestAssembly testAssembly1 = mate.EnumerateTestAssemblies().Where(a => a.AssemblyFilename == mockAssembly1.FileName).Single();
TestAssembly testAssembly2 = mate.EnumerateTestAssemblies().Where(a => a.AssemblyFilename == mockAssembly2.FileName).Single();
TestMethod assembly1Method = testAssembly1.EnumerateTestMethods().Single();
TestMethod assembly2Method = testAssembly1.EnumerateTestMethods().Single();
mate.Run(mate.EnumerateTestMethods(), callback.Object);
callback.Verify(c => c.AssemblyStart(testAssembly1));
callback.Verify(c => c.TestStart(assembly1Method));
callback.Verify(c => c.TestFinished(assembly1Method));
callback.Verify(c => c.AssemblyFinished(testAssembly1, 1, 0, 0, It.IsAny<double>()));
callback.Verify(c => c.AssemblyStart(testAssembly2));
callback.Verify(c => c.TestStart(assembly2Method));
callback.Verify(c => c.TestFinished(assembly2Method));
callback.Verify(c => c.AssemblyFinished(testAssembly2, 1, 0, 0, It.IsAny<double>()));
}
}
public XmlNode RunWithMate()
{
var mate = new MultiAssemblyTestEnvironment();
mate.Load(FileName);
string xml = mate.Run(mate.EnumerateTestMethods(), new Callback());
var doc = new XmlDocument();
doc.LoadXml(xml);
// ChildNodes[0] == <assemblies>
// ChildNodes[0].ChildNodes[0] == first <assembly> node
return doc.ChildNodes[0].ChildNodes[0];
}
static int RunProject(XunitProject project, bool teamcity, bool silent, ParallelXunitCommandLine parallelXunitCommandLine)
{
if (!parallelXunitCommandLine.Start.HasValue || !parallelXunitCommandLine.End.HasValue)
return -1;
int totalAssemblies = 0;
int totalTests = 0;
int totalFailures = 0;
int totalSkips = 0;
double totalTime = 0;
var mate = new MultiAssemblyTestEnvironment();
foreach (XunitProjectAssembly assembly in project.Assemblies)
{
parallelXunitCommandLine.SetOutputArguments(assembly.Output);
TestAssembly testAssembly = mate.Load(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy);
List<IResultXmlTransform> transforms = TransformFactory.GetAssemblyTransforms(assembly);
try
{
var start = parallelXunitCommandLine.Start.Value;
var end = parallelXunitCommandLine.End.Value;
var methods = new List<TestMethod>(testAssembly.EnumerateTestMethods(project.Filters.Filter)).Skip(start).Take(end - start).ToList();
if (methods.Count == 0)
{
Console.WriteLine("Skipping assembly (no tests match the specified filter).");
continue;
}
var callback = CreateCallback(teamcity, silent, methods.Count);
var assemblyXml = testAssembly.Run(methods, callback);
++totalAssemblies;
totalTests += callback.TotalTests;
totalFailures += callback.TotalFailures;
totalSkips += callback.TotalSkips;
totalTime += callback.TotalTime;
ParallelXunitCommandLine.WriteResultFile(assembly.AssemblyFilename, totalTests, totalFailures, totalSkips, totalTime, start, end);
foreach (var transform in transforms)
transform.Transform(assemblyXml);
}
catch (ArgumentException ex)
{
Console.WriteLine(ex.Message);
}
mate.Unload(testAssembly);
}
if (!teamcity && totalAssemblies > 1)
{
Console.WriteLine();
Console.WriteLine("=== {0} total, {1} failed, {2} skipped, took {3} seconds ===",
totalTests, totalFailures, totalSkips, totalTime.ToString("0.000", CultureInfo.InvariantCulture));
}
return totalFailures;
}
static int RunProject(XunitProject project, bool teamcity, bool silent, string[] args)
{
var defaultArgs = args.Aggregate((a, b) => a + " " + b);
var mate = new MultiAssemblyTestEnvironment();
foreach (XunitProjectAssembly assembly in project.Assemblies)
{
TestAssembly testAssembly = mate.Load(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy);
var methods = new List<TestMethod>(testAssembly.EnumerateTestMethods(project.Filters.Filter));
var methodsCount = methods.Count;
ParallelXunitCommandLine.DeleteResultFiles(assembly.AssemblyFilename);
using (var waiter = new ProcessWaiter())
{
waiter.DefaultArgs = defaultArgs;
waiter.TeamCity = teamcity;
waiter.Silent = silent;
Console.WriteLine("Running with {0} threads.", waiter.MaximumNumberOfProcesses);
Console.WriteLine("Total number of tests: {0}.", methodsCount);
IParallelTestMethodRunnerCallback callback = CreateCallback(teamcity, silent, methodsCount);
callback.ParentAssemblyStart(testAssembly);
TestBatch batch;
var batcher = new TestMethodBatcher(methodsCount);
while ((batch = batcher.GetNextBatch()).Size > 0)
{
waiter.StartProcess(batch);
}
if (!teamcity)
{
waiter.PrintSummary();
}
else
{
waiter.WaitForAllProcesses();
}
var testResult = ParallelXunitCommandLine.AggregateTestResults(assembly.AssemblyFilename);
callback.ParentAssemblyFinished(testAssembly, testResult.Total, testResult.Failed, testResult.Skiped);
mate.Unload(testAssembly);
}
}
return 0;
}
public void NullFilterThrows()
{
var mate = new MultiAssemblyTestEnvironment();
Exception ex = Record.Exception(() => mate.EnumerateTestMethods(null).ToList());
Assert.IsType<ArgumentNullException>(ex);
}
public void MultiAssemblyGetTraitsAcceptanceTest()
{
string code1 =
@"
using System;
using Xunit;
public class MockTestClass
{
[Fact]
[Trait(""Trait1"", ""Value1"")]
public void Value1Test()
{
}
[Fact]
[Trait(""Trait1"", ""Value2"")]
public void Value2Test()
{
}
[Fact]
[Trait(""Trait2"", ""Value1"")]
public void Trait2Value1Test()
{
}
}
";
string code2 =
@"
using System;
using Xunit;
public class MockTestClass
{
[Fact]
[Trait(""Trait1"", ""Value1"")]
public void OtherTest1()
{
}
[Fact]
[Trait(""Trait3"", ""Value42"")]
public void Crazy()
{
}
}
";
using (MockAssembly mockAssembly1 = new MockAssembly())
using (MockAssembly mockAssembly2 = new MockAssembly())
{
mockAssembly1.Compile(code1);
mockAssembly2.Compile(code2);
MultiAssemblyTestEnvironment mate = new MultiAssemblyTestEnvironment();
mate.Load(mockAssembly1.FileName);
mate.Load(mockAssembly2.FileName);
MultiValueDictionary<string, string> result = mate.EnumerateTraits();
var trait1 = result["Trait1"];
Assert.Equal(2, trait1.Count());
Assert.Contains("Value1", trait1);
Assert.Contains("Value2", trait1);
var trait2 = result["Trait2"];
Assert.Single(trait2);
Assert.Contains("Value1", trait2);
var trait3 = result["Trait3"];
Assert.Single(trait3);
Assert.Contains("Value42", trait3);
}
}
public void SingleAssemblyAcceptanceTest()
{
string code =
@"
using System;
using Xunit;
public class MockTestClass
{
[Fact]
public void SuccessTest()
{
Assert.Equal(2, 2);
}
[Fact]
public void FailureTest()
{
Assert.Equal(2, 3);
}
[Fact(Skip=""I'm too lazy to run today"")]
public void SkippingTest()
{
Assert.Equal(2, 4);
}
}
";
using (MockAssembly mockAssembly = new MockAssembly())
{
mockAssembly.Compile(code);
Mock<ITestMethodRunnerCallback> callback = new Mock<ITestMethodRunnerCallback>();
callback.Setup(c => c.TestStart(It.IsAny<TestMethod>())).Returns(true);
callback.Setup(c => c.TestFinished(It.IsAny<TestMethod>())).Returns(true);
MultiAssemblyTestEnvironment mate = new MultiAssemblyTestEnvironment();
mate.Load(mockAssembly.FileName);
TestAssembly testAssembly = mate.EnumerateTestAssemblies().Single();
TestMethod passingMethod = testAssembly.EnumerateTestMethods().Where(m => m.MethodName == "SuccessTest").Single();
TestMethod failingMethod = testAssembly.EnumerateTestMethods().Where(m => m.MethodName == "FailureTest").Single();
TestMethod skippedMethod = testAssembly.EnumerateTestMethods().Where(m => m.MethodName == "SkippingTest").Single();
mate.Run(mate.EnumerateTestMethods(), callback.Object);
callback.Verify(c => c.AssemblyStart(testAssembly));
callback.Verify(c => c.TestStart(passingMethod));
callback.Verify(c => c.TestFinished(passingMethod));
callback.Verify(c => c.TestStart(failingMethod));
callback.Verify(c => c.TestFinished(failingMethod));
callback.Verify(c => c.TestStart(skippedMethod), Times.Never());
callback.Verify(c => c.TestFinished(skippedMethod));
callback.Verify(c => c.AssemblyFinished(testAssembly, 3, 1, 1, It.IsAny<double>()));
var passingMethodResult = Assert.IsType<TestPassedResult>(passingMethod.RunResults[0]);
Assert.Null(passingMethodResult.Output);
var failingMethodResult = Assert.IsType<TestFailedResult>(failingMethod.RunResults[0]);
Assert.Null(failingMethodResult.Output);
Assert.Equal("Xunit.Sdk.EqualException", failingMethodResult.ExceptionType);
var skippedMethodResult = Assert.IsType<TestSkippedResult>(skippedMethod.RunResults[0]);
}
}
public void MultiAssemblyTraitFilterAcceptanceTest()
{
string code =
@"
using System;
using Xunit;
public class MockTestClass
{
[Fact]
[Trait(""Trait1"", ""Value1"")]
public void Value1Test()
{
}
[Fact]
[Trait(""Trait1"", ""Value2"")]
public void Value2Test()
{
}
}
";
using (MockAssembly mockAssembly1 = new MockAssembly())
using (MockAssembly mockAssembly2 = new MockAssembly())
{
mockAssembly1.Compile(code);
mockAssembly2.Compile(code);
Mock<ITestMethodRunnerCallback> callback = new Mock<ITestMethodRunnerCallback>();
callback.Setup(c => c.TestStart(It.IsAny<TestMethod>())).Returns(true);
callback.Setup(c => c.TestFinished(It.IsAny<TestMethod>())).Returns(true);
MultiAssemblyTestEnvironment mate = new MultiAssemblyTestEnvironment();
mate.Load(mockAssembly1.FileName);
mate.Load(mockAssembly2.FileName);
TestAssembly testAssembly1 = mate.EnumerateTestAssemblies().Where(a => a.AssemblyFilename == mockAssembly1.FileName).Single();
TestAssembly testAssembly2 = mate.EnumerateTestAssemblies().Where(a => a.AssemblyFilename == mockAssembly2.FileName).Single();
TestMethod assembly1Value1Method = testAssembly1.EnumerateTestMethods().Where(m => m.MethodName == "Value1Test").Single();
TestMethod assembly1Value2Method = testAssembly1.EnumerateTestMethods().Where(m => m.MethodName == "Value2Test").Single();
TestMethod assembly2Value1Method = testAssembly2.EnumerateTestMethods().Where(m => m.MethodName == "Value1Test").Single();
TestMethod assembly2Value2Method = testAssembly2.EnumerateTestMethods().Where(m => m.MethodName == "Value2Test").Single();
mate.Run(mate.EnumerateTestMethods(m => m.Traits["Trait1"].FirstOrDefault() == "Value1"), callback.Object);
callback.Verify(c => c.TestStart(assembly1Value1Method));
callback.Verify(c => c.TestStart(assembly1Value2Method), Times.Never());
callback.Verify(c => c.TestStart(assembly2Value1Method));
callback.Verify(c => c.TestStart(assembly2Value2Method), Times.Never());
}
}
public void MultiAssemblySearchFilterAcceptanceTest()
{
string code =
@"
using System;
using Xunit;
public class MockTestClass
{
[Fact]
public void Test1()
{
}
[Fact]
public void Test2()
{
}
}
";
using (MockAssembly mockAssembly1 = new MockAssembly())
using (MockAssembly mockAssembly2 = new MockAssembly())
{
mockAssembly1.Compile(code);
mockAssembly2.Compile(code);
Mock<ITestMethodRunnerCallback> callback = new Mock<ITestMethodRunnerCallback>();
callback.Setup(c => c.TestStart(It.IsAny<TestMethod>())).Returns(true);
callback.Setup(c => c.TestFinished(It.IsAny<TestMethod>())).Returns(true);
MultiAssemblyTestEnvironment mate = new MultiAssemblyTestEnvironment();
mate.Load(mockAssembly1.FileName);
mate.Load(mockAssembly2.FileName);
TestAssembly testAssembly1 = mate.EnumerateTestAssemblies().Where(a => a.AssemblyFilename == mockAssembly1.FileName).Single();
TestAssembly testAssembly2 = mate.EnumerateTestAssemblies().Where(a => a.AssemblyFilename == mockAssembly2.FileName).Single();
TestMethod assembly1Test1Method = testAssembly1.EnumerateTestMethods().Where(m => m.MethodName == "Test1").Single();
TestMethod assembly1Test2Method = testAssembly1.EnumerateTestMethods().Where(m => m.MethodName == "Test2").Single();
TestMethod assembly2Test1Method = testAssembly2.EnumerateTestMethods().Where(m => m.MethodName == "Test1").Single();
TestMethod assembly2Test2Method = testAssembly2.EnumerateTestMethods().Where(m => m.MethodName == "Test2").Single();
mate.Run(mate.EnumerateTestMethods(m => m.MethodName.Contains("t2")), callback.Object);
callback.Verify(c => c.TestStart(assembly1Test1Method), Times.Never());
callback.Verify(c => c.TestStart(assembly1Test2Method));
callback.Verify(c => c.TestStart(assembly2Test1Method), Times.Never());
callback.Verify(c => c.TestStart(assembly2Test2Method));
}
}
static int RunProject(XunitProject project, bool teamcity, bool silent, string[] args)
{
var defaultArgs = args.Aggregate((a, b) => a + " " + b);
var mate = new MultiAssemblyTestEnvironment();
foreach (XunitProjectAssembly assembly in project.Assemblies)
{
TestAssembly testAssembly = mate.Load(assembly.AssemblyFilename, assembly.ConfigFilename, assembly.ShadowCopy);
var methods = new List<TestMethod>(testAssembly.EnumerateTestMethods(project.Filters.Filter));
var batcher = new TestMethodBatcher(methods.Count);
var waiter = new ProcessWaiter();
waiter.DefaultArgs = defaultArgs;
waiter.TeamCity = teamcity;
waiter.Silent = silent;
Console.WriteLine("Running with {0} threads.", waiter.MaximumNumberOfProcesses);
Console.WriteLine("Total number of tests: {0}.", methods.Count);
TestBatch batch;
while ((batch = batcher.GetNextBatch()).Size > 0)
{
waiter.StartProcess(batch);
}
waiter.PrintSummary();
mate.Unload(testAssembly);
}
return 0;
}