public void and_a_namespace_is_specified_it_should_only_run_tests_contained_by_these_namespaces_and_all_sub_namespaces()
{
// Note that the namespaces varies between MbUnit.Tests and MbUnitTests.Tests
var settings = new RunSettings(new AssemblyOptions(getAssembly()), new string[] { }, null);
settings.Assembly.AddNamespace("AutoTest.TestRunners.MbUnit.Tests.TestResource");
Assert.That(_runner.Run(settings).Count(), Is.EqualTo(1));
}
public void Should_run_all_tests()
{
var settings = new RunSettings(new AssemblyOptions(Path.GetFullPath(@"AutoTest.TestRunners.MSTest.Tests.TestResource.dll")), new string[] {}, null);
var runner = new Runner();
var result = runner.Run(settings);
Assert.That(result.Count(), Is.EqualTo(10));
}
public IEnumerable<TestResult> Run(Plugin plugin, string id, RunSettings settings)
{
_directories.Add(Path.GetDirectoryName(settings.Assembly.Assembly));
_directories.Add(Path.GetDirectoryName(plugin.Assembly));
Logger.Write("About to create plugin {0} in {1} for {2}", plugin.Type, plugin.Assembly, id);
var runner = getRunner(plugin);
try
{
if (runner == null)
return _results;
Logger.Write("Matching plugin identifier ({0}) to test identifier ({1})", runner.Identifier, id);
if (!runner.Identifier.ToLower().Equals(id.ToLower()))
return _results;
Logger.Write("Checking whether assembly contains tests for {0}", id);
if (!runner.ContainsTestsFor(settings.Assembly.Assembly))
return _results;
Logger.Write("Starting test run");
return runner.Run(settings);
}
catch
{
throw;
}
finally
{
AppDomain.CurrentDomain.AssemblyResolve -= CurrentDomain_AssemblyResolve;
}
}
public void and_a_test_is_specified_it_should_only_run_specified_tests()
{
var settings = new RunSettings(new AssemblyOptions(getAssembly()), new string[] { }, null);
settings.Assembly.AddTest("AutoTest.TestRunners.MbUnitTests.Tests.TestResource.ClassContainingTests.A_passing_test");
settings.Assembly.AddTest("AutoTest.TestRunners.MbUnitTests.Tests.TestResource.ClassContainingTests.A_failing_test");
Assert.That(_runner.Run(settings).Count(), Is.EqualTo(2));
}
public void When_told_to_run_all_tests_it_reports_all_tests()
{
var runner = new Runner();
var options = new AssemblyOptions(getAssembly());
var settings = new RunSettings(options, new string[] { }, null);
var result = runner.Run(settings);
Assert.That(result.Count(), Is.EqualTo(5));
}
public void and_a_namespace_is_specified_it_should_only_run_tests_contained_by_these_namespaces()
{
var settings = new RunSettings(new AssemblyOptions(getAssembly()), new string[] { }, null);
settings.Assembly.AddNamespace("AutoTest.TestRunners.MbUnit.Tests.TestResource.AnotherNamespace");
Assert.That(_runner.Run(settings).Count(), Is.EqualTo(1));
Assert.That(_runner.Run(settings).ElementAt(0).TestName,
Is.EqualTo("AutoTest.TestRunners.MbUnit.Tests.TestResource.AnotherNamespace.TestsInAnotherNamespace.Even_another_test"));
}
public void when_told_to_run_all_on_type_it_runs_all_on_type()
{
var runner = new Runner();
var options = new AssemblyOptions(getAssembly());
options.AddMember("AutoTest.TestRunners.SimpleTesting.Tests.Resources.SimpleTestingTests");
var settings = new RunSettings(options, new string[] { }, null);
var result = runner.Run(settings);
Assert.That(result.Count(), Is.EqualTo(3));
}
public IEnumerable<TestResult> Run(RunSettings settings)
{
_results = new List<TestResult>();
var thread = new Thread(run);
thread.SetApartmentState(ApartmentState.STA);
thread.Start(settings);
thread.Join();
return _results;
}
public void Should_not_run_test_with_ignore_attribute()
{
var assemblyPath = Path.GetFullPath(@"AutoTest.TestRunners.MSTest.Tests.TestResource.dll");
var assembly = new AssemblyOptions(assemblyPath);
assembly.AddTest("AutoTest.TestRunners.MSTest.Tests.TestResource.TestFixture1.Ignore_Attrib_test");
var settings = new RunSettings(assembly, new string[] { }, null);
var runner = new Runner();
var result = runner.Run(settings);
Assert.That(result.Count(), Is.EqualTo(0));
}
public IEnumerable<TestResult> Run(RunSettings settings)
{
_results = new List<TestResult>();
var listener = new TestListener(_feedback, settings.Assembly.Assembly);
var assembly = getAssembly(settings.Assembly.Assembly);
var runner = new AppDomainRunner(listener, Machine.Specifications.Runner.RunOptions.Default);
runTests(settings, assembly, runner);
_results.AddRange(listener.Results);
return _results;
}
public IEnumerable<TestResult> Run(RunSettings settings)
{
var runner = new NUnitRunner();
runner.Initialize();
var parser = new NUnitOptionsParser(settings);
parser.Parse();
var results = new List<TestResult>();
foreach (var option in parser.Options)
results.AddRange(runner.Execute(option));
return results;
}
private static bool shouldRun(string methodName, RunSettings settings)
{
var assembly = settings.Assembly;
if (assembly.Tests.Contains(methodName))
return true;
if (assembly.Members.Any(x => methodName.StartsWith(x)))
return true;
if (assembly.Namespaces.Any(x => methodName.StartsWith(x)))
return true;
return false;
}
private void runTests(RunSettings settings, IGrouping<Type, MethodInfo> fixture)
{
log("Running fixture {0}", fixture.Key);
if (_channel != null)
_channel.TestStarted(fixture.Key.ToString());
new MSTestTestFixture(fixture.Key)
.Run(fixture.ToList()).ToList()
.ForEach(result =>
{
var item = getResult(settings, fixture, result);
_results.Add(item);
_channel.TestFinished(item);
});
}
public void and_it_passes_it_should_return_passing_result()
{
var settings = new RunSettings(new AssemblyOptions(getAssembly()), new string[] { }, null);
settings.Assembly.AddTest("AutoTest.TestRunners.MbUnitTests.Tests.TestResource.ClassContainingTests.A_passing_test");
var result = _runner.Run(settings);
var test = result.ElementAt(0);
Assert.That(test.Assembly, Is.EqualTo(getAssembly()));
Assert.That(test.DurationInMilliseconds, Is.GreaterThan(0));
Assert.That(test.Runner, Is.EqualTo("MbUnit"));
Assert.That(test.State, Is.EqualTo(Shared.Results.TestState.Passed));
Assert.That(test.TestFixture, Is.EqualTo("AutoTest.TestRunners.MbUnitTests.Tests.TestResource.ClassContainingTests"));
Assert.That(test.TestName, Is.EqualTo("AutoTest.TestRunners.MbUnitTests.Tests.TestResource.ClassContainingTests.A_passing_test"));
}
private void runTests(RunSettings settings, Assembly assembly, AppDomainRunner runner)
{
if (runAllTests(settings))
{
runner.RunAssembly(assembly);
return;
}
foreach (var member in settings.Assembly.Tests)
runner.RunMember(assembly, assembly.GetType(member));
foreach (var member in settings.Assembly.Members)
runner.RunMember(assembly, assembly.GetType(member));
foreach (var ns in settings.Assembly.Namespaces)
runner.RunNamespace(assembly, ns);
}
public void Should_run_full_fixture()
{
var assemblyPath = Path.GetFullPath(@"AutoTest.TestRunners.MSTest.Tests.TestResource.dll");
var assembly = new AssemblyOptions(assemblyPath);
assembly.AddMember("AutoTest.TestRunners.MSTest.Tests.TestResource.TestFixture2");
var settings = new RunSettings(assembly, new string[] { });
var runner = new Runner();
var result = runner.Run(settings);
Assert.That(result.Count(), Is.EqualTo(2));
var test = result.ElementAt(0);
Assert.That(test.TestFixture, Is.EqualTo("AutoTest.TestRunners.MSTest.Tests.TestResource.TestFixture2"));
Assert.That(test.TestName, Is.EqualTo("AutoTest.TestRunners.MSTest.Tests.TestResource.TestFixture2.Another_passing_test"));
Assert.That(result.ElementAt(1).TestFixture, Is.EqualTo("AutoTest.TestRunners.MSTest.Tests.TestResource.TestFixture2"));
}
public void When_told_to_run_a_test_that_is_ignored_it_reports_a_ignored_result()
{
var runner = new Runner();
var options = new AssemblyOptions(getAssembly());
options.AddTest("AutoTest.TestRunners.MSpec.Tests.TestResource.Ignored_test");
var settings = new RunSettings(options, new string[] { }, null);
var result = runner.Run(settings);
Assert.That(result.Count(), Is.EqualTo(1));
var test = result.ElementAt(0);
Assert.That(test.State, Is.EqualTo(Shared.Results.TestState.Ignored));
Assert.That(test.Assembly, Is.EqualTo(getAssembly()));
Assert.That(test.Runner, Is.EqualTo("MSpec"));
Assert.That(test.TestFixture, Is.EqualTo("AutoTest.TestRunners.MSpec.Tests.TestResource.Ignored_test"));
Assert.That(test.TestName, Is.EqualTo("AutoTest.TestRunners.MSpec.Tests.TestResource.Ignored_test"));
}
public void when_running_named_test_that_passes()
{
var runner = new Runner();
var options = new AssemblyOptions(getAssembly());
options.AddTest("AutoTest.TestRunners.SimpleTesting.Tests.Resources.SimpleTestingTests.a_passing_test");
var settings = new RunSettings(options, new string[] { }, null);
var result = runner.Run(settings);
Assert.That(result.Count(), Is.EqualTo(1));
Assert.That(result.First().State == TestState.Passed);
Assert.That(!result.First().StackLines.Any());
Assert.That(result.First().TestDisplayName == "a_passing_test");
Assert.That(result.First().Runner == "SimpleTesting");
Assert.AreEqual("AutoTest.TestRunners.SimpleTesting.Tests.Resources.SimpleTestingTests.a_passing_test", result.First().TestName);
Assert.AreEqual("SimpleTestingTests", result.First().TestFixture);
}
private void runTests(RunSettings settings, IGrouping<Type, MethodInfo> fixture)
{
var list = fixture.ToList();
try
{
_logger.Write("Running fixture {0}", fixture.Key);
using (var runner = new MSTestTestRunner(fixture.Key))
{
list.Select(test => runner.Run(test)).ToList().ForEach(x => _results.Add(getResult(settings, fixture, x)));
}
}
catch (Exception ex)
{
list.ForEach(test => _results.Add(getResult(settings, fixture, new celer.Core.RunResult(test, false, false, ex, 0))));
}
}
public void Should_run_single_failing_test()
{
var assemblyPath = Path.GetFullPath(@"AutoTest.TestRunners.MSTest.Tests.TestResource.dll");
var assembly = new AssemblyOptions(assemblyPath);
assembly.AddTest("AutoTest.TestRunners.MSTest.Tests.TestResource.TestFixture1.Failing_test");
var settings = new RunSettings(assembly, new string[] { }, null);
var runner = new Runner();
var result = runner.Run(settings);
Assert.That(result.Count(), Is.EqualTo(1));
var test = result.ElementAt(0);
Assert.That(test.Runner, Is.EqualTo("MSTest"));
Assert.That(test.Assembly, Is.EqualTo(assemblyPath));
Assert.That(test.State, Is.EqualTo(Shared.Results.TestState.Failed));
Assert.That(test.TestFixture, Is.EqualTo("AutoTest.TestRunners.MSTest.Tests.TestResource.TestFixture1"));
Assert.That(test.TestName, Is.EqualTo("AutoTest.TestRunners.MSTest.Tests.TestResource.TestFixture1.Failing_test"));
}
public void Should_run_single_inconclusive_test()
{
var assemblyPath = Path.GetFullPath(@"AutoTest.TestRunners.MSTest.Tests.TestResource.dll");
var assembly = new AssemblyOptions(assemblyPath);
assembly.AddTest("AutoTest.TestRunners.MSTest.Tests.TestResource.TestFixture1.Inconclusive_test");
var settings = new RunSettings(assembly, new string[] { });
var runner = new Runner();
var result = runner.Run(settings);
Assert.That(result.Count(), Is.EqualTo(1));
var test = result.ElementAt(0);
Assert.That(test.Runner, Is.EqualTo("MSTest"));
Assert.That(test.Assembly, Is.EqualTo(assemblyPath));
Assert.That(test.State, Is.EqualTo(Shared.Results.TestState.Ignored));
Assert.That(test.TestFixture, Is.EqualTo("AutoTest.TestRunners.MSTest.Tests.TestResource.TestFixture1"));
Assert.That(test.TestName, Is.EqualTo("AutoTest.TestRunners.MSTest.Tests.TestResource.TestFixture1.Inconclusive_test"));
Assert.That(test.DurationInMilliseconds, Is.GreaterThan(0));
}
public IEnumerable<TestResult> Run(bool startLogger, Plugin plugin, string id, RunSettings settings)
{
if (startLogger)
Logger.SetLogger(new ConsoleLogger());
IEnumerable<TestResult> resultSet = null;
var directories = new List<string>();
directories.Add(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location));
directories.Add(Path.GetDirectoryName(settings.Assembly.Assembly));
directories.Add(Path.GetDirectoryName(plugin.Assembly));
using (var resolver = new AssemblyResolver(directories.ToArray()))
{
Logger.Write("About to create plugin {0} in {1} for {2}", plugin.Type, plugin.Assembly, id);
var runner = getRunner(plugin);
var currentDirectory = Environment.CurrentDirectory;
try
{
if (runner == null)
return _results;
using (var server = new PipeServer(settings.PipeName))
{
Logger.Write("Matching plugin identifier ({0}) to test identifier ({1})", runner.Identifier, id);
if (!runner.Identifier.ToLower().Equals(id.ToLower()) && !id.ToLower().Equals("any"))
return _results;
Logger.Write("Checking whether assembly contains tests for {0}", id);
if (!settings.Assembly.IsVerified && !runner.ContainsTestsFor(settings.Assembly.Assembly))
return _results;
Logger.Write("Initializing channel");
runner.SetLiveFeedbackChannel(new TestFeedbackProvider(server));
var newCurrent = Path.GetDirectoryName(settings.Assembly.Assembly);
Logger.Write("Setting current directory to " + newCurrent);
Environment.CurrentDirectory = newCurrent;
Logger.Write("Starting test run");
resultSet = runner.Run(settings);
}
}
catch
{
throw;
}
finally
{
Environment.CurrentDirectory = currentDirectory;
}
}
return resultSet;
}
public IEnumerable<TestResult> Run(RunSettings settings)
{
var assembly = getAssembly(settings.Assembly.Assembly);
var generator = new LinFu.DynamicProxy.ProxyFactory();
var proxy = new TestListenerProxy(_feedback, settings.Assembly.Assembly);
_results = new List<TestResult>();
var mspec = Assembly.LoadFrom("Machine.Specifications.dll");
var type = mspec.GetType("Machine.Specifications.Runner.Impl.AppDomainRunner");
var iTestListener = mspec.GetType("Machine.Specifications.Runner.ISpecificationRunListener");
var dflt = mspec.GetType("Machine.Specifications.Runner.RunOptions").GetProperty("Default").GetValue(null, null);
var listener = generator.CreateProxy(iTestListener, proxy, new Type[] {});
var runner = Activator.CreateInstance(type, new object[] {listener, dflt});
runTests(settings, assembly, runner);
_results.AddRange(proxy.Results);
return _results;
}
public IEnumerable<TestResult> Run(RunSettings settings)
{
_results = new List<TestResult>();
var assembly = getAssembly(settings.Assembly.Assembly);
if (_results.Count != 0)
{
}
else
{
foreach (var result in runTests(settings, assembly))
{
_results.Add(result);
}
}
return _results;
}
public IEnumerable<TestResult> Run(Plugin plugin, RunSettings options)
{
return plugin.New().Run(options);
}
private bool shouldRunAllTests(RunSettings settings)
{
return
!settings.Assembly.Tests.Any() &&
!settings.Assembly.Members.Any() &&
!settings.Assembly.Namespaces.Any();
}
private IEnumerable<TestResult> runTests(RunSettings settings, Assembly assembly)
{
if (shouldRunAllTests(settings))
{
return runAllTests(assembly);
}
var results = new List<TestResult>();
foreach (var member in settings.Assembly.Members)
results.AddRange(runAllOnType(assembly.GetType(member)));
results.AddRange(runAllNamed(assembly, settings.Assembly.Tests));
//foreach (var ns in settings.Assembly.Namespaces)
// results.AddRange(runAllInNameSpace(assembly));
return results;
}
public IEnumerable<AutoTest.TestRunners.Shared.Results.TestResult> Run(RunSettings settings)
{
var logger = new XUnitLogger();
XunitProject project = new XunitProject();
var runner = settings.Assembly;
// Set assembly externally as XUnit screws up the casing
logger.SetCurrentAssembly(runner.Assembly);
XunitProjectAssembly assembly = new XunitProjectAssembly
{
AssemblyFilename = runner.Assembly,
ConfigFilename = null,
ShadowCopy = false
};
project.AddAssembly(assembly);
foreach (XunitProjectAssembly asm in project.Assemblies)
{
using (ExecutorWrapper wrapper = new ExecutorWrapper(asm.AssemblyFilename, asm.ConfigFilename, asm.ShadowCopy))
{
try
{
var xunitRunner = new TestRunner(wrapper, logger);
//Run all tests
if (runner.Tests.Count() == 0 && runner.Members.Count() == 0 && runner.Namespaces.Count() == 0)
xunitRunner.RunAssembly();
//Run tests
if (runner.Tests.Count() > 0)
{
foreach (var test in runner.Tests)
xunitRunner.RunTest(test.Substring(0, test.LastIndexOf(".")), test.Substring(test.LastIndexOf(".") + 1, test.Length - (test.LastIndexOf(".") + 1)));
}
//Run members
if (runner.Members.Count() > 0)
{
foreach (var member in runner.Members)
xunitRunner.RunClass(member);
}
//Run namespaces
if (runner.Namespaces.Count() > 0)
{
var loadedAssembly = Assembly.LoadFrom(runner.Assembly);
var types = loadedAssembly.GetExportedTypes();
loadedAssembly = null;
foreach (var ns in runner.Namespaces)
{
foreach (Type type in types)
if (ns == null || type.Namespace == ns)
xunitRunner.RunClass(type.FullName);
}
}
}
catch (ArgumentException ex)
{
Console.WriteLine(ex.Message);
}
}
}
return logger.Results;
}
private TestResult getResult(RunSettings settings, IGrouping<Type, MethodInfo> fixture, celer.Core.RunResult x)
{
var result = new TestResult(Identifier, settings.Assembly.Assembly, fixture.Key.FullName, x.MillisecondsSpent, fixture.Key.FullName + "." + x.Test.Name, getState(x), getMessage(x));
result.AddStackLines(getStackLines(x.Exception));
return result;
}
IEnumerable<Shared.Results.TestResult> Shared.IAutoTestNetTestRunner.Run(RunSettings settings)
{
return null;
}
