public void and_a_namespace_is_specified_it_should_only_run_tests_contained_by_these_namespaces_and_all_sub_namespaces() { // Note that the namespaces varies between MbUnit.Tests and MbUnitTests.Tests var settings = new RunSettings(new AssemblyOptions(getAssembly()), new string[] { }, null); settings.Assembly.AddNamespace("AutoTest.TestRunners.MbUnit.Tests.TestResource"); Assert.That(_runner.Run(settings).Count(), Is.EqualTo(1)); }
public void Should_run_all_tests() { var settings = new RunSettings(new AssemblyOptions(Path.GetFullPath(@"AutoTest.TestRunners.MSTest.Tests.TestResource.dll")), new string[] {}, null); var runner = new Runner(); var result = runner.Run(settings); Assert.That(result.Count(), Is.EqualTo(10)); }
public IEnumerable<TestResult> Run(Plugin plugin, string id, RunSettings settings) { _directories.Add(Path.GetDirectoryName(settings.Assembly.Assembly)); _directories.Add(Path.GetDirectoryName(plugin.Assembly)); Logger.Write("About to create plugin {0} in {1} for {2}", plugin.Type, plugin.Assembly, id); var runner = getRunner(plugin); try { if (runner == null) return _results; Logger.Write("Matching plugin identifier ({0}) to test identifier ({1})", runner.Identifier, id); if (!runner.Identifier.ToLower().Equals(id.ToLower())) return _results; Logger.Write("Checking whether assembly contains tests for {0}", id); if (!runner.ContainsTestsFor(settings.Assembly.Assembly)) return _results; Logger.Write("Starting test run"); return runner.Run(settings); } catch { throw; } finally { AppDomain.CurrentDomain.AssemblyResolve -= CurrentDomain_AssemblyResolve; } }
public void and_a_test_is_specified_it_should_only_run_specified_tests() { var settings = new RunSettings(new AssemblyOptions(getAssembly()), new string[] { }, null); settings.Assembly.AddTest("AutoTest.TestRunners.MbUnitTests.Tests.TestResource.ClassContainingTests.A_passing_test"); settings.Assembly.AddTest("AutoTest.TestRunners.MbUnitTests.Tests.TestResource.ClassContainingTests.A_failing_test"); Assert.That(_runner.Run(settings).Count(), Is.EqualTo(2)); }
public void When_told_to_run_all_tests_it_reports_all_tests() { var runner = new Runner(); var options = new AssemblyOptions(getAssembly()); var settings = new RunSettings(options, new string[] { }, null); var result = runner.Run(settings); Assert.That(result.Count(), Is.EqualTo(5)); }
public void and_a_namespace_is_specified_it_should_only_run_tests_contained_by_these_namespaces() { var settings = new RunSettings(new AssemblyOptions(getAssembly()), new string[] { }, null); settings.Assembly.AddNamespace("AutoTest.TestRunners.MbUnit.Tests.TestResource.AnotherNamespace"); Assert.That(_runner.Run(settings).Count(), Is.EqualTo(1)); Assert.That(_runner.Run(settings).ElementAt(0).TestName, Is.EqualTo("AutoTest.TestRunners.MbUnit.Tests.TestResource.AnotherNamespace.TestsInAnotherNamespace.Even_another_test")); }
public void when_told_to_run_all_on_type_it_runs_all_on_type() { var runner = new Runner(); var options = new AssemblyOptions(getAssembly()); options.AddMember("AutoTest.TestRunners.SimpleTesting.Tests.Resources.SimpleTestingTests"); var settings = new RunSettings(options, new string[] { }, null); var result = runner.Run(settings); Assert.That(result.Count(), Is.EqualTo(3)); }
public IEnumerable<TestResult> Run(RunSettings settings) { _results = new List<TestResult>(); var thread = new Thread(run); thread.SetApartmentState(ApartmentState.STA); thread.Start(settings); thread.Join(); return _results; }
public void Should_not_run_test_with_ignore_attribute() { var assemblyPath = Path.GetFullPath(@"AutoTest.TestRunners.MSTest.Tests.TestResource.dll"); var assembly = new AssemblyOptions(assemblyPath); assembly.AddTest("AutoTest.TestRunners.MSTest.Tests.TestResource.TestFixture1.Ignore_Attrib_test"); var settings = new RunSettings(assembly, new string[] { }, null); var runner = new Runner(); var result = runner.Run(settings); Assert.That(result.Count(), Is.EqualTo(0)); }
public IEnumerable<TestResult> Run(RunSettings settings) { _results = new List<TestResult>(); var listener = new TestListener(_feedback, settings.Assembly.Assembly); var assembly = getAssembly(settings.Assembly.Assembly); var runner = new AppDomainRunner(listener, Machine.Specifications.Runner.RunOptions.Default); runTests(settings, assembly, runner); _results.AddRange(listener.Results); return _results; }
public IEnumerable<TestResult> Run(RunSettings settings) { var runner = new NUnitRunner(); runner.Initialize(); var parser = new NUnitOptionsParser(settings); parser.Parse(); var results = new List<TestResult>(); foreach (var option in parser.Options) results.AddRange(runner.Execute(option)); return results; }
private static bool shouldRun(string methodName, RunSettings settings) { var assembly = settings.Assembly; if (assembly.Tests.Contains(methodName)) return true; if (assembly.Members.Any(x => methodName.StartsWith(x))) return true; if (assembly.Namespaces.Any(x => methodName.StartsWith(x))) return true; return false; }
private void runTests(RunSettings settings, IGrouping<Type, MethodInfo> fixture) { log("Running fixture {0}", fixture.Key); if (_channel != null) _channel.TestStarted(fixture.Key.ToString()); new MSTestTestFixture(fixture.Key) .Run(fixture.ToList()).ToList() .ForEach(result => { var item = getResult(settings, fixture, result); _results.Add(item); _channel.TestFinished(item); }); }
public void and_it_passes_it_should_return_passing_result() { var settings = new RunSettings(new AssemblyOptions(getAssembly()), new string[] { }, null); settings.Assembly.AddTest("AutoTest.TestRunners.MbUnitTests.Tests.TestResource.ClassContainingTests.A_passing_test"); var result = _runner.Run(settings); var test = result.ElementAt(0); Assert.That(test.Assembly, Is.EqualTo(getAssembly())); Assert.That(test.DurationInMilliseconds, Is.GreaterThan(0)); Assert.That(test.Runner, Is.EqualTo("MbUnit")); Assert.That(test.State, Is.EqualTo(Shared.Results.TestState.Passed)); Assert.That(test.TestFixture, Is.EqualTo("AutoTest.TestRunners.MbUnitTests.Tests.TestResource.ClassContainingTests")); Assert.That(test.TestName, Is.EqualTo("AutoTest.TestRunners.MbUnitTests.Tests.TestResource.ClassContainingTests.A_passing_test")); }
private void runTests(RunSettings settings, Assembly assembly, AppDomainRunner runner) { if (runAllTests(settings)) { runner.RunAssembly(assembly); return; } foreach (var member in settings.Assembly.Tests) runner.RunMember(assembly, assembly.GetType(member)); foreach (var member in settings.Assembly.Members) runner.RunMember(assembly, assembly.GetType(member)); foreach (var ns in settings.Assembly.Namespaces) runner.RunNamespace(assembly, ns); }
public void Should_run_full_fixture() { var assemblyPath = Path.GetFullPath(@"AutoTest.TestRunners.MSTest.Tests.TestResource.dll"); var assembly = new AssemblyOptions(assemblyPath); assembly.AddMember("AutoTest.TestRunners.MSTest.Tests.TestResource.TestFixture2"); var settings = new RunSettings(assembly, new string[] { }); var runner = new Runner(); var result = runner.Run(settings); Assert.That(result.Count(), Is.EqualTo(2)); var test = result.ElementAt(0); Assert.That(test.TestFixture, Is.EqualTo("AutoTest.TestRunners.MSTest.Tests.TestResource.TestFixture2")); Assert.That(test.TestName, Is.EqualTo("AutoTest.TestRunners.MSTest.Tests.TestResource.TestFixture2.Another_passing_test")); Assert.That(result.ElementAt(1).TestFixture, Is.EqualTo("AutoTest.TestRunners.MSTest.Tests.TestResource.TestFixture2")); }
public void When_told_to_run_a_test_that_is_ignored_it_reports_a_ignored_result() { var runner = new Runner(); var options = new AssemblyOptions(getAssembly()); options.AddTest("AutoTest.TestRunners.MSpec.Tests.TestResource.Ignored_test"); var settings = new RunSettings(options, new string[] { }, null); var result = runner.Run(settings); Assert.That(result.Count(), Is.EqualTo(1)); var test = result.ElementAt(0); Assert.That(test.State, Is.EqualTo(Shared.Results.TestState.Ignored)); Assert.That(test.Assembly, Is.EqualTo(getAssembly())); Assert.That(test.Runner, Is.EqualTo("MSpec")); Assert.That(test.TestFixture, Is.EqualTo("AutoTest.TestRunners.MSpec.Tests.TestResource.Ignored_test")); Assert.That(test.TestName, Is.EqualTo("AutoTest.TestRunners.MSpec.Tests.TestResource.Ignored_test")); }
public void when_running_named_test_that_passes() { var runner = new Runner(); var options = new AssemblyOptions(getAssembly()); options.AddTest("AutoTest.TestRunners.SimpleTesting.Tests.Resources.SimpleTestingTests.a_passing_test"); var settings = new RunSettings(options, new string[] { }, null); var result = runner.Run(settings); Assert.That(result.Count(), Is.EqualTo(1)); Assert.That(result.First().State == TestState.Passed); Assert.That(!result.First().StackLines.Any()); Assert.That(result.First().TestDisplayName == "a_passing_test"); Assert.That(result.First().Runner == "SimpleTesting"); Assert.AreEqual("AutoTest.TestRunners.SimpleTesting.Tests.Resources.SimpleTestingTests.a_passing_test", result.First().TestName); Assert.AreEqual("SimpleTestingTests", result.First().TestFixture); }
private void runTests(RunSettings settings, IGrouping<Type, MethodInfo> fixture) { var list = fixture.ToList(); try { _logger.Write("Running fixture {0}", fixture.Key); using (var runner = new MSTestTestRunner(fixture.Key)) { list.Select(test => runner.Run(test)).ToList().ForEach(x => _results.Add(getResult(settings, fixture, x))); } } catch (Exception ex) { list.ForEach(test => _results.Add(getResult(settings, fixture, new celer.Core.RunResult(test, false, false, ex, 0)))); } }
public void Should_run_single_failing_test() { var assemblyPath = Path.GetFullPath(@"AutoTest.TestRunners.MSTest.Tests.TestResource.dll"); var assembly = new AssemblyOptions(assemblyPath); assembly.AddTest("AutoTest.TestRunners.MSTest.Tests.TestResource.TestFixture1.Failing_test"); var settings = new RunSettings(assembly, new string[] { }, null); var runner = new Runner(); var result = runner.Run(settings); Assert.That(result.Count(), Is.EqualTo(1)); var test = result.ElementAt(0); Assert.That(test.Runner, Is.EqualTo("MSTest")); Assert.That(test.Assembly, Is.EqualTo(assemblyPath)); Assert.That(test.State, Is.EqualTo(Shared.Results.TestState.Failed)); Assert.That(test.TestFixture, Is.EqualTo("AutoTest.TestRunners.MSTest.Tests.TestResource.TestFixture1")); Assert.That(test.TestName, Is.EqualTo("AutoTest.TestRunners.MSTest.Tests.TestResource.TestFixture1.Failing_test")); }
public void Should_run_single_inconclusive_test() { var assemblyPath = Path.GetFullPath(@"AutoTest.TestRunners.MSTest.Tests.TestResource.dll"); var assembly = new AssemblyOptions(assemblyPath); assembly.AddTest("AutoTest.TestRunners.MSTest.Tests.TestResource.TestFixture1.Inconclusive_test"); var settings = new RunSettings(assembly, new string[] { }); var runner = new Runner(); var result = runner.Run(settings); Assert.That(result.Count(), Is.EqualTo(1)); var test = result.ElementAt(0); Assert.That(test.Runner, Is.EqualTo("MSTest")); Assert.That(test.Assembly, Is.EqualTo(assemblyPath)); Assert.That(test.State, Is.EqualTo(Shared.Results.TestState.Ignored)); Assert.That(test.TestFixture, Is.EqualTo("AutoTest.TestRunners.MSTest.Tests.TestResource.TestFixture1")); Assert.That(test.TestName, Is.EqualTo("AutoTest.TestRunners.MSTest.Tests.TestResource.TestFixture1.Inconclusive_test")); Assert.That(test.DurationInMilliseconds, Is.GreaterThan(0)); }
public IEnumerable<TestResult> Run(bool startLogger, Plugin plugin, string id, RunSettings settings) { if (startLogger) Logger.SetLogger(new ConsoleLogger()); IEnumerable<TestResult> resultSet = null; var directories = new List<string>(); directories.Add(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)); directories.Add(Path.GetDirectoryName(settings.Assembly.Assembly)); directories.Add(Path.GetDirectoryName(plugin.Assembly)); using (var resolver = new AssemblyResolver(directories.ToArray())) { Logger.Write("About to create plugin {0} in {1} for {2}", plugin.Type, plugin.Assembly, id); var runner = getRunner(plugin); var currentDirectory = Environment.CurrentDirectory; try { if (runner == null) return _results; using (var server = new PipeServer(settings.PipeName)) { Logger.Write("Matching plugin identifier ({0}) to test identifier ({1})", runner.Identifier, id); if (!runner.Identifier.ToLower().Equals(id.ToLower()) && !id.ToLower().Equals("any")) return _results; Logger.Write("Checking whether assembly contains tests for {0}", id); if (!settings.Assembly.IsVerified && !runner.ContainsTestsFor(settings.Assembly.Assembly)) return _results; Logger.Write("Initializing channel"); runner.SetLiveFeedbackChannel(new TestFeedbackProvider(server)); var newCurrent = Path.GetDirectoryName(settings.Assembly.Assembly); Logger.Write("Setting current directory to " + newCurrent); Environment.CurrentDirectory = newCurrent; Logger.Write("Starting test run"); resultSet = runner.Run(settings); } } catch { throw; } finally { Environment.CurrentDirectory = currentDirectory; } } return resultSet; }
public IEnumerable<TestResult> Run(RunSettings settings) { var assembly = getAssembly(settings.Assembly.Assembly); var generator = new LinFu.DynamicProxy.ProxyFactory(); var proxy = new TestListenerProxy(_feedback, settings.Assembly.Assembly); _results = new List<TestResult>(); var mspec = Assembly.LoadFrom("Machine.Specifications.dll"); var type = mspec.GetType("Machine.Specifications.Runner.Impl.AppDomainRunner"); var iTestListener = mspec.GetType("Machine.Specifications.Runner.ISpecificationRunListener"); var dflt = mspec.GetType("Machine.Specifications.Runner.RunOptions").GetProperty("Default").GetValue(null, null); var listener = generator.CreateProxy(iTestListener, proxy, new Type[] {}); var runner = Activator.CreateInstance(type, new object[] {listener, dflt}); runTests(settings, assembly, runner); _results.AddRange(proxy.Results); return _results; }
public IEnumerable<TestResult> Run(RunSettings settings) { _results = new List<TestResult>(); var assembly = getAssembly(settings.Assembly.Assembly); if (_results.Count != 0) { } else { foreach (var result in runTests(settings, assembly)) { _results.Add(result); } } return _results; }
public IEnumerable<TestResult> Run(Plugin plugin, RunSettings options) { return plugin.New().Run(options); }
private bool shouldRunAllTests(RunSettings settings) { return !settings.Assembly.Tests.Any() && !settings.Assembly.Members.Any() && !settings.Assembly.Namespaces.Any(); }
private IEnumerable<TestResult> runTests(RunSettings settings, Assembly assembly) { if (shouldRunAllTests(settings)) { return runAllTests(assembly); } var results = new List<TestResult>(); foreach (var member in settings.Assembly.Members) results.AddRange(runAllOnType(assembly.GetType(member))); results.AddRange(runAllNamed(assembly, settings.Assembly.Tests)); //foreach (var ns in settings.Assembly.Namespaces) // results.AddRange(runAllInNameSpace(assembly)); return results; }
public IEnumerable<AutoTest.TestRunners.Shared.Results.TestResult> Run(RunSettings settings) { var logger = new XUnitLogger(); XunitProject project = new XunitProject(); var runner = settings.Assembly; // Set assembly externally as XUnit screws up the casing logger.SetCurrentAssembly(runner.Assembly); XunitProjectAssembly assembly = new XunitProjectAssembly { AssemblyFilename = runner.Assembly, ConfigFilename = null, ShadowCopy = false }; project.AddAssembly(assembly); foreach (XunitProjectAssembly asm in project.Assemblies) { using (ExecutorWrapper wrapper = new ExecutorWrapper(asm.AssemblyFilename, asm.ConfigFilename, asm.ShadowCopy)) { try { var xunitRunner = new TestRunner(wrapper, logger); //Run all tests if (runner.Tests.Count() == 0 && runner.Members.Count() == 0 && runner.Namespaces.Count() == 0) xunitRunner.RunAssembly(); //Run tests if (runner.Tests.Count() > 0) { foreach (var test in runner.Tests) xunitRunner.RunTest(test.Substring(0, test.LastIndexOf(".")), test.Substring(test.LastIndexOf(".") + 1, test.Length - (test.LastIndexOf(".") + 1))); } //Run members if (runner.Members.Count() > 0) { foreach (var member in runner.Members) xunitRunner.RunClass(member); } //Run namespaces if (runner.Namespaces.Count() > 0) { var loadedAssembly = Assembly.LoadFrom(runner.Assembly); var types = loadedAssembly.GetExportedTypes(); loadedAssembly = null; foreach (var ns in runner.Namespaces) { foreach (Type type in types) if (ns == null || type.Namespace == ns) xunitRunner.RunClass(type.FullName); } } } catch (ArgumentException ex) { Console.WriteLine(ex.Message); } } } return logger.Results; }
private TestResult getResult(RunSettings settings, IGrouping<Type, MethodInfo> fixture, celer.Core.RunResult x) { var result = new TestResult(Identifier, settings.Assembly.Assembly, fixture.Key.FullName, x.MillisecondsSpent, fixture.Key.FullName + "." + x.Test.Name, getState(x), getMessage(x)); result.AddStackLines(getStackLines(x.Exception)); return result; }
IEnumerable<Shared.Results.TestResult> Shared.IAutoTestNetTestRunner.Run(RunSettings settings) { return null; }