private void MarkTestFailed( Test test, TestResult suiteResult, EventListener listener, ITestFilter filter) { if (test is TestSuite) { listener.SuiteStarted(test.TestName); TestResult result = new TestResult(new TestInfo(test)); string msg = this.FixtureType == null ? "Parent SetUp failed" : string.Format("Parent SetUp failed in {0}", this.FixtureType.Name); result.Failure(msg, null, FailureSite.Parent); MarkTestsFailed(test.Tests, result, listener, filter); suiteResult.AddResult(result); listener.SuiteFinished(result); } else { listener.TestStarted(test.TestName); TestResult result = new TestResult(new TestInfo(test)); string msg = this.FixtureType == null ? "TestFixtureSetUp failed" : string.Format("TestFixtureSetUp failed in {0}", this.FixtureType.Name); result.Failure(msg, null, FailureSite.Parent); suiteResult.AddResult(result); listener.TestFinished(result); } }
private void RunAllTests( TestResult suiteResult, EventListener listener, ITestFilter filter) { if (Properties.Contains("Timeout")) { TestContext.TestCaseTimeout = (int)Properties["Timeout"]; } foreach (Test test in ArrayList.Synchronized(Tests)) { if (filter.Pass(test)) { RunState saveRunState = test.RunState; if (test.RunState == RunState.Runnable && this.RunState != RunState.Runnable && this.RunState != RunState.Explicit) { test.RunState = this.RunState; test.IgnoreReason = this.IgnoreReason; } TestResult result = test.Run(listener, filter); suiteResult.AddResult(result); if (saveRunState != test.RunState) { test.RunState = saveRunState; test.IgnoreReason = null; } } } }
private void MarkTestNotRun( Test test, ResultState resultState, string ignoreReason, TestResult suiteResult, EventListener listener, ITestFilter filter) { if (test is TestSuite) { listener.SuiteStarted(test.TestName); TestResult result = new TestResult(new TestInfo(test)); result.SetResult(resultState, ignoreReason, null); MarkTestsNotRun(test.Tests, resultState, ignoreReason, suiteResult, listener, filter); suiteResult.AddResult(result); listener.SuiteFinished(result); } else { listener.TestStarted(test.TestName); TestResult result = new TestResult(new TestInfo(test)); result.SetResult(resultState, ignoreReason, null); suiteResult.AddResult(result); listener.TestFinished(result); } }
private void RunAllTests( TestResult suiteResult, EventListener listener, ITestFilter filter) { if (Properties.Contains("Timeout")) { TestExecutionContext.CurrentContext.TestCaseTimeout = (int)Properties["Timeout"]; } IDictionary settings = TestExecutionContext.CurrentContext.TestPackage.Settings; bool stopOnError = settings.Contains("StopOnError") && (bool)settings["StopOnError"]; foreach (Test test in ArrayList.Synchronized(Tests)) { if (filter.Pass(test)) { RunState saveRunState = test.RunState; if (test.RunState == RunState.Runnable && this.RunState != RunState.Runnable && this.RunState != RunState.Explicit) { test.RunState = this.RunState; test.IgnoreReason = this.IgnoreReason; } TestResult result = test.Run(listener, filter); log.Debug("Test result = " + result.ResultState); suiteResult.AddResult(result); log.Debug("Suite result = " + suiteResult.ResultState); if (saveRunState != test.RunState) { test.RunState = saveRunState; test.IgnoreReason = null; } if (result.ResultState == ResultState.Cancelled) { break; } if ((result.IsError || result.IsFailure) && stopOnError) { break; } } } }
private void AddResult(TestResult model, TestResult otherChild) { if (model.Results != null && model.Results.IsFixedSize) { var results = new ArrayList(model.Results); // Finding result by ReferenceEquals not to be tight to private variable name var resultsField = model.GetType() .GetFields(BindingFlags.Default | BindingFlags.Instance | BindingFlags.NonPublic) .FirstOrDefault(f => ReferenceEquals(f.GetValue(model), model.Results)); if (resultsField != null) resultsField.SetValue(model, results); } model.AddResult(otherChild); }
public TestResult CreateTestResult(TestType type, string fullName, ResultState state = ResultState.Success, Func<IEnumerable<TestResult>> children = null, string stackTrace = null, string description = null, IList categories = null, string agentName = null) { description = description ?? RandomValuesGenerator.GetRandomValue<string>(); agentName = agentName ?? RandomValuesGenerator.GetRandomValue<string>(); var splitted = (fullName ?? string.Empty).Split(new[]{'.'}, StringSplitOptions.RemoveEmptyEntries); var childResults = children != null ? children() : new TestResult[0]; var testResult = new TestResult(new TestInfoWrapper { TestName = new TestName { FullName = fullName, Name = splitted.Length > 0 ? splitted[splitted.Length - 1] : string.Empty }, Categories = categories, IsSuite = type != TestType.TestMethod, TestCount = type == TestType.TestMethod ? 1 : RandomValuesGenerator.GetRandomValue<int>(), TestType = type.ToString() }) { AssertCount = 1, Time = RandomValuesGenerator.GetRandomValue<double>() }; if (state != ResultState.Success) testResult.SetResult(state, description, stackTrace); else testResult.Success(description); testResult.SetAgentName(agentName); foreach (var childResult in childResults) { testResult.AddResult(childResult); } return testResult; }
public void SaveResults(string fileName) { TestResult all = new TestResult(new TestName()); foreach (TestResult r in mResultList) all.AddResult(r); new XmlResultWriter(fileName).SaveTestResult(all); }
private TestResult MergeResults(ITest test, List<TestResult> results) { if (results.Count == 1) return results[0]; var rv = new TestResult(test); foreach (var item in from result in (from r in results where r.Results != null from TestResult r2 in r.Results where r2.Executed select r2) group result by result.Name into g let mergeSet = g.ToList() select MergeResults(mergeSet[0].Test, mergeSet)) { rv.AddResult(item); rv.Time += item.Time; } return rv; }
/// <summary> /// When the complete rnning of the test is done, let us add /// on all the relevant items. /// </summary> public void RunFinished(TestResult result ) { if( !result.IsFailure ){ return; } foreach(TestResult t in this._results){ result.AddResult(t); } }
public void WhenResultIsCleared_NestedResultsAreAlsoCleared() { TestResult testCaseResult = new TestResult( testCase ); testCaseResult.Success(); TestResult testSuiteResult = new TestResult( testFixture ); testSuiteResult.AddResult( testCaseResult ); testSuiteResult.Success(); TestSuiteTreeNode node1 = new TestSuiteTreeNode( testSuiteResult ); TestSuiteTreeNode node2 = new TestSuiteTreeNode( testCaseResult ); node1.Nodes.Add( node2 ); Assert.AreEqual( TestSuiteTreeNode.SuccessIndex, node1.ImageIndex ); Assert.AreEqual( TestSuiteTreeNode.SuccessIndex, node1.SelectedImageIndex ); Assert.AreEqual( TestSuiteTreeNode.SuccessIndex, node2.ImageIndex ); Assert.AreEqual( TestSuiteTreeNode.SuccessIndex, node2.SelectedImageIndex ); node1.ClearResults(); Assert.AreEqual( TestSuiteTreeNode.InitIndex, node1.ImageIndex ); Assert.AreEqual( TestSuiteTreeNode.InitIndex, node1.SelectedImageIndex ); Assert.AreEqual( TestSuiteTreeNode.InitIndex, node2.ImageIndex ); Assert.AreEqual( TestSuiteTreeNode.InitIndex, node2.SelectedImageIndex ); }
private void MarkTestNotRun( Test test, ResultState resultState, string ignoreReason, TestResult suiteResult, EventListener listener, ITestFilter filter) { if (test is TestSuite) { listener.SuiteStarted(test.TestName); TestResult result = new TestResult( new TestInfo(test) ); result.SetResult( resultState, ignoreReason, null ); MarkTestsNotRun(test.Tests, resultState, ignoreReason, result, listener, filter); suiteResult.AddResult(result); listener.SuiteFinished(result); } else { listener.TestStarted(test.TestName); TestResult result = new TestResult( new TestInfo(test) ); result.SetResult( resultState, ignoreReason, null ); suiteResult.AddResult(result); listener.TestFinished(result); } }
private void MarkTestFailed( Test test, TestResult suiteResult, EventListener listener, ITestFilter filter) { if (test is TestSuite) { listener.SuiteStarted(test.TestName); TestResult result = new TestResult( new TestInfo(test) ); string msg = this.FixtureType == null ? "Parent SetUp failed" : string.Format( "Parent SetUp failed in {0}", this.FixtureType.Name ); result.Failure(msg, null, FailureSite.Parent); MarkTestsFailed(test.Tests, result, listener, filter); suiteResult.AddResult(result); listener.SuiteFinished(result); } else { listener.TestStarted(test.TestName); TestResult result = new TestResult( new TestInfo(test) ); string msg = this.FixtureType == null ? "TestFixtureSetUp failed" : string.Format( "TestFixtureSetUp failed in {0}", this.FixtureType.Name ); result.Failure(msg, null, FailureSite.Parent); suiteResult.AddResult(result); listener.TestFinished(result); } }
/// <summary> /// Runs all of the tests defined in the specified assembly, specifically not running /// any of the tests in the calling type. /// </summary> /// <param name="testNames"> /// The tests. /// </param> /// <param name="assembly"> /// The assembly. /// </param> /// <param name="numberOfThreads"> /// The number of threads. /// </param> /// <returns> /// The result of running the suite of tests. /// </returns> public TestResult RunTestsInParallel(IEnumerable<string> testNames, Assembly assembly, int numberOfThreads) { // Initialize NUnit host service CoreExtensions.Host.InitializeService(); _assemblyLocation = assembly.Location; // Queue up the tests foreach (var test in testNames) { _testQueue.Enqueue(test); } // Allocate threads if (_testQueue.Count < numberOfThreads) numberOfThreads = _testQueue.Count; var threads = new List<Thread>(); for (var i = 1; i <= numberOfThreads; i++) { threads.Add(new Thread(RunQueuedTests)); } // Start test execution foreach (var thread in threads) { thread.Start(); } // Wait for completion foreach (var thread in threads) { thread.Join(); } // Build up the results for return var finalResult = new TestResult(new TestName()); foreach (var result in _results) { if (result != null) { finalResult.AddResult(result); } } return finalResult; }
protected virtual TestResult InternalRun(ITestFilter filter, TestResult result) { if (this.runInParallel) { foreach (TestRunner runner in runners) if (filter.Pass(runner.Test)) runner.BeginRun(this, filter); result = this.EndRun(); } else { foreach (TestRunner runner in runners) if (filter.Pass(runner.Test)) result.AddResult(runner.Run(this, filter)); } return result; }
public override TestResult Run(EventListener listener, ITestFilter filter) { listener.SuiteStarted(TestName); var result = new TestResult(this); try { ITestResult resultAdapter = new NUnitTestResultAdapter(result); fixture.WithInitialized(_ => { foreach(var item in Tests.Cast<Test>().Where(filter.Pass).Select(x => x.Run(listener, filter))) result.AddResult(item); }, ex => { resultAdapter.BeforeFailure(ex); foreach(Test item in Tests) { var failure = new TestResult(item); listener.TestStarted(item.TestName); failure.Error(ex, FailureSite.SetUp); listener.TestFinished(failure); result.AddResult(failure); } }, resultAdapter.AfterFailure); } finally { listener.SuiteFinished(result); } return result; }
public virtual TestResult Run(EventListener listener, ITestFilter filter, bool tracing, LoggingThreshold logLevel) { Log.Info("Run - EventListener={0}", listener.GetType().Name); // Save active listener for derived classes this.listener = listener; ITest[] tests = new ITest[runners.Count]; for( int index = 0; index < runners.Count; index++ ) tests[index] = ((TestRunner)runners[index]).Test; string name = this.testName.Name; int count = this.CountTestCases(filter); Log.Info("Signalling RunStarted({0},{1})", name, count); this.listener.RunStarted(name, count); long startTime = DateTime.Now.Ticks; TestResult result = new TestResult(new TestInfo(testName, tests)); if (this.runInParallel) { foreach (TestRunner runner in runners) if (filter.Pass(runner.Test)) runner.BeginRun(this, filter, tracing, logLevel); result = this.EndRun(); } else { foreach (TestRunner runner in runners) if (filter.Pass(runner.Test)) result.AddResult(runner.Run(this, filter, tracing, logLevel)); } long stopTime = DateTime.Now.Ticks; double time = ((double)(stopTime - startTime)) / (double)TimeSpan.TicksPerSecond; result.Time = time; this.listener.RunFinished( result ); this.testResult = result; return result; }
private void RunAllTests( TestResult suiteResult, EventListener listener, ITestFilter filter ) { if (Properties.Contains("Timeout")) TestContext.TestCaseTimeout = (int)Properties["Timeout"]; foreach (Test test in ArrayList.Synchronized(Tests)) { if (filter.Pass(test)) { RunState saveRunState = test.RunState; if (test.RunState == RunState.Runnable && this.RunState != RunState.Runnable && this.RunState != RunState.Explicit ) { test.RunState = this.RunState; test.IgnoreReason = this.IgnoreReason; } TestResult result = test.Run(listener, filter); suiteResult.AddResult(result); if (saveRunState != test.RunState) { test.RunState = saveRunState; test.IgnoreReason = null; } if (result.ResultState == ResultState.Cancelled) break; } } }
public virtual TestResult EndRun() { Log.Info("EndRun"); TestResult suiteResult = new TestResult(Test as TestInfo); foreach (TestRunner runner in runners) suiteResult.AddResult(runner.EndRun()); return suiteResult; }
protected override int DoRun(Console.ConsoleOptions options, bool redirectOutput, bool redirectError, TestPackage package, TextWriter outWriter, TextWriter errorWriter, TestFilter testFilter, out TestResult result, EventCollector collector) { result = null; var testRunner1 = new DefaultTestRunnerFactory().MakeTestRunner(package); try { testRunner1.Load(package); if (testRunner1.Test == null) { testRunner1.Unload(); System.Console.Error.WriteLine("Unable to locate fixture {0}", options.fixture); return FIXTURE_NOT_FOUND; } } finally { var disp = testRunner1 as IDisposable; if (disp != null) disp.Dispose(); } result = new TestResult(new TestName { Name = "Global" }); var timer = new Stopwatch(); timer.Start(); var testRunnerId = 0; { var syncTestRunner = new DefaultTestRunnerFactory().MakeTestRunner(package); var syncFilter = new AndFilter(testFilter, new SynchronousFilter()); var logger = new ConsoleLoggingEventListener(collector); result.AddResult(RunPartition(redirectOutput, redirectError, package, outWriter, errorWriter, syncFilter, syncTestRunner, logger)); testRunnerId = syncTestRunner.ID; } var dep = 0; var consoleOptions = options as ConsoleOptions; if (options != null) dep = consoleOptions.degreeofparallelism; if (dep == 0) dep = 4; System.Console.WriteLine("Degree of Parallelism: {0}", dep); var state = new AsynchronousFilterState(dep); var asyncTestRunner = new ParallelTestRunner(testRunnerId + 1, dep, state); result.AddResult(RunPartition(redirectOutput, redirectError, package, outWriter, errorWriter, testFilter, asyncTestRunner, collector)); if (consoleOptions != null && consoleOptions.retestfailures) { var failedTests = (from test in Flatten(result) where test.Result.IsFailure select test).ToList(); var failedTestFilter = new SimpleNameFilter(failedTests.Select(t => t.Result.FullName)); var retestTestRunner = new DefaultTestRunnerFactory().MakeTestRunner(package); var retestFilter = new AndFilter(testFilter, failedTestFilter); var logger = new ConsoleLoggingEventListener(collector); var retestResults = RunPartition(redirectOutput, redirectError, package, outWriter, errorWriter, retestFilter, retestTestRunner, logger); var newTests = Flatten(retestResults).ToDictionary(test => test.Result.FullName); foreach (var failedTest in failedTests) { var newTest = newTests[failedTest.Result.FullName]; if (newTest.Result.IsSuccess) { failedTest.Parent.Results.Remove(failedTest.Result); failedTest.Parent.Results.Add(newTest.Result); } } } result = MergeResults(result.Test, (TestResult)result.Results[0], (TestResult)result.Results[1]); timer.Stop(); result.Time = timer.ElapsedTicks / (double)TimeSpan.TicksPerSecond; return 0; }
public void ClearNestedResults() { TestResult testCaseResult = new TestResult( testCaseInfo ); testCaseResult.Success(); TestResult testSuiteResult = new TestResult( fixtureInfo ); testSuiteResult.AddResult( testCaseResult ); testSuiteResult.Success(); TestSuiteTreeNode node1 = new TestSuiteTreeNode( testSuiteResult ); TestSuiteTreeNode node2 = new TestSuiteTreeNode( testCaseResult ); node1.Nodes.Add( node2 ); Assert.AreEqual( TestSuiteTreeNode.SuccessIndex, node1.ImageIndex ); Assert.AreEqual( TestSuiteTreeNode.SuccessIndex, node1.SelectedImageIndex ); Assert.AreEqual( TestSuiteTreeNode.SuccessIndex, node2.ImageIndex ); Assert.AreEqual( TestSuiteTreeNode.SuccessIndex, node2.SelectedImageIndex ); node1.ClearResults(); Assert.AreEqual( TestSuiteTreeNode.InitIndex, node1.ImageIndex ); Assert.AreEqual( TestSuiteTreeNode.InitIndex, node1.SelectedImageIndex ); Assert.AreEqual( TestSuiteTreeNode.InitIndex, node2.ImageIndex ); Assert.AreEqual( TestSuiteTreeNode.InitIndex, node2.SelectedImageIndex ); }
private void RunAllTests(TestResult suiteResult, EventListener listener, ITestFilter filter ) { if (Properties.Contains("Timeout")) TestExecutionContext.CurrentContext.TestCaseTimeout = (int)Properties["Timeout"]; IDictionary settings = TestExecutionContext.CurrentContext.TestPackage.Settings; bool stopOnError = settings.Contains("StopOnError") && (bool)settings["StopOnError"]; foreach (Test test in ArrayList.Synchronized(Tests)) { if (filter.Pass(test)) { RunState saveRunState = test.RunState; if (test.RunState == RunState.Runnable && this.RunState != RunState.Runnable && this.RunState != RunState.Explicit ) { test.RunState = this.RunState; test.IgnoreReason = this.IgnoreReason; } TestResult result = test.Run(listener, filter); log.Debug("Test result = " + result.ResultState); suiteResult.AddResult(result); log.Debug("Suite result = " + suiteResult.ResultState); if (saveRunState != test.RunState) { test.RunState = saveRunState; test.IgnoreReason = null; } if (result.ResultState == ResultState.Cancelled) break; if ((result.IsError || result.IsFailure || result.ResultState == ResultState.NotRunnable) && stopOnError) break; } } }