private void MarkTestFailed(
Test test, TestResult suiteResult, EventListener listener, ITestFilter filter)
{
if (test is TestSuite)
{
listener.SuiteStarted(test.TestName);
TestResult result = new TestResult(new TestInfo(test));
string msg = this.FixtureType == null
? "Parent SetUp failed"
: string.Format("Parent SetUp failed in {0}", this.FixtureType.Name);
result.Failure(msg, null, FailureSite.Parent);
MarkTestsFailed(test.Tests, result, listener, filter);
suiteResult.AddResult(result);
listener.SuiteFinished(result);
}
else
{
listener.TestStarted(test.TestName);
TestResult result = new TestResult(new TestInfo(test));
string msg = this.FixtureType == null
? "TestFixtureSetUp failed"
: string.Format("TestFixtureSetUp failed in {0}", this.FixtureType.Name);
result.Failure(msg, null, FailureSite.Parent);
suiteResult.AddResult(result);
listener.TestFinished(result);
}
}
private void RunAllTests(
TestResult suiteResult, EventListener listener, ITestFilter filter)
{
if (Properties.Contains("Timeout"))
{
TestContext.TestCaseTimeout = (int)Properties["Timeout"];
}
foreach (Test test in ArrayList.Synchronized(Tests))
{
if (filter.Pass(test))
{
RunState saveRunState = test.RunState;
if (test.RunState == RunState.Runnable && this.RunState != RunState.Runnable && this.RunState != RunState.Explicit)
{
test.RunState = this.RunState;
test.IgnoreReason = this.IgnoreReason;
}
TestResult result = test.Run(listener, filter);
suiteResult.AddResult(result);
if (saveRunState != test.RunState)
{
test.RunState = saveRunState;
test.IgnoreReason = null;
}
}
}
}
private void MarkTestNotRun(
Test test, ResultState resultState, string ignoreReason, TestResult suiteResult, EventListener listener, ITestFilter filter)
{
if (test is TestSuite)
{
listener.SuiteStarted(test.TestName);
TestResult result = new TestResult(new TestInfo(test));
result.SetResult(resultState, ignoreReason, null);
MarkTestsNotRun(test.Tests, resultState, ignoreReason, suiteResult, listener, filter);
suiteResult.AddResult(result);
listener.SuiteFinished(result);
}
else
{
listener.TestStarted(test.TestName);
TestResult result = new TestResult(new TestInfo(test));
result.SetResult(resultState, ignoreReason, null);
suiteResult.AddResult(result);
listener.TestFinished(result);
}
}
private void RunAllTests(
TestResult suiteResult, EventListener listener, ITestFilter filter)
{
if (Properties.Contains("Timeout"))
{
TestExecutionContext.CurrentContext.TestCaseTimeout = (int)Properties["Timeout"];
}
IDictionary settings = TestExecutionContext.CurrentContext.TestPackage.Settings;
bool stopOnError = settings.Contains("StopOnError") && (bool)settings["StopOnError"];
foreach (Test test in ArrayList.Synchronized(Tests))
{
if (filter.Pass(test))
{
RunState saveRunState = test.RunState;
if (test.RunState == RunState.Runnable && this.RunState != RunState.Runnable && this.RunState != RunState.Explicit)
{
test.RunState = this.RunState;
test.IgnoreReason = this.IgnoreReason;
}
TestResult result = test.Run(listener, filter);
log.Debug("Test result = " + result.ResultState);
suiteResult.AddResult(result);
log.Debug("Suite result = " + suiteResult.ResultState);
if (saveRunState != test.RunState)
{
test.RunState = saveRunState;
test.IgnoreReason = null;
}
if (result.ResultState == ResultState.Cancelled)
{
break;
}
if ((result.IsError || result.IsFailure) && stopOnError)
{
break;
}
}
}
}
private void AddResult(TestResult model, TestResult otherChild)
{
if (model.Results != null && model.Results.IsFixedSize)
{
var results = new ArrayList(model.Results);
// Finding result by ReferenceEquals not to be tight to private variable name
var resultsField = model.GetType()
.GetFields(BindingFlags.Default | BindingFlags.Instance | BindingFlags.NonPublic)
.FirstOrDefault(f => ReferenceEquals(f.GetValue(model), model.Results));
if (resultsField != null)
resultsField.SetValue(model, results);
}
model.AddResult(otherChild);
}
public TestResult CreateTestResult(TestType type, string fullName,
ResultState state = ResultState.Success,
Func<IEnumerable<TestResult>> children = null,
string stackTrace = null,
string description = null,
IList categories = null,
string agentName = null)
{
description = description ?? RandomValuesGenerator.GetRandomValue<string>();
agentName = agentName ?? RandomValuesGenerator.GetRandomValue<string>();
var splitted = (fullName ?? string.Empty).Split(new[]{'.'}, StringSplitOptions.RemoveEmptyEntries);
var childResults = children != null ? children() : new TestResult[0];
var testResult = new TestResult(new TestInfoWrapper
{
TestName = new TestName
{
FullName = fullName,
Name = splitted.Length > 0 ? splitted[splitted.Length - 1] : string.Empty
},
Categories = categories,
IsSuite = type != TestType.TestMethod,
TestCount = type == TestType.TestMethod ? 1 : RandomValuesGenerator.GetRandomValue<int>(),
TestType = type.ToString()
})
{
AssertCount = 1,
Time = RandomValuesGenerator.GetRandomValue<double>()
};
if (state != ResultState.Success)
testResult.SetResult(state,
description,
stackTrace);
else
testResult.Success(description);
testResult.SetAgentName(agentName);
foreach (var childResult in childResults)
{
testResult.AddResult(childResult);
}
return testResult;
}
public void SaveResults(string fileName)
{
TestResult all = new TestResult(new TestName());
foreach (TestResult r in mResultList) all.AddResult(r);
new XmlResultWriter(fileName).SaveTestResult(all);
}
private TestResult MergeResults(ITest test, List<TestResult> results)
{
if (results.Count == 1)
return results[0];
var rv = new TestResult(test);
foreach (var item in from result in
(from r in results
where r.Results != null
from TestResult r2 in r.Results
where r2.Executed
select r2)
group result by result.Name into g
let mergeSet = g.ToList()
select MergeResults(mergeSet[0].Test, mergeSet))
{
rv.AddResult(item);
rv.Time += item.Time;
}
return rv;
}
/// <summary>
/// When the complete rnning of the test is done, let us add
/// on all the relevant items.
/// </summary>
public void RunFinished(TestResult result )
{
if( !result.IsFailure ){
return;
}
foreach(TestResult t in this._results){
result.AddResult(t);
}
}
public void WhenResultIsCleared_NestedResultsAreAlsoCleared()
{
TestResult testCaseResult = new TestResult( testCase );
testCaseResult.Success();
TestResult testSuiteResult = new TestResult( testFixture );
testSuiteResult.AddResult( testCaseResult );
testSuiteResult.Success();
TestSuiteTreeNode node1 = new TestSuiteTreeNode( testSuiteResult );
TestSuiteTreeNode node2 = new TestSuiteTreeNode( testCaseResult );
node1.Nodes.Add( node2 );
Assert.AreEqual( TestSuiteTreeNode.SuccessIndex, node1.ImageIndex );
Assert.AreEqual( TestSuiteTreeNode.SuccessIndex, node1.SelectedImageIndex );
Assert.AreEqual( TestSuiteTreeNode.SuccessIndex, node2.ImageIndex );
Assert.AreEqual( TestSuiteTreeNode.SuccessIndex, node2.SelectedImageIndex );
node1.ClearResults();
Assert.AreEqual( TestSuiteTreeNode.InitIndex, node1.ImageIndex );
Assert.AreEqual( TestSuiteTreeNode.InitIndex, node1.SelectedImageIndex );
Assert.AreEqual( TestSuiteTreeNode.InitIndex, node2.ImageIndex );
Assert.AreEqual( TestSuiteTreeNode.InitIndex, node2.SelectedImageIndex );
}
private void MarkTestNotRun(
Test test, ResultState resultState, string ignoreReason, TestResult suiteResult, EventListener listener, ITestFilter filter)
{
if (test is TestSuite)
{
listener.SuiteStarted(test.TestName);
TestResult result = new TestResult( new TestInfo(test) );
result.SetResult( resultState, ignoreReason, null );
MarkTestsNotRun(test.Tests, resultState, ignoreReason, result, listener, filter);
suiteResult.AddResult(result);
listener.SuiteFinished(result);
}
else
{
listener.TestStarted(test.TestName);
TestResult result = new TestResult( new TestInfo(test) );
result.SetResult( resultState, ignoreReason, null );
suiteResult.AddResult(result);
listener.TestFinished(result);
}
}
private void MarkTestFailed(
Test test, TestResult suiteResult, EventListener listener, ITestFilter filter)
{
if (test is TestSuite)
{
listener.SuiteStarted(test.TestName);
TestResult result = new TestResult( new TestInfo(test) );
string msg = this.FixtureType == null
? "Parent SetUp failed"
: string.Format( "Parent SetUp failed in {0}", this.FixtureType.Name );
result.Failure(msg, null, FailureSite.Parent);
MarkTestsFailed(test.Tests, result, listener, filter);
suiteResult.AddResult(result);
listener.SuiteFinished(result);
}
else
{
listener.TestStarted(test.TestName);
TestResult result = new TestResult( new TestInfo(test) );
string msg = this.FixtureType == null
? "TestFixtureSetUp failed"
: string.Format( "TestFixtureSetUp failed in {0}", this.FixtureType.Name );
result.Failure(msg, null, FailureSite.Parent);
suiteResult.AddResult(result);
listener.TestFinished(result);
}
}
/// <summary>
/// Runs all of the tests defined in the specified assembly, specifically not running
/// any of the tests in the calling type.
/// </summary>
/// <param name="testNames">
/// The tests.
/// </param>
/// <param name="assembly">
/// The assembly.
/// </param>
/// <param name="numberOfThreads">
/// The number of threads.
/// </param>
/// <returns>
/// The result of running the suite of tests.
/// </returns>
public TestResult RunTestsInParallel(IEnumerable<string> testNames, Assembly assembly, int numberOfThreads)
{
// Initialize NUnit host service
CoreExtensions.Host.InitializeService();
_assemblyLocation = assembly.Location;
// Queue up the tests
foreach (var test in testNames)
{
_testQueue.Enqueue(test);
}
// Allocate threads
if (_testQueue.Count < numberOfThreads) numberOfThreads = _testQueue.Count;
var threads = new List<Thread>();
for (var i = 1; i <= numberOfThreads; i++)
{
threads.Add(new Thread(RunQueuedTests));
}
// Start test execution
foreach (var thread in threads)
{
thread.Start();
}
// Wait for completion
foreach (var thread in threads)
{
thread.Join();
}
// Build up the results for return
var finalResult = new TestResult(new TestName());
foreach (var result in _results)
{
if (result != null)
{
finalResult.AddResult(result);
}
}
return finalResult;
}
protected virtual TestResult InternalRun(ITestFilter filter, TestResult result)
{
if (this.runInParallel)
{
foreach (TestRunner runner in runners)
if (filter.Pass(runner.Test))
runner.BeginRun(this, filter);
result = this.EndRun();
}
else
{
foreach (TestRunner runner in runners)
if (filter.Pass(runner.Test))
result.AddResult(runner.Run(this, filter));
}
return result;
}
public override TestResult Run(EventListener listener, ITestFilter filter)
{
listener.SuiteStarted(TestName);
var result = new TestResult(this);
try {
ITestResult resultAdapter = new NUnitTestResultAdapter(result);
fixture.WithInitialized(_ => {
foreach(var item in Tests.Cast<Test>().Where(filter.Pass).Select(x => x.Run(listener, filter)))
result.AddResult(item);
}, ex => {
resultAdapter.BeforeFailure(ex);
foreach(Test item in Tests) {
var failure = new TestResult(item);
listener.TestStarted(item.TestName);
failure.Error(ex, FailureSite.SetUp);
listener.TestFinished(failure);
result.AddResult(failure);
}
}, resultAdapter.AfterFailure);
} finally {
listener.SuiteFinished(result);
}
return result;
}
public virtual TestResult Run(EventListener listener, ITestFilter filter, bool tracing, LoggingThreshold logLevel)
{
Log.Info("Run - EventListener={0}", listener.GetType().Name);
// Save active listener for derived classes
this.listener = listener;
ITest[] tests = new ITest[runners.Count];
for( int index = 0; index < runners.Count; index++ )
tests[index] = ((TestRunner)runners[index]).Test;
string name = this.testName.Name;
int count = this.CountTestCases(filter);
Log.Info("Signalling RunStarted({0},{1})", name, count);
this.listener.RunStarted(name, count);
long startTime = DateTime.Now.Ticks;
TestResult result = new TestResult(new TestInfo(testName, tests));
if (this.runInParallel)
{
foreach (TestRunner runner in runners)
if (filter.Pass(runner.Test))
runner.BeginRun(this, filter, tracing, logLevel);
result = this.EndRun();
}
else
{
foreach (TestRunner runner in runners)
if (filter.Pass(runner.Test))
result.AddResult(runner.Run(this, filter, tracing, logLevel));
}
long stopTime = DateTime.Now.Ticks;
double time = ((double)(stopTime - startTime)) / (double)TimeSpan.TicksPerSecond;
result.Time = time;
this.listener.RunFinished( result );
this.testResult = result;
return result;
}
private void RunAllTests(
TestResult suiteResult, EventListener listener, ITestFilter filter )
{
if (Properties.Contains("Timeout"))
TestContext.TestCaseTimeout = (int)Properties["Timeout"];
foreach (Test test in ArrayList.Synchronized(Tests))
{
if (filter.Pass(test))
{
RunState saveRunState = test.RunState;
if (test.RunState == RunState.Runnable && this.RunState != RunState.Runnable && this.RunState != RunState.Explicit )
{
test.RunState = this.RunState;
test.IgnoreReason = this.IgnoreReason;
}
TestResult result = test.Run(listener, filter);
suiteResult.AddResult(result);
if (saveRunState != test.RunState)
{
test.RunState = saveRunState;
test.IgnoreReason = null;
}
if (result.ResultState == ResultState.Cancelled)
break;
}
}
}
public virtual TestResult EndRun()
{
Log.Info("EndRun");
TestResult suiteResult = new TestResult(Test as TestInfo);
foreach (TestRunner runner in runners)
suiteResult.AddResult(runner.EndRun());
return suiteResult;
}
protected override int DoRun(Console.ConsoleOptions options, bool redirectOutput, bool redirectError, TestPackage package, TextWriter outWriter, TextWriter errorWriter, TestFilter testFilter, out TestResult result, EventCollector collector)
{
result = null;
var testRunner1 = new DefaultTestRunnerFactory().MakeTestRunner(package);
try
{
testRunner1.Load(package);
if (testRunner1.Test == null)
{
testRunner1.Unload();
System.Console.Error.WriteLine("Unable to locate fixture {0}", options.fixture);
return FIXTURE_NOT_FOUND;
}
}
finally
{
var disp = testRunner1 as IDisposable;
if (disp != null)
disp.Dispose();
}
result = new TestResult(new TestName { Name = "Global" });
var timer = new Stopwatch();
timer.Start();
var testRunnerId = 0;
{
var syncTestRunner = new DefaultTestRunnerFactory().MakeTestRunner(package);
var syncFilter = new AndFilter(testFilter, new SynchronousFilter());
var logger = new ConsoleLoggingEventListener(collector);
result.AddResult(RunPartition(redirectOutput, redirectError, package, outWriter, errorWriter, syncFilter, syncTestRunner, logger));
testRunnerId = syncTestRunner.ID;
}
var dep = 0;
var consoleOptions = options as ConsoleOptions;
if (options != null)
dep = consoleOptions.degreeofparallelism;
if (dep == 0)
dep = 4;
System.Console.WriteLine("Degree of Parallelism: {0}", dep);
var state = new AsynchronousFilterState(dep);
var asyncTestRunner = new ParallelTestRunner(testRunnerId + 1, dep, state);
result.AddResult(RunPartition(redirectOutput, redirectError, package, outWriter, errorWriter, testFilter, asyncTestRunner, collector));
if (consoleOptions != null && consoleOptions.retestfailures)
{
var failedTests = (from test in Flatten(result)
where test.Result.IsFailure
select test).ToList();
var failedTestFilter = new SimpleNameFilter(failedTests.Select(t => t.Result.FullName));
var retestTestRunner = new DefaultTestRunnerFactory().MakeTestRunner(package);
var retestFilter = new AndFilter(testFilter, failedTestFilter);
var logger = new ConsoleLoggingEventListener(collector);
var retestResults = RunPartition(redirectOutput, redirectError, package, outWriter, errorWriter, retestFilter, retestTestRunner, logger);
var newTests = Flatten(retestResults).ToDictionary(test => test.Result.FullName);
foreach (var failedTest in failedTests)
{
var newTest = newTests[failedTest.Result.FullName];
if (newTest.Result.IsSuccess)
{
failedTest.Parent.Results.Remove(failedTest.Result);
failedTest.Parent.Results.Add(newTest.Result);
}
}
}
result = MergeResults(result.Test, (TestResult)result.Results[0], (TestResult)result.Results[1]);
timer.Stop();
result.Time = timer.ElapsedTicks / (double)TimeSpan.TicksPerSecond;
return 0;
}
public void ClearNestedResults()
{
TestResult testCaseResult = new TestResult( testCaseInfo );
testCaseResult.Success();
TestResult testSuiteResult = new TestResult( fixtureInfo );
testSuiteResult.AddResult( testCaseResult );
testSuiteResult.Success();
TestSuiteTreeNode node1 = new TestSuiteTreeNode( testSuiteResult );
TestSuiteTreeNode node2 = new TestSuiteTreeNode( testCaseResult );
node1.Nodes.Add( node2 );
Assert.AreEqual( TestSuiteTreeNode.SuccessIndex, node1.ImageIndex );
Assert.AreEqual( TestSuiteTreeNode.SuccessIndex, node1.SelectedImageIndex );
Assert.AreEqual( TestSuiteTreeNode.SuccessIndex, node2.ImageIndex );
Assert.AreEqual( TestSuiteTreeNode.SuccessIndex, node2.SelectedImageIndex );
node1.ClearResults();
Assert.AreEqual( TestSuiteTreeNode.InitIndex, node1.ImageIndex );
Assert.AreEqual( TestSuiteTreeNode.InitIndex, node1.SelectedImageIndex );
Assert.AreEqual( TestSuiteTreeNode.InitIndex, node2.ImageIndex );
Assert.AreEqual( TestSuiteTreeNode.InitIndex, node2.SelectedImageIndex );
}
private void RunAllTests(TestResult suiteResult, EventListener listener, ITestFilter filter )
{
if (Properties.Contains("Timeout"))
TestExecutionContext.CurrentContext.TestCaseTimeout = (int)Properties["Timeout"];
IDictionary settings = TestExecutionContext.CurrentContext.TestPackage.Settings;
bool stopOnError = settings.Contains("StopOnError") && (bool)settings["StopOnError"];
foreach (Test test in ArrayList.Synchronized(Tests))
{
if (filter.Pass(test))
{
RunState saveRunState = test.RunState;
if (test.RunState == RunState.Runnable && this.RunState != RunState.Runnable && this.RunState != RunState.Explicit )
{
test.RunState = this.RunState;
test.IgnoreReason = this.IgnoreReason;
}
TestResult result = test.Run(listener, filter);
log.Debug("Test result = " + result.ResultState);
suiteResult.AddResult(result);
log.Debug("Suite result = " + suiteResult.ResultState);
if (saveRunState != test.RunState)
{
test.RunState = saveRunState;
test.IgnoreReason = null;
}
if (result.ResultState == ResultState.Cancelled)
break;
if ((result.IsError || result.IsFailure || result.ResultState == ResultState.NotRunnable) && stopOnError)
break;
}
}
}
