/// <summary>
/// Override Run, setting Fixture to that of the Parent.
/// </summary>
/// <param name="listener"></param>
/// <param name="filter"></param>
/// <returns></returns>
public override TestResult Run(EventListener listener, ITestFilter filter)
{
if (this.Parent != null)
{
this.Fixture = this.Parent.Fixture;
TestSuite suite = this.Parent as TestSuite;
if (suite != null)
{
this.setUpMethods = suite.GetSetUpMethods();
this.tearDownMethods = suite.GetTearDownMethods();
}
}
// DYNAMIC: Get the parameters, and add the methods here.
TestResult result = base.Run(listener, filter);
if (this.isTheory && result.ResultState == ResultState.Inconclusive)
{
result.SetResult(
ResultState.Failure,
this.TestCount == 0
? "No test cases were provided"
: "All test cases were inconclusive",
null);
}
return(result);
}
TestResult RunTest(EventListener listener)
{
listener.TestStarted(base.TestName);
TestResult nunitTestResult = new TestResult(this);
if (_pendingException != null)
{
nunitTestResult.Failure(_pendingException.Message, _pendingException.StackTrace, FailureSite.Test);
}
else if (RunState == RunState.NotRunnable)
{
nunitTestResult.SetResult(ResultState.NotRunnable, IgnoreReason, "", FailureSite.Test);
}
else
{
var testResult = SpecificationRunner.RunTest(this._testContext, new List<string>());
NativeTestResult.ApplyToNunitResult(testResult, nunitTestResult);
}
listener.TestFinished(nunitTestResult);
return nunitTestResult;
}
protected virtual void RecordException( Exception exception, TestResult testResult, FailureSite failureSite )
{
if (exception is NUnitException)
exception = exception.InnerException;
testResult.SetResult(NUnitFramework.GetResultState(exception), exception, failureSite);
}
protected virtual void RecordException(Exception exception, TestResult testResult)
{
if (exception is NUnitException)
{
exception = exception.InnerException;
}
testResult.SetResult(NUnitFramework.GetResultState(exception), exception);
}
/// <summary>
/// The doRun method is used to run a test internally.
/// It assumes that the caller is taking care of any
/// TestFixtureSetUp and TestFixtureTearDown needed.
/// </summary>
/// <param name="testResult">The result in which to record success or failure</param>
public virtual TestResult RunTest()
{
DateTime start = DateTime.Now;
TestResult testResult = new TestResult(this);
TestExecutionContext.CurrentContext.CurrentResult = testResult;
try
{
RunSetUp();
RunTestCase( testResult );
}
catch(Exception ex)
{
// doTestCase handles its own exceptions so
// if we're here it's a setup exception
if (ex is ThreadAbortException)
Thread.ResetAbort();
RecordException(ex, testResult, FailureSite.SetUp);
}
finally
{
RunTearDown( testResult );
DateTime stop = DateTime.Now;
TimeSpan span = stop.Subtract(start);
testResult.Time = (double)span.Ticks / (double)TimeSpan.TicksPerSecond;
if (testResult.IsSuccess)
{
if (this.Properties.Contains("MaxTime"))
{
int elapsedTime = (int)Math.Round(testResult.Time * 1000.0);
int maxTime = (int)this.Properties["MaxTime"];
if (maxTime > 0 && elapsedTime > maxTime)
testResult.Failure(
string.Format("Elapsed time of {0}ms exceeds maximum of {1}ms",
elapsedTime, maxTime),
null);
}
if (testResult.IsSuccess && testResult.Message == null &&
Environment.CurrentDirectory != TestExecutionContext.CurrentContext.prior.CurrentDirectory)
{
// TODO: Introduce a warning result state in NUnit 3.0
testResult.SetResult(ResultState.Success, "Warning: Test changed the CurrentDirectory", null);
}
}
}
log.Debug("Test result = " + testResult.ResultState);
return testResult;
}
public void WhenResultIsSet_IndexReflectsResultState(ResultState resultState, int expectedIndex)
{
TestSuiteTreeNode node = new TestSuiteTreeNode(new TestInfo(testCase));
TestResult result = new TestResult(testCase);
result.SetResult(resultState, null, null);
node.Result = result;
Assert.AreEqual(expectedIndex, node.ImageIndex);
Assert.AreEqual(expectedIndex, node.SelectedImageIndex);
Assert.AreEqual(resultState.ToString(), node.StatusText);
}
public override TestResult Run(EventListener listener, ITestFilter filter)
{
listener.TestStarted(base.TestName);
long ticks = DateTime.Now.Ticks;
TestResult result = new TestResult(this);
result.SetResult(ResultState.NotRunnable, "Specification not implemented.", null, FailureSite.Test);
double num3 = ((double)(DateTime.Now.Ticks - ticks)) / 10000000.0;
result.Time = num3;
listener.TestFinished(result);
return result;
}
public void SetResult_Inconclusive()
{
TestSuiteTreeNode node = new TestSuiteTreeNode(testCaseInfo);
TestResult result = new TestResult(testCaseInfo);
result.SetResult(ResultState.Inconclusive, null, null);
node.Result = result;
Assert.AreEqual("MockTest1", node.Result.Name);
Assert.AreEqual(TestSuiteTreeNode.InconclusiveIndex, node.ImageIndex);
Assert.AreEqual(TestSuiteTreeNode.InconclusiveIndex, node.SelectedImageIndex);
Assert.AreEqual(result.ResultState.ToString(), node.StatusText);
}
protected virtual void RecordException( Exception exception, TestResult testResult, FailureSite failureSite )
{
if (exception is NUnitException)
exception = exception.InnerException;
// Ensure that once a test is cancelled, it stays cancelled
ResultState finalResultState = testResult.ResultState == ResultState.Cancelled
? ResultState.Cancelled
: NUnitFramework.GetResultState(exception);
testResult.SetResult(finalResultState, exception, failureSite);
}
public override TestResult Run(EventListener listener, ITestFilter filter)
{
// TODO: Implement logic required for filtering.
listener.TestStarted(this.TestName);
long startTime = DateTime.Now.Ticks;
var result = new TestResult(this);
try
{
_engine.ExecuteScriptFile(_scriptPath);
result.Success();
}
catch (AssertionException assertEx)
{
result.SetResult(ResultState.Failure, assertEx.Message, assertEx.StackTrace, FailureSite.Test);
}
catch (InconclusiveException inconclusiveEx)
{
result.SetResult(ResultState.Inconclusive, inconclusiveEx.Message, inconclusiveEx.StackTrace);
}
catch (Exception ex)
{
result.Error(ex);
}
finally
{
long stopTime = DateTime.Now.Ticks;
double time = ((double)(stopTime - startTime)) / (double)TimeSpan.TicksPerSecond;
result.Time = time;
listener.TestFinished(result);
}
return result;
}
public void ProcessException(Exception exception, TestResult testResult)
{
if (exception is NUnitException)
{
exception = exception.InnerException;
}
if (IsExpectedExceptionType(exception))
{
if (IsExpectedMessageMatch(exception))
{
if (exceptionHandler != null)
{
Reflect.InvokeMethod(exceptionHandler, testMethod.Fixture, exception);
}
testResult.Success();
}
else
{
testResult.Failure(WrongTextMessage(exception), GetStackTrace(exception));
}
}
else
{
switch (NUnitFramework.GetResultState(exception))
{
case ResultState.Failure:
testResult.Failure(exception.Message, exception.StackTrace);
break;
case ResultState.Ignored:
testResult.Ignore(exception);
break;
case ResultState.Inconclusive:
testResult.SetResult(ResultState.Inconclusive, exception, FailureSite.Test);
break;
case ResultState.Success:
testResult.Success(exception.Message);
break;
default:
testResult.Failure(WrongTypeMessage(exception), GetStackTrace(exception));
break;
}
}
}
public void TestFinished_OutcomesAreCorrectlyTranslated(ResultState resultState, TestOutcome outcome, string message)
{
fakeNUnitResult.SetResult(resultState, message, null);
listener.TestFinished(fakeNUnitResult);
Assume.That(testLog.Events.Count, Is.EqualTo(2));
Assume.That(
testLog.Events[0].EventType,
Is.EqualTo(FakeFrameworkHandle.EventType.RecordEnd));
Assume.That(
testLog.Events[1].EventType,
Is.EqualTo(FakeFrameworkHandle.EventType.RecordResult));
Assert.AreEqual(outcome, testLog.Events[0].TestOutcome);
Assert.AreEqual(outcome, testLog.Events[1].TestResult.Outcome);
Assert.AreEqual(message, testLog.Events[1].TestResult.ErrorMessage);
}
public TestResult CreateTestResult(TestType type, string fullName,
ResultState state = ResultState.Success,
Func<IEnumerable<TestResult>> children = null,
string stackTrace = null,
string description = null,
IList categories = null,
string agentName = null)
{
description = description ?? RandomValuesGenerator.GetRandomValue<string>();
agentName = agentName ?? RandomValuesGenerator.GetRandomValue<string>();
var splitted = (fullName ?? string.Empty).Split(new[]{'.'}, StringSplitOptions.RemoveEmptyEntries);
var childResults = children != null ? children() : new TestResult[0];
var testResult = new TestResult(new TestInfoWrapper
{
TestName = new TestName
{
FullName = fullName,
Name = splitted.Length > 0 ? splitted[splitted.Length - 1] : string.Empty
},
Categories = categories,
IsSuite = type != TestType.TestMethod,
TestCount = type == TestType.TestMethod ? 1 : RandomValuesGenerator.GetRandomValue<int>(),
TestType = type.ToString()
})
{
AssertCount = 1,
Time = RandomValuesGenerator.GetRandomValue<double>()
};
if (state != ResultState.Success)
testResult.SetResult(state,
description,
stackTrace);
else
testResult.Success(description);
testResult.SetAgentName(agentName);
foreach (var childResult in childResults)
{
testResult.AddResult(childResult);
}
return testResult;
}
public static TestResult ApplyToNunitResult(TestResultShim shim, TestResult result)
{
switch (shim.Status)
{
case TestResultShim.Result.Successs:
result.Success();
break;
case TestResultShim.Result.Error:
result.SetResult(ResultState.Error, shim.FailureReason, shim.FailureStackTrace, GetNUnitFailureSite(shim.FailureSite));
break;
default:
throw new ArgumentOutOfRangeException();
}
result.Time = shim.ExecutionTime.Seconds;
return result;
}
public void SetUp()
{
MethodInfo fakeTestMethod = GetType().GetMethod("FakeTestMethod", BindingFlags.Instance | BindingFlags.NonPublic);
fakeNUnitTest = new NUnitTestMethod(fakeTestMethod);
fakeNUnitResult = new NUnitTestResult(fakeNUnitTest);
fakeNUnitResult.SetResult(ResultState.Success, "It passed!", null);
fakeNUnitResult.Time = 1.234;
testLog = new FakeFrameworkHandle();
testConverter = new TestConverter(new TestLogger(), ThisAssemblyPath);
testConverter.ConvertTestCase(fakeNUnitTest);
Assert.NotNull(testConverter.GetCachedTestCase(fakeNUnitTest.TestName.UniqueName));
listener = new NUnitEventListener(testLog, testConverter);
}
public void SetUp()
{
MethodInfo fakeTestMethod = GetType().GetMethod("FakeTestMethod", BindingFlags.Instance | BindingFlags.NonPublic);
fakeNUnitTest = new NUnitTestMethod(fakeTestMethod);
fakeNUnitResult = new NUnitTestResult(fakeNUnitTest);
fakeNUnitResult.SetResult(ResultState.Success, "It passed!", null);
fakeNUnitResult.Time = 1.234;
testLog = new FakeFrameworkHandle();
testConverter = new TestConverter(new TestLogger(), ThisAssemblyPath);
testConverter.ConvertTestCase(fakeNUnitTest);
Assert.NotNull(testConverter.GetCachedTestCase(fakeNUnitTest.TestName.UniqueName));
listener = new NUnitEventListener(testLog, testConverter);
}
public void CanMakeTestResultFromNUnitTestResult()
{
// This should put the TestCase in the cache
var cachedTestCase = testConverter.ConvertTestCase(fakeNUnitTest);
var nunitResult = new NUnitTestResult(fakeNUnitTest);
nunitResult.SetResult(ResultState.Success, "It passed!", null);
nunitResult.Time = 1.234;
var testResult = testConverter.ConvertTestResult(nunitResult);
var testCase = testResult.TestCase;
Assert.That(testCase, Is.SameAs(cachedTestCase));
CheckTestCase(testCase);
Assert.That(testResult.Outcome, Is.EqualTo(TestOutcome.Passed));
Assert.That(testResult.ErrorMessage, Is.EqualTo("It passed!"));
Assert.That(testResult.Duration, Is.EqualTo(TimeSpan.FromSeconds(1.234)));
}
public void CanMakeTestResultFromNUnitTestResult()
{
// This should put the TestCase in the cache
var cachedTestCase = testConverter.ConvertTestCase(fakeNUnitTest);
var nunitResult = new NUnitTestResult(fakeNUnitTest);
nunitResult.SetResult(ResultState.Success, "It passed!", null);
nunitResult.Time = 1.234;
var testResult = testConverter.ConvertTestResult(nunitResult);
var testCase = testResult.TestCase;
Assert.That(testCase, Is.SameAs(cachedTestCase));
CheckTestCase(testCase);
Assert.That(testResult.Outcome, Is.EqualTo(TestOutcome.Passed));
Assert.That(testResult.ErrorMessage, Is.EqualTo("It passed!"));
Assert.That(testResult.Duration, Is.EqualTo(TimeSpan.FromSeconds(1.234)));
}
private void MarkTestNotRun(
Test test, ResultState resultState, string ignoreReason, TestResult suiteResult, EventListener listener, ITestFilter filter)
{
if (test is TestSuite)
{
listener.SuiteStarted(test.TestName);
TestResult result = new TestResult(new TestInfo(test));
result.SetResult(resultState, ignoreReason, null);
MarkTestsNotRun(test.Tests, resultState, ignoreReason, suiteResult, listener, filter);
suiteResult.AddResult(result);
listener.SuiteFinished(result);
}
else
{
listener.TestStarted(test.TestName);
TestResult result = new TestResult(new TestInfo(test));
result.SetResult(resultState, ignoreReason, null);
suiteResult.AddResult(result);
listener.TestFinished(result);
}
}
/// <summary>
/// Override Run, setting Fixture to that of the Parent.
/// </summary>
/// <param name="listener"></param>
/// <param name="filter"></param>
/// <returns></returns>
public override TestResult Run(EventListener listener, ITestFilter filter)
{
if (this.Parent != null)
{
this.Fixture = this.Parent.Fixture;
TestSuite suite = this.Parent as TestSuite;
if (suite != null)
{
SetUpMethods = suite.SetUpMethods;
TearDownMethods = suite.TearDownMethods;
}
}
#if CLR_2_0 || CLR_4_0
this.actions = ActionsHelper.GetActionsFromAttributeProvider(this.method);
#endif
// DYNAMIC: Get the parameters, and add the methods here.
TestResult result = base.Run(listener, filter);
if (this.isTheory && result.ResultState == ResultState.Inconclusive)
{
result.SetResult(
ResultState.Failure,
this.TestCount == 0
? "No test cases were provided"
: "All test cases were inconclusive",
null);
}
Fixture = null;
SetUpMethods = null;
TearDownMethods = null;
#if CLR_2_0 || CLR_4_0
this.actions = null;
#endif
return(result);
}
private TestResult NUnitTestResult(IResultSummary concordionResult, string resultPath)
{
var testResult = new TestResult(this);
if (concordionResult.HasExceptions)
{
var errorDetails = concordionResult.ErrorDetails.First();
testResult.Error(errorDetails.Exception);
testResult.SetResult(testResult.ResultState,
resultPath + Environment.NewLine + testResult.Message,
testResult.StackTrace);
}
else if (concordionResult.HasFailures)
{
var failureDetails = concordionResult.FailureDetails.First();
testResult.Failure(resultPath + Environment.NewLine + failureDetails.Message, failureDetails.StackTrace);
} else
{
testResult.Success(resultPath);
}
return testResult;
}
/// <summary>
/// Gets the no available agents failure.
/// </summary>
/// <param name="test">The test.</param>
/// <param name="exception">The exception.</param>
/// <returns></returns>
public static TestResult GetNoAvailableAgentsFailure(TestUnitWithMetadata test, Exception exception)
{
var result = GetResultForTest(test.Test, exception);
TestResult suite = result;
while (suite.HasResults)
{
suite = (TestResult)suite.Results[0];
}
// A test can be either test suite or test case
// Here it is a suite
if (test.Test.Info.IsSuite && test.Children.Any())
{
foreach (var child in test.Children)
{
var childResult = new TestResult(child.Test.Info);
childResult.SetResult(ResultState.NotRunnable, exception, FailureSite.Parent);
suite.AddResult(childResult);
}
}
return result;
}
public void ProcessException(Exception exception, TestResult testResult)
{
if (exception is NUnitException)
exception = exception.InnerException;
if (IsExpectedExceptionType(exception))
{
if (IsExpectedMessageMatch(exception))
{
if (exceptionHandler != null)
Reflect.InvokeMethod(exceptionHandler, testMethod.Fixture, exception);
testResult.Success();
}
else
{
testResult.Failure(WrongTextMessage(exception), GetStackTrace(exception));
}
}
else
{
switch (NUnitFramework.GetResultState(exception))
{
case ResultState.Failure:
testResult.Failure(exception.Message, exception.StackTrace);
break;
case ResultState.Ignored:
testResult.Ignore(exception);
break;
case ResultState.Inconclusive:
testResult.SetResult(ResultState.Inconclusive, exception, FailureSite.Test);
break;
case ResultState.Success:
testResult.Success(exception.Message);
break;
default:
testResult.Failure(WrongTypeMessage(exception), GetStackTrace(exception));
break;
}
}
}
protected virtual void RecordException( Exception exception, TestResult testResult, FailureSite failureSite )
{
if (exception is NUnitException)
exception = exception.InnerException;
// Ensure that once a test is cancelled, it stays cancelled
ResultState finalResultState = testResult.ResultState == ResultState.Cancelled
? ResultState.Cancelled
: NUnitFramework.GetResultState(exception);
testResult.SetResult(finalResultState, exception, failureSite);
}
/// <summary>
/// The doRun method is used to run a test internally.
/// It assumes that the caller is taking care of any
/// TestFixtureSetUp and TestFixtureTearDown needed.
/// </summary>
/// <param name="testResult">The result in which to record success or failure</param>
public virtual TestResult RunTest()
{
DateTime start = DateTime.Now;
TestResult testResult = new TestResult(this);
TestExecutionContext.CurrentContext.CurrentResult = testResult;
try
{
RunSetUp();
#if CLR_2_0 || CLR_4_0
RunBeforeActions(testResult);
#endif
RunTestCase( testResult );
}
catch(Exception ex)
{
// doTestCase handles its own exceptions so
// if we're here it's a setup exception
if (ex is ThreadAbortException)
Thread.ResetAbort();
RecordException(ex, testResult, FailureSite.SetUp);
}
finally
{
#if CLR_2_0 || CLR_4_0
RunAfterActions(testResult);
#endif
RunTearDown( testResult );
DateTime stop = DateTime.Now;
TimeSpan span = stop.Subtract(start);
testResult.Time = (double)span.Ticks / (double)TimeSpan.TicksPerSecond;
if (testResult.IsSuccess)
{
if (this.Properties.Contains("MaxTime"))
{
int elapsedTime = (int)Math.Round(testResult.Time * 1000.0);
int maxTime = (int)this.Properties["MaxTime"];
if (maxTime > 0 && elapsedTime > maxTime)
testResult.Failure(
string.Format("Elapsed time of {0}ms exceeds maximum of {1}ms",
elapsedTime, maxTime),
null);
}
if (testResult.IsSuccess && testResult.Message == null &&
Environment.CurrentDirectory != TestExecutionContext.CurrentContext.prior.CurrentDirectory)
{
// TODO: Introduce a warning result state in NUnit 3.0
testResult.SetResult(ResultState.Success, "Warning: Test changed the CurrentDirectory", null);
}
}
}
log.Debug("Test result = " + testResult.ResultState);
return testResult;
}
private void MarkTestNotRun(
Test test, ResultState resultState, string ignoreReason, TestResult suiteResult, EventListener listener, ITestFilter filter)
{
if (test is TestSuite)
{
listener.SuiteStarted(test.TestName);
TestResult result = new TestResult( new TestInfo(test) );
result.SetResult( resultState, ignoreReason, null );
MarkTestsNotRun(test.Tests, resultState, ignoreReason, result, listener, filter);
suiteResult.AddResult(result);
listener.SuiteFinished(result);
}
else
{
listener.TestStarted(test.TestName);
TestResult result = new TestResult( new TestInfo(test) );
result.SetResult( resultState, ignoreReason, null );
suiteResult.AddResult(result);
listener.TestFinished(result);
}
}
/// <summary>
/// For each test that finished, if it failed, we need to
/// do some work.
/// </summary>
public void TestFinished(TestResult result)
{
if( ! result.IsFailure ){
return;
}
TestName name = new TestName();
string[] bits = this.items[this.random.Next(0, this.items.Count)];
try {
if( !string.IsNullOrEmpty(result.Description) ){
string prefix = "{TDG=";
int startIndex = result.Description.IndexOf(prefix);
if( startIndex > -1 ){
int endIndex = result.Description.IndexOf("}", startIndex);
int actualSI = startIndex + prefix.Length;
string component = result.Description.Substring(actualSI, endIndex - actualSI);
string[] componentBits = component.Split(new char[1]{','}, StringSplitOptions.RemoveEmptyEntries);
foreach(string s in componentBits){
string[] kvp = s.Split(':');
switch(kvp[0].ToLower()){
case "pref":
bits = this.items[this.random.Next(this.itemMap[kvp[1]][0], this.itemMap[kvp[1]][1])];
break;
case "weight":
// 'weight' is at 'bits[2]', so we will adjust it if they have
// specified so.
bits[2] = kvp[1];
break;
}
}
}
}
} catch(Exception){
// we can ignore this exception (it amounts to ignoring any customisation they've done
// of the execution; perhaps the data was not in the correct format, either way, it is
// not critical.
}
string type = bits[0].ToLower();
name.FullName = string.Format("TDG action of type '{0}'.", type);
string actionDetails = bits[1];
switch(type){
case "exercise":
if( !string.IsNullOrEmpty(bits[2]) &&
!string.IsNullOrEmpty(bits[3]) &&
!string.IsNullOrEmpty(bits[4])
) {
// Need to set the variables into the string
actionDetails = string.Format(
actionDetails,
this.random.Next(
int.Parse(bits[2]) * int.Parse(bits[3]),
int.Parse(bits[2]) * int.Parse(bits[4]) + 1
)
);
}
break;
default:
// Nothing to do, just display
break;
}
TestResult test = new TestResult(name);
// For NUnit-2.5.2.9222
// test.SetResult(ResultState.Failure, new Exception(actionDetails));
// For NUnit-2.6.0.12051
test.SetResult(ResultState.Failure, new Exception(actionDetails), FailureSite.Test);
_results.Add(test);
}
/// <summary>
/// Gets the result.
/// </summary>
/// <param name="exception">The exception.</param>
/// <param name="description">The description.</param>
/// <param name="testType">Type of the test.</param>
/// <param name="fullName">The full name.</param>
/// <param name="isSuite">if set to <c>true</c> [is suite].</param>
/// <returns></returns>
public static TestResult GetResult(Exception exception, string description, string testType, string fullName, bool isSuite = true)
{
var result = new TestResult(new TestDataProvider
{
TestName = new TestName
{
FullName = fullName,
Name = fullName
},
IsSuite = isSuite,
TestType = testType,
});
result.SetResult(ResultState.NotRunnable,
exception == null ? description : exception.Message,
exception == null ? null : exception.StackTrace);
return result;
}
private static void HandleException(Exception e, TestResult testResult)
{
if (e is NUnitException)
e = e.InnerException;
testResult.SetResult(NUnitFramework.GetResultState(e), e);
}
public static void Error(TestResult testResult, Exception exception, IEnumerable<string> traceMessages, FailureSite failureSite = FailureSite.Test)
{
traceMessages = traceMessages ?? new List<string>();
testResult.SetResult(ResultState.Error, BuildMessage(exception), BuildStackTrace(exception, testResult.Test, traceMessages), failureSite);
}
protected virtual void RecordException( Exception exception, TestResult testResult, FailureSite failureSite )
{
if (exception is NUnitException)
exception = exception.InnerException;
testResult.SetResult(NUnitFramework.GetResultState(exception), exception, failureSite);
}