/// <summary>
/// Private method to return a test case, optionally ignored on the Linux platform.
/// We use this since the Platform attribute is not supported on TestCaseData.
/// </summary>
private TestCaseData GetTestCase(MethodInfo method, ResultState resultState, int assertionCount, bool ignoreOnLinux)
{
var data = new TestCaseData(method, resultState, assertionCount);
if (ON_LINUX && ignoreOnLinux)
data = data.Ignore("Intermittent failure on Linux");
return data;
}
public void Label_ConstructorWithOneArguments_ReturnsStringEmpty()
{
// Arrange N/A
ResultState resultState = new ResultState(TestStatus.Failed);
Assert.AreEqual(string.Empty, resultState.Label);
}
public void Label_ConstructorWithTwoArguments_ReturnsConstructorArgumentLabel(string label)
{
// Arrange N/A
ResultState resultState = new ResultState(TestStatus.Failed, label);
Assert.AreEqual(label, resultState.Label);
}
public void Status_ConstructorWithTwoArguments_ReturnsConstructorArgumentStatus(TestStatus status)
{
// Arrange N/A
ResultState resultState = new ResultState(status, string.Empty);
Assert.AreEqual(status, resultState.Status);
}
public void RunTests(MethodInfo method, ResultState resultState, int assertionCount)
{
var test = _builder.BuildFrom(method);
var result = TestBuilder.RunTest(test, _testObject);
Assert.That(result.ResultState, Is.EqualTo(resultState), "Wrong result state");
Assert.That(result.AssertCount, Is.EqualTo(assertionCount), "Wrong assertion count");
}
public void Label_ConstructorWithTwoArgumentsLabelArgumentIsNull_ReturnsEmptyString()
{
// Arrange N/A
ResultState resultState = new ResultState(TestStatus.Failed, null);
Assert.AreEqual(string.Empty, resultState.Label);
}
public static void IsRunnable(Type type, string name, ResultState resultState)
{
Test test = TestBuilder.MakeTestFromMethod(type, name);
Assert.That(test.RunState, Is.EqualTo(RunState.Runnable));
object testObject = Reflect.Construct(type);
ITestResult result = TestBuilder.RunTest(test, testObject);
if (result.HasChildren) // In case it's a parameterized method
result = result.Children.ToArray()[0];
Assert.That(result.ResultState, Is.EqualTo(resultState));
}
public static void IsRunnable(Type type, string name, ResultState resultState)
{
Test test = TestBuilder.MakeTestFromMethod(type, name);
Assert.That(test.RunState, Is.EqualTo(RunState.Runnable));
object testObject = Activator.CreateInstance(type);
ITestResult result = TestBuilder.RunTest(test, testObject);
if (result.HasChildren) // In case it's a parameterized method
result = (ITestResult)result.Children[0];
Assert.That(result.ResultState, Is.EqualTo(resultState));
}
public void AddTestRun(int suiteId, int testCaseId, ResultState caseResult, string errorOuput, string comment)
{
if (RunData.Testrun == null) throw new Exception("InitializeTestRun method has not been called, please call this at the start of the test run");
//Get test points
ITestPointCollection testpoints = RunData.Plan.QueryTestPoints("SELECT * FROM TestPoint WHERE SuiteId = " + suiteId + " and TestCaseId = " + testCaseId);
foreach (ITestPoint tp in testpoints)
{
RunData.Testrun.AddTestPoint(tp, null);
}
RunData.RunCacheData.Add(new RunCache() { CaseResult = caseResult, Comment = comment, ErrorOutput = errorOuput });
}
private static TestOutcome TranslateTestResult(ResultState result)
{
if (result == ResultState.Success) return TestOutcome.Passed;
if (result == ResultState.Cancelled) return TestOutcome.Aborted;
if (result == ResultState.ChildFailure) return TestOutcome.Blocked;
if (result == ResultState.Error) return TestOutcome.Error;
if (result == ResultState.Explicit) return TestOutcome.Failed;
if (result == ResultState.Failure) return TestOutcome.Failed;
if (result == ResultState.Ignored) return TestOutcome.NotExecuted;
if (result == ResultState.Inconclusive) return TestOutcome.Inconclusive;
if (result == ResultState.NotRunnable) return TestOutcome.NotExecuted;
if (result == ResultState.SetUpError) return TestOutcome.Blocked;
if (result == ResultState.SetUpFailure) return TestOutcome.Blocked;
if (result == ResultState.Cancelled) return TestOutcome.Aborted;
if (result == ResultState.Skipped) return TestOutcome.NotExecuted;
if (result == ResultState.TearDownError) return TestOutcome.Unspecified;
throw new Exception(String.Format("Could not translate result from NUnit result was {0}", result));
}
private void SkipChildren(TestSuite suite, ResultState resultState, string message)
{
foreach (Test child in suite.Tests)
{
if (_childFilter.Pass(child))
{
TestResult childResult = child.MakeTestResult();
childResult.SetResult(resultState, message);
Result.AddResult(childResult);
// Some runners may depend on getting the TestFinished event
// even for tests that have been skipped at a higher level.
Context.Listener.TestFinished(childResult);
if (child.IsSuite)
SkipChildren((TestSuite)child, resultState, message);
}
}
}
private void SkipFixture(ResultState resultState, string message, string stackTrace)
{
Result.SetResult(resultState.WithSite(FailureSite.SetUp), message, StackFilter.Filter(stackTrace));
SkipChildren(_suite, resultState.WithSite(FailureSite.Parent), "OneTimeSetUp: " + message);
}
private void SkipFixture(ResultState resultState, string message, string stackTrace)
{
Result.SetResult(resultState, message, stackTrace);
SkipChildren();
}
/// <summary>
/// Test whether this ResultState has the same Status and Label
/// as another one. In other words, the whether two are equal
/// ignoring the Site.
/// </summary>
/// <param name="other"></param>
/// <returns></returns>
public bool Matches(ResultState other)
{
return(Status == other.Status && Label == other.Label);
}
private ITestResult CheckResult(string methodName, ResultState expectedResultState, params string[] assertionMessageRegex)
{
ITestResult result = TestBuilder.RunTestCase(typeof(AssertMultipleFixture), methodName);
Assert.That(result.ResultState, Is.EqualTo(expectedResultState), "ResultState");
Assert.That(result.AssertionResults.Count, Is.EqualTo(assertionMessageRegex.Length), "Number of AssertionResults");
Assert.That(result.StackTrace, Is.Not.Null.And.Contains(methodName), "StackTrace");
if (result.AssertionResults.Count > 0)
{
int numFailures = result.AssertionResults.Count;
if (expectedResultState == ResultState.Error)
--numFailures;
Assert.That(result.Message, Contains.Substring("One or more failures in Multiple Assert block"));
int i = 0;
foreach (var assertion in result.AssertionResults)
{
// Since the order of argument evaluation is not guaranteed, we don't
// want 'i' to appear more than once in the Assert statement.
string errmsg = string.Format("AssertionResult {0}", i + 1);
Assert.That(assertion.Message, Does.Match(assertionMessageRegex[i++]), errmsg);
Assert.That(result.Message, Contains.Substring(assertion.Message), errmsg);
#if !PORTABLE || NETSTANDARD1_6
// NOTE: This test expects the stack trace to contain the name of the method
// that actually caused the failure. To ensure it is not optimized away, we
// compile the testdata assembly with optimizations disabled.
Assert.That(assertion.StackTrace, Is.Not.Null.And.Contains(methodName), errmsg);
#endif
}
}
return result;
}
/// <summary>
/// Set the result of the test
/// </summary>
/// <param name="resultState">The ResultState to use in the result</param>
/// <param name="message">A message associated with the result state</param>
/// <param name="stackTrace">Stack trace giving the location of the command</param>
public void SetResult(ResultState resultState, string message, string stackTrace)
{
ResultState = resultState;
Message = message;
StackTrace = stackTrace;
// Set pseudo-counts for a test case
//if (IsTestCase(test))
//{
// passCount = 0;
// failCount = 0;
// skipCount = 0;
// inconclusiveCount = 0;
// switch (ResultState.Status)
// {
// case TestStatus.Passed:
// passCount++;
// break;
// case TestStatus.Failed:
// failCount++;
// break;
// case TestStatus.Skipped:
// skipCount++;
// break;
// default:
// case TestStatus.Inconclusive:
// inconclusiveCount++;
// break;
// }
//}
}
/// <summary>
/// Private method to return a test case, optionally ignored on the Linux platform.
/// We use this since the Platform attribute is not supported on TestCaseData.
/// </summary>
private static TestCaseData GetTestCase(IMethodInfo method, ResultState resultState, int assertionCount, bool ignoreThis)
{
var data = new TestCaseData(method, resultState, assertionCount);
if (PLATFORM_IGNORE && ignoreThis)
data = data.Ignore("Intermittent failure on Linux and under Portable build");
return data;
}
public void Status_ConstructorWithOneArguments_ReturnsConstructorArgumentStatus(TestStatus status)
{
ResultState resultState = new ResultState(status);
Assert.AreEqual(status, resultState.Status);
}
public void AddSiteToResult(TestStatus status, string label, FailureSite site)
{
var result = new ResultState(status, label).WithSite(site);
Assert.That(result.Status, Is.EqualTo(status));
Assert.That(result.Label, Is.EqualTo(label));
Assert.That(result.Site, Is.EqualTo(site));
}
public void TestEquality_Null()
{
var rs = new ResultState(TestStatus.Passed);
Assert.AreNotEqual(null, rs);
Assert.AreNotEqual(rs, null);
Assert.False(rs.Equals(null));
}
public void TestEquality_WrongType()
{
var rs = new ResultState(TestStatus.Passed);
var s = "123";
Assert.AreNotEqual(rs, s);
Assert.AreNotEqual(s, rs);
Assert.False(rs.Equals(s));
}
public void Site_ConstructorWithOneArguments_ReturnsTest()
{
ResultState resultState = new ResultState(TestStatus.Failed);
Assert.AreEqual(FailureSite.Test, resultState.Site);
}
/// <summary>
/// Set the result of the test
/// </summary>
/// <param name="resultState">The ResultState to use in the result</param>
public void SetResult(ResultState resultState)
{
SetResult(resultState, null, null);
}
public void Site_ConstructorWithTwoArguments_ReturnsTest(string label)
{
ResultState resultState = new ResultState(TestStatus.Failed, label);
Assert.AreEqual(FailureSite.Test, resultState.Site);
}
public void Site_ConstructorWithThreeArguments_ReturnsSite(string label, FailureSite site)
{
ResultState resultState = new ResultState(TestStatus.Failed, label, site);
Assert.AreEqual(site, resultState.Site);
}
public void ToString_Constructor_ReturnsExepectedString(TestStatus status, string label, string expected)
{
// Arrange N/A
ResultState resultState = new ResultState(status, label);
Assert.AreEqual(expected, resultState.ToString());
}
private void SkipChildren(TestSuite suite, ResultState resultState, string message)
{
foreach (Test child in suite.Tests)
{
if (_childFilter.Pass(child))
{
TestResult childResult = child.MakeTestResult();
childResult.SetResult(resultState, message);
Result.AddResult(childResult);
if (child.IsSuite)
SkipChildren((TestSuite)child, resultState, message);
}
}
}
/// <summary>
/// Set the result of the test
/// </summary>
/// <param name="resultState">The ResultState to use in the result</param>
/// <param name="message">A message associated with the result state</param>
public void SetResult(ResultState resultState, string message)
{
SetResult(resultState, message, null);
}
/// <summary>
/// Test whether this ResultState has the same Status and Label
/// as another one. In other words, the whether two are equal
/// ignoring the Site.
/// </summary>
/// <param name="other"></param>
/// <returns></returns>
public bool Matches(ResultState other)
{
return Status == other.Status && Label == other.Label;
}
private string TranslateResult(ResultState resultState)
{
switch (resultState.Status)
{
default:
case TestStatus.Passed:
return "Success";
case TestStatus.Inconclusive:
return "Inconclusive";
case TestStatus.Failed:
switch (resultState.Label)
{
case "Error":
case "Cancelled":
return resultState.Label;
default:
return "Failure";
}
case TestStatus.Skipped:
switch (resultState.Label)
{
case "Ignored":
return "Ignored";
case "Invalid":
return "NotRunnable";
default:
return "Skipped";
}
}
}
/// <summary>
/// Construct a test result given a Test
/// </summary>
/// <param name="test">The test to be used</param>
public TestResult(ITest test)
{
this.test = test;
this.resultState = ResultState.Inconclusive;
}
