private TestResult SimulateTest(TestNode test, bool ignore)
{
if (test.RunState != RunState.Runnable)
{
ignore = true;
}
if (test.IsSuite)
{
FireSuiteStarting(test.TestName);
TestSuiteResult result = new TestSuiteResult(test, test.TestName.Name);
foreach (TestNode childTest in test.Tests)
{
result.AddResult(SimulateTest(childTest, ignore));
}
FireSuiteFinished(result);
return(result);
}
else
{
FireTestStarting(test.TestName);
TestCaseResult result = new TestCaseResult(test);
result.RunState = ignore ? RunState.Ignored : RunState.Executed;
FireTestFinished(result);
return(result);
}
}
public static TestSuiteResult RunChildCommands(TestSuiteCommand command, TestExecutionContext context)
{
TestSuiteResult suiteResult = TestExecutionContext.CurrentContext.CurrentResult as TestSuiteResult;
suiteResult.SetResult(ResultState.Success);
foreach (TestCommand childCommand in command.Children)
{
TestResult childResult = CommandRunner.Execute(childCommand);
suiteResult.AddResult(childResult);
if (childResult.ResultState == ResultState.Cancelled)
{
break;
}
if (childResult.ResultState.Status == TestStatus.Failed && TestExecutionContext.CurrentContext.StopOnError)
{
break;
}
}
return(suiteResult);
}
private void RunAllTests(
TestSuiteResult suiteResult, EventListener listener, ITestFilter filter)
{
Random rnd = new Random();
foreach (Test test in ArrayList.Synchronized(Tests).Cast <Test>().OrderBy(ts => rnd.Next()))
{
if (filter.Pass(test))
{
RunState saveRunState = test.RunState;
if (test.RunState == RunState.Runnable && this.RunState != RunState.Runnable && this.RunState != RunState.Explicit)
{
test.RunState = this.RunState;
test.IgnoreReason = this.IgnoreReason;
}
TestResult result = test.Run(listener, filter);
suiteResult.AddResult(result);
if (saveRunState != test.RunState)
{
test.RunState = saveRunState;
test.IgnoreReason = null;
}
}
}
}
public void ClearResults()
{
TestCaseResult testCaseResult = new TestCaseResult(testCaseInfo);
testCaseResult.Success();
TestSuiteResult testSuiteResult = new TestSuiteResult(fixtureInfo, "MockTestFixture");
testSuiteResult.AddResult(testCaseResult);
testSuiteResult.RunState = RunState.Executed;
TestSuiteTreeNode node1 = new TestSuiteTreeNode(testSuiteResult);
TestSuiteTreeNode node2 = new TestSuiteTreeNode(testCaseResult);
node1.Nodes.Add(node2);
Assert.AreEqual(TestSuiteTreeNode.SuccessIndex, node1.ImageIndex);
Assert.AreEqual(TestSuiteTreeNode.SuccessIndex, node1.SelectedImageIndex);
Assert.AreEqual(TestSuiteTreeNode.SuccessIndex, node2.ImageIndex);
Assert.AreEqual(TestSuiteTreeNode.SuccessIndex, node2.SelectedImageIndex);
node1.ClearResults();
Assert.AreEqual(TestSuiteTreeNode.InitIndex, node1.ImageIndex);
Assert.AreEqual(TestSuiteTreeNode.InitIndex, node1.SelectedImageIndex);
Assert.AreEqual(TestSuiteTreeNode.InitIndex, node2.ImageIndex);
Assert.AreEqual(TestSuiteTreeNode.InitIndex, node2.SelectedImageIndex);
}
private TestResult SimulateTest(Test test)
{
if (test.IsSuite)
{
FireSuiteStarting(test);
TestSuiteResult result = new TestSuiteResult(test, test.Name);
foreach (Test childTest in test.Tests)
{
result.AddResult(SimulateTest(childTest));
}
FireSuiteFinished(result);
return(result);
}
else
{
FireTestStarting(test);
TestCaseResult result = new TestCaseResult(test as TestCase);
result.Executed = test.ShouldRun && !test.IsExplicit;
FireTestFinished(result);
return(result);
}
}
private IEnumerable RunChildren()
{
int childCount = Children.Count;
if (childCount == 0)
{
throw new InvalidOperationException("RunChildren called but item has no children");
}
_childTestCountdown = new CountdownEvent(childCount);
foreach (UnityWorkItem child in Children)
{
if (CheckForCancellation())
{
yield break;
}
var unityTestExecutionContext = new UnityTestExecutionContext(Context);
child.InitializeContext(unityTestExecutionContext);
var enumerable = child.Execute().GetEnumerator();
while (true)
{
if (!enumerable.MoveNext())
{
break;
}
ResultedInDomainReload |= child.ResultedInDomainReload;
yield return(enumerable.Current);
}
_suiteResult.AddResult(child.Result);
childCount--;
}
if (childCount > 0)
{
while (childCount-- > 0)
{
CountDownChildTest();
}
}
}
public override void Run(IList <TestAssemblyInfo> testAssemblies)
{
if (testAssemblies == null)
{
throw new ArgumentNullException(nameof(testAssemblies));
}
var builder = new NUnitLiteTestAssemblyBuilder();
var runner = new NUnitLiteTestAssemblyRunner(builder);
var testSuite = new TestSuite(global::Android.App.Application.Context.PackageName);
results = new TestSuiteResult(testSuite);
foreach (TestAssemblyInfo assemblyInfo in testAssemblies)
{
if (assemblyInfo == null || assemblyInfo.Assembly == null)
{
continue;
}
if (!runner.Load(assemblyInfo.Assembly, builderSettings))
{
OnWarning($"Failed to load tests from assembly '{assemblyInfo.Assembly}");
continue;
}
if (runner.LoadedTest is NUnitTest tests)
{
testSuite.Add(tests);
}
// Messy API. .Run returns ITestResult which is, in reality, an instance of TestResult since that's
// what WorkItem returns and we need an instance of TestResult to add it to TestSuiteResult. So, cast
// the return to TestResult and hope for the best.
ITestResult result = null;
try {
OnAssemblyStart(assemblyInfo.Assembly);
result = runner.Run(this, Filter);
} finally {
OnAssemblyFinish(assemblyInfo.Assembly);
}
if (result == null)
{
continue;
}
var testResult = result as TestResult;
if (testResult == null)
{
throw new InvalidOperationException($"Unexpected test result type '{result.GetType ()}'");
}
results.AddResult(testResult);
}
LogFailureSummary();
}
private TestSuiteResult MockSuiteResult()
{
TestSuiteResult result = new TestSuiteResult(null, "base");
TestSuiteResult level1SuiteA = new TestSuiteResult(null, "level 1 A");
result.AddResult(level1SuiteA);
TestSuiteResult level1SuiteB = new TestSuiteResult(null, "level 1 B");
result.AddResult(level1SuiteB);
testCase = new TestCaseResult("a test case");
level1SuiteA.AddResult(testCase);
level1SuiteB.AddResult(new TestCaseResult("a successful test"));
return(result);
}
private void MarkTestNotRun(
Test test, RunState runState, string ignoreReason, TestSuiteResult suiteResult, EventListener listener, ITestFilter filter)
{
if (test is TestSuite)
{
listener.SuiteStarted(test.TestName);
TestSuiteResult result = new TestSuiteResult(new TestInfo(test), test.TestName.FullName);
result.NotRun(runState, ignoreReason, null);
MarkTestsNotRun(test.Tests, runState, ignoreReason, suiteResult, listener, filter);
suiteResult.AddResult(result);
listener.SuiteFinished(result);
}
else
{
listener.TestStarted(test.TestName);
TestCaseResult result = new TestCaseResult(new TestInfo(test));
result.NotRun(runState, ignoreReason, null);
suiteResult.AddResult(result);
listener.TestFinished(result);
}
}
private void SkipChildren(TestSuite suite, ResultState resultState, string message)
{
foreach (Test child in suite.Tests)
{
if (_childFilter.Pass(child))
{
TestResult childResult = child.MakeTestResult();
childResult.SetResult(resultState, message);
_suiteResult.AddResult(childResult);
// Some runners may depend on getting the TestFinished event
// even for tests that have been skipped at a higher level.
Context.Listener.TestFinished(childResult);
if (child.IsSuite)
{
SkipChildren((TestSuite)child, resultState, message);
}
}
}
}
public bool Run(TestInfo test, TestSuiteResult suite_result, string test_name, EventListener listener, ITestFilter filter, int timeout)
{
listener = new ProxyListener(listener);
TestRunnerDelegate runner = new TestRunnerDelegate(delegate {
return(server.Run(test_name, listener, filter));
});
IAsyncResult ar = runner.BeginInvoke(test_name, null, filter, null, null);
if (!ar.AsyncWaitHandle.WaitOne(timeout) || !ar.IsCompleted)
{
TestCaseResult error = new TestCaseResult(test);
string msg = String.Format("Timeout after {0} ms", timeout);
error.Failure(msg, null, FailureSite.Parent);
suite_result.AddResult(error);
return(false);
}
try {
TestResult result = runner.EndInvoke(ar);
if (result != null)
{
suite_result.AddResult(result);
return(true);
}
TestCaseResult error = new TestCaseResult(test);
error.Failure("Unknown error", null, FailureSite.Parent);
suite_result.AddResult(error);
return(false);
} catch (Exception ex) {
TestCaseResult error = new TestCaseResult(test);
string msg = String.Format("Unknown exception: {0}", ex);
error.Failure(msg, null, FailureSite.Parent);
suite_result.AddResult(error);
return(false);
}
}
private TestSuiteResult NotRunTestSuite()
{
TestSuiteResult result = new TestSuiteResult("RootSuite");
result.RunState = RunState.Executed;
TestCaseResult testCaseResult = new TestCaseResult("NonRunTestCase");
testCaseResult.Ignore("No Reason");
result.AddResult(testCaseResult);
return(result);
}
private TestSuiteResult MockSuiteResult(string suiteName, bool failure)
{
TestSuiteResult result = new TestSuiteResult(null, suiteName);
result.Time = time;
result.Executed = true;
TestSuiteResult level1SuiteA = new TestSuiteResult(null, "level 1 A");
result.AddResult(level1SuiteA);
level1SuiteA.Executed = true;
TestSuiteResult level1SuiteB = new TestSuiteResult(null, "level 1 B");
result.AddResult(level1SuiteB);
level1SuiteB.Executed = true;
testCase = new TestCaseResult("a test case");
if (failure)
{
testCase.Failure("argument exception", null);
}
else
{
testCase.Success();
}
level1SuiteA.AddResult(testCase);
testCase = new TestCaseResult("a successful test");
testCase.Success();
level1SuiteB.AddResult(testCase);
testCase = new TestCaseResult("a not run test");
testCase.NotRun("test not run");
level1SuiteB.AddResult(testCase);
return(result);
}
private TestSuiteResult NotRunTestSuite()
{
TestSuiteResult result = new TestSuiteResult(null, "RootSuite");
result.Executed = false;
TestCaseResult testCaseResult = new TestCaseResult("NonRunTestCase");
testCaseResult.NotRun("No Reason");
result.AddResult(testCaseResult);
return(result);
}
/// <summary>
/// Run selected tests synchronously or asynchronously, notifying the listener interface as it progresses.
/// </summary>
/// <param name="assemblies">The list of test assembly runners to run.</param>
/// <param name="listener">Interface to receive EventListener notifications.</param>
/// <param name="filter">A test filter used to select tests to be run.</param>
/// <param name="runAsync"><c>true</c> to run tests asynchronously, otherwise <c>false</c> to run synchronously.</param>
/// <param name="results">The <see cref="ITestResult" /> object to add the test results to.</param>
private static void RunTests(IList <ITestAssemblyRunner> assemblies,
ITestListener listener, ITestFilter filter, bool runAsync, ref TestSuiteResult results)
{
filter = ValidateFilter(filter);
Queue <ITestAssemblyRunner> testQueue = new Queue <ITestAssemblyRunner>(assemblies);
// Check each test not ran when first encountered to see if it is ready to be run
while (testQueue.Count > 0)
{
// Run test if not currently running, otherwise queue up to be checked again later
ITestAssemblyRunner test = testQueue.Dequeue();
if (!test.IsTestRunning)
{
if (runAsync)
{
test.RunAsync(listener, filter);
}
else
{
test.Run(listener, filter);
}
}
else
{
// Test not ready to run so re-enqueue
testQueue.Enqueue(test);
}
// Slow down the polling loop to give time for tests to complete
if (runAsync)
{
Thread.Sleep(10);
}
}
// Wait for tests to complete
while (assemblies.Any(test => test.IsTestRunning || !test.IsTestComplete))
// Slow down the polling loop to give time for tests to complete
{
Thread.Sleep(10);
}
// Add individual test runner results to overall result
foreach (ITestAssemblyRunner test in assemblies)
{
if (test.Result != null)
{
results.AddResult(test.Result);
}
}
}
private void SkipChildren(TestSuite suite, ResultState resultState, string message)
{
foreach (Test child in suite.Tests)
{
if (_childFilter.Pass(child))
{
TestResult childResult = child.MakeTestResult();
childResult.SetResult(resultState, message);
_suiteResult.AddResult(childResult);
#if TIZEN
//[tronghieu.d] - return FAIL TC in case it is skipped.
if (tsettings.IsSlaveMode && childResult.FullName.Equals(tsettings.Testcase_ID))
{
tsettings.TCResult = "FAIL";
tsettings.TCMessage = childResult.Message;
string json = "case_id=" + tsettings.Testcase_ID + "&result=" + tsettings.TCResult + "&msg=" + tsettings.TCMessage + "\n" + childResult.StackTrace;
if (!tsettings.IsManual)
{
tsettings.RequestPOST("commit_result", json);
tsettings.NextStepRequest();
}
else
{
tsettings.RequestPOST("commit_manual_result", json);
}
}
#endif
// Some runners may depend on getting the TestFinished event
// even for tests that have been skipped at a higher level.
Context.Listener.TestFinished(childResult);
if (child.IsSuite)
{
SkipChildren((TestSuite)child, resultState, message);
}
}
}
}
private void MarkTestFailed(
Test test, TestSuiteResult suiteResult, EventListener listener, ITestFilter filter)
{
if (test is TestSuite)
{
listener.SuiteStarted(test.TestName);
TestSuiteResult result = new TestSuiteResult(new TestInfo(test), test.TestName.FullName);
string msg = string.Format("Parent SetUp failed in {0}", this.FixtureType.Name);
result.Failure(msg, null, FailureSite.Parent);
MarkTestsFailed(test.Tests, suiteResult, listener, filter);
suiteResult.AddResult(result);
listener.SuiteFinished(result);
}
else
{
listener.TestStarted(test.TestName);
TestCaseResult result = new TestCaseResult(new TestInfo(test));
string msg = string.Format("TestFixtureSetUp failed in {0}", this.FixtureType.Name);
result.Failure(msg, null, FailureSite.Parent);
suiteResult.AddResult(result);
listener.TestFinished(result);
}
}
//private TestCaseResult testCase;
private TestSuiteResult MockSuiteResult(bool failure)
{
TestCaseResult testCaseResult = new TestCaseResult("a test case");
if (failure)
{
testCaseResult.Failure("case failed", null);
}
TestSuiteResult level1SuiteA = new TestSuiteResult("level 1 A");
level1SuiteA.AddResult(testCaseResult);
TestSuiteResult level1SuiteB = new TestSuiteResult("level 1 B");
level1SuiteB.AddResult(new TestCaseResult("a successful test"));
TestSuiteResult result = new TestSuiteResult("base");
result.AddResult(level1SuiteA);
result.AddResult(level1SuiteB);
return(result);
}
/// <inheritdoc />
public ITestResult GetTestResults()
{
lock (TestSuite)
{
TestSuiteResult results = new TestSuiteResult(TestSuite);
foreach (ITestAssemblyRunner runner in TestSuite.TestAssemblyRunners)
{
if (runner.Result != null)
{
results.AddResult(runner.Result);
}
}
return(results);
}
}
public override TestResult Run(EventListener listener, ITestFilter filter)
{
TestSuiteResult suite_result = new TestSuiteResult(new TestInfo(this), TestName.FullName);
DebuggerTestHost host = DebuggerTestHost.Create();
if (host == null)
{
TestCaseResult error = new TestCaseResult(new TestInfo(this));
string msg = String.Format("Failed to create DebuggerTestHost in {0}", FixtureType.Name);
error.Failure(msg, null, FailureSite.Parent);
suite_result.AddResult(error);
return(suite_result);
}
int timeout;
if (Attribute.Timeout != 0)
{
timeout = Attribute.Timeout;
}
else
{
timeout = DebuggerTestAddIn.DefaultTimeout;
}
int repeat = 1;
if (Attribute.Repeat != 0)
{
repeat = Attribute.Repeat;
}
try {
for (int i = 0; i < repeat; i++)
{
if (!host.Run(new TestInfo(this), suite_result, Type.AssemblyQualifiedName, listener, filter, timeout))
{
break;
}
}
return(suite_result);
} finally {
host.Dispose();
}
}
public override TestResult Run(EventListener listener)
{
TestSuiteResult suiteResult = new TestSuiteResult(this, testCase.Name);
for (int i = 0; i < repeatCount; i++)
{
TestResult testResult = testCase.Run(NullListener.NULL);
suiteResult.AddResult(testResult);
if (!testResult.IsSuccess)
{
break;
}
}
return(suiteResult);
}
private static ITestResult MergeTestResults(IEnumerable <ITestResult> partialResults)
{
if (partialResults == null)
{
throw new ArgumentNullException(nameof(partialResults));
}
var nodesToMerge = new List <ITestResult>();
using (var en = partialResults.GetEnumerator())
{
if (!en.MoveNext())
{
throw new ArgumentException("There must be at least one result.", nameof(partialResults));
}
var first = en.Current;
if (!en.MoveNext())
{
return(first);
}
nodesToMerge.Add(first);
do
{
var next = en.Current;
if (first.Test != next.Test)
{
throw new InvalidOperationException("Result nodes for different tests cannot be merged.");
}
nodesToMerge.Add(next);
} while (en.MoveNext());
}
var merged = new TestSuiteResult((TestSuite)nodesToMerge[0].Test);
foreach (var childNodesToMerge in nodesToMerge.SelectMany(n => n.Children).GroupBy(c => c.Test))
{
merged.AddResult(MergeTestResults(childNodesToMerge));
}
return(merged);
}
private static string ConstructXmlDocumentForAbnormalExit(ITest test, TestCaseResult caseResult)
{
TestResult currentResult = caseResult;
ITest currentTest = test;
while (currentTest.Parent != null)
{
var suiteResult = new TestSuiteResult((TestSuite)currentTest.Parent);
suiteResult.AddResult(currentResult);
currentResult = suiteResult;
currentTest = currentTest.Parent;
}
var node = currentResult.ToXml(true);
var res = @"<?xml version=""1.0"" encoding=""utf-8"" ?>
<test-run fullname=""" + currentTest.FullName + @""">
" + node.OuterXml + @"</test-run>";
return(res);
}
/// <inheritdoc />
public ITestResult GetTestResults(Assembly assembly)
{
if (assembly == null)
{
throw ExceptionHelper.ThrowArgumentNullException(nameof(assembly));
}
lock (TestSuite)
{
TestSuiteResult results = new TestSuiteResult(TestSuite);
ITestAssemblyRunner runner = TestSuite.GetTestAssemblyRunner(assembly);
if (runner?.Result != null)
{
results.AddResult(runner.Result);
}
return(results);
}
}
/// <inheritdoc />
public async Task <ITestResult> GetTestResultsAsync()
{
Task <ITestResult> task = Task.Run(() =>
{
lock (TestSuite)
{
TestSuiteResult results = new TestSuiteResult(TestSuite);
foreach (ITestAssemblyRunner runner in TestSuite.TestAssemblyRunners)
{
if (runner.Result != null)
{
results.AddResult(runner.Result);
}
}
return((ITestResult)results);
}
});
return(await task.ConfigureAwait(false));
}
public virtual TestResult Run(EventListener listener, ITestFilter filter)
{
// Save active listener for derived classes
this.listener = listener;
ITest[] tests = new ITest[runners.Count];
for (int index = 0; index < runners.Count; index++)
{
tests[index] = ((TestRunner)runners[index]).Test;
}
this.listener.RunStarted(this.Test.TestName.Name, this.CountTestCases(filter));
this.listener.SuiteStarted(this.Test.TestName);
long startTime = DateTime.Now.Ticks;
TestSuiteResult result = new TestSuiteResult(new TestInfo(testName, tests), projectName);
result.RunState = RunState.Executed;
foreach (TestRunner runner in runners)
{
if (filter.Pass(runner.Test))
{
result.AddResult(runner.Run(this, filter));
}
}
long stopTime = DateTime.Now.Ticks;
double time = ((double)(stopTime - startTime)) / (double)TimeSpan.TicksPerSecond;
result.Time = time;
this.listener.SuiteFinished(result);
this.listener.RunFinished(result);
this.testResult = result;
return(result);
}
/// <inheritdoc />
public async Task <ITestResult> GetTestResultsAsync(Assembly assembly)
{
if (assembly == null)
{
throw ExceptionHelper.ThrowArgumentNullException(nameof(assembly));
}
Task <ITestResult> task = Task.Run(() =>
{
lock (TestSuite)
{
TestSuiteResult results = new TestSuiteResult(TestSuite);
ITestAssemblyRunner runner = TestSuite.GetTestAssemblyRunner(assembly);
if (runner?.Result != null)
{
results.AddResult(runner.Result);
}
return((ITestResult)results);
}
});
return(await task.ConfigureAwait(false));
}
public override void Run(IList <TestAssemblyInfo> testAssemblies)
{
if (testAssemblies == null)
{
throw new ArgumentNullException(nameof(testAssemblies));
}
if (AssemblyFilters == null || AssemblyFilters.Count == 0)
{
runAssemblyByDefault = true;
}
else
{
runAssemblyByDefault = AssemblyFilters.Values.Any(v => !v);
}
var builder = new NUnitLiteTestAssemblyBuilder();
var runner = new NUnitLiteTestAssemblyRunner(builder, new FinallyDelegate());
var testSuite = new TestSuite(NSBundle.MainBundle.BundleIdentifier);
results = new TestSuiteResult(testSuite);
TotalTests = 0;
foreach (TestAssemblyInfo assemblyInfo in testAssemblies)
{
if (assemblyInfo == null || assemblyInfo.Assembly == null || !ShouldRunAssembly(assemblyInfo))
{
continue;
}
if (!runner.Load(assemblyInfo.Assembly, builderSettings))
{
OnWarning($"Failed to load tests from assembly '{assemblyInfo.Assembly}");
continue;
}
if (runner.LoadedTest is NUnitTest tests)
{
TotalTests += tests.TestCaseCount;
testSuite.Add(tests);
}
// Messy API. .Run returns ITestResult which is, in reality, an instance of TestResult since that's
// what WorkItem returns and we need an instance of TestResult to add it to TestSuiteResult. So, cast
// the return to TestResult and hope for the best.
ITestResult result = null;
try {
OnAssemblyStart(assemblyInfo.Assembly);
result = runner.Run(this, Filter);
} finally {
OnAssemblyFinish(assemblyInfo.Assembly);
}
if (result == null)
{
continue;
}
var testResult = result as TestResult;
if (testResult == null)
{
throw new InvalidOperationException($"Unexpected test result type '{result.GetType ()}'");
}
results.AddResult(testResult);
}
// NUnitLite doesn't report filtered tests at all, but we can calculate here
FilteredTests = TotalTests - ExecutedTests;
LogFailureSummary();
}