/// <summary>
/// Start asynchronous execution of a test.
/// </summary>
/// <param name="listener">The ITestEventListener to use for this run</param>
/// <param name="filter">The TestFilter to use for this run</param>
public void Start(ITestEventListener listener, TestFilter filter)
{
_worker.DoWork += (s, ea) =>
_result = _runner.Run(listener, filter);
_worker.RunWorkerAsync();
}
public void FilterWithOneTest()
{
string text = "<filter><tests><test>My.Test.Name</test></tests></filter>";
TestFilter filter = new TestFilter(text);
Assert.That(filter.Text, Is.EqualTo(text));
Assert.That(filter.Xml.Name, Is.EqualTo("filter"));
Assert.That(filter.Xml.SelectSingleNode("tests/test").InnerText, Is.EqualTo("My.Test.Name"));
}
public int CountTestCases(TestFilter filter)
{
CallbackHandler handler = new CallbackHandler();
CreateObject(COUNT_ACTION, _frameworkController, filter.Text, handler);
return int.Parse(handler.Result);
}
public void FilterWithThreeTests()
{
string text = "<filter><tests><test>My.First.Test</test><test>My.Second.Test</test><test>My.Third.Test</test></tests></filter>";
TestFilter filter = new TestFilter(text);
Assert.That(filter.Text, Is.EqualTo(text));
Assert.That(filter.Xml.Name, Is.EqualTo("filter"));
XmlNodeList testNodes = filter.Xml.SelectNodes("tests/test");
Assert.That(testNodes.Count, Is.EqualTo(3));
Assert.That(testNodes[0].InnerText, Is.EqualTo("My.First.Test"));
Assert.That(testNodes[1].InnerText, Is.EqualTo("My.Second.Test"));
Assert.That(testNodes[2].InnerText, Is.EqualTo("My.Third.Test"));
}
private TestResults RunTests(TestPackage package, TestFilter filter, Messages.OnMessage[] messageSubscribers)
{
XmlNode result;
using (new SaveConsoleOutput())
using (ITestRunner runner = _engine.GetRunner(package))
using (var ms = new MemoryStream())
using (var output = CreateOutputWriter(ms))
{
try
{
var labels = "ON";
var eventHandler = new TestEventHandler(output, labels, messageSubscribers);
result = runner.Run(eventHandler, filter);
var reporter = new ResultReporter(result, output, _options);
reporter.ReportResults();
output.Flush();
if (reporter.Summary.UnexpectedError)
return new TestResults(TestResults.Code.UnexpectedError, GetResultText(ms), reporter.Summary);
return new TestResults(reporter.Summary.InvalidAssemblies > 0
? TestResults.Code.InvalidAssembly
: GetCode( reporter.Summary.FailureCount + reporter.Summary.ErrorCount + reporter.Summary.InvalidCount),
GetResultText(ms), reporter.Summary);
}
catch (NUnitEngineException ex)
{
output.WriteLine(ex.Message);
output.Flush();
return new TestResults(TestResults.Code.InvalidArg, GetResultText(ms));
}
catch (FileNotFoundException ex)
{
output.WriteLine(ex.Message);
output.Flush();
return new TestResults(TestResults.Code.InvalidAssembly, GetResultText(ms));
}
catch (DirectoryNotFoundException ex)
{
output.WriteLine(ex.Message);
output.Flush();
return new TestResults(TestResults.Code.InvalidAssembly, GetResultText(ms));
}
catch (Exception ex)
{
output.WriteLine(ex.ToString());
output.Flush();
return new TestResults(TestResults.Code.UnexpectedError, GetResultText(ms));
}
}
}
public void FilterWithThreeTests()
{
string text = "<filter><tests><test>My.First.Test</test><test>My.Second.Test</test><test>My.Third.Test</test></tests></filter>";
TestFilter filter = new TestFilter(text);
Assert.That(filter.Text, Is.EqualTo(text));
}
public void FilterWithOneTest()
{
string text = "<filter><tests><test>My.Test.Name</test></tests></filter>";
TestFilter filter = new TestFilter(text);
Assert.That(filter.Text, Is.EqualTo(text));
}
private void RunAssembly(string assemblyName, IFrameworkHandle frameworkHandle)
{
#if LAUNCHDEBUGGER
if (!Debugger.IsAttached)
Debugger.Launch();
#endif
_testRunner = GetRunnerFor(assemblyName);
try
{
var loadResult = _testRunner.Explore(TestFilter.Empty);
if (loadResult.Name == "test-run")
loadResult = loadResult.FirstChild;
if (loadResult.GetAttribute("runstate") == "Runnable")
{
TestLog.SendInformationalMessage(string.Format("Loading tests from {0}", assemblyName));
var nunitTestCases = loadResult.SelectNodes("//test-case");
using (var testConverter = new TestConverter(TestLog, assemblyName))
{
var loadedTestCases = new List<TestCase>();
// As a side effect of calling TestConverter.ConvertTestCase,
// the converter's cache of all test cases is populated as well.
// All future calls to convert a test case may now use the cache.
foreach (XmlNode testNode in nunitTestCases)
loadedTestCases.Add(testConverter.ConvertTestCase(testNode));
// If we have a TFS Filter, convert it to an nunit filter
if (_tfsFilter != null && _tfsFilter.HasTfsFilterValue)
{
var filteredTestCases = _tfsFilter.CheckFilter(loadedTestCases);
var testCases = filteredTestCases as TestCase[] ?? filteredTestCases.ToArray();
TestLog.SendInformationalMessage(string.Format("TFS Filter detected: LoadedTestCases {0}, Filterered Test Cases {1}", loadedTestCases.Count, testCases.Count()));
_nunitFilter = MakeTestFilter(testCases);
}
using (var listener = new NUnitEventListener(frameworkHandle, testConverter))
{
try
{
_testRunner.Run(listener, _nunitFilter);
}
catch (NullReferenceException)
{
// this happens during the run when CancelRun is called.
TestLog.SendDebugMessage("Nullref caught");
}
}
}
}
else
TestLog.NUnitLoadError(assemblyName);
}
catch (BadImageFormatException)
{
// we skip the native c++ binaries that we don't support.
TestLog.AssemblyNotSupportedWarning(assemblyName);
}
catch (System.IO.FileNotFoundException ex)
{
// Probably from the GetExportedTypes in NUnit.core, attempting to find an assembly, not a problem if it is not NUnit here
TestLog.DependentAssemblyNotFoundWarning(ex.FileName, assemblyName);
}
catch (Exception ex)
{
if (ex is TargetInvocationException)
ex = ex.InnerException;
TestLog.SendErrorMessage("Exception thrown executing tests in " + assemblyName, ex);
}
_testRunner.Dispose();
}
private int RunTests(TestPackage package, TestFilter filter)
{
foreach (var spec in _options.ResultOutputSpecifications)
{
var outputPath = Path.Combine(_workDirectory, spec.OutputPath);
GetResultWriter(spec).CheckWritability(outputPath);
}
// TODO: Incorporate this in EventCollector?
RedirectErrorOutputAsRequested();
var labels = _options.DisplayTestLabels != null
? _options.DisplayTestLabels.ToUpperInvariant()
: "ON";
XmlNode result;
try
{
using (new SaveConsoleOutput())
using (new ColorConsole(ColorStyle.Output))
using (ITestRunner runner = _engine.GetRunner(package))
using (var output = CreateOutputWriter())
{
var eventHandler = new TestEventHandler(output, labels, _options.TeamCity);
result = runner.Run(eventHandler, filter);
}
}
finally
{
RestoreErrorOutput();
}
var writer = new ColorConsoleWriter(!_options.NoColor);
var reporter = new ResultReporter(result, writer, _options);
reporter.ReportResults();
foreach (var spec in _options.ResultOutputSpecifications)
{
var outputPath = Path.Combine(_workDirectory, spec.OutputPath);
GetResultWriter(spec).WriteResultFile(result, outputPath);
_outWriter.WriteLine("Results ({0}) saved as {1}", spec.Format, spec.OutputPath);
}
if (reporter.Summary.UnexpectedError)
return ConsoleRunner.UNEXPECTED_ERROR;
return reporter.Summary.InvalidAssemblies > 0
? ConsoleRunner.INVALID_ASSEMBLY
: reporter.Summary.FailureCount + reporter.Summary.ErrorCount + reporter.Summary.InvalidCount;
}
/// <summary>
/// Executes the tests in an assembly.
/// </summary>
/// <param name="listener">An ITestEventHandler that receives progress notices</param>
/// <param name="filter">A filter that controls which tests are executed</param>
/// <returns>An Xml string representing the result</returns>
public string Run(ITestEventListener listener, TestFilter filter)
{
CallbackHandler handler = new RunTestsCallbackHandler(listener);
log.Info("Running {0} - see separate log file", Path.GetFileName(_testAssemblyPath));
CreateObject(RUN_ACTION, _frameworkController, filter.Text, handler);
return handler.Result;
}
/// <summary>
/// Returns information about the tests in an assembly.
/// </summary>
/// <param name="filter">A filter indicating which tests to include</param>
/// <returns>An Xml string representing the tests</returns>
public string Explore(TestFilter filter)
{
CallbackHandler handler = new CallbackHandler();
log.Info("Exploring {0} - see separate log file", Path.GetFileName(_testAssemblyPath));
CreateObject(EXPLORE_ACTION, _frameworkController, filter.Text, handler);
return handler.Result;
}
private int RunTests(TestPackage package, TestFilter filter)
{
foreach (var spec in _options.ResultOutputSpecifications)
{
var outputPath = Path.Combine(_workDirectory, spec.OutputPath);
GetResultWriter(spec).CheckWritability(outputPath);
}
// TODO: Incorporate this in EventCollector?
RedirectErrorOutputAsRequested();
var labels = _options.DisplayTestLabels != null
? _options.DisplayTestLabels.ToUpperInvariant()
: "ON";
XmlNode result = null;
NUnitEngineException engineException = null;
try
{
using (new SaveConsoleOutput())
using (new ColorConsole(ColorStyle.Output))
using (ITestRunner runner = _engine.GetRunner(package))
using (var output = CreateOutputWriter())
{
var eventHandler = new TestEventHandler(output, labels);
result = runner.Run(eventHandler, filter);
}
}
catch (NUnitEngineException ex)
{
engineException = ex;
}
finally
{
RestoreErrorOutput();
}
var writer = new ColorConsoleWriter(!_options.NoColor);
if (result != null)
{
var reporter = new ResultReporter(result, writer, _options);
reporter.ReportResults();
foreach (var spec in _options.ResultOutputSpecifications)
{
var outputPath = Path.Combine(_workDirectory, spec.OutputPath);
GetResultWriter(spec).WriteResultFile(result, outputPath);
_outWriter.WriteLine("Results ({0}) saved as {1}", spec.Format, spec.OutputPath);
}
// Since we got a result, we display any engine exception as a warning
if (engineException != null)
writer.WriteLine(ColorStyle.Warning, Environment.NewLine + engineException.Message);
if (reporter.Summary.UnexpectedError)
return ConsoleRunner.UNEXPECTED_ERROR;
if (reporter.Summary.InvalidAssemblies > 0)
return ConsoleRunner.INVALID_ASSEMBLY;
return reporter.Summary.InvalidTestFixtures > 0
? ConsoleRunner.INVALID_TEST_FIXTURE
: reporter.Summary.FailureCount + reporter.Summary.ErrorCount + reporter.Summary.InvalidCount;
}
// If we got here, it's because we had an exception, but check anyway
if (engineException != null)
writer.WriteLine(ColorStyle.Error, engineException.Message);
return ConsoleRunner.UNEXPECTED_ERROR;
}
private int ExploreTests(TestPackage package, TestFilter filter)
{
XmlNode result = null;
using (var runner = _engine.GetRunner(package))
result = runner.Explore(filter);
if (_options.ExploreOutputSpecifications.Count == 0)
{
new TestCaseOutputWriter().WriteResultFile(result, Console.Out);
}
else
{
var outputManager = new OutputManager(result, _workDirectory);
foreach (OutputSpecification spec in _options.ExploreOutputSpecifications)
outputManager.WriteTestFile(spec);
}
return ConsoleRunner.OK;
}
private int RunTests(TestPackage package, TestFilter filter)
{
// TODO: We really need options as resolved by engine for most of these
DisplayRequestedOptions();
// TODO: Incorporate this in EventCollector?
RedirectOutputAsRequested();
var labels = _options.DisplayTestLabels != null
? _options.DisplayTestLabels.ToUpperInvariant()
: "ON";
TestEventHandler eventHandler = new TestEventHandler(_outWriter, labels);
XmlNode result = null;
// Save things that might be messed up by a bad test
TextWriter savedOut = Console.Out;
TextWriter savedError = Console.Error;
DateTime startTime = DateTime.Now;
try
{
using ( new ColorConsole( ColorStyle.Output ) )
using (ITestRunner runner = _engine.GetRunner(package))
{
result = runner.Run(eventHandler, filter);
}
}
finally
{
Console.SetOut(savedOut);
Console.SetError(savedError);
RestoreOutput();
}
//Console.WriteLine();
ResultReporter reporter = new ResultReporter(result, _options);
reporter.ReportResults();
// TODO: Inject this?
var outputManager = new OutputManager(result, _workDirectory);
foreach (var outputSpec in _options.ResultOutputSpecifications)
outputManager.WriteResultFile(outputSpec, startTime);
return reporter.Summary.ErrorsAndFailures;
}
private int RunTests(TestPackage package, TestFilter filter)
{
// TODO: We really need options as resolved by engine for most of these
DisplayRequestedOptions();
foreach (var spec in _options.ResultOutputSpecifications)
GetResultWriter(spec).CheckWritability(spec.OutputPath);
// TODO: Incorporate this in EventCollector?
RedirectErrorOutputAsRequested();
var labels = _options.DisplayTestLabels != null
? _options.DisplayTestLabels.ToUpperInvariant()
: "ON";
XmlNode result;
try
{
using (new SaveConsoleOutput())
using (new ColorConsole(ColorStyle.Output))
using (ITestRunner runner = _engine.GetRunner(package))
using (var output = CreateOutputWriter())
{
var eventHandler = new TestEventHandler(output, labels, _options.TeamCity);
result = runner.Run(eventHandler, filter);
}
}
finally
{
RestoreErrorOutput();
}
var writer = new ColorConsoleWriter(!_options.NoColor);
var reporter = new ResultReporter(result, writer, _options);
reporter.ReportResults();
foreach (var spec in _options.ResultOutputSpecifications)
{
GetResultWriter(spec).WriteResultFile(result, spec.OutputPath);
_outWriter.WriteLine("Results ({0}) saved as {1}", spec.Format, spec.OutputPath);
}
return reporter.Summary.FailureCount + reporter.Summary.ErrorCount + reporter.Summary.InvalidCount;
}
private int ExploreTests(TestPackage package, TestFilter filter)
{
XmlNode result;
using (var runner = _engine.GetRunner(package))
result = runner.Explore(filter);
if (_options.ExploreOutputSpecifications.Count == 0)
{
_resultService.GetResultWriter("cases", null).WriteResultFile(result, Console.Out);
}
else
{
foreach (OutputSpecification spec in _options.ExploreOutputSpecifications)
{
_resultService.GetResultWriter(spec.Format, new object[] {spec.Transform}).WriteResultFile(result, spec.OutputPath);
_outWriter.WriteLine("Results ({0}) saved as {1}", spec.Format, spec.OutputPath);
}
}
return ConsoleRunner.OK;
}
public int CountTestCases(TestFilter filter)
{
return _runner.CountTestCases(filter);
}
/// <summary>
/// Called by the VisualStudio IDE when selected tests are to be run. Never called from TFS Build.
/// </summary>
/// <param name="tests">The tests to be run</param>
/// <param name="runContext">The RunContext</param>
/// <param name="frameworkHandle">The FrameworkHandle</param>
public void RunTests(IEnumerable<TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
#if LAUNCHDEBUGGER
if (!Debugger.IsAttached)
Debugger.Launch();
#endif
Initialize(frameworkHandle);
var enableShutdown = (UseVsKeepEngineRunning) ? !runContext.KeepAlive : true;
frameworkHandle.EnableShutdownAfterTestRun = enableShutdown;
Debug("executing tests", "EnableShutdown set to " + enableShutdown);
var assemblyGroups = tests.GroupBy(tc => tc.Source);
foreach (var assemblyGroup in assemblyGroups)
{
var assemblyName = assemblyGroup.Key;
if (Debugger.IsAttached)
TestLog.SendInformationalMessage("Debugging selected tests in " + assemblyName);
else
TestLog.SendInformationalMessage("Running selected tests in " + assemblyName);
_nunitFilter = MakeTestFilter(assemblyGroup);
RunAssembly(assemblyName, frameworkHandle);
}
Info("executing tests", "finished");
Unload();
}
internal string ExploreInternal(TestFilter filter)
{
return _runner.Explore(filter).OuterXml;
}
internal string RunInternal(ITestEventListener listener, TestFilter filter)
{
return _runner.Run(listener, filter).OuterXml;
}
private void RunAssembly(string assemblyName, TestFilter filter)
{
#if LAUNCHDEBUGGER
if (!Debugger.IsAttached)
Debugger.Launch();
#endif
_activeRunner = GetRunnerFor(assemblyName);
try
{
var loadResult = _activeRunner.Explore(TestFilter.Empty);
if (loadResult.Name == "test-run")
loadResult = loadResult.FirstChild;
if (loadResult.GetAttribute("runstate") == "Runnable")
{
var nunitTestCases = loadResult.SelectNodes("//test-case");
using (var testConverter = new TestConverter(TestLog, assemblyName))
{
var loadedTestCases = new List<TestCase>();
// As a side effect of calling TestConverter.ConvertTestCase,
// the converter's cache of all test cases is populated as well.
// All future calls to convert a test case may now use the cache.
foreach (XmlNode testNode in nunitTestCases)
loadedTestCases.Add(testConverter.ConvertTestCase(testNode));
TestLog.Info(string.Format("NUnit3TestExecutor converted {0} of {1} NUnit test cases", loadedTestCases.Count, nunitTestCases.Count));
// If we have a TFS Filter, convert it to an nunit filter
if (TfsFilter != null && !TfsFilter.IsEmpty)
{
// NOTE This overwrites filter used in call
var filterBuilder = new NUnitTestFilterBuilder(TestEngine.Services.GetService<ITestFilterService>());
filter = filterBuilder.ConvertTfsFilterToNUnitFilter(TfsFilter, loadedTestCases);
}
if (filter == NUnitTestFilterBuilder.NoTestsFound)
{
TestLog.Info("Skipping assembly - no matching test cases found");
return;
}
using (var listener = new NUnitEventListener(FrameworkHandle, testConverter))
{
try
{
_activeRunner.Run(listener, filter);
}
catch (NullReferenceException)
{
// this happens during the run when CancelRun is called.
TestLog.Debug("Nullref caught");
}
}
}
}
else
TestLog.Info("NUnit failed to load " + assemblyName);
}
catch (BadImageFormatException)
{
// we skip the native c++ binaries that we don't support.
TestLog.Warning("Assembly not supported: " + assemblyName);
}
catch (System.IO.FileNotFoundException ex)
{
// Probably from the GetExportedTypes in NUnit.core, attempting to find an assembly, not a problem if it is not NUnit here
TestLog.Warning("Dependent Assembly " + ex.FileName + " of " + assemblyName + " not found. Can be ignored if not a NUnit project.");
}
catch (Exception ex)
{
if (ex is TargetInvocationException)
ex = ex.InnerException;
TestLog.Error("Exception thrown executing tests in " + assemblyName, ex);
}
_activeRunner.Dispose();
_activeRunner = null;
}
