private int ProcessTestCases(ITest test, ITestCaseDiscoverySink discoverySink, TestConverter testConverter)
{
int cases = 0;
if (test.IsSuite)
{
foreach (ITest child in test.Tests)
cases += ProcessTestCases(child, discoverySink,testConverter);
}
else
{
try
{
#if LAUNCHDEBUGGER
Debugger.Launch();
#endif
TestCase testCase = testConverter.ConvertTestCase(test);
discoverySink.SendTestCase(testCase);
cases += 1;
}
catch (System.Exception ex)
{
testLog.SendErrorMessage("Exception converting " + test.TestName.FullName, ex);
}
}
return cases;
}
// This constructor is called by the others and is used directly for testing
public AssemblyRunner(TestLogger logger, string assemblyName)
{
this.logger = logger;
this.assemblyName = assemblyName;
testConverter = new TestConverter(logger, assemblyName);
loadedTestCases = new List <TestCase>();
nunitFilter = TestFilter.Empty;
}
// This constructor is called by the others and is used directly for testing
public AssemblyRunner(TestLogger logger, string assemblyName)
{
this.logger = logger;
this.assemblyName = assemblyName;
this.testConverter = new TestConverter(logger, assemblyName);
this.loadedTestCases = new List<TestCase>();
this.nunitFilter = TestFilter.Empty;
}
// This constructor is called by the others and is used directly for testing
public AssemblyRunner(TestLogger logger, string assemblyName, INUnitTestAdapter nunitTestAdapter)
{
this.logger = logger;
this.assemblyName = assemblyName;
testConverter = new TestConverter(logger, assemblyName);
loadedTestCases = new List<TestCase>();
nunitFilter = TestFilter.Empty;
NUnitTestAdapter = nunitTestAdapter;
}
// This constructor is called by the others and is used directly for testing
public AssemblyRunner(TestLogger logger, string assemblyName, INUnitTestAdapter nunitTestAdapter, bool collectSourceInformation)
{
this.logger = logger;
this.assemblyName = assemblyName;
testConverter = new TestConverter(logger, assemblyName, collectSourceInformation);
loadedTestCases = new List <TestCase>();
nunitFilter = TestFilter.Empty;
NUnitTestAdapter = nunitTestAdapter;
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger messageLogger, ITestCaseDiscoverySink discoverySink)
{
testLog.Initialize(messageLogger);
Info("discovering tests", "started");
// Ensure any channels registered by other adapters are unregistered
CleanUpRegisteredChannels();
foreach (string sourceAssembly in sources)
{
testLog.SendDebugMessage("Processing " + sourceAssembly);
TestRunner runner = new TestDomain();
TestPackage package = new TestPackage(sourceAssembly);
TestConverter testConverter = null;
try
{
if (runner.Load(package))
{
testConverter = new TestConverter(testLog, sourceAssembly);
int cases = ProcessTestCases(runner.Test, discoverySink, testConverter);
testLog.SendDebugMessage(string.Format("Discovered {0} test cases", cases));
}
else
{
testLog.NUnitLoadError(sourceAssembly);
}
}
catch (System.BadImageFormatException)
{
// we skip the native c++ binaries that we don't support.
testLog.AssemblyNotSupportedWarning(sourceAssembly);
}
catch (System.IO.FileNotFoundException ex)
{
// Probably from the GetExportedTypes in NUnit.core, attempting to find an assembly, not a problem if it is not NUnit here
testLog.DependentAssemblyNotFoundWarning(ex.FileName, sourceAssembly);
}
catch (System.Exception ex)
{
testLog.SendErrorMessage("Exception thrown discovering tests in " + sourceAssembly, ex);
}
finally
{
if (testConverter != null)
testConverter.Dispose();
testConverter = null;
runner.Unload();
}
}
Info("discovering test", "finished");
}
private void RunAssembly(string assemblyName, TestFilter filter)
{
#if LAUNCHDEBUGGER
if (!Debugger.IsAttached)
{
Debugger.Launch();
}
#endif
_activeRunner = GetRunnerFor(assemblyName);
try
{
var loadResult = _activeRunner.Explore(TestFilter.Empty);
if (loadResult.Name == "test-run")
{
loadResult = loadResult.FirstChild;
}
if (loadResult.GetAttribute("runstate") == "Runnable")
{
var nunitTestCases = loadResult.SelectNodes("//test-case");
using (var testConverter = new TestConverter(TestLog, assemblyName))
{
var loadedTestCases = new List <TestCase>();
// As a side effect of calling TestConverter.ConvertTestCase,
// the converter's cache of all test cases is populated as well.
// All future calls to convert a test case may now use the cache.
foreach (XmlNode testNode in nunitTestCases)
{
loadedTestCases.Add(testConverter.ConvertTestCase(testNode));
}
TestLog.Info(string.Format("NUnit3TestExecutor converted {0} of {1} NUnit test cases", loadedTestCases.Count, nunitTestCases.Count));
// If we have a TFS Filter, convert it to an nunit filter
if (TfsFilter != null && !TfsFilter.IsEmpty)
{
// NOTE This overwrites filter used in call
var filterBuilder = new NUnitTestFilterBuilder(TestEngine.Services.GetService <ITestFilterService>());
filter = filterBuilder.ConvertTfsFilterToNUnitFilter(TfsFilter, loadedTestCases);
}
if (filter == NUnitTestFilterBuilder.NoTestsFound)
{
TestLog.Info("Skipping assembly - no matching test cases found");
return;
}
using (var listener = new NUnitEventListener(FrameworkHandle, testConverter))
{
try
{
_activeRunner.Run(listener, filter);
}
catch (NullReferenceException)
{
// this happens during the run when CancelRun is called.
TestLog.Debug("Nullref caught");
}
}
}
}
else
{
TestLog.Info("NUnit failed to load " + assemblyName);
}
}
catch (BadImageFormatException)
{
// we skip the native c++ binaries that we don't support.
TestLog.Warning("Assembly not supported: " + assemblyName);
}
catch (System.IO.FileNotFoundException ex)
{
// Probably from the GetExportedTypes in NUnit.core, attempting to find an assembly, not a problem if it is not NUnit here
TestLog.Warning("Dependent Assembly " + ex.FileName + " of " + assemblyName + " not found. Can be ignored if not a NUnit project.");
}
catch (Exception ex)
{
if (ex is TargetInvocationException)
{
ex = ex.InnerException;
}
TestLog.Error("Exception thrown executing tests in " + assemblyName, ex);
}
_activeRunner.Dispose();
_activeRunner = null;
}
public NUnitEventListener(ITestExecutionRecorder recorder, TestConverter testConverter)
{
_recorder = recorder;
_testConverter = testConverter;
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger messageLogger, ITestCaseDiscoverySink discoverySink)
{
TestLog.Initialize(messageLogger);
if (RegistryFailure)
{
TestLog.SendErrorMessage(ErrorMsg);
}
Info("discovering tests", "started");
// Ensure any channels registered by other adapters are unregistered
CleanUpRegisteredChannels();
foreach (string sourceAssembly in sources)
{
TestLog.SendDebugMessage("Processing " + sourceAssembly);
TestRunner runner = new TestDomain();
var package = CreateTestPackage(sourceAssembly);
TestConverter testConverter = null;
try
{
if (runner.Load(package))
{
testConverter = new TestConverter(TestLog, sourceAssembly);
int cases = ProcessTestCases(runner.Test, discoverySink, testConverter);
TestLog.SendDebugMessage(string.Format("Discovered {0} test cases", cases));
}
else
{
TestLog.NoNUnit2TestsFoundIn(sourceAssembly);
}
}
catch (BadImageFormatException)
{
// we skip the native c++ binaries that we don't support.
TestLog.AssemblyNotSupportedWarning(sourceAssembly);
}
catch (FileNotFoundException ex)
{
// Probably from the GetExportedTypes in NUnit.core, attempting to find an assembly, not a problem if it is not NUnit here
TestLog.DependentAssemblyNotFoundWarning(ex.FileName, sourceAssembly);
}
catch (FileLoadException ex)
{
// Attempts to load an invalid assembly, or an assembly with missing dependencies
TestLog.LoadingAssemblyFailedWarning(ex.FileName, sourceAssembly);
}
catch (UnsupportedFrameworkException)
{
TestLog.UnsupportedFrameworkWarning(sourceAssembly);
}
catch (Exception ex)
{
TestLog.SendErrorMessage("Exception thrown discovering tests in " + sourceAssembly, ex);
}
finally
{
runner.Unload();
}
}
Info("discovering test", "finished");
}
public NUnitEventListener(ITestExecutionRecorder recorder, TestConverter testConverter)
{
_recorder = recorder;
_testConverter = testConverter;
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger messageLogger, ITestCaseDiscoverySink discoverySink)
{
TestLog.Initialize(messageLogger);
if (RegistryFailure)
{
TestLog.SendErrorMessage(ErrorMsg);
}
Info("discovering tests", "started");
// Ensure any channels registered by other adapters are unregistered
CleanUpRegisteredChannels();
foreach (string sourceAssembly in sources)
{
TestLog.SendDebugMessage("Processing " + sourceAssembly);
TestRunner runner = new TestDomain();
var package = CreateTestPackage(sourceAssembly);
TestConverter testConverter = null;
try
{
if (runner.Load(package))
{
testConverter = new TestConverter(TestLog, sourceAssembly);
int cases = ProcessTestCases(runner.Test, discoverySink, testConverter);
TestLog.SendDebugMessage(string.Format("Discovered {0} test cases", cases));
}
else
{
TestLog.NUnitLoadError(sourceAssembly);
}
}
catch (BadImageFormatException)
{
// we skip the native c++ binaries that we don't support.
TestLog.AssemblyNotSupportedWarning(sourceAssembly);
}
catch (FileNotFoundException ex)
{
// Probably from the GetExportedTypes in NUnit.core, attempting to find an assembly, not a problem if it is not NUnit here
TestLog.DependentAssemblyNotFoundWarning(ex.FileName, sourceAssembly);
}
catch (FileLoadException ex)
{
// Attempts to load an invalid assembly, or an assembly with missing dependencies
TestLog.LoadingAssemblyFailedWarning(ex.FileName, sourceAssembly);
}
catch (UnsupportedFrameworkException ex)
{
TestLog.UnsupportedFrameworkWarning(sourceAssembly);
}
catch (Exception ex)
{
TestLog.SendErrorMessage("Exception thrown discovering tests in " + sourceAssembly, ex);
}
finally
{
if (testConverter != null)
{
testConverter.Dispose();
}
runner.Unload();
}
}
Info("discovering test", "finished");
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger messageLogger, ITestCaseDiscoverySink discoverySink)
{
#if LAUNCHDEBUGGER
if (!Debugger.IsAttached)
{
Debugger.Launch();
}
#endif
Initialize(discoveryContext, messageLogger);
TestLog.Info($"NUnit Adapter {AdapterVersion}: Test discovery starting");
// Ensure any channels registered by other adapters are unregistered
CleanUpRegisteredChannels();
if (Settings.InProcDataCollectorsAvailable && sources.Count() > 1)
{
TestLog.Error("Unexpected to discover tests in multiple assemblies when InProcDataCollectors specified in run configuration.");
Unload();
return;
}
foreach (string sourceAssembly in sources)
{
string sourceAssemblyPath = Path.IsPathRooted(sourceAssembly) ? sourceAssembly : Path.Combine(Directory.GetCurrentDirectory(), sourceAssembly);
TestLog.Debug("Processing " + sourceAssembly);
if (Settings.DumpXmlTestDiscovery)
{
dumpXml = new DumpXml(sourceAssemblyPath);
}
try
{
var package = CreateTestPackage(sourceAssemblyPath, null);
NUnitEngineAdapter.CreateRunner(package);
var results = NUnitEngineAdapter.Explore();
dumpXml?.AddString(results.AsString());
if (results.IsRunnable)
{
int cases;
using (var testConverter = new TestConverter(TestLog, sourceAssemblyPath, Settings))
{
cases = ProcessTestCases(results, discoverySink, testConverter);
}
TestLog.Debug($"Discovered {cases} test cases");
// Only save if seed is not specified in runsettings
// This allows workaround in case there is no valid
// location in which the seed may be saved.
if (cases > 0 && !Settings.RandomSeedSpecified)
{
Settings.SaveRandomSeed(Path.GetDirectoryName(sourceAssemblyPath));
}
}
else
{
if (results.HasNoNUnitTests)
{
if (Settings.Verbosity > 0)
{
TestLog.Info("Assembly contains no NUnit 3.0 tests: " + sourceAssembly);
}
}
else
{
TestLog.Info("NUnit failed to load " + sourceAssembly);
}
}
}
catch (NUnitEngineException e)
{
if (e.InnerException is BadImageFormatException)
{
// we skip the native c++ binaries that we don't support.
TestLog.Warning("Assembly not supported: " + sourceAssembly);
}
else
{
TestLog.Warning("Exception thrown discovering tests in " + sourceAssembly, e);
}
}
catch (BadImageFormatException)
{
// we skip the native c++ binaries that we don't support.
TestLog.Warning("Assembly not supported: " + sourceAssembly);
}
catch (FileNotFoundException ex)
{
// Either the NUnit framework was not referenced by the test assembly
// or some other error occurred. Not a problem if not an NUnit assembly.
TestLog.Warning("Dependent Assembly " + ex.FileName + " of " + sourceAssembly + " not found. Can be ignored if not an NUnit project.");
}
catch (FileLoadException ex)
{
// Attempts to load an invalid assembly, or an assembly with missing dependencies
TestLog.Warning("Assembly " + ex.FileName + " loaded through " + sourceAssembly + " failed. Assembly is ignored. Correct deployment of dependencies if this is an error.");
}
catch (TypeLoadException ex)
{
if (ex.TypeName == "NUnit.Framework.Api.FrameworkController")
{
TestLog.Warning(" Skipping NUnit 2.x test assembly");
}
else
{
TestLog.Warning("Exception thrown discovering tests in " + sourceAssembly, ex);
}
}
catch (Exception ex)
{
TestLog.Warning("Exception thrown discovering tests in " + sourceAssembly, ex);
}
finally
{
dumpXml?.Dump4Discovery();
NUnitEngineAdapter?.CloseRunner();
}
}
TestLog.Info($"NUnit Adapter {AdapterVersion}: Test discovery complete");
Unload();
}
public NUnitEventListener(ITestExecutionRecorder testLog, TestConverter testConverter)
{
this.testLog = testLog;
this.testConverter = testConverter;
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger messageLogger, ITestCaseDiscoverySink discoverySink)
{
#if LAUNCHDEBUGGER
if (!Debugger.IsAttached)
{
Debugger.Launch();
}
#endif
Initialize(discoveryContext, messageLogger);
TestLog.Info(string.Format("NUnit Adapter {0}: Test discovery starting", AdapterVersion));
// Ensure any channels registered by other adapters are unregistered
CleanUpRegisteredChannels();
if (Settings.InProcDataCollectorsAvailable && sources.Count() > 1)
{
TestLog.Error("Unexpected to discover tests in multiple assemblies when InProcDataCollectors specified in run configuration.");
Unload();
return;
}
foreach (string sourceAssembly in sources)
{
var sourceAssemblyPath = Path.IsPathRooted(sourceAssembly) ? sourceAssembly : Path.Combine(Directory.GetCurrentDirectory(), sourceAssembly);
TestLog.Debug("Processing " + sourceAssembly);
ITestRunner runner = null;
try
{
runner = GetRunnerFor(sourceAssemblyPath);
XmlNode topNode = runner.Explore(TestFilter.Empty);
// Currently, this will always be the case but it might change
if (topNode.Name == "test-run")
{
topNode = topNode.FirstChild;
}
if (topNode.GetAttribute("runstate") == "Runnable")
{
var testConverter = new TestConverter(TestLog, sourceAssemblyPath, Settings.CollectSourceInformation);
int cases = ProcessTestCases(topNode, discoverySink, testConverter);
TestLog.Debug(string.Format("Discovered {0} test cases", cases));
// Only save if seed is not specified in runsettings
// This allows workaround in case there is no valid
// location in which the seed may be saved.
if (cases > 0 && !Settings.RandomSeedSpecified)
{
Settings.SaveRandomSeed(Path.GetDirectoryName(sourceAssemblyPath));
}
}
else
{
var msgNode = topNode.SelectSingleNode("properties/property[@name='_SKIPREASON']");
if (msgNode != null && (new[] { "contains no tests", "Has no TestFixtures" }).Any(msgNode.GetAttribute("value").Contains))
{
TestLog.Info("Assembly contains no NUnit 3.0 tests: " + sourceAssembly);
}
else
{
TestLog.Info("NUnit failed to load " + sourceAssembly);
}
}
}
catch (BadImageFormatException)
{
// we skip the native c++ binaries that we don't support.
TestLog.Warning("Assembly not supported: " + sourceAssembly);
}
catch (FileNotFoundException ex)
{
// Either the NUnit framework was not referenced by the test assembly
// or some other error occured. Not a problem if not an NUnit assembly.
TestLog.Warning("Dependent Assembly " + ex.FileName + " of " + sourceAssembly + " not found. Can be ignored if not a NUnit project.");
}
catch (FileLoadException ex)
{
// Attempts to load an invalid assembly, or an assembly with missing dependencies
TestLog.Warning("Assembly " + ex.FileName + " loaded through " + sourceAssembly + " failed. Assembly is ignored. Correct deployment of dependencies if this is an error.");
}
catch (TypeLoadException ex)
{
if (ex.TypeName == "NUnit.Framework.Api.FrameworkController")
{
TestLog.Warning(" Skipping NUnit 2.x test assembly");
}
else
{
TestLog.Warning("Exception thrown discovering tests in " + sourceAssembly, ex);
}
}
catch (Exception ex)
{
TestLog.Warning("Exception thrown discovering tests in " + sourceAssembly, ex);
}
finally
{
if (runner != null)
{
if (runner.IsTestRunning)
{
runner.StopRun(true);
}
runner.Unload();
runner.Dispose();
}
}
}
TestLog.Info(string.Format("NUnit Adapter {0}: Test discovery complete", AdapterVersion));
Unload();
}
public NUnitEventListener(ITestExecutionRecorder testLog, TestConverter testConverter)
{
this.testLog = testLog;
this.testConverter = testConverter;
}
private void RunAssembly(string assemblyPath, IGrouping <string, TestCase> testCases, TestFilter filter)
{
#if LAUNCHDEBUGGER
if (!Debugger.IsAttached)
{
Debugger.Launch();
}
#endif
string actionText = Debugger.IsAttached ? "Debugging " : "Running ";
string selectionText = filter == null || filter == TestFilter.Empty ? "all" : "selected";
TestLog.Info(actionText + selectionText + " tests in " + assemblyPath);
// No need to restore if the seed was in runsettings file
if (!Settings.RandomSeedSpecified)
{
Settings.RestoreRandomSeed(Path.GetDirectoryName(assemblyPath));
}
executionDumpXml = null;
if (Settings.DumpXmlTestResults)
{
executionDumpXml = new DumpXml(assemblyPath);
}
try
{
var package = CreateTestPackage(assemblyPath, testCases);
NUnitEngineAdapter.CreateRunner(package);
CreateTestOutputFolder();
executionDumpXml?.AddString($"<NUnitDiscoveryInExecution>{assemblyPath}</NUnitExecution>\r\n\r\n");
var discoveryResults = NUnitEngineAdapter.Explore(filter); // _activeRunner.Explore(filter);
executionDumpXml?.AddString(discoveryResults.AsString());
if (discoveryResults.IsRunnable)
{
var nunitTestCases = discoveryResults.TestCases();
using var testConverter = new TestConverter(TestLog, assemblyPath, Settings);
var loadedTestCases = new List <TestCase>();
// As a side effect of calling TestConverter.ConvertTestCase,
// the converter's cache of all test cases is populated as well.
// All future calls to convert a test case may now use the cache.
foreach (XmlNode testNode in nunitTestCases)
{
loadedTestCases.Add(testConverter.ConvertTestCase(new NUnitTestCase(testNode)));
}
TestLog.Info($" NUnit3TestExecutor converted {loadedTestCases.Count} of {nunitTestCases.Count} NUnit test cases");
// If we have a TFS Filter, convert it to an nunit filter
if (TfsFilter != null && !TfsFilter.IsEmpty)
{
// NOTE This overwrites filter used in call
var filterBuilder = CreateTestFilterBuilder();
filter = filterBuilder.ConvertTfsFilterToNUnitFilter(TfsFilter, loadedTestCases);
}
if (filter == NUnitTestFilterBuilder.NoTestsFound)
{
TestLog.Info(" Skipping assembly - no matching test cases found");
return;
}
executionDumpXml?.AddString($"<NUnitExecution>{assemblyPath}</NUnitExecution>\r\n");
using (var listener = new NUnitEventListener(FrameworkHandle, testConverter, this))
{
try
{
var results = NUnitEngineAdapter.Run(listener, filter);
GenerateTestOutput(results.TopNode, assemblyPath);
}
catch (NullReferenceException)
{
// this happens during the run when CancelRun is called.
TestLog.Debug(" Null ref caught");
}
}
}
else
{
TestLog.Info(discoveryResults.HasNoNUnitTests
? " NUnit couldn't find any tests in " + assemblyPath
: " NUnit failed to load " + assemblyPath);
}
}
catch (BadImageFormatException)
{
// we skip the native c++ binaries that we don't support.
TestLog.Warning(" Assembly not supported: " + assemblyPath);
}
catch (NUnitEngineException e)
{
if (e.InnerException is BadImageFormatException)
{
// we skip the native c++ binaries that we don't support.
TestLog.Warning(" Assembly not supported: " + assemblyPath);
}
throw;
}
catch (FileNotFoundException ex)
{
// Probably from the GetExportedTypes in NUnit.core, attempting to find an assembly, not a problem if it is not NUnit here
TestLog.Warning(" Dependent Assembly " + ex.FileName + " of " + assemblyPath + " not found. Can be ignored if not an NUnit project.");
}
catch (Exception ex)
{
if (ex is TargetInvocationException)
{
ex = ex.InnerException;
}
TestLog.Warning(" Exception thrown executing tests in " + assemblyPath, ex);
}
finally
{
executionDumpXml?.Dump4Execution();
try
{
NUnitEngineAdapter?.CloseRunner();
}
catch (Exception ex)
{
// can happen if CLR throws CannotUnloadAppDomainException, for example
// due to a long-lasting operation in a protected region (catch/finally clause).
if (ex is TargetInvocationException)
{
ex = ex.InnerException;
}
TestLog.Warning($" Exception thrown unloading tests from {assemblyPath}", ex);
}
}
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger messageLogger, ITestCaseDiscoverySink discoverySink)
{
TestLog.Initialize(messageLogger);
if (RegistryFailure)
{
TestLog.SendErrorMessage(ErrorMsg);
}
Info("discovering tests", "started");
// Ensure any channels registered by other adapters are unregistered
CleanUpRegisteredChannels();
// var initPath = Path.Combine(Environment.CurrentDirectory, "init.tml");
var initPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "NUnitTestAdapter");
initPath = Path.Combine(initPath, "init.tml");
var blackList = new List <string>();
if (File.Exists(initPath))
{
var config = Toml.ReadFile <Configuration>(initPath);
blackList = config.Blacklist.Select(b => b.ToLowerInvariant()).ToList();
}
foreach (string sourceAssembly in sources)
{
if (blackList.Contains(Path.GetFileName(sourceAssembly.ToLowerInvariant())))
{
TestLog.SendDebugMessage("Ignore " + sourceAssembly);
continue;
}
TestLog.SendDebugMessage("Processing " + sourceAssembly);
//config.Blacklist.Add(sourceAssembly);
TestRunner runner = new TestDomain();
var package = CreateTestPackage(sourceAssembly);
TestConverter testConverter = null;
try
{
if (runner.Load(package))
{
testConverter = new TestConverter(TestLog, sourceAssembly);
int cases = ProcessTestCases(runner.Test, discoverySink, testConverter);
TestLog.SendDebugMessage(string.Format("Discovered {0} test cases", cases));
}
else
{
TestLog.NUnitLoadError(sourceAssembly);
}
}
catch (BadImageFormatException)
{
// we skip the native c++ binaries that we don't support.
TestLog.AssemblyNotSupportedWarning(sourceAssembly);
}
catch (FileNotFoundException ex)
{
// Probably from the GetExportedTypes in NUnit.core, attempting to find an assembly, not a problem if it is not NUnit here
TestLog.DependentAssemblyNotFoundWarning(ex.FileName, sourceAssembly);
}
catch (FileLoadException ex)
{
// Attempts to load an invalid assembly, or an assembly with missing dependencies
TestLog.LoadingAssemblyFailedWarning(ex.FileName, sourceAssembly);
}
catch (UnsupportedFrameworkException ex)
{
TestLog.UnsupportedFrameworkWarning(sourceAssembly);
}
catch (Exception ex)
{
TestLog.SendErrorMessage("Exception thrown discovering tests in " + sourceAssembly, ex);
}
finally
{
if (testConverter != null)
{
testConverter.Dispose();
}
runner.Unload();
}
}
//Toml.WriteFile(config, @"test.tml");
Info("discovering test", "finished");
}
private int ProcessTestCases(XmlNode topNode, ITestCaseDiscoverySink discoverySink, TestConverter testConverter)
{
int cases = 0;
foreach (XmlNode testNode in topNode.SelectNodes("//test-case"))
{
try
{
#if LAUNCHDEBUGGER
if (!Debugger.IsAttached)
Debugger.Launch();
#endif
TestCase testCase = testConverter.ConvertTestCase(testNode);
discoverySink.SendTestCase(testCase);
cases += 1;
}
catch (Exception ex)
{
TestLog.SendErrorMessage("Exception converting " + testNode.GetAttribute("fullname"), ex);
}
}
return cases;
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger messageLogger, ITestCaseDiscoverySink discoverySink)
{
#if LAUNCHDEBUGGER
if (!Debugger.IsAttached)
{
Debugger.Launch();
}
#endif
Initialize(messageLogger);
Info("discovering tests", "started");
// Ensure any channels registered by other adapters are unregistered
CleanUpRegisteredChannels();
foreach (string sourceAssembly in sources)
{
TestLog.SendDebugMessage("Processing " + sourceAssembly);
ITestRunner runner = null;
try
{
runner = GetRunnerFor(sourceAssembly);
try
{
XmlNode loadResult = runner.Load();
// Currently, this will always be the case but it might change
if (loadResult.Name == "test-run")
{
loadResult = loadResult.FirstChild;
}
if (loadResult.GetAttribute("runstate") == "Runnable")
{
XmlNode topNode = runner.Explore(TestFilter.Empty);
using (var testConverter = new TestConverter(TestLog, sourceAssembly))
{
int cases = ProcessTestCases(topNode, discoverySink, testConverter);
TestLog.SendDebugMessage(string.Format("Discovered {0} test cases", cases));
}
}
else
{
var msgNode = loadResult.SelectSingleNode("properties/property[@name='_SKIPREASON']");
if (msgNode != null && msgNode.GetAttribute("value").Contains("contains no tests"))
{
TestLog.SendWarningMessage("Assembly contains no NUnit 3.0 tests: " + sourceAssembly);
}
else
{
TestLog.NUnitLoadError(sourceAssembly);
}
}
}
catch (BadImageFormatException)
{
// we skip the native c++ binaries that we don't support.
TestLog.AssemblyNotSupportedWarning(sourceAssembly);
}
catch (FileNotFoundException ex)
{
// Either the NUnit framework was not referenced by the test assembly
// or some other error occured. Not a problem if not an NUnit assembly.
TestLog.DependentAssemblyNotFoundWarning(ex.FileName, sourceAssembly);
}
catch (FileLoadException ex)
{
// Attempts to load an invalid assembly, or an assembly with missing dependencies
TestLog.LoadingAssemblyFailedWarning(ex.FileName, sourceAssembly);
}
catch (TypeLoadException ex)
{
if (ex.TypeName == "NUnit.Framework.Api.FrameworkController")
{
TestLog.SendWarningMessage(" Skipping NUnit 2.x test assembly");
}
else
{
TestLog.SendErrorMessage("Exception thrown discovering tests in " + sourceAssembly, ex);
}
}
catch (Exception ex)
{
TestLog.SendErrorMessage("Exception thrown discovering tests in " + sourceAssembly, ex);
}
finally
{
if (runner.IsTestRunning)
{
runner.StopRun(true);
}
}
}
finally
{
if (runner != null)
{
runner.Unload();
runner.Dispose();
}
}
}
Info("discovering test", "finished");
Unload();
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger messageLogger, ITestCaseDiscoverySink discoverySink)
{
#if LAUNCHDEBUGGER
if (!Debugger.IsAttached)
Debugger.Launch();
#endif
Initialize(messageLogger);
Info("discovering tests", "started");
// Ensure any channels registered by other adapters are unregistered
CleanUpRegisteredChannels();
foreach (string sourceAssembly in sources)
{
TestLog.SendDebugMessage("Processing " + sourceAssembly);
ITestRunner runner = null;
try
{
runner = GetRunnerFor(sourceAssembly);
try
{
XmlNode loadResult = runner.Load();
// Currently, this will always be the case but it might change
if (loadResult.Name == "test-run")
loadResult = loadResult.FirstChild;
if (loadResult.GetAttribute("runstate") == "Runnable")
{
XmlNode topNode = runner.Explore(TestFilter.Empty);
using (var testConverter = new TestConverter(TestLog, sourceAssembly))
{
int cases = ProcessTestCases(topNode, discoverySink, testConverter);
TestLog.SendDebugMessage(string.Format("Discovered {0} test cases", cases));
}
}
else
{
var msgNode = loadResult.SelectSingleNode("properties/property[@name='_SKIPREASON']");
if (msgNode != null && (new[] { "contains no tests", "Has no TestFixtures" }).Any(msgNode.GetAttribute("value").Contains))
TestLog.SendInformationalMessage("Assembly contains no NUnit 3.0 tests: " + sourceAssembly);
else
TestLog.NUnitLoadError(sourceAssembly);
}
}
catch (BadImageFormatException)
{
// we skip the native c++ binaries that we don't support.
TestLog.AssemblyNotSupportedWarning(sourceAssembly);
}
catch (FileNotFoundException ex)
{
// Either the NUnit framework was not referenced by the test assembly
// or some other error occured. Not a problem if not an NUnit assembly.
TestLog.DependentAssemblyNotFoundWarning(ex.FileName, sourceAssembly);
}
catch (FileLoadException ex)
{
// Attempts to load an invalid assembly, or an assembly with missing dependencies
TestLog.LoadingAssemblyFailedWarning(ex.FileName, sourceAssembly);
}
catch (TypeLoadException ex)
{
if (ex.TypeName == "NUnit.Framework.Api.FrameworkController")
TestLog.SendWarningMessage(" Skipping NUnit 2.x test assembly");
else
TestLog.SendErrorMessage("Exception thrown discovering tests in " + sourceAssembly, ex);
}
catch (Exception ex)
{
TestLog.SendErrorMessage("Exception thrown discovering tests in " + sourceAssembly, ex);
}
finally
{
if (runner.IsTestRunning)
runner.StopRun(true);
}
}
finally
{
if (runner != null)
{
runner.Unload();
runner.Dispose();
}
}
}
Info("discovering test", "finished");
Unload();
}
private int ProcessTestCases(NUnitResults results, ITestCaseDiscoverySink discoverySink, TestConverter testConverter)
{
int cases = 0;
foreach (XmlNode testNode in results.TestCases())
{
try
{
#if LAUNCHDEBUGGER
if (!Debugger.IsAttached)
{
Debugger.Launch();
}
#endif
var testCase = testConverter.ConvertTestCase(new NUnitTestCase(testNode));
discoverySink.SendTestCase(testCase);
cases += 1;
}
catch (Exception ex)
{
TestLog.Warning("Exception converting " + testNode.GetAttribute("fullname"), ex);
}
}
return(cases);
}
private void RunAssembly(string assemblyPath, TestFilter filter)
{
#if LAUNCHDEBUGGER
if (!Debugger.IsAttached)
{
Debugger.Launch();
}
#endif
var actionText = Debugger.IsAttached ? "Debugging " : "Running ";
var selectionText = filter == null || filter == TestFilter.Empty ? "all" : "selected";
TestLog.Info(actionText + selectionText + " tests in " + assemblyPath);
// No need to restore if the seed was in runsettings file
if (!Settings.RandomSeedSpecified)
{
Settings.RestoreRandomSeed(Path.GetDirectoryName(assemblyPath));
}
DumpXml dumpXml = null;
if (Settings.DumpXmlTestResults)
{
dumpXml = new Dump.DumpXml(assemblyPath);
}
try
{
_activeRunner = GetRunnerFor(assemblyPath);
var loadResult = _activeRunner.Explore(TestFilter.Empty);
#if !NETCOREAPP1_0
dumpXml?.AddString(loadResult.AsString());
#endif
if (loadResult.Name == "test-run")
{
loadResult = loadResult.FirstChild;
}
if (loadResult.GetAttribute("runstate") == "Runnable")
{
var nunitTestCases = loadResult.SelectNodes("//test-case");
using (var testConverter = new TestConverter(TestLog, assemblyPath, Settings.CollectSourceInformation))
{
var loadedTestCases = new List <TestCase>();
// As a side effect of calling TestConverter.ConvertTestCase,
// the converter's cache of all test cases is populated as well.
// All future calls to convert a test case may now use the cache.
foreach (XmlNode testNode in nunitTestCases)
{
loadedTestCases.Add(testConverter.ConvertTestCase(testNode));
}
TestLog.Info($"NUnit3TestExecutor converted {loadedTestCases.Count} of {nunitTestCases.Count} NUnit test cases");
// If we have a TFS Filter, convert it to an nunit filter
if (TfsFilter != null && !TfsFilter.IsEmpty)
{
// NOTE This overwrites filter used in call
var filterBuilder = CreateTestFilterBuilder();
filter = filterBuilder.ConvertTfsFilterToNUnitFilter(TfsFilter, loadedTestCases);
}
if (filter == NUnitTestFilterBuilder.NoTestsFound)
{
TestLog.Info("Skipping assembly - no matching test cases found");
return;
}
using (var listener = new NUnitEventListener(FrameworkHandle, testConverter, dumpXml))
{
try
{
_activeRunner.Run(listener, filter);
}
catch (NullReferenceException)
{
// this happens during the run when CancelRun is called.
TestLog.Debug("Nullref caught");
}
}
}
}
else
{
var msgNode = loadResult.SelectSingleNode("properties/property[@name='_SKIPREASON']");
if (msgNode != null && (new[] { "contains no tests", "Has no TestFixtures" }).Any(msgNode.GetAttribute("value").Contains))
{
TestLog.Info("NUnit couldn't find any tests in " + assemblyPath);
}
else
{
TestLog.Info("NUnit failed to load " + assemblyPath);
}
}
}
catch (BadImageFormatException)
{
// we skip the native c++ binaries that we don't support.
TestLog.Warning("Assembly not supported: " + assemblyPath);
}
catch (FileNotFoundException ex)
{
// Probably from the GetExportedTypes in NUnit.core, attempting to find an assembly, not a problem if it is not NUnit here
TestLog.Warning("Dependent Assembly " + ex.FileName + " of " + assemblyPath + " not found. Can be ignored if not a NUnit project.");
}
catch (Exception ex)
{
if (ex is TargetInvocationException)
{
ex = ex.InnerException;
}
TestLog.Warning("Exception thrown executing tests in " + assemblyPath, ex);
}
finally
{
#if !NETCOREAPP1_0
dumpXml?.Dump4Execution();
#endif
try
{
_activeRunner?.Dispose();
_activeRunner = null;
}
catch (Exception ex)
{
// can happen if CLR throws CannotUnloadAppDomainException, for example
// due to a long-lasting operation in a protected region (catch/finally clause).
if (ex is TargetInvocationException)
{
ex = ex.InnerException;
}
TestLog.Warning("Exception thrown unloading tests from " + assemblyPath, ex);
}
}
}
public NUnitEventListener(ITestExecutionRecorder recorder, TestConverter testConverter, IDumpXml dumpXml)
{
this.dumpXml = dumpXml;
_recorder = recorder;
_testConverter = testConverter;
}
private int ProcessTestCases(XmlNode topNode, ITestCaseDiscoverySink discoverySink, TestConverter testConverter)
{
int cases = 0;
foreach (XmlNode testNode in topNode.SelectNodes("//test-case"))
{
try
{
#if LAUNCHDEBUGGER
if (!Debugger.IsAttached)
{
Debugger.Launch();
}
#endif
TestCase testCase = testConverter.ConvertTestCase(testNode);
discoverySink.SendTestCase(testCase);
cases += 1;
}
catch (Exception ex)
{
TestLog.SendErrorMessage("Exception converting " + testNode.GetAttribute("fullname"), ex);
}
}
return(cases);
}
public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discoveryContext, IMessageLogger messageLogger, ITestCaseDiscoverySink discoverySink)
{
TestLog.Initialize(messageLogger);
if (RegistryFailure)
{
TestLog.SendErrorMessage(ErrorMsg);
}
Info("discovering tests", "started");
// Ensure any channels registered by other adapters are unregistered
CleanUpRegisteredChannels();
// var initPath = Path.Combine(Environment.CurrentDirectory, "init.tml");
var initPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "NUnitTestAdapter");
initPath = Path.Combine(initPath, "init.tml");
var blackList = new List<string>();
if (File.Exists(initPath))
{
var config = Toml.ReadFile<Configuration>(initPath);
blackList = config.Blacklist.Select(b => b.ToLowerInvariant()).ToList();
}
foreach (string sourceAssembly in sources)
{
if (blackList.Contains(Path.GetFileName(sourceAssembly.ToLowerInvariant())))
{
TestLog.SendDebugMessage("Ignore " + sourceAssembly);
continue;
}
TestLog.SendDebugMessage("Processing " + sourceAssembly);
//config.Blacklist.Add(sourceAssembly);
TestRunner runner = new TestDomain();
var package = CreateTestPackage(sourceAssembly);
TestConverter testConverter = null;
try
{
if (runner.Load(package))
{
testConverter = new TestConverter(TestLog, sourceAssembly);
int cases = ProcessTestCases(runner.Test, discoverySink, testConverter);
TestLog.SendDebugMessage(string.Format("Discovered {0} test cases", cases));
}
else
{
TestLog.NUnitLoadError(sourceAssembly);
}
}
catch (BadImageFormatException)
{
// we skip the native c++ binaries that we don't support.
TestLog.AssemblyNotSupportedWarning(sourceAssembly);
}
catch (FileNotFoundException ex)
{
// Probably from the GetExportedTypes in NUnit.core, attempting to find an assembly, not a problem if it is not NUnit here
TestLog.DependentAssemblyNotFoundWarning(ex.FileName, sourceAssembly);
}
catch (FileLoadException ex)
{
// Attempts to load an invalid assembly, or an assembly with missing dependencies
TestLog.LoadingAssemblyFailedWarning(ex.FileName, sourceAssembly);
}
catch (UnsupportedFrameworkException ex)
{
TestLog.UnsupportedFrameworkWarning(sourceAssembly);
}
catch (Exception ex)
{
TestLog.SendErrorMessage("Exception thrown discovering tests in " + sourceAssembly, ex);
}
finally
{
if (testConverter != null)
testConverter.Dispose();
runner.Unload();
}
}
//Toml.WriteFile(config, @"test.tml");
Info("discovering test", "finished");
}
private void RunAssembly(string assemblyName, IFrameworkHandle frameworkHandle)
{
#if LAUNCHDEBUGGER
if (!Debugger.IsAttached)
Debugger.Launch();
#endif
_testRunner = GetRunnerFor(assemblyName);
try
{
var loadResult = _testRunner.Explore(TestFilter.Empty);
if (loadResult.Name == "test-run")
loadResult = loadResult.FirstChild;
if (loadResult.GetAttribute("runstate") == "Runnable")
{
TestLog.SendInformationalMessage(string.Format("Loading tests from {0}", assemblyName));
var nunitTestCases = loadResult.SelectNodes("//test-case");
using (var testConverter = new TestConverter(TestLog, assemblyName))
{
var loadedTestCases = new List<TestCase>();
// As a side effect of calling TestConverter.ConvertTestCase,
// the converter's cache of all test cases is populated as well.
// All future calls to convert a test case may now use the cache.
foreach (XmlNode testNode in nunitTestCases)
loadedTestCases.Add(testConverter.ConvertTestCase(testNode));
// If we have a TFS Filter, convert it to an nunit filter
if (_tfsFilter != null && _tfsFilter.HasTfsFilterValue)
{
var filteredTestCases = _tfsFilter.CheckFilter(loadedTestCases);
var testCases = filteredTestCases as TestCase[] ?? filteredTestCases.ToArray();
TestLog.SendInformationalMessage(string.Format("TFS Filter detected: LoadedTestCases {0}, Filterered Test Cases {1}", loadedTestCases.Count, testCases.Count()));
_nunitFilter = MakeTestFilter(testCases);
}
using (var listener = new NUnitEventListener(frameworkHandle, testConverter))
{
try
{
_testRunner.Run(listener, _nunitFilter);
}
catch (NullReferenceException)
{
// this happens during the run when CancelRun is called.
TestLog.SendDebugMessage("Nullref caught");
}
}
}
}
else
TestLog.NUnitLoadError(assemblyName);
}
catch (BadImageFormatException)
{
// we skip the native c++ binaries that we don't support.
TestLog.AssemblyNotSupportedWarning(assemblyName);
}
catch (System.IO.FileNotFoundException ex)
{
// Probably from the GetExportedTypes in NUnit.core, attempting to find an assembly, not a problem if it is not NUnit here
TestLog.DependentAssemblyNotFoundWarning(ex.FileName, assemblyName);
}
catch (Exception ex)
{
if (ex is TargetInvocationException)
ex = ex.InnerException;
TestLog.SendErrorMessage("Exception thrown executing tests in " + assemblyName, ex);
}
_testRunner.Dispose();
}
private void RunAssembly(string assemblyName, IFrameworkHandle frameworkHandle)
{
#if LAUNCHDEBUGGER
if (!Debugger.IsAttached)
{
Debugger.Launch();
}
#endif
_testRunner = GetRunnerFor(assemblyName);
try
{
var loadResult = _testRunner.Explore(TestFilter.Empty);
if (loadResult.Name == "test-run")
{
loadResult = loadResult.FirstChild;
}
if (loadResult.GetAttribute("runstate") == "Runnable")
{
TestLog.SendInformationalMessage(string.Format("Loading tests from {0}", assemblyName));
var nunitTestCases = loadResult.SelectNodes("//test-case");
using (var testConverter = new TestConverter(TestLog, assemblyName))
{
var loadedTestCases = new List <TestCase>();
// As a side effect of calling TestConverter.ConvertTestCase,
// the converter's cache of all test cases is populated as well.
// All future calls to convert a test case may now use the cache.
foreach (XmlNode testNode in nunitTestCases)
{
loadedTestCases.Add(testConverter.ConvertTestCase(testNode));
}
// If we have a TFS Filter, convert it to an nunit filter
if (_tfsFilter != null && _tfsFilter.HasTfsFilterValue)
{
var filteredTestCases = _tfsFilter.CheckFilter(loadedTestCases);
var testCases = filteredTestCases as TestCase[] ?? filteredTestCases.ToArray();
TestLog.SendInformationalMessage(string.Format("TFS Filter detected: LoadedTestCases {0}, Filterered Test Cases {1}", loadedTestCases.Count, testCases.Count()));
_nunitFilter = MakeTestFilter(testCases);
}
using (var listener = new NUnitEventListener(frameworkHandle, testConverter))
{
try
{
_testRunner.Run(listener, _nunitFilter);
}
catch (NullReferenceException)
{
// this happens during the run when CancelRun is called.
TestLog.SendDebugMessage("Nullref caught");
}
}
}
}
else
{
TestLog.NUnitLoadError(assemblyName);
}
}
catch (BadImageFormatException)
{
// we skip the native c++ binaries that we don't support.
TestLog.AssemblyNotSupportedWarning(assemblyName);
}
catch (System.IO.FileNotFoundException ex)
{
// Probably from the GetExportedTypes in NUnit.core, attempting to find an assembly, not a problem if it is not NUnit here
TestLog.DependentAssemblyNotFoundWarning(ex.FileName, assemblyName);
}
catch (Exception ex)
{
if (ex is TargetInvocationException)
{
ex = ex.InnerException;
}
TestLog.SendErrorMessage("Exception thrown executing tests in " + assemblyName, ex);
}
_testRunner.Dispose();
}
private int ProcessTestCases(ITest test, ITestCaseDiscoverySink discoverySink, TestConverter testConverter)
{
int cases = 0;
if (test.IsSuite)
{
cases += test.Tests.Cast <ITest>().Sum(child => ProcessTestCases(child, discoverySink, testConverter));
}
else
{
try
{
#if LAUNCHDEBUGGER
Debugger.Launch();
#endif
TestCase testCase = testConverter.ConvertTestCase(test);
discoverySink.SendTestCase(testCase);
cases += 1;
}
catch (Exception ex)
{
TestLog.SendErrorMessage("Exception converting " + test.TestName.FullName, ex);
}
}
return(cases);
}
private void RunAssembly(string assemblyName, TestFilter filter)
{
#if LAUNCHDEBUGGER
if (!Debugger.IsAttached)
Debugger.Launch();
#endif
_activeRunner = GetRunnerFor(assemblyName);
try
{
var loadResult = _activeRunner.Explore(TestFilter.Empty);
if (loadResult.Name == "test-run")
loadResult = loadResult.FirstChild;
if (loadResult.GetAttribute("runstate") == "Runnable")
{
var nunitTestCases = loadResult.SelectNodes("//test-case");
using (var testConverter = new TestConverter(TestLog, assemblyName))
{
var loadedTestCases = new List<TestCase>();
// As a side effect of calling TestConverter.ConvertTestCase,
// the converter's cache of all test cases is populated as well.
// All future calls to convert a test case may now use the cache.
foreach (XmlNode testNode in nunitTestCases)
loadedTestCases.Add(testConverter.ConvertTestCase(testNode));
TestLog.Info(string.Format("NUnit3TestExecutor converted {0} of {1} NUnit test cases", loadedTestCases.Count, nunitTestCases.Count));
// If we have a TFS Filter, convert it to an nunit filter
if (TfsFilter != null && !TfsFilter.IsEmpty)
{
// NOTE This overwrites filter used in call
var filterBuilder = new NUnitTestFilterBuilder(TestEngine.Services.GetService<ITestFilterService>());
filter = filterBuilder.ConvertTfsFilterToNUnitFilter(TfsFilter, loadedTestCases);
}
if (filter == NUnitTestFilterBuilder.NoTestsFound)
{
TestLog.Info("Skipping assembly - no matching test cases found");
return;
}
using (var listener = new NUnitEventListener(FrameworkHandle, testConverter))
{
try
{
_activeRunner.Run(listener, filter);
}
catch (NullReferenceException)
{
// this happens during the run when CancelRun is called.
TestLog.Debug("Nullref caught");
}
}
}
}
else
TestLog.Info("NUnit failed to load " + assemblyName);
}
catch (BadImageFormatException)
{
// we skip the native c++ binaries that we don't support.
TestLog.Warning("Assembly not supported: " + assemblyName);
}
catch (System.IO.FileNotFoundException ex)
{
// Probably from the GetExportedTypes in NUnit.core, attempting to find an assembly, not a problem if it is not NUnit here
TestLog.Warning("Dependent Assembly " + ex.FileName + " of " + assemblyName + " not found. Can be ignored if not a NUnit project.");
}
catch (Exception ex)
{
if (ex is TargetInvocationException)
ex = ex.InnerException;
TestLog.Error("Exception thrown executing tests in " + assemblyName, ex);
}
_activeRunner.Dispose();
_activeRunner = null;
}
