// The TestEngineResult returned to MasterTestRunner contains no info
// about projects. At this point, if there are any projects, the result
// needs to be modified to include info about them. Doing it this way
// allows the lower-level runners to be completely ignorant of projects
private TestEngineResult PrepareResult(TestEngineResult result)
{
if (result == null) throw new ArgumentNullException("result");
// See if we have any projects to deal with. At this point,
// any subpackage, which itself has subpackages, is a project
// we expanded.
bool hasProjects = false;
foreach (var p in TestPackage.SubPackages)
hasProjects |= p.HasSubPackages();
// If no Projects, there's nothing to do
if (!hasProjects)
return result;
// If there is just one subpackage, it has to be a project and we don't
// need to rebuild the XML but only wrap it with a project result.
if (TestPackage.SubPackages.Count == 1)
return result.MakeProjectResult(TestPackage.SubPackages[0]);
// Most complex case - we need to work with the XML in order to
// examine and rebuild the result to include project nodes.
// NOTE: The algorithm used here relies on the ordering of nodes in the
// result matching the ordering of subpackages under the top-level package.
// If that should change in the future, then we would need to implement
// identification and summarization of projects into each of the lower-
// level TestEngineRunners. In that case, we will be warned by failures
// of some of the MasterTestRunnerTests.
// Start a fresh TestEngineResult for top level
var topLevelResult = new TestEngineResult();
int nextTest = 0;
foreach (var subPackage in TestPackage.SubPackages)
{
if (subPackage.HasSubPackages())
{
// This is a project, create an intermediate result
var projectResult = new TestEngineResult();
// Now move any children of this project under it. As noted
// above, we must rely on ordering here because (1) the
// fullname attribute is not reliable on all nunit framework
// versions, (2) we may have duplicates of the same assembly
// and (3) we have no info about the id of each assembly.
int numChildren = subPackage.SubPackages.Count;
while (numChildren-- > 0)
projectResult.Add(result.XmlNodes[nextTest++]);
topLevelResult.Add(projectResult.MakeProjectResult(subPackage).Xml);
}
else
{
// Add the next assembly package to our new result
topLevelResult.Add(result.XmlNodes[nextTest++]);
}
}
return topLevelResult;
}
/// <summary>
/// Run the tests in the loaded TestPackage.
/// </summary>
/// <param name="listener">An ITestEventHandler to receive events</param>
/// <param name="filter">A TestFilter used to select tests</param>
/// <returns>
/// A TestEngineResult giving the result of the test execution
/// </returns>
protected override TestEngineResult RunTests(ITestEventListener listener, TestFilter filter)
{
var result = new TestEngineResult();
foreach (IFrameworkDriver driver in _drivers)
{
result.Add(driver.Run(listener, filter.Text));
}
if (IsProjectPackage(TestPackage))
{
result = result.MakePackageResult(TestPackage.Name, TestPackage.FullName);
}
var packages = TestPackage.SubPackages;
if (packages.Count == 0)
{
packages.Add(TestPackage);
}
foreach (var package in packages)
{
_assemblyResolver.RemovePathFromFile(package.FullName);
}
return(result);
}
/// <summary>
/// Load a TestPackage for exploration or execution
/// </summary>
/// <returns>A TestEngineResult.</returns>
protected override TestEngineResult LoadPackage()
{
var result = new TestEngineResult();
// DirectRunner may be called with a single-assembly package
// or a set of assemblies as subpackages.
var packages = TestPackage.SubPackages;
if (packages.Count == 0)
{
packages.Add(TestPackage);
}
var driverService = new DriverService();
foreach (var subPackage in packages)
{
var testFile = subPackage.FullName;
bool skipNonTestAssemblies = subPackage.GetSetting(EnginePackageSettings.SkipNonTestAssemblies, false);
IFrameworkDriver driver = driverService.GetDriver(testFile, skipNonTestAssemblies);
driver.ID = TestPackage.ID;
result.Add(driver.Load(testFile, subPackage.Settings));
_drivers.Add(driver);
}
return(result);
}
/// <summary>
/// Load a TestPackage for exploration or execution
/// </summary>
/// <returns>A TestEngineResult.</returns>
protected override TestEngineResult LoadPackage()
{
var result = new TestEngineResult();
// DirectRunner may be called with a single-assembly package,
// a set of assemblies as subpackages or even an arbitrary
// hierarchy of packages and subpackages with assemblies
// found in the terminal nodes.
var packagesToLoad = TestPackage.Select(p => !p.HasSubPackages());
var driverService = Services.GetService <IDriverService>();
foreach (var subPackage in packagesToLoad)
{
var testFile = subPackage.FullName;
string targetFramework = subPackage.GetSetting(InternalEnginePackageSettings.ImageTargetFrameworkName, (string)null);
bool skipNonTestAssemblies = subPackage.GetSetting(EnginePackageSettings.SkipNonTestAssemblies, false);
if (_assemblyResolver != null && !TestDomain.IsDefaultAppDomain() &&
subPackage.GetSetting(InternalEnginePackageSettings.ImageRequiresDefaultAppDomainAssemblyResolver, false))
{
// It's OK to do this in the loop because the Add method
// checks to see if the path is already present.
_assemblyResolver.AddPathFromFile(testFile);
}
IFrameworkDriver driver = driverService.GetDriver(TestDomain, testFile, targetFramework, skipNonTestAssemblies);
driver.ID = subPackage.ID;
result.Add(LoadDriver(driver, testFile, subPackage));
_drivers.Add(driver);
}
return(result);
}
/// <summary>
/// Run the tests in the loaded TestPackage.
/// </summary>
/// <param name="listener">An ITestEventHandler to receive events</param>
/// <param name="filter">A TestFilter used to select tests</param>
/// <returns>
/// A TestEngineResult giving the result of the test execution
/// </returns>
protected override TestEngineResult RunTests(ITestEventListener listener, TestFilter filter)
{
EnsurePackageIsLoaded();
var result = new TestEngineResult();
foreach (IFrameworkDriver driver in _drivers)
{
string driverResult;
try
{
driverResult = driver.Run(listener, filter.Text);
}
catch (Exception ex) when(!(ex is NUnitEngineException))
{
throw new NUnitEngineException("An exception occurred in the driver while running tests.", ex);
}
result.Add(driverResult);
}
#if !NETSTANDARD1_6
if (_assemblyResolver != null)
{
foreach (var package in TestPackage.AssemblyPackages())
{
_assemblyResolver.RemovePathFromFile(package.FullName);
}
}
#endif
return(result);
}
/// <summary>
/// Load a TestPackage for exploration or execution
/// </summary>
/// <returns>A TestEngineResult.</returns>
protected override TestEngineResult LoadPackage()
{
var result = new TestEngineResult();
// DirectRunner may be called with a single-assembly package
// or a set of assemblies as subpackages.
var packages = TestPackage.SubPackages;
if (packages.Count == 0)
packages.Add(TestPackage);
var driverService = Services.GetService<IDriverService>();
foreach (var subPackage in packages)
{
var testFile = subPackage.FullName;
IFrameworkDriver driver = driverService.GetDriver(TestDomain, testFile);
driver.ID = TestPackage.ID;
result.Add(driver.Load(testFile, subPackage.Settings));
_drivers.Add(driver);
}
if (IsProjectPackage(TestPackage))
result = result.MakePackageResult(TestPackage.Name, TestPackage.FullName);
return result;
}
/// <summary>
/// Explore a TestPackage and return information about
/// the tests found.
/// </summary>
/// <param name="package">The TestPackage to be explored</param>
/// <returns>A TestEngineResult.</returns>
protected override TestEngineResult ExploreTests(TestFilter filter)
{
TestEngineResult result = new TestEngineResult();
foreach (IFrameworkDriver driver in _drivers)
result.Add(driver.Explore(filter));
return IsProjectPackage(this.TestPackage)
? result.MakePackageResult(TestPackage.Name, TestPackage.FullName)
: result;
}
/// <summary>
/// Run the tests in the loaded TestPackage.
/// </summary>
/// <param name="listener">An ITestEventHandler to receive events</param>
/// <param name="filter">A TestFilter used to select tests</param>
/// <returns>
/// A TestEngineResult giving the result of the test execution
/// </returns>
protected override TestEngineResult RunTests(ITestEventListener listener, TestFilter filter)
{
var result = new TestEngineResult();
foreach (IFrameworkDriver driver in _drivers)
result.Add(driver.Run(listener, filter.Text));
if (IsProjectPackage(TestPackage))
result = result.MakePackageResult(TestPackage.Name, TestPackage.FullName);
return result;
}
/// <summary>
/// Explores a previously loaded TestPackage and returns information
/// about the tests found.
/// </summary>
/// <param name="filter">The TestFilter to be used to select tests</param>
/// <returns>
/// A TestEngineResult.
/// </returns>
protected override TestEngineResult ExploreTests(TestFilter filter)
{
var result = new TestEngineResult();
foreach (IFrameworkDriver driver in _drivers)
result.Add(driver.Explore(filter.Text));
if (IsProjectPackage(TestPackage))
result = result.MakePackageResult(TestPackage.Name, TestPackage.FullName);
return result;
}
/// <summary>
/// Run the tests in a loaded TestPackage
/// </summary>
/// <param name="filter">A TestFilter used to select tests</param>
/// <returns>
/// A TestEngineResult giving the result of the test execution. The
/// top-level node of the result is <direct-runner> and wraps
/// all the <test-assembly> elements returned by the drivers.
/// </returns>
protected override TestEngineResult RunTests(ITestEventListener listener, TestFilter filter)
{
TestEngineResult result = new TestEngineResult();
foreach (IFrameworkDriver driver in _drivers)
{
result.Add(driver.Run(listener, filter));
}
return(IsProjectPackage(this.TestPackage)
? result.MakePackageResult(TestPackage.Name, TestPackage.FullName)
: result);
}
/// <summary>
/// Run the tests in a loaded TestPackage
/// </summary>
/// <param name="filter">A TestFilter used to select tests</param>
/// <returns>
/// A TestEngineResult giving the result of the test execution. The
/// top-level node of the result is <direct-runner> and wraps
/// all the <test-assembly> elements returned by the drivers.
/// </returns>
public override TestEngineResult Run(ITestEventHandler listener, TestFilter filter)
{
TestEngineResult result = new TestEngineResult();
foreach (IFrameworkDriver driver in drivers)
{
result.Add(driver.Run(listener, filter));
}
return(IsProjectPackage(this.package)
? result.MakePackageResult(package.Name, package.FullName)
: result);
}
/// <summary>
/// Explores a previously loaded TestPackage and returns information
/// about the tests found.
/// </summary>
/// <param name="filter">The TestFilter to be used to select tests</param>
/// <returns>
/// A TestEngineResult.
/// </returns>
public override TestEngineResult Explore(TestFilter filter)
{
EnsurePackageIsLoaded();
var result = new TestEngineResult();
foreach (IFrameworkDriver driver in _drivers)
{
result.Add(driver.Explore(filter.Text));
}
return(result);
}
/// <summary>
/// Explore a TestPackage and return information about
/// the tests found.
/// </summary>
/// <param name="package">The TestPackage to be explored</param>
/// <returns>A TestEngineResult.</returns>
protected override TestEngineResult ExploreTests(TestFilter filter)
{
TestEngineResult result = new TestEngineResult();
foreach (IFrameworkDriver driver in _drivers)
{
result.Add(driver.Explore(filter));
}
return(IsProjectPackage(this.TestPackage)
? result.MakePackageResult(TestPackage.Name, TestPackage.FullName)
: result);
}
/// <summary>
/// Run the tests in the loaded TestPackage.
/// </summary>
/// <param name="listener">An ITestEventHandler to receive events</param>
/// <param name="filter">A TestFilter used to select tests</param>
/// <returns>
/// A TestEngineResult giving the result of the test execution
/// </returns>
protected override TestEngineResult RunTests(ITestEventListener listener, TestFilter filter)
{
EnsurePackageIsLoaded();
var result = new TestEngineResult();
foreach (var driver in _drivers)
{
result.Add(driver.Run(listener, filter.Text));
}
return(result);
}
/// <summary>
/// Merges multiple test engine results into a single result. The
/// result element contains all the XML nodes found in the input.
/// </summary>
/// <param name="results">A collection of <see cref="TestEngineResult"/> to merge.</param>
/// <returns>A TestEngineResult merging all the input results.</returns>
/// <remarks>Used by AbstractTestRunner MakePackageResult method.</remarks>
public static TestEngineResult Merge(IList <TestEngineResult> results)
{
var mergedResult = new TestEngineResult();
foreach (TestEngineResult result in results)
{
foreach (XmlNode node in result.XmlNodes)
{
mergedResult.Add(node);
}
}
return(mergedResult);
}
/// <summary>
/// Load a TestPackage for exploration or execution
/// </summary>
/// <param name="package">The TestPackage to be loaded</param>
/// <returns>A TestEngineResult.</returns>
protected override TestEngineResult LoadPackage()
{
TestEngineResult result = new TestEngineResult();
foreach (string testFile in TestPackage.TestFiles)
{
IFrameworkDriver driver = Services.DriverFactory.GetDriver(TestDomain, testFile, TestPackage.Settings);
result.Add(driver.Load());
_drivers.Add(driver);
}
return IsProjectPackage(TestPackage)
? result.MakePackageResult(TestPackage.Name, TestPackage.FullName)
: result;
}
/// <summary>
/// Load a TestPackage for exploration or execution
/// </summary>
/// <param name="package">The TestPackage to be loaded</param>
/// <returns>A TestEngineResult.</returns>
protected override TestEngineResult LoadPackage()
{
TestEngineResult result = new TestEngineResult();
foreach (string testFile in TestPackage.TestFiles)
{
IFrameworkDriver driver = Services.DriverFactory.GetDriver(TestDomain, testFile, TestPackage.Settings);
result.Add(driver.Load());
_drivers.Add(driver);
}
return(IsProjectPackage(TestPackage)
? result.MakePackageResult(TestPackage.Name, TestPackage.FullName)
: result);
}
/// <summary>
/// Load a TestPackage for exploration or execution
/// </summary>
/// <param name="package">The TestPackage to be loaded</param>
/// <returns>A TestEngineResult.</returns>
public override TestEngineResult Load(TestPackage package)
{
this.package = package;
TestEngineResult result = new TestEngineResult();
foreach (string testFile in package.TestFiles)
{
IFrameworkDriver driver = Services.DriverFactory.GetDriver(TestDomain, testFile, package.Settings);
result.Add(driver.Load());
drivers.Add(driver);
}
return(IsProjectPackage(this.package)
? result.MakePackageResult(package.Name, package.FullName)
: result);
}
/// <summary>
/// Run the tests in the loaded TestPackage.
/// </summary>
/// <param name="listener">An ITestEventHandler to receive events</param>
/// <param name="filter">A TestFilter used to select tests</param>
/// <returns>
/// A TestEngineResult giving the result of the test execution
/// </returns>
protected override TestEngineResult RunTests(ITestEventListener listener, TestFilter filter)
{
var result = new TestEngineResult();
foreach (IFrameworkDriver driver in _drivers)
{
result.Add(driver.Run(listener, filter));
}
#if NUNIT_ENGINE
if (IsProjectPackage(TestPackage))
{
result = result.MakePackageResult(TestPackage.Name, TestPackage.FullName);
}
#endif
return(result);
}
/// <summary>
/// Explores a previously loaded TestPackage and returns information
/// about the tests found.
/// </summary>
/// <param name="filter">The TestFilter to be used to select tests</param>
/// <returns>
/// A TestEngineResult.
/// </returns>
protected override TestEngineResult ExploreTests(TestFilter filter)
{
var result = new TestEngineResult();
foreach (IFrameworkDriver driver in _drivers)
{
result.Add(driver.Explore(filter.Text));
}
#if NUNIT_ENGINE
if (IsProjectPackage(TestPackage))
{
result = result.MakePackageResult(TestPackage.Name, TestPackage.FullName);
}
#endif
return(result);
}
/// <summary>
/// Load a TestPackage for exploration or execution
/// </summary>
/// <returns>A TestEngineResult.</returns>
protected override TestEngineResult LoadPackage()
{
var result = new TestEngineResult();
// DirectRunner may be called with a single-assembly package
// or a set of assemblies as subpackages.
var packages = TestPackage.SubPackages;
if (packages.Count == 0)
{
packages.Add(TestPackage);
}
var driverService = Services.GetService <IDriverService>();
foreach (var subPackage in packages)
{
var testFile = subPackage.FullName;
if (_assemblyResolver != null && !TestDomain.IsDefaultAppDomain() &&
subPackage.GetSetting(PackageSettings.ImageRequiresDefaultAppDomainAssemblyResolver, false))
{
// It's OK to do this in the loop because the Add method
// checks to see if the path is already present.
_assemblyResolver.AddPathFromFile(testFile);
}
var targetFramework = subPackage.GetSetting(PackageSettings.ImageTargetFrameworkName, (string)null);
IFrameworkDriver driver = driverService.GetDriver(TestDomain, testFile, targetFramework);
driver.ID = TestPackage.ID;
result.Add(driver.Load(testFile, subPackage.Settings));
_drivers.Add(driver);
}
if (IsProjectPackage(TestPackage))
{
result = result.MakePackageResult(TestPackage.Name, TestPackage.FullName);
}
return(result);
}
/// <summary>
/// Load a TestPackage for exploration or execution
/// </summary>
/// <returns>A TestEngineResult.</returns>
protected override TestEngineResult LoadPackage()
{
var result = new TestEngineResult();
// DirectRunner may be called with a single-assembly package
// or a set of assemblies as subpackages.
var packages = TestPackage.SubPackages;
if (packages.Count == 0)
{
packages.Add(TestPackage);
}
var driverService = Services.GetService <IDriverService>();
foreach (var subPackage in packages)
{
var testFile = subPackage.FullName;
string targetFramework = subPackage.GetSetting(InternalEnginePackageSettings.ImageTargetFrameworkName, (string)null);
bool skipNonTestAssemblies = subPackage.GetSetting(EnginePackageSettings.SkipNonTestAssemblies, false);
#if !NETSTANDARD1_6
if (_assemblyResolver != null && !TestDomain.IsDefaultAppDomain() &&
subPackage.GetSetting(InternalEnginePackageSettings.ImageRequiresDefaultAppDomainAssemblyResolver, false))
{
// It's OK to do this in the loop because the Add method
// checks to see if the path is already present.
_assemblyResolver.AddPathFromFile(testFile);
}
IFrameworkDriver driver = driverService.GetDriver(TestDomain, testFile, targetFramework, skipNonTestAssemblies);
#else
IFrameworkDriver driver = driverService.GetDriver(testFile, skipNonTestAssemblies);
#endif
driver.ID = TestPackage.ID;
result.Add(LoadDriver(driver, testFile, subPackage));
_drivers.Add(driver);
}
return(result);
}
/// <summary>
/// Explores a previously loaded TestPackage and returns information
/// about the tests found.
/// </summary>
/// <param name="filter">The TestFilter to be used to select tests</param>
/// <returns>
/// A TestEngineResult.
/// </returns>
public override TestEngineResult Explore(TestFilter filter)
{
EnsurePackageIsLoaded();
var result = new TestEngineResult();
foreach (IFrameworkDriver driver in _drivers)
{
string driverResult;
try
{
driverResult = driver.Explore(filter.Text);
}
catch (Exception ex) when(!(ex is NUnitEngineException))
{
throw new NUnitEngineException("An exception occurred in the driver while exploring tests.", ex);
}
result.Add(driverResult);
}
return(result);
}
/// <summary>
/// Load a TestPackage for exploration or execution
/// </summary>
/// <returns>A TestEngineResult.</returns>
protected override TestEngineResult LoadPackage()
{
var result = new TestEngineResult();
// DirectRunner may be called with a single-assembly package
// or a set of assemblies as subpackages.
var packages = TestPackage.SubPackages;
if (packages.Count == 0)
packages.Add(TestPackage);
var driverService = Services.GetService<IDriverService>();
foreach (var subPackage in packages)
{
var testFile = subPackage.FullName;
if (_assemblyResolver != null && !TestDomain.IsDefaultAppDomain()
&& subPackage.GetSetting(InternalEnginePackageSettings.ImageRequiresDefaultAppDomainAssemblyResolver, false))
{
// It's OK to do this in the loop because the Add method
// checks to see if the path is already present.
_assemblyResolver.AddPathFromFile(testFile);
}
var targetFramework = subPackage.GetSetting(InternalEnginePackageSettings.ImageTargetFrameworkName, (string)null);
IFrameworkDriver driver = driverService.GetDriver(TestDomain, testFile, targetFramework);
driver.ID = TestPackage.ID;
result.Add(driver.Load(testFile, subPackage.Settings));
_drivers.Add(driver);
}
if (IsProjectPackage(TestPackage))
result = result.MakePackageResult(TestPackage.Name, TestPackage.FullName);
return result;
}
/// <summary>
/// Run the tests in the loaded TestPackage.
/// </summary>
/// <param name="listener">An ITestEventHandler to receive events</param>
/// <param name="filter">A TestFilter used to select tests</param>
/// <returns>
/// A TestEngineResult giving the result of the test execution
/// </returns>
protected override TestEngineResult RunTests(ITestEventListener listener, TestFilter filter)
{
var result = new TestEngineResult();
foreach (IFrameworkDriver driver in _drivers)
{
result.Add(driver.Run(listener, filter.Text));
}
if (IsProjectPackage(TestPackage))
result = result.MakePackageResult(TestPackage.Name, TestPackage.FullName);
if (_assemblyResolver != null)
{
var packages = TestPackage.SubPackages;
if (packages.Count == 0)
packages.Add(TestPackage);
foreach (var package in packages)
_assemblyResolver.RemovePathFromFile(package.FullName);
}
return result;
}
private TestEngineResult AddMetadata(TestEngineResult input)
{
input.Add("<settings><setting name=\"Setting1Name\" value=\"Setting1Value\"></setting><setting name=\"Setting2Name\" value=\"Setting2Value\"></setting></settings>");
return(input.Aggregate("test-run start-time=\"2015-10-19 02:12:28Z\" end-time=\"2015-10-19 02:12:29Z\" duration=\"0.348616\"", string.Empty, string.Empty));
}
/// <summary>
/// Merges multiple test engine results into a single result. The
/// result element contains all the XML nodes found in the input.
/// </summary>
/// <param name="results">A collection of <see cref="TestEngineResult"/> to merge.</param>
/// <returns>A TestEngineResult merging all the input results.</returns>
/// <remarks>Used by AbstractTestRunner MakePackageResult method.</remarks>
public static TestEngineResult Merge(IList<TestEngineResult> results)
{
var mergedResult = new TestEngineResult();
foreach (TestEngineResult result in results)
foreach (XmlNode node in result.XmlNodes)
mergedResult.Add(node);
return mergedResult;
}
