public void IsComplete_TrueIfComplete()
{
var testRun = new TestRun(null);
var result = new TestEngineResult();
testRun.SetResult(result);
Assert.IsTrue(testRun.IsComplete);
}
public void SetUp()
{
result1 = new TestEngineResult(resultText1);
result2 = new TestEngineResult(resultText2);
twoResults = new TestEngineResult[] { result1, result2 };
twoNodes = new XmlNode[] { result1.Xml, result2.Xml };
}
public void SetResult_ThrowsIfSetTwice()
{
var testRun = new TestRun(null);
var result = new TestEngineResult();
testRun.SetResult(result);
Assert.Throws<InvalidOperationException>(() => testRun.SetResult(result));
}
/// <summary>
/// Load a TestPackage for exploration or execution
/// </summary>
/// <returns>A TestEngineResult.</returns>
protected override TestEngineResult LoadPackage()
{
var result = new TestEngineResult();
// DirectRunner may be called with a single-assembly package
// or a set of assemblies as subpackages.
var packages = TestPackage.SubPackages;
if (packages.Count == 0)
packages.Add(TestPackage);
var driverService = Services.GetService<IDriverService>();
foreach (var subPackage in packages)
{
var testFile = subPackage.FullName;
IFrameworkDriver driver = driverService.GetDriver(TestDomain, testFile);
driver.ID = TestPackage.ID;
result.Add(driver.Load(testFile, subPackage.Settings));
_drivers.Add(driver);
}
if (IsProjectPackage(TestPackage))
result = result.MakePackageResult(TestPackage.Name, TestPackage.FullName);
return result;
}
/// <summary>
/// Merges multiple test engine results into a single result. The
/// result element contains all the XML nodes found in the input.
/// </summary>
/// <param name="results">A list of TestEngineResults</param>
/// <returns>A TestEngineResult merging all the imput results</returns>
/// <remarks>Used by AbstractTestRunner MakePackageResult method.</remarks>
public static TestEngineResult Merge(IList<TestEngineResult> results)
{
TestEngineResult mergedResult = new TestEngineResult();
foreach (TestEngineResult result in results)
foreach (XmlNode node in result.XmlNodes)
mergedResult.Add(node);
return mergedResult;
}
/// <summary>
/// Explore a TestPackage and return information about
/// the tests found.
/// </summary>
/// <param name="package">The TestPackage to be explored</param>
/// <returns>A TestEngineResult.</returns>
protected override TestEngineResult ExploreTests(TestFilter filter)
{
TestEngineResult result = new TestEngineResult();
foreach (IFrameworkDriver driver in _drivers)
result.Add(driver.Explore(filter));
return IsProjectPackage(this.TestPackage)
? result.MakePackageResult(TestPackage.Name, TestPackage.FullName)
: result;
}
/// <summary>
/// Explores a previously loaded TestPackage and returns information
/// about the tests found.
/// </summary>
/// <param name="filter">The TestFilter to be used to select tests</param>
/// <returns>
/// A TestEngineResult.
/// </returns>
protected override TestEngineResult ExploreTests(TestFilter filter)
{
var result = new TestEngineResult();
foreach (IFrameworkDriver driver in _drivers)
result.Add(driver.Explore(filter.Text));
if (IsProjectPackage(TestPackage))
result = result.MakePackageResult(TestPackage.Name, TestPackage.FullName);
return result;
}
public void CanCreateFromXmlString()
{
TestEngineResult result = new TestEngineResult(xmlText);
Assert.True(result.IsSingle);
Assert.That(result.Xml.Name, Is.EqualTo("test-assembly"));
Assert.That(result.Xml.Attributes["result"].Value, Is.EqualTo("Passed"));
Assert.That(result.Xml.Attributes["total"].Value, Is.EqualTo("23"));
Assert.That(result.Xml.Attributes["passed"].Value, Is.EqualTo("23"));
Assert.That(result.Xml.Attributes["failed"].Value, Is.EqualTo("0"));
Assert.That(result.Xml.Attributes["inconclusive"].Value, Is.EqualTo("0"));
Assert.That(result.Xml.Attributes["skipped"].Value, Is.EqualTo("0"));
Assert.That(result.Xml.Attributes["asserts"].Value, Is.EqualTo("40"));
}
/// <summary>
/// Merges multiple test engine results into a single result. The
/// result element contains all the XML nodes found in the input.
/// </summary>
/// <param name="results">A list of TestEngineResults</param>
/// <returns>A TestEngineResult merging all the imput results</returns>
/// <remarks>Used by AbstractTestRunner MakePackageResult method.</remarks>
public static TestEngineResult Merge(IList <TestEngineResult> results)
{
var mergedResult = new TestEngineResult();
foreach (TestEngineResult result in results)
{
foreach (XmlNode node in result.XmlNodes)
{
mergedResult.Add(node);
}
}
return(mergedResult);
}
private TestEngineResult PrepareResult(TestEngineResult result)
{
if (result == null)
{
throw new ArgumentNullException("result");
}
if (!IsProjectPackage(TestPackage))
{
return(result);
}
return(result.MakePackageResult(TestPackage.Name, TestPackage.FullName));
}
public void CanCreateFromXmlString()
{
TestEngineResult result = new TestEngineResult(xmlText);
Assert.That(result.IsSingle, Is.True);
Assert.That(result.Xml.Name, Is.EqualTo("test-assembly"));
Assert.That(result.Xml.Attributes["result"].Value, Is.EqualTo("Passed"));
Assert.That(result.Xml.Attributes["total"].Value, Is.EqualTo("23"));
Assert.That(result.Xml.Attributes["passed"].Value, Is.EqualTo("23"));
Assert.That(result.Xml.Attributes["failed"].Value, Is.EqualTo("0"));
Assert.That(result.Xml.Attributes["inconclusive"].Value, Is.EqualTo("0"));
Assert.That(result.Xml.Attributes["skipped"].Value, Is.EqualTo("0"));
Assert.That(result.Xml.Attributes["asserts"].Value, Is.EqualTo("40"));
}
public void Wait_ReturnsFalseTillTestCompletes()
{
var result = new TestEngineResult("<test-assembly />");
Assert.IsFalse(_asyncResult.Wait(0),
"Expected wait to be false because test hasn't completed yet");
_asyncResult.SetResult(result);
Assert.IsTrue(_asyncResult.Wait(0),
"Expected wait to be true because the test is complete");
Assert.AreEqual(result.Xml, _asyncResult.EngineResult.Xml);
}
public void Wait_ReturnsFalseTillTestCompletes()
{
var result = new TestEngineResult("<test-assembly />");
Assert.IsFalse(_asyncResult.Wait(0),
"Expected wait to be false because test hasn't completed yet");
_asyncResult.SetResult(result);
Assert.IsTrue(_asyncResult.Wait(0),
"Expected wait to be true because the test is complete");
Assert.AreEqual(result.Xml, _asyncResult.EngineResult.Xml);
}
/// <summary>
/// Run the tests in a loaded TestPackage
/// </summary>
/// <param name="listener">An ITestEventHandler to receive events</param>
/// <param name="filter">A TestFilter used to select tests</param>
/// <returns>
/// A TestEngineResult giving the result of the test execution.
/// </returns>
protected override TestEngineResult RunTests(ITestEventListener listener, TestFilter filter)
{
var results = new List <TestEngineResult>();
bool disposeRunners = TestPackage.GetSetting(EnginePackageSettings.DisposeRunners, false);
if (LevelOfParallelism <= 1)
{
foreach (ITestEngineRunner runner in _runners)
{
results.Add(runner.Run(listener, filter));
if (disposeRunners)
{
runner.Dispose();
}
}
}
else
{
var workerPool = new ParallelTaskWorkerPool(LevelOfParallelism);
var tasks = new List <TestExecutionTask>();
foreach (ITestEngineRunner runner in _runners)
{
var task = new TestExecutionTask(runner, listener, filter, disposeRunners);
tasks.Add(task);
workerPool.Enqueue(task);
}
workerPool.Start();
workerPool.WaitAll();
foreach (var task in tasks)
{
results.Add(task.Result());
}
}
if (disposeRunners)
{
_runners.Clear();
}
TestEngineResult result = ResultHelper.Merge(results);
return(IsProjectPackage(TestPackage)
? result.MakePackageResult(TestPackage.Name, TestPackage.FullName)
: result);
}
public void CanCreateFromXmlNode()
{
XmlNode node = XmlHelper.CreateTopLevelElement("test-assembly");
XmlHelper.AddAttribute(node, "result", "Passed");
XmlHelper.AddAttribute(node, "total", "23");
XmlHelper.AddAttribute(node, "passed", "23");
XmlHelper.AddAttribute(node, "failed", "0");
XmlHelper.AddAttribute(node, "inconclusive", "0");
XmlHelper.AddAttribute(node, "skipped", "0");
XmlHelper.AddAttribute(node, "asserts", "40");
TestEngineResult result = new TestEngineResult(node);
Assert.True(result.IsSingle);
Assert.That(result.Xml.OuterXml, Is.EqualTo(xmlText));
}
public void Wait_ReturnsFalseTillTestCompletes()
{
var testRun = new TestRun(null);
var result = new TestEngineResult("<test-assembly />");
Assert.IsFalse(testRun.Wait(TimeSpan.FromMilliseconds(0)),
"Expected wait to be false because test hasn't completed yet");
testRun.SetResult(result);
Assert.IsTrue(testRun.Wait(TimeSpan.FromMilliseconds(0)),
"Expected wait to be true because the test is complete");
Assert.AreEqual(result.Xml, testRun.Result);
}
/// <summary>
/// Load a TestPackage for exploration or execution
/// </summary>
/// <param name="package">The TestPackage to be loaded</param>
/// <returns>A TestEngineResult.</returns>
protected override TestEngineResult LoadPackage()
{
TestEngineResult result = new TestEngineResult();
foreach (string testFile in TestPackage.TestFiles)
{
IFrameworkDriver driver = Services.DriverFactory.GetDriver(TestDomain, testFile, TestPackage.Settings);
result.Add(driver.Load());
_drivers.Add(driver);
}
return IsProjectPackage(TestPackage)
? result.MakePackageResult(TestPackage.Name, TestPackage.FullName)
: result;
}
public void Wait_ReturnsFalseTillTestCompletes()
{
var testRun = new TestRun(null);
var result = new TestEngineResult("<test-assembly />");
Assert.IsFalse(testRun.Wait(TimeSpan.FromMilliseconds(0)),
"Expected wait to be false because test hasn't completed yet");
testRun.SetResult(result);
Assert.IsTrue(testRun.Wait(TimeSpan.FromMilliseconds(0)),
"Expected wait to be true because the test is complete");
Assert.AreEqual(result.Xml, testRun.Result);
}
/// <summary>
/// Explore a TestPackage and return information about
/// the tests found.
/// </summary>
/// <param name="package">The TestPackage to be explored</param>
/// <returns>A TestEngineResult.</returns>
protected override TestEngineResult ExploreTests(TestFilter filter)
{
List <TestEngineResult> results = new List <TestEngineResult>();
foreach (ITestEngineRunner runner in _runners)
{
results.Add(runner.Explore(filter));
}
TestEngineResult result = ResultHelper.Merge(results);
return(IsProjectPackage(this.TestPackage)
? result.MakePackageResult(TestPackage.Name, TestPackage.FullName)
: result);
}
/// <summary>
/// Load a TestPackage for exploration or execution
/// </summary>
/// <param name="package">The TestPackage to be loaded</param>
/// <returns>A TestEngineResult.</returns>
protected override TestEngineResult LoadPackage()
{
TestEngineResult result = new TestEngineResult();
foreach (string testFile in TestPackage.TestFiles)
{
IFrameworkDriver driver = Services.DriverFactory.GetDriver(TestDomain, testFile, TestPackage.Settings);
result.Add(driver.Load());
_drivers.Add(driver);
}
return(IsProjectPackage(TestPackage)
? result.MakePackageResult(TestPackage.Name, TestPackage.FullName)
: result);
}
/// <summary>
/// Load a TestPackage for exploration or execution
/// </summary>
/// <param name="package">The TestPackage to be loaded</param>
/// <returns>A TestEngineResult.</returns>
public override TestEngineResult Load(TestPackage package)
{
this.package = package;
TestEngineResult result = new TestEngineResult();
foreach (string testFile in package.TestFiles)
{
IFrameworkDriver driver = Services.DriverFactory.GetDriver(TestDomain, testFile, package.Settings);
result.Add(driver.Load());
drivers.Add(driver);
}
return(IsProjectPackage(this.package)
? result.MakePackageResult(package.Name, package.FullName)
: result);
}
/// <summary>
/// Explores a previously loaded TestPackage and returns information
/// about the tests found.
/// </summary>
/// <param name="filter">The TestFilter to be used to select tests</param>
/// <returns>
/// A TestEngineResult.
/// </returns>
protected override TestEngineResult ExploreTests(TestFilter filter)
{
var result = new TestEngineResult();
foreach (IFrameworkDriver driver in _drivers)
{
result.Add(driver.Explore(filter.Text));
}
if (IsProjectPackage(TestPackage))
{
result = result.MakePackageResult(TestPackage.Name, TestPackage.FullName);
}
return(result);
}
/// <summary>
/// Explore a TestPackage and return information about
/// the tests found.
/// </summary>
/// <param name="package">The TestPackage to be explored</param>
/// <returns>A TestEngineResult.</returns>
public override TestEngineResult Explore(TestFilter filter)
{
List <TestEngineResult> results = new List <TestEngineResult>();
// TODO: Eliminate need for implicit cast to AbstractTestRunner
foreach (AbstractTestRunner runner in runners)
{
results.Add(runner.Explore(filter));
}
TestEngineResult result = ResultHelper.Merge(results);
return(IsProjectPackage(this.package)
? result.MakePackageResult(package.Name, package.FullName)
: result);
}
/// <summary>
/// Run the tests in the loaded TestPackage and return a test result. The tests
/// are run synchronously and the listener interface is notified as it progresses.
/// </summary>
/// <param name="listener">An ITestEventHandler to receive events</param>
/// <param name="filter">A TestFilter used to select tests</param>
/// <returns>A TestEngineResult giving the result of the test execution</returns>
private TestEngineResult RunTests(ITestEventListener listener, TestFilter filter)
{
var eventDispatcher = new TestEventDispatcher();
if (listener != null)
{
eventDispatcher.Listeners.Add(listener);
}
#if !NETSTANDARD1_6
foreach (var extension in _extensionService.GetExtensions <ITestEventListener>())
{
eventDispatcher.Listeners.Add(extension);
}
#endif
IsTestRunning = true;
eventDispatcher.OnTestEvent(string.Format("<start-run count='{0}'/>", CountTests(filter)));
DateTime startTime = DateTime.UtcNow;
long startTicks = Stopwatch.GetTimestamp();
TestEngineResult result = _engineRunner.Run(eventDispatcher, filter).Aggregate("test-run", TestPackage.Name, TestPackage.FullName);
// These are inserted in reverse order, since each is added as the first child.
InsertFilterElement(result.Xml, filter);
#if !NETSTANDARD1_6
InsertCommandLineElement(result.Xml);
result.Xml.AddAttribute("engine-version", Assembly.GetExecutingAssembly().GetName().Version.ToString());
result.Xml.AddAttribute("clr-version", Environment.Version.ToString());
#else
result.Xml.AddAttribute("engine-version", typeof(MasterTestRunner).GetTypeInfo().Assembly.GetName().Version.ToString());
result.Xml.AddAttribute("clr-version", Microsoft.DotNet.InternalAbstractions.RuntimeEnvironment.GetRuntimeIdentifier());
#endif
double duration = (double)(Stopwatch.GetTimestamp() - startTicks) / Stopwatch.Frequency;
result.Xml.AddAttribute("start-time", XmlConvert.ToString(startTime, "u"));
result.Xml.AddAttribute("end-time", XmlConvert.ToString(DateTime.UtcNow, "u"));
result.Xml.AddAttribute("duration", duration.ToString("0.000000", NumberFormatInfo.InvariantInfo));
IsTestRunning = false;
eventDispatcher.OnTestEvent(result.Xml.OuterXml);
return(result);
}
public void AggregateTestResult()
{
TestEngineResult combined = result2.Aggregate("test-run", "NAME", "FULLNAME");
Assert.That(combined.IsSingle);
XmlNode combinedNode = combined.Xml;
Assert.That(combinedNode.Name, Is.EqualTo("test-run"));
Assert.That(combinedNode.Attributes["result"].Value, Is.EqualTo("Failed"));
Assert.That(combinedNode.Attributes["total"].Value, Is.EqualTo("42"));
Assert.That(combinedNode.Attributes["passed"].Value, Is.EqualTo("31"));
Assert.That(combinedNode.Attributes["failed"].Value, Is.EqualTo("4"));
Assert.That(combinedNode.Attributes["inconclusive"].Value, Is.EqualTo("5"));
Assert.That(combinedNode.Attributes["skipped"].Value, Is.EqualTo("2"));
Assert.That(combinedNode.Attributes["asserts"].Value, Is.EqualTo("53"));
}
public void CanCreateFromXmlNode()
{
XmlNode node = XmlHelper.CreateTopLevelElement("test-assembly");
XmlHelper.AddAttribute(node, "result", "Passed");
XmlHelper.AddAttribute(node, "total", "23");
XmlHelper.AddAttribute(node, "passed", "23");
XmlHelper.AddAttribute(node, "failed", "0");
XmlHelper.AddAttribute(node, "inconclusive", "0");
XmlHelper.AddAttribute(node, "skipped", "0");
XmlHelper.AddAttribute(node, "asserts", "40");
TestEngineResult result = new TestEngineResult(node);
Assert.That(result.IsSingle, Is.True);
Assert.That(result.Xml.OuterXml, Is.EqualTo(xmlText));
}
/// <summary>
/// Run the tests in the loaded TestPackage and return a test result. The tests
/// are run synchronously and the listener interface is notified as it progresses.
/// </summary>
/// <param name="listener">An ITestEventHandler to receive events</param>
/// <param name="filter">A TestFilter used to select tests</param>
/// <returns>A TestEngineResult giving the result of the test execution</returns>
public override TestEngineResult Run(ITestEventHandler listener, TestFilter filter)
{
DateTime startTime = DateTime.UtcNow;
long startTicks = Stopwatch.GetTimestamp();
TestEngineResult result = realRunner.Run(listener, filter).Aggregate("test-run", package.Name, package.FullName);
result.Xml.InsertEnvironmentElement();
double duration = (double)(Stopwatch.GetTimestamp() - startTicks) / Stopwatch.Frequency;
result.Xml.AddAttribute("start-time", XmlConvert.ToString(startTime, "u"));
result.Xml.AddAttribute("end-time", XmlConvert.ToString(DateTime.UtcNow, "u"));
result.Xml.AddAttribute("duration", duration.ToString("0.000000", NumberFormatInfo.InvariantInfo));
return(result);
}
/// <summary>
/// Run the tests in the loaded TestPackage.
/// </summary>
/// <param name="listener">An ITestEventHandler to receive events</param>
/// <param name="filter">A TestFilter used to select tests</param>
/// <returns>
/// A TestEngineResult giving the result of the test execution
/// </returns>
protected override TestEngineResult RunTests(ITestEventListener listener, TestFilter filter)
{
var result = new TestEngineResult();
foreach (IFrameworkDriver driver in _drivers)
{
result.Add(driver.Run(listener, filter.Text));
}
#if NUNIT_ENGINE
if (IsProjectPackage(TestPackage))
{
result = result.MakePackageResult(TestPackage.Name, TestPackage.FullName);
}
#endif
return(result);
}
/// <summary>
/// Load a TestPackage for exploration or execution
/// </summary>
/// <returns>A TestEngineResult.</returns>
protected override TestEngineResult LoadPackage()
{
var result = new TestEngineResult();
// DirectRunner may be called with a single-assembly package
// or a set of assemblies as subpackages.
var packages = TestPackage.SubPackages;
if (packages.Count == 0)
{
packages.Add(TestPackage);
}
var driverService = Services.GetService <IDriverService>();
foreach (var subPackage in packages)
{
var testFile = subPackage.FullName;
if (_assemblyResolver != null && !TestDomain.IsDefaultAppDomain() &&
subPackage.GetSetting(PackageSettings.ImageRequiresDefaultAppDomainAssemblyResolver, false))
{
// It's OK to do this in the loop because the Add method
// checks to see if the path is already present.
_assemblyResolver.AddPathFromFile(testFile);
}
var targetFramework = subPackage.GetSetting(PackageSettings.ImageTargetFrameworkName, (string)null);
IFrameworkDriver driver = driverService.GetDriver(TestDomain, testFile, targetFramework);
driver.ID = TestPackage.ID;
result.Add(driver.Load(testFile, subPackage.Settings));
_drivers.Add(driver);
}
if (IsProjectPackage(TestPackage))
{
result = result.MakePackageResult(TestPackage.Name, TestPackage.FullName);
}
return(result);
}
public void MergeAndAggregateTestResults()
{
TestEngineResult combined = ResultHelper.Merge(twoResults).Aggregate("test-suite", "Project", "NAME", "FULLNAME");
Assert.That(combined.IsSingle);
XmlNode combinedNode = combined.Xml;
Assert.That(combinedNode.Name, Is.EqualTo("test-suite"));
Assert.That(combinedNode.Attributes["type"].Value, Is.EqualTo("Project"));
Assert.That(combinedNode.Attributes["name"].Value, Is.EqualTo("NAME"));
Assert.That(combinedNode.Attributes["fullname"].Value, Is.EqualTo("FULLNAME"));
Assert.That(combinedNode.Attributes["result"].Value, Is.EqualTo("Failed"));
Assert.That(combinedNode.Attributes["total"].Value, Is.EqualTo("65"));
Assert.That(combinedNode.Attributes["passed"].Value, Is.EqualTo("54"));
Assert.That(combinedNode.Attributes["failed"].Value, Is.EqualTo("4"));
Assert.That(combinedNode.Attributes["inconclusive"].Value, Is.EqualTo("5"));
Assert.That(combinedNode.Attributes["skipped"].Value, Is.EqualTo("2"));
Assert.That(combinedNode.Attributes["asserts"].Value, Is.EqualTo("93"));
}
public void Execute()
{
_hasExecuted = true;
try
{
_result = _runner.Run(_listener, _filter);
}
finally
{
try
{
if (_disposeRunner)
_runner.Dispose();
}
catch (Exception e)
{
_unloadException = e;
}
}
}
/// <summary>
/// Load a TestPackage for exploration or execution
/// </summary>
/// <returns>A TestEngineResult.</returns>
protected override TestEngineResult LoadPackage()
{
var result = new TestEngineResult();
// DirectRunner may be called with a single-assembly package
// or a set of assemblies as subpackages.
var packages = TestPackage.SubPackages;
if (packages.Count == 0)
{
packages.Add(TestPackage);
}
var driverService = Services.GetService <IDriverService>();
foreach (var subPackage in packages)
{
var testFile = subPackage.FullName;
string targetFramework = subPackage.GetSetting(InternalEnginePackageSettings.ImageTargetFrameworkName, (string)null);
bool skipNonTestAssemblies = subPackage.GetSetting(EnginePackageSettings.SkipNonTestAssemblies, false);
#if !NETSTANDARD1_6
if (_assemblyResolver != null && !TestDomain.IsDefaultAppDomain() &&
subPackage.GetSetting(InternalEnginePackageSettings.ImageRequiresDefaultAppDomainAssemblyResolver, false))
{
// It's OK to do this in the loop because the Add method
// checks to see if the path is already present.
_assemblyResolver.AddPathFromFile(testFile);
}
IFrameworkDriver driver = driverService.GetDriver(TestDomain, testFile, targetFramework, skipNonTestAssemblies);
#else
IFrameworkDriver driver = driverService.GetDriver(testFile, skipNonTestAssemblies);
#endif
driver.ID = TestPackage.ID;
result.Add(LoadDriver(driver, testFile, subPackage));
_drivers.Add(driver);
}
return(result);
}
/// <summary>
/// Load a TestPackage for exploration or execution
/// </summary>
/// <param name="package">The TestPackage to be loaded</param>
/// <returns>A TestEngineResult.</returns>
public override TestEngineResult Load(TestPackage package)
{
this.package = package;
List <TestEngineResult> loadResults = new List <TestEngineResult>();
foreach (string testFile in package.TestFiles)
{
// TODO: Should get the appropriate driver for the file
IFrameworkDriver driver = Services.DriverFactory.GetDriver(TestDomain, testFile, package.Settings);
TestEngineResult driverResult = driver.Load();
loadResults.Add(driverResult);
if (!driverResult.HasErrors)
{
drivers.Add(driver);
}
}
return(MakePackageResult(loadResults));
}
/// <summary>
/// Run the tests in the loaded TestPackage.
/// </summary>
/// <param name="listener">An ITestEventHandler to receive events</param>
/// <param name="filter">A TestFilter used to select tests</param>
/// <returns>
/// A TestEngineResult giving the result of the test execution
/// </returns>
protected override TestEngineResult RunTests(ITestEventListener listener, TestFilter filter)
{
EnsurePackageIsLoaded();
var result = new TestEngineResult();
foreach (IFrameworkDriver driver in _drivers)
{
string driverResult;
try
{
driverResult = driver.Run(listener, filter.Text);
}
catch (Exception ex) when(!(ex is NUnitEngineException))
{
throw new NUnitEngineException("An exception occurred in the driver while running tests.", ex);
}
result.Add(driverResult);
}
#if !NETSTANDARD1_6
if (_assemblyResolver != null)
{
var packages = TestPackage.SubPackages;
if (packages.Count == 0)
{
packages.Add(TestPackage);
}
foreach (var package in packages)
{
_assemblyResolver.RemovePathFromFile(package.FullName);
}
}
#endif
return(result);
}
/// <summary>
/// Loads the assembly that contains tests and run the tests.
/// </summary>
/// <param name="testAssembly">
/// Assembly that will be containing tests
/// </param>
public void LoadTestAndRunningAssembly(Assembly testAssembly)
{
Uri uri = new Uri(testAssembly.CodeBase);
this.localDirectory = Path.GetDirectoryName(uri.LocalPath);
this.engine = TestEngineActivator.CreateInstance(null, InternalTraceLevel.Off);
var settings = new Dictionary <string, object>();
var runner = new DefaultTestAssemblyRunner(new DefaultTestAssemblyBuilder());
Assert.True(runner.Load(testAssembly, settings), "Unable to load Executing Assembly.");
// Convert our own framework XmlNode to a TestEngineResult
var package = new TestPackage(GetLocalPath(uri.AbsolutePath));
this.engineResult = TestEngineResult.MakeTestRunResult(
package,
DateTime.Now,
new TestEngineResult(
runner.Run(TestListener.NULL, NUnit.Framework.Internal.TestFilter.Empty).ToXml(true).OuterXml));
}
/// <summary>
/// Load a TestPackage for exploration or execution
/// </summary>
/// <returns>A TestEngineResult.</returns>
protected override TestEngineResult LoadPackage()
{
var result = new TestEngineResult();
// DirectRunner may be called with a single-assembly package,
// a set of assemblies as subpackages or even an arbitrary
// hierarchy of packages and subpackages with assemblies
// found in the terminal nodes.
var packagesToLoad = TestPackage.Select(p => !p.HasSubPackages());
var driverService = Services.GetService <IDriverService>();
_drivers.Clear();
foreach (var subPackage in packagesToLoad)
{
var testFile = subPackage.FullName;
string targetFramework = subPackage.GetSetting(InternalEnginePackageSettings.ImageTargetFrameworkName, (string)null);
bool skipNonTestAssemblies = subPackage.GetSetting(EnginePackageSettings.SkipNonTestAssemblies, false);
if (_assemblyResolver != null && !TestDomain.IsDefaultAppDomain() &&
subPackage.GetSetting(InternalEnginePackageSettings.ImageRequiresDefaultAppDomainAssemblyResolver, false))
{
// It's OK to do this in the loop because the Add method
// checks to see if the path is already present.
_assemblyResolver.AddPathFromFile(testFile);
}
IFrameworkDriver driver = driverService.GetDriver(TestDomain, testFile, targetFramework, skipNonTestAssemblies);
driver.ID = subPackage.ID;
result.Add(LoadDriver(driver, testFile, subPackage));
_drivers.Add(driver);
}
return(result);
}
/// <summary>
/// Load a TestPackage for exploration or execution
/// </summary>
/// <returns>A TestEngineResult.</returns>
protected override TestEngineResult LoadPackage()
{
var result = new TestEngineResult();
// DirectRunner may be called with a single-assembly package
// or a set of assemblies as subpackages.
var packages = TestPackage.SubPackages;
if (packages.Count == 0)
packages.Add(TestPackage);
var driverService = Services.GetService<IDriverService>();
foreach (var subPackage in packages)
{
var testFile = subPackage.FullName;
if (_assemblyResolver != null && !TestDomain.IsDefaultAppDomain()
&& subPackage.GetSetting(InternalEnginePackageSettings.ImageRequiresDefaultAppDomainAssemblyResolver, false))
{
// It's OK to do this in the loop because the Add method
// checks to see if the path is already present.
_assemblyResolver.AddPathFromFile(testFile);
}
var targetFramework = subPackage.GetSetting(InternalEnginePackageSettings.ImageTargetFrameworkName, (string)null);
IFrameworkDriver driver = driverService.GetDriver(TestDomain, testFile, targetFramework);
driver.ID = TestPackage.ID;
result.Add(driver.Load(testFile, subPackage.Settings));
_drivers.Add(driver);
}
if (IsProjectPackage(TestPackage))
result = result.MakePackageResult(TestPackage.Name, TestPackage.FullName);
return result;
}
/// <summary>
/// Explores a previously loaded TestPackage and returns information
/// about the tests found.
/// </summary>
/// <param name="filter">The TestFilter to be used to select tests</param>
/// <returns>
/// A TestEngineResult.
/// </returns>
public override TestEngineResult Explore(TestFilter filter)
{
EnsurePackageIsLoaded();
var result = new TestEngineResult();
foreach (IFrameworkDriver driver in _drivers)
{
string driverResult;
try
{
driverResult = driver.Explore(filter.Text);
}
catch (Exception ex) when(!(ex is NUnitEngineException))
{
throw new NUnitEngineException("An exception occurred in the driver while exploring tests.", ex);
}
result.Add(driverResult);
}
return(result);
}
/// <summary>
/// Runs the test in given assembly against the provided source assembly.
/// </summary>
/// <param name="request">
/// The request.
/// </param>
/// <returns>
/// The <see cref="TestResult"/>.
/// </returns>
public TestResult RunTests(TestRunRequest request)
{
Uri uri = new Uri(request.TestAssembly.CodeBase);
this.localDirectory = Path.GetDirectoryName(uri.LocalPath);
this.engine = TestEngineActivator.CreateInstance(null, InternalTraceLevel.Off);
var settings = new Dictionary <string, object>();
var runner = new DefaultTestAssemblyRunner(new DefaultTestAssemblyBuilder());
runner.Load(request.TestAssembly, settings);
// Convert our own framework XmlNode to a TestEngineResult
var package = new TestPackage(this.GetLocalPath(uri.AbsolutePath));
this.engineResult = TestEngineResult.MakeTestRunResult(
package,
DateTime.Now,
new TestEngineResult(
runner.Run(TestListener.NULL, NUnit.Framework.Internal.TestFilter.Empty).ToXml(true).OuterXml));
return(this.SummaryTransformTest());
}
/// <summary>
/// Run the tests in the loaded TestPackage and return a test result. The tests
/// are run synchronously and the listener interface is notified as it progresses.
/// </summary>
/// <param name="listener">An ITestEventHandler to receive events</param>
/// <param name="filter">A TestFilter used to select tests</param>
/// <returns>A TestEngineResult giving the result of the test execution</returns>
protected override TestEngineResult RunTests(ITestEventListener listener, TestFilter filter)
{
IsTestRunning = true;
if (listener != null)
{
listener.OnTestEvent(string.Format("<start-run count='{0}'/>", CountTestCases(filter)));
}
DateTime startTime = DateTime.UtcNow;
long startTicks = Stopwatch.GetTimestamp();
TestEngineResult result = _realRunner.Run(listener, filter).Aggregate("test-run", TestPackage.Name, TestPackage.FullName);
// These are inserted in reverse order, since each is added as the first child.
InsertFilterElement(result.Xml, filter);
InsertCommandLineElement(result.Xml);
result.Xml.AddAttribute("engine-version", Assembly.GetExecutingAssembly().GetName().Version.ToString());
result.Xml.AddAttribute("clr-version", Environment.Version.ToString());
double duration = (double)(Stopwatch.GetTimestamp() - startTicks) / Stopwatch.Frequency;
result.Xml.AddAttribute("start-time", XmlConvert.ToString(startTime, "u"));
result.Xml.AddAttribute("end-time", XmlConvert.ToString(DateTime.UtcNow, "u"));
result.Xml.AddAttribute("duration", duration.ToString("0.000000", NumberFormatInfo.InvariantInfo));
IsTestRunning = false;
if (listener != null)
{
listener.OnTestEvent(result.Xml.OuterXml);
}
return(result);
}
/// <summary>
/// Run the tests in the loaded TestPackage.
/// </summary>
/// <param name="listener">An ITestEventHandler to receive events</param>
/// <param name="filter">A TestFilter used to select tests</param>
/// <returns>
/// A TestEngineResult giving the result of the test execution
/// </returns>
protected override TestEngineResult RunTests(ITestEventListener listener, TestFilter filter)
{
var result = new TestEngineResult();
foreach (IFrameworkDriver driver in _drivers)
{
result.Add(driver.Run(listener, filter.Text));
}
if (IsProjectPackage(TestPackage))
result = result.MakePackageResult(TestPackage.Name, TestPackage.FullName);
if (_assemblyResolver != null)
{
var packages = TestPackage.SubPackages;
if (packages.Count == 0)
packages.Add(TestPackage);
foreach (var package in packages)
_assemblyResolver.RemovePathFromFile(package.FullName);
}
return result;
}
/// <summary>
/// Run the tests in the loaded TestPackage.
/// </summary>
/// <param name="listener">An ITestEventHandler to receive events</param>
/// <param name="filter">A TestFilter used to select tests</param>
/// <returns>
/// A TestEngineResult giving the result of the test execution
/// </returns>
protected override TestEngineResult RunTests(ITestEventListener listener, TestFilter filter)
{
var result = new TestEngineResult();
foreach (IFrameworkDriver driver in _drivers)
result.Add(driver.Run(listener, filter.Text));
if (IsProjectPackage(TestPackage))
result = result.MakePackageResult(TestPackage.Name, TestPackage.FullName);
return result;
}
public void SetResult_ThrowsIfSetTwice()
{
var result = new TestEngineResult();
_asyncResult.SetResult(result);
Assert.Throws<InvalidOperationException>(() => _asyncResult.SetResult(result));
}
public void IsComplete_TrueIfComplete()
{
var result = new TestEngineResult();
_asyncResult.SetResult(result);
Assert.IsTrue(_asyncResult.IsComplete);
}
/// <summary>
/// Aggregate all the separate assembly results of a project as a single node.
/// </summary>
/// <param name="result">A new TestEngineResult with xml nodes for each assembly or project.</param>
/// <param name="name">The name of the <see cref="TestEngineResult"/>.</param>
/// <param name="fullName">The full name of the <see cref="TestEngineResult"/>.</param>
/// <returns>A TestEngineResult with a single top-level element.</returns>
public static TestEngineResult MakePackageResult(this TestEngineResult result, string name, string fullName)
{
return(Aggregate(result, TEST_SUITE_ELEMENT, PROJECT_SUITE_TYPE, name, fullName));
}
/// <summary>
/// Aggregate the XmlNodes under a TestEngineResult into a single XmlNode.
/// </summary>
/// <param name="result">A new TestEngineResult with xml nodes for each assembly or project.</param>
/// <param name="elementName">The root node name under which to aggregate the results.</param>
/// <param name="name">The name of the <see cref="TestEngineResult"/>.</param>
/// <param name="fullName">The full name of the <see cref="TestEngineResult"/>.</param>
/// <returns>A TestEngineResult with a single top-level element.</returns>
public static TestEngineResult Aggregate(this TestEngineResult result, string elementName, string name, string fullName)
{
return(new TestEngineResult(Aggregate(elementName, name, fullName, result.XmlNodes)));
}
public void InitializeTestEngineResult()
{
// Save the local directory - used by GetLocalPath
Uri uri = new Uri(Assembly.GetExecutingAssembly().CodeBase);
localDirectory = Path.GetDirectoryName(uri.LocalPath);
// Create a fresh copy of the engine, since we can't use the
// one that is running this test.
engine = TestEngineActivator.CreateInstance();
engine.InternalTraceLevel = InternalTraceLevel.Off;
// Create a new DefaultAssemblyRunner, which is actually a framework class,
// because we can't use the one that's currently running this test.
var runner = new Runner(new Builder());
var assemblyPath = GetLocalPath("mock-nunit-assembly.dll");
var settings = new Dictionary<string, object>();
// Make sure the runner loaded the mock assembly.
Assert.That(
runner.Load(assemblyPath, settings).RunState.ToString(),
Is.EqualTo("Runnable"),
"Unable to load mock-assembly.dll");
// Run the tests, saving the result as an XML string
var xmlText = runner.Run(TestListener.NULL, TestFilter.Empty).ToXml(true).OuterXml;
// Create a TestEngineResult from the string, just as the TestEngine does,
// then add a test-run element to the result, wrapping the result so it
// looks just like what the engine would return!
this.EngineResult = new TestEngineResult(xmlText).Aggregate("test-run", "NAME", "FULLNAME");
}
// The TestEngineResult returned to MasterTestRunner contains no info
// about projects. At this point, if there are any projects, the result
// needs to be modified to include info about them. Doing it this way
// allows the lower-level runners to be completely ignorant of projects
private TestEngineResult PrepareResult(TestEngineResult result)
{
if (result == null)
{
throw new ArgumentNullException("result");
}
// See if we have any projects to deal with. At this point,
// any subpackage, which itself has subpackages, is a project
// we expanded.
bool hasProjects = false;
foreach (var p in TestPackage.SubPackages)
{
hasProjects |= p.HasSubPackages();
}
// If no Projects, there's nothing to do
if (!hasProjects)
{
return(result);
}
// If there is just one subpackage, it has to be a project and we don't
// need to rebuild the XML but only wrap it with a project result.
if (TestPackage.SubPackages.Count == 1)
{
return(result.MakeProjectResult(TestPackage.SubPackages[0]));
}
// Most complex case - we need to work with the XML in order to
// examine and rebuild the result to include project nodes.
// NOTE: The algorithm used here relies on the ordering of nodes in the
// result matching the ordering of subpackages under the top-level package.
// If that should change in the future, then we would need to implement
// identification and summarization of projects into each of the lower-
// level TestEngineRunners. In that case, we will be warned by failures
// of some of the MasterTestRunnerTests.
// Start a fresh TestEngineResult for top level
var topLevelResult = new TestEngineResult();
int nextTest = 0;
foreach (var subPackage in TestPackage.SubPackages)
{
if (subPackage.HasSubPackages())
{
// This is a project, create an intermediate result
var projectResult = new TestEngineResult();
// Now move any children of this project under it. As noted
// above, we must rely on ordering here because (1) the
// fullname attribute is not reliable on all nunit framework
// versions, (2) we may have duplicates of the same assembly
// and (3) we have no info about the id of each assembly.
int numChildren = subPackage.SubPackages.Count;
while (numChildren-- > 0)
{
projectResult.Add(result.XmlNodes[nextTest++]);
}
topLevelResult.Add(projectResult.MakeProjectResult(subPackage).Xml);
}
else
{
// Add the next assembly package to our new result
topLevelResult.Add(result.XmlNodes[nextTest++]);
}
}
return(topLevelResult);
}
public void Execute()
{
_result = _runner.Run(_listener, _filter);
if (_disposeRunner)
_runner.Dispose();
}