/// <summary>
/// Cancel the ongoing test run. If no test is running, the call is ignored.
/// </summary>
/// <param name="force">If true, cancel any ongoing test threads, otherwise wait for them to complete.</param>
public void StopRun(bool force)
{
_engineRunner.StopRun(force);
// Frameworks should handle StopRun(true) by cancelling all tests and notifying
// us of the completion of any tests that were running. However, this feature
// may be absent in some frameworks or may be broken and we may not pass on the
// notifications needed by some runners. In fact, such a bug is present in the
// NUnit framework through release 3.12 and motivated the following code.
//
// We try to make up for the potential problem here by notifying the listeners
// of the completion of every pending WorkItem, one that started but never
// sent a completion event. Since we have so far only noted this failure wrt
// test suites and fixtures, those are the only ones we currently track.
//
// Note that this code only deals with notifications. If the framework did not
// actually stop the run, that's a different problem, which has to be handled
// at a lower level within the engine.
if (force && !_workItemTracker.WaitForCompletion(WAIT_FOR_CANCEL_TO_COMPLETE))
{
_workItemTracker.IssuePendingNotifications(_eventDispatcher);
// Indicate we are no longer running
IsTestRunning = false;
// Signal completion of the run
_eventDispatcher.OnTestEvent($"<test-run id='{TestPackage.ID}' result='Failed' label='Cancelled' />");
}
}
/// <summary>
/// Cancel the ongoing test run. If no test is running, the call is ignored.
/// </summary>
/// <param name="force">If true, cancel any ongoing test threads, otherwise wait for them to complete.</param>
public void StopRun(bool force)
{
_engineRunner.StopRun(force);
if (force)
{
// Frameworks should handle StopRun(true) by cancelling all tests and notifying
// us of the completion of any tests that were running. However, this feature
// may be absent in some frameworks or may be broken and we may not pass on the
// notifications needed by some runners. In fact, such a bug is present in the
// NUnit framework through release 3.12 and motivated the following code.
//
// We try to make up for the potential problem here by notifying the listeners
// of the completion of every pending WorkItem, that is, one that started but
// never sent a completion event.
if (!_workItemTracker.WaitForCompletion(WAIT_FOR_CANCEL_TO_COMPLETE))
{
_workItemTracker.SendPendingTestCompletionEvents(_eventDispatcher);
// Indicate we are no longer running
IsTestRunning = false;
// Signal completion of the run
_eventDispatcher.OnTestEvent($"<test-run id='{TestPackage.ID}' result='Failed' label='Cancelled' />");
// Since we were not notified of the completion of some items, we can't trust
// that they were actually stopped by the framework. To make sure nothing is
// left running, we unload the tests. By unloading only the lower-level engine
// runner and not the MasterTestRunner itself, we allow the tests to be loaded
_engineRunner.Unload();
}
}
}
/// <summary>
/// Run the tests in the loaded TestPackage and return a test result. The tests
/// are run synchronously and the listener interface is notified as it progresses.
/// </summary>
/// <param name="listener">An ITestEventHandler to receive events</param>
/// <param name="filter">A TestFilter used to select tests</param>
/// <returns>A TestEngineResult giving the result of the test execution</returns>
private TestEngineResult RunTests(ITestEventListener listener, TestFilter filter)
{
var eventDispatcher = new TestEventDispatcher();
if (listener != null)
{
eventDispatcher.Listeners.Add(listener);
}
foreach (var extension in _extensionService.GetExtensions <ITestEventListener>())
{
eventDispatcher.Listeners.Add(extension);
}
IsTestRunning = true;
string clrVersion;
string engineVersion;
clrVersion = Environment.Version.ToString();
engineVersion = Assembly.GetExecutingAssembly().GetName().Version.ToString();
var startTime = DateTime.UtcNow;
var startRunNode = XmlHelper.CreateTopLevelElement("start-run");
startRunNode.AddAttribute("count", CountTests(filter).ToString());
startRunNode.AddAttribute("start-time", XmlConvert.ToString(startTime, "u"));
startRunNode.AddAttribute("engine-version", engineVersion);
startRunNode.AddAttribute("clr-version", clrVersion);
InsertCommandLineElement(startRunNode);
eventDispatcher.OnTestEvent(startRunNode.OuterXml);
long startTicks = Stopwatch.GetTimestamp();
TestEngineResult result = PrepareResult(GetEngineRunner().Run(eventDispatcher, filter)).MakeTestRunResult(TestPackage);
// These are inserted in reverse order, since each is added as the first child.
InsertFilterElement(result.Xml, filter);
InsertCommandLineElement(result.Xml);
result.Xml.AddAttribute("engine-version", engineVersion);
result.Xml.AddAttribute("clr-version", clrVersion);
double duration = (double)(Stopwatch.GetTimestamp() - startTicks) / Stopwatch.Frequency;
result.Xml.AddAttribute("start-time", XmlConvert.ToString(startTime, "u"));
result.Xml.AddAttribute("end-time", XmlConvert.ToString(DateTime.UtcNow, "u"));
result.Xml.AddAttribute("duration", duration.ToString("0.000000", NumberFormatInfo.InvariantInfo));
IsTestRunning = false;
eventDispatcher.OnTestEvent(result.Xml.OuterXml);
return(result);
}
/// <summary>
/// Run the tests in the loaded TestPackage and return a test result. The tests
/// are run synchronously and the listener interface is notified as it progresses.
/// </summary>
/// <param name="listener">An ITestEventHandler to receive events</param>
/// <param name="filter">A TestFilter used to select tests</param>
/// <returns>A TestEngineResult giving the result of the test execution</returns>
private TestEngineResult RunTests(ITestEventListener listener, TestFilter filter)
{
var eventDispatcher = new TestEventDispatcher();
if (listener != null)
{
eventDispatcher.Listeners.Add(listener);
}
#if !NETSTANDARD1_6
foreach (var extension in _extensionService.GetExtensions <ITestEventListener>())
{
eventDispatcher.Listeners.Add(extension);
}
#endif
IsTestRunning = true;
eventDispatcher.OnTestEvent(string.Format("<start-run count='{0}'/>", CountTests(filter)));
DateTime startTime = DateTime.UtcNow;
long startTicks = Stopwatch.GetTimestamp();
TestEngineResult result = _engineRunner.Run(eventDispatcher, filter).Aggregate("test-run", TestPackage.Name, TestPackage.FullName);
// These are inserted in reverse order, since each is added as the first child.
InsertFilterElement(result.Xml, filter);
#if !NETSTANDARD1_6
InsertCommandLineElement(result.Xml);
result.Xml.AddAttribute("engine-version", Assembly.GetExecutingAssembly().GetName().Version.ToString());
result.Xml.AddAttribute("clr-version", Environment.Version.ToString());
#else
result.Xml.AddAttribute("engine-version", typeof(MasterTestRunner).GetTypeInfo().Assembly.GetName().Version.ToString());
result.Xml.AddAttribute("clr-version", Microsoft.DotNet.InternalAbstractions.RuntimeEnvironment.GetRuntimeIdentifier());
#endif
double duration = (double)(Stopwatch.GetTimestamp() - startTicks) / Stopwatch.Frequency;
result.Xml.AddAttribute("start-time", XmlConvert.ToString(startTime, "u"));
result.Xml.AddAttribute("end-time", XmlConvert.ToString(DateTime.UtcNow, "u"));
result.Xml.AddAttribute("duration", duration.ToString("0.000000", NumberFormatInfo.InvariantInfo));
IsTestRunning = false;
eventDispatcher.OnTestEvent(result.Xml.OuterXml);
return(result);
}
/// <summary>
/// Run the tests in the loaded TestPackage and return a test result. The tests
/// are run synchronously and the listener interface is notified as it progresses.
/// </summary>
/// <param name="listener">An ITestEventHandler to receive events</param>
/// <param name="filter">A TestFilter used to select tests</param>
/// <returns>A TestEngineResult giving the result of the test execution</returns>
private TestEngineResult RunTests(ITestEventListener listener, TestFilter filter)
{
var eventDispatcher = new TestEventDispatcher();
if (listener != null)
eventDispatcher.Listeners.Add(listener);
#if !NETSTANDARD1_6
foreach (var extension in _extensionService.GetExtensions<ITestEventListener>())
eventDispatcher.Listeners.Add(extension);
#endif
IsTestRunning = true;
string clrVersion;
string engineVersion;
#if !NETSTANDARD1_6
clrVersion = Environment.Version.ToString();
engineVersion = Assembly.GetExecutingAssembly().GetName().Version.ToString();
#else
clrVersion = Microsoft.DotNet.InternalAbstractions.RuntimeEnvironment.GetRuntimeIdentifier();
engineVersion = typeof(MasterTestRunner).GetTypeInfo().Assembly.GetName().Version.ToString();
#endif
var startTime = DateTime.UtcNow;
long startTicks = Stopwatch.GetTimestamp();
try
{
var startRunNode = XmlHelper.CreateTopLevelElement("start-run");
startRunNode.AddAttribute("count", CountTests(filter).ToString());
startRunNode.AddAttribute("start-time", XmlConvert.ToString(startTime, "u"));
startRunNode.AddAttribute("engine-version", engineVersion);
startRunNode.AddAttribute("clr-version", clrVersion);
#if !NETSTANDARD1_6
InsertCommandLineElement(startRunNode);
#endif
eventDispatcher.OnTestEvent(startRunNode.OuterXml);
TestEngineResult result = PrepareResult(GetEngineRunner().Run(eventDispatcher, filter)).MakeTestRunResult(TestPackage);
// These are inserted in reverse order, since each is added as the first child.
InsertFilterElement(result.Xml, filter);
#if !NETSTANDARD1_6
InsertCommandLineElement(result.Xml);
#endif
result.Xml.AddAttribute("engine-version", engineVersion);
result.Xml.AddAttribute("clr-version", clrVersion);
double duration = (double)(Stopwatch.GetTimestamp() - startTicks) / Stopwatch.Frequency;
result.Xml.AddAttribute("start-time", XmlConvert.ToString(startTime, "u"));
result.Xml.AddAttribute("end-time", XmlConvert.ToString(DateTime.UtcNow, "u"));
result.Xml.AddAttribute("duration", duration.ToString("0.000000", NumberFormatInfo.InvariantInfo));
IsTestRunning = false;
eventDispatcher.OnTestEvent(result.Xml.OuterXml);
return result;
}
catch(Exception ex)
{
IsTestRunning = false;
var resultXml = XmlHelper.CreateTopLevelElement("test-run");
resultXml.AddAttribute("id", TestPackage.ID);
resultXml.AddAttribute("result", "Failed");
resultXml.AddAttribute("label", "Error");
resultXml.AddAttribute("engine-version", engineVersion);
resultXml.AddAttribute("clr-version", clrVersion);
double duration = (double)(Stopwatch.GetTimestamp() - startTicks) / Stopwatch.Frequency;
resultXml.AddAttribute("start-time", XmlConvert.ToString(startTime, "u"));
resultXml.AddAttribute("end-time", XmlConvert.ToString(DateTime.UtcNow, "u"));
resultXml.AddAttribute("duration", duration.ToString("0.000000", NumberFormatInfo.InvariantInfo));
eventDispatcher.OnTestEvent(resultXml.OuterXml);
eventDispatcher.OnTestEvent($"<unhandled-exception message='{ex.Message}' stacktrace='{ex.StackTrace}'/>");
return new TestEngineResult(resultXml);
}
}
/// <summary>
/// Run the tests in the loaded TestPackage and return a test result. The tests
/// are run synchronously and the listener interface is notified as it progresses.
/// </summary>
/// <param name="listener">An ITestEventHandler to receive events</param>
/// <param name="filter">A TestFilter used to select tests</param>
/// <returns>A TestEngineResult giving the result of the test execution</returns>
protected override TestEngineResult RunTests(ITestEventListener listener, TestFilter filter)
{
var eventDispatcher = new TestEventDispatcher();
if (listener != null)
eventDispatcher.Listeners.Add(listener);
foreach (var extension in _extensionService.GetExtensions<ITestEventListener>())
eventDispatcher.Listeners.Add(extension);
IsTestRunning = true;
eventDispatcher.OnTestEvent(string.Format("<start-run count='{0}'/>", CountTestCases(filter)));
DateTime startTime = DateTime.UtcNow;
long startTicks = Stopwatch.GetTimestamp();
TestEngineResult result = _realRunner.Run(eventDispatcher, filter).Aggregate("test-run", TestPackage.Name, TestPackage.FullName);
// These are inserted in reverse order, since each is added as the first child.
InsertFilterElement(result.Xml, filter);
InsertCommandLineElement(result.Xml);
result.Xml.AddAttribute("engine-version", Assembly.GetExecutingAssembly().GetName().Version.ToString());
result.Xml.AddAttribute("clr-version", Environment.Version.ToString());
double duration = (double)(Stopwatch.GetTimestamp() - startTicks) / Stopwatch.Frequency;
result.Xml.AddAttribute("start-time", XmlConvert.ToString(startTime, "u"));
result.Xml.AddAttribute("end-time", XmlConvert.ToString(DateTime.UtcNow, "u"));
result.Xml.AddAttribute("duration", duration.ToString("0.000000", NumberFormatInfo.InvariantInfo));
IsTestRunning = false;
eventDispatcher.OnTestEvent(result.Xml.OuterXml);
return result;
}
