private void RunInternal(IEnumerable <TestMethod> tests)
{
var testMethods = tests as IList <TestMethod> ?? tests.ToList();
if (!testMethods.Any())
{
return;
}
var modules = testMethods.GroupBy(test => test.Declaration.QualifiedName.QualifiedModuleName);
foreach (var module in modules)
{
var testInitialize = module.Key.FindTestInitializeMethods(_state).ToList();
var testCleanup = module.Key.FindTestCleanupMethods(_state).ToList();
var capturedModule = module;
var moduleTestMethods = testMethods
.Where(test => test.Declaration.QualifiedName.QualifiedModuleName.ProjectId == capturedModule.Key.ProjectId &&
test.Declaration.QualifiedName.QualifiedModuleName.ComponentName == capturedModule.Key.ComponentName);
var fakes = _fakesFactory.Create();
try
{
Run(module.Key.FindModuleInitializeMethods(_state));
foreach (var test in moduleTestMethods)
{
// no need to run setup/teardown for ignored tests
if (test.Declaration.Annotations.Any(a => a.AnnotationType == AnnotationType.IgnoreTest))
{
test.UpdateResult(TestOutcome.Ignored);
OnTestCompleted();
continue;
}
var stopwatch = new Stopwatch();
stopwatch.Start();
try
{
fakes.StartTest();
Run(testInitialize);
test.Run();
Run(testCleanup);
}
catch (COMException ex)
{
Logger.Error(ex, "Unexpected COM exception while running tests.", test.Declaration?.QualifiedName);
test.UpdateResult(TestOutcome.Inconclusive, RubberduckUI.Assert_ComException);
}
finally
{
fakes.StopTest();
}
stopwatch.Stop();
test.Result.SetDuration(stopwatch.ElapsedMilliseconds);
OnTestCompleted();
Model.AddExecutedTest(test);
}
Run(module.Key.FindModuleCleanupMethods(_state));
}
finally
{
_fakesFactory.Release(fakes);
}
}
}
private void RunWhileSuspendedOnUiThread(IEnumerable <TestMethod> tests)
{
var testMethods = tests as IList <TestMethod> ?? tests.ToList();
if (!testMethods.Any())
{
return;
}
_lastRun.Clear();
try
{
EnsureRubberduckIsReferencedForEarlyBoundTests();
}
catch (InvalidOperationException e)
{
Logger.Warn(e);
foreach (var test in testMethods)
{
OnTestCompleted(test, new TestResult(TestOutcome.Failed, AssertMessages.Prerequisite_EarlyBindingReferenceMissing));
}
return;
}
var overallTime = new Stopwatch();
overallTime.Start();
try
{
var testsByModule = testMethods.GroupBy(test => test.Declaration.QualifiedName.QualifiedModuleName)
.ToDictionary(grouping => grouping.Key, grouping => grouping.ToList());
foreach (var moduleName in testsByModule.Keys)
{
var testInitialize = TestDiscovery.FindTestInitializeMethods(moduleName, _state).ToList();
var testCleanup = TestDiscovery.FindTestCleanupMethods(moduleName, _state).ToList();
var moduleTestMethods = testsByModule[moduleName];
var fakes = _fakesFactory.Create();
using (var typeLibWrapper = _wrapperProvider.TypeLibWrapperFromProject(moduleName.ProjectId))
{
try
{
_declarationRunner.RunDeclarations(typeLibWrapper, TestDiscovery.FindModuleInitializeMethods(moduleName, _state));
}
catch (COMException ex)
{
Logger.Error(ex, "Unexpected COM exception while initializing tests for module {0}. The module will be skipped.", moduleName.Name);
foreach (var method in moduleTestMethods)
{
OnTestCompleted(method, new TestResult(TestOutcome.Unknown, AssertMessages.TestRunner_ModuleInitializeFailure));
}
continue;
}
foreach (var test in moduleTestMethods)
{
OnTestStarted(test);
// no need to run setup/teardown for ignored tests
if (test.Declaration.Annotations.Any(a => a.Annotation is IgnoreTestAnnotation))
{
OnTestCompleted(test, new TestResult(TestOutcome.Ignored));
continue;
}
try
{
fakes.StartTest();
try
{
_declarationRunner.RunDeclarations(typeLibWrapper, testInitialize);
}
catch (COMException trace)
{
OnTestCompleted(test, new TestResult(TestOutcome.Inconclusive, AssertMessages.TestRunner_TestInitializeFailure));
Logger.Trace(trace, "Unexpected COMException when running TestInitialize");
continue;
}
// The message pump is flushed here to catch cancellation requests. This is the only place inside the main test running
// loop where this is "safe" to do - any other location risks either potentially misses test teardown or risks not knowing
// what teardown needs to be done via VBA.
_uiDispatcher.FlushMessageQueue();
if (CancellationRequested)
{
RunTestCleanup(typeLibWrapper, testCleanup);
fakes.StopTest();
break;
}
var result = RunTestMethod(typeLibWrapper, test);
// we can trigger this event, because cleanup can fail without affecting the result
OnTestCompleted(test, result);
RunTestCleanup(typeLibWrapper, testCleanup);
}
finally
{
fakes.StopTest();
}
}
try
{
_declarationRunner.RunDeclarations(typeLibWrapper, TestDiscovery.FindModuleCleanupMethods(moduleName, _state));
}
catch (COMException ex)
{
// FIXME somehow notify the user of this mess
Logger.Error(ex,
"Unexpected COM exception while cleaning up tests for module {0}. Aborting any further unit tests",
moduleName.Name);
break;
}
}
}
}
catch (Exception ex)
{
// FIXME somehow notify the user of this mess
Logger.Error(ex, "Unexpected expection while running unit tests; unit tests will be aborted");
}
CancellationRequested = false;
overallTime.Stop();
TestRunCompleted?.Invoke(this, new TestRunCompletedEventArgs(overallTime.ElapsedMilliseconds));
}
private void RunWhileSuspended(IEnumerable <TestMethod> tests)
{
var testMethods = tests as IList <TestMethod> ?? tests.ToList();
if (!testMethods.Any())
{
return;
}
LastRun.Clear();
foreach (var resultAggregator in resultsByOutcome.Values)
{
resultAggregator.Clear();
}
try
{
EnsureRubberduckIsReferencedForEarlyBoundTests();
}
catch (InvalidOperationException e)
{
Logger.Warn(e);
foreach (var test in testMethods)
{
OnTestCompleted(test, new TestResult(TestOutcome.Failed, AssertMessages.Prerequisite_EarlyBindingReferenceMissing, 0));
}
return;
}
var overallTime = new Stopwatch();
overallTime.Start();
try
{
var testsByModule = testMethods.GroupBy(test => test.Declaration.QualifiedName.QualifiedModuleName)
.ToDictionary(grouping => grouping.Key, grouping => grouping.ToList());
foreach (var moduleName in testsByModule.Keys)
{
var testInitialize = TestDiscovery.FindTestInitializeMethods(moduleName, _state).ToList();
var testCleanup = TestDiscovery.FindTestCleanupMethods(moduleName, _state).ToList();
var moduleTestMethods = testsByModule[moduleName];
var fakes = _fakesFactory.Create();
using (var typeLibWrapper = _wrapperProvider.TypeLibWrapperFromProject(moduleName.ProjectId))
{
try
{
_declarationRunner.RunDeclarations(typeLibWrapper, TestDiscovery.FindModuleInitializeMethods(moduleName, _state));
}
catch (COMException ex)
{
Logger.Error(ex, "Unexpected COM exception while initializing tests for module {0}. The module will be skipped.", moduleName.Name);
foreach (var method in moduleTestMethods)
{
OnTestCompleted(method, new TestResult(TestOutcome.Unknown, AssertMessages.TestRunner_ModuleInitializeFailure));
}
continue;
}
foreach (var test in moduleTestMethods)
{
// no need to run setup/teardown for ignored tests
if (test.Declaration.Annotations.Any(a => a.AnnotationType == AnnotationType.IgnoreTest))
{
OnTestCompleted(test, new TestResult(TestOutcome.Ignored));
continue;
}
try
{
fakes.StartTest();
try
{
_declarationRunner.RunDeclarations(typeLibWrapper, testInitialize);
}
catch (COMException trace)
{
OnTestCompleted(test, new TestResult(TestOutcome.Inconclusive, AssertMessages.TestRunner_TestInitializeFailure));
Logger.Trace(trace, "Unexpected COMException when running TestInitialize");
continue;
}
var result = RunTestMethod(typeLibWrapper, test);
// we can trigger this event, because cleanup can fail without affecting the result
OnTestCompleted(test, result);
try
{
_declarationRunner.RunDeclarations(typeLibWrapper, testCleanup);
}
catch (COMException cleanupFail)
{
// Apparently the user doesn't need to know when test results for subsequent tests could be incorrect
Logger.Trace(cleanupFail, "Unexpected COMException when running TestCleanup");
}
}
finally
{
fakes.StopTest();
}
}
try
{
_declarationRunner.RunDeclarations(typeLibWrapper, TestDiscovery.FindModuleCleanupMethods(moduleName, _state));
}
catch (COMException ex)
{
// FIXME somehow notify the user of this mess
Logger.Error(ex,
"Unexpected COM exception while cleaning up tests for module {0}. Aborting any further unit tests",
moduleName.Name);
break;
}
}
}
}
catch (Exception ex)
{
// FIXME somehow notify the user of this mess
Logger.Error(ex, "Unexpected expection while running unit tests; unit tests will be aborted");
}
overallTime.Stop();
TestRunCompleted?.Invoke(this, new TestRunCompletedEventArgs(overallTime.ElapsedMilliseconds));
}