/// <summary>
/// Provides a test batching strategy based on the provided arguments
/// </summary>
/// <param name="strategy">The base strategy to provide</param>
/// <param name="settings">Adapter settings currently in use</param>
/// <returns>An ITestBatchingStrategy instance or null if one cannot be provided</returns>
private ITestBatchingStrategy GetBatchStrategy(TestBatch.Strategy strategy, BoostTestAdapterSettings settings)
{
TestBatch.CommandLineArgsBuilder argsBuilder = GetDefaultArguments;
if (strategy != Strategy.TestCase)
{
argsBuilder = GetBatchedTestRunsArguments;
}
switch (strategy)
{
case Strategy.Source: return(new SourceTestBatchStrategy(this._testRunnerFactory, settings, argsBuilder));
case Strategy.TestSuite: return(new TestSuiteTestBatchStrategy(this._testRunnerFactory, settings, argsBuilder));
case Strategy.TestCase: return(new IndividualTestBatchStrategy(this._testRunnerFactory, settings, argsBuilder));
}
return(null);
}
/// <summary>
/// Execute the tests one by one. Run Selected
/// </summary>
/// <param name="tests">Testcases object</param>
/// <param name="runContext">Solution properties</param>
/// <param name="frameworkHandle">Unit test framework handle</param>
/// <remarks>Entry point of the execution procedure whenever the user requests to run one or a specific lists of tests</remarks>
public void RunTests(IEnumerable <VSTestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle)
{
Code.Require(tests, "tests");
Code.Require(runContext, "runContext");
Code.Require(frameworkHandle, "frameworkHandle");
SetUp(frameworkHandle);
Logger.Debug("IRunContext.IsDataCollectionEnabled: {0}", runContext.IsDataCollectionEnabled);
Logger.Debug("IRunContext.RunSettings.SettingsXml: {0}", runContext.RunSettings.SettingsXml);
BoostTestAdapterSettings settings = BoostTestAdapterSettingsProvider.GetSettings(runContext);
// Batch tests into grouped runs based on test source and test suite so that we minimize symbol reloading
//
// NOTE Required batching at test suite level since Boost Unit Test Framework command-line arguments only allow
// multiple test name specification for tests which reside in the same test suite
//
// NOTE For code-coverage speed is given preference over adapter responsiveness.
TestBatch.Strategy strategy = ((runContext.IsDataCollectionEnabled) ? TestBatch.Strategy.TestSuite : settings.TestBatchStrategy);
// Source strategy is invalid in such context since explicit tests are chosen. TestSuite is used instead.
if (strategy == Strategy.Source)
{
strategy = Strategy.TestSuite;
}
ITestBatchingStrategy batchStrategy = GetBatchStrategy(strategy, settings, runContext);
if (batchStrategy == null)
{
Logger.Error("No valid test batching strategy was found. Tests skipped.");
}
else
{
// NOTE Apply distinct to avoid duplicate test cases. Common issue when using BOOST_DATA_TEST_CASE.
IEnumerable <TestRun> batches = batchStrategy.BatchTests(tests.Distinct(new TestCaseComparer()));
RunBoostTests(batches, runContext, frameworkHandle);
}
TearDown();
}
/// <summary>
/// Provides a test batching strategy based on the provided arguments
/// </summary>
/// <param name="strategy">The base strategy to provide</param>
/// <param name="settings">Adapter settings currently in use</param>
/// <param name="runContext">The RunContext for this TestCase. Determines whether the test should be debugged or not.</param>
/// <returns>An ITestBatchingStrategy instance or null if one cannot be provided</returns>
private ITestBatchingStrategy GetBatchStrategy(TestBatch.Strategy strategy, BoostTestAdapterSettings settings, IRunContext runContext)
{
TestBatch.CommandLineArgsBuilder argsBuilder = (string _source, BoostTestAdapterSettings _settings) =>
{
return(GetDefaultArguments(_source, _settings, runContext.IsBeingDebugged));
};
if (strategy != Strategy.TestCase)
{
// Disable stdout, stderr and memory leak detection since it is difficult
// to distinguish from which test does portions of the output map to
argsBuilder = (string _source, BoostTestAdapterSettings _settings) =>
{
var args = GetDefaultArguments(_source, _settings, runContext.IsBeingDebugged);
// Disable standard error/standard output capture
args.StandardOutFile = null;
args.StandardErrorFile = null;
// Disable memory leak detection
args.DetectMemoryLeaks = 0;
return(args);
};
}
switch (strategy)
{
case Strategy.Source: return(new SourceTestBatchStrategy(_testRunnerFactory, settings, argsBuilder));
case Strategy.TestSuite: return(new TestSuiteTestBatchStrategy(_testRunnerFactory, settings, argsBuilder));
case Strategy.TestCase: return(new IndividualTestBatchStrategy(_testRunnerFactory, settings, argsBuilder));
case Strategy.One: return(new OneShotTestBatchStrategy(_testRunnerFactory, settings, argsBuilder));
}
return(null);
}
/// <summary>
/// Execute the tests one by one. Run All.
/// </summary>
/// <param name="sources">Collection of test modules (exe/dll)</param>
/// <param name="runContext">Solution properties</param>
/// <param name="frameworkHandle">Unit test framework handle</param>
/// <remarks>Entry point of the execution procedure whenever the user requests to run all the tests</remarks>
public void RunTests(IEnumerable <string> sources,
IRunContext runContext,
IFrameworkHandle frameworkHandle)
{
Code.Require(sources, "sources");
Code.Require(runContext, "runContext");
Code.Require(frameworkHandle, "frameworkHandle");
SetUp(frameworkHandle);
Logger.Debug("IRunContext.IsDataCollectionEnabled: {0}", runContext.IsDataCollectionEnabled);
Logger.Debug("IRunContext.RunSettings.SettingsXml: {0}", runContext.RunSettings.SettingsXml);
BoostTestAdapterSettings settings = BoostTestAdapterSettingsProvider.GetSettings(runContext);
foreach (string source in sources)
{
if (_cancelled)
{
break;
}
var discoverer = _boostTestDiscovererFactory.GetDiscoverer(source, settings);
if (discoverer != null)
{
try
{
DefaultTestCaseDiscoverySink sink = new DefaultTestCaseDiscoverySink();
// NOTE IRunContext implements IDiscoveryContext
// NOTE IFrameworkHandle implements IMessageLogger
// Re-discover tests so that we could make use of the RunTests overload which takes an enumeration of test cases.
// This is necessary since we need to run tests one by one in order to have the test adapter remain responsive
// and have a list of tests over which we can generate test results for.
discoverer.DiscoverTests(new[] { source }, runContext, sink);
// The following ensures that only test cases that are not disabled are run when the user presses "Run all"
// This, however, can be overridden by the .runsettings file supplied
IEnumerable <TestCase> testsToRun = GetTestsToRun(settings, sink.Tests);
// Batch tests into grouped runs based by source so that we avoid reloading symbols per test run
// Batching by source since this overload is called when 'Run All...' or equivalent is triggered
// NOTE For code-coverage speed is given preference over adapter responsiveness.
TestBatch.Strategy strategy = ((runContext.IsDataCollectionEnabled) ? TestBatch.Strategy.Source : settings.TestBatchStrategy);
ITestBatchingStrategy batchStrategy = GetBatchStrategy(strategy, settings, runContext);
if (batchStrategy == null)
{
Logger.Error("No valid test batching strategy was found for {0}. Source skipped.", source);
continue;
}
IEnumerable <TestRun> batches = batchStrategy.BatchTests(testsToRun);
// Delegate to the RunBoostTests overload which takes an enumeration of test batches
RunBoostTests(batches, runContext, frameworkHandle);
}
catch (Exception ex)
{
Logger.Error("Exception caught while running tests from {0} ({1})", source, ex.Message);
}
}
}
TearDown();
}
