/// <summary> /// Provides a test batching strategy based on the provided arguments /// </summary> /// <param name="strategy">The base strategy to provide</param> /// <param name="settings">Adapter settings currently in use</param> /// <returns>An ITestBatchingStrategy instance or null if one cannot be provided</returns> private ITestBatchingStrategy GetBatchStrategy(TestBatch.Strategy strategy, BoostTestAdapterSettings settings) { TestBatch.CommandLineArgsBuilder argsBuilder = GetDefaultArguments; if (strategy != Strategy.TestCase) { argsBuilder = GetBatchedTestRunsArguments; } switch (strategy) { case Strategy.Source: return(new SourceTestBatchStrategy(this._testRunnerFactory, settings, argsBuilder)); case Strategy.TestSuite: return(new TestSuiteTestBatchStrategy(this._testRunnerFactory, settings, argsBuilder)); case Strategy.TestCase: return(new IndividualTestBatchStrategy(this._testRunnerFactory, settings, argsBuilder)); } return(null); }
/// <summary> /// Execute the tests one by one. Run Selected /// </summary> /// <param name="tests">Testcases object</param> /// <param name="runContext">Solution properties</param> /// <param name="frameworkHandle">Unit test framework handle</param> /// <remarks>Entry point of the execution procedure whenever the user requests to run one or a specific lists of tests</remarks> public void RunTests(IEnumerable <VSTestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle) { Code.Require(tests, "tests"); Code.Require(runContext, "runContext"); Code.Require(frameworkHandle, "frameworkHandle"); SetUp(frameworkHandle); Logger.Debug("IRunContext.IsDataCollectionEnabled: {0}", runContext.IsDataCollectionEnabled); Logger.Debug("IRunContext.RunSettings.SettingsXml: {0}", runContext.RunSettings.SettingsXml); BoostTestAdapterSettings settings = BoostTestAdapterSettingsProvider.GetSettings(runContext); // Batch tests into grouped runs based on test source and test suite so that we minimize symbol reloading // // NOTE Required batching at test suite level since Boost Unit Test Framework command-line arguments only allow // multiple test name specification for tests which reside in the same test suite // // NOTE For code-coverage speed is given preference over adapter responsiveness. TestBatch.Strategy strategy = ((runContext.IsDataCollectionEnabled) ? TestBatch.Strategy.TestSuite : settings.TestBatchStrategy); // Source strategy is invalid in such context since explicit tests are chosen. TestSuite is used instead. if (strategy == Strategy.Source) { strategy = Strategy.TestSuite; } ITestBatchingStrategy batchStrategy = GetBatchStrategy(strategy, settings, runContext); if (batchStrategy == null) { Logger.Error("No valid test batching strategy was found. Tests skipped."); } else { // NOTE Apply distinct to avoid duplicate test cases. Common issue when using BOOST_DATA_TEST_CASE. IEnumerable <TestRun> batches = batchStrategy.BatchTests(tests.Distinct(new TestCaseComparer())); RunBoostTests(batches, runContext, frameworkHandle); } TearDown(); }
/// <summary> /// Provides a test batching strategy based on the provided arguments /// </summary> /// <param name="strategy">The base strategy to provide</param> /// <param name="settings">Adapter settings currently in use</param> /// <param name="runContext">The RunContext for this TestCase. Determines whether the test should be debugged or not.</param> /// <returns>An ITestBatchingStrategy instance or null if one cannot be provided</returns> private ITestBatchingStrategy GetBatchStrategy(TestBatch.Strategy strategy, BoostTestAdapterSettings settings, IRunContext runContext) { TestBatch.CommandLineArgsBuilder argsBuilder = (string _source, BoostTestAdapterSettings _settings) => { return(GetDefaultArguments(_source, _settings, runContext.IsBeingDebugged)); }; if (strategy != Strategy.TestCase) { // Disable stdout, stderr and memory leak detection since it is difficult // to distinguish from which test does portions of the output map to argsBuilder = (string _source, BoostTestAdapterSettings _settings) => { var args = GetDefaultArguments(_source, _settings, runContext.IsBeingDebugged); // Disable standard error/standard output capture args.StandardOutFile = null; args.StandardErrorFile = null; // Disable memory leak detection args.DetectMemoryLeaks = 0; return(args); }; } switch (strategy) { case Strategy.Source: return(new SourceTestBatchStrategy(_testRunnerFactory, settings, argsBuilder)); case Strategy.TestSuite: return(new TestSuiteTestBatchStrategy(_testRunnerFactory, settings, argsBuilder)); case Strategy.TestCase: return(new IndividualTestBatchStrategy(_testRunnerFactory, settings, argsBuilder)); case Strategy.One: return(new OneShotTestBatchStrategy(_testRunnerFactory, settings, argsBuilder)); } return(null); }
/// <summary> /// Execute the tests one by one. Run All. /// </summary> /// <param name="sources">Collection of test modules (exe/dll)</param> /// <param name="runContext">Solution properties</param> /// <param name="frameworkHandle">Unit test framework handle</param> /// <remarks>Entry point of the execution procedure whenever the user requests to run all the tests</remarks> public void RunTests(IEnumerable <string> sources, IRunContext runContext, IFrameworkHandle frameworkHandle) { Code.Require(sources, "sources"); Code.Require(runContext, "runContext"); Code.Require(frameworkHandle, "frameworkHandle"); SetUp(frameworkHandle); Logger.Debug("IRunContext.IsDataCollectionEnabled: {0}", runContext.IsDataCollectionEnabled); Logger.Debug("IRunContext.RunSettings.SettingsXml: {0}", runContext.RunSettings.SettingsXml); BoostTestAdapterSettings settings = BoostTestAdapterSettingsProvider.GetSettings(runContext); foreach (string source in sources) { if (_cancelled) { break; } var discoverer = _boostTestDiscovererFactory.GetDiscoverer(source, settings); if (discoverer != null) { try { DefaultTestCaseDiscoverySink sink = new DefaultTestCaseDiscoverySink(); // NOTE IRunContext implements IDiscoveryContext // NOTE IFrameworkHandle implements IMessageLogger // Re-discover tests so that we could make use of the RunTests overload which takes an enumeration of test cases. // This is necessary since we need to run tests one by one in order to have the test adapter remain responsive // and have a list of tests over which we can generate test results for. discoverer.DiscoverTests(new[] { source }, runContext, sink); // The following ensures that only test cases that are not disabled are run when the user presses "Run all" // This, however, can be overridden by the .runsettings file supplied IEnumerable <TestCase> testsToRun = GetTestsToRun(settings, sink.Tests); // Batch tests into grouped runs based by source so that we avoid reloading symbols per test run // Batching by source since this overload is called when 'Run All...' or equivalent is triggered // NOTE For code-coverage speed is given preference over adapter responsiveness. TestBatch.Strategy strategy = ((runContext.IsDataCollectionEnabled) ? TestBatch.Strategy.Source : settings.TestBatchStrategy); ITestBatchingStrategy batchStrategy = GetBatchStrategy(strategy, settings, runContext); if (batchStrategy == null) { Logger.Error("No valid test batching strategy was found for {0}. Source skipped.", source); continue; } IEnumerable <TestRun> batches = batchStrategy.BatchTests(testsToRun); // Delegate to the RunBoostTests overload which takes an enumeration of test batches RunBoostTests(batches, runContext, frameworkHandle); } catch (Exception ex) { Logger.Error("Exception caught while running tests from {0} ({1})", source, ex.Message); } } } TearDown(); }