public void TestEmptyBoostReport()
{
const string logFile = "BoostTestAdapterNunit.Resources.ReportsLogs.Empty.sample.test.log.xml";
const string reportFile = "BoostTestAdapterNunit.Resources.ReportsLogs.Empty.sample.test.report.xml";
this.TestResourceProvider = CreateDefaultResourceFactory(logFile, reportFile);
this.Executor.RunTests(
new string[] { DefaultSource },
this.RunContext,
this.FrameworkHandle
);
// Ensure that a single MockBoostTestRunner was provisioned
Assert.That(this.RunnerFactory.ProvisionedRunners.Count, Is.EqualTo(1));
MockBoostTestRunner runner = this.RunnerFactory.LastTestRunner as MockBoostTestRunner;
Assert.That(runner, Is.Not.Null);
// Ensure that the tests are marked as failed due to empty report file
Assert.That(this.FrameworkHandle.Results.Count(), Is.EqualTo(1));
foreach (VSTestResult result in this.FrameworkHandle.Results)
{
Assert.That(result.ComputerName, Is.EqualTo(Environment.MachineName));
Assert.That(result.TestCase.Source, Is.EqualTo(DefaultSource));
Assert.That(result.Outcome, Is.EqualTo(TestOutcome.Failed));
}
}
public void TestGlobalFixtureException()
{
const string logFile = "BoostTestAdapterNunit.Resources.ReportsLogs.GlobalFixtureError.sample.test.log.xml";
const string reportFile = "BoostTestAdapterNunit.Resources.ReportsLogs.GlobalFixtureError.sample.test.report.xml";
this.TestResourceProvider = CreateDefaultResourceFactory(logFile, reportFile);
this.Executor.RunTests(
new VSTestCase[] { CreateTestCase("Test1", DefaultSource), CreateTestCase("Test2", DefaultSource) },
this.RunContext,
this.FrameworkHandle
);
// Ensure that a single MockBoostTestRunner was provisioned
Assert.That(this.RunnerFactory.ProvisionedRunners.Count, Is.EqualTo(1));
MockBoostTestRunner runner = this.RunnerFactory.LastTestRunner as MockBoostTestRunner;
Assert.That(runner, Is.Not.Null);
// Ensure that the tests are marked as failed and that the content of the message is equivalent to that of the report
string reportContent = TestHelper.ReadEmbeddedResource(reportFile);
Assert.That(this.FrameworkHandle.Results.Count(), Is.EqualTo(2));
foreach (VSTestResult result in this.FrameworkHandle.Results)
{
Assert.That(result.ComputerName, Is.EqualTo(Environment.MachineName));
Assert.That(result.TestCase.Source, Is.EqualTo(DefaultSource));
Assert.That(result.Outcome, Is.EqualTo(TestOutcome.Failed));
Assert.That(result.ErrorMessage, Is.EqualTo(reportContent));
}
}
public void DisableStdOutErrRedirection()
{
this.RunContext.RegisterSettingProvider(BoostTestAdapterSettings.XmlRootName, new BoostTestAdapterSettingsProvider());
this.RunContext.LoadSettings("<RunSettings><BoostTest><EnableStdOutRedirection>false</EnableStdOutRedirection><EnableStdErrRedirection>false</EnableStdErrRedirection></BoostTest></RunSettings>");
this.Executor.RunTests(
new string[] { DefaultSource },
this.RunContext,
this.FrameworkHandle
);
AssertDefaultTestResultProperties(this.FrameworkHandle.Results);
foreach (IBoostTestRunner runner in this.RunnerFactory.ProvisionedRunners)
{
Assert.That(runner, Is.TypeOf <MockBoostTestRunner>());
MockBoostTestRunner testRunner = (MockBoostTestRunner)runner;
foreach (BoostTestRunnerCommandLineArgs args in testRunner.Args)
{
Assert.That(args.StandardOutFile, Is.Null);
Assert.That(args.StandardErrorFile, Is.Null);
}
}
}
public void DebugTestSelection()
{
this.RunContext.IsBeingDebugged = true;
this.Executor.RunTests(
GetDefaultTests(),
this.RunContext,
this.FrameworkHandle
);
MockBoostTestRunner runner = this.RunnerFactory.LastTestRunner as MockBoostTestRunner;
Assert.That(runner, Is.Not.Null);
Assert.That(runner.DebugExecution, Is.True);
AssertDefaultTestResultProperties(this.FrameworkHandle.Results);
}
public void TestIndividualTestExecutionRuns()
{
this.RunContext.RegisterSettingProvider(BoostTestAdapterSettings.XmlRootName, new BoostTestAdapterSettingsProvider());
this.RunContext.LoadSettings("<RunSettings><BoostTest><BatchTestRuns>false</BatchTestRuns></BoostTest></RunSettings>");
VSTestCase[] testCases = new VSTestCase[]
{
CreateTestCase("A/1", DefaultSource), CreateTestCase("A/2", DefaultSource),
CreateTestCase("B/1", DefaultSource)
};
this.Executor.RunTests(
testCases,
this.RunContext,
this.FrameworkHandle
);
List <string> testIdentifiers = testCases.Select(test => test.FullyQualifiedName).ToList();
foreach (IBoostTestRunner runner in this.RunnerFactory.ProvisionedRunners)
{
Assert.That(runner, Is.TypeOf <MockBoostTestRunner>());
MockBoostTestRunner testRunner = (MockBoostTestRunner)runner;
// TestRunners may be provisioned but left unused. We are only interested in the ones used to execute tests.
if (testRunner.RunCount == 0)
{
continue;
}
Assert.That(runner.Source, Is.EqualTo(DefaultSource));
foreach (BoostTestRunnerCommandLineArgs args in testRunner.Args)
{
// One test at a time should be invoked
Assert.That(args.Tests.Count, Is.EqualTo(1));
Assert.That(testIdentifiers.Remove(args.Tests[0]), Is.True);
}
}
// All tests should be have been invoked
Assert.That(testIdentifiers, Is.Empty);
}
public void RunTestsWithTestSettings()
{
this.RunContext.RegisterSettingProvider(BoostTestAdapterSettings.XmlRootName, new BoostTestAdapterSettingsProvider());
this.RunContext.LoadEmbeddedSettings("BoostTestAdapterNunit.Resources.Settings.sample.runsettings");
this.Executor.RunTests(
GetDefaultTests(),
this.RunContext,
this.FrameworkHandle
);
AssertDefaultTestResultProperties(this.FrameworkHandle.Results);
MockBoostTestRunner runner = this.RunnerFactory.LastTestRunner as MockBoostTestRunner;
Assert.That(runner, Is.Not.Null);
Assert.That(runner.RunCount, Is.EqualTo(1));
Assert.That(runner.Settings.First().Timeout, Is.EqualTo(600000));
}
public void TestTestSuiteBatchedRuns()
{
this.RunContext.RegisterSettingProvider(BoostTestAdapterSettings.XmlRootName, new BoostTestAdapterSettingsProvider());
this.RunContext.LoadSettings("<RunSettings><BoostTest><TestBatchStrategy>TestSuite</TestBatchStrategy></BoostTest></RunSettings>");
string otherSource = "OtherSource";
this.Executor.RunTests(
new VSTestCase[] { CreateTestCase("A/Test1", DefaultSource), CreateTestCase("A/Test2", DefaultSource), CreateTestCase("B/Test1", DefaultSource), CreateTestCase("A/Test1", otherSource) },
this.RunContext,
this.FrameworkHandle
);
List <KeyValuePair <string, IList <string> > > expectedBatches = new List <KeyValuePair <string, IList <string> > >
{
new KeyValuePair <string, IList <string> >(DefaultSource, new List <string> {
"A/Test1", "A/Test2"
}),
new KeyValuePair <string, IList <string> >(DefaultSource, new List <string> {
"B/Test1"
}),
new KeyValuePair <string, IList <string> >(otherSource, new List <string> {
"A/Test1"
})
};
// A runner per test suite per module should be provisioned
foreach (IBoostTestRunner runner in this.RunnerFactory.ProvisionedRunners)
{
Assert.That(runner, Is.TypeOf <MockBoostTestRunner>());
MockBoostTestRunner testRunner = (MockBoostTestRunner)runner;
for (int i = 0; i < testRunner.RunCount; ++i)
{
KeyValuePair <string, IList <string> > run = new KeyValuePair <string, IList <string> >(testRunner.Source, testRunner.Args[i].Tests);
Assert.That(expectedBatches.RemoveAll(batch => (batch.Key == testRunner.Source) && (!batch.Value.Except(testRunner.Args[i].Tests).Any())), Is.EqualTo(1));
}
}
Assert.That(expectedBatches, Is.Empty);
}
public void TestCodeCoverageSelection()
{
this.RunContext.IsDataCollectionEnabled = true;
this.Executor.RunTests(
new VSTestCase[] { CreateTestCase("Test1", DefaultSource), CreateTestCase("Test2", DefaultSource) },
this.RunContext,
this.FrameworkHandle
);
IList <MockBoostTestRunner> runners = this.RunnerFactory.ProvisionedRunners.OfType <MockBoostTestRunner>().ToList();
Assert.That(runners.GroupBy(runner => runner.Source).Count(), Is.EqualTo(1));
MockBoostTestRunner testRunner = runners.FirstOrDefault(runner => runner.RunCount == 1);
Assert.That(testRunner, Is.Not.Null);
// All selected tests are executed
Assert.That(testRunner.Args.First().Tests.Count, Is.EqualTo(2));
}
public void TestCodeCoverage()
{
this.RunContext.IsDataCollectionEnabled = true;
this.Executor.RunTests(
new string[] { DefaultSource },
this.RunContext,
this.FrameworkHandle
);
IList <MockBoostTestRunner> runners = this.RunnerFactory.ProvisionedRunners.OfType <MockBoostTestRunner>().ToList();
// Although multiple runners (one per testcase) will be provisioned, only one type of runner (specific to DefaultSource) is used
Assert.That(runners.GroupBy(runner => runner.Source).Count(), Is.EqualTo(1));
// Only one runner is executed and that runner is only executed once
MockBoostTestRunner testRunner = runners.FirstOrDefault(runner => runner.RunCount == 1);
Assert.That(testRunner, Is.Not.Null);
// All tests are executed
Assert.That(testRunner.Args.First().Tests, Is.Empty);
}
public void TestModuleBatchedRuns()
{
this.RunContext.RegisterSettingProvider(BoostTestAdapterSettings.XmlRootName, new BoostTestAdapterSettingsProvider());
this.RunContext.LoadSettings("<RunSettings><BoostTest><TestBatchStrategy>Source</TestBatchStrategy></BoostTest></RunSettings>");
string OtherSource = "OtherSource";
string YetAnotherSource = "YetAnotherSource";
List <string> sources = new List <string> {
DefaultSource, OtherSource, YetAnotherSource
};
this.TestCaseProvider = (string source) =>
{
IEnumerable <VSTestCase> tests = GetTests(source);
if (!tests.Any())
{
if (source == OtherSource)
{
// 10 tests in 2 suites
List <string> fullyQualifiedNames = new List <string> {
"Suite1/Test1", "Suite1/Test2", "Suite1/Test3", "Suite1/Test4", "Suite1/Test5"
, "Suite2/Test1", "Suite2/Test2", "Suite2/Test3", "Suite2/Test4", "Suite2/Test5"
};
return(GenerateTestCases(fullyQualifiedNames, OtherSource));
}
if (source == YetAnotherSource)
{
// 5 tests in 2 suites
List <string> fullyQualifiedNames = new List <string> {
"Suite1/Test1", "Suite1/Test2", "Suite1/Test3", "Suite2/Test1", "Suite2/Test2"
};
return(GenerateTestCases(fullyQualifiedNames, YetAnotherSource));
}
}
return(tests);
};
this.Executor.RunTests(
sources,
this.RunContext,
this.FrameworkHandle
);
// One runner per source be provisioned for the available source
foreach (IBoostTestRunner runner in this.RunnerFactory.ProvisionedRunners)
{
Assert.That(runner, Is.TypeOf <MockBoostTestRunner>());
MockBoostTestRunner testRunner = (MockBoostTestRunner)runner;
// 1 Module/Source -> 1 Execution for all contained tests within
Assert.That(testRunner.RunCount, Is.EqualTo(1));
Assert.That(sources.Remove(runner.Source), Is.True);
// No tests should be specified on the command line implying all tests are to be executed
Assert.That(testRunner.Args.First().Tests.Count, Is.EqualTo(0));
}
Assert.That(sources, Is.Empty);
}
