/// <summary>
/// Sets the working environment (i.e. WorkingDirectory and Environment) properties of the command line arguments
/// based on the provided details
/// </summary>
/// <param name="args">The arguments which to set</param>
/// <param name="source">The base source which will be executed</param>
/// <param name="settings">The BoostTestAdapterSettings which are currently applied</param>
/// <param name="vs">The current Visual Studio instance (if available)</param>
/// <exception cref="COMException"></exception>
public static void SetWorkingEnvironment(this BoostTestRunnerCommandLineArgs args, string source, BoostTestAdapterSettings settings, IVisualStudio vs)
{
Code.Require(args, "args");
Code.Require(source, "source");
Code.Require(settings, "settings");
// Default working directory
args.WorkingDirectory = Path.GetDirectoryName(source);
// Working directory extracted from test settings
if (!string.IsNullOrEmpty(settings.WorkingDirectory) && Directory.Exists(settings.WorkingDirectory))
{
args.WorkingDirectory = settings.WorkingDirectory;
}
if (vs != null)
{
// Visual Studio configuration has higher priority over settings (if available)
IVSDebugConfiguration vsConfiguration = LocateVSDebugConfiguration(source, vs);
if (vsConfiguration != null)
{
args.WorkingDirectory = vsConfiguration.WorkingDirectory;
args.SetEnvironment(vsConfiguration.Environment);
}
}
// Enforce windows style backward slashes
args.WorkingDirectory = args.WorkingDirectory.Replace('/', '\\');
}
/// <summary>
/// Provides a collection of ISourceFilters based on the provided BoostTestAdapterSettings.
/// </summary>
/// <param name="settings">The BoostTestAdapterSettings</param>
/// <returns>An collection of ISourceFilters based on the provided settings.</returns>
public static ISourceFilter[] Get(BoostTestAdapterSettings settings)
{
Utility.Code.Require(settings, "settings");
if (settings.ConditionalInclusionsFilteringEnabled)
{
return new ISourceFilter[]
{
new QuotedStringsFilter(),
new MultilineCommentFilter(),
new SingleLineCommentFilter(),
new ConditionalInclusionsFilter(
new ExpressionEvaluation()
)
};
}
else
{
Logger.Warn("Conditional inclusions filtering is disabled.");
return new ISourceFilter[]
{
new QuotedStringsFilter(),
new MultilineCommentFilter(),
new SingleLineCommentFilter(),
//conditional inclusions filter omitted
};
}
}
public IEnumerable<FactoryResult> GetDiscoverers(IReadOnlyCollection<string> sources, BoostTestAdapterSettings settings)
{
var tmpSources = new List<string>(sources);
var discoverers = new List<FactoryResult>();
// sources that can be run on the external runner
if (settings.ExternalTestRunner != null)
{
var extSources = tmpSources
.Where(s => settings.ExternalTestRunner.ExtensionType == Path.GetExtension(s))
.ToList();
discoverers.Add(new FactoryResult()
{
Discoverer = new ExternalDiscoverer(settings.ExternalTestRunner),
Sources = extSources
});
tmpSources.RemoveAll(s => extSources.Contains(s));
}
// sources that support list-content parameter
var listContentHelper = new StubListContentHelper();
var listContentSources = tmpSources
.Where(s => Path.GetExtension(s) == BoostTestDiscoverer.ExeExtension)
.Where(listContentHelper.IsListContentSupported)
.ToList();
if (listContentSources.Count > 0)
{
discoverers.Add(new FactoryResult()
{
Discoverer = new ListContentDiscoverer(listContentHelper),
Sources = listContentSources
});
tmpSources.RemoveAll(s => listContentSources.Contains(s));
}
// sources that NOT support the list-content parameter
var sourceCodeSources = tmpSources
.Where(s => Path.GetExtension(s) == BoostTestDiscoverer.ExeExtension)
.ToList();
if (sourceCodeSources.Count > 0)
{
discoverers.Add(new FactoryResult()
{
Discoverer = new SourceCodeDiscoverer(_dummySolution.Provider),
Sources = sourceCodeSources
});
}
return discoverers;
}
public void Load(XmlReader reader)
{
Utility.Code.Require(reader, "reader");
// NOTE This method gets called if the settings name matches the node name as expected.
if (reader.Read() && reader.Name.Equals(BoostTestAdapterSettings.XmlRootName))
{
XmlSerializer deserializer = new XmlSerializer(typeof(BoostTestAdapterSettings));
this.Settings = deserializer.Deserialize(reader) as BoostTestAdapterSettings;
}
}
public void Load(XmlReader reader)
{
Utility.Code.Require(reader, "reader");
// NOTE This method gets called if the settings name matches the node name as expected.
if (reader.Read() && reader.Name.Equals(BoostTestAdapterSettings.XmlRootName))
{
XmlSerializer deserializer = new XmlSerializer(typeof(BoostTestAdapterSettings));
this.Settings = deserializer.Deserialize(reader) as BoostTestAdapterSettings;
}
}
/// <summary>
/// Tests the contents of a BoostTestAdapterSettings instance, making sure they comply to the default expected values.
/// </summary>
/// <param name="settings">The BoostTestAdapterSettings instance to test</param>
private void AssertDefaultSettings(BoostTestAdapterSettings settings)
{
Assert.That(settings.ExecutionTimeoutMilliseconds, Is.EqualTo(-1));
Assert.That(settings.DiscoveryTimeoutMilliseconds, Is.EqualTo(5000));
Assert.That(settings.FailTestOnMemoryLeak, Is.False);
Assert.That(settings.ConditionalInclusionsFilteringEnabled, Is.True);
Assert.That(settings.LogLevel, Is.EqualTo(LogLevel.TestSuite));
Assert.That(settings.ExternalTestRunner, Is.Null);
Assert.That(settings.DetectFloatingPointExceptions, Is.False);
Assert.That(settings.CatchSystemErrors, Is.True);
Assert.That(settings.TestBatchStrategy, Is.EqualTo(Strategy.TestCase));
Assert.That(settings.UseListContent, Is.False);
}
/// <summary>
/// Builds a BoostTestAdapterSettings structure based on the information located within the IDiscoveryContext instance.
/// </summary>
/// <param name="context">The discovery context instance</param>
/// <returns>A BoostTestRunnerSettings instance based on the information identified via the provided IDiscoveryContext instance.</returns>
public static BoostTestAdapterSettings GetSettings(IDiscoveryContext context)
{
Utility.Code.Require(context, "context");
BoostTestAdapterSettings settings = new BoostTestAdapterSettings();
BoostTestAdapterSettingsProvider provider = (context.RunSettings == null) ? null : context.RunSettings.GetSettings(BoostTestAdapterSettings.XmlRootName) as BoostTestAdapterSettingsProvider;
if (provider != null)
{
settings = provider.Settings;
}
// Return defaults
return settings;
}
/// <summary>
/// Compares the serialized content of the settings structure against an Xml embedded resource string.
/// </summary>
/// <param name="settings">The settings structure whose serialization is to be compared</param>
/// <param name="resource">The path to an embedded resource which contains the serialized Xml content to compare against</param>
private void Compare(BoostTestAdapterSettings settings, string resource)
{
XmlElement element = settings.ToXml();
using (Stream stream = TestHelper.LoadEmbeddedResource(resource))
{
XmlDocument doc = new XmlDocument();
doc.Load(stream);
XmlNode root = doc.DocumentElement.SelectSingleNode("/RunSettings/BoostTest");
XmlComparer comparer = new XmlComparer();
comparer.CompareXML(element, root, XmlNodeTypeFilter.DefaultFilter);
}
}
/// <summary>
/// Builds a BoostTestAdapterSettings structure based on the information located within the IDiscoveryContext instance.
/// </summary>
/// <param name="context">The discovery context instance</param>
/// <returns>A BoostTestRunnerSettings instance based on the information identified via the provided IDiscoveryContext instance.</returns>
public static BoostTestAdapterSettings GetSettings(IDiscoveryContext context)
{
Utility.Code.Require(context, "context");
BoostTestAdapterSettings settings = new BoostTestAdapterSettings();
BoostTestAdapterSettingsProvider provider = (context.RunSettings == null) ? null : context.RunSettings.GetSettings(BoostTestAdapterSettings.XmlRootName) as BoostTestAdapterSettingsProvider;
if (provider != null)
{
settings = provider.Settings;
}
// Return defaults
return(settings);
}
/// <summary>
/// Returns an IBoostTestDiscoverer based on the provided source.
/// </summary>
/// <param name="source">Source to be associated.</param>
/// <param name="settings">A structure which states particular features of interest in the manufactured product.</param>
/// <returns>An IBoostTestDiscoverer instance or null if one cannot be provided.</returns>
public IBoostTestDiscoverer GetDiscoverer(string source, Settings.BoostTestAdapterSettings settings)
{
var list = new[] { source };
var results = GetDiscoverers(list, settings);
if (results != null)
{
var result = results.FirstOrDefault(x => x.Sources.Contains(source));
if (result != null)
{
return(result.Discoverer);
}
}
return(null);
}
public Type TestDiscovererProvisioning(string source, bool useListContent, string externalExtension)
{
ExternalBoostTestRunnerSettings externalSettings = null;
if (!string.IsNullOrEmpty(externalExtension))
{
externalSettings = new ExternalBoostTestRunnerSettings { ExtensionType = externalExtension };
}
BoostTestAdapterSettings settings = new BoostTestAdapterSettings()
{
ExternalTestRunner = externalSettings,
UseListContent = useListContent
};
IBoostTestDiscoverer discoverer = this.Factory.GetDiscoverer(source, settings);
return (discoverer == null) ? null : discoverer.GetType();
}
public IEnumerable<FactoryResult> GetDiscoverers(IReadOnlyCollection<string> sources, BoostTestAdapterSettings settings)
{
var tmpSources = new List<string>(sources);
var discoverers = new List<FactoryResult>();
// sources that can be run on the external runner
if (settings.ExternalTestRunner != null)
{
var extSources = tmpSources
.Where(s => settings.ExternalTestRunner.ExtensionType.IsMatch(Path.GetExtension(s)))
.ToList();
discoverers.Add(new FactoryResult()
{
Discoverer = new ExternalDiscoverer(settings.ExternalTestRunner, _dummySolution.Provider),
Sources = extSources
});
tmpSources.RemoveAll(s => extSources.Contains(s));
}
// sources that support list-content parameter
var listContentSources = tmpSources
.Where(s => (s == ("ListContentSupport" + BoostTestDiscoverer.ExeExtension)))
.ToList();
if (listContentSources.Count > 0)
{
IBoostTestRunnerFactory factory = A.Fake<IBoostTestRunnerFactory>();
A.CallTo(() => factory.GetRunner(A<string>._, A<BoostTestRunnerFactoryOptions>._)).ReturnsLazily((string source, BoostTestRunnerFactoryOptions options) => new StubListContentRunner(source));
discoverers.Add(new FactoryResult()
{
Discoverer = new ListContentDiscoverer(factory, _dummySolution.Provider),
Sources = listContentSources
});
tmpSources.RemoveAll(s => listContentSources.Contains(s));
}
return discoverers;
}
public void CorrectMultipleBoostTestDiscovererDispatchingWithExternalDll()
{
var sources = new[]
{
"ListContentSupport" + BoostTestDiscoverer.ExeExtension,
"ParseSources1" + BoostTestDiscoverer.ExeExtension,
"ParseSources2" + BoostTestDiscoverer.ExeExtension,
"DllProject1" + BoostTestDiscoverer.DllExtension,
"DllProject2" + BoostTestDiscoverer.DllExtension,
};
var stubListContentHelper = new StubListContentHelper();
var boostTestDiscovererFactory = new BoostTestDiscovererFactory(stubListContentHelper);
BoostTestAdapterSettings settings = new BoostTestAdapterSettings();
settings.ExternalTestRunner = new ExternalBoostTestRunnerSettings
{
ExtensionType = BoostTestDiscoverer.DllExtension
};
var results = boostTestDiscovererFactory.GetDiscoverers(sources, settings);
Assert.That(results.Count(), Is.EqualTo(3));
Assert.That(results.FirstOrDefault(x => x.Discoverer is ListContentDiscoverer), Is.Not.Null);
var lcd = results.First(x => x.Discoverer is ListContentDiscoverer);
Assert.That(lcd.Sources, Is.EqualTo(new[] { "ListContentSupport" + BoostTestDiscoverer.ExeExtension }));
Assert.That(results.FirstOrDefault(x => x.Discoverer is SourceCodeDiscoverer), Is.Not.Null);
var scd = results.First(x => x.Discoverer is SourceCodeDiscoverer);
Assert.That(scd.Sources, Is.EqualTo(new[] { "ParseSources1" + BoostTestDiscoverer.ExeExtension,
"ParseSources2" + BoostTestDiscoverer.ExeExtension }));
Assert.That(results.FirstOrDefault(x => x.Discoverer is ExternalDiscoverer), Is.Not.Null);
var exd = results.First(x => x.Discoverer is ExternalDiscoverer);
Assert.That(exd.Sources, Is.EqualTo(new[] { "DllProject1" + BoostTestDiscoverer.DllExtension,
"DllProject2" + BoostTestDiscoverer.DllExtension }));
}
public void CorrectSingleProjectBoostTestDiscovererDispatchingExternalExe()
{
var stubListContentHelper = new StubListContentHelper();
var boostTestDiscovererFactory = new BoostTestDiscovererFactory(stubListContentHelper);
BoostTestAdapterSettings settings = new BoostTestAdapterSettings();
settings.ExternalTestRunner = new ExternalBoostTestRunnerSettings
{
ExtensionType = BoostTestDiscoverer.ExeExtension
};
// source that supports --list-content parameter
var source = "ListContentSupport" + BoostTestDiscoverer.ExeExtension;
var discoverer = boostTestDiscovererFactory.GetDiscoverer(source, settings);
Assert.That(discoverer, Is.Not.Null);
Assert.That(discoverer, Is.AssignableFrom(typeof(ExternalDiscoverer)));
// source that NOT supports --list-content parameter
source = "ParseSources" + BoostTestDiscoverer.ExeExtension;
discoverer = boostTestDiscovererFactory.GetDiscoverer(source, settings);
Assert.That(discoverer, Is.Not.Null);
Assert.That(discoverer, Is.AssignableFrom(typeof(ExternalDiscoverer)));
// source dll project
source = "DllProject" + BoostTestDiscoverer.DllExtension;
discoverer = boostTestDiscovererFactory.GetDiscoverer(source, settings);
Assert.That(discoverer, Is.Null);
}
public SourceTestBatchStrategy(IBoostTestRunnerFactory testRunnerFactory, BoostTestAdapterSettings settings, CommandLineArgsBuilder argsBuilder)
: base(testRunnerFactory, settings, argsBuilder)
{
}
public void SerialiseExternalTestRunnerDiscoveryMapSettings()
{
BoostTestAdapterSettings settings = new BoostTestAdapterSettings();
settings.ExternalTestRunner = new ExternalBoostTestRunnerSettings
{
ExtensionType = ".dll",
DiscoveryMethodType = DiscoveryMethodType.DiscoveryFileMap,
ExecutionCommandLine = new CommandLine("C:\\ExternalTestRunner.exe", "--test \"{source}\"")
};
settings.ExternalTestRunner.DiscoveryFileMap.Add("test_1.dll", "C:\\tests\\test_1.xml");
settings.ExternalTestRunner.DiscoveryFileMap.Add("test_2.dll", "C:\\tests\\test_2.xml");
Compare(settings, "BoostTestAdapterNunit.Resources.Settings.externalTestRunner.runsettings");
}
/// <summary>
/// Generates TestResults based on Boost Test result output.
/// </summary>
/// <param name="testRun">The tests which have been executed in the prior test run.</param>
/// <param name="start">The test execution start time.</param>
/// <param name="end">The test execution end time.</param>
/// <param name="settings">boost test adapter settings</param>
/// <returns>A Visual Studio TestResult related to the executed test.</returns>
private static IEnumerable<VSTestResult> GenerateTestResults(TestRun testRun, DateTimeOffset start, DateTimeOffset end, BoostTestAdapterSettings settings)
{
TestResultCollection results = new TestResultCollection();
try
{
results.Parse(testRun.Arguments, settings);
}
catch (XmlException)
{
string text = ((File.Exists(testRun.Arguments.ReportFile)) ? File.ReadAllText(testRun.Arguments.ReportFile) : string.Empty);
if (text.Trim().StartsWith(TestNotFound, StringComparison.Ordinal))
{
return testRun.Tests.Select(GenerateNotFoundResult);
}
else
{
// Represent result parsing exception as a test fatal error
if (string.IsNullOrEmpty(text))
{
text = "Boost Test result file was not found or is empty.";
}
return testRun.Tests.Select(test => {
Boost.Results.TestResult exception = new Boost.Results.TestResult(results);
exception.Unit = Boost.Test.TestUnit.FromFullyQualifiedName(test.FullyQualifiedName);
// NOTE Divide by 10 to compensate for duration calculation described in VSTestResult.AsVSTestResult(this Boost.Results.TestResult, VSTestCase)
exception.Duration = ((ulong)(end - start).Ticks) / 10;
exception.Result = TestResultType.Failed;
exception.LogEntries.Add(new Boost.Results.LogEntryTypes.LogEntryFatalError(text));
return GenerateResult(test, exception, start, end);
});
}
}
return testRun.Tests.
Select(test =>
{
// Locate the test result associated to the current test
Boost.Results.TestResult result = results[test.FullyQualifiedName];
return (result == null) ? null : GenerateResult(test, result, start, end);
}).
Where(result => (result != null));
}
/// <summary>
/// Factory function which returns an appropriate BoostTestRunnerCommandLineArgs structure
/// </summary>
/// <param name="source">The TestCases source</param>
/// <param name="settings">The Boost Test adapter settings currently in use</param>
/// <returns>A BoostTestRunnerCommandLineArgs structure for the provided source</returns>
private BoostTestRunnerCommandLineArgs GetDefaultArguments(string source, BoostTestAdapterSettings settings)
{
BoostTestRunnerCommandLineArgs args = settings.CommandLineArgs.Clone();
args.WorkingDirectory = Path.GetDirectoryName(source);
string filename = Path.GetFileName(source);
// Specify log and report file information
args.LogFormat = OutputFormat.XML;
args.LogLevel = settings.LogLevel;
args.LogFile = SanitizeFileName(filename + FileExtensions.LogFile);
args.ReportFormat = OutputFormat.XML;
args.ReportLevel = ReportLevel.Detailed;
args.ReportFile = SanitizeFileName(filename + FileExtensions.ReportFile);
args.StandardOutFile = SanitizeFileName(filename + FileExtensions.StdOutFile);
args.StandardErrorFile = SanitizeFileName(filename + FileExtensions.StdErrFile);
return args;
}
/// <summary>
/// Generates TestResults based on Boost Test result output.
/// </summary>
/// <param name="testRun">The tests which have been executed in the prior test run.</param>
/// <param name="start">The test execution start time.</param>
/// <param name="settings">boost test adapter settings</param>
/// <returns>A Visual Studio TestResult related to the executed test.</returns>
private static IEnumerable<VSTestResult> GenerateTestResults(TestRun testRun, DateTimeOffset start, BoostTestAdapterSettings settings)
{
return GenerateTestResults(testRun, start, DateTimeOffset.Now, settings);
}
/// <summary>
/// Retrieves and assigns parameters by resolving configurations from different possible resources
/// </summary>
/// <param name="source">The TestCases source</param>
/// <param name="settings">The Boost Test adapter settings currently in use</param>
/// <returns>A string for the default working directory</returns>
private void GetDebugConfigurationProperties(string source, BoostTestAdapterSettings settings, BoostTestRunnerCommandLineArgs args)
{
try
{
args.SetWorkingEnvironment(source, settings, ((_vsProvider == null) ? null : _vsProvider.Instance));
}
catch (COMException ex)
{
Logger.Exception(ex, "Could not retrieve WorkingDirectory from Visual Studio Configuration-{0}", ex.Message);
}
}
/// <summary>
/// Tests the contents of a BoostTestAdapterSettings instance, making sure they comply to the default expected values.
/// </summary>
/// <param name="settings">The BoostTestAdapterSettings instance to test</param>
private void AssertDefaultSettings(BoostTestAdapterSettings settings)
{
Assert.That(settings.ExecutionTimeoutMilliseconds, Is.EqualTo(-1));
Assert.That(settings.DiscoveryTimeoutMilliseconds, Is.EqualTo(30000));
Assert.That(settings.FailTestOnMemoryLeak, Is.False);
Assert.That(settings.LogLevel, Is.EqualTo(LogLevel.TestSuite));
Assert.That(settings.ExternalTestRunner, Is.Null);
Assert.That(settings.DetectFloatingPointExceptions, Is.False);
Assert.That(settings.CatchSystemErrors, Is.True);
Assert.That(settings.TestBatchStrategy, Is.EqualTo(Strategy.TestCase));
Assert.That(settings.ForceListContent, Is.False);
Assert.That(settings.WorkingDirectory, Is.Null);
Assert.That(settings.EnableStdOutRedirection, Is.True);
Assert.That(settings.EnableStdErrRedirection, Is.True);
Assert.That(settings.Filters, Is.EqualTo(TestSourceFilter.Empty));
Assert.That(settings.RunDisabledTests, Is.False);
}
/// <summary>
/// Constructor
/// </summary>
/// <param name="testRunnerFactory">Factory which provides a test runner for a specific test source</param>
/// <param name="settings">Adapter settings which are currently in use</param>
/// <param name="argsBuilder">Factory function which populates a BoostTestRunnerCommandLineArgs based on the provided source and settings</param>
protected TestBatchStrategy(IBoostTestRunnerFactory testRunnerFactory, BoostTestAdapterSettings settings, CommandLineArgsBuilder argsBuilder)
{
this.TestRunnerFactory = testRunnerFactory;
this.Settings = settings;
this.ArgsBuilder = argsBuilder;
}
/// <summary>
/// Factory function which returns an appropriate BoostTestRunnerCommandLineArgs structure
/// </summary>
/// <param name="source">The TestCases source</param>
/// <param name="settings">The Boost Test adapter settings currently in use</param>
/// <returns>A BoostTestRunnerCommandLineArgs structure for the provided source</returns>
private BoostTestRunnerCommandLineArgs GetDefaultArguments(string source, BoostTestAdapterSettings settings)
{
BoostTestRunnerCommandLineArgs args = settings.CommandLineArgs.Clone();
GetDebugConfigurationProperties(source, settings, args);
// Specify log and report file information
args.LogFormat = OutputFormat.XML;
args.LogLevel = settings.LogLevel;
args.LogFile = TestPathGenerator.Generate(source, FileExtensions.LogFile);
args.ReportFormat = OutputFormat.XML;
args.ReportLevel = ReportLevel.Detailed;
args.ReportFile = TestPathGenerator.Generate(source, FileExtensions.ReportFile);
args.StandardOutFile = ((settings.EnableStdOutRedirection) ? TestPathGenerator.Generate(source, FileExtensions.StdOutFile) : null);
args.StandardErrorFile = ((settings.EnableStdErrRedirection) ? TestPathGenerator.Generate(source, FileExtensions.StdErrFile) : null);
return args;
}
/// <summary>
/// Associates each source with the correct IBoostTestDiscoverer implementation.
/// </summary>
/// <param name="sources">List of the sources to be associated.</param>
/// <param name="settings">A structure which states particular features of interest in the manufactured product.</param>
/// <returns>A list of FactoryResult. Each value contains an instance of IBoostTestDiscoverer and a list of sources that should be analyzed by the discoverer.</returns>
/// <remarks>The returned IEnumerable is always an already initialised List instance.</remarks>
public IEnumerable <FactoryResult> GetDiscoverers(IReadOnlyCollection <string> sources, Settings.BoostTestAdapterSettings settings)
{
var discoverers = new List <FactoryResult>();
if ((sources == null) || (!sources.Any()))
{
return(discoverers);
}
// Use default settings in case they are not provided by client code
settings = settings ?? new BoostTestAdapterSettings();
// sources that can be run on the external runner
var externalDiscovererSources = new List <string>();
// sources that support list-content parameter
var listContentDiscovererSources = new List <string>();
var factory = GetTestRunnerFactory();
foreach (var source in sources)
{
string extension = Path.GetExtension(source);
if (settings.ExternalTestRunner != null)
{
if (settings.ExternalTestRunner.ExtensionType.IsMatch(extension))
{
externalDiscovererSources.Add(source);
continue;
}
}
// Skip modules which are not .exe
if (string.Compare(extension, BoostTestDiscoverer.ExeExtension, StringComparison.OrdinalIgnoreCase) != 0)
{
continue;
}
// Ensure that the source is a Boost.Test module if it supports '--list_content'
if (IsListContentSupported(factory, source, settings))
{
listContentDiscovererSources.Add(source);
}
}
if ((externalDiscovererSources.Any()) && (settings != null))
{
discoverers.Add(new FactoryResult()
{
Discoverer = new ExternalDiscoverer(settings.ExternalTestRunner),
Sources = externalDiscovererSources
});
}
if (listContentDiscovererSources.Any())
{
discoverers.Add(new FactoryResult()
{
Discoverer = new ListContentDiscoverer(factory, _vsInstanceProvider),
Sources = listContentDiscovererSources
});
}
return(discoverers);
}
/// <summary>
/// Determines whether the provided source has --list_content capabilities
/// </summary>
/// <param name="source">The source to test</param>
/// <param name="settings">Test adapter settings</param>
/// <returns>true if the source has list content capabilities; false otherwise</returns>
private bool IsListContentSupported(string source, BoostTestAdapterSettings settings)
{
BoostTestRunnerFactoryOptions options = new BoostTestRunnerFactoryOptions()
{
ExternalTestRunnerSettings = settings.ExternalTestRunner
};
IBoostTestRunner runner = _factory.GetRunner(source, options);
return (runner != null) && runner.ListContentSupported;
}
public void SerializeSettings()
{
BoostTestAdapterSettings settings = new BoostTestAdapterSettings();
settings.TimeoutMilliseconds = 600000;
settings.FailTestOnMemoryLeak = true;
settings.ExternalTestRunner = new ExternalBoostTestRunnerSettings()
{
ExtensionType = ".dll",
DiscoveryMethodType = DiscoveryMethodType.DiscoveryCommandLine,
DiscoveryCommandLine = new CommandLine("C:\\ExternalTestRunner.exe", "--test \"{source}\" --list-debug \"{out}\""),
ExecutionCommandLine = new CommandLine("C:\\ExternalTestRunner.exe", "--test \"{source}\"")
};
Compare(settings, "BoostTestAdapterNunit.Resources.Settings.sample.runsettings");
}
/// <summary>
/// Associates each source with the correct IBoostTestDiscoverer implementation.
/// </summary>
/// <param name="sources">List of the sources to be associated.</param>
/// <param name="settings">A structure which states particular features of interest in the manufactured product.</param>
/// <returns>A list of FactoryResult. Each value contains an instance of IBoostTestDiscoverer and a list of sources that should be analyzed by the discoverer.</returns>
/// <remarks>The returned IEnumerable is always an already initialised List instance.</remarks>
public IEnumerable <FactoryResult> GetDiscoverers(IReadOnlyCollection <string> sources, Settings.BoostTestAdapterSettings settings)
{
var discoverers = new List <FactoryResult>();
if ((sources == null) || (!sources.Any()))
{
return(discoverers);
}
// Use default settings in case they are not provided by client code
settings = settings ?? new BoostTestAdapterSettings();
// sources that can be run on the external runner
var externalDiscovererSources = new List <string>();
// sources that support list-content parameter
var listContentDiscovererSources = new List <string>();
// sources that do NOT support the list-content parameter
var sourceCodeDiscovererSources = new List <string>();
foreach (var source in sources)
{
string extension = Path.GetExtension(source);
if (settings.ExternalTestRunner != null)
{
if (settings.ExternalTestRunner.ExtensionType.IsMatch(extension))
{
externalDiscovererSources.Add(source);
continue;
}
}
// Skip modules which are not .exe
if (extension != BoostTestDiscoverer.ExeExtension)
{
continue;
}
if ((settings.UseListContent) && ((settings.ForceListContent) || IsListContentSupported(source, settings)))
{
listContentDiscovererSources.Add(source);
}
else
{
sourceCodeDiscovererSources.Add(source);
}
}
if ((externalDiscovererSources.Any()) && (settings != null))
{
discoverers.Add(new FactoryResult()
{
Discoverer = new ExternalDiscoverer(settings.ExternalTestRunner),
Sources = externalDiscovererSources
});
}
if (listContentDiscovererSources.Any())
{
discoverers.Add(new FactoryResult()
{
Discoverer = new ListContentDiscoverer(),
Sources = listContentDiscovererSources
});
}
if (sourceCodeDiscovererSources.Any())
{
discoverers.Add(new FactoryResult()
{
Discoverer = new SourceCodeDiscoverer(),
Sources = sourceCodeDiscovererSources
});
}
return(discoverers);
}
public void DefaultSettings()
{
BoostTestAdapterSettings settings = new BoostTestAdapterSettings();
AssertDefaultSettings(settings);
}
/// <summary>
/// Provides a test batching strategy based on the provided arguments
/// </summary>
/// <param name="strategy">The base strategy to provide</param>
/// <param name="settings">Adapter settings currently in use</param>
/// <returns>An ITestBatchingStrategy instance or null if one cannot be provided</returns>
private ITestBatchingStrategy GetBatchStrategy(TestBatch.Strategy strategy, BoostTestAdapterSettings settings)
{
TestBatch.CommandLineArgsBuilder argsBuilder = GetDefaultArguments;
if (strategy != Strategy.TestCase)
{
argsBuilder = GetBatchedTestRunsArguments;
}
switch (strategy)
{
case Strategy.Source: return new SourceTestBatchStrategy(this._testRunnerFactory, settings, argsBuilder);
case Strategy.TestSuite: return new TestSuiteTestBatchStrategy(this._testRunnerFactory, settings, argsBuilder);
case Strategy.TestCase: return new IndividualTestBatchStrategy(this._testRunnerFactory, settings, argsBuilder);
}
return null;
}
public Type TestDiscovererProvisioning(string source, ListContentUse listContent, string externalExtension)
{
ExternalBoostTestRunnerSettings externalSettings = null;
if (!string.IsNullOrEmpty(externalExtension))
{
externalSettings = new ExternalBoostTestRunnerSettings { ExtensionType = new Regex(externalExtension) };
}
BoostTestAdapterSettings settings = new BoostTestAdapterSettings()
{
ExternalTestRunner = externalSettings,
ForceListContent = (listContent == ListContentUse.ForceUse)
};
IBoostTestDiscoverer discoverer = this.DiscovererFactory.GetDiscoverer(source, settings);
return (discoverer == null) ? null : discoverer.GetType();
}
/// <summary>
/// Factory function which returns an appropriate BoostTestRunnerCommandLineArgs structure for batched test runs
/// </summary>
/// <param name="source">The TestCases source</param>
/// <param name="settings">The Boost Test adapter settings currently in use</param>
/// <returns>A BoostTestRunnerCommandLineArgs structure for the provided source</returns>
private BoostTestRunnerCommandLineArgs GetBatchedTestRunsArguments(string source, BoostTestAdapterSettings settings)
{
BoostTestRunnerCommandLineArgs args = GetDefaultArguments(source, settings);
// Disable standard error/standard output capture
args.StandardOutFile = null;
args.StandardErrorFile = null;
// Disable memory leak detection
args.DetectMemoryLeaks = 0;
return args;
}
/// <summary>
/// Filters out any tests which are not intended to run
/// </summary>
/// <param name="settings">Adapter settings which determines test filtering</param>
/// <param name="tests">The entire test corpus</param>
/// <returns>A test corpus which contains only the test which are intended to run</returns>
private static IEnumerable<TestCase> GetTestsToRun(BoostTestAdapterSettings settings, IEnumerable<TestCase> tests)
{
IEnumerable<TestCase> testsToRun = tests;
if (!settings.RunDisabledTests)
{
testsToRun = tests.Where((test) =>
{
foreach (var trait in test.Traits)
{
if ((trait.Name == VSTestModel.StatusTrait) && (trait.Value == VSTestModel.TestEnabled))
{
return true;
}
}
return false;
});
}
return testsToRun;
}
/// <summary>
/// Generates TestResults based on Boost Test result output.
/// </summary>
/// <param name="testRun">The tests which have been executed in the prior test run.</param>
/// <param name="start">The test execution start time.</param>
/// <param name="end">The test execution end time.</param>
/// <param name="settings">boost test adapter settings</param>
/// <returns>A Visual Studio TestResult related to the executed test.</returns>
private static IEnumerable<VSTestResult> GenerateTestResults(TestRun testRun, DateTimeOffset start, DateTimeOffset end, BoostTestAdapterSettings settings)
{
TestResultCollection results = new TestResultCollection();
try
{
results.Parse(testRun.Arguments, settings);
}
catch (XmlException)
{
string text = File.ReadAllText(testRun.Arguments.ReportFile);
if (text.Trim().StartsWith(TestNotFound, StringComparison.Ordinal))
{
return testRun.Tests.Select(GenerateNotFoundResult);
}
else
{
// Re-throw the exception
throw;
}
}
return testRun.Tests.
Select(test =>
{
// Locate the test result associated to the current test
Boost.Results.TestResult result = results[test.FullyQualifiedName];
if (result != null)
{
// Convert the Boost.Test.Result data structure into an equivalent Visual Studio model
VSTestResult vsResult = result.AsVSTestResult(test);
vsResult.StartTime = start;
vsResult.EndTime = end;
return vsResult;
}
return null;
}).
Where(result => (result != null));
}
/// <summary>
/// Tests the contents of a BoostTestAdapterSettings instance, making sure they comply to the default expected values.
/// </summary>
/// <param name="settings">The BoostTestAdapterSettings instance to test</param>
private void AssertDefaultSettings(BoostTestAdapterSettings settings)
{
Assert.That(settings.TimeoutMilliseconds, Is.EqualTo(-1));
Assert.That(settings.FailTestOnMemoryLeak, Is.False);
Assert.That(settings.ConditionalInclusionsFilteringEnabled, Is.True);
Assert.That(settings.LogLevel, Is.EqualTo(LogLevel.TestSuite));
Assert.That(settings.ExternalTestRunner, Is.Null);
}
public IBoostTestDiscoverer GetDiscoverer(string source, BoostTestAdapterSettings settings)
{
throw new NotImplementedException();
}
/// <summary>
/// Returns an IBoostTestDiscoverer based on the provided source.
/// </summary>
/// <param name="factory">The factory to utilise.</param>
/// <param name="source">Source to be associated.</param>
/// <param name="settings">A structure which states particular features of interest in the manufactured product.</param>
/// <returns>An IBoostTestDiscoverer instance or null if one cannot be provided.</returns>
public static IBoostTestDiscoverer GetDiscoverer(this IBoostTestDiscovererFactory factory, string source, Settings.BoostTestAdapterSettings settings)
{
Utility.Code.Require(factory, "factory");
var list = new[] { source };
var results = factory.GetDiscoverers(list, settings);
if (results != null)
{
var result = results.FirstOrDefault(x => x.Sources.Contains(source));
if (result != null)
{
return(result.Discoverer);
}
}
return(null);
}
