public async Task Should_Allow_The_CoverageOutputManager_To_SetProjectCoverageOutputFolder()
{
var mockCoverageToolOutputManager = mocker.GetMock <ICoverageToolOutputManager>();
mockCoverageToolOutputManager.Setup(om => om.SetProjectCoverageOutputFolder(It.IsAny <List <ICoverageProject> >())).
Callback <List <ICoverageProject> >(coverageProjects =>
{
coverageProjects[0].CoverageOutputFolder = "Set by ICoverageToolOutputManager";
});
ICoverageProject coverageProjectAfterCoverageOutputManager = null;
var coverageUtilManager = mocker.GetMock <ICoverageUtilManager>();
coverageUtilManager.Setup(mgr => mgr.RunCoverageAsync(It.IsAny <ICoverageProject>(), It.IsAny <CancellationToken>()))
.Callback <ICoverageProject, CancellationToken>((cp, _) =>
{
coverageProjectAfterCoverageOutputManager = cp;
});
await ReloadSuitableCoverageProject(mockCoverageProject => {
mockCoverageProject.SetupProperty(cp => cp.CoverageOutputFolder);
mockCoverageProject.Setup(p => p.StepAsync("Run Coverage Tool", It.IsAny <Func <ICoverageProject, Task> >())).Callback <string, Func <ICoverageProject, Task> >((_, runCoverTool) =>
{
runCoverTool(mockCoverageProject.Object);
});
});
Assert.AreEqual(coverageProjectAfterCoverageOutputManager.CoverageOutputFolder, "Set by ICoverageToolOutputManager");
}
public async Task RunAsync(ICoverageProject project, CancellationToken cancellationToken)
{
var title = $"Coverlet Run ({project.ProjectName})";
var coverletSettings = GetCoverletSettings(project);
logger.Log($"{title} Arguments {Environment.NewLine}{string.Join($"{Environment.NewLine}", coverletSettings)}");
var result = await processUtil
.ExecuteAsync(GetExecuteRequest(project, string.Join(" ", coverletSettings)), cancellationToken);
/*
* 0 - Success.
* 1 - If any test fails.
* 2 - Coverage percentage is below threshold.
* 3 - Test fails and also coverage percentage is below threshold.
*/
if (result.ExitCode > 3)
{
logger.Log($"{title} Error. Exit code: {result.ExitCode}");
logger.Log($"{title} Error. Output: ", result.Output);
throw new Exception(result.Output);
}
logger.Log(title, result.Output);
}
internal List <string> GetCoverletSettings(ICoverageProject project)
{
var coverletSettings = new List <string>();
coverletSettings.Add($@"""{project.TestDllFile}""");
coverletSettings.Add($@"--format ""cobertura""");
foreach (var value in (project.Settings.Exclude ?? new string[0]).Where(x => !string.IsNullOrWhiteSpace(x)))
{
coverletSettings.Add($@"--exclude ""{value.Replace("\"", "\\\"").Trim(' ', '\'')}""");
}
foreach (var referencedProjectExcludedFromCodeCoverage in project.ExcludedReferencedProjects)
{
coverletSettings.Add($@"--exclude ""[{referencedProjectExcludedFromCodeCoverage}]*""");
}
foreach (var value in (project.Settings.Include ?? new string[0]).Where(x => !string.IsNullOrWhiteSpace(x)))
{
coverletSettings.Add($@"--include ""{value.Replace("\"", "\\\"").Trim(' ', '\'')}""");
}
foreach (var includedReferencedProject in project.IncludedReferencedProjects)
{
coverletSettings.Add($@"--include ""[{includedReferencedProject}]*""");
}
foreach (var value in (project.Settings.ExcludeByFile ?? new string[0]).Where(x => !string.IsNullOrWhiteSpace(x)))
{
coverletSettings.Add($@"--exclude-by-file ""{value.Replace("\"", "\\\"").Trim(' ', '\'')}""");
}
foreach (var value in (project.Settings.ExcludeByAttribute ?? new string[0]).Where(x => !string.IsNullOrWhiteSpace(x)))
{
coverletSettings.Add($@"--exclude-by-attribute ""{value.Replace("\"", "\\\"").Trim(' ', '\'', '[', ']')}""");
}
if (project.Settings.IncludeTestAssembly)
{
coverletSettings.Add("--include-test-assembly");
}
coverletSettings.Add($@"--target ""dotnet""");
coverletSettings.Add($@"--threshold-type line");
coverletSettings.Add($@"--threshold-stat total");
coverletSettings.Add($@"--threshold 0");
coverletSettings.Add($@"--output ""{ project.CoverageOutputFile }""");
var runSettings = !string.IsNullOrWhiteSpace(project.RunSettingsFile) ? $@"--settings """"{project.RunSettingsFile}""""" : default;
coverletSettings.Add($@"--targetargs ""test """"{project.TestDllFile}"""" --nologo --blame {runSettings} --results-directory """"{project.CoverageOutputFolder}"""" --diag """"{project.CoverageOutputFolder}/diagnostics.log"""" """);
return(coverletSettings);
}
public Task RunCoverletAsync(ICoverageProject project, CancellationToken cancellationToken)
{
if (coverletDataCollectorUtil.CanUseDataCollector(project))
{
return(coverletDataCollectorUtil.RunAsync(cancellationToken));
}
return(coverletGlobalUtil.RunAsync(project, cancellationToken));
}
public async Task <IAppOptions> GetSettingsAsync(ICoverageProject coverageProject)
{
var projectDirectory = Path.GetDirectoryName(coverageProject.ProjectFile);
var settingsFilesElements = fccSettingsFilesProvider.Provide(projectDirectory);
var projectSettingsElement = await coverageProjectSettingsProvider.ProvideAsync(coverageProject);
return(settingsMerger.Merge(appOptionsProvider.Get(), settingsFilesElements, projectSettingsElement));
}
public Task <bool> RunCoverletAsync(ICoverageProject project, bool throwError = false)
{
if (coverletDataCollectorUtil.CanUseDataCollector(project))
{
return(coverletDataCollectorUtil.RunAsync(throwError));
}
return(coverletGlobalUtil.RunAsync(project, throwError));
}
private XElement ProjectSettingsElementFromFCCLabelledPropertyGroup(ICoverageProject coverageProject)
{
/*
* <PropertyGroup Label="FineCodeCoverage">
* ...
* </PropertyGroup>
*/
return(coverageProject.ProjectFileXElement.XPathSelectElement($"/PropertyGroup[@Label='{Vsix.Code}']"));
}
private const string zipDirectoryName = "coverlet"; //backwards compatibility
public ExecuteRequest GetRequest(ICoverageProject coverageProject, string coverletSettings)
{
return(new ExecuteRequest
{
FilePath = coverletExePath,
Arguments = coverletSettings,
WorkingDirectory = coverageProject.ProjectOutputFolder
});
}
public async Task <XElement> ProvideAsync(ICoverageProject coverageProject)
{
var settingsElement = ProjectSettingsElementFromFCCLabelledPropertyGroup(coverageProject);
if (settingsElement == null)
{
settingsElement = await vsBuildFCCSettingsProvider.GetSettingsAsync(coverageProject.Id);
}
return(settingsElement);
}
public async Task RunCoverageAsync(ICoverageProject project, CancellationToken cancellationToken)
{
if (project.IsDotNetSdkStyle())
{
await coverletUtil.RunCoverletAsync(project, cancellationToken);
}
else
{
await openCoverUtil.RunOpenCoverAsync(project, cancellationToken);
}
}
private System.Threading.Tasks.Task RunCoverToolAsync(ICoverageProject project)
{
if (project.IsDotNetSdkStyle())
{
return(coverletUtil.RunCoverletAsync(project, true));
}
else
{
return(openCoverUtil.RunOpenCoverAsync(project, true));
}
}
// for now FCCCoverletConsoleExeProvider can return null for exe path
internal ExecuteRequest GetExecuteRequest(ICoverageProject project, string coverletSettings)
{
foreach (var exeProvider in executors)
{
var executeRequest = exeProvider.GetRequest(project, coverletSettings);
if (executeRequest != null)
{
return(executeRequest);
}
}
return(null); //todo change to throw when using zip file
}
public async Task Should_Run_Coverage_ThrowingErrors_But_Safely_With_StepAsync()
{
ICoverageProject coverageProject = null;
await ReloadSuitableCoverageProject(mockCoverageProject => {
coverageProject = mockCoverageProject.Object;
mockCoverageProject.Setup(p => p.StepAsync("Run Coverage Tool", It.IsAny <Func <ICoverageProject, Task> >())).Callback <string, Func <ICoverageProject, Task> >((_, runCoverTool) =>
{
runCoverTool(coverageProject);
});
});
mocker.Verify <ICoverageUtilManager>(coverageUtilManager => coverageUtilManager.RunCoverageAsync(coverageProject, It.IsAny <CancellationToken>()));
}
private async Task RunAndProcessReportAsync(IEnumerable <Uri> resultsUris, string[] expectedCoberturaFiles)
{
autoMocker = new AutoMoqer();
var mockToolFolder = autoMocker.GetMock <IToolFolder>();
mockToolFolder.Setup(tf => tf.EnsureUnzipped(
It.IsAny <string>(),
It.IsAny <string>(),
It.IsAny <ZipDetails>(),
It.IsAny <CancellationToken>()
)).Returns("ZipDestination");
var msCodeCoverageRunSettingsService = autoMocker.Create <MsCodeCoverageRunSettingsService>();
msCodeCoverageRunSettingsService.threadHelper = new TestThreadHelper();
var mockFccEngine = new Mock <IFCCEngine>();
msCodeCoverageRunSettingsService.Initialize("", mockFccEngine.Object, CancellationToken.None);
var mockOperation = new Mock <IOperation>();
mockOperation.Setup(operation => operation.GetRunSettingsDataCollectorResultUri(new Uri(RunSettingsHelper.MsDataCollectorUri))).Returns(resultsUris);
// IsCollecting
var mockTestOperation = new Mock <ITestOperation>();
runSettingsCoverageProject = CreateCoverageProject(".runsettings");
var coverageProjects = new List <ICoverageProject>
{
CreateCoverageProject(null),
runSettingsCoverageProject
};
mockTestOperation.Setup(testOperation => testOperation.GetCoverageProjectsAsync()).ReturnsAsync(coverageProjects);
var mockAppOptionsProvider = autoMocker.GetMock <IAppOptionsProvider>();
mockAppOptionsProvider.Setup(appOptionsProvider => appOptionsProvider.Get()).Returns(new Mock <IAppOptions>().Object);
await msCodeCoverageRunSettingsService.IsCollectingAsync(mockTestOperation.Object);
await msCodeCoverageRunSettingsService.CollectAsync(mockOperation.Object, mockTestOperation.Object);
mockFccEngine.Verify(engine => engine.RunAndProcessReport(
It.Is <string[]>(coberturaFiles => !expectedCoberturaFiles.Except(coberturaFiles).Any() && !coberturaFiles.Except(expectedCoberturaFiles).Any()), It.IsAny <Action>()
)
);
}
public IRunSettingsTemplateReplacements Create(ICoverageProject coverageProject, string testAdapter)
{
var projectSettings = coverageProject.Settings;
var additionalModulePathsExclude = coverageProject.ExcludedReferencedProjects.Select(
rp => MsCodeCoverageRegex.RegexModuleName(rp)).ToList();
if (!projectSettings.IncludeTestAssembly)
{
additionalModulePathsExclude.Add(MsCodeCoverageRegex.RegexEscapePath(coverageProject.TestDllFile));
}
var additionalModulePathsInclude = coverageProject.IncludedReferencedProjects.Select(rp => MsCodeCoverageRegex.RegexModuleName(rp)).ToList();
var settings = new CombinedIncludesExcludesOptions(projectSettings, additionalModulePathsInclude, additionalModulePathsExclude);
return(new RunSettingsTemplateReplacements(settings, coverageProject.CoverageOutputFolder, projectSettings.Enabled.ToString(), testAdapter));
}
public ExecuteRequest GetRequest(ICoverageProject coverageProject, string coverletSettings)
{
if (coverageProject.Settings.CoverletConsoleGlobal)
{
var details = dotNetToolListCoverlet.Global();
if (details == null)
{
return(null);
}
return(new ExecuteRequest
{
FilePath = details.Command,
Arguments = coverletSettings,
WorkingDirectory = coverageProject.ProjectOutputFolder
});
}
return(null);
}
public ExecuteRequest GetRequest(ICoverageProject coverageProject, string coverletSettings)
{
var coverletConsoleCustomPath = coverageProject.Settings.CoverletConsoleCustomPath;
if (string.IsNullOrWhiteSpace(coverletConsoleCustomPath))
{
return(null);
}
if (File.Exists(coverletConsoleCustomPath) && Path.GetExtension(coverletConsoleCustomPath) == ".exe")
{
return(new ExecuteRequest
{
FilePath = coverletConsoleCustomPath,
Arguments = coverletSettings,
WorkingDirectory = coverageProject.ProjectOutputFolder
});
}
return(null);
}
public ExecuteRequest GetRequest(ICoverageProject coverageProject, string coverletSettings)
{
if (coverageProject.Settings.CoverletConsoleLocal)
{
foreach (var configContainingDirectory in dotNetConfigFinder.GetConfigDirectories(coverageProject.ProjectOutputFolder))
{
var coverletToolDetails = dotnetToolListCoverlet.Local(configContainingDirectory);
if (coverletToolDetails != null)
{
return(new ExecuteRequest
{
FilePath = "dotnet",
Arguments = coverletToolDetails.Command + " " + coverletSettings,
WorkingDirectory = configContainingDirectory
});
}
}
return(null);
}
return(null);
}
public bool CanUseDataCollector(ICoverageProject coverageProject)
{
runSettingsCoverletConfiguration = runSettingsCoverletConfigurationFactory.Create();
this.coverageProject = coverageProject;
if (coverageProject.RunSettingsFile != null)
{
var runSettingsXml = fileUtil.ReadAllText(coverageProject.RunSettingsFile);
runSettingsCoverletConfiguration.Read(runSettingsXml);
switch (runSettingsCoverletConfiguration.CoverletDataCollectorState)
{
case CoverletDataCollectorState.Disabled:
return(false);
case CoverletDataCollectorState.Enabled:
return(!OverriddenFromProjectFile());
}
}
return(HasSetUseDataCollectorInProjectFile());
}
public async Task Should_Run_The_Appropriate_Cover_Tool_Based_On_IsDotNetSdkStyle(bool isDotNetSdkStyle)
{
Task waitForCoverage = null;
ICoverageProject coverageProject = null;
await ReloadSuitableCoverageProject(mockCoverageProject => {
coverageProject = mockCoverageProject.Object;
mockCoverageProject.Setup(p => p.IsDotNetSdkStyle()).Returns(isDotNetSdkStyle);
mockCoverageProject.Setup(p => p.StepAsync("Run Coverage Tool", It.IsAny <Func <ICoverageProject, Task> >())).Callback <string, Func <ICoverageProject, Task> >((_, runCoverTool) =>
{
waitForCoverage = runCoverTool(coverageProject);
});
});
if (isDotNetSdkStyle)
{
mocker.Verify <ICoverletUtil>(coverlet => coverlet.RunCoverletAsync(coverageProject, true));
}
else
{
mocker.Verify <IOpenCoverUtil>(openCover => openCover.RunOpenCoverAsync(coverageProject, true));
}
}
public async Task <bool> RunAsync(ICoverageProject project, bool throwError = false)
{
var title = $"Coverlet Run ({project.ProjectName})";
var coverletSettings = GetCoverletSettings(project);
logger.Log($"{title} Arguments {Environment.NewLine}{string.Join($"{Environment.NewLine}", coverletSettings)}");
var result = await processUtil
.ExecuteAsync(GetExecuteRequest(project, string.Join(" ", coverletSettings)));
if (result != null)
{
/*
* 0 - Success.
* 1 - If any test fails.
* 2 - Coverage percentage is below threshold.
* 3 - Test fails and also coverage percentage is below threshold.
*/
if (result.ExitCode > 3)
{
if (throwError)
{
throw new Exception(result.Output);
}
logger.Log($"{title} Error", result.Output);
return(false);
}
logger.Log(title, result.Output);
return(true);
}
return(false);
}
public string CoverageToolName(ICoverageProject project)
{
return(project.IsDotNetSdkStyle() ? "Coverlet" : "OpenCover");
}
public async Task RunOpenCoverAsync(ICoverageProject project, CancellationToken cancellationToken)
{
var title = $"OpenCover Run ({project.ProjectName})";
var opencoverSettings = new List <string>();
opencoverSettings.Add($@" -mergebyhash ");
opencoverSettings.Add($@" -hideskipped:all ");
{
// -register:
var registerValue = "path32";
if (project.Is64Bit)
{
registerValue = "path64";
}
opencoverSettings.Add($@" -register:{registerValue} ");
}
{
// -target:
opencoverSettings.Add($@" ""-target:{msTestPlatformUtil.MsTestPlatformExePath}"" ");
}
{
// -filter:
var filters = new List <string>();
var defaultFilter = "+[*]*";
foreach (var value in (project.Settings.Include ?? new string[0]).Where(x => !string.IsNullOrWhiteSpace(x)))
{
filters.Add($@"+{value.Replace("\"", "\\\"").Trim(' ', '\'')}");
}
foreach (var includedReferencedProject in project.IncludedReferencedProjects)
{
filters.Add($@"+[{includedReferencedProject}]*");
}
if (!filters.Any())
{
filters.Add(defaultFilter);
}
foreach (var value in (project.Settings.Exclude ?? new string[0]).Where(x => !string.IsNullOrWhiteSpace(x)))
{
filters.Add($@"-{value.Replace("\"", "\\\"").Trim(' ', '\'')}");
}
foreach (var referencedProjectExcludedFromCodeCoverage in project.ExcludedReferencedProjects)
{
filters.Add($@"-[{referencedProjectExcludedFromCodeCoverage}]*");
}
if (filters.Any(x => !x.Equals(defaultFilter)))
{
opencoverSettings.Add($@" ""-filter:{string.Join(" ", filters.Distinct())}"" ");
}
}
{
// -excludebyfile:
var excludes = new List <string>();
foreach (var value in (project.Settings.ExcludeByFile ?? new string[0]).Where(x => !string.IsNullOrWhiteSpace(x)))
{
excludes.Add(value.Replace("\"", "\\\"").Trim(' ', '\''));
}
if (excludes.Any())
{
opencoverSettings.Add($@" ""-excludebyfile:{string.Join(";", excludes)}"" ");
}
}
{
// -excludebyattribute:
var excludes = new List <string>()
{
// coverlet knows these implicitly
"ExcludeFromCoverage",
"ExcludeFromCodeCoverage"
};
foreach (var value in (project.Settings.ExcludeByAttribute ?? new string[0]).Where(x => !string.IsNullOrWhiteSpace(x)))
{
excludes.Add(value.Replace("\"", "\\\"").Trim(' ', '\''));
}
foreach (var exclude in excludes.ToArray())
{
var excludeAlternateName = default(string);
if (exclude.EndsWith("Attribute", StringComparison.OrdinalIgnoreCase))
{
// remove 'Attribute' suffix
excludeAlternateName = exclude.Substring(0, exclude.IndexOf("Attribute", StringComparison.OrdinalIgnoreCase));
}
else
{
// add 'Attribute' suffix
excludeAlternateName = $"{exclude}Attribute";
}
excludes.Add(excludeAlternateName);
}
excludes = excludes.Distinct(StringComparer.OrdinalIgnoreCase).OrderBy(x => x).ToList();
if (excludes.Any())
{
opencoverSettings.Add($@" ""-excludebyattribute:(*.{string.Join(")|(*.", excludes)})"" ");
}
}
if (!project.Settings.IncludeTestAssembly)
{
// deleting the pdb of the test assembly seems to work; this is a VERY VERY shameful hack :(
var testDllPdbFile = Path.Combine(project.ProjectOutputFolder, Path.GetFileNameWithoutExtension(project.TestDllFile)) + ".pdb";
File.Delete(testDllPdbFile);
// filtering out the test-assembly blows up the entire process and nothing gets instrumented or analysed
//var nameOnlyOfDll = Path.GetFileNameWithoutExtension(project.TestDllFileInWorkFolder);
//filters.Add($@"-[{nameOnlyOfDll}]*");
}
var runSettings = !string.IsNullOrWhiteSpace(project.RunSettingsFile) ? $@"/Settings:\""{project.RunSettingsFile}\""" : default;
opencoverSettings.Add($@" ""-targetargs:\""{project.TestDllFile}\"" {runSettings}"" ");
opencoverSettings.Add($@" ""-output:{ project.CoverageOutputFile }"" ");
logger.Log($"{title} Arguments {Environment.NewLine}{string.Join($"{Environment.NewLine}", opencoverSettings)}");
var result = await processUtil
.ExecuteAsync(new ExecuteRequest
{
FilePath = GetOpenCoverExePath(project.Settings.OpenCoverCustomPath),
Arguments = string.Join(" ", opencoverSettings),
WorkingDirectory = project.ProjectOutputFolder
}, cancellationToken);
if (result.ExitCode != 0)
{
throw new Exception(result.Output);
}
logger.Log(title, result.Output);
}
internal static string GeneratedProjectRunSettingsFilePath(ICoverageProject coverageProject)
{
return(Path.Combine(coverageProject.CoverageOutputFolder, $"{coverageProject.ProjectName}-{fccGeneratedRunSettingsSuffix}.runsettings"));
}
private ITemplateReplacementResult ReplaceTemplate(ICoverageProject coverageProject, string runSettingsTemplate, string fccMsTestAdapterPath)
{
var replacements = runSettingsTemplateReplacementsFactory.Create(coverageProject, fccMsTestAdapterPath);
return(this.runSettingsTemplate.ReplaceTemplate(runSettingsTemplate, replacements, coverageProject.IsDotNetFramework));
}
