public void Test_MultipleTargetFrameworkReport_CoverletOutput_Folder_FileNameWithExtension_SpecifyFramework()
{
using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject();
string[] targetFrameworks = new string[] { "netcoreapp2.2", "netcoreapp2.1" };
UpdateProjectTargetFramework(clonedTemplateProject, targetFrameworks);
Assert.True(clonedTemplateProject.IsMultipleTargetFramework());
string[] frameworks = clonedTemplateProject.GetTargetFrameworks();
Assert.Equal(2, frameworks.Length);
string framework = frameworks.FirstOrDefault();
Assert.True(DotnetCli($"test -f {framework} \"{clonedTemplateProject.ProjectRootPath}\" /p:CollectCoverage=true /p:Include=\"[{ClonedTemplateProject.AssemblyName}]*DeepThought\" /p:IncludeTestAssembly=true /p:CoverletOutput=\"{clonedTemplateProject.ProjectRootPath}\"\\file.ext", out string standardOutput, out string standardError, clonedTemplateProject.ProjectRootPath !), standardOutput);
Assert.Contains("Test Run Successful.", standardOutput);
Assert.Contains("| coverletsamplelib.integration.template | 100% | 100% | 100% |", standardOutput);
foreach (string targetFramework in targetFrameworks)
{
if (framework == targetFramework)
{
Assert.True(File.Exists(Path.Combine(clonedTemplateProject.ProjectRootPath, $"file.{targetFramework}.ext")));
}
else
{
Assert.False(File.Exists(Path.Combine(clonedTemplateProject.ProjectRootPath, $"file.{targetFramework}.ext")));
}
}
AssertCoverage(clonedTemplateProject, "file.*.ext");
}
private ClonedTemplateProject PrepareTemplateProject()
{
ClonedTemplateProject clonedTemplateProject = CloneTemplateProject();
UpdateNugeConfigtWithLocalPackageFolder(clonedTemplateProject.ProjectRootPath !);
AddCoverletCollectosRef(clonedTemplateProject.ProjectRootPath !);
return(clonedTemplateProject);
}
public void TestMsbuild_CoverletOutput_Folder_FileNameWithDoubleExtension()
{
using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject();
Assert.True(DotnetCli($"test \"{clonedTemplateProject.ProjectRootPath}\" /p:CollectCoverage=true /p:Include=\"[{ClonedTemplateProject.AssemblyName}]*DeepThought\" /p:IncludeTestAssembly=true /p:CoverletOutput=\"{clonedTemplateProject.ProjectRootPath}\"\\file.ext1.ext2", out string standardOutput, out string standardError), standardOutput);
Assert.Contains("Test Run Successful.", standardOutput);
Assert.Contains("| coverletsamplelib.integration.template | 100% | 100% | 100% |", standardOutput);
Assert.True(File.Exists(Path.Combine(clonedTemplateProject.ProjectRootPath, "file.ext1.ext2")));
AssertCoverage(clonedTemplateProject, "file.ext1.ext2");
}
public void TestVsTest_Test()
{
using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject();
Assert.True(DotnetCli($"test -c {_buildConfiguration} \"{clonedTemplateProject.ProjectRootPath}\" --collect:\"XPlat Code Coverage\" --diag:{Path.Combine(clonedTemplateProject.ProjectRootPath, "log.txt")}", out string standardOutput, out string standardError, clonedTemplateProject.ProjectRootPath !), standardOutput);
// We don't have any result to check because tests and code to instrument are in same assembly so we need to pass
// IncludeTestAssembly=true we do it in other test
Assert.Contains("Passed!", standardOutput);
AssertCollectorsInjection(clonedTemplateProject);
}
private protected override void AssertCollectorsInjection(ClonedTemplateProject clonedTemplateProject)
{
// Check out/in process collectors injection
Assert.Contains("[coverlet]", File.ReadAllText(clonedTemplateProject.GetFiles("log.datacollector.*.txt").Single()));
// There is a bug in this SDK version https://github.com/microsoft/vstest/pull/2221
// in-proc coverlet.collector.dll collector with version != 1.0.0.0 won't be loaded
// Assert.Contains("[coverlet]", File.ReadAllText(clonedTemplateProject.GetFiles("log.host.*.txt").Single()));
}
public void TestMsbuild_NoCoverletOutput()
{
using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject();
Assert.True(DotnetCli($"test -c {_buildConfiguration} \"{clonedTemplateProject.ProjectRootPath}\" /p:CollectCoverage=true /p:Include=\"[{ClonedTemplateProject.AssemblyName}]*DeepThought\" /p:IncludeTestAssembly=true", out string standardOutput, out string standardError), standardOutput);
Assert.Contains("Passed!", standardOutput);
Assert.Contains("| coverletsamplelib.integration.template | 100% | 100% | 100% |", standardOutput);
Assert.True(File.Exists(Path.Combine(clonedTemplateProject.ProjectRootPath, "coverage.json")));
AssertCoverage(clonedTemplateProject);
}
public void TestVsTest_Test_Settings()
{
using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject();
string runSettingsPath = AddCollectorRunsettingsFile(clonedTemplateProject.ProjectRootPath !);
Assert.True(DotnetCli($"test \"{clonedTemplateProject.ProjectRootPath}\" --collect:\"XPlat Code Coverage\" --settings \"{runSettingsPath}\" --diag:{Path.Combine(clonedTemplateProject.ProjectRootPath, "log.txt")}", out string standardOutput, out string standardError), standardOutput);
Assert.Contains("Test Run Successful.", standardOutput);
AssertCoverage(clonedTemplateProject);
AssertCollectorsInjection(clonedTemplateProject);
}
public void TestMsbuild_CoverletOutput_Folder_FileNameExtension_SpecifyFramework()
{
using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject();
Assert.False(clonedTemplateProject.IsMultipleTargetFramework());
string framework = clonedTemplateProject.GetTargetFrameworks().Single();
Assert.True(DotnetCli($"test -f {framework} \"{clonedTemplateProject.ProjectRootPath}\" /p:CollectCoverage=true /p:Include=\"[{ClonedTemplateProject.AssemblyName}]*DeepThought\" /p:IncludeTestAssembly=true /p:CoverletOutput=\"{clonedTemplateProject.ProjectRootPath}\"\\file.ext", out string standardOutput, out string standardError), standardOutput);
Assert.Contains("Test Run Successful.", standardOutput);
Assert.Contains("| coverletsamplelib.integration.template | 100% | 100% | 100% |", standardOutput);
Assert.True(File.Exists(Path.Combine(clonedTemplateProject.ProjectRootPath, "file.ext")));
AssertCoverage(clonedTemplateProject, "file.ext");
}
public void StandAlone()
{
using ClonedTemplateProject clonedTemplateProject = CloneTemplateProject();
UpdateNugeConfigtWithLocalPackageFolder(clonedTemplateProject.ProjectRootPath !);
string coverletToolCommandPath = InstallTool(clonedTemplateProject.ProjectRootPath !);
DotnetCli($"build {clonedTemplateProject.ProjectRootPath}", out string standardOutput, out string standardError);
string publishedTestFile = clonedTemplateProject.GetFiles("*" + ClonedTemplateProject.AssemblyName + ".dll").Single(f => !f.Contains("obj") && !f.Contains("ref"));
RunCommand(coverletToolCommandPath, $"\"{Path.GetDirectoryName(publishedTestFile)}\" --target \"dotnet\" --targetargs \"{publishedTestFile}\" --output \"{clonedTemplateProject.ProjectRootPath}\"\\", out standardOutput, out standardError);
Assert.Contains("Hello World!", standardOutput);
AssertCoverage(clonedTemplateProject, standardOutput: standardOutput);
}
private ClonedTemplateProject PrepareTemplateProject()
{
if (TestSDKVersion is null)
{
throw new ArgumentNullException("Invalid TestSDKVersion");
}
ClonedTemplateProject clonedTemplateProject = CloneTemplateProject(testSDKVersion: TestSDKVersion);
UpdateNugeConfigtWithLocalPackageFolder(clonedTemplateProject.ProjectRootPath !);
AddCoverletCollectosRef(clonedTemplateProject.ProjectRootPath !);
return(clonedTemplateProject);
}
public void DotnetTool()
{
using ClonedTemplateProject clonedTemplateProject = CloneTemplateProject();
UpdateNugeConfigtWithLocalPackageFolder(clonedTemplateProject.ProjectRootPath !);
string coverletToolCommandPath = InstallTool(clonedTemplateProject.ProjectRootPath !);
DotnetCli($"build {clonedTemplateProject.ProjectRootPath}", out string standardOutput, out string standardError);
string publishedTestFile = clonedTemplateProject.GetFiles("*" + ClonedTemplateProject.AssemblyName + ".dll").Single(f => !f.Contains("obj"));
RunCommand(coverletToolCommandPath, $"\"{publishedTestFile}\" --target \"dotnet\" --targetargs \"test {Path.Combine(clonedTemplateProject.ProjectRootPath, ClonedTemplateProject.ProjectFileName)} --no-build\" --include-test-assembly --output \"{clonedTemplateProject.ProjectRootPath}\"\\", out standardOutput, out standardError);
Assert.Contains("Test Run Successful.", standardOutput);
AssertCoverage(clonedTemplateProject, standardOutput: standardOutput);
}
private protected void PinSDK(ClonedTemplateProject project, string sdkVersion)
{
if (string.IsNullOrEmpty(sdkVersion))
{
throw new ArgumentException("Invalid sdkVersion", nameof(sdkVersion));
}
if (!File.Exists(project.ProjectFileNamePath))
{
throw new FileNotFoundException("coverlet.integration.template.csproj not found", "coverlet.integration.template.csproj");
}
File.WriteAllText(Path.Combine(project.ProjectRootPath, "global.json"), $"{{ \"sdk\": {{ \"version\": \"{sdkVersion}\" }} }}");
}
public void TestVsTest_VsTest_Settings()
{
using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject();
string runSettingsPath = AddCollectorRunsettingsFile(clonedTemplateProject.ProjectRootPath !);
Assert.True(DotnetCli($"publish \"{clonedTemplateProject.ProjectRootPath}\"", out string standardOutput, out string standardError), standardOutput);
string publishedTestFile = clonedTemplateProject.GetFiles("*" + ClonedTemplateProject.AssemblyName + ".dll").Single(f => f.Contains("publish"));
Assert.NotNull(publishedTestFile);
Assert.True(DotnetCli($"vstest \"{publishedTestFile}\" --collect:\"XPlat Code Coverage\" --ResultsDirectory:\"{clonedTemplateProject.ProjectRootPath}\" /settings:\"{runSettingsPath}\" --diag:{Path.Combine(clonedTemplateProject.ProjectRootPath, "log.txt")}", out standardOutput, out standardError), standardOutput);
Assert.Contains("Test Run Successful.", standardOutput);
AssertCoverage(clonedTemplateProject);
AssertCollectorsInjection(clonedTemplateProject);
}
public void TestVsTest_VsTest()
{
using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject();
string runSettingsPath = AddCollectorRunsettingsFile(clonedTemplateProject.ProjectRootPath !);
Assert.True(DotnetCli($"publish {clonedTemplateProject.ProjectRootPath}", out string standardOutput, out string standardError), standardOutput);
string publishedTestFile = clonedTemplateProject.GetFiles("*" + ClonedTemplateProject.AssemblyName + ".dll").Single(f => f.Contains("publish"));
Assert.NotNull(publishedTestFile);
Assert.True(DotnetCli($"vstest \"{publishedTestFile}\" --collect:\"XPlat Code Coverage\" --diag:{Path.Combine(clonedTemplateProject.ProjectRootPath, "log.txt")}", out standardOutput, out standardError), standardOutput);
// We don't have any result to check because tests and code to instrument are in same assembly so we need to pass
// IncludeTestAssembly=true we do it in other test
Assert.Contains("Test Run Successful.", standardOutput);
AssertCollectorsInjection(clonedTemplateProject);
}
public void Test_MultipleTargetFrameworkReport_NoCoverletOutput()
{
using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject();
string[] targetFrameworks = new string[] { "netcoreapp2.2", "netcoreapp2.1" };
UpdateProjectTargetFramework(clonedTemplateProject, targetFrameworks);
Assert.True(DotnetCli($"test \"{clonedTemplateProject.ProjectRootPath}\" /p:CollectCoverage=true /p:Include=\"[{ClonedTemplateProject.AssemblyName}]*DeepThought\" /p:IncludeTestAssembly=true", out string standardOutput, out string standardError, clonedTemplateProject.ProjectRootPath !), standardOutput);
Assert.Contains("Test Run Successful.", standardOutput);
Assert.Contains("| coverletsamplelib.integration.template | 100% | 100% | 100% |", standardOutput);
foreach (string targetFramework in targetFrameworks)
{
Assert.True(File.Exists(Path.Combine(clonedTemplateProject.ProjectRootPath, $"coverage.{targetFramework}.json")));
}
AssertCoverage(clonedTemplateProject, "coverage.*.json");
}
public void Test_MultipleTargetFrameworkReport_CoverletOutput_Folder_FileNameWithDoubleExtension()
{
using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject();
string[] targetFrameworks = new string[] { "net5.0", "netcoreapp3.1" };
UpdateProjectTargetFramework(clonedTemplateProject, targetFrameworks);
Assert.True(DotnetCli($"test -c {_buildConfiguration} \"{clonedTemplateProject.ProjectRootPath}\" /p:CollectCoverage=true /p:Include=\"[{ClonedTemplateProject.AssemblyName}]*DeepThought\" /p:IncludeTestAssembly=true /p:CoverletOutput=\"{clonedTemplateProject.ProjectRootPath}\"\\file.ext1.ext2", out string standardOutput, out string standardError, clonedTemplateProject.ProjectRootPath !), standardOutput);
Assert.Contains("Passed!", standardOutput);
Assert.Contains("| coverletsamplelib.integration.template | 100% | 100% | 100% |", standardOutput);
foreach (string targetFramework in targetFrameworks)
{
Assert.True(File.Exists(Path.Combine(clonedTemplateProject.ProjectRootPath, $"file.ext1.{targetFramework}.ext2")));
}
AssertCoverage(clonedTemplateProject, "file.ext1.*.ext2");
}
private protected void AssertCoverage(ClonedTemplateProject clonedTemplateProject, string filter = "coverage.json")
{
bool coverageChecked = false;
foreach (string coverageFile in clonedTemplateProject.GetFiles(filter))
{
JsonConvert.DeserializeObject <Modules>(File.ReadAllText(coverageFile))
.Document("DeepThought.cs")
.Class("Coverlet.Integration.Template.DeepThought")
.Method("System.Int32 Coverlet.Integration.Template.DeepThought::AnswerToTheUltimateQuestionOfLifeTheUniverseAndEverything()")
.AssertLinesCovered((6, 1), (7, 1), (8, 1));
coverageChecked = true;
}
Assert.True(coverageChecked, "Coverage check fail");
}
public void Test_MultipleTargetFrameworkReport_CoverletOutput_Folder()
{
using ClonedTemplateProject clonedTemplateProject = PrepareTemplateProject();
string[] targetFrameworks = new string[] { "netcoreapp3.1", "net5.0" };
UpdateProjectTargetFramework(clonedTemplateProject, targetFrameworks);
Assert.True(DotnetCli($"test -c {_buildConfiguration} \"{clonedTemplateProject.ProjectRootPath}\" /p:CollectCoverage=true /p:Include=\"[{ClonedTemplateProject.AssemblyName}]*DeepThought\" /p:IncludeTestAssembly=true /p:CoverletOutput=\"{clonedTemplateProject.ProjectRootPath}\"\\", out string standardOutput, out string standardError, clonedTemplateProject.ProjectRootPath !), standardOutput);
Assert.Contains("Passed!", standardOutput);
Assert.Contains("| coverletsamplelib.integration.template | 100% | 100% | 100% |", standardOutput);
foreach (string targetFramework in targetFrameworks)
{
string fileToCheck = Path.Combine(clonedTemplateProject.ProjectRootPath, $"coverage.{targetFramework}.json");
Assert.True(File.Exists(fileToCheck), $"Expected file '{fileToCheck}'\nOutput:\n{standardOutput}");
}
AssertCoverage(clonedTemplateProject, "coverage.*.json");
}
public void DotnetTool()
{
if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
// Disabled for the moment on unix system we get an exception(folder access denied) during tool installation
return;
}
using ClonedTemplateProject clonedTemplateProject = CloneTemplateProject();
UpdateNugeConfigtWithLocalPackageFolder(clonedTemplateProject.ProjectRootPath !);
string coverletToolCommandPath = InstallTool(clonedTemplateProject.ProjectRootPath !);
DotnetCli($"build {clonedTemplateProject.ProjectRootPath}", out string standardOutput, out string standardError);
string publishedTestFile = clonedTemplateProject.GetFiles("*" + ClonedTemplateProject.AssemblyName + ".dll").Single(f => !f.Contains("obj"));
RunCommand(coverletToolCommandPath, $"\"{publishedTestFile}\" --target \"dotnet\" --targetargs \"test {Path.Combine(clonedTemplateProject.ProjectRootPath, ClonedTemplateProject.ProjectFileName)} --no-build\" --include-test-assembly --output \"{clonedTemplateProject.ProjectRootPath}\"\\", out standardOutput, out standardError);
Assert.Contains("Test Run Successful.", standardOutput);
AssertCoverage(clonedTemplateProject, standardOutput: standardOutput);
}
private protected void UpdateProjectTargetFramework(ClonedTemplateProject project, params string[] targetFrameworks)
{
if (targetFrameworks is null || targetFrameworks.Length == 0)
{
throw new ArgumentException("Invalid targetFrameworks", nameof(targetFrameworks));
}
if (!File.Exists(project.ProjectFileNamePath))
{
throw new FileNotFoundException("coverlet.integration.template.csproj not found", "coverlet.integration.template.csproj");
}
XDocument xml;
using (var csprojStream = File.OpenRead(project.ProjectFileNamePath))
{
xml = XDocument.Load(csprojStream);
}
xml.Element("Project") !
.Element("PropertyGroup") !
.Element("TargetFramework") !
.Remove();
XElement targetFrameworkElement;
if (targetFrameworks.Length == 1)
{
targetFrameworkElement = new XElement("TargetFramework", targetFrameworks[0]);
}
else
{
targetFrameworkElement = new XElement("TargetFrameworks", string.Join(';', targetFrameworks));
}
xml.Element("Project") !.Element("PropertyGroup") !.Add(targetFrameworkElement);
xml.Save(project.ProjectFileNamePath);
}
private protected virtual void AssertCollectorsInjection(ClonedTemplateProject clonedTemplateProject)
{
// Check out/in process collectors injection
Assert.Contains("[coverlet]Initializing CoverletCoverageDataCollector with configuration:", File.ReadAllText(clonedTemplateProject.GetFiles("log.datacollector.*.txt").Single()));
Assert.Contains("[coverlet]Initialize CoverletInProcDataCollector", File.ReadAllText(clonedTemplateProject.GetFiles("log.host.*.txt").Single()));
}
private void AssertCollectorsInjection(ClonedTemplateProject clonedTemplateProject)
{
// Check out/in process collectors injection
Assert.Contains("[coverlet]", File.ReadAllText(clonedTemplateProject.GetFiles("log.datacollector.*.txt").Single()));
Assert.Contains("[coverlet]", File.ReadAllText(clonedTemplateProject.GetFiles("log.host.*.txt").Single()));
}