public void TestWillNotBuildTheProjectIfNoBuildArgsIsGiven() { // Copy and restore VSTestCore project in output directory of project dotnet-vstest.Tests var testProjectDirectory = this.CopyAndRestoreVSTestDotNetCoreTestApp("5"); string configuration = Environment.GetEnvironmentVariable("CONFIGURATION") ?? "Debug"; string expectedError = Path.Combine(testProjectDirectory, "bin", configuration, "netcoreapp3.0", "VSTestCore.dll"); expectedError = "The test source file " + "\"" + expectedError + "\"" + " provided was not found."; // Call test CommandResult result = new DotnetTestCommand() .WithWorkingDirectory(testProjectDirectory) .ExecuteWithCapturedOutput("--no-build -v:m"); // Verify if (!DotnetUnderTest.IsLocalized()) { result.StdOut.Should().NotContain("Restore"); // https://github.com/dotnet/sdk/issues/3684 // Disable expected error check, it is sometimes giving the following error: // The argument /opt/code/artifacts-ubuntu.18.04/tmp/Debug/bin/5/VSTestCore/bin/Debug/netcoreapp3.0/VSTestCore.dll is invalid. Please use the /help option to check the list of valid arguments //result.StdErr.Should().Contain(expectedError); } result.ExitCode.Should().Be(1); }
public void ItTestsWithTheSpecifiedRuntimeOption() { var testInstance = TestAssets.Get("XunitCore") .CreateInstance() .WithSourceFiles(); var rootPath = testInstance.Root.FullName; var rid = DotnetLegacyRuntimeIdentifiers.InferLegacyRestoreRuntimeIdentifier(); new BuildCommand() .WithWorkingDirectory(rootPath) .ExecuteWithCapturedOutput($"--runtime {rid}") .Should() .Pass() .And.NotHaveStdErr(); var result = new DotnetTestCommand() .WithWorkingDirectory(rootPath) .ExecuteWithCapturedOutput($"{TestBase.ConsoleLoggerOutputNormal} --no-build --runtime {rid}"); result .Should() .NotHaveStdErrContaining("MSB1001") .And .HaveStdOutContaining(rid); if (!DotnetUnderTest.IsLocalized()) { result.StdOut.Should().Contain("Total tests: 2"); result.StdOut.Should().Contain("Passed: 1"); result.StdOut.Should().Contain("Failed: 1"); } result.ExitCode.Should().Be(1); }
public void XunitSingleTFM() { // Copy XunitCore project in output directory of project dotnet-vstest.Tests string testAppName = "XunitCore"; var testInstance = TestAssets.Get(testAppName) .CreateInstance("4") .WithSourceFiles(); var testProjectDirectory = testInstance.Root.FullName; // Restore project XunitCore new RestoreCommand() .WithWorkingDirectory(testProjectDirectory) .Execute() .Should() .Pass(); // Call test CommandResult result = new DotnetTestCommand() .WithWorkingDirectory(testProjectDirectory) .ExecuteWithCapturedOutput(TestBase.ConsoleLoggerOutputNormal); // Verify if (!DotnetUnderTest.IsLocalized()) { result.StdOut.Should().Contain("Total tests: 2. Passed: 1. Failed: 1. Skipped: 0."); result.StdOut.Should().Contain("Passed TestNamespace.VSTestXunitTests.VSTestXunitPassTest"); result.StdOut.Should().Contain("Failed TestNamespace.VSTestXunitTests.VSTestXunitFailTest"); } result.ExitCode.Should().Be(1); }
public void TestsFromAGivenContainerShouldRunWithExpectedOutput() { var testAppName = "VSTestCore"; var testRoot = TestAssets.Get(testAppName) .CreateInstance() .WithSourceFiles() .WithRestoreFiles() .Root; var configuration = Environment.GetEnvironmentVariable("CONFIGURATION") ?? "Debug"; new BuildCommand() .WithWorkingDirectory(testRoot) .Execute() .Should().Pass(); var outputDll = testRoot .GetDirectory("bin", configuration, "netcoreapp3.0") .GetFile($"{testAppName}.dll"); var argsForVstest = $"\"{outputDll.FullName}\" --logger:console;verbosity=normal"; // Call vstest var result = new VSTestCommand().ExecuteWithCapturedOutput(argsForVstest); if (!DotnetUnderTest.IsLocalized()) { result.StdOut .Should().Contain("Total tests: 2. Passed: 1. Failed: 1. Skipped: 0.") .And.Contain("Passed VSTestPassTest") .And.Contain("Failed VSTestFailTest"); } result.ExitCode.Should().Be(1); }
public void ItAcceptsMultipleLoggersAsCliArguments() { // Copy and restore VSTestCore project in output directory of project dotnet-vstest.Tests var testProjectDirectory = this.CopyAndRestoreVSTestDotNetCoreTestApp("10"); var trxFileNamePattern = "custom*.trx"; string trxLoggerDirectory = Path.Combine(testProjectDirectory, "RD"); // Delete trxLoggerDirectory if it exist if (Directory.Exists(trxLoggerDirectory)) { Directory.Delete(trxLoggerDirectory, true); } // Call test with logger enable CommandResult result = new DotnetTestCommand() .WithWorkingDirectory(testProjectDirectory) .ExecuteWithCapturedOutput("--logger \"trx;logfilename=custom.trx\" --logger console;verbosity=normal -- RunConfiguration.ResultsDirectory=" + trxLoggerDirectory); // Verify if (!DotnetUnderTest.IsLocalized()) { // We append current date time to trx file name, hence modifying this check Assert.True(Directory.EnumerateFiles(trxLoggerDirectory, trxFileNamePattern).Any()); result.StdOut.Should().Contain("\u221a VSTestPassTest"); result.StdOut.Should().Contain("X VSTestFailTest"); } // Cleanup trxLoggerDirectory if it exist if (Directory.Exists(trxLoggerDirectory)) { Directory.Delete(trxLoggerDirectory, true); } }
public void MStestMultiTFM() { var testProjectDirectory = TestAssets.Get("VSTestMulti") .CreateInstance("1") .WithSourceFiles() .WithNuGetConfig(new RepoDirectoriesProvider().TestPackages) .Root; var runtime = DotnetLegacyRuntimeIdentifiers.InferLegacyRestoreRuntimeIdentifier(); new RestoreCommand() .WithWorkingDirectory(testProjectDirectory) .WithRuntime(runtime) .Execute() .Should().Pass(); var result = new DotnetTestCommand() .WithWorkingDirectory(testProjectDirectory) .WithRuntime(runtime) .ExecuteWithCapturedOutput(TestBase.ConsoleLoggerOutputNormal); if (!DotnetUnderTest.IsLocalized()) { result.StdOut .Should().Contain("Total tests: 3. Passed: 2. Failed: 1. Skipped: 0.", "because .NET 4.6 tests will pass") .And.Contain("Passed VSTestPassTestDesktop", "because .NET 4.6 tests will pass") .And.Contain("Total tests: 3. Passed: 1. Failed: 2. Skipped: 0.", "because netcoreapp2.0 tests will fail") .And.Contain("Failed VSTestFailTestNetCoreApp", "because netcoreapp2.0 tests will fail"); } result.ExitCode.Should().Be(1); }
public void ItCreatesCoverageFileInResultsDirectory() { var testProjectDirectory = this.CopyAndRestoreVSTestDotNetCoreTestApp("12"); string resultsDirectory = Path.Combine(testProjectDirectory, "RD"); // Delete resultsDirectory if it exist if (Directory.Exists(resultsDirectory)) { Directory.Delete(resultsDirectory, true); } // Call test CommandResult result = new DotnetTestCommand() .WithWorkingDirectory(testProjectDirectory) .ExecuteWithCapturedOutput( "--collect \"Code Coverage\" " + "--results-directory " + resultsDirectory); // Verify test results if (!DotnetUnderTest.IsLocalized()) { result.StdOut.Should().Contain("Total tests: 2. Passed: 1. Failed: 1. Skipped: 0."); } // Verify coverage file. DirectoryInfo d = new DirectoryInfo(resultsDirectory); FileInfo[] coverageFileInfos = d.GetFiles("*.coverage", SearchOption.AllDirectories); Assert.Equal(1, coverageFileInfos.Length); result.ExitCode.Should().Be(1); }
public void ItImplicitlyRestoresAProjectWhenTesting() { string testAppName = "VSTestCore"; var testInstance = TestAssets.Get(testAppName) .CreateInstance() .WithSourceFiles(); var testProjectDirectory = testInstance.Root.FullName; CommandResult result = new DotnetTestCommand() .WithWorkingDirectory(testProjectDirectory) .ExecuteWithCapturedOutput(TestBase.ConsoleLoggerOutputNormal); if (!DotnetUnderTest.IsLocalized()) { result.StdOut.Should().Contain("Total tests: 2"); result.StdOut.Should().Contain("Passed: 1"); result.StdOut.Should().Contain("Total tests: 2"); result.StdOut.Should().Contain("Passed: 1"); result.StdOut.Should().Contain("Failed: 1"); result.StdOut.Should().Contain("\u221a VSTestPassTest"); result.StdOut.Should().Contain("X VSTestFailTest"); } result.ExitCode.Should().Be(1); }
public void WhenSwitchIsSkippedThenItPrintsError() { var cmd = new DotnetCommand().Execute("new Web1.1"); cmd.ExitCode.Should().NotBe(0); if (!DotnetUnderTest.IsLocalized()) { cmd.StdErr.Should().StartWith("No templates matched the input template name: Web1.1."); } }
public void WhenTemplateNameIsNotUniquelyMatchedThenItIndicatesProblemToUser() { var cmd = new DotnetCommand().Execute("new c"); cmd.ExitCode.Should().NotBe(0); if (!DotnetUnderTest.IsLocalized()) { cmd.StdErr.Should().StartWith("Unable to determine the desired template from the input template name: c."); } }
public void ItBuildsAndTestsAppWhenRestoringToSpecificDirectory() { // Creating folder with name short name "RestoreTest" to avoid PathTooLongException var rootPath = TestAssets.Get("VSTestCore") .CreateInstance("8") .WithSourceFiles() .WithVersionVariables() .Root.FullName; string pkgDir; if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { // Moving pkgs folder on top to avoid PathTooLongException pkgDir = Path.Combine(RepoDirectoriesProvider.TestWorkingFolder, "pkgs"); } else { pkgDir = Path.Combine(rootPath, "pkgs"); } string args = $"--packages \"{pkgDir}\""; new RestoreCommand() .WithWorkingDirectory(rootPath) .Execute(args) .Should() .Pass(); new BuildCommand() .WithWorkingDirectory(rootPath) .ExecuteWithCapturedOutput("--no-restore") .Should() .Pass() .And.NotHaveStdErr(); CommandResult result = new DotnetTestCommand() .WithWorkingDirectory(rootPath) .ExecuteWithCapturedOutput($"{TestBase.ConsoleLoggerOutputNormal} --no-restore"); if (!DotnetUnderTest.IsLocalized()) { result.StdOut.Should().Contain("Total tests: 2"); result.StdOut.Should().Contain("Passed: 1"); result.StdOut.Should().Contain("Failed: 1"); result.StdOut.Should().Contain("\u221a VSTestPassTest"); result.StdOut.Should().Contain("X VSTestFailTest"); } result.ExitCode.Should().Be(1); }
public void ItAcceptsNoLogoAsCliArguments() { // Copy and restore VSTestCore project in output directory of project dotnet-vstest.Tests var testProjectDirectory = this.CopyAndRestoreVSTestDotNetCoreTestApp("14"); // Call test with logger enable CommandResult result = new DotnetTestCommand() .WithWorkingDirectory(testProjectDirectory) .ExecuteWithCapturedOutput("--nologo"); // Verify if (!DotnetUnderTest.IsLocalized()) { result.StdOut.Should().NotContain("Microsoft (R) Test Execution Command Line Tool Version"); result.StdOut.Should().Contain("Total tests: 2. Passed: 1. Failed: 1. Skipped: 0."); } }
public void ItDoesNotImplicitlyBuildAProjectWhenPackagingWithTheNoBuildOption() { var testInstance = TestAssets.Get("TestAppSimple") .CreateInstance() .WithSourceFiles(); var result = new PackCommand() .WithWorkingDirectory(testInstance.Root) .ExecuteWithCapturedOutput("--no-build"); result.Should().Fail(); if (!DotnetUnderTest.IsLocalized()) { result.Should().NotHaveStdOutContaining("Restore") .And.HaveStdOutContaining("project.assets.json"); } }
public void ItDoesNotImplicitlyBuildAProjectWhenRunningWithTheNoBuildOption() { var testAppName = "MSBuildTestApp"; var testInstance = TestAssets.Get(testAppName) .CreateInstance() .WithSourceFiles(); var result = new RunCommand() .WithWorkingDirectory(testInstance.Root.FullName) .ExecuteWithCapturedOutput("--no-build -v:m"); result.Should().Fail(); if (!DotnetUnderTest.IsLocalized()) { result.Should().NotHaveStdOutContaining("Restore"); } }
public void DotnetBuildDoesNotPrintCopyrightInfo() { var testInstance = TestAssets.Get("MSBuildTestApp") .CreateInstance() .WithSourceFiles() .WithRestoreFiles(); var cmd = new BuildCommand() .WithWorkingDirectory(testInstance.Root) .ExecuteWithCapturedOutput("--nologo"); cmd.Should().Pass(); if (!DotnetUnderTest.IsLocalized()) { cmd.Should().NotHaveStdOutContaining("Copyright (C) Microsoft Corporation. All rights reserved."); } }
public void MSTestSingleTFM() { var testProjectDirectory = this.CopyAndRestoreVSTestDotNetCoreTestApp("3"); // Call test CommandResult result = new DotnetTestCommand() .WithWorkingDirectory(testProjectDirectory) .ExecuteWithCapturedOutput(TestBase.ConsoleLoggerOutputNormal); // Verify if (!DotnetUnderTest.IsLocalized()) { result.StdOut.Should().Contain("Total tests: 2. Passed: 1. Failed: 1. Skipped: 0."); result.StdOut.Should().Contain("Passed TestNamespace.VSTestTests.VSTestPassTest"); result.StdOut.Should().Contain("Failed TestNamespace.VSTestTests.VSTestFailTest"); } result.ExitCode.Should().Be(1); }
public void GivenAFailingTestItDisplaysFailureDetails() { var testInstance = TestAssets.Get("XunitCore") .CreateInstance() .WithSourceFiles(); var result = new DotnetTestCommand() .WithWorkingDirectory(testInstance.Root.FullName) .ExecuteWithCapturedOutput(); result.ExitCode.Should().Be(1); if (!DotnetUnderTest.IsLocalized()) { result.StdOut.Should().Contain("Failed TestNamespace.VSTestXunitTests.VSTestXunitFailTest"); result.StdOut.Should().Contain("Assert.Equal() Failure"); result.StdOut.Should().Contain("Total tests: 2. Passed: 1. Failed: 1. Skipped: 0."); } }
public void ItUsesVerbosityPassedToDefineVerbosityOfConsoleLoggerOfTheTests() { // Copy and restore VSTestCore project in output directory of project dotnet-vstest.Tests var testProjectDirectory = this.CopyAndRestoreVSTestDotNetCoreTestApp("9"); // Call test CommandResult result = new DotnetTestCommand() .WithWorkingDirectory(testProjectDirectory) .ExecuteWithCapturedOutput("-v q"); // Verify if (!DotnetUnderTest.IsLocalized()) { result.StdOut.Should().Contain("Total tests: 2. Passed: 1. Failed: 1. Skipped: 0."); result.StdOut.Should().NotContain("Passed TestNamespace.VSTestTests.VSTestPassTest"); result.StdOut.Should().NotContain("Failed TestNamespace.VSTestTests.VSTestFailTest"); } result.ExitCode.Should().Be(1); }
public void ItRunsWithTheSpecifiedVerbosity() { var testAppName = "MSBuildTestApp"; var testInstance = TestAssets.Get(testAppName) .CreateInstance() .WithSourceFiles(); var result = new RunCommand() .WithWorkingDirectory(testInstance.Root.FullName) .ExecuteWithCapturedOutput("-v:n"); result.Should().Pass() .And.HaveStdOutContaining("Hello World!"); if (!DotnetUnderTest.IsLocalized()) { result.Should().HaveStdOutContaining("Restore") .And.HaveStdOutContaining("CoreCompile"); } }
public void XunitMultiTFM() { // Copy XunitMulti project in output directory of project dotnet-test.Tests string testAppName = "XunitMulti"; var testInstance = TestAssets.Get(testAppName) .CreateInstance("2") .WithSourceFiles() .WithVersionVariables(); var testProjectDirectory = testInstance.Root.FullName; // Restore project XunitMulti new RestoreCommand() .WithWorkingDirectory(testProjectDirectory) .Execute() .Should() .Pass(); // Call test CommandResult result = new DotnetTestCommand() .WithWorkingDirectory(testProjectDirectory) .ExecuteWithCapturedOutput(TestBase.ConsoleLoggerOutputNormal); // Verify if (!DotnetUnderTest.IsLocalized()) { // for target framework net46 result.StdOut.Should().Contain("Total tests: 3"); result.StdOut.Should().Contain("Passed: 2"); result.StdOut.Should().Contain("Failed: 1"); result.StdOut.Should().Contain("\u221a TestNamespace.VSTestXunitTests.VSTestXunitPassTestDesktop"); // for target framework netcoreapp1.0 result.StdOut.Should().Contain("Total tests: 3"); result.StdOut.Should().Contain("Passed: 1"); result.StdOut.Should().Contain("Failed: 2"); result.StdOut.Should().Contain("X TestNamespace.VSTestXunitTests.VSTestXunitFailTestNetCoreApp"); } result.ExitCode.Should().Be(1); }
public void ItShouldShowWarningMessageOnCollectCodeCoverage() { var testProjectDirectory = this.CopyAndRestoreVSTestDotNetCoreTestApp("13"); // Call test CommandResult result = new DotnetTestCommand() .WithWorkingDirectory(testProjectDirectory) .ExecuteWithCapturedOutput( "--collect \"Code Coverage\" " + "--filter VSTestPassTest"); // Verify test results if (!DotnetUnderTest.IsLocalized()) { result.StdOut.Should().Contain("No code coverage data available. Code coverage is currently supported only on Windows."); result.StdOut.Should().Contain("Total tests: 1. Passed: 1. Failed: 0. Skipped: 0."); result.StdOut.Should().Contain("Test Run Successful."); } result.ExitCode.Should().Be(0); }
public void ItCreatesCoverageFileWhenCodeCoverageEnabledByRunsettings() { var testProjectDirectory = this.CopyAndRestoreVSTestDotNetCoreTestApp("11"); string resultsDirectory = Path.Combine(testProjectDirectory, "RD"); // Delete resultsDirectory if it exist if (Directory.Exists(resultsDirectory)) { Directory.Delete(resultsDirectory, true); } var settingsPath = Path.Combine(AppContext.BaseDirectory, "CollectCodeCoverage.runsettings"); // Call test CommandResult result = new DotnetTestCommand() .WithWorkingDirectory(testProjectDirectory) .ExecuteWithCapturedOutput( "--settings " + settingsPath + " --results-directory " + resultsDirectory); File.WriteAllText(Path.Combine(testProjectDirectory, "output.txt"), result.StdOut + Environment.NewLine + result.StdErr); // Verify test results if (!DotnetUnderTest.IsLocalized()) { result.StdOut.Should().Contain("Total tests: 2"); result.StdOut.Should().Contain("Passed: 1"); result.StdOut.Should().Contain("Failed: 1"); } // Verify coverage file. DirectoryInfo d = new DirectoryInfo(resultsDirectory); FileInfo[] coverageFileInfos = d.GetFiles("*.coverage", SearchOption.AllDirectories); Assert.Single(coverageFileInfos); result.ExitCode.Should().Be(1); }
public void TestWillNotBuildTheProjectIfNoBuildArgsIsGiven() { // Copy and restore VSTestCore project in output directory of project dotnet-vstest.Tests var testProjectDirectory = this.CopyAndRestoreVSTestDotNetCoreTestApp("5"); string configuration = Environment.GetEnvironmentVariable("CONFIGURATION") ?? "Debug"; string expectedError = Path.Combine(testProjectDirectory, "bin", configuration, "netcoreapp2.1", "VSTestCore.dll"); expectedError = "The test source file " + "\"" + expectedError + "\"" + " provided was not found."; // Call test CommandResult result = new DotnetTestCommand() .WithWorkingDirectory(testProjectDirectory) .ExecuteWithCapturedOutput("--no-build"); // Verify if (!DotnetUnderTest.IsLocalized()) { result.StdErr.Should().Contain(expectedError); } result.ExitCode.Should().Be(1); }
public void ItShouldShowImportantMessageWhenInteractiveFlagIsPassed() { string testAppName = "VSTestCore"; var testInstance = TestAssets.Get(testAppName) .CreateInstance() .WithSourceFiles() .WithProjectChanges(ProjectModification.AddDisplayMessageBeforeVsTestToProject); var testProjectDirectory = testInstance.Root.FullName; // Call test CommandResult result = new DotnetTestCommand() .WithWorkingDirectory(testProjectDirectory) .ExecuteWithCapturedOutput("--interactive"); // Verify if (!DotnetUnderTest.IsLocalized()) { result.StdOut.Should().Contain("Important text"); } result.ExitCode.Should().Be(1); }
public void ItBuildsAndTestsAppWhenRestoringToSpecificDirectory() { // Creating folder with name short name "RestoreTest" to avoid PathTooLongException var rootPath = TestAssets.Get("VSTestCore").CreateInstance("8").WithSourceFiles().Root.FullName; // Moving pkgs folder on top to avoid PathTooLongException string dir = @"..\..\..\..\pkgs"; string fullPath = Path.GetFullPath(Path.Combine(rootPath, dir)); string args = $"--packages \"{dir}\""; new RestoreCommand() .WithWorkingDirectory(rootPath) .Execute(args) .Should() .Pass(); new BuildCommand() .WithWorkingDirectory(rootPath) .ExecuteWithCapturedOutput("--no-restore") .Should() .Pass() .And.NotHaveStdErr(); CommandResult result = new DotnetTestCommand() .WithWorkingDirectory(rootPath) .ExecuteWithCapturedOutput($"{TestBase.ConsoleLoggerOutputNormal} --no-restore"); if (!DotnetUnderTest.IsLocalized()) { result.StdOut.Should().Contain("Total tests: 2. Passed: 1. Failed: 1. Skipped: 0."); result.StdOut.Should().Contain("Passed TestNamespace.VSTestTests.VSTestPassTest"); result.StdOut.Should().Contain("Failed TestNamespace.VSTestTests.VSTestFailTest"); } result.ExitCode.Should().Be(1); }
public void MStestMultiTFM() { var testProjectDirectory = TestAssets.Get("VSTestMulti") .CreateInstance("1") .WithSourceFiles() .WithVersionVariables() .WithNuGetConfig(RepoDirectoriesProvider.TestPackages) .Root; var runtime = EnvironmentInfo.GetCompatibleRid(); new RestoreCommand() .WithWorkingDirectory(testProjectDirectory) .WithRuntime(runtime) .Execute() .Should().Pass(); var result = new DotnetTestCommand() .WithWorkingDirectory(testProjectDirectory) .WithRuntime(runtime) .ExecuteWithCapturedOutput(TestBase.ConsoleLoggerOutputNormal); if (!DotnetUnderTest.IsLocalized()) { result.StdOut .Should().Contain("Total tests: 3") .And.Contain("Passed: 2") .And.Contain("Failed: 1") .And.Contain("\u221a VSTestPassTestDesktop", "because .NET 4.6 tests will pass") .And.Contain("Total tests: 3") .And.Contain("Passed: 1") .And.Contain("Failed: 2") .And.Contain("X VSTestFailTestNetCoreApp", "because netcoreapp2.0 tests will fail"); } result.ExitCode.Should().Be(1); }