public void StackOverflowExceptionShouldBeLoggedToConsoleAndDiagLogFile(RunnerInfo runnerInfo)
{
AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
var resultsDir = GetResultsDirectory();
if (IntegrationTestEnvironment.BuildConfiguration.Equals("release", StringComparison.OrdinalIgnoreCase))
{
// On release, x64 builds, recursive calls may be replaced with loops (tail call optimization)
Assert.Inconclusive("On StackOverflowException testhost not exited in release configuration.");
return;
}
var diagLogFilePath = Path.Combine(resultsDir, $"std_error_log_{Guid.NewGuid()}.txt");
File.Delete(diagLogFilePath);
var assemblyPaths = this.BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"');
var arguments = PrepareArguments(assemblyPaths, this.GetTestAdapterPath(), string.Empty, this.FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: resultsDir);
arguments = string.Concat(arguments, " /testcasefilter:ExitWithStackoverFlow");
arguments = string.Concat(arguments, $" /diag:{diagLogFilePath}");
this.InvokeVsTest(arguments);
var errorMessage = "Process is terminated due to StackOverflowException.";
if (runnerInfo.TargetFramework.StartsWith("netcoreapp2."))
{
errorMessage = "Process is terminating due to StackOverflowException.";
}
this.ExitCodeEquals(1);
FileAssert.Contains(diagLogFilePath, errorMessage);
this.StdErrorContains(errorMessage);
File.Delete(diagLogFilePath);
TryRemoveDirectory(resultsDir);
}
public void LegacySettingsAssemblyResolution(RunnerInfo runnerInfo)
{
AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
var resultsDir = GetResultsDirectory();
var testAssemblyPath = this.GetAssetFullPath("LegacySettingsUnitTestProject.dll");
var runsettingsFormat = @"<RunSettings>
<MSTest><ForcedLegacyMode>true</ForcedLegacyMode></MSTest>
<LegacySettings>
<Execution>
<TestTypeSpecific>
<UnitTestRunConfig testTypeId=""13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b"">
<AssemblyResolution>
<TestDirectory useLoadContext=""true"" />
<RuntimeResolution>
<Directory path=""{0}"" includeSubDirectories=""true"" />
</RuntimeResolution>
</AssemblyResolution>
</UnitTestRunConfig>
</TestTypeSpecific>
</Execution>
</LegacySettings>
</RunSettings>";
var testAssemblyDirectory = Path.Combine(this.testEnvironment.TestAssetsPath, "LegacySettingsUnitTestProject", "DependencyAssembly");
var runsettingsXml = string.Format(runsettingsFormat, testAssemblyDirectory);
File.WriteAllText(this.runsettingsPath, runsettingsXml);
var arguments = PrepareArguments(testAssemblyPath, string.Empty, this.runsettingsPath, this.FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: resultsDir);
arguments = string.Concat(arguments, " /testcasefilter:Name=DependencyTest");
this.InvokeVsTest(arguments);
this.ValidateSummaryStatus(1, 0, 0);
TryRemoveDirectory(resultsDir);
}
public void OlderOrderedTestsShouldWorkFine(RunnerInfo runnerInfo)
{
AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
var resultsDir = GetResultsDirectory();
if (runnerInfo.RunnerFramework.StartsWith("netcoreapp"))
{
Assert.Inconclusive(" Ordered tests are not supported with .Netcore runner.");
return;
}
var orderedTestFileAbsolutePath = Path.Combine(this.testEnvironment.TestAssetsPath, "MstestV1UnitTestProject");
if (IntegrationTestEnvironment.BuildConfiguration.Equals("release", StringComparison.OrdinalIgnoreCase))
{
orderedTestFileAbsolutePath = Path.Combine(orderedTestFileAbsolutePath, "MixedTestsRelease.orderedtest");
}
else
{
orderedTestFileAbsolutePath = Path.Combine(orderedTestFileAbsolutePath, "MixedTestsDebug.orderedtest");
}
var arguments = PrepareArguments(
orderedTestFileAbsolutePath,
this.GetTestAdapterPath(),
string.Empty, this.FrameworkArgValue,
runnerInfo.InIsolationValue, resultsDirectory: resultsDir);
this.InvokeVsTest(arguments);
this.ValidatePassedTests("PassingTest1");
this.ValidatePassedTests("PassingTest2");
this.ValidateFailedTests("FailingTest1");
this.ValidateSkippedTests("FailingTest2");
// Parent test result should fail as inner results contain failing test.
this.ValidateSummaryStatus(2, 1, 1);
TryRemoveDirectory(resultsDir);
}
public void WildCardPatternShouldCorrectlyWorkForRelativeAssemblyPath(RunnerInfo runnerInfo)
{
AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
var testAssembly = this.GetSampleTestAssembly();
testAssembly = testAssembly.Replace("SimpleTestProject.dll", "*TestProj*.dll");
var wildCardIndex = testAssembly.IndexOfAny(new char[] { '*' });
var testAssemblyDirectory = testAssembly.Substring(0, wildCardIndex);
testAssembly = testAssembly.Substring(wildCardIndex);
Directory.SetCurrentDirectory(testAssemblyDirectory);
var arguments = PrepareArguments(
testAssembly,
this.GetTestAdapterPath(),
string.Empty, string.Empty,
runnerInfo.InIsolationValue);
this.InvokeVsTest(arguments);
this.ValidateSummaryStatus(1, 1, 1);
}
public void IncompatibleSourcesWarningShouldBeDisplayedInTheConsoleOnlyWhenRunningIn32BitOS(RunnerInfo runnerInfo)
{
AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
var expectedWarningContains = @"Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.5.1 framework and X86 platform. SimpleTestProject2.dll is built for Framework .NETFramework,Version=v4.5.1 and Platform X64";
var assemblyPaths =
this.BuildMultipleAssemblyPath("SimpleTestProject2.dll");
var arguments = PrepareArguments(assemblyPaths, this.GetTestAdapterPath(), string.Empty, this.FrameworkArgValue, runnerInfo.InIsolationValue);
this.InvokeVsTest(arguments);
this.ValidateSummaryStatus(1, 1, 1);
this.ExitCodeEquals(1);
// If we are running this test on 64 bit OS, it should not output any warning
if (Environment.Is64BitOperatingSystem)
{
this.StdOutputDoesNotContains(expectedWarningContains);
}
// If we are running this test on 32 bit OS, it should output warning message
else
{
this.StdOutputContains(expectedWarningContains);
}
}
public void UnhandleExceptionExceptionShouldBeLoggedToDiagLogFile(RunnerInfo runnerInfo)
{
AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
var resultsDir = GetResultsDirectory();
var diagLogFilePath = Path.Combine(resultsDir, $"std_error_log_{Guid.NewGuid()}.txt");
File.Delete(diagLogFilePath);
var assemblyPaths =
this.BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"');
var arguments = PrepareArguments(assemblyPaths, this.GetTestAdapterPath(), string.Empty, this.FrameworkArgValue, runnerInfo.InIsolationValue, resultsDirectory: resultsDir);
arguments = string.Concat(arguments, " /testcasefilter:ExitwithUnhandleException");
arguments = string.Concat(arguments, $" /diag:{diagLogFilePath}");
this.InvokeVsTest(arguments);
var errorFirstLine = "Test host standard error line: Unhandled Exception: System.InvalidOperationException: Operation is not valid due to the current state of the object.";
FileAssert.Contains(diagLogFilePath, errorFirstLine);
File.Delete(diagLogFilePath);
TryRemoveDirectory(resultsDir);
}
public void RunSpecificTestsShouldWorkWithFrameworkInCompatibleWarning(RunnerInfo runnerInfo)
{
AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
var resultsDir = GetResultsDirectory();
var arguments = PrepareArguments(GetSampleTestAssembly(), string.Empty, string.Empty, string.Empty, resultsDirectory: resultsDir);
arguments = string.Concat(arguments, " ", "/tests:PassingTest");
arguments = string.Concat(arguments, " ", "/Framework:Framework40");
this.InvokeVsTest(arguments);
if (runnerInfo.TargetFramework.Contains("netcore"))
{
this.StdOutputContains("No test is available");
}
else
{
this.StdOutputContains("Following DLL(s) do not match current settings, which are .NETFramework,Version=v4.0 framework and X86 platform.");
this.ValidateSummaryStatus(1, 0, 0);
}
TryRemoveDirectory(resultsDir);
}
public void ResultsDirectoryRelativePathShouldWork(RunnerInfo runnerInfo)
{
AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
var arguments = PrepareArguments(this.GetSampleTestAssembly(), this.GetTestAdapterPath(), string.Empty, this.FrameworkArgValue, runnerInfo.InIsolationValue);
var trxFileName = "TestResults.trx";
var relativeDirectory = @"relative\directory";
var resultsDirectory = Path.Combine(Directory.GetCurrentDirectory(), relativeDirectory);
var trxFilePath = Path.Combine(resultsDirectory, trxFileName);
arguments = string.Concat(arguments, $" /logger:\"trx;LogFileName={trxFileName}\"");
arguments = string.Concat(arguments, $" /ResultsDirectory:{relativeDirectory}");
if (Directory.Exists(resultsDirectory))
{
Directory.Delete(resultsDirectory, true);
}
this.InvokeVsTest(arguments);
Assert.IsTrue(File.Exists(trxFilePath), $"Expected Trx file: {trxFilePath} not created in results directory");
TryRemoveDirectory(resultsDirectory);
}
public void PassingInvalidArgumentsToVsTestConsoleShouldNotPrintHelpMessage(RunnerInfo runnerInfo)
{
AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
var arguments = PrepareArguments(this.GetSampleTestAssembly(), this.GetTestAdapterPath(), string.Empty, this.FrameworkArgValue);
arguments = string.Concat(arguments, " /badArgument");
this.InvokeVsTest(arguments);
//Check for help usage, description and arguments text.
this.StdOutputDoesNotContains("Usage: vstest.console.exe");
this.StdOutputDoesNotContains("Description: Runs tests from the specified files.");
this.StdOutputDoesNotContains("Arguments:");
//Check for help options text
this.StdOutputDoesNotContains("Options:");
//Check for help examples text
this.StdOutputDoesNotContains("To run tests: >vstest.console.exe tests.dll");
//Check for message which guides using help option
this.StdErrorContains("Please use the /help option to check the list of valid arguments");
}
public void XUnitRunAllTestExecution(RunnerInfo runnerInfo)
{
AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
string testAssemblyPath = null;
// Xunit >= 2.2 won't support net451, Minimum target framework it supports is net452.
if (this.testEnvironment.TargetFramework.Equals("net451"))
{
testAssemblyPath = testEnvironment.GetTestAsset("XUTestProject.dll", "net46");
}
else
{
testAssemblyPath = testEnvironment.GetTestAsset("XUTestProject.dll");
}
var arguments = PrepareArguments(
testAssemblyPath,
this.GetTestAdapterPath(UnitTestFramework.XUnit),
string.Empty,
runnerInfo.InIsolationValue);
this.InvokeVsTest(arguments);
this.ValidateSummaryStatus(1, 1, 0);
}
public void CLIRunsettingsShouldWinBetweenCLISwitchesAndCLIRunsettings(RunnerInfo runnerInfo)
{
AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
var targetPlatform = "x86";
var testhostProcessName = new[] { "testhost.x86", "dotnet" };
var expectedNumOfProcessCreated = GetExpectedNumOfProcessCreatedForWithoutParallel();
// Pass parallel
var additionalArgs = "/Parallel";
// Pass non parallel
var runSettingsArgs = String.Join(
" ",
new string[]
{
"RunConfiguration.MaxCpuCount=1",
string.Concat("RunConfiguration.TargetPlatform=", targetPlatform),
string.Concat("RunConfiguration.TargetFrameworkVersion=", this.GetTargetFramworkForRunsettings()),
string.Concat("RunConfiguration.TestAdaptersPaths=", this.GetTestAdapterPath())
});
this.RunTestWithRunSettings(null, runSettingsArgs, additionalArgs, testhostProcessName, expectedNumOfProcessCreated);
}
public void RunTestExecutionWithDisableAppDomain(RunnnerInfo runnerInfo)
{
AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
var testAppDomainDetailFileName = Path.Combine(Path.GetTempPath(), "appdomain_test.txt");
var dataCollectorAppDomainDetailFileName = Path.Combine(Path.GetTempPath(), "appdomain_datacollector.txt");
// Delete test output files if already exist
File.Delete(testAppDomainDetailFileName);
File.Delete(dataCollectorAppDomainDetailFileName);
var runsettingsFilePath = this.GetInProcDataCollectionRunsettingsFile(true);
var arguments = PrepareArguments(
this.GetSampleTestAssembly(),
this.GetTestAdapterPath(),
runsettingsFilePath,
this.FrameworkArgValue,
runnerInfo.InIsolationValue);
this.InvokeVsTest(arguments);
Assert.IsTrue(IsFilesContentEqual(testAppDomainDetailFileName, dataCollectorAppDomainDetailFileName), "Different AppDomains, test: {0} datacollector: {1}", File.ReadAllText(testAppDomainDetailFileName), File.ReadAllText(dataCollectorAppDomainDetailFileName));
this.ValidateSummaryStatus(1, 1, 1);
File.Delete(runsettingsFilePath);
}
public void EnvironmentVariablesSettingsShouldSetEnvironmentVariables(RunnerInfo runnerInfo)
{
AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
var testAssemblyPath = this.GetAssetFullPath("EnvironmentVariablesTestProject.dll");
var runsettingsXml = @"<RunSettings>
<RunConfiguration>
<EnvironmentVariables>
<RANDOM_PATH>C:\temp</RANDOM_PATH>
</EnvironmentVariables>
</RunConfiguration>
</RunSettings>";
File.WriteAllText(this.runsettingsPath, runsettingsXml);
var arguments = PrepareArguments(
testAssemblyPath,
string.Empty,
this.runsettingsPath, this.FrameworkArgValue, runnerInfo.InIsolationValue);
this.InvokeVsTest(arguments);
this.ValidateSummaryStatus(1, 0, 0);
}
public void DiscoverTestsShouldPublishMetrics(RunnerInfo runnerInfo)
{
AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
this.DiscoverTests(runnerInfo.RunnerFramework);
}
public void CodedWebTestRunAllTests(RunnerInfo runnerInfo)
{
AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
CodedWebTestRunAllTests(runnerInfo.RunnerFramework);
}
public void CPPRunAllTestExecutionPlatformx64(RunnerInfo runnerInfo)
{
AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerInfo);
CppRunAllTests(runnerInfo.RunnerFramework, "x64");
}
public void CPPRunAllTestExecutionPlatformx64(string runnerFramework, string targetFramework, string targetRuntime)
{
AcceptanceTestBase.SetTestEnvironment(this.testEnvironment, runnerFramework, targetFramework, targetRuntime);
CppRunAllTests(runnerFramework, "x64");
}
