private void DoRunCancelingTests(bool killProcesses, int lower)
{
MockOptions.Setup(o => o.KillProcessesOnCancel).Returns(killProcesses);
List <Model.TestCase> testCasesToRun = TestDataCreator.GetTestCases("Crashing.LongRunning", "LongRunningTests.Test2");
testCasesToRun.Should().HaveCount(2);
var stopwatch = new Stopwatch();
var executor = new TestExecutor(TestEnvironment.Logger, TestEnvironment.Options, MockDebuggerAttacher.Object);
var canceller = new Thread(() =>
{
Thread.Sleep(WaitBeforeCancelInMs);
executor.Cancel();
});
canceller.Start();
stopwatch.Start();
executor.RunTests(testCasesToRun.Select(tc => tc.ToVsTestCase()), MockRunContext.Object, MockFrameworkHandle.Object);
stopwatch.Stop();
canceller.Join();
stopwatch.ElapsedMilliseconds.Should().BeInRange(lower, lower + OverheadInMs);
}
public virtual void RunTests_WorkingDir_IsSetCorrectly()
{
TestCase testCase = TestDataCreator.GetTestCases("WorkingDir.IsSolutionDirectory").First();
MockOptions.Setup(o => o.WorkingDir).Returns(SettingsWrapper.ExecutableDirPlaceholder);
RunAndVerifySingleTest(testCase, VsTestOutcome.Failed);
MockFrameworkHandle.Reset();
MockOptions.Setup(o => o.WorkingDir).Returns(SettingsWrapper.SolutionDirPlaceholder);
RunAndVerifySingleTest(testCase, VsTestOutcome.Passed);
}
public virtual void RunTests_TestDirectoryViaUserParams_IsPassedViaCommandLineArg()
{
TestCase testCase = TestDataCreator.GetTestCases("CommandArgs.TestDirectoryIsSet").First();
RunAndVerifySingleTest(testCase, VsTestOutcome.Failed);
MockFrameworkHandle.Reset();
MockOptions.Setup(o => o.AdditionalTestExecutionParam).Returns("-testdirectory=\"" + SettingsWrapper.TestDirPlaceholder + "\"");
RunAndVerifySingleTest(testCase, VsTestOutcome.Passed);
}
public void RunTests_WorkingDirSetForProject_TestPasses()
{
TestCase testCase = TestDataCreator.GetTestCases("WorkingDir.IsSolutionDirectory").First();
var settings = CreateSettings("foo", SettingsWrapper.SolutionDirPlaceholder);
var runner = new SequentialTestRunner("", 0, "", MockFrameworkReporter.Object, TestEnvironment.Logger, settings, new SchedulingAnalyzer(TestEnvironment.Logger));
runner.RunTests(testCase.Yield(), false, ProcessExecutorFactory);
MockLogger.Verify(l => l.LogError(It.IsAny <string>()), Times.Never);
MockFrameworkReporter.Verify(r => r.ReportTestResults(
It.Is <IEnumerable <TestResult> >(tr => CheckSingleResultHasOutcome(tr, TestOutcome.Passed))), Times.Once);
}
public void RunTests_WorkingDirNotSet_TestFails()
{
var testCase = TestDataCreator.GetTestCases("WorkingDir.IsSolutionDirectory").First();
var settings = CreateSettings(null, null);
var runner = new SequentialTestRunner("", 0, "", MockFrameworkReporter.Object, TestEnvironment.Logger, settings, new SchedulingAnalyzer(TestEnvironment.Logger));
var executor = new ProcessExecutor(null, MockLogger.Object);
runner.RunTests(testCase.Yield(), false, null, executor);
MockLogger.Verify(l => l.LogError(It.IsAny <string>()), Times.Never);
MockFrameworkReporter.Verify(r => r.ReportTestResults(
It.Is <IEnumerable <TestResult> >(tr => CheckSingleResultHasOutcome(tr, TestOutcome.Failed))), Times.Once);
}
private void RunExitCodeTest(string testCaseName, VsTestOutcome outcome)
{
string exitCodeTestName = "ExitCode";
MockOptions.Setup(o => o.ExitCodeTestCase).Returns(exitCodeTestName);
TestCase testCase = TestDataCreator.GetTestCases(testCaseName).First();
TestExecutor executor = new TestExecutor(TestEnvironment.Logger, TestEnvironment.Options, MockDebuggerAttacher.Object);
executor.RunTests(testCase.ToVsTestCase().Yield(), MockRunContext.Object, MockFrameworkHandle.Object);
MockFrameworkHandle.Verify(h => h.RecordResult(It.Is <VsTestResult>(result =>
result.TestCase.FullyQualifiedName == testCaseName &&
result.Outcome == outcome
)),
Times.Once);
// ReSharper disable once PossibleNullReferenceException
string finalName = exitCodeTestName + "." + Path.GetFileName(testCase.Source).Replace(".", "_");
bool outputAvailable = MockOptions.Object.UseNewTestExecutionFramework ||
!MockRunContext.Object.IsBeingDebugged;
Func <VsTestResult, bool> errorMessagePredicate = outcome == VsTestOutcome.Failed
? result => result.ErrorMessage.Contains("Exit code: 1") &&
(!outputAvailable || result.ErrorMessage.Contains("The result code output")) &&
!result.ErrorMessage.Contains(StreamingStandardOutputTestResultParser.GtaExitCodeOutputBegin) &&
!result.ErrorMessage.Contains(StreamingStandardOutputTestResultParser.GtaExitCodeOutputEnd) &&
!result.ErrorMessage.Contains("Some more output")
: (Func <VsTestResult, bool>)(result => string.IsNullOrEmpty(result.ErrorMessage) || result.ErrorMessage.Contains("The result code output"));
MockFrameworkHandle.Verify(h => h.RecordResult(It.Is <VsTestResult>(result =>
result.TestCase.FullyQualifiedName == finalName &&
result.Outcome == outcome &&
errorMessagePredicate(result)
)),
Times.Once);
MockFrameworkHandle.Verify(h => h.RecordResult(It.IsAny <VsTestResult>()), Times.Exactly(2));
if (!outputAvailable && outcome == VsTestOutcome.Failed)
{
MockLogger.Verify(l => l.LogWarning(It.Is <string>(msg => msg.Contains("Result code") &&
msg.Contains(SettingsWrapper.OptionUseNewTestExecutionFramework))), Times.Once);
}
MockLogger.Verify(l => l.DebugWarning(It.Is <string>(msg => msg.Contains("main method") && msg.Contains("exit code"))), Times.Never);
}
public void RunTests_CancelingExecutor_StopsTestExecution()
{
List <Model.TestCase> testCasesToRun = TestDataCreator.GetTestCases("Crashing.LongRunning", "LongRunningTests.Test2");
testCasesToRun.Count.Should().Be(2);
Stopwatch stopwatch = new Stopwatch();
TestExecutor executor = new TestExecutor(TestEnvironment.Logger, TestEnvironment.Options);
Thread thread = new Thread(() => executor.RunTests(testCasesToRun.Select(DataConversionExtensions.ToVsTestCase), MockRunContext.Object, MockFrameworkHandle.Object));
stopwatch.Start();
thread.Start();
Thread.Sleep(1000);
executor.Cancel();
thread.Join();
stopwatch.Stop();
stopwatch.ElapsedMilliseconds.Should().BeInRange(3000, // 1st test should be executed
4000); // 2nd test should not be executed
}
private void DoRunCancelingTests(bool killProcesses, int lower, int upper)
{
MockOptions.Setup(o => o.KillProcessesOnCancel).Returns(killProcesses);
List <TestCase> testCasesToRun = TestDataCreator.GetTestCases("Crashing.LongRunning", "LongRunningTests.Test2");
var stopwatch = new Stopwatch();
var runner = new SequentialTestRunner("", 0, "", MockFrameworkReporter.Object, TestEnvironment.Logger, TestEnvironment.Options, new SchedulingAnalyzer(TestEnvironment.Logger));
var thread = new Thread(() => runner.RunTests(testCasesToRun, false, ProcessExecutorFactory));
stopwatch.Start();
thread.Start();
Thread.Sleep(1000);
runner.Cancel();
thread.Join();
stopwatch.Stop();
testCasesToRun.Should().HaveCount(2);
MockLogger.Verify(l => l.LogError(It.IsAny <string>()), Times.Never);
stopwatch.ElapsedMilliseconds.Should().BeGreaterThan(lower); // 1st test should be executed
stopwatch.ElapsedMilliseconds.Should().BeLessThan(upper); // 2nd test should not be executed
}
public void RunTests_CancelingDuringTestExecution_StopsTestExecution()
{
List <TestCase> allTestCases = TestDataCreator.AllTestCasesExceptLoadTests;
List <TestCase> testCasesToRun = TestDataCreator.GetTestCases("Crashing.LongRunning", "LongRunningTests.Test2");
var stopwatch = new Stopwatch();
var runner = new SequentialTestRunner("", MockFrameworkReporter.Object, TestEnvironment.Logger, TestEnvironment.Options, new SchedulingAnalyzer(TestEnvironment.Logger));
var executor = new ProcessExecutor(null, MockLogger.Object);
var thread = new Thread(() => runner.RunTests(allTestCases, testCasesToRun, "", "", "", false, null, executor));
stopwatch.Start();
thread.Start();
Thread.Sleep(1000);
runner.Cancel();
thread.Join();
stopwatch.Stop();
testCasesToRun.Count.Should().Be(2);
MockLogger.Verify(l => l.LogError(It.IsAny <string>()), Times.Never);
stopwatch.ElapsedMilliseconds.Should().BeGreaterThan(2000); // 1st test should be executed
stopwatch.ElapsedMilliseconds.Should().BeLessThan(3000); // 2nd test should not be executed
}