public void RunTests_ParallelTestExecution_SpeedsUpTestExecution()
{
MockOptions.Setup(o => o.ParallelTestExecution).Returns(false);
Stopwatch stopwatch = new Stopwatch();
TestExecutor executor = new TestExecutor(TestEnvironment);
IEnumerable<string> testsToRun = TestResources.SampleTests.Yield();
stopwatch.Start();
executor.RunTests(testsToRun, MockRunContext.Object, MockFrameworkHandle.Object);
stopwatch.Stop();
long sequentialDuration = stopwatch.ElapsedMilliseconds;
MockOptions.Setup(o => o.ParallelTestExecution).Returns(true);
MockOptions.Setup(o => o.MaxNrOfThreads).Returns(Environment.ProcessorCount);
executor = new TestExecutor(TestEnvironment);
testsToRun = TestResources.SampleTests.Yield();
stopwatch.Restart();
executor.RunTests(testsToRun, MockRunContext.Object, MockFrameworkHandle.Object);
stopwatch.Stop();
long parallelDuration = stopwatch.ElapsedMilliseconds;
sequentialDuration.Should().BeGreaterThan(4000); // 2 long tests, 2 seconds per test
parallelDuration.Should().BeGreaterThan(2000);
parallelDuration.Should().BeLessThan(4000); // 2 seconds per long test + some time for the rest
}
private void RunAndVerifySingleTest(TestCase testCase, VsTestOutcome expectedOutcome)
{
TestExecutor executor = new TestExecutor(TestEnvironment);
executor.RunTests(testCase.ToVsTestCase().Yield(), MockRunContext.Object, MockFrameworkHandle.Object);
foreach (VsTestOutcome outcome in Enum.GetValues(typeof(VsTestOutcome)))
{
MockFrameworkHandle.Verify(h => h.RecordEnd(It.IsAny<VsTestCase>(), It.Is<VsTestOutcome>(to => to == outcome)),
Times.Exactly(outcome == expectedOutcome ? 1 : 0));
}
}
public void RunTests_CancelingExecutor_StopsTestExecution()
{
List<Model.TestCase> testCasesToRun = TestDataCreator.GetTestCasesOfSampleTests("Crashing.LongRunning", "LongRunningTests.Test3");
Stopwatch stopwatch = new Stopwatch();
TestExecutor executor = new TestExecutor(TestEnvironment);
Thread thread = new Thread(() => executor.RunTests(testCasesToRun.Select(DataConversionExtensions.ToVsTestCase), MockRunContext.Object, MockFrameworkHandle.Object));
stopwatch.Start();
thread.Start();
Thread.Sleep(1000);
executor.Cancel();
thread.Join();
stopwatch.Stop();
stopwatch.ElapsedMilliseconds.Should().BeInRange(3000, // 1st test should be executed
4000); // 2nd test should not be executed
}
protected void RunMemoryLeakTest(string executable, string testCaseName, VsTestOutcome testOutcome, VsTestOutcome leakCheckOutcome, Func <string, bool> errorMessagePredicate)
{
string exitCodeTestName = "MemoryLeakTest";
MockOptions.Setup(o => o.ExitCodeTestCase).Returns(exitCodeTestName);
MockOptions.Setup(o => o.AdditionalTestExecutionParam).Returns("-is_run_by_gta");
var testCases = new GoogleTestDiscoverer(MockLogger.Object, TestEnvironment.Options,
new ProcessExecutorFactory(), new DefaultDiaResolverFactory()).GetTestsFromExecutable(executable);
var testCase = testCases.Single(tc => tc.DisplayName == testCaseName);
var executor = new TestExecutor(TestEnvironment.Logger, TestEnvironment.Options, MockDebuggerAttacher.Object);
executor.RunTests(testCase.Yield().Select(tc => tc.ToVsTestCase()), MockRunContext.Object, MockFrameworkHandle.Object);
var invocations = new List <IInvocation>();
foreach (var invocation in MockFrameworkHandle.Invocations)
{
invocations.Add(invocation);
}
MockFrameworkHandle.Verify(h => h.RecordResult(It.Is <VsTestResult>(result =>
result.TestCase.FullyQualifiedName == testCaseName &&
result.Outcome == testOutcome
)),
Times.Once);
MockFrameworkHandle.Verify(h => h.RecordResult(It.Is <VsTestResult>(result =>
result.TestCase.FullyQualifiedName.StartsWith("MemoryLeakTest") &&
result.Outcome == leakCheckOutcome &&
(result.ErrorMessage == null || !result.ErrorMessage.Contains(StreamingStandardOutputTestResultParser.GtaExitCodeOutputBegin)) &&
(result.ErrorMessage == null || !result.ErrorMessage.Contains(StreamingStandardOutputTestResultParser.GtaExitCodeOutputEnd)) &&
errorMessagePredicate(result.ErrorMessage)
)),
Times.Once);
MockLogger.Verify(l => l.DebugWarning(It.Is <string>(msg => msg.Contains("main method") && msg.Contains("exit code"))), Times.Never);
}
public virtual void RunTests_TestDirectoryViaUserParams_IsPassedViaCommandLineArg()
{
TestCase testCase = TestDataCreator.GetTestCasesOfSampleTests("CommandArgs.TestDirectoryIsSet").First();
TestExecutor executor = new TestExecutor(TestEnvironment);
executor.RunTests(testCase.ToVsTestCase().Yield(), MockRunContext.Object, MockFrameworkHandle.Object);
MockFrameworkHandle.Verify(h => h.RecordEnd(It.IsAny <Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase>(), It.Is <Microsoft.VisualStudio.TestPlatform.ObjectModel.TestOutcome>(to => to == Microsoft.VisualStudio.TestPlatform.ObjectModel.TestOutcome.Passed)),
Times.Exactly(0));
MockFrameworkHandle.Verify(h => h.RecordEnd(It.IsAny <Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase>(), It.Is <Microsoft.VisualStudio.TestPlatform.ObjectModel.TestOutcome>(to => to == Microsoft.VisualStudio.TestPlatform.ObjectModel.TestOutcome.Failed)),
Times.Exactly(1));
MockFrameworkHandle.Reset();
MockOptions.Setup(o => o.AdditionalTestExecutionParam).Returns("-testdirectory=\"" + SettingsWrapper.TestDirPlaceholder + "\"");
executor = new TestExecutor(TestEnvironment);
executor.RunTests(testCase.ToVsTestCase().Yield(), MockRunContext.Object, MockFrameworkHandle.Object);
MockFrameworkHandle.Verify(h => h.RecordEnd(It.IsAny <Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase>(), It.Is <Microsoft.VisualStudio.TestPlatform.ObjectModel.TestOutcome>(to => to == Microsoft.VisualStudio.TestPlatform.ObjectModel.TestOutcome.Failed)),
Times.Exactly(0));
MockFrameworkHandle.Verify(h => h.RecordEnd(It.IsAny <Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase>(), It.Is <Microsoft.VisualStudio.TestPlatform.ObjectModel.TestOutcome>(to => to == Microsoft.VisualStudio.TestPlatform.ObjectModel.TestOutcome.Passed)),
Times.Exactly(1));
}
private void RunAndVerifyTests(string executable, int nrOfPassedTests, int nrOfFailedTests, int nrOfUnexecutedTests, int nrOfNotFoundTests = 0)
{
TestExecutor executor = new TestExecutor(TestEnvironment);
executor.RunTests(executable.Yield(), MockRunContext.Object, MockFrameworkHandle.Object);
CheckMockInvocations(nrOfPassedTests, nrOfFailedTests, nrOfUnexecutedTests, nrOfNotFoundTests);
}
public virtual void RunTests_WithoutPathExtension_ExecutionFails()
{
var executor = new TestExecutor(TestEnvironment);
executor.RunTests(TestResources.PathExtensionTestsExe.Yield(), MockRunContext.Object, MockFrameworkHandle.Object);
MockFrameworkHandle.Verify(h => h.RecordResult(It.IsAny<VsTestResult>()), Times.Never);
MockLogger.Verify(l => l.LogWarning(It.IsAny<string>()), Times.Once);
}
public virtual void RunTests_WithPathExtension_ExecutionOk()
{
MockOptions.Setup(o => o.PathExtension).Returns(SettingsWrapper.ExecutableDirPlaceholder + @"\..\lib");
var executor = new TestExecutor(TestEnvironment);
executor.RunTests(TestResources.PathExtensionTestsExe.Yield(), MockRunContext.Object, MockFrameworkHandle.Object);
MockFrameworkHandle.Verify(h => h.RecordResult(It.Is<VsTestResult>(tr => tr.Outcome == VsTestOutcome.Passed)), Times.Exactly(29));
MockLogger.Verify(l => l.LogError(It.IsAny<string>()), Times.Never);
}
public virtual void RunTests_HardCrashingX86Tests_CorrectTestResults()
{
TestExecutor executor = new TestExecutor(TestEnvironment);
executor.RunTests(TestResources.HardCrashingSampleTests.Yield(), MockRunContext.Object, MockFrameworkHandle.Object);
CheckMockInvocations(1, 2, 0, 3);
}