public void RunTests_ParallelTestExecution_SpeedsUpTestExecution() { MockOptions.Setup(o => o.ParallelTestExecution).Returns(false); Stopwatch stopwatch = new Stopwatch(); TestExecutor executor = new TestExecutor(TestEnvironment); IEnumerable<string> testsToRun = TestResources.SampleTests.Yield(); stopwatch.Start(); executor.RunTests(testsToRun, MockRunContext.Object, MockFrameworkHandle.Object); stopwatch.Stop(); long sequentialDuration = stopwatch.ElapsedMilliseconds; MockOptions.Setup(o => o.ParallelTestExecution).Returns(true); MockOptions.Setup(o => o.MaxNrOfThreads).Returns(Environment.ProcessorCount); executor = new TestExecutor(TestEnvironment); testsToRun = TestResources.SampleTests.Yield(); stopwatch.Restart(); executor.RunTests(testsToRun, MockRunContext.Object, MockFrameworkHandle.Object); stopwatch.Stop(); long parallelDuration = stopwatch.ElapsedMilliseconds; sequentialDuration.Should().BeGreaterThan(4000); // 2 long tests, 2 seconds per test parallelDuration.Should().BeGreaterThan(2000); parallelDuration.Should().BeLessThan(4000); // 2 seconds per long test + some time for the rest }
private void RunAndVerifySingleTest(TestCase testCase, VsTestOutcome expectedOutcome) { TestExecutor executor = new TestExecutor(TestEnvironment); executor.RunTests(testCase.ToVsTestCase().Yield(), MockRunContext.Object, MockFrameworkHandle.Object); foreach (VsTestOutcome outcome in Enum.GetValues(typeof(VsTestOutcome))) { MockFrameworkHandle.Verify(h => h.RecordEnd(It.IsAny<VsTestCase>(), It.Is<VsTestOutcome>(to => to == outcome)), Times.Exactly(outcome == expectedOutcome ? 1 : 0)); } }
public void RunTests_CancelingExecutor_StopsTestExecution() { List<Model.TestCase> testCasesToRun = TestDataCreator.GetTestCasesOfSampleTests("Crashing.LongRunning", "LongRunningTests.Test3"); Stopwatch stopwatch = new Stopwatch(); TestExecutor executor = new TestExecutor(TestEnvironment); Thread thread = new Thread(() => executor.RunTests(testCasesToRun.Select(DataConversionExtensions.ToVsTestCase), MockRunContext.Object, MockFrameworkHandle.Object)); stopwatch.Start(); thread.Start(); Thread.Sleep(1000); executor.Cancel(); thread.Join(); stopwatch.Stop(); stopwatch.ElapsedMilliseconds.Should().BeInRange(3000, // 1st test should be executed 4000); // 2nd test should not be executed }
protected void RunMemoryLeakTest(string executable, string testCaseName, VsTestOutcome testOutcome, VsTestOutcome leakCheckOutcome, Func <string, bool> errorMessagePredicate) { string exitCodeTestName = "MemoryLeakTest"; MockOptions.Setup(o => o.ExitCodeTestCase).Returns(exitCodeTestName); MockOptions.Setup(o => o.AdditionalTestExecutionParam).Returns("-is_run_by_gta"); var testCases = new GoogleTestDiscoverer(MockLogger.Object, TestEnvironment.Options, new ProcessExecutorFactory(), new DefaultDiaResolverFactory()).GetTestsFromExecutable(executable); var testCase = testCases.Single(tc => tc.DisplayName == testCaseName); var executor = new TestExecutor(TestEnvironment.Logger, TestEnvironment.Options, MockDebuggerAttacher.Object); executor.RunTests(testCase.Yield().Select(tc => tc.ToVsTestCase()), MockRunContext.Object, MockFrameworkHandle.Object); var invocations = new List <IInvocation>(); foreach (var invocation in MockFrameworkHandle.Invocations) { invocations.Add(invocation); } MockFrameworkHandle.Verify(h => h.RecordResult(It.Is <VsTestResult>(result => result.TestCase.FullyQualifiedName == testCaseName && result.Outcome == testOutcome )), Times.Once); MockFrameworkHandle.Verify(h => h.RecordResult(It.Is <VsTestResult>(result => result.TestCase.FullyQualifiedName.StartsWith("MemoryLeakTest") && result.Outcome == leakCheckOutcome && (result.ErrorMessage == null || !result.ErrorMessage.Contains(StreamingStandardOutputTestResultParser.GtaExitCodeOutputBegin)) && (result.ErrorMessage == null || !result.ErrorMessage.Contains(StreamingStandardOutputTestResultParser.GtaExitCodeOutputEnd)) && errorMessagePredicate(result.ErrorMessage) )), Times.Once); MockLogger.Verify(l => l.DebugWarning(It.Is <string>(msg => msg.Contains("main method") && msg.Contains("exit code"))), Times.Never); }
public virtual void RunTests_TestDirectoryViaUserParams_IsPassedViaCommandLineArg() { TestCase testCase = TestDataCreator.GetTestCasesOfSampleTests("CommandArgs.TestDirectoryIsSet").First(); TestExecutor executor = new TestExecutor(TestEnvironment); executor.RunTests(testCase.ToVsTestCase().Yield(), MockRunContext.Object, MockFrameworkHandle.Object); MockFrameworkHandle.Verify(h => h.RecordEnd(It.IsAny <Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase>(), It.Is <Microsoft.VisualStudio.TestPlatform.ObjectModel.TestOutcome>(to => to == Microsoft.VisualStudio.TestPlatform.ObjectModel.TestOutcome.Passed)), Times.Exactly(0)); MockFrameworkHandle.Verify(h => h.RecordEnd(It.IsAny <Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase>(), It.Is <Microsoft.VisualStudio.TestPlatform.ObjectModel.TestOutcome>(to => to == Microsoft.VisualStudio.TestPlatform.ObjectModel.TestOutcome.Failed)), Times.Exactly(1)); MockFrameworkHandle.Reset(); MockOptions.Setup(o => o.AdditionalTestExecutionParam).Returns("-testdirectory=\"" + SettingsWrapper.TestDirPlaceholder + "\""); executor = new TestExecutor(TestEnvironment); executor.RunTests(testCase.ToVsTestCase().Yield(), MockRunContext.Object, MockFrameworkHandle.Object); MockFrameworkHandle.Verify(h => h.RecordEnd(It.IsAny <Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase>(), It.Is <Microsoft.VisualStudio.TestPlatform.ObjectModel.TestOutcome>(to => to == Microsoft.VisualStudio.TestPlatform.ObjectModel.TestOutcome.Failed)), Times.Exactly(0)); MockFrameworkHandle.Verify(h => h.RecordEnd(It.IsAny <Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase>(), It.Is <Microsoft.VisualStudio.TestPlatform.ObjectModel.TestOutcome>(to => to == Microsoft.VisualStudio.TestPlatform.ObjectModel.TestOutcome.Passed)), Times.Exactly(1)); }
private void RunAndVerifyTests(string executable, int nrOfPassedTests, int nrOfFailedTests, int nrOfUnexecutedTests, int nrOfNotFoundTests = 0) { TestExecutor executor = new TestExecutor(TestEnvironment); executor.RunTests(executable.Yield(), MockRunContext.Object, MockFrameworkHandle.Object); CheckMockInvocations(nrOfPassedTests, nrOfFailedTests, nrOfUnexecutedTests, nrOfNotFoundTests); }
public virtual void RunTests_WithoutPathExtension_ExecutionFails() { var executor = new TestExecutor(TestEnvironment); executor.RunTests(TestResources.PathExtensionTestsExe.Yield(), MockRunContext.Object, MockFrameworkHandle.Object); MockFrameworkHandle.Verify(h => h.RecordResult(It.IsAny<VsTestResult>()), Times.Never); MockLogger.Verify(l => l.LogWarning(It.IsAny<string>()), Times.Once); }
public virtual void RunTests_WithPathExtension_ExecutionOk() { MockOptions.Setup(o => o.PathExtension).Returns(SettingsWrapper.ExecutableDirPlaceholder + @"\..\lib"); var executor = new TestExecutor(TestEnvironment); executor.RunTests(TestResources.PathExtensionTestsExe.Yield(), MockRunContext.Object, MockFrameworkHandle.Object); MockFrameworkHandle.Verify(h => h.RecordResult(It.Is<VsTestResult>(tr => tr.Outcome == VsTestOutcome.Passed)), Times.Exactly(29)); MockLogger.Verify(l => l.LogError(It.IsAny<string>()), Times.Never); }
public virtual void RunTests_HardCrashingX86Tests_CorrectTestResults() { TestExecutor executor = new TestExecutor(TestEnvironment); executor.RunTests(TestResources.HardCrashingSampleTests.Yield(), MockRunContext.Object, MockFrameworkHandle.Object); CheckMockInvocations(1, 2, 0, 3); }