Inheritance: ITestExecutor
        public void RunTests_ParallelTestExecution_SpeedsUpTestExecution()
        {
            MockOptions.Setup(o => o.ParallelTestExecution).Returns(false);

            Stopwatch stopwatch = new Stopwatch();
            TestExecutor executor = new TestExecutor(TestEnvironment);
            IEnumerable<string> testsToRun = TestResources.SampleTests.Yield();
            stopwatch.Start();
            executor.RunTests(testsToRun, MockRunContext.Object, MockFrameworkHandle.Object);
            stopwatch.Stop();
            long sequentialDuration = stopwatch.ElapsedMilliseconds;

            MockOptions.Setup(o => o.ParallelTestExecution).Returns(true);
            MockOptions.Setup(o => o.MaxNrOfThreads).Returns(Environment.ProcessorCount);

            executor = new TestExecutor(TestEnvironment);
            testsToRun = TestResources.SampleTests.Yield();
            stopwatch.Restart();
            executor.RunTests(testsToRun, MockRunContext.Object, MockFrameworkHandle.Object);
            stopwatch.Stop();
            long parallelDuration = stopwatch.ElapsedMilliseconds;

            sequentialDuration.Should().BeGreaterThan(4000); // 2 long tests, 2 seconds per test
            parallelDuration.Should().BeGreaterThan(2000);
            parallelDuration.Should().BeLessThan(4000); // 2 seconds per long test + some time for the rest
        }
        private void RunAndVerifySingleTest(TestCase testCase, VsTestOutcome expectedOutcome)
        {
            TestExecutor executor = new TestExecutor(TestEnvironment);
            executor.RunTests(testCase.ToVsTestCase().Yield(), MockRunContext.Object, MockFrameworkHandle.Object);

            foreach (VsTestOutcome outcome in Enum.GetValues(typeof(VsTestOutcome)))
            {
                MockFrameworkHandle.Verify(h => h.RecordEnd(It.IsAny<VsTestCase>(), It.Is<VsTestOutcome>(to => to == outcome)),
                    Times.Exactly(outcome == expectedOutcome ? 1 : 0));
            }
        }
        public void RunTests_CancelingExecutor_StopsTestExecution()
        {
            List<Model.TestCase> testCasesToRun = TestDataCreator.GetTestCasesOfSampleTests("Crashing.LongRunning", "LongRunningTests.Test3");

            Stopwatch stopwatch = new Stopwatch();
            TestExecutor executor = new TestExecutor(TestEnvironment);
            Thread thread = new Thread(() => executor.RunTests(testCasesToRun.Select(DataConversionExtensions.ToVsTestCase), MockRunContext.Object, MockFrameworkHandle.Object));

            stopwatch.Start();
            thread.Start();
            Thread.Sleep(1000);
            executor.Cancel();
            thread.Join();
            stopwatch.Stop();

            stopwatch.ElapsedMilliseconds.Should().BeInRange(3000,  // 1st test should be executed
                                                             4000); // 2nd test should not be executed 
        }
        protected void RunMemoryLeakTest(string executable, string testCaseName, VsTestOutcome testOutcome, VsTestOutcome leakCheckOutcome, Func <string, bool> errorMessagePredicate)
        {
            string exitCodeTestName = "MemoryLeakTest";

            MockOptions.Setup(o => o.ExitCodeTestCase).Returns(exitCodeTestName);
            MockOptions.Setup(o => o.AdditionalTestExecutionParam).Returns("-is_run_by_gta");

            var testCases = new GoogleTestDiscoverer(MockLogger.Object, TestEnvironment.Options,
                                                     new ProcessExecutorFactory(), new DefaultDiaResolverFactory()).GetTestsFromExecutable(executable);
            var testCase = testCases.Single(tc => tc.DisplayName == testCaseName);

            var executor = new TestExecutor(TestEnvironment.Logger, TestEnvironment.Options, MockDebuggerAttacher.Object);

            executor.RunTests(testCase.Yield().Select(tc => tc.ToVsTestCase()), MockRunContext.Object, MockFrameworkHandle.Object);

            var invocations = new List <IInvocation>();

            foreach (var invocation in MockFrameworkHandle.Invocations)
            {
                invocations.Add(invocation);
            }

            MockFrameworkHandle.Verify(h => h.RecordResult(It.Is <VsTestResult>(result =>
                                                                                result.TestCase.FullyQualifiedName == testCaseName &&
                                                                                result.Outcome == testOutcome
                                                                                )),
                                       Times.Once);

            MockFrameworkHandle.Verify(h => h.RecordResult(It.Is <VsTestResult>(result =>
                                                                                result.TestCase.FullyQualifiedName.StartsWith("MemoryLeakTest") &&
                                                                                result.Outcome == leakCheckOutcome &&
                                                                                (result.ErrorMessage == null || !result.ErrorMessage.Contains(StreamingStandardOutputTestResultParser.GtaExitCodeOutputBegin)) &&
                                                                                (result.ErrorMessage == null || !result.ErrorMessage.Contains(StreamingStandardOutputTestResultParser.GtaExitCodeOutputEnd)) &&
                                                                                errorMessagePredicate(result.ErrorMessage)
                                                                                )),
                                       Times.Once);

            MockLogger.Verify(l => l.DebugWarning(It.Is <string>(msg => msg.Contains("main method") && msg.Contains("exit code"))), Times.Never);
        }
Ejemplo n.º 5
0
        public virtual void RunTests_TestDirectoryViaUserParams_IsPassedViaCommandLineArg()
        {
            TestCase testCase = TestDataCreator.GetTestCasesOfSampleTests("CommandArgs.TestDirectoryIsSet").First();

            TestExecutor executor = new TestExecutor(TestEnvironment);

            executor.RunTests(testCase.ToVsTestCase().Yield(), MockRunContext.Object, MockFrameworkHandle.Object);

            MockFrameworkHandle.Verify(h => h.RecordEnd(It.IsAny <Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase>(), It.Is <Microsoft.VisualStudio.TestPlatform.ObjectModel.TestOutcome>(to => to == Microsoft.VisualStudio.TestPlatform.ObjectModel.TestOutcome.Passed)),
                                       Times.Exactly(0));
            MockFrameworkHandle.Verify(h => h.RecordEnd(It.IsAny <Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase>(), It.Is <Microsoft.VisualStudio.TestPlatform.ObjectModel.TestOutcome>(to => to == Microsoft.VisualStudio.TestPlatform.ObjectModel.TestOutcome.Failed)),
                                       Times.Exactly(1));

            MockFrameworkHandle.Reset();
            MockOptions.Setup(o => o.AdditionalTestExecutionParam).Returns("-testdirectory=\"" + SettingsWrapper.TestDirPlaceholder + "\"");

            executor = new TestExecutor(TestEnvironment);
            executor.RunTests(testCase.ToVsTestCase().Yield(), MockRunContext.Object, MockFrameworkHandle.Object);

            MockFrameworkHandle.Verify(h => h.RecordEnd(It.IsAny <Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase>(), It.Is <Microsoft.VisualStudio.TestPlatform.ObjectModel.TestOutcome>(to => to == Microsoft.VisualStudio.TestPlatform.ObjectModel.TestOutcome.Failed)),
                                       Times.Exactly(0));
            MockFrameworkHandle.Verify(h => h.RecordEnd(It.IsAny <Microsoft.VisualStudio.TestPlatform.ObjectModel.TestCase>(), It.Is <Microsoft.VisualStudio.TestPlatform.ObjectModel.TestOutcome>(to => to == Microsoft.VisualStudio.TestPlatform.ObjectModel.TestOutcome.Passed)),
                                       Times.Exactly(1));
        }
        private void RunAndVerifyTests(string executable, int nrOfPassedTests, int nrOfFailedTests, int nrOfUnexecutedTests, int nrOfNotFoundTests = 0)
        {
            TestExecutor executor = new TestExecutor(TestEnvironment);
            executor.RunTests(executable.Yield(), MockRunContext.Object, MockFrameworkHandle.Object);

            CheckMockInvocations(nrOfPassedTests, nrOfFailedTests, nrOfUnexecutedTests, nrOfNotFoundTests);
        }
        public virtual void RunTests_WithoutPathExtension_ExecutionFails()
        {
            var executor = new TestExecutor(TestEnvironment);
            executor.RunTests(TestResources.PathExtensionTestsExe.Yield(), MockRunContext.Object, MockFrameworkHandle.Object);

            MockFrameworkHandle.Verify(h => h.RecordResult(It.IsAny<VsTestResult>()), Times.Never);
            MockLogger.Verify(l => l.LogWarning(It.IsAny<string>()), Times.Once);
        }
        public virtual void RunTests_WithPathExtension_ExecutionOk()
        {
            MockOptions.Setup(o => o.PathExtension).Returns(SettingsWrapper.ExecutableDirPlaceholder + @"\..\lib");

            var executor = new TestExecutor(TestEnvironment);
            executor.RunTests(TestResources.PathExtensionTestsExe.Yield(), MockRunContext.Object, MockFrameworkHandle.Object);

            MockFrameworkHandle.Verify(h => h.RecordResult(It.Is<VsTestResult>(tr => tr.Outcome == VsTestOutcome.Passed)), Times.Exactly(29));
            MockLogger.Verify(l => l.LogError(It.IsAny<string>()), Times.Never);
        }
        public virtual void RunTests_HardCrashingX86Tests_CorrectTestResults()
        {
            TestExecutor executor = new TestExecutor(TestEnvironment);
            executor.RunTests(TestResources.HardCrashingSampleTests.Yield(), MockRunContext.Object, MockFrameworkHandle.Object);

            CheckMockInvocations(1, 2, 0, 3);
        }