private static void DiscoverTests(TestEnvironment testEnv, string[] sources, MockRunSettings runSettings, TestInfo[] expectedTests) { var discoveryContext = new MockDiscoveryContext(runSettings); var discoverySink = new MockTestCaseDiscoverySink(); var logger = new MockMessageLogger(); ITestDiscoverer discoverer = null; switch (testEnv.TestFramework) { case FrameworkPytest: discoverer = new PytestTestDiscoverer(); break; case FrameworkUnittest: discoverer = new UnittestTestDiscoverer(); break; default: Assert.Fail($"unknown testframework: {testEnv.TestFramework.ToString()}"); break; } discoverer.DiscoverTests(sources, discoveryContext, logger, discoverySink); ValidateDiscoveredTests(testEnv.TestFramework, discoverySink.Tests, expectedTests); }
public void DiscoverPytestNotInstalled() { var testEnv = TestEnvironment.GetOrCreate(Version, FrameworkPytest, installFramework: false); var testFilePath = Path.Combine(testEnv.SourceFolderPath, "test_pt.py"); File.Copy(TestData.GetPath("TestData", "TestDiscoverer", "BasicPytest", "test_pt.py"), testFilePath); var runSettings = new MockRunSettings( new MockRunSettingsXmlBuilder(testEnv.TestFramework, testEnv.InterpreterPath, testEnv.ResultsFolderPath, testEnv.SourceFolderPath) .WithTestFilesFromFolder(testEnv.SourceFolderPath) .ToXml() ); var discoveryContext = new MockDiscoveryContext(runSettings); var discoverySink = new MockTestCaseDiscoverySink(); var logger = new MockMessageLogger(); var discoverer = new PytestTestDiscoverer(); discoverer.DiscoverTests(new[] { testFilePath }, discoveryContext, logger, discoverySink); Assert.AreEqual(0, discoverySink.Tests.Count); var errors = string.Join(Environment.NewLine, logger.GetErrors()); AssertUtil.Contains(errors, string.Format(ImportErrorFormat, "pytest")); }
public void DiscoverUnittestTimeoutError() { var testEnv = TestEnvironment.GetOrCreate(Version, FrameworkUnittest); var testFilePath = Path.Combine(testEnv.SourceFolderPath, "test_ut.py"); File.Copy(TestData.GetPath("TestData", "TestDiscoverer", "Timeout", "test_timeout_ut.py"), testFilePath); int waitTimeInSeconds = 1; var runSettings = new MockRunSettings( new MockRunSettingsXmlBuilder(testEnv.TestFramework, testEnv.InterpreterPath, testEnv.ResultsFolderPath, testEnv.SourceFolderPath, waitTimeInSeconds) .WithTestFilesFromFolder(testEnv.SourceFolderPath) .ToXml() ); var discoveryContext = new MockDiscoveryContext(runSettings); var discoverySink = new MockTestCaseDiscoverySink(); var logger = new MockMessageLogger(); var discoverer = new UnittestTestDiscoverer(); discoverer.DiscoverTests(new[] { testFilePath }, discoveryContext, logger, discoverySink); Assert.AreEqual(0, discoverySink.Tests.Count); var errors = string.Join(Environment.NewLine, logger.GetErrors()); AssertUtil.Contains( errors, Strings.PythonTestDiscovererTimeoutErrorMessage ); }
public void DiscoverUnitTestSyntaxErrorPartialResults() { // one file has a valid passing test, // the other has an unknown module import at global scope var testEnv = TestEnvironment.GetOrCreate(Version, FrameworkUnittest); FileUtils.CopyDirectory(TestData.GetPath("TestData", "TestDiscoverer", "SyntaxErrorUnittest"), testEnv.SourceFolderPath); var testFilePath1 = Path.Combine(testEnv.SourceFolderPath, "test_basic_ut.py"); var testFilePath2 = Path.Combine(testEnv.SourceFolderPath, "test_syntax_error_ut.py"); var expectedTests = new[] { new TestInfo("test_ut_fail", "test_basic_ut.py::TestClassUT::test_ut_fail", testFilePath1, 4), new TestInfo("test_ut_pass", "test_basic_ut.py::TestClassUT::test_ut_pass", testFilePath1, 7), }; var runSettings = new MockRunSettings( new MockRunSettingsXmlBuilder(testEnv.TestFramework, testEnv.InterpreterPath, testEnv.ResultsFolderPath, testEnv.SourceFolderPath) .WithTestFile(testFilePath1) .WithTestFile(testFilePath2) .ToXml() ); var discoveryContext = new MockDiscoveryContext(runSettings); var discoverySink = new MockTestCaseDiscoverySink(); var logger = new MockMessageLogger(); var discoverer = new UnittestTestDiscoverer(); discoverer.DiscoverTests(new[] { testFilePath1, testFilePath2 }, discoveryContext, logger, discoverySink); ValidateDiscoveredTests(testEnv.TestFramework, discoverySink.Tests, expectedTests); }
public void DiscoverPytestSyntaxErrorLogErrors() { // one file has a valid passing test, // the other has a test with a syntax error in it var testEnv = TestEnvironment.GetOrCreate(Version, FrameworkPytest); FileUtils.CopyDirectory(TestData.GetPath("TestData", "TestDiscoverer", "SyntaxErrorPytest"), testEnv.SourceFolderPath); var testFilePath1 = Path.Combine(testEnv.SourceFolderPath, "test_basic.py"); var testFilePath2 = Path.Combine(testEnv.SourceFolderPath, "test_syntax_error.py"); var expectedTests = new[] { new TestInfo("test_success", "test_basic.py::test_basic::test_success", testFilePath1, 1), }; var runSettings = new MockRunSettings( new MockRunSettingsXmlBuilder(testEnv.TestFramework, testEnv.InterpreterPath, testEnv.ResultsFolderPath, testEnv.SourceFolderPath) .WithTestFile(testFilePath1) .WithTestFile(testFilePath2) .ToXml() ); var discoveryContext = new MockDiscoveryContext(runSettings); var discoverySink = new MockTestCaseDiscoverySink(); var logger = new MockMessageLogger(); var discoverer = new PytestTestDiscoverer(); discoverer.DiscoverTests(new[] { testFilePath1, testFilePath2 }, discoveryContext, logger, discoverySink); var errors = string.Join(Environment.NewLine, logger.GetErrors()); AssertUtil.Contains(errors, "SyntaxError: invalid syntax" ); }
private MockMessageLogger NewMethod() { TestEnvironment testEnv = TestEnvironment.GetOrCreate(Version, FrameworkUnittest); FileUtils.CopyDirectory(TestData.GetPath("TestData", "TestDiscoverer", "SyntaxErrorUnittest"), testEnv.SourceFolderPath); var testFilePath1 = Path.Combine(testEnv.SourceFolderPath, "test_basic_ut.py"); var testFilePath2 = Path.Combine(testEnv.SourceFolderPath, "test_syntax_error_ut.py"); TestInfo[] expectedTests = new[] { new TestInfo("test_ut_fail", "test_basic_ut.py::TestClassUT::test_ut_fail", testFilePath1, 4), new TestInfo("test_ut_pass", "test_basic_ut.py::TestClassUT::test_ut_pass", testFilePath1, 7), }; MockRunSettings runSettings = new MockRunSettings( new MockRunSettingsXmlBuilder(testEnv.TestFramework, testEnv.InterpreterPath, testEnv.ResultsFolderPath, testEnv.SourceFolderPath) .WithTestFile(testFilePath1) .WithTestFile(testFilePath2) .ToXml() ); MockDiscoveryContext discoveryContext = new MockDiscoveryContext(runSettings); MockTestCaseDiscoverySink discoverySink = new MockTestCaseDiscoverySink(); MockMessageLogger logger = new MockMessageLogger(); var discoverer = new UnittestTestDiscoverer(); discoverer.DiscoverTests(new[] { testFilePath1, testFilePath2 }, discoveryContext, logger, discoverySink); return(logger); }
public void TestDiscover() { var ctx = new MockDiscoveryContext(new MockRunSettings(_runSettings)); var sink = new MockTestCaseDiscoverySink(); var logger = new MockMessageLogger(); const string projectPath = @"C:\Visual Studio 2015\Projects\PythonApplication107\PythonApplication107\PythonApplication107.pyproj"; const string testFilePath = @"C:\Visual Studio 2015\Projects\PythonApplication107\PythonApplication107\test1.py"; new TestDiscoverer().DiscoverTests( new[] { testFilePath }, ctx, logger, sink ); PrintTestCases(sink.Tests); var expectedTests = new[] { TestInfo.FromRelativePaths("Test_test1", "test_A", projectPath, testFilePath, 17, TestOutcome.Passed), TestInfo.FromRelativePaths("Test_test1", "test_B", projectPath, testFilePath, 21, TestOutcome.Passed), TestInfo.FromRelativePaths("Test_test2", "test_C", projectPath, testFilePath, 48, TestOutcome.Passed) }; Assert.AreEqual(expectedTests.Length, sink.Tests.Count); foreach (var expectedTest in expectedTests) { var expectedFullyQualifiedName = TestReader.MakeFullyQualifiedTestName(expectedTest.RelativeClassFilePath, expectedTest.ClassName, expectedTest.MethodName); var actualTestCase = sink.Tests.SingleOrDefault(tc => tc.FullyQualifiedName == expectedFullyQualifiedName); Assert.IsNotNull(actualTestCase, expectedFullyQualifiedName); Assert.AreEqual(expectedTest.MethodName, actualTestCase.DisplayName, expectedFullyQualifiedName); Assert.AreEqual(new Uri(PythonConstants.TestExecutorUriString), actualTestCase.ExecutorUri); Assert.AreEqual(expectedTest.SourceCodeLineNumber, actualTestCase.LineNumber, expectedFullyQualifiedName); Assert.IsTrue(IsSameFile(expectedTest.SourceCodeFilePath, actualTestCase.CodeFilePath), expectedFullyQualifiedName); sink.Tests.Remove(actualTestCase); } Debug.WriteLine(""); Debug.WriteLine(""); Debug.WriteLine(""); Debug.WriteLine(""); PrintTestCases(sink.Tests); }
public void DiscoverUnitTestSyntaxErrorLogErrors() { // one file has a valid passing test, // the other has an unknown module import at global scope var testEnv = TestEnvironment.GetOrCreate(Version, FrameworkUnittest); FileUtils.CopyDirectory(TestData.GetPath("TestData", "TestDiscoverer", "SyntaxErrorUnittest"), testEnv.SourceFolderPath); var testFilePath1 = Path.Combine(testEnv.SourceFolderPath, "test_basic_ut.py"); var testFilePath2 = Path.Combine(testEnv.SourceFolderPath, "test_syntax_error_ut.py"); var expectedTests = new[] { new TestInfo("test_ut_fail", "test_basic_ut.py::TestClassUT::test_ut_fail", testFilePath1, 4), new TestInfo("test_ut_pass", "test_basic_ut.py::TestClassUT::test_ut_pass", testFilePath1, 7), }; var runSettings = new MockRunSettings( new MockRunSettingsXmlBuilder(testEnv.TestFramework, testEnv.InterpreterPath, testEnv.ResultsFolderPath, testEnv.SourceFolderPath) .WithTestFile(testFilePath1) .WithTestFile(testFilePath2) .ToXml() ); var discoveryContext = new MockDiscoveryContext(runSettings); var discoverySink = new MockTestCaseDiscoverySink(); var logger = new MockMessageLogger(); var discoverer = new UnittestTestDiscoverer(); discoverer.DiscoverTests(new[] { testFilePath1, testFilePath2 }, discoveryContext, logger, discoverySink); var errors = string.Join(Environment.NewLine, logger.GetErrors()); if (Version.Version > PythonLanguageVersion.V27) { AssertUtil.Contains(errors, "SyntaxError: invalid syntax" ); } else { Assert.Inconclusive("Python 2.7 unittest errors are not currently being printed to error logs"); } }
public void TestDiscover() { var ctx = new MockDiscoveryContext(); var sink = new MockTestCaseDiscoverySink(); var logger = new MockMessageLogger(); new TestDiscoverer().DiscoverTests( new[] { TestInfo.TestAdapterLibProjectFilePath, TestInfo.TestAdapterAProjectFilePath, TestInfo.TestAdapterBProjectFilePath }, ctx, logger, sink ); PrintTestCases(sink.Tests); var expectedTests = TestInfo.TestAdapterATests.Concat(TestInfo.TestAdapterBTests).ToArray(); Assert.AreEqual(expectedTests.Length, sink.Tests.Count); foreach (var expectedTest in expectedTests) { var expectedFullyQualifiedName = TestAnalyzer.MakeFullyQualifiedTestName(expectedTest.RelativeClassFilePath, expectedTest.ClassName, expectedTest.MethodName); var actualTestCase = sink.Tests.SingleOrDefault(tc => tc.FullyQualifiedName == expectedFullyQualifiedName); Assert.IsNotNull(actualTestCase, expectedFullyQualifiedName); Assert.AreEqual(expectedTest.MethodName, actualTestCase.DisplayName, expectedFullyQualifiedName); Assert.AreEqual(new Uri(TestExecutor.ExecutorUriString), actualTestCase.ExecutorUri); Assert.AreEqual(expectedTest.SourceCodeLineNumber, actualTestCase.LineNumber, expectedFullyQualifiedName); Assert.IsTrue(IsSameFile(expectedTest.SourceCodeFilePath, actualTestCase.CodeFilePath), expectedFullyQualifiedName); Assert.IsTrue(IsSameFile(expectedTest.ProjectFilePath, actualTestCase.Source), expectedFullyQualifiedName); sink.Tests.Remove(actualTestCase); } Debug.WriteLine(""); Debug.WriteLine(""); Debug.WriteLine(""); Debug.WriteLine(""); PrintTestCases(sink.Tests); }