private static void DiscoverTests(TestEnvironment testEnv, string[] sources, MockRunSettings runSettings, TestInfo[] expectedTests)
{
var discoveryContext = new MockDiscoveryContext(runSettings);
var discoverySink = new MockTestCaseDiscoverySink();
var logger = new MockMessageLogger();
ITestDiscoverer discoverer = null;
switch (testEnv.TestFramework)
{
case FrameworkPytest:
discoverer = new PytestTestDiscoverer();
break;
case FrameworkUnittest:
discoverer = new UnittestTestDiscoverer();
break;
default:
Assert.Fail($"unknown testframework: {testEnv.TestFramework.ToString()}");
break;
}
discoverer.DiscoverTests(sources, discoveryContext, logger, discoverySink);
ValidateDiscoveredTests(testEnv.TestFramework, discoverySink.Tests, expectedTests);
}
public void DiscoverPytestNotInstalled()
{
var testEnv = TestEnvironment.GetOrCreate(Version, FrameworkPytest, installFramework: false);
var testFilePath = Path.Combine(testEnv.SourceFolderPath, "test_pt.py");
File.Copy(TestData.GetPath("TestData", "TestDiscoverer", "BasicPytest", "test_pt.py"), testFilePath);
var runSettings = new MockRunSettings(
new MockRunSettingsXmlBuilder(testEnv.TestFramework, testEnv.InterpreterPath, testEnv.ResultsFolderPath, testEnv.SourceFolderPath)
.WithTestFilesFromFolder(testEnv.SourceFolderPath)
.ToXml()
);
var discoveryContext = new MockDiscoveryContext(runSettings);
var discoverySink = new MockTestCaseDiscoverySink();
var logger = new MockMessageLogger();
var discoverer = new PytestTestDiscoverer();
discoverer.DiscoverTests(new[] { testFilePath }, discoveryContext, logger, discoverySink);
Assert.AreEqual(0, discoverySink.Tests.Count);
var errors = string.Join(Environment.NewLine, logger.GetErrors());
AssertUtil.Contains(errors, string.Format(ImportErrorFormat, "pytest"));
}
public void DiscoverUnittestTimeoutError()
{
var testEnv = TestEnvironment.GetOrCreate(Version, FrameworkUnittest);
var testFilePath = Path.Combine(testEnv.SourceFolderPath, "test_ut.py");
File.Copy(TestData.GetPath("TestData", "TestDiscoverer", "Timeout", "test_timeout_ut.py"), testFilePath);
int waitTimeInSeconds = 1;
var runSettings = new MockRunSettings(
new MockRunSettingsXmlBuilder(testEnv.TestFramework, testEnv.InterpreterPath, testEnv.ResultsFolderPath, testEnv.SourceFolderPath, waitTimeInSeconds)
.WithTestFilesFromFolder(testEnv.SourceFolderPath)
.ToXml()
);
var discoveryContext = new MockDiscoveryContext(runSettings);
var discoverySink = new MockTestCaseDiscoverySink();
var logger = new MockMessageLogger();
var discoverer = new UnittestTestDiscoverer();
discoverer.DiscoverTests(new[] { testFilePath }, discoveryContext, logger, discoverySink);
Assert.AreEqual(0, discoverySink.Tests.Count);
var errors = string.Join(Environment.NewLine, logger.GetErrors());
AssertUtil.Contains(
errors,
Strings.PythonTestDiscovererTimeoutErrorMessage
);
}
public void DiscoverUnitTestSyntaxErrorPartialResults()
{
// one file has a valid passing test,
// the other has an unknown module import at global scope
var testEnv = TestEnvironment.GetOrCreate(Version, FrameworkUnittest);
FileUtils.CopyDirectory(TestData.GetPath("TestData", "TestDiscoverer", "SyntaxErrorUnittest"), testEnv.SourceFolderPath);
var testFilePath1 = Path.Combine(testEnv.SourceFolderPath, "test_basic_ut.py");
var testFilePath2 = Path.Combine(testEnv.SourceFolderPath, "test_syntax_error_ut.py");
var expectedTests = new[] {
new TestInfo("test_ut_fail", "test_basic_ut.py::TestClassUT::test_ut_fail", testFilePath1, 4),
new TestInfo("test_ut_pass", "test_basic_ut.py::TestClassUT::test_ut_pass", testFilePath1, 7),
};
var runSettings = new MockRunSettings(
new MockRunSettingsXmlBuilder(testEnv.TestFramework, testEnv.InterpreterPath, testEnv.ResultsFolderPath, testEnv.SourceFolderPath)
.WithTestFile(testFilePath1)
.WithTestFile(testFilePath2)
.ToXml()
);
var discoveryContext = new MockDiscoveryContext(runSettings);
var discoverySink = new MockTestCaseDiscoverySink();
var logger = new MockMessageLogger();
var discoverer = new UnittestTestDiscoverer();
discoverer.DiscoverTests(new[] { testFilePath1, testFilePath2 }, discoveryContext, logger, discoverySink);
ValidateDiscoveredTests(testEnv.TestFramework, discoverySink.Tests, expectedTests);
}
public void DiscoverPytestSyntaxErrorLogErrors()
{
// one file has a valid passing test,
// the other has a test with a syntax error in it
var testEnv = TestEnvironment.GetOrCreate(Version, FrameworkPytest);
FileUtils.CopyDirectory(TestData.GetPath("TestData", "TestDiscoverer", "SyntaxErrorPytest"), testEnv.SourceFolderPath);
var testFilePath1 = Path.Combine(testEnv.SourceFolderPath, "test_basic.py");
var testFilePath2 = Path.Combine(testEnv.SourceFolderPath, "test_syntax_error.py");
var expectedTests = new[] {
new TestInfo("test_success", "test_basic.py::test_basic::test_success", testFilePath1, 1),
};
var runSettings = new MockRunSettings(
new MockRunSettingsXmlBuilder(testEnv.TestFramework, testEnv.InterpreterPath, testEnv.ResultsFolderPath, testEnv.SourceFolderPath)
.WithTestFile(testFilePath1)
.WithTestFile(testFilePath2)
.ToXml()
);
var discoveryContext = new MockDiscoveryContext(runSettings);
var discoverySink = new MockTestCaseDiscoverySink();
var logger = new MockMessageLogger();
var discoverer = new PytestTestDiscoverer();
discoverer.DiscoverTests(new[] { testFilePath1, testFilePath2 }, discoveryContext, logger, discoverySink);
var errors = string.Join(Environment.NewLine, logger.GetErrors());
AssertUtil.Contains(errors,
"SyntaxError: invalid syntax"
);
}
private MockMessageLogger NewMethod()
{
TestEnvironment testEnv = TestEnvironment.GetOrCreate(Version, FrameworkUnittest);
FileUtils.CopyDirectory(TestData.GetPath("TestData", "TestDiscoverer", "SyntaxErrorUnittest"), testEnv.SourceFolderPath);
var testFilePath1 = Path.Combine(testEnv.SourceFolderPath, "test_basic_ut.py");
var testFilePath2 = Path.Combine(testEnv.SourceFolderPath, "test_syntax_error_ut.py");
TestInfo[] expectedTests = new[] {
new TestInfo("test_ut_fail", "test_basic_ut.py::TestClassUT::test_ut_fail", testFilePath1, 4),
new TestInfo("test_ut_pass", "test_basic_ut.py::TestClassUT::test_ut_pass", testFilePath1, 7),
};
MockRunSettings runSettings = new MockRunSettings(
new MockRunSettingsXmlBuilder(testEnv.TestFramework, testEnv.InterpreterPath, testEnv.ResultsFolderPath, testEnv.SourceFolderPath)
.WithTestFile(testFilePath1)
.WithTestFile(testFilePath2)
.ToXml()
);
MockDiscoveryContext discoveryContext = new MockDiscoveryContext(runSettings);
MockTestCaseDiscoverySink discoverySink = new MockTestCaseDiscoverySink();
MockMessageLogger logger = new MockMessageLogger();
var discoverer = new UnittestTestDiscoverer();
discoverer.DiscoverTests(new[] { testFilePath1, testFilePath2 }, discoveryContext, logger, discoverySink);
return(logger);
}
public void TestDiscover()
{
var ctx = new MockDiscoveryContext(new MockRunSettings(_runSettings));
var sink = new MockTestCaseDiscoverySink();
var logger = new MockMessageLogger();
const string projectPath = @"C:\Visual Studio 2015\Projects\PythonApplication107\PythonApplication107\PythonApplication107.pyproj";
const string testFilePath = @"C:\Visual Studio 2015\Projects\PythonApplication107\PythonApplication107\test1.py";
new TestDiscoverer().DiscoverTests(
new[] { testFilePath },
ctx,
logger,
sink
);
PrintTestCases(sink.Tests);
var expectedTests = new[] {
TestInfo.FromRelativePaths("Test_test1", "test_A", projectPath, testFilePath, 17, TestOutcome.Passed),
TestInfo.FromRelativePaths("Test_test1", "test_B", projectPath, testFilePath, 21, TestOutcome.Passed),
TestInfo.FromRelativePaths("Test_test2", "test_C", projectPath, testFilePath, 48, TestOutcome.Passed)
};
Assert.AreEqual(expectedTests.Length, sink.Tests.Count);
foreach (var expectedTest in expectedTests)
{
var expectedFullyQualifiedName = TestReader.MakeFullyQualifiedTestName(expectedTest.RelativeClassFilePath, expectedTest.ClassName, expectedTest.MethodName);
var actualTestCase = sink.Tests.SingleOrDefault(tc => tc.FullyQualifiedName == expectedFullyQualifiedName);
Assert.IsNotNull(actualTestCase, expectedFullyQualifiedName);
Assert.AreEqual(expectedTest.MethodName, actualTestCase.DisplayName, expectedFullyQualifiedName);
Assert.AreEqual(new Uri(PythonConstants.TestExecutorUriString), actualTestCase.ExecutorUri);
Assert.AreEqual(expectedTest.SourceCodeLineNumber, actualTestCase.LineNumber, expectedFullyQualifiedName);
Assert.IsTrue(IsSameFile(expectedTest.SourceCodeFilePath, actualTestCase.CodeFilePath), expectedFullyQualifiedName);
sink.Tests.Remove(actualTestCase);
}
Debug.WriteLine("");
Debug.WriteLine("");
Debug.WriteLine("");
Debug.WriteLine("");
PrintTestCases(sink.Tests);
}
public void DiscoverUnitTestSyntaxErrorLogErrors()
{
// one file has a valid passing test,
// the other has an unknown module import at global scope
var testEnv = TestEnvironment.GetOrCreate(Version, FrameworkUnittest);
FileUtils.CopyDirectory(TestData.GetPath("TestData", "TestDiscoverer", "SyntaxErrorUnittest"), testEnv.SourceFolderPath);
var testFilePath1 = Path.Combine(testEnv.SourceFolderPath, "test_basic_ut.py");
var testFilePath2 = Path.Combine(testEnv.SourceFolderPath, "test_syntax_error_ut.py");
var expectedTests = new[] {
new TestInfo("test_ut_fail", "test_basic_ut.py::TestClassUT::test_ut_fail", testFilePath1, 4),
new TestInfo("test_ut_pass", "test_basic_ut.py::TestClassUT::test_ut_pass", testFilePath1, 7),
};
var runSettings = new MockRunSettings(
new MockRunSettingsXmlBuilder(testEnv.TestFramework, testEnv.InterpreterPath, testEnv.ResultsFolderPath, testEnv.SourceFolderPath)
.WithTestFile(testFilePath1)
.WithTestFile(testFilePath2)
.ToXml()
);
var discoveryContext = new MockDiscoveryContext(runSettings);
var discoverySink = new MockTestCaseDiscoverySink();
var logger = new MockMessageLogger();
var discoverer = new UnittestTestDiscoverer();
discoverer.DiscoverTests(new[] { testFilePath1, testFilePath2 }, discoveryContext, logger, discoverySink);
var errors = string.Join(Environment.NewLine, logger.GetErrors());
if (Version.Version > PythonLanguageVersion.V27)
{
AssertUtil.Contains(errors,
"SyntaxError: invalid syntax"
);
}
else
{
Assert.Inconclusive("Python 2.7 unittest errors are not currently being printed to error logs");
}
}
public void TestDiscover() {
var ctx = new MockDiscoveryContext(new MockRunSettings(_runSettings));
var sink = new MockTestCaseDiscoverySink();
var logger = new MockMessageLogger();
const string projectPath = @"C:\Visual Studio 2015\Projects\PythonApplication107\PythonApplication107\PythonApplication107.pyproj";
const string testFilePath = @"C:\Visual Studio 2015\Projects\PythonApplication107\PythonApplication107\test1.py";
new TestDiscoverer().DiscoverTests(
new[] { testFilePath },
ctx,
logger,
sink
);
PrintTestCases(sink.Tests);
var expectedTests = new[] {
TestInfo.FromRelativePaths("Test_test1", "test_A", projectPath, testFilePath, 17, TestOutcome.Passed),
TestInfo.FromRelativePaths("Test_test1", "test_B", projectPath, testFilePath, 21, TestOutcome.Passed),
TestInfo.FromRelativePaths("Test_test2", "test_C", projectPath, testFilePath, 48, TestOutcome.Passed)
};
Assert.AreEqual(expectedTests.Length, sink.Tests.Count);
foreach (var expectedTest in expectedTests) {
var expectedFullyQualifiedName = TestReader.MakeFullyQualifiedTestName(expectedTest.RelativeClassFilePath, expectedTest.ClassName, expectedTest.MethodName);
var actualTestCase = sink.Tests.SingleOrDefault(tc => tc.FullyQualifiedName == expectedFullyQualifiedName);
Assert.IsNotNull(actualTestCase, expectedFullyQualifiedName);
Assert.AreEqual(expectedTest.MethodName, actualTestCase.DisplayName, expectedFullyQualifiedName);
Assert.AreEqual(new Uri(PythonConstants.TestExecutorUriString), actualTestCase.ExecutorUri);
Assert.AreEqual(expectedTest.SourceCodeLineNumber, actualTestCase.LineNumber, expectedFullyQualifiedName);
Assert.IsTrue(IsSameFile(expectedTest.SourceCodeFilePath, actualTestCase.CodeFilePath), expectedFullyQualifiedName);
sink.Tests.Remove(actualTestCase);
}
Debug.WriteLine("");
Debug.WriteLine("");
Debug.WriteLine("");
Debug.WriteLine("");
PrintTestCases(sink.Tests);
}
public void TestDiscover()
{
var ctx = new MockDiscoveryContext();
var sink = new MockTestCaseDiscoverySink();
var logger = new MockMessageLogger();
new TestDiscoverer().DiscoverTests(
new[] { TestInfo.TestAdapterLibProjectFilePath, TestInfo.TestAdapterAProjectFilePath, TestInfo.TestAdapterBProjectFilePath },
ctx,
logger,
sink
);
PrintTestCases(sink.Tests);
var expectedTests = TestInfo.TestAdapterATests.Concat(TestInfo.TestAdapterBTests).ToArray();
Assert.AreEqual(expectedTests.Length, sink.Tests.Count);
foreach (var expectedTest in expectedTests)
{
var expectedFullyQualifiedName = TestAnalyzer.MakeFullyQualifiedTestName(expectedTest.RelativeClassFilePath, expectedTest.ClassName, expectedTest.MethodName);
var actualTestCase = sink.Tests.SingleOrDefault(tc => tc.FullyQualifiedName == expectedFullyQualifiedName);
Assert.IsNotNull(actualTestCase, expectedFullyQualifiedName);
Assert.AreEqual(expectedTest.MethodName, actualTestCase.DisplayName, expectedFullyQualifiedName);
Assert.AreEqual(new Uri(TestExecutor.ExecutorUriString), actualTestCase.ExecutorUri);
Assert.AreEqual(expectedTest.SourceCodeLineNumber, actualTestCase.LineNumber, expectedFullyQualifiedName);
Assert.IsTrue(IsSameFile(expectedTest.SourceCodeFilePath, actualTestCase.CodeFilePath), expectedFullyQualifiedName);
Assert.IsTrue(IsSameFile(expectedTest.ProjectFilePath, actualTestCase.Source), expectedFullyQualifiedName);
sink.Tests.Remove(actualTestCase);
}
Debug.WriteLine("");
Debug.WriteLine("");
Debug.WriteLine("");
Debug.WriteLine("");
PrintTestCases(sink.Tests);
}
public void TestDiscover() {
var ctx = new MockDiscoveryContext();
var sink = new MockTestCaseDiscoverySink();
var logger = new MockMessageLogger();
new TestDiscoverer().DiscoverTests(
new[] { TestInfo.TestAdapterLibProjectFilePath, TestInfo.TestAdapterAProjectFilePath, TestInfo.TestAdapterBProjectFilePath },
ctx,
logger,
sink
);
PrintTestCases(sink.Tests);
var expectedTests = TestInfo.TestAdapterATests.Concat(TestInfo.TestAdapterBTests).ToArray();
Assert.AreEqual(expectedTests.Length, sink.Tests.Count);
foreach (var expectedTest in expectedTests) {
var expectedFullyQualifiedName = TestAnalyzer.MakeFullyQualifiedTestName(expectedTest.RelativeClassFilePath, expectedTest.ClassName, expectedTest.MethodName);
var actualTestCase = sink.Tests.SingleOrDefault(tc => tc.FullyQualifiedName == expectedFullyQualifiedName);
Assert.IsNotNull(actualTestCase, expectedFullyQualifiedName);
Assert.AreEqual(expectedTest.MethodName, actualTestCase.DisplayName, expectedFullyQualifiedName);
Assert.AreEqual(new Uri(TestExecutor.ExecutorUriString), actualTestCase.ExecutorUri);
Assert.AreEqual(expectedTest.SourceCodeLineNumber, actualTestCase.LineNumber, expectedFullyQualifiedName);
Assert.IsTrue(IsSameFile(expectedTest.SourceCodeFilePath, actualTestCase.CodeFilePath), expectedFullyQualifiedName);
Assert.IsTrue(IsSameFile(expectedTest.ProjectFilePath, actualTestCase.Source), expectedFullyQualifiedName);
sink.Tests.Remove(actualTestCase);
}
Debug.WriteLine("");
Debug.WriteLine("");
Debug.WriteLine("");
Debug.WriteLine("");
PrintTestCases(sink.Tests);
}
