private string GetSearchPaths(IEnumerable <TestCase> tests, PythonProjectSettings settings)
{
var paths = settings.SearchPath.ToList();
HashSet <string> knownModulePaths = new HashSet <string>();
foreach (var test in tests)
{
string testFilePath = PathUtils.GetAbsoluteFilePath(settings.ProjectHome, test.CodeFilePath);
var modulePath = ModulePath.FromFullPath(testFilePath);
if (knownModulePaths.Add(modulePath.LibraryPath))
{
paths.Insert(0, modulePath.LibraryPath);
}
}
paths.Insert(0, settings.WorkingDirectory);
if (_debugMode == PythonDebugMode.PythonOnly)
{
paths.Insert(0, PtvsdSearchPath);
}
string searchPaths = string.Join(
";",
paths.Where(Directory.Exists).Distinct(StringComparer.OrdinalIgnoreCase)
);
return(searchPaths);
}
private void RunTestGroup(
IGrouping <PythonProjectSettings, TestCase> testGroup,
IRunContext runContext,
IFrameworkHandle frameworkHandle
)
{
PythonProjectSettings settings = testGroup.Key;
if (settings == null || settings.TestFramework != TestFrameworkType.Pytest)
{
return;
}
var testConfig = new PytestConfiguration(runContext);
using (var executor = new ExecutorService(
testConfig,
settings,
frameworkHandle,
runContext)
) {
executor.Run(testGroup, _cancelRequested);
}
var testResults = ParseResults(testConfig.ResultsXmlPath, testGroup, frameworkHandle);
foreach (var result in testResults)
{
if (_cancelRequested.WaitOne(0))
{
break;
}
frameworkHandle.RecordResult(result);
}
}
public TestRunner(
IFrameworkHandle frameworkHandle,
IRunContext runContext,
IEnumerable <TestCase> tests,
string codeCoverageFile,
PythonProjectSettings settings,
VisualStudioProxy app,
ManualResetEvent cancelRequested)
{
_frameworkHandle = frameworkHandle;
_context = runContext;
_tests = tests.ToArray();
_codeCoverageFile = codeCoverageFile;
_settings = settings;
_app = app;
_cancelRequested = cancelRequested;
_dryRun = ExecutorService.IsDryRun(runContext.RunSettings);
_showConsole = ExecutorService.ShouldShowConsole(runContext.RunSettings);
_env = new Dictionary <string, string>();
_searchPaths = GetSearchPaths(tests, settings);
ExecutorService.GetDebugSettings(_app, _context, _settings, out _debugMode, out _debugSecret, out _debugPort);
_socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.IP);
_socket.Bind(new IPEndPoint(IPAddress.Loopback, 0));
_socket.Listen(0);
_socket.BeginAccept(AcceptConnection, _socket);
}
private Dictionary<string, PythonProjectSettings> GetSourceToSettings(IRunSettings settings) {
var doc = Read(settings.SettingsXml);
XPathNodeIterator nodes = doc.CreateNavigator().Select("/RunSettings/Python/TestCases/Project");
Dictionary<string, PythonProjectSettings> res = new Dictionary<string, PythonProjectSettings>();
foreach (XPathNavigator project in nodes) {
PythonProjectSettings projSettings = new PythonProjectSettings(
project.GetAttribute("home", ""),
project.GetAttribute("workingDir", ""),
project.GetAttribute("interpreter", ""),
project.GetAttribute("pathEnv", ""),
project.GetAttribute("nativeDebugging", "").IsTrue()
);
foreach (XPathNavigator environment in project.Select("Environment/Variable")) {
projSettings.Environment[environment.GetAttribute("name", "")] = environment.GetAttribute("value", "");
}
string djangoSettings = project.GetAttribute("djangoSettingsModule", "");
if (!String.IsNullOrWhiteSpace(djangoSettings)) {
projSettings.Environment["DJANGO_SETTINGS_MODULE"] = djangoSettings;
}
foreach (XPathNavigator searchPath in project.Select("SearchPaths/Search")) {
projSettings.SearchPath.Add(searchPath.GetAttribute("value", ""));
}
foreach (XPathNavigator test in project.Select("Test")) {
string testFile = test.GetAttribute("file", "");
Debug.Assert(!string.IsNullOrWhiteSpace(testFile));
res[testFile] = projSettings;
}
}
return res;
}
private void RunTestGroup(
IGrouping <PythonProjectSettings, TestCase> testGroup,
IRunContext runContext,
IFrameworkHandle frameworkHandle
)
{
PythonProjectSettings settings = testGroup.Key;
if (settings == null || settings.TestFramework != TestFrameworkType.Pytest)
{
return;
}
using (var executor = new ExecutorService(settings, frameworkHandle, runContext)) {
bool codeCoverage = ExecutorService.EnableCodeCoverage(runContext);
string covPath = null;
if (codeCoverage)
{
covPath = ExecutorService.GetCoveragePath(testGroup);
}
var resultsXML = executor.Run(testGroup, covPath, _cancelRequested);
// Default TestResults
var pytestIdToResultsMap = testGroup.Select(tc => new TestResult(tc)
{
Outcome = TestOutcome.Skipped
})
.ToDictionary(tr => tr.TestCase.GetPropertyValue <string>(Pytest.Constants.PytestIdProperty, String.Empty), tr => tr);
if (File.Exists(resultsXML))
{
try {
var doc = JunitXmlTestResultParser.Read(resultsXML);
Parse(doc, pytestIdToResultsMap, frameworkHandle);
} catch (Exception ex) {
frameworkHandle.SendMessage(TestMessageLevel.Error, ex.Message);
}
}
else
{
frameworkHandle.SendMessage(TestMessageLevel.Error, Strings.PytestResultsXmlNotFound.FormatUI(resultsXML));
}
foreach (var result in pytestIdToResultsMap.Values)
{
if (_cancelRequested.WaitOne(0))
{
break;
}
frameworkHandle.RecordResult(result);
}
if (codeCoverage)
{
ExecutorService.AttachCoverageResults(frameworkHandle, covPath);
}
}
}
public ExecutorService(PythonProjectSettings projectSettings, IFrameworkHandle frameworkHandle, IRunContext runContext)
{
_projectSettings = projectSettings;
_frameworkHandle = frameworkHandle;
_runContext = runContext;
_app = VisualStudioProxy.FromEnvironmentVariable(PythonConstants.PythonToolsProcessIdEnvironmentVariable);
GetDebugSettings(_app, _runContext, _projectSettings, out _debugMode, out _debugSecret, out _debugPort);
}
public TestRunner(
IFrameworkHandle frameworkHandle,
IRunContext runContext,
IEnumerable <TestCase> tests,
string codeCoverageFile,
PythonProjectSettings settings,
VisualStudioProxy app,
ManualResetEvent cancelRequested)
{
_frameworkHandle = frameworkHandle;
_context = runContext;
_tests = tests.ToArray();
_codeCoverageFile = codeCoverageFile;
_settings = settings;
_app = app;
_cancelRequested = cancelRequested;
_dryRun = IsDryRun(runContext.RunSettings);
_showConsole = ShouldShowConsole(runContext.RunSettings);
_env = new Dictionary <string, string>();
_debugMode = PythonDebugMode.None;
if (runContext.IsBeingDebugged && _app != null)
{
_debugMode = settings.EnableNativeCodeDebugging ? PythonDebugMode.PythonAndNative : PythonDebugMode.PythonOnly;
}
_searchPaths = GetSearchPaths(tests, settings);
if (_debugMode == PythonDebugMode.PythonOnly)
{
if (_settings.UseLegacyDebugger)
{
var secretBuffer = new byte[24];
RandomNumberGenerator.Create().GetNonZeroBytes(secretBuffer);
_debugSecret = Convert.ToBase64String(secretBuffer)
.Replace('+', '-')
.Replace('/', '_')
.TrimEnd('=');
}
else
{
_debugSecret = "";
}
SocketUtils.GetRandomPortListener(IPAddress.Loopback, out _debugPort).Stop();
}
_socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.IP);
_socket.Bind(new IPEndPoint(IPAddress.Loopback, 0));
_socket.Listen(0);
_socket.BeginAccept(AcceptConnection, _socket);
}
private string GetSearchPaths(IEnumerable <TestCase> tests, PythonProjectSettings settings)
{
var paths = settings.SearchPath;
paths.Insert(0, settings.WorkingDirectory);
string searchPaths = string.Join(
";",
paths.Where(Directory.Exists).Distinct(StringComparer.OrdinalIgnoreCase)
);
return(searchPaths);
}
internal ExecutorService(
ITestConfiguration config,
PythonProjectSettings projectSettings,
IFrameworkHandle frameworkHandle,
IRunContext runContext
)
{
_testConfig = config ?? throw new ArgumentNullException(nameof(config));
_projectSettings = projectSettings ?? throw new ArgumentNullException(nameof(projectSettings));
_frameworkHandle = frameworkHandle ?? throw new ArgumentNullException(nameof(frameworkHandle));
_runContext = runContext ?? throw new ArgumentNullException(nameof(runContext));;
_app = VisualStudioProxy.FromEnvironmentVariable(PythonConstants.PythonToolsProcessIdEnvironmentVariable);
GetDebugSettings(_app, _runContext, _projectSettings, out _debugMode, out _debugSecret, out _debugPort);
}
public string[] GetArguments(IEnumerable <string> sources, PythonProjectSettings projSettings, string outputfilename)
{
var arguments = new List <string>();
arguments.Add(DiscoveryAdapterPath);
arguments.Add("discover");
arguments.Add("pytest");
arguments.Add("--output-file");
arguments.Add(outputfilename);
//Note pytest specific arguments go after this separator
arguments.Add("--");
arguments.Add("--cache-clear");
arguments.Add(String.Format("--rootdir={0}", projSettings.ProjectHome));
return(arguments.ToArray());
}
public static IPythonTestDiscoverer GetDiscoverer(PythonProjectSettings settings)
{
switch (settings.TestFramework)
{
case TestFrameworkType.Pytest:
return(new TestDiscovererPytest(settings));
case TestFrameworkType.UnitTest:
return(new TestDiscovererUnitTest(settings));
case TestFrameworkType.None:
default:
throw new NotImplementedException($"CreateDiscoveryService TestFrameworkType:{settings.TestFramework.ToString()} not supported");
}
}
public string[] GetArguments(IEnumerable <string> sources, PythonProjectSettings settings, string outputfilename)
{
var arguments = new List <string>();
arguments.Add(DiscoveryAdapterPath);
arguments.Add("discover");
arguments.Add("unittest");
arguments.Add("--output-file");
arguments.Add(outputfilename);
//Note unittest specific options go after this separator
arguments.Add("--");
arguments.Add(settings.UnitTestRootDir);
arguments.Add(settings.UnitTestPattern);
return(arguments.ToArray());
}
public IList <string> GetExecutionArguments(IEnumerable <TestCase> tests, PythonProjectSettings settings)
{
if (tests is null)
{
throw new ArgumentNullException(nameof(tests));
}
if (settings is null)
{
throw new ArgumentNullException(nameof(settings));
}
var args = new List <string>();
// For a small set of tests, we'll pass them on the command
// line. Once we exceed a certain (arbitrary) number, create
// a test list on disk so that we do not overflow the
// 32K argument limit.
var testIds = tests.Select(t => t.GetPropertyValue <string>(Pytest.Constants.PytestIdProperty, default));
if (testIds.Count() > 5)
{
var testListFilePath = TestUtils.CreateTestListFile(testIds);
args.Add(testListFilePath);
}
else
{
args.Add("dummyfilename"); //expected not to exist, but script excepts something
foreach (var testId in testIds)
{
args.Add(testId);
}
}
// output results to xml file
args.Add(String.Format("--junitxml={0}", ResultsXmlPath));
args.Add(String.Format("--rootdir={0}", settings.ProjectHome));
args.Add("-o");
args.Add("junit_logging=all");
args.Add("-o");
args.Add("junit_family=xunit1");
return(args);
}
private void DiscoverTestGroup(
IGrouping <PythonProjectSettings, string> testGroup,
IDiscoveryContext discoveryContext,
IMessageLogger logger,
ITestCaseDiscoverySink discoverySink
)
{
PythonProjectSettings settings = testGroup.Key;
if (settings == null || settings.TestFramework != _frameworkType)
{
return;
}
try {
DiscoverTests(testGroup, settings, logger, discoverySink);
} catch (Exception ex) {
logger.SendMessage(TestMessageLevel.Error, ex.Message);
}
}
abstract public void DiscoverTests(
IEnumerable <string> sources,
PythonProjectSettings settings,
IMessageLogger logger,
ITestCaseDiscoverySink discoverySink
);
internal static void GetDebugSettings(VisualStudioProxy app, IRunContext runContext, PythonProjectSettings projectSettings, out PythonDebugMode debugMode, out string debugSecret, out int debugPort)
{
debugMode = PythonDebugMode.None;
debugSecret = "";
debugPort = 0;
if (runContext.IsBeingDebugged && app != null)
{
debugMode = projectSettings.EnableNativeCodeDebugging ? PythonDebugMode.PythonAndNative : PythonDebugMode.PythonOnly;
}
if (debugMode == PythonDebugMode.PythonOnly)
{
if (projectSettings.UseLegacyDebugger)
{
var secretBuffer = new byte[24];
RandomNumberGenerator.Create().GetNonZeroBytes(secretBuffer);
debugSecret = Convert.ToBase64String(secretBuffer)
.Replace('+', '-')
.Replace('/', '_')
.TrimEnd('=');
}
SocketUtils.GetRandomPortListener(IPAddress.Loopback, out debugPort).Stop();
}
}
public TestDiscovererPytest(PythonProjectSettings settings)
{
_settings = settings;
}
private Dictionary <string, string> InitializeEnvironment(IEnumerable <string> sources, PythonProjectSettings projSettings)
{
var pythonPathVar = projSettings.PathEnv;
var pythonPath = GetSearchPaths(sources, projSettings);
var env = new Dictionary <string, string>();
if (!string.IsNullOrWhiteSpace(pythonPathVar))
{
env[pythonPathVar] = pythonPath;
}
foreach (var envVar in projSettings.Environment)
{
env[envVar.Key] = envVar.Value;
}
env["PYTHONUNBUFFERED"] = "1";
return(env);
}
public PythonTestCase(PythonProjectSettings settings, TestCase testCase, PythonDebugMode debugMode) {
Settings = settings;
TestCase = testCase;
TestAnalyzer.ParseFullyQualifiedTestName(
testCase.FullyQualifiedName,
out TestFile,
out TestClass,
out TestMethod
);
TestFilePath = CommonUtils.GetAbsoluteFilePath(Settings.ProjectHome, TestFile);
ModulePath = ModulePath.FromFullPath(TestFilePath);
WorkingDirectory = Settings.WorkingDir;
var paths = settings.SearchPath.ToList();
paths.Insert(0, ModulePath.LibraryPath);
paths.Insert(0, WorkingDirectory);
if (debugMode == PythonDebugMode.PythonOnly) {
paths.Insert(0, PtvsdSearchPath);
}
SearchPaths = string.Join(
";",
paths.Where(Directory.Exists).Distinct(StringComparer.OrdinalIgnoreCase)
);
var arguments = new List<string> {
TestLauncherPath,
"-m", ModulePath.ModuleName,
"-t", string.Format("{0}.{1}", TestClass, TestMethod)
};
if (debugMode == PythonDebugMode.PythonOnly) {
var secretBuffer = new byte[24];
RandomNumberGenerator.Create().GetNonZeroBytes(secretBuffer);
DebugSecret = Convert.ToBase64String(secretBuffer)
.Replace('+', '-')
.Replace('/', '_')
.TrimEnd('=');
DebugPort = GetFreePort();
arguments.AddRange(new[] {
"-s", DebugSecret,
"-p", DebugPort.ToString()
});
} else if (debugMode == PythonDebugMode.PythonAndNative) {
arguments.Add("-x");
}
Arguments = arguments;
Environment = new Dictionary<string, string>();
if (!string.IsNullOrEmpty(settings.DjangoSettingsModule)) {
Environment["DJANGO_SETTINGS_MODULE"] = settings.DjangoSettingsModule;
}
foreach (var envVar in settings.Environment) {
Environment[envVar.Key] = envVar.Value;
}
}
public string[] GetArguments(IEnumerable <TestCase> tests, string outputfile, string coveragePath, PythonProjectSettings settings)
{
var arguments = new List <string> {
TestLauncherPath,
_projectSettings.WorkingDirectory,
"pytest",
_debugSecret,
_debugPort.ToString(),
GetDebuggerSearchPath(_projectSettings.UseLegacyDebugger),
_debugMode == PythonDebugMode.PythonAndNative ? "mixed" : string.Empty,
coveragePath ?? string.Empty
};
// For a small set of tests, we'll pass them on the command
// line. Once we exceed a certain (arbitrary) number, create
// a test list on disk so that we do not overflow the
// 32K argument limit.
var testIds = tests.Select(t => t.GetPropertyValue <string>(Pytest.Constants.PytestIdProperty, default));
if (testIds.Count() > 5)
{
var testListFilePath = TestUtils.CreateTestListFile(testIds);
arguments.Add(testListFilePath);
}
else
{
arguments.Add("dummyfilename"); //expected not to exist, but script excepts something
foreach (var testId in testIds)
{
arguments.Add(testId);
}
}
// output results to xml file
arguments.Add(String.Format("--junitxml={0}", outputfile));
arguments.Add(String.Format("--rootdir={0}", settings.ProjectHome));
return(arguments.ToArray());
}
public override void DiscoverTests(
IEnumerable <string> sources,
PythonProjectSettings settings,
IMessageLogger logger,
ITestCaseDiscoverySink discoverySink
)
{
if (sources is null)
{
throw new ArgumentNullException(nameof(sources));
}
if (discoverySink is null)
{
throw new ArgumentNullException(nameof(discoverySink));
}
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
var workspaceText = settings.IsWorkspace ? Strings.WorkspaceText : Strings.ProjectText;
LogInfo(Strings.PythonTestDiscovererStartedMessage.FormatUI(PythonConstants.PytestText, settings.ProjectName, workspaceText, settings.DiscoveryWaitTimeInSeconds));
var env = InitializeEnvironment(sources, settings);
var outputFilePath = Path.GetTempFileName();
var arguments = GetArguments(sources, settings, outputFilePath);
LogInfo("cd " + settings.WorkingDirectory);
LogInfo("set " + settings.PathEnv + "=" + env[settings.PathEnv]);
LogInfo($"{settings.InterpreterPath} {string.Join(" ", arguments)}");
try {
var stdout = ProcessExecute.RunWithTimeout(
settings.InterpreterPath,
env,
arguments,
settings.WorkingDirectory,
settings.PathEnv,
settings.DiscoveryWaitTimeInSeconds
);
if (!String.IsNullOrEmpty(stdout))
{
Error(stdout);
}
} catch (TimeoutException) {
Error(Strings.PythonTestDiscovererTimeoutErrorMessage);
return;
}
if (!File.Exists(outputFilePath))
{
Error(Strings.PythonDiscoveryResultsNotFound.FormatUI(outputFilePath));
return;
}
string json = File.ReadAllText(outputFilePath);
if (string.IsNullOrEmpty(json))
{
return;
}
try {
var results = JsonConvert.DeserializeObject <List <PytestDiscoveryResults> >(json);
var testcases = ParseDiscoveryResults(results, settings.ProjectHome);
foreach (var tc in testcases)
{
// Note: Test Explorer will show a key not found exception if we use a source path that doesn't match a test container's source.
if (settings.TestContainerSources.TryGetValue(tc.CodeFilePath, out _))
{
discoverySink.SendTestCase(tc);
}
}
} catch (InvalidOperationException ex) {
Error("Failed to parse: {0}".FormatInvariant(ex.Message));
Error(json);
} catch (JsonException ex) {
Error("Failed to parse: {0}".FormatInvariant(ex.Message));
Error(json);
}
}
