private string GetSearchPaths(IEnumerable <TestCase> tests, PythonProjectSettings settings) { var paths = settings.SearchPath.ToList(); HashSet <string> knownModulePaths = new HashSet <string>(); foreach (var test in tests) { string testFilePath = PathUtils.GetAbsoluteFilePath(settings.ProjectHome, test.CodeFilePath); var modulePath = ModulePath.FromFullPath(testFilePath); if (knownModulePaths.Add(modulePath.LibraryPath)) { paths.Insert(0, modulePath.LibraryPath); } } paths.Insert(0, settings.WorkingDirectory); if (_debugMode == PythonDebugMode.PythonOnly) { paths.Insert(0, PtvsdSearchPath); } string searchPaths = string.Join( ";", paths.Where(Directory.Exists).Distinct(StringComparer.OrdinalIgnoreCase) ); return(searchPaths); }
private void RunTestGroup( IGrouping <PythonProjectSettings, TestCase> testGroup, IRunContext runContext, IFrameworkHandle frameworkHandle ) { PythonProjectSettings settings = testGroup.Key; if (settings == null || settings.TestFramework != TestFrameworkType.Pytest) { return; } var testConfig = new PytestConfiguration(runContext); using (var executor = new ExecutorService( testConfig, settings, frameworkHandle, runContext) ) { executor.Run(testGroup, _cancelRequested); } var testResults = ParseResults(testConfig.ResultsXmlPath, testGroup, frameworkHandle); foreach (var result in testResults) { if (_cancelRequested.WaitOne(0)) { break; } frameworkHandle.RecordResult(result); } }
public TestRunner( IFrameworkHandle frameworkHandle, IRunContext runContext, IEnumerable <TestCase> tests, string codeCoverageFile, PythonProjectSettings settings, VisualStudioProxy app, ManualResetEvent cancelRequested) { _frameworkHandle = frameworkHandle; _context = runContext; _tests = tests.ToArray(); _codeCoverageFile = codeCoverageFile; _settings = settings; _app = app; _cancelRequested = cancelRequested; _dryRun = ExecutorService.IsDryRun(runContext.RunSettings); _showConsole = ExecutorService.ShouldShowConsole(runContext.RunSettings); _env = new Dictionary <string, string>(); _searchPaths = GetSearchPaths(tests, settings); ExecutorService.GetDebugSettings(_app, _context, _settings, out _debugMode, out _debugSecret, out _debugPort); _socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.IP); _socket.Bind(new IPEndPoint(IPAddress.Loopback, 0)); _socket.Listen(0); _socket.BeginAccept(AcceptConnection, _socket); }
private Dictionary<string, PythonProjectSettings> GetSourceToSettings(IRunSettings settings) { var doc = Read(settings.SettingsXml); XPathNodeIterator nodes = doc.CreateNavigator().Select("/RunSettings/Python/TestCases/Project"); Dictionary<string, PythonProjectSettings> res = new Dictionary<string, PythonProjectSettings>(); foreach (XPathNavigator project in nodes) { PythonProjectSettings projSettings = new PythonProjectSettings( project.GetAttribute("home", ""), project.GetAttribute("workingDir", ""), project.GetAttribute("interpreter", ""), project.GetAttribute("pathEnv", ""), project.GetAttribute("nativeDebugging", "").IsTrue() ); foreach (XPathNavigator environment in project.Select("Environment/Variable")) { projSettings.Environment[environment.GetAttribute("name", "")] = environment.GetAttribute("value", ""); } string djangoSettings = project.GetAttribute("djangoSettingsModule", ""); if (!String.IsNullOrWhiteSpace(djangoSettings)) { projSettings.Environment["DJANGO_SETTINGS_MODULE"] = djangoSettings; } foreach (XPathNavigator searchPath in project.Select("SearchPaths/Search")) { projSettings.SearchPath.Add(searchPath.GetAttribute("value", "")); } foreach (XPathNavigator test in project.Select("Test")) { string testFile = test.GetAttribute("file", ""); Debug.Assert(!string.IsNullOrWhiteSpace(testFile)); res[testFile] = projSettings; } } return res; }
private void RunTestGroup( IGrouping <PythonProjectSettings, TestCase> testGroup, IRunContext runContext, IFrameworkHandle frameworkHandle ) { PythonProjectSettings settings = testGroup.Key; if (settings == null || settings.TestFramework != TestFrameworkType.Pytest) { return; } using (var executor = new ExecutorService(settings, frameworkHandle, runContext)) { bool codeCoverage = ExecutorService.EnableCodeCoverage(runContext); string covPath = null; if (codeCoverage) { covPath = ExecutorService.GetCoveragePath(testGroup); } var resultsXML = executor.Run(testGroup, covPath, _cancelRequested); // Default TestResults var pytestIdToResultsMap = testGroup.Select(tc => new TestResult(tc) { Outcome = TestOutcome.Skipped }) .ToDictionary(tr => tr.TestCase.GetPropertyValue <string>(Pytest.Constants.PytestIdProperty, String.Empty), tr => tr); if (File.Exists(resultsXML)) { try { var doc = JunitXmlTestResultParser.Read(resultsXML); Parse(doc, pytestIdToResultsMap, frameworkHandle); } catch (Exception ex) { frameworkHandle.SendMessage(TestMessageLevel.Error, ex.Message); } } else { frameworkHandle.SendMessage(TestMessageLevel.Error, Strings.PytestResultsXmlNotFound.FormatUI(resultsXML)); } foreach (var result in pytestIdToResultsMap.Values) { if (_cancelRequested.WaitOne(0)) { break; } frameworkHandle.RecordResult(result); } if (codeCoverage) { ExecutorService.AttachCoverageResults(frameworkHandle, covPath); } } }
public ExecutorService(PythonProjectSettings projectSettings, IFrameworkHandle frameworkHandle, IRunContext runContext) { _projectSettings = projectSettings; _frameworkHandle = frameworkHandle; _runContext = runContext; _app = VisualStudioProxy.FromEnvironmentVariable(PythonConstants.PythonToolsProcessIdEnvironmentVariable); GetDebugSettings(_app, _runContext, _projectSettings, out _debugMode, out _debugSecret, out _debugPort); }
public TestRunner( IFrameworkHandle frameworkHandle, IRunContext runContext, IEnumerable <TestCase> tests, string codeCoverageFile, PythonProjectSettings settings, VisualStudioProxy app, ManualResetEvent cancelRequested) { _frameworkHandle = frameworkHandle; _context = runContext; _tests = tests.ToArray(); _codeCoverageFile = codeCoverageFile; _settings = settings; _app = app; _cancelRequested = cancelRequested; _dryRun = IsDryRun(runContext.RunSettings); _showConsole = ShouldShowConsole(runContext.RunSettings); _env = new Dictionary <string, string>(); _debugMode = PythonDebugMode.None; if (runContext.IsBeingDebugged && _app != null) { _debugMode = settings.EnableNativeCodeDebugging ? PythonDebugMode.PythonAndNative : PythonDebugMode.PythonOnly; } _searchPaths = GetSearchPaths(tests, settings); if (_debugMode == PythonDebugMode.PythonOnly) { if (_settings.UseLegacyDebugger) { var secretBuffer = new byte[24]; RandomNumberGenerator.Create().GetNonZeroBytes(secretBuffer); _debugSecret = Convert.ToBase64String(secretBuffer) .Replace('+', '-') .Replace('/', '_') .TrimEnd('='); } else { _debugSecret = ""; } SocketUtils.GetRandomPortListener(IPAddress.Loopback, out _debugPort).Stop(); } _socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.IP); _socket.Bind(new IPEndPoint(IPAddress.Loopback, 0)); _socket.Listen(0); _socket.BeginAccept(AcceptConnection, _socket); }
private string GetSearchPaths(IEnumerable <TestCase> tests, PythonProjectSettings settings) { var paths = settings.SearchPath; paths.Insert(0, settings.WorkingDirectory); string searchPaths = string.Join( ";", paths.Where(Directory.Exists).Distinct(StringComparer.OrdinalIgnoreCase) ); return(searchPaths); }
internal ExecutorService( ITestConfiguration config, PythonProjectSettings projectSettings, IFrameworkHandle frameworkHandle, IRunContext runContext ) { _testConfig = config ?? throw new ArgumentNullException(nameof(config)); _projectSettings = projectSettings ?? throw new ArgumentNullException(nameof(projectSettings)); _frameworkHandle = frameworkHandle ?? throw new ArgumentNullException(nameof(frameworkHandle)); _runContext = runContext ?? throw new ArgumentNullException(nameof(runContext));; _app = VisualStudioProxy.FromEnvironmentVariable(PythonConstants.PythonToolsProcessIdEnvironmentVariable); GetDebugSettings(_app, _runContext, _projectSettings, out _debugMode, out _debugSecret, out _debugPort); }
public string[] GetArguments(IEnumerable <string> sources, PythonProjectSettings projSettings, string outputfilename) { var arguments = new List <string>(); arguments.Add(DiscoveryAdapterPath); arguments.Add("discover"); arguments.Add("pytest"); arguments.Add("--output-file"); arguments.Add(outputfilename); //Note pytest specific arguments go after this separator arguments.Add("--"); arguments.Add("--cache-clear"); arguments.Add(String.Format("--rootdir={0}", projSettings.ProjectHome)); return(arguments.ToArray()); }
public static IPythonTestDiscoverer GetDiscoverer(PythonProjectSettings settings) { switch (settings.TestFramework) { case TestFrameworkType.Pytest: return(new TestDiscovererPytest(settings)); case TestFrameworkType.UnitTest: return(new TestDiscovererUnitTest(settings)); case TestFrameworkType.None: default: throw new NotImplementedException($"CreateDiscoveryService TestFrameworkType:{settings.TestFramework.ToString()} not supported"); } }
public string[] GetArguments(IEnumerable <string> sources, PythonProjectSettings settings, string outputfilename) { var arguments = new List <string>(); arguments.Add(DiscoveryAdapterPath); arguments.Add("discover"); arguments.Add("unittest"); arguments.Add("--output-file"); arguments.Add(outputfilename); //Note unittest specific options go after this separator arguments.Add("--"); arguments.Add(settings.UnitTestRootDir); arguments.Add(settings.UnitTestPattern); return(arguments.ToArray()); }
public IList <string> GetExecutionArguments(IEnumerable <TestCase> tests, PythonProjectSettings settings) { if (tests is null) { throw new ArgumentNullException(nameof(tests)); } if (settings is null) { throw new ArgumentNullException(nameof(settings)); } var args = new List <string>(); // For a small set of tests, we'll pass them on the command // line. Once we exceed a certain (arbitrary) number, create // a test list on disk so that we do not overflow the // 32K argument limit. var testIds = tests.Select(t => t.GetPropertyValue <string>(Pytest.Constants.PytestIdProperty, default)); if (testIds.Count() > 5) { var testListFilePath = TestUtils.CreateTestListFile(testIds); args.Add(testListFilePath); } else { args.Add("dummyfilename"); //expected not to exist, but script excepts something foreach (var testId in testIds) { args.Add(testId); } } // output results to xml file args.Add(String.Format("--junitxml={0}", ResultsXmlPath)); args.Add(String.Format("--rootdir={0}", settings.ProjectHome)); args.Add("-o"); args.Add("junit_logging=all"); args.Add("-o"); args.Add("junit_family=xunit1"); return(args); }
private void DiscoverTestGroup( IGrouping <PythonProjectSettings, string> testGroup, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink ) { PythonProjectSettings settings = testGroup.Key; if (settings == null || settings.TestFramework != _frameworkType) { return; } try { DiscoverTests(testGroup, settings, logger, discoverySink); } catch (Exception ex) { logger.SendMessage(TestMessageLevel.Error, ex.Message); } }
abstract public void DiscoverTests( IEnumerable <string> sources, PythonProjectSettings settings, IMessageLogger logger, ITestCaseDiscoverySink discoverySink );
internal static void GetDebugSettings(VisualStudioProxy app, IRunContext runContext, PythonProjectSettings projectSettings, out PythonDebugMode debugMode, out string debugSecret, out int debugPort) { debugMode = PythonDebugMode.None; debugSecret = ""; debugPort = 0; if (runContext.IsBeingDebugged && app != null) { debugMode = projectSettings.EnableNativeCodeDebugging ? PythonDebugMode.PythonAndNative : PythonDebugMode.PythonOnly; } if (debugMode == PythonDebugMode.PythonOnly) { if (projectSettings.UseLegacyDebugger) { var secretBuffer = new byte[24]; RandomNumberGenerator.Create().GetNonZeroBytes(secretBuffer); debugSecret = Convert.ToBase64String(secretBuffer) .Replace('+', '-') .Replace('/', '_') .TrimEnd('='); } SocketUtils.GetRandomPortListener(IPAddress.Loopback, out debugPort).Stop(); } }
public TestDiscovererPytest(PythonProjectSettings settings) { _settings = settings; }
private Dictionary <string, string> InitializeEnvironment(IEnumerable <string> sources, PythonProjectSettings projSettings) { var pythonPathVar = projSettings.PathEnv; var pythonPath = GetSearchPaths(sources, projSettings); var env = new Dictionary <string, string>(); if (!string.IsNullOrWhiteSpace(pythonPathVar)) { env[pythonPathVar] = pythonPath; } foreach (var envVar in projSettings.Environment) { env[envVar.Key] = envVar.Value; } env["PYTHONUNBUFFERED"] = "1"; return(env); }
public PythonTestCase(PythonProjectSettings settings, TestCase testCase, PythonDebugMode debugMode) { Settings = settings; TestCase = testCase; TestAnalyzer.ParseFullyQualifiedTestName( testCase.FullyQualifiedName, out TestFile, out TestClass, out TestMethod ); TestFilePath = CommonUtils.GetAbsoluteFilePath(Settings.ProjectHome, TestFile); ModulePath = ModulePath.FromFullPath(TestFilePath); WorkingDirectory = Settings.WorkingDir; var paths = settings.SearchPath.ToList(); paths.Insert(0, ModulePath.LibraryPath); paths.Insert(0, WorkingDirectory); if (debugMode == PythonDebugMode.PythonOnly) { paths.Insert(0, PtvsdSearchPath); } SearchPaths = string.Join( ";", paths.Where(Directory.Exists).Distinct(StringComparer.OrdinalIgnoreCase) ); var arguments = new List<string> { TestLauncherPath, "-m", ModulePath.ModuleName, "-t", string.Format("{0}.{1}", TestClass, TestMethod) }; if (debugMode == PythonDebugMode.PythonOnly) { var secretBuffer = new byte[24]; RandomNumberGenerator.Create().GetNonZeroBytes(secretBuffer); DebugSecret = Convert.ToBase64String(secretBuffer) .Replace('+', '-') .Replace('/', '_') .TrimEnd('='); DebugPort = GetFreePort(); arguments.AddRange(new[] { "-s", DebugSecret, "-p", DebugPort.ToString() }); } else if (debugMode == PythonDebugMode.PythonAndNative) { arguments.Add("-x"); } Arguments = arguments; Environment = new Dictionary<string, string>(); if (!string.IsNullOrEmpty(settings.DjangoSettingsModule)) { Environment["DJANGO_SETTINGS_MODULE"] = settings.DjangoSettingsModule; } foreach (var envVar in settings.Environment) { Environment[envVar.Key] = envVar.Value; } }
public string[] GetArguments(IEnumerable <TestCase> tests, string outputfile, string coveragePath, PythonProjectSettings settings) { var arguments = new List <string> { TestLauncherPath, _projectSettings.WorkingDirectory, "pytest", _debugSecret, _debugPort.ToString(), GetDebuggerSearchPath(_projectSettings.UseLegacyDebugger), _debugMode == PythonDebugMode.PythonAndNative ? "mixed" : string.Empty, coveragePath ?? string.Empty }; // For a small set of tests, we'll pass them on the command // line. Once we exceed a certain (arbitrary) number, create // a test list on disk so that we do not overflow the // 32K argument limit. var testIds = tests.Select(t => t.GetPropertyValue <string>(Pytest.Constants.PytestIdProperty, default)); if (testIds.Count() > 5) { var testListFilePath = TestUtils.CreateTestListFile(testIds); arguments.Add(testListFilePath); } else { arguments.Add("dummyfilename"); //expected not to exist, but script excepts something foreach (var testId in testIds) { arguments.Add(testId); } } // output results to xml file arguments.Add(String.Format("--junitxml={0}", outputfile)); arguments.Add(String.Format("--rootdir={0}", settings.ProjectHome)); return(arguments.ToArray()); }
public override void DiscoverTests( IEnumerable <string> sources, PythonProjectSettings settings, IMessageLogger logger, ITestCaseDiscoverySink discoverySink ) { if (sources is null) { throw new ArgumentNullException(nameof(sources)); } if (discoverySink is null) { throw new ArgumentNullException(nameof(discoverySink)); } _logger = logger ?? throw new ArgumentNullException(nameof(logger)); var workspaceText = settings.IsWorkspace ? Strings.WorkspaceText : Strings.ProjectText; LogInfo(Strings.PythonTestDiscovererStartedMessage.FormatUI(PythonConstants.PytestText, settings.ProjectName, workspaceText, settings.DiscoveryWaitTimeInSeconds)); var env = InitializeEnvironment(sources, settings); var outputFilePath = Path.GetTempFileName(); var arguments = GetArguments(sources, settings, outputFilePath); LogInfo("cd " + settings.WorkingDirectory); LogInfo("set " + settings.PathEnv + "=" + env[settings.PathEnv]); LogInfo($"{settings.InterpreterPath} {string.Join(" ", arguments)}"); try { var stdout = ProcessExecute.RunWithTimeout( settings.InterpreterPath, env, arguments, settings.WorkingDirectory, settings.PathEnv, settings.DiscoveryWaitTimeInSeconds ); if (!String.IsNullOrEmpty(stdout)) { Error(stdout); } } catch (TimeoutException) { Error(Strings.PythonTestDiscovererTimeoutErrorMessage); return; } if (!File.Exists(outputFilePath)) { Error(Strings.PythonDiscoveryResultsNotFound.FormatUI(outputFilePath)); return; } string json = File.ReadAllText(outputFilePath); if (string.IsNullOrEmpty(json)) { return; } try { var results = JsonConvert.DeserializeObject <List <PytestDiscoveryResults> >(json); var testcases = ParseDiscoveryResults(results, settings.ProjectHome); foreach (var tc in testcases) { // Note: Test Explorer will show a key not found exception if we use a source path that doesn't match a test container's source. if (settings.TestContainerSources.TryGetValue(tc.CodeFilePath, out _)) { discoverySink.SendTestCase(tc); } } } catch (InvalidOperationException ex) { Error("Failed to parse: {0}".FormatInvariant(ex.Message)); Error(json); } catch (JsonException ex) { Error("Failed to parse: {0}".FormatInvariant(ex.Message)); Error(json); } }