private static void FullStdLibTest(PythonVersion v) { v.AssertInstalled(); var factory = new AstPythonInterpreterFactory(v.Configuration, null); var modules = ModulePath.GetModulesInLib(v.PrefixPath).ToList(); bool anySuccess = false; bool anyExtensionSuccess = false, anyExtensionSeen = false; using (var analyzer = new PythonAnalysis(factory)) { foreach (var modName in modules) { anyExtensionSeen |= modName.IsNativeExtension; var mod = analyzer.Analyzer.Interpreter.ImportModule(modName.ModuleName); if (mod == null) { Trace.TraceWarning("failed to import {0} from {1}".FormatInvariant(modName.ModuleName, modName.SourceFile)); } else { anySuccess = true; anyExtensionSuccess |= modName.IsNativeExtension; mod.GetMemberNames(analyzer.ModuleContext).ToList(); } } } Assert.IsTrue(anySuccess, "failed to import any modules at all"); Assert.IsTrue(anyExtensionSuccess || !anyExtensionSeen, "failed to import all extension modules"); }
/// <summary> /// Determines whether the interpreter factory contains the specified /// modules. /// </summary> /// <returns>The names of the modules that were found.</returns> public static async Task <bool> HasModuleAsync(this IPythonInterpreterFactory factory, string moduleName, IInterpreterOptionsService interpreterOptions = null) { if (interpreterOptions != null) { foreach (var pm in interpreterOptions.GetPackageManagers(factory)) { if ((await pm.GetInstalledPackageAsync(new PackageSpec(moduleName), CancellationToken.None)).IsValid) { return(true); } } } return(await Task.Run(() => { foreach (var mp in ModulePath.GetModulesInLib(factory.Configuration)) { if (mp.ModuleName == moduleName) { return true; } } return false; })); }
private static void FullStdLibTest(PythonVersion v) { v.AssertInstalled(); var modules = ModulePath.GetModulesInLib(v.PrefixPath).ToList(); var paths = modules.Select(m => m.LibraryPath).Distinct().ToArray(); bool anySuccess = false; using (var analyzer = new PythonAnalysis(v.Version)) { analyzer.SetSearchPaths(paths); foreach (var modName in modules) { if (modName.IsCompiled || modName.IsNativeExtension) { continue; } var mod = analyzer.Analyzer.Interpreter.ImportModule(modName.ModuleName); if (mod == null) { Trace.TraceWarning("failed to import {0} from {1}".FormatInvariant(modName.ModuleName, modName.SourceFile)); } else { anySuccess = true; mod.GetMemberNames(analyzer.ModuleContext).ToList(); } } } Assert.IsTrue(anySuccess, "failed to import any modules at all"); }
/// <summary> /// Determines whether the interpreter factory contains the specified /// modules. /// </summary> /// <returns>The names of the modules that were found.</returns> public static async Task <bool> HasModuleAsync(this IPythonInterpreterFactory factory, string moduleName, IInterpreterOptionsService interpreterOptions = null) { if (interpreterOptions != null) { foreach (var pm in interpreterOptions.GetPackageManagers(factory)) { if ((await pm.GetInstalledPackageAsync(new PackageSpec(moduleName), CancellationToken.None)).IsValid) { return(true); } } } return(await Task.Run(() => { var configuration = factory.Configuration; var prefixPath = configuration.GetPrefixPath(); var libraryPath = !string.IsNullOrEmpty(configuration.LibraryPath) ? configuration.LibraryPath : Path.Combine(prefixPath, "Lib"); var sitePackagesPath = !string.IsNullOrEmpty(configuration.SitePackagesPath) ? configuration.SitePackagesPath : Path.Combine(libraryPath, "site-packages"); var requiresInitPyFiles = ModulePath.PythonVersionRequiresInitPyFiles(configuration.Version); foreach (var mp in ModulePath.GetModulesInLib(libraryPath, sitePackagesPath, requiresInitPyFiles)) { if (mp.ModuleName == moduleName) { return true; } } return false; })); }
private string[] GetMissingModules(HashSet <string> existingDatabase) { return(ModulePath.GetModulesInLib(this) .Select(mp => mp.ModuleName) .Concat(RequiredBuiltinModules) .Where(name => !existingDatabase.Contains(name)) .OrderBy(name => name, StringComparer.InvariantCultureIgnoreCase) .ToArray()); }
/// <summary> /// Determines whether the interpreter factory contains the specified /// modules. /// </summary> /// <returns>The names of the modules that were found.</returns> public static async Task <bool> HasModuleAsync(this IPythonInterpreterFactory factory, string moduleName, IInterpreterOptionsService interpreterOptions = null) { if (interpreterOptions != null) { foreach (var pm in interpreterOptions.GetPackageManagers(factory)) { if ((await pm.GetInstalledPackageAsync(new PackageSpec(moduleName), CancellationToken.None)).IsValid) { return(true); } } } var withDb = factory as LegacyDB.PythonInterpreterFactoryWithDatabase; if (withDb != null && withDb.IsCurrent) { var db = withDb.GetCurrentDatabase(); if (db.GetModule(moduleName) != null) { return(true); } // Always stop searching after this step return(false); } if (withDb != null) { try { var paths = await LegacyDB.PythonTypeDatabase.GetDatabaseSearchPathsAsync(withDb); if (LegacyDB.PythonTypeDatabase.GetDatabaseExpectedModules(withDb.Configuration.Version, paths) .SelectMany() .Any(g => g.ModuleName == moduleName)) { return(true); } } catch (InvalidOperationException) { } } return(await Task.Run(() => { foreach (var mp in ModulePath.GetModulesInLib(factory.Configuration)) { if (mp.ModuleName == moduleName) { return true; } } return false; })); }
/// <summary> /// Determines whether the interpreter factory contains the specified /// modules. /// </summary> /// <returns>The names of the modules that were found.</returns> internal static HashSet <string> FindModules(this IPythonInterpreterFactory factory, params string[] moduleNames) { var expected = new HashSet <string>(moduleNames); var result = new HashSet <string>(); foreach (var mp in ModulePath.GetModulesInLib(factory)) { if (expected.Count == 0) { break; } if (expected.Remove(mp.ModuleName)) { result.Add(mp.ModuleName); } } return(result); }
/// <summary> /// Determines whether the interpreter factory contains the specified /// modules. /// </summary> /// <returns>The names of the modules that were found.</returns> public static async Task <HashSet <string> > FindModulesAsync(this IPythonInterpreterFactory factory, params string[] moduleNames) { var withDb = factory as PythonInterpreterFactoryWithDatabase; if (withDb != null && withDb.IsCurrent) { var db = withDb.GetCurrentDatabase(); var set = new HashSet <string>(moduleNames.Where(m => db.GetModule(m) != null)); return(set); } var expected = new HashSet <string>(moduleNames); if (withDb != null) { var paths = PythonTypeDatabase.GetCachedDatabaseSearchPaths(withDb.DatabasePath) ?? await PythonTypeDatabase.GetUncachedDatabaseSearchPathsAsync(withDb.Configuration.InterpreterPath).ConfigureAwait(false); var db = PythonTypeDatabase.GetDatabaseExpectedModules(withDb.Configuration.Version, paths) .SelectMany() .Select(g => g.ModuleName); expected.IntersectWith(db); return(expected); } return(await Task.Run(() => { var result = new HashSet <string>(); foreach (var mp in ModulePath.GetModulesInLib(factory)) { if (expected.Count == 0) { break; } if (expected.Remove(mp.ModuleName)) { result.Add(mp.ModuleName); } } return result; })); }
private async Task FullStdLibTest(PythonVersion v, params string[] skipModules) { v.AssertInstalled(); var factory = new AstPythonInterpreterFactory(v.Configuration, new InterpreterFactoryCreationOptions { DatabasePath = TestData.GetTempPath(), UseExistingCache = false }); var modules = ModulePath.GetModulesInLib(v.PrefixPath).ToList(); var skip = new HashSet <string>(skipModules); skip.UnionWith(new[] { "matplotlib.backends._backend_gdk", "matplotlib.backends._backend_gtkagg", "matplotlib.backends._gtkagg", }); bool anySuccess = false; bool anyExtensionSuccess = false, anyExtensionSeen = false; bool anyParseError = false; using (var analyzer = new PythonAnalysis(factory)) { try { var tasks = new List <Task <Tuple <ModulePath, IPythonModule> > >(); var interp = (AstPythonInterpreter)analyzer.Analyzer.Interpreter; foreach (var m in skip) { interp.AddUnimportableModule(m); } foreach (var r in modules .Where(m => !skip.Contains(m.ModuleName)) .GroupBy(m => { int i = m.FullName.IndexOf('.'); return(i <= 0 ? m.FullName : m.FullName.Remove(i)); }) .AsParallel() .SelectMany(g => g.Select(m => Tuple.Create(m, interp.ImportModule(m.ModuleName)))) ) { var modName = r.Item1; var mod = r.Item2; anyExtensionSeen |= modName.IsNativeExtension; if (mod == null) { Trace.TraceWarning("failed to import {0} from {1}", modName.ModuleName, modName.SourceFile); } else if (mod is AstScrapedPythonModule smod) { if (smod.ParseErrors?.Any() ?? false) { anyParseError = true; Trace.TraceError("Parse errors in {0}", modName.SourceFile); foreach (var e in smod.ParseErrors) { Trace.TraceError(e); } } else { anySuccess = true; anyExtensionSuccess |= modName.IsNativeExtension; mod.GetMemberNames(analyzer.ModuleContext).ToList(); } } else if (mod is AstPythonModule) { // pass } else { Trace.TraceError("imported {0} as type {1}", modName.ModuleName, mod.GetType().FullName); } } } finally { _analysisLog = analyzer.GetLogContent(CultureInfo.InvariantCulture); } } Assert.IsTrue(anySuccess, "failed to import any modules at all"); Assert.IsTrue(anyExtensionSuccess || !anyExtensionSeen, "failed to import all extension modules"); Assert.IsFalse(anyParseError, "parse errors occurred"); }
private async Task FullStdLibTest(InterpreterConfiguration configuration, params string[] skipModules) { configuration.AssertInstalled(); var moduleUri = TestData.GetDefaultModuleUri(); var moduleDirectory = Path.GetDirectoryName(moduleUri.LocalPath); var services = await CreateServicesAsync(moduleDirectory, configuration); var interpreter = services.GetService <IPythonInterpreter>(); var modules = ModulePath.GetModulesInLib(configuration.LibraryPath, configuration.SitePackagesPath).ToList(); var skip = new HashSet <string>(skipModules); skip.UnionWith(new[] { @"matplotlib.backends._backend_gdk", @"matplotlib.backends._backend_gtkagg", @"matplotlib.backends._gtkagg", "test.test_pep3131", "test.test_unicode_identifiers", "test.test_super" // nonlocal syntax error }); skip.UnionWith(modules.Select(m => m.FullName) .Where(n => n.StartsWith(@"test.badsyntax") || n.StartsWith("test.bad_coding"))); var anySuccess = false; var anyExtensionSuccess = false; var anyExtensionSeen = false; var anyParseError = false; foreach (var m in skip) { ((MainModuleResolution)interpreter.ModuleResolution).AddUnimportableModule(m); } var set = modules .Where(m => !skip.Contains(m.ModuleName)) .GroupBy(m => { var i = m.FullName.IndexOf('.'); return(i <= 0 ? m.FullName : m.FullName.Remove(i)); }) .SelectMany(g => g.Select(m => Tuple.Create(m, m.ModuleName))) .ToArray(); set = set.Where(x => x.Item2 != null && x.Item2.Contains("grammar")).ToArray(); var sb = new StringBuilder(); foreach (var r in set) { var module = interpreter.ModuleResolution.GetOrLoadModule(r.Item2); if (module != null) { sb.AppendLine($"import {module.Name}"); } } await GetAnalysisAsync(sb.ToString(), services); foreach (var r in set) { var modName = r.Item1; var mod = interpreter.ModuleResolution.GetOrLoadModule(r.Item2); anyExtensionSeen |= modName.IsNativeExtension; switch (mod) { case null: Trace.TraceWarning("failed to import {0} from {1}", modName.ModuleName, modName.SourceFile); break; case CompiledPythonModule compiledPythonModule: var errors = compiledPythonModule.GetParseErrors().ToArray(); if (errors.Any()) { anyParseError = true; Trace.TraceError("Parse errors in {0}", modName.SourceFile); foreach (var e in errors) { Trace.TraceError(e.Message); } } else { anySuccess = true; anyExtensionSuccess |= modName.IsNativeExtension; } break; case IPythonModule _: { var filteredErrors = ((IDocument)mod).GetParseErrors().Where(e => !e.Message.Contains("encoding problem")).ToArray(); if (filteredErrors.Any()) { // Do not fail due to errors in installed packages if (!mod.FilePath.Contains("site-packages")) { anyParseError = true; } Trace.TraceError("Parse errors in {0}", modName.SourceFile); foreach (var e in filteredErrors) { Trace.TraceError(e.Message); } } else { anySuccess = true; anyExtensionSuccess |= modName.IsNativeExtension; } break; } } } Assert.IsTrue(anySuccess, "failed to import any modules at all"); Assert.IsTrue(anyExtensionSuccess || !anyExtensionSeen, "failed to import all extension modules"); Assert.IsFalse(anyParseError, "parse errors occurred"); }
/// <summary> /// Determines whether the interpreter factory contains the specified /// modules. /// </summary> /// <returns>The names of the modules that were found.</returns> public static async Task <HashSet <string> > FindModulesAsync(this IPythonInterpreterFactory factory, params string[] moduleNames) { var finding = new HashSet <string>(moduleNames); var found = new HashSet <string>(); var withPackages = factory.PackageManager; if (withPackages != null) { foreach (var m in finding) { if ((await withPackages.GetInstalledPackageAsync(new PackageSpec(m), CancellationToken.None)).IsValid) { found.Add(m); } } finding.ExceptWith(found); if (!finding.Any()) { // Found all of them, so stop searching return(found); } } var withDb = factory as PythonInterpreterFactoryWithDatabase; if (withDb != null && withDb.IsCurrent) { var db = withDb.GetCurrentDatabase(); found.UnionWith(finding.Where(m => db.GetModule(m) != null)); // Always stop searching after this step return(found); } if (withDb != null) { try { var paths = await PythonTypeDatabase.GetDatabaseSearchPathsAsync(withDb); found.UnionWith(PythonTypeDatabase.GetDatabaseExpectedModules(withDb.Configuration.Version, paths) .SelectMany() .Select(g => g.ModuleName) .Where(m => finding.Contains(m))); } catch (InvalidOperationException) { } finding.ExceptWith(found); if (!finding.Any()) { // Found all of them, so stop searching return(found); } } return(await Task.Run(() => { foreach (var mp in ModulePath.GetModulesInLib(factory.Configuration)) { if (finding.Remove(mp.ModuleName)) { found.Add(mp.ModuleName); } if (!finding.Any()) { break; } } return found; })); }
/// <summary> /// Determines whether the interpreter factory contains the specified /// modules. /// </summary> /// <returns>The names of the modules that were found.</returns> public static async Task <HashSet <string> > FindModulesAsync(this IPythonInterpreterFactory factory, params string[] moduleNames) { var withPackages = factory as IPackageManager; if (withPackages != null) { var res = new HashSet <string>(); foreach (var m in moduleNames) { if ((await withPackages.GetInstalledPackageAsync(new PackageSpec(m), CancellationToken.None)).IsValid) { res.Add(m); } } if (res.SetEquals(moduleNames)) { return(res); } } var withDb = factory as PythonInterpreterFactoryWithDatabase; if (withDb != null && withDb.IsCurrent) { var db = withDb.GetCurrentDatabase(); var set = new HashSet <string>(moduleNames.Where(m => db.GetModule(m) != null)); return(set); } var expected = new HashSet <string>(moduleNames); if (withDb != null) { try { var paths = PythonTypeDatabase.GetCachedDatabaseSearchPaths(withDb.DatabasePath) ?? await PythonTypeDatabase.GetUncachedDatabaseSearchPathsAsync(withDb.Configuration.InterpreterPath).ConfigureAwait(false); var db = PythonTypeDatabase.GetDatabaseExpectedModules(withDb.Configuration.Version, paths) .SelectMany() .Select(g => g.ModuleName); expected.IntersectWith(db); return(expected); } catch (InvalidOperationException) { } } return(await Task.Run(() => { var result = new HashSet <string>(); foreach (var mp in ModulePath.GetModulesInLib(factory.Configuration)) { if (expected.Count == 0) { break; } if (expected.Remove(mp.ModuleName)) { result.Add(mp.ModuleName); } } return result; })); }