private ModulePath?FindModuleInSearchPath(IReadOnlyList <string> searchPaths, IReadOnlyDictionary <string, string> packages, string name)
{
if (searchPaths == null || searchPaths.Count == 0)
{
return(null);
}
var i = name.IndexOf('.');
var firstBit = i < 0 ? name : name.Remove(i);
ModulePath mp;
Func <string, bool> isPackage = IsPackage;
if (firstBit.EndsWithOrdinal("-stubs", ignoreCase: true))
{
isPackage = _fs.DirectoryExists;
}
var requireInitPy = ModulePath.PythonVersionRequiresInitPyFiles(Configuration.Version);
if (packages != null && packages.TryGetValue(firstBit, out var searchPath) && !string.IsNullOrEmpty(searchPath))
{
if (ModulePath.FromBasePathAndName_NoThrow(searchPath, name, isPackage, null, requireInitPy, out mp, out _, out _, out _))
{
return(mp);
}
}
if (searchPaths.MaybeEnumerate()
.Any(sp => ModulePath.FromBasePathAndName_NoThrow(sp, name, isPackage, null, requireInitPy, out mp, out _, out _, out _)))
{
return(mp);
}
return(null);
}
internal async Task <IReadOnlyDictionary <string, string> > GetImportableModulesAsync(CancellationToken cancellationToken)
{
var spp = _searchPathPackages;
if (spp != null)
{
return(spp);
}
var sp = await GetSearchPathsAsync(cancellationToken).ConfigureAwait(false);
if (sp == null)
{
return(null);
}
var requireInitPy = ModulePath.PythonVersionRequiresInitPyFiles(Configuration.Version);
var packageDict = await GetImportableModulesAsync(sp, requireInitPy, cancellationToken).ConfigureAwait(false);
if (!packageDict.Any())
{
return(null);
}
lock (_searchPathsLock) {
if (_searchPathPackages != null)
{
return(_searchPathPackages);
}
_searchPathPackages = packageDict;
return(packageDict);
}
}
/// <summary>
/// Determines whether the interpreter factory contains the specified
/// modules.
/// </summary>
/// <returns>The names of the modules that were found.</returns>
public static async Task <bool> HasModuleAsync(this IPythonInterpreterFactory factory, string moduleName, IInterpreterOptionsService interpreterOptions = null)
{
if (interpreterOptions != null)
{
foreach (var pm in interpreterOptions.GetPackageManagers(factory))
{
if ((await pm.GetInstalledPackageAsync(new PackageSpec(moduleName), CancellationToken.None)).IsValid)
{
return(true);
}
}
}
return(await Task.Run(() => {
var configuration = factory.Configuration;
var prefixPath = configuration.GetPrefixPath();
var libraryPath = !string.IsNullOrEmpty(configuration.LibraryPath) ? configuration.LibraryPath : Path.Combine(prefixPath, "Lib");
var sitePackagesPath = !string.IsNullOrEmpty(configuration.SitePackagesPath) ? configuration.SitePackagesPath : Path.Combine(libraryPath, "site-packages");
var requiresInitPyFiles = ModulePath.PythonVersionRequiresInitPyFiles(configuration.Version);
foreach (var mp in GetModulesInLib(libraryPath, sitePackagesPath, requiresInitPyFiles))
{
if (mp.ModuleName == moduleName)
{
return true;
}
}
return false;
}));
}
public AstModuleResolution(AstModuleCache moduleCache, InterpreterConfiguration configuration, AnalysisLogWriter log)
{
_moduleCache = moduleCache;
_configuration = configuration;
_log = log;
_requireInitPy = ModulePath.PythonVersionRequiresInitPyFiles(_configuration.Version);
}
private async Task GetCodeActionsAsync(IDocumentAnalysis analysis,
Diagnostic[] diagnostics,
Input input,
List <CodeAction> codeActions,
CancellationToken cancellationToken)
{
var importFullNameMap = new Dictionary <string, ImportInfo>();
await AddCandidatesFromIndexAsync(analysis, input.Identifier, importFullNameMap, cancellationToken);
var interpreter = analysis.Document.Interpreter;
var pathResolver = interpreter.ModuleResolution.CurrentPathResolver;
// find installed modules matching the given name. this will include submodules
var languageVersion = Parsing.PythonLanguageVersionExtensions.ToVersion(interpreter.LanguageVersion);
var includeImplicit = !ModulePath.PythonVersionRequiresInitPyFiles(languageVersion);
foreach (var moduleFullName in pathResolver.GetAllImportableModulesByName(input.Identifier, includeImplicit))
{
cancellationToken.ThrowIfCancellationRequested();
importFullNameMap[moduleFullName] = new ImportInfo(moduleImported: false, memberImported: false, isModule: true);
}
// find members matching the given name from modules already loaded.
var moduleInfo = new ModuleInfo(analysis);
foreach (var module in interpreter.ModuleResolution.GetImportedModules(cancellationToken))
{
if (module.ModuleType == ModuleType.Unresolved)
{
continue;
}
// module name is full module name that you can use in import xxxx directly
CollectCandidates(moduleInfo.Reset(module), input.Identifier, importFullNameMap, cancellationToken);
Debug.Assert(moduleInfo.NameParts.Count == 1 && moduleInfo.NameParts[0] == module.Name);
}
// check quick bail out case where we know what module we are looking for
if (input.ModuleFullNameOpt != null)
{
if (importFullNameMap.ContainsKey(input.ModuleFullNameOpt))
{
// add code action if the module exist, otherwise, bail out empty
codeActions.AddIfNotNull(CreateCodeAction(analysis, input.Context, input.ModuleFullNameOpt, diagnostics, locallyInserted: false, cancellationToken));
}
return;
}
// regular case
FilterCandidatesBasedOnContext(analysis, input.Context, importFullNameMap, cancellationToken);
// this will create actual code fix with certain orders
foreach (var fullName in OrderFullNames(importFullNameMap))
{
cancellationToken.ThrowIfCancellationRequested();
codeActions.AddIfNotNull(CreateCodeAction(analysis, input.Context, fullName, diagnostics, locallyInserted: false, cancellationToken));
}
}
private async Task <IReadOnlyCollection <string> > GetPackagesFromDirectoryAsync(string searchPath, CancellationToken cancellationToken)
{
return(ModulePath.GetModulesInPath(
searchPath,
recurse: false,
requireInitPy: ModulePath.PythonVersionRequiresInitPyFiles(_langVersion)
).Select(mp => mp.ModuleName).Where(n => !string.IsNullOrEmpty(n)).ToList());
}
public AstModuleResolution(IPythonInterpreter interpreter, ConcurrentDictionary <string, IPythonModule> modules, AstModuleCache astModuleCache, InterpreterConfiguration configuration, AnalysisLogWriter log)
{
_interpreter = interpreter;
_modules = modules;
_astModuleCache = astModuleCache;
_configuration = configuration;
_log = log;
_requireInitPy = ModulePath.PythonVersionRequiresInitPyFiles(_configuration.Version);
}
protected ModuleResolutionBase(string root, IServiceContainer services)
{
_root = root;
_services = services;
_interpreter = services.GetService <IPythonInterpreter>();
_fs = services.GetService <IFileSystem>();
_log = services.GetService <ILogger>();
_requireInitPy = ModulePath.PythonVersionRequiresInitPyFiles(_interpreter.Configuration.Version);
}
private IEnumerable <IFileSystemInfo> LibraryFiles(PathResolverSnapshot snapshot)
{
if (snapshot == null)
{
return(Enumerable.Empty <IFileSystemInfo>());
}
var includeImplicit = !ModulePath.PythonVersionRequiresInitPyFiles(_version.ToVersion());
return(snapshot.GetAllImportableModuleFilePaths(includeImplicit).Select(p => new FileInfoProxy(new FileInfo(p))));
}
public ModuleResolution(string root, IServiceContainer services)
{
_root = root;
_services = services;
_interpreter = services.GetService <IPythonInterpreter>();
_fs = services.GetService <IFileSystem>();
_log = services.GetService <ILogger>();
_requireInitPy = ModulePath.PythonVersionRequiresInitPyFiles(_interpreter.Configuration.Version);
// TODO: merge with user-provided stub paths
_typeStubPaths = GetTypeShedPaths(_interpreter.Configuration?.TypeshedPath).ToArray();
}
private static IEnumerable <CompletionItem> GetAllImportableModules(CompletionContext context)
{
var interpreter = context.Analysis.Document.Interpreter;
var languageVersion = interpreter.LanguageVersion.ToVersion();
var includeImplicit = !ModulePath.PythonVersionRequiresInitPyFiles(languageVersion);
var modules = interpreter.ModuleResolution.CurrentPathResolver.GetAllImportableModuleNames(includeImplicit);
return(modules
.Where(n => !string.IsNullOrEmpty(n))
.Distinct()
.Select(n => CompletionItemSource.CreateCompletionItem(n, CompletionItemKind.Module)));
}
private IPythonModule LoadModuleFromDirectory(string searchPath, string moduleName)
{
Func <string, bool> isPackage = null;
if (!ModulePath.PythonVersionRequiresInitPyFiles(_langVersion))
{
isPackage = Directory.Exists;
}
ModulePath package;
try {
package = ModulePath.FromBasePathAndName(searchPath, moduleName, isPackage);
} catch (ArgumentException) {
return(null);
}
EnsureSearchPathDB();
if (package.IsNativeExtension || package.IsCompiled)
{
_searchPathDb.LoadExtensionModule(package);
}
else
{
_searchPathDb.AddModule(package.FullName, AstPythonModule.FromFile(
this,
package.SourceFile,
_factory.GetLanguageVersion(),
package.FullName
));
}
var mod = _searchPathDb.GetModule(package.FullName);
if (!package.IsSpecialName)
{
int i = package.FullName.LastIndexOf('.');
if (i >= 1)
{
var parent = package.FullName.Remove(i);
var parentMod = _searchPathDb.GetModule(parent) as AstPythonModule;
if (parentMod != null)
{
parentMod.AddChildModule(package.Name, mod);
}
}
}
return(mod);
}
/// <summary>
/// Returns ModulePaths representing the modules that should be analyzed
/// for the given search paths.
/// </summary>
/// <param name="languageVersion">
/// The Python language version to assume. This affects whether
/// namespace packages are supported or not.
/// </param>
/// <param name="searchPaths">A sequence of paths to search.</param>
/// <returns>
/// All the expected modules, grouped based on codependency. When
/// analyzing modules, all those in the same list should be analyzed
/// together.
/// </returns>
/// <remarks>Added in 2.2</remarks>
public static IEnumerable <List <ModulePath> > GetDatabaseExpectedModules(
Version languageVersion,
IEnumerable <PythonLibraryPath> searchPaths
)
{
var requireInitPy = ModulePath.PythonVersionRequiresInitPyFiles(languageVersion);
var stdlibGroup = new List <ModulePath>();
var packages = new List <List <ModulePath> > {
stdlibGroup
};
foreach (var path in searchPaths ?? Enumerable.Empty <PythonLibraryPath>())
{
if (path.IsStandardLibrary)
{
stdlibGroup.AddRange(ModulePath.GetModulesInPath(
path.Path,
includeTopLevelFiles: true,
recurse: true,
// Always require __init__.py for stdlib folders
// Otherwise we will probably include libraries multiple
// times, and while Python 3.3+ allows this, it's really
// not a good idea.
requireInitPy: true
));
}
else
{
packages.Add(ModulePath.GetModulesInPath(
path.Path,
includeTopLevelFiles: true,
recurse: false,
basePackage: path.ModulePrefix
).ToList());
packages.AddRange(ModulePath.GetModulesInPath(
path.Path,
includeTopLevelFiles: false,
recurse: true,
basePackage: path.ModulePrefix,
requireInitPy: requireInitPy
).GroupBy(g => g.LibraryPath).Select(g => g.ToList()));
}
}
return(packages);
}
private async Task <ModulePath?> FindModuleInUserSearchPathAsync(string name, CancellationToken cancel)
{
var searchPaths = _userSearchPaths;
if (searchPaths == null || searchPaths.Count == 0)
{
return(null);
}
var packages = await GetUserSearchPathPackagesAsync(cancel);
var i = name.IndexOf('.');
var firstBit = i < 0 ? name : name.Remove(i);
string searchPath;
ModulePath mp;
Func <string, bool> isPackage = _factory.IsPackage;
if (firstBit.EndsWithOrdinal("-stubs", ignoreCase: true))
{
isPackage = Directory.Exists;
}
var requireInitPy = ModulePath.PythonVersionRequiresInitPyFiles(_factory.Configuration.Version);
if (packages != null && packages.TryGetValue(firstBit, out searchPath) && !string.IsNullOrEmpty(searchPath))
{
if (ModulePath.FromBasePathAndName_NoThrow(searchPath, name, isPackage, null, requireInitPy, out mp, out _, out _, out _))
{
ImportedFromUserSearchPath(name);
return(mp);
}
}
foreach (var sp in searchPaths.MaybeEnumerate())
{
if (ModulePath.FromBasePathAndName_NoThrow(sp, name, isPackage, null, requireInitPy, out mp, out _, out _, out _))
{
ImportedFromUserSearchPath(name);
return(mp);
}
}
return(null);
}
private ModulePath?FindModuleInSearchPath(IReadOnlyList <string> searchPaths, IReadOnlyDictionary <string, string> packages, string name)
{
if (searchPaths == null || searchPaths.Count == 0)
{
return(null);
}
_log?.Log(TraceLevel.Verbose, "FindModule", name, "system", string.Join(", ", searchPaths));
int i = name.IndexOf('.');
var firstBit = i < 0 ? name : name.Remove(i);
string searchPath;
ModulePath mp;
Func <string, bool> isPackage = IsPackage;
if (firstBit.EndsWithOrdinal("-stubs", ignoreCase: true))
{
isPackage = Directory.Exists;
}
var requireInitPy = ModulePath.PythonVersionRequiresInitPyFiles(_configuration.Version);
if (packages != null && packages.TryGetValue(firstBit, out searchPath) && !string.IsNullOrEmpty(searchPath))
{
if (ModulePath.FromBasePathAndName_NoThrow(searchPath, name, isPackage, null, requireInitPy, out mp, out _, out _, out _))
{
return(mp);
}
}
foreach (var sp in searchPaths.MaybeEnumerate())
{
if (ModulePath.FromBasePathAndName_NoThrow(sp, name, isPackage, null, requireInitPy, out mp, out _, out _, out _))
{
return(mp);
}
}
return(null);
}
private async Task <IReadOnlyDictionary <string, string> > GetUserSearchPathPackagesAsync(CancellationToken cancellationToken)
{
_log?.Log(TraceLevel.Verbose, "GetUserSearchPathPackagesAsync");
var ussp = _userSearchPathPackages;
if (ussp == null)
{
IReadOnlyList <string> usp;
lock (_userSearchPathsLock) {
usp = _userSearchPaths;
ussp = _userSearchPathPackages;
}
if (ussp != null || usp == null || !usp.Any())
{
return(ussp);
}
_log?.Log(TraceLevel.Verbose, "GetImportableModulesAsync");
var requireInitPy = ModulePath.PythonVersionRequiresInitPyFiles(_factory.Configuration.Version);
ussp = await AstPythonInterpreterFactory.GetImportableModulesAsync(usp, requireInitPy, cancellationToken);
lock (_userSearchPathsLock) {
if (_userSearchPathPackages == null)
{
_userSearchPathPackages = ussp;
}
else
{
ussp = _userSearchPathPackages;
}
}
}
_log?.Log(TraceLevel.Verbose, "GetUserSearchPathPackagesAsync", ussp.Keys.Cast <object>().ToArray());
return(ussp);
}
private IPythonModule LoadModuleFromDirectory(string searchPath, string moduleName)
{
Func <string, bool> isPackage = null;
if (!ModulePath.PythonVersionRequiresInitPyFiles(_langVersion))
{
isPackage = Directory.Exists;
}
ModulePath package;
try {
package = ModulePath.FromBasePathAndName(searchPath, moduleName, isPackage);
} catch (ArgumentException) {
return(null);
}
var db = EnsureSearchPathDB();
if (package.IsNativeExtension || package.IsCompiled)
{
db.LoadExtensionModule(package);
}
else
{
db.AddModule(package.FullName, PythonModuleLoader.FromFile(
this,
package.SourceFile,
_factory.GetLanguageVersion(),
package.FullName
));
}
if (db != _searchPathDb)
{
// Racing with the DB being invalidated.
// It's okay if we miss it here, so don't worry
// about taking the lock.
return(null);
}
var mod = db.GetModule(package.FullName);
if (!package.IsSpecialName)
{
var i = package.FullName.LastIndexOf('.');
if (i >= 1)
{
var parent = package.FullName.Remove(i);
var parentMod = db.GetModule(parent) as AstPythonModule;
if (parentMod != null)
{
parentMod.AddChildModule(package.Name, mod);
}
}
}
lock (_searchPathDbLock) {
if (db != _searchPathDb)
{
// Raced with the DB being invalidated
return(null);
}
}
return(mod);
}
/// <summary>
/// Determines whether the specified directory is an importable package.
/// </summary>
private bool IsPackage(string directory)
{
return(ModulePath.PythonVersionRequiresInitPyFiles(_configuration.Version) ?
!string.IsNullOrEmpty(ModulePath.GetPackageInitPy(directory)) :
Directory.Exists(directory));
}
/// <summary> /// Determines whether the specified directory is an importable package. /// </summary> private bool IsPackage(string directory) => ModulePath.PythonVersionRequiresInitPyFiles(Configuration.Version) ? !string.IsNullOrEmpty(ModulePath.GetPackageInitPy(directory)) : _fs.DirectoryExists(directory);