private IPythonModule ImportFromSearchPaths(string name, PathResolverSnapshot pathResolver)
{
var moduleImport = pathResolver.GetModuleImportFromModuleName(name);
if (moduleImport == null)
{
_log?.Log(TraceLevel.Verbose, "ImportNotFound", name);
return(null);
}
if (moduleImport.IsBuiltin)
{
_log?.Log(TraceLevel.Info, "ImportBuiltins", name, _configuration.InterpreterPath);
return(new AstBuiltinPythonModule(name, _configuration.InterpreterPath));
}
if (moduleImport.IsCompiled)
{
_log?.Log(TraceLevel.Verbose, "ImportScraped", moduleImport.FullName, moduleImport.ModulePath, moduleImport.RootPath);
return(new AstScrapedPythonModule(moduleImport.FullName, moduleImport.ModulePath));
}
_log?.Log(TraceLevel.Verbose, "Import", moduleImport.FullName, moduleImport.ModulePath);
return(PythonModuleLoader.FromFile(_interpreter, moduleImport.ModulePath, _configuration.Version.ToLanguageVersion(), moduleImport.FullName));
}
private async Task <IPythonModule> ImportFromTypeStubsAsync(string name, TryImportModuleContext context, CancellationToken cancellationToken)
{
var mp = FindModuleInSearchPath(context.TypeStubPaths, null, name);
if (mp == null)
{
int i = name.IndexOf('.');
if (i == 0)
{
Debug.Fail("Invalid module name");
return(null);
}
var stubName = i < 0 ? (name + "-stubs") : (name.Remove(i)) + "-stubs" + name.Substring(i);
ModulePath?stubMp = null;
if (context.FindModuleInUserSearchPathAsync != null)
{
try {
stubMp = await context.FindModuleInUserSearchPathAsync(stubName, cancellationToken);
} catch (Exception ex) {
_log?.Log(TraceLevel.Error, "Exception", ex.ToString());
_log?.Flush();
return(null);
}
}
if (stubMp == null)
{
stubMp = await FindModuleInSearchPathAsync(stubName, cancellationToken);
}
if (stubMp != null)
{
mp = new ModulePath(name, stubMp?.SourceFile, stubMp?.LibraryPath);
}
}
if (mp == null && context.TypeStubPaths != null && context.TypeStubPaths.Count > 0)
{
mp = FindModuleInSearchPath(context.TypeStubPaths.SelectMany(GetTypeShedPaths).ToArray(), null, name);
}
if (mp == null)
{
return(null);
}
if (mp.Value.IsCompiled)
{
Debug.Fail("Unsupported native module in typeshed");
return(null);
}
_log?.Log(TraceLevel.Verbose, "ImportTypeStub", mp?.FullName, FastRelativePath(mp?.SourceFile));
return(PythonModuleLoader.FromTypeStub(context.Interpreter, mp?.SourceFile, LanguageVersion, mp?.FullName));
}
private async Task <IPythonModule> ImportFromSearchPathsAsync(string name, TryImportModuleContext context, CancellationToken cancellationToken)
{
ModulePath?mmp = null;
if (context.FindModuleInUserSearchPathAsync != null)
{
try {
mmp = await context.FindModuleInUserSearchPathAsync(name, cancellationToken);
} catch (Exception ex) {
_log?.Log(TraceLevel.Error, "Exception", ex.ToString());
_log?.Flush();
return(null);
}
}
if (!mmp.HasValue)
{
mmp = await FindModuleInSearchPathAsync(name, cancellationToken);
}
if (!mmp.HasValue)
{
_log?.Log(TraceLevel.Verbose, "ImportNotFound", name);
return(null);
}
var mp = mmp.Value;
IPythonModule module;
if (mp.IsCompiled)
{
_log?.Log(TraceLevel.Verbose, "ImportScraped", mp.FullName, FastRelativePath(mp.SourceFile));
module = new AstScrapedPythonModule(mp.FullName, mp.SourceFile);
}
else
{
_log?.Log(TraceLevel.Verbose, "Import", mp.FullName, FastRelativePath(mp.SourceFile));
module = PythonModuleLoader.FromFile(context.Interpreter, mp.SourceFile, LanguageVersion, mp.FullName);
}
return(module);
}
private IPythonModule ImportFromTypeStubs(string name, IReadOnlyList <string> typeStubPaths, PathResolverSnapshot pathResolver)
{
var mp = FindModuleInSearchPath(typeStubPaths, null, name);
if (mp == null)
{
var i = name.IndexOf('.');
if (i == 0)
{
Debug.Fail("Invalid module name");
return(null);
}
foreach (var stubPath in pathResolver.GetPossibleModuleStubPaths(name))
{
if (File.Exists(stubPath))
{
return(PythonModuleLoader.FromTypeStub(_interpreter, stubPath, _configuration.Version.ToLanguageVersion(), name));
}
}
}
if (mp == null && typeStubPaths != null && typeStubPaths.Count > 0)
{
mp = FindModuleInSearchPath(typeStubPaths.SelectMany(GetTypeShedPaths).ToArray(), null, name);
}
if (mp == null)
{
return(null);
}
if (mp.Value.IsCompiled)
{
Debug.Fail("Unsupported native module in typeshed");
return(null);
}
_log?.Log(TraceLevel.Verbose, "ImportTypeStub", mp?.FullName, mp?.SourceFile);
return(PythonModuleLoader.FromTypeStub(_interpreter, mp?.SourceFile, _configuration.Version.ToLanguageVersion(), mp?.FullName));
}
public IPythonModule ImportFromCache(string name, IPythonInterpreter interpreter)
{
if (string.IsNullOrEmpty(DatabasePath))
{
return(null);
}
var cache = GetCacheFilePath("python.{0}.pyi".FormatInvariant(name));
if (!File.Exists(cache))
{
cache = GetCacheFilePath("python._{0}.pyi".FormatInvariant(name));
if (!File.Exists(cache))
{
cache = GetCacheFilePath("{0}.pyi".FormatInvariant(name));
if (!File.Exists(cache))
{
return(null);
}
}
}
return(PythonModuleLoader.FromTypeStub(interpreter, cache, _configuration.Version.ToLanguageVersion(), name));
}
private IPythonModule ImportFromCache(string name, TryImportModuleContext context)
{
if (string.IsNullOrEmpty(CreationOptions.DatabasePath))
{
return(null);
}
var cache = GetCacheFilePath("python.{0}.pyi".FormatInvariant(name));
if (!File.Exists(cache))
{
cache = GetCacheFilePath("python._{0}.pyi".FormatInvariant(name));
if (!File.Exists(cache))
{
cache = GetCacheFilePath("{0}.pyi".FormatInvariant(name));
if (!File.Exists(cache))
{
return(null);
}
}
}
return(PythonModuleLoader.FromTypeStub(context.Interpreter, cache, LanguageVersion, name));
}
private async Task <IPythonModule> ImportFromSearchPathsAsyncWorker(string name, TryImportModuleContext context)
{
ModulePath?mmp = null;
if (context.FindModuleInUserSearchPathAsync != null)
{
try {
mmp = await context.FindModuleInUserSearchPathAsync(name);
} catch (Exception ex) {
_log?.Log(TraceLevel.Error, "Exception", ex.ToString());
_log?.Flush();
return(null);
}
}
if (!mmp.HasValue)
{
mmp = await FindModuleInSearchPathAsync(name);
}
if (!mmp.HasValue)
{
_log?.Log(TraceLevel.Verbose, "ImportNotFound", name);
return(null);
}
var mp = mmp.Value;
#if USE_TYPESHED
lock (_typeShedPathsLock) {
if (_typeShedPaths == null)
{
var typeshed = FindModuleInSearchPath(_factory.GetSearchPaths(), _factory.GetImportableModules(), "typeshed");
if (typeshed.HasValue)
{
_typeShedPaths = GetTypeShedPaths(PathUtils.GetParent(typeshed.Value.SourceFile)).ToArray();
}
else
{
_typeShedPaths = Array.Empty <string>();
}
}
if (_typeShedPaths.Any())
{
var mtsp = FindModuleInSearchPath(_typeShedPaths, null, mp.FullName);
if (mtsp.HasValue)
{
mp = mtsp.Value;
}
}
}
#endif
if (mp.IsCompiled)
{
_log?.Log(TraceLevel.Verbose, "ImportScraped", mp.FullName, FastRelativePath(mp.SourceFile));
return(new AstScrapedPythonModule(mp.FullName, mp.SourceFile));
}
_log?.Log(TraceLevel.Verbose, "Import", mp.FullName, FastRelativePath(mp.SourceFile));
return(PythonModuleLoader.FromFile(context.Interpreter, mp.SourceFile, LanguageVersion, mp.FullName));
}
public TryImportModuleResult TryImportModule(
string name,
out IPythonModule module,
TryImportModuleContext context
)
{
module = null;
if (string.IsNullOrEmpty(name))
{
return(TryImportModuleResult.ModuleNotFound);
}
Debug.Assert(!name.EndsWithOrdinal("."), $"{name} should not end with '.'");
// Handle builtins explicitly
if (name == BuiltinModuleName)
{
Debug.Fail($"Interpreters must handle import {name} explicitly");
return(TryImportModuleResult.NotSupported);
}
var modules = context?.ModuleCache;
int importTimeout = context?.Timeout ?? 5000;
SentinelModule sentinalValue = null;
if (modules != null)
{
// Return any existing module
if (modules.TryGetValue(name, out module) && module != null)
{
if (module is SentinelModule smod)
{
// If we are importing this module on another thread, allow
// time for it to complete. This does not block if we are
// importing on the current thread or the module is not
// really being imported.
var newMod = smod.WaitForImport(importTimeout);
if (newMod is SentinelModule)
{
_log?.Log(TraceLevel.Warning, "RecursiveImport", name);
module = newMod;
}
else if (newMod == null)
{
_log?.Log(TraceLevel.Warning, "ImportTimeout", name);
module = null;
return(TryImportModuleResult.Timeout);
}
else
{
module = newMod;
}
}
return(TryImportModuleResult.Success);
}
// Set up a sentinel so we can detect recursive imports
sentinalValue = new SentinelModule(name, true);
if (!modules.TryAdd(name, sentinalValue))
{
// Try to get the new module, in case we raced with a .Clear()
if (modules.TryGetValue(name, out module) && !(module is SentinelModule))
{
return(module == null ? TryImportModuleResult.ModuleNotFound : TryImportModuleResult.Success);
}
// If we reach here, the race is too complicated to recover
// from. Signal the caller to try importing again.
_log?.Log(TraceLevel.Warning, "RetryImport", name);
return(TryImportModuleResult.NeedRetry);
}
}
// Do normal searches
if (!string.IsNullOrEmpty(Configuration?.InterpreterPath))
{
var importTask = ImportFromSearchPathsAsync(name, context);
if (importTask.Wait(importTimeout))
{
module = importTask.Result;
}
else
{
_log?.Log(TraceLevel.Error, "ImportTimeout", name, "ImportFromSearchPaths");
module = null;
}
module = module ?? ImportFromBuiltins(name, context.BuiltinModule as AstBuiltinsPythonModule);
}
if (module == null)
{
module = ImportFromCache(name);
}
if (modules != null)
{
// Replace our sentinel, or if we raced, get the current
// value and abandon the one we just created.
if (!modules.TryUpdate(name, module, sentinalValue))
{
// Try to get the new module, in case we raced
if (modules.TryGetValue(name, out module) && !(module is SentinelModule))
{
return(module == null ? TryImportModuleResult.ModuleNotFound : TryImportModuleResult.Success);
}
// If we reach here, the race is too complicated to recover
// from. Signal the caller to try importing again.
_log?.Log(TraceLevel.Warning, "RetryImport", name);
return(TryImportModuleResult.NeedRetry);
}
sentinalValue.Complete(module);
sentinalValue.Dispose();
}
// Also search for type stub packages if enabled and we are not a blacklisted module
if (module != null && module.Name != "typing")
{
// Note that this currently only looks in the typeshed package, as type stub
// packages are not yet standardised so we don't know where to look.
// The details will be in PEP 561.
if (context.TypeStubPaths?.Any() == true)
{
var mtsp = FindModuleInSearchPath(context.TypeStubPaths, null, module.Name);
if (mtsp.HasValue)
{
var mp = mtsp.Value;
if (mp.IsCompiled)
{
Debug.Fail("Unsupported native module in typeshed");
}
else
{
_log?.Log(TraceLevel.Verbose, "ImportTypeShed", mp.FullName, FastRelativePath(mp.SourceFile));
var tsModule = PythonModuleLoader.FromFile(context.Interpreter, mp.SourceFile, LanguageVersion, mp.FullName);
if (tsModule != null)
{
if (context.MergeTypeStubPackages)
{
module = AstPythonMultipleModules.Combine(module, tsModule);
}
else
{
module = tsModule;
}
}
}
}
}
}
return(module == null ? TryImportModuleResult.ModuleNotFound : TryImportModuleResult.Success);
}