private async Task <IPythonModule> ImportFromSearchPathsAsync(string name, TryImportModuleContext context)
{
try {
return(await ImportFromSearchPathsAsyncWorker(name, context).ConfigureAwait(false));
} catch (Exception ex) {
_log?.Log(TraceLevel.Error, "ImportFromSearchPathsAsync", ex.ToString());
throw;
}
}
private async Task <IPythonModule> ImportFromTypeStubsAsync(string name, TryImportModuleContext context, CancellationToken cancellationToken)
{
var mp = FindModuleInSearchPath(context.TypeStubPaths, null, name);
if (mp == null)
{
int i = name.IndexOf('.');
if (i == 0)
{
Debug.Fail("Invalid module name");
return(null);
}
var stubName = i < 0 ? (name + "-stubs") : (name.Remove(i)) + "-stubs" + name.Substring(i);
ModulePath?stubMp = null;
if (context.FindModuleInUserSearchPathAsync != null)
{
try {
stubMp = await context.FindModuleInUserSearchPathAsync(stubName, cancellationToken);
} catch (Exception ex) {
_log?.Log(TraceLevel.Error, "Exception", ex.ToString());
_log?.Flush();
return(null);
}
}
if (stubMp == null)
{
stubMp = await FindModuleInSearchPathAsync(stubName, cancellationToken);
}
if (stubMp != null)
{
mp = new ModulePath(name, stubMp?.SourceFile, stubMp?.LibraryPath);
}
}
if (mp == null && context.TypeStubPaths != null && context.TypeStubPaths.Count > 0)
{
mp = FindModuleInSearchPath(context.TypeStubPaths.SelectMany(GetTypeShedPaths).ToArray(), null, name);
}
if (mp == null)
{
return(null);
}
if (mp.Value.IsCompiled)
{
Debug.Fail("Unsupported native module in typeshed");
return(null);
}
_log?.Log(TraceLevel.Verbose, "ImportTypeStub", mp?.FullName, FastRelativePath(mp?.SourceFile));
return(PythonModuleLoader.FromTypeStub(context.Interpreter, mp?.SourceFile, LanguageVersion, mp?.FullName));
}
private async Task <IPythonModule> ImportFromSearchPathsAsync(string name, TryImportModuleContext context, CancellationToken cancellationToken)
{
ModulePath?mmp = null;
if (context.FindModuleInUserSearchPathAsync != null)
{
try {
mmp = await context.FindModuleInUserSearchPathAsync(name, cancellationToken);
} catch (Exception ex) {
_log?.Log(TraceLevel.Error, "Exception", ex.ToString());
_log?.Flush();
return(null);
}
}
if (!mmp.HasValue)
{
mmp = await FindModuleInSearchPathAsync(name, cancellationToken);
}
if (!mmp.HasValue)
{
_log?.Log(TraceLevel.Verbose, "ImportNotFound", name);
return(null);
}
var mp = mmp.Value;
IPythonModule module;
if (mp.IsCompiled)
{
_log?.Log(TraceLevel.Verbose, "ImportScraped", mp.FullName, FastRelativePath(mp.SourceFile));
module = new AstScrapedPythonModule(mp.FullName, mp.SourceFile);
}
else
{
_log?.Log(TraceLevel.Verbose, "Import", mp.FullName, FastRelativePath(mp.SourceFile));
module = PythonModuleLoader.FromFile(context.Interpreter, mp.SourceFile, LanguageVersion, mp.FullName);
}
return(module);
}
private IPythonModule ImportFromCache(string name, TryImportModuleContext context)
{
if (string.IsNullOrEmpty(CreationOptions.DatabasePath))
{
return(null);
}
var cache = GetCacheFilePath("python.{0}.pyi".FormatInvariant(name));
if (!File.Exists(cache))
{
cache = GetCacheFilePath("python._{0}.pyi".FormatInvariant(name));
if (!File.Exists(cache))
{
cache = GetCacheFilePath("{0}.pyi".FormatInvariant(name));
if (!File.Exists(cache))
{
return(null);
}
}
}
return(PythonModuleLoader.FromTypeStub(context.Interpreter, cache, LanguageVersion, name));
}
public async Task <IPythonModule> ImportModuleAsync(string name, CancellationToken token)
{
if (name == BuiltinModuleName)
{
if (_builtinModule == null)
{
_modules[BuiltinModuleName] = _builtinModule = new AstBuiltinsPythonModule(_factory.LanguageVersion);
_builtinModuleNames = null;
}
return(_builtinModule);
}
Debug.Assert(_analyzer != null);
var ctxt = new TryImportModuleContext {
Interpreter = this,
ModuleCache = _modules,
BuiltinModule = _builtinModule,
FindModuleInUserSearchPathAsync = FindModuleInUserSearchPathAsync,
TypeStubPaths = _analyzer.Limits.UseTypeStubPackages ? _analyzer.GetTypeStubPaths() : null,
MergeTypeStubPackages = !_analyzer.Limits.UseTypeStubPackagesExclusively
};
for (var retries = 5; retries > 0; --retries)
{
// The call should be cancelled by the cancellation token, but since we
// are blocking here we wait for slightly longer. Timeouts are handled
// gracefully by TryImportModuleAsync(), so we want those to trigger if
// possible, but if all else fails then we'll abort and treat it as an
// error.
// (And if we've got a debugger attached, don't time out at all.)
TryImportModuleResult result;
try {
result = await _factory.ModuleResolution.TryImportModuleAsync(name, ctxt, token);
} catch (OperationCanceledException) {
Log.Log(TraceLevel.Error, "ImportTimeout", name);
Debug.Fail("Import timeout");
return(null);
}
switch (result.Status)
{
case TryImportModuleResultCode.Success:
return(result.Module);
case TryImportModuleResultCode.ModuleNotFound:
Log?.Log(TraceLevel.Info, "ImportNotFound", name);
return(null);
case TryImportModuleResultCode.NeedRetry:
case TryImportModuleResultCode.Timeout:
break;
case TryImportModuleResultCode.NotSupported:
Log?.Log(TraceLevel.Error, "ImportNotSupported", name);
return(null);
}
}
// Never succeeded, so just log the error and fail
Log?.Log(TraceLevel.Error, "RetryImport", name);
return(null);
}
public async Task <TryImportModuleResult> TryImportModuleAsync(
string name,
TryImportModuleContext context,
CancellationToken cancellationToken
)
{
IPythonModule module = null;
if (string.IsNullOrEmpty(name))
{
return(TryImportModuleResult.ModuleNotFound);
}
Debug.Assert(!name.EndsWithOrdinal("."), $"{name} should not end with '.'");
// Handle builtins explicitly
if (name == BuiltinModuleName)
{
Debug.Fail($"Interpreters must handle import {name} explicitly");
return(TryImportModuleResult.NotSupported);
}
var modules = context?.ModuleCache;
SentinelModule sentinalValue = null;
if (modules != null)
{
// Return any existing module
if (modules.TryGetValue(name, out module) && module != null)
{
if (module is SentinelModule smod)
{
// If we are importing this module on another thread, allow
// time for it to complete. This does not block if we are
// importing on the current thread or the module is not
// really being imported.
try {
module = await smod.WaitForImportAsync(cancellationToken);
} catch (OperationCanceledException) {
_log?.Log(TraceLevel.Warning, "ImportTimeout", name);
return(TryImportModuleResult.Timeout);
}
if (module is SentinelModule)
{
_log?.Log(TraceLevel.Warning, "RecursiveImport", name);
}
}
return(new TryImportModuleResult(module));
}
// Set up a sentinel so we can detect recursive imports
sentinalValue = new SentinelModule(name, true);
if (!modules.TryAdd(name, sentinalValue))
{
// Try to get the new module, in case we raced with a .Clear()
if (modules.TryGetValue(name, out module) && !(module is SentinelModule))
{
return(new TryImportModuleResult(module));
}
// If we reach here, the race is too complicated to recover
// from. Signal the caller to try importing again.
_log?.Log(TraceLevel.Warning, "RetryImport", name);
return(TryImportModuleResult.NeedRetry);
}
}
// Do normal searches
if (!string.IsNullOrEmpty(Configuration?.InterpreterPath))
{
try {
module = await ImportFromSearchPathsAsync(name, context, cancellationToken);
} catch (OperationCanceledException) {
_log?.Log(TraceLevel.Error, "ImportTimeout", name, "ImportFromSearchPaths");
return(TryImportModuleResult.Timeout);
}
if (module == null)
{
module = ImportFromBuiltins(name, context.BuiltinModule as AstBuiltinsPythonModule);
}
}
if (module == null)
{
module = ImportFromCache(name, context);
}
if (modules != null)
{
if (sentinalValue == null)
{
_log?.Log(TraceLevel.Error, "RetryImport", name, "sentinalValue==null");
Debug.Fail("Sentinal module was never created");
return(TryImportModuleResult.NeedRetry);
}
// Replace our sentinel, or if we raced, get the current
// value and abandon the one we just created.
if (!modules.TryUpdate(name, module, sentinalValue))
{
// Try to get the new module, in case we raced
if (modules.TryGetValue(name, out module) && !(module is SentinelModule))
{
return(new TryImportModuleResult(module));
}
// If we reach here, the race is too complicated to recover
// from. Signal the caller to try importing again.
_log?.Log(TraceLevel.Warning, "RetryImport", name);
return(TryImportModuleResult.NeedRetry);
}
sentinalValue.Complete(module);
}
// Also search for type stub packages if enabled and we are not a blacklisted module
if (module != null && context?.TypeStubPaths != null && module.Name != "typing")
{
var tsModule = await ImportFromTypeStubsAsync(module.Name, context, cancellationToken);
if (tsModule != null)
{
if (context.MergeTypeStubPackages)
{
module = AstPythonMultipleMembers.CombineAs <IPythonModule>(module, tsModule);
}
else
{
module = tsModule;
}
}
}
return(new TryImportModuleResult(module));
}
private async Task <IPythonModule> ImportFromSearchPathsAsyncWorker(string name, TryImportModuleContext context)
{
ModulePath?mmp = null;
if (context.FindModuleInUserSearchPathAsync != null)
{
try {
mmp = await context.FindModuleInUserSearchPathAsync(name);
} catch (Exception ex) {
_log?.Log(TraceLevel.Error, "Exception", ex.ToString());
_log?.Flush();
return(null);
}
}
if (!mmp.HasValue)
{
mmp = await FindModuleInSearchPathAsync(name);
}
if (!mmp.HasValue)
{
_log?.Log(TraceLevel.Verbose, "ImportNotFound", name);
return(null);
}
var mp = mmp.Value;
#if USE_TYPESHED
lock (_typeShedPathsLock) {
if (_typeShedPaths == null)
{
var typeshed = FindModuleInSearchPath(_factory.GetSearchPaths(), _factory.GetImportableModules(), "typeshed");
if (typeshed.HasValue)
{
_typeShedPaths = GetTypeShedPaths(PathUtils.GetParent(typeshed.Value.SourceFile)).ToArray();
}
else
{
_typeShedPaths = Array.Empty <string>();
}
}
if (_typeShedPaths.Any())
{
var mtsp = FindModuleInSearchPath(_typeShedPaths, null, mp.FullName);
if (mtsp.HasValue)
{
mp = mtsp.Value;
}
}
}
#endif
if (mp.IsCompiled)
{
_log?.Log(TraceLevel.Verbose, "ImportScraped", mp.FullName, FastRelativePath(mp.SourceFile));
return(new AstScrapedPythonModule(mp.FullName, mp.SourceFile));
}
_log?.Log(TraceLevel.Verbose, "Import", mp.FullName, FastRelativePath(mp.SourceFile));
return(PythonModuleLoader.FromFile(context.Interpreter, mp.SourceFile, LanguageVersion, mp.FullName));
}
public TryImportModuleResult TryImportModule(
string name,
out IPythonModule module,
TryImportModuleContext context
)
{
module = null;
if (string.IsNullOrEmpty(name))
{
return(TryImportModuleResult.ModuleNotFound);
}
Debug.Assert(!name.EndsWithOrdinal("."), $"{name} should not end with '.'");
// Handle builtins explicitly
if (name == BuiltinModuleName)
{
Debug.Fail($"Interpreters must handle import {name} explicitly");
return(TryImportModuleResult.NotSupported);
}
var modules = context?.ModuleCache;
int importTimeout = context?.Timeout ?? 5000;
SentinelModule sentinalValue = null;
if (modules != null)
{
// Return any existing module
if (modules.TryGetValue(name, out module) && module != null)
{
if (module is SentinelModule smod)
{
// If we are importing this module on another thread, allow
// time for it to complete. This does not block if we are
// importing on the current thread or the module is not
// really being imported.
var newMod = smod.WaitForImport(importTimeout);
if (newMod is SentinelModule)
{
_log?.Log(TraceLevel.Warning, "RecursiveImport", name);
module = newMod;
}
else if (newMod == null)
{
_log?.Log(TraceLevel.Warning, "ImportTimeout", name);
module = null;
return(TryImportModuleResult.Timeout);
}
else
{
module = newMod;
}
}
return(TryImportModuleResult.Success);
}
// Set up a sentinel so we can detect recursive imports
sentinalValue = new SentinelModule(name, true);
if (!modules.TryAdd(name, sentinalValue))
{
// Try to get the new module, in case we raced with a .Clear()
if (modules.TryGetValue(name, out module) && !(module is SentinelModule))
{
return(module == null ? TryImportModuleResult.ModuleNotFound : TryImportModuleResult.Success);
}
// If we reach here, the race is too complicated to recover
// from. Signal the caller to try importing again.
_log?.Log(TraceLevel.Warning, "RetryImport", name);
return(TryImportModuleResult.NeedRetry);
}
}
// Do normal searches
if (!string.IsNullOrEmpty(Configuration?.InterpreterPath))
{
var importTask = ImportFromSearchPathsAsync(name, context);
if (importTask.Wait(importTimeout))
{
module = importTask.Result;
}
else
{
_log?.Log(TraceLevel.Error, "ImportTimeout", name, "ImportFromSearchPaths");
module = null;
}
module = module ?? ImportFromBuiltins(name, context.BuiltinModule as AstBuiltinsPythonModule);
}
if (module == null)
{
module = ImportFromCache(name);
}
if (modules != null)
{
// Replace our sentinel, or if we raced, get the current
// value and abandon the one we just created.
if (!modules.TryUpdate(name, module, sentinalValue))
{
// Try to get the new module, in case we raced
if (modules.TryGetValue(name, out module) && !(module is SentinelModule))
{
return(module == null ? TryImportModuleResult.ModuleNotFound : TryImportModuleResult.Success);
}
// If we reach here, the race is too complicated to recover
// from. Signal the caller to try importing again.
_log?.Log(TraceLevel.Warning, "RetryImport", name);
return(TryImportModuleResult.NeedRetry);
}
sentinalValue.Complete(module);
sentinalValue.Dispose();
}
return(TryImportModuleResult.Success);
}
public TryImportModuleResult TryImportModule(
string name,
out IPythonModule module,
TryImportModuleContext context
)
{
module = null;
if (string.IsNullOrEmpty(name))
{
return(TryImportModuleResult.ModuleNotFound);
}
Debug.Assert(!name.EndsWithOrdinal("."), $"{name} should not end with '.'");
// Handle builtins explicitly
if (name == BuiltinModuleName)
{
Debug.Fail($"Interpreters must handle import {name} explicitly");
return(TryImportModuleResult.NotSupported);
}
var modules = context?.ModuleCache;
int importTimeout = context?.Timeout ?? 5000;
SentinelModule sentinalValue = null;
if (modules != null)
{
// Return any existing module
if (modules.TryGetValue(name, out module) && module != null)
{
if (module is SentinelModule smod)
{
// If we are importing this module on another thread, allow
// time for it to complete. This does not block if we are
// importing on the current thread or the module is not
// really being imported.
var newMod = smod.WaitForImport(importTimeout);
if (newMod is SentinelModule)
{
_log?.Log(TraceLevel.Warning, "RecursiveImport", name);
module = newMod;
}
else if (newMod == null)
{
_log?.Log(TraceLevel.Warning, "ImportTimeout", name);
module = null;
return(TryImportModuleResult.Timeout);
}
else
{
module = newMod;
}
}
return(TryImportModuleResult.Success);
}
// Set up a sentinel so we can detect recursive imports
sentinalValue = new SentinelModule(name, true);
if (!modules.TryAdd(name, sentinalValue))
{
// Try to get the new module, in case we raced with a .Clear()
if (modules.TryGetValue(name, out module) && !(module is SentinelModule))
{
return(module == null ? TryImportModuleResult.ModuleNotFound : TryImportModuleResult.Success);
}
// If we reach here, the race is too complicated to recover
// from. Signal the caller to try importing again.
_log?.Log(TraceLevel.Warning, "RetryImport", name);
return(TryImportModuleResult.NeedRetry);
}
}
// Do normal searches
if (!string.IsNullOrEmpty(Configuration?.InterpreterPath))
{
var importTask = ImportFromSearchPathsAsync(name, context);
if (importTask.Wait(importTimeout))
{
module = importTask.Result;
}
else
{
_log?.Log(TraceLevel.Error, "ImportTimeout", name, "ImportFromSearchPaths");
module = null;
}
module = module ?? ImportFromBuiltins(name, context.BuiltinModule as AstBuiltinsPythonModule);
}
if (module == null)
{
module = ImportFromCache(name);
}
if (modules != null)
{
// Replace our sentinel, or if we raced, get the current
// value and abandon the one we just created.
if (!modules.TryUpdate(name, module, sentinalValue))
{
// Try to get the new module, in case we raced
if (modules.TryGetValue(name, out module) && !(module is SentinelModule))
{
return(module == null ? TryImportModuleResult.ModuleNotFound : TryImportModuleResult.Success);
}
// If we reach here, the race is too complicated to recover
// from. Signal the caller to try importing again.
_log?.Log(TraceLevel.Warning, "RetryImport", name);
return(TryImportModuleResult.NeedRetry);
}
sentinalValue.Complete(module);
sentinalValue.Dispose();
}
// Also search for type stub packages if enabled and we are not a blacklisted module
if (module != null && module.Name != "typing")
{
// Note that this currently only looks in the typeshed package, as type stub
// packages are not yet standardised so we don't know where to look.
// The details will be in PEP 561.
if (context.TypeStubPaths?.Any() == true)
{
var mtsp = FindModuleInSearchPath(context.TypeStubPaths, null, module.Name);
if (mtsp.HasValue)
{
var mp = mtsp.Value;
if (mp.IsCompiled)
{
Debug.Fail("Unsupported native module in typeshed");
}
else
{
_log?.Log(TraceLevel.Verbose, "ImportTypeShed", mp.FullName, FastRelativePath(mp.SourceFile));
var tsModule = PythonModuleLoader.FromFile(context.Interpreter, mp.SourceFile, LanguageVersion, mp.FullName);
if (tsModule != null)
{
if (context.MergeTypeStubPackages)
{
module = AstPythonMultipleModules.Combine(module, tsModule);
}
else
{
module = tsModule;
}
}
}
}
}
}
return(module == null ? TryImportModuleResult.ModuleNotFound : TryImportModuleResult.Success);
}