private async Task <IPythonModule> ImportFromSearchPathsAsync(string name, TryImportModuleContext context) { try { return(await ImportFromSearchPathsAsyncWorker(name, context).ConfigureAwait(false)); } catch (Exception ex) { _log?.Log(TraceLevel.Error, "ImportFromSearchPathsAsync", ex.ToString()); throw; } }
private async Task <IPythonModule> ImportFromTypeStubsAsync(string name, TryImportModuleContext context, CancellationToken cancellationToken) { var mp = FindModuleInSearchPath(context.TypeStubPaths, null, name); if (mp == null) { int i = name.IndexOf('.'); if (i == 0) { Debug.Fail("Invalid module name"); return(null); } var stubName = i < 0 ? (name + "-stubs") : (name.Remove(i)) + "-stubs" + name.Substring(i); ModulePath?stubMp = null; if (context.FindModuleInUserSearchPathAsync != null) { try { stubMp = await context.FindModuleInUserSearchPathAsync(stubName, cancellationToken); } catch (Exception ex) { _log?.Log(TraceLevel.Error, "Exception", ex.ToString()); _log?.Flush(); return(null); } } if (stubMp == null) { stubMp = await FindModuleInSearchPathAsync(stubName, cancellationToken); } if (stubMp != null) { mp = new ModulePath(name, stubMp?.SourceFile, stubMp?.LibraryPath); } } if (mp == null && context.TypeStubPaths != null && context.TypeStubPaths.Count > 0) { mp = FindModuleInSearchPath(context.TypeStubPaths.SelectMany(GetTypeShedPaths).ToArray(), null, name); } if (mp == null) { return(null); } if (mp.Value.IsCompiled) { Debug.Fail("Unsupported native module in typeshed"); return(null); } _log?.Log(TraceLevel.Verbose, "ImportTypeStub", mp?.FullName, FastRelativePath(mp?.SourceFile)); return(PythonModuleLoader.FromTypeStub(context.Interpreter, mp?.SourceFile, LanguageVersion, mp?.FullName)); }
private async Task <IPythonModule> ImportFromSearchPathsAsync(string name, TryImportModuleContext context, CancellationToken cancellationToken) { ModulePath?mmp = null; if (context.FindModuleInUserSearchPathAsync != null) { try { mmp = await context.FindModuleInUserSearchPathAsync(name, cancellationToken); } catch (Exception ex) { _log?.Log(TraceLevel.Error, "Exception", ex.ToString()); _log?.Flush(); return(null); } } if (!mmp.HasValue) { mmp = await FindModuleInSearchPathAsync(name, cancellationToken); } if (!mmp.HasValue) { _log?.Log(TraceLevel.Verbose, "ImportNotFound", name); return(null); } var mp = mmp.Value; IPythonModule module; if (mp.IsCompiled) { _log?.Log(TraceLevel.Verbose, "ImportScraped", mp.FullName, FastRelativePath(mp.SourceFile)); module = new AstScrapedPythonModule(mp.FullName, mp.SourceFile); } else { _log?.Log(TraceLevel.Verbose, "Import", mp.FullName, FastRelativePath(mp.SourceFile)); module = PythonModuleLoader.FromFile(context.Interpreter, mp.SourceFile, LanguageVersion, mp.FullName); } return(module); }
private IPythonModule ImportFromCache(string name, TryImportModuleContext context) { if (string.IsNullOrEmpty(CreationOptions.DatabasePath)) { return(null); } var cache = GetCacheFilePath("python.{0}.pyi".FormatInvariant(name)); if (!File.Exists(cache)) { cache = GetCacheFilePath("python._{0}.pyi".FormatInvariant(name)); if (!File.Exists(cache)) { cache = GetCacheFilePath("{0}.pyi".FormatInvariant(name)); if (!File.Exists(cache)) { return(null); } } } return(PythonModuleLoader.FromTypeStub(context.Interpreter, cache, LanguageVersion, name)); }
public async Task <IPythonModule> ImportModuleAsync(string name, CancellationToken token) { if (name == BuiltinModuleName) { if (_builtinModule == null) { _modules[BuiltinModuleName] = _builtinModule = new AstBuiltinsPythonModule(_factory.LanguageVersion); _builtinModuleNames = null; } return(_builtinModule); } Debug.Assert(_analyzer != null); var ctxt = new TryImportModuleContext { Interpreter = this, ModuleCache = _modules, BuiltinModule = _builtinModule, FindModuleInUserSearchPathAsync = FindModuleInUserSearchPathAsync, TypeStubPaths = _analyzer.Limits.UseTypeStubPackages ? _analyzer.GetTypeStubPaths() : null, MergeTypeStubPackages = !_analyzer.Limits.UseTypeStubPackagesExclusively }; for (var retries = 5; retries > 0; --retries) { // The call should be cancelled by the cancellation token, but since we // are blocking here we wait for slightly longer. Timeouts are handled // gracefully by TryImportModuleAsync(), so we want those to trigger if // possible, but if all else fails then we'll abort and treat it as an // error. // (And if we've got a debugger attached, don't time out at all.) TryImportModuleResult result; try { result = await _factory.ModuleResolution.TryImportModuleAsync(name, ctxt, token); } catch (OperationCanceledException) { Log.Log(TraceLevel.Error, "ImportTimeout", name); Debug.Fail("Import timeout"); return(null); } switch (result.Status) { case TryImportModuleResultCode.Success: return(result.Module); case TryImportModuleResultCode.ModuleNotFound: Log?.Log(TraceLevel.Info, "ImportNotFound", name); return(null); case TryImportModuleResultCode.NeedRetry: case TryImportModuleResultCode.Timeout: break; case TryImportModuleResultCode.NotSupported: Log?.Log(TraceLevel.Error, "ImportNotSupported", name); return(null); } } // Never succeeded, so just log the error and fail Log?.Log(TraceLevel.Error, "RetryImport", name); return(null); }
public async Task <TryImportModuleResult> TryImportModuleAsync( string name, TryImportModuleContext context, CancellationToken cancellationToken ) { IPythonModule module = null; if (string.IsNullOrEmpty(name)) { return(TryImportModuleResult.ModuleNotFound); } Debug.Assert(!name.EndsWithOrdinal("."), $"{name} should not end with '.'"); // Handle builtins explicitly if (name == BuiltinModuleName) { Debug.Fail($"Interpreters must handle import {name} explicitly"); return(TryImportModuleResult.NotSupported); } var modules = context?.ModuleCache; SentinelModule sentinalValue = null; if (modules != null) { // Return any existing module if (modules.TryGetValue(name, out module) && module != null) { if (module is SentinelModule smod) { // If we are importing this module on another thread, allow // time for it to complete. This does not block if we are // importing on the current thread or the module is not // really being imported. try { module = await smod.WaitForImportAsync(cancellationToken); } catch (OperationCanceledException) { _log?.Log(TraceLevel.Warning, "ImportTimeout", name); return(TryImportModuleResult.Timeout); } if (module is SentinelModule) { _log?.Log(TraceLevel.Warning, "RecursiveImport", name); } } return(new TryImportModuleResult(module)); } // Set up a sentinel so we can detect recursive imports sentinalValue = new SentinelModule(name, true); if (!modules.TryAdd(name, sentinalValue)) { // Try to get the new module, in case we raced with a .Clear() if (modules.TryGetValue(name, out module) && !(module is SentinelModule)) { return(new TryImportModuleResult(module)); } // If we reach here, the race is too complicated to recover // from. Signal the caller to try importing again. _log?.Log(TraceLevel.Warning, "RetryImport", name); return(TryImportModuleResult.NeedRetry); } } // Do normal searches if (!string.IsNullOrEmpty(Configuration?.InterpreterPath)) { try { module = await ImportFromSearchPathsAsync(name, context, cancellationToken); } catch (OperationCanceledException) { _log?.Log(TraceLevel.Error, "ImportTimeout", name, "ImportFromSearchPaths"); return(TryImportModuleResult.Timeout); } if (module == null) { module = ImportFromBuiltins(name, context.BuiltinModule as AstBuiltinsPythonModule); } } if (module == null) { module = ImportFromCache(name, context); } if (modules != null) { if (sentinalValue == null) { _log?.Log(TraceLevel.Error, "RetryImport", name, "sentinalValue==null"); Debug.Fail("Sentinal module was never created"); return(TryImportModuleResult.NeedRetry); } // Replace our sentinel, or if we raced, get the current // value and abandon the one we just created. if (!modules.TryUpdate(name, module, sentinalValue)) { // Try to get the new module, in case we raced if (modules.TryGetValue(name, out module) && !(module is SentinelModule)) { return(new TryImportModuleResult(module)); } // If we reach here, the race is too complicated to recover // from. Signal the caller to try importing again. _log?.Log(TraceLevel.Warning, "RetryImport", name); return(TryImportModuleResult.NeedRetry); } sentinalValue.Complete(module); } // Also search for type stub packages if enabled and we are not a blacklisted module if (module != null && context?.TypeStubPaths != null && module.Name != "typing") { var tsModule = await ImportFromTypeStubsAsync(module.Name, context, cancellationToken); if (tsModule != null) { if (context.MergeTypeStubPackages) { module = AstPythonMultipleMembers.CombineAs <IPythonModule>(module, tsModule); } else { module = tsModule; } } } return(new TryImportModuleResult(module)); }
private async Task <IPythonModule> ImportFromSearchPathsAsyncWorker(string name, TryImportModuleContext context) { ModulePath?mmp = null; if (context.FindModuleInUserSearchPathAsync != null) { try { mmp = await context.FindModuleInUserSearchPathAsync(name); } catch (Exception ex) { _log?.Log(TraceLevel.Error, "Exception", ex.ToString()); _log?.Flush(); return(null); } } if (!mmp.HasValue) { mmp = await FindModuleInSearchPathAsync(name); } if (!mmp.HasValue) { _log?.Log(TraceLevel.Verbose, "ImportNotFound", name); return(null); } var mp = mmp.Value; #if USE_TYPESHED lock (_typeShedPathsLock) { if (_typeShedPaths == null) { var typeshed = FindModuleInSearchPath(_factory.GetSearchPaths(), _factory.GetImportableModules(), "typeshed"); if (typeshed.HasValue) { _typeShedPaths = GetTypeShedPaths(PathUtils.GetParent(typeshed.Value.SourceFile)).ToArray(); } else { _typeShedPaths = Array.Empty <string>(); } } if (_typeShedPaths.Any()) { var mtsp = FindModuleInSearchPath(_typeShedPaths, null, mp.FullName); if (mtsp.HasValue) { mp = mtsp.Value; } } } #endif if (mp.IsCompiled) { _log?.Log(TraceLevel.Verbose, "ImportScraped", mp.FullName, FastRelativePath(mp.SourceFile)); return(new AstScrapedPythonModule(mp.FullName, mp.SourceFile)); } _log?.Log(TraceLevel.Verbose, "Import", mp.FullName, FastRelativePath(mp.SourceFile)); return(PythonModuleLoader.FromFile(context.Interpreter, mp.SourceFile, LanguageVersion, mp.FullName)); }
public TryImportModuleResult TryImportModule( string name, out IPythonModule module, TryImportModuleContext context ) { module = null; if (string.IsNullOrEmpty(name)) { return(TryImportModuleResult.ModuleNotFound); } Debug.Assert(!name.EndsWithOrdinal("."), $"{name} should not end with '.'"); // Handle builtins explicitly if (name == BuiltinModuleName) { Debug.Fail($"Interpreters must handle import {name} explicitly"); return(TryImportModuleResult.NotSupported); } var modules = context?.ModuleCache; int importTimeout = context?.Timeout ?? 5000; SentinelModule sentinalValue = null; if (modules != null) { // Return any existing module if (modules.TryGetValue(name, out module) && module != null) { if (module is SentinelModule smod) { // If we are importing this module on another thread, allow // time for it to complete. This does not block if we are // importing on the current thread or the module is not // really being imported. var newMod = smod.WaitForImport(importTimeout); if (newMod is SentinelModule) { _log?.Log(TraceLevel.Warning, "RecursiveImport", name); module = newMod; } else if (newMod == null) { _log?.Log(TraceLevel.Warning, "ImportTimeout", name); module = null; return(TryImportModuleResult.Timeout); } else { module = newMod; } } return(TryImportModuleResult.Success); } // Set up a sentinel so we can detect recursive imports sentinalValue = new SentinelModule(name, true); if (!modules.TryAdd(name, sentinalValue)) { // Try to get the new module, in case we raced with a .Clear() if (modules.TryGetValue(name, out module) && !(module is SentinelModule)) { return(module == null ? TryImportModuleResult.ModuleNotFound : TryImportModuleResult.Success); } // If we reach here, the race is too complicated to recover // from. Signal the caller to try importing again. _log?.Log(TraceLevel.Warning, "RetryImport", name); return(TryImportModuleResult.NeedRetry); } } // Do normal searches if (!string.IsNullOrEmpty(Configuration?.InterpreterPath)) { var importTask = ImportFromSearchPathsAsync(name, context); if (importTask.Wait(importTimeout)) { module = importTask.Result; } else { _log?.Log(TraceLevel.Error, "ImportTimeout", name, "ImportFromSearchPaths"); module = null; } module = module ?? ImportFromBuiltins(name, context.BuiltinModule as AstBuiltinsPythonModule); } if (module == null) { module = ImportFromCache(name); } if (modules != null) { // Replace our sentinel, or if we raced, get the current // value and abandon the one we just created. if (!modules.TryUpdate(name, module, sentinalValue)) { // Try to get the new module, in case we raced if (modules.TryGetValue(name, out module) && !(module is SentinelModule)) { return(module == null ? TryImportModuleResult.ModuleNotFound : TryImportModuleResult.Success); } // If we reach here, the race is too complicated to recover // from. Signal the caller to try importing again. _log?.Log(TraceLevel.Warning, "RetryImport", name); return(TryImportModuleResult.NeedRetry); } sentinalValue.Complete(module); sentinalValue.Dispose(); } return(TryImportModuleResult.Success); }
public TryImportModuleResult TryImportModule( string name, out IPythonModule module, TryImportModuleContext context ) { module = null; if (string.IsNullOrEmpty(name)) { return(TryImportModuleResult.ModuleNotFound); } Debug.Assert(!name.EndsWithOrdinal("."), $"{name} should not end with '.'"); // Handle builtins explicitly if (name == BuiltinModuleName) { Debug.Fail($"Interpreters must handle import {name} explicitly"); return(TryImportModuleResult.NotSupported); } var modules = context?.ModuleCache; int importTimeout = context?.Timeout ?? 5000; SentinelModule sentinalValue = null; if (modules != null) { // Return any existing module if (modules.TryGetValue(name, out module) && module != null) { if (module is SentinelModule smod) { // If we are importing this module on another thread, allow // time for it to complete. This does not block if we are // importing on the current thread or the module is not // really being imported. var newMod = smod.WaitForImport(importTimeout); if (newMod is SentinelModule) { _log?.Log(TraceLevel.Warning, "RecursiveImport", name); module = newMod; } else if (newMod == null) { _log?.Log(TraceLevel.Warning, "ImportTimeout", name); module = null; return(TryImportModuleResult.Timeout); } else { module = newMod; } } return(TryImportModuleResult.Success); } // Set up a sentinel so we can detect recursive imports sentinalValue = new SentinelModule(name, true); if (!modules.TryAdd(name, sentinalValue)) { // Try to get the new module, in case we raced with a .Clear() if (modules.TryGetValue(name, out module) && !(module is SentinelModule)) { return(module == null ? TryImportModuleResult.ModuleNotFound : TryImportModuleResult.Success); } // If we reach here, the race is too complicated to recover // from. Signal the caller to try importing again. _log?.Log(TraceLevel.Warning, "RetryImport", name); return(TryImportModuleResult.NeedRetry); } } // Do normal searches if (!string.IsNullOrEmpty(Configuration?.InterpreterPath)) { var importTask = ImportFromSearchPathsAsync(name, context); if (importTask.Wait(importTimeout)) { module = importTask.Result; } else { _log?.Log(TraceLevel.Error, "ImportTimeout", name, "ImportFromSearchPaths"); module = null; } module = module ?? ImportFromBuiltins(name, context.BuiltinModule as AstBuiltinsPythonModule); } if (module == null) { module = ImportFromCache(name); } if (modules != null) { // Replace our sentinel, or if we raced, get the current // value and abandon the one we just created. if (!modules.TryUpdate(name, module, sentinalValue)) { // Try to get the new module, in case we raced if (modules.TryGetValue(name, out module) && !(module is SentinelModule)) { return(module == null ? TryImportModuleResult.ModuleNotFound : TryImportModuleResult.Success); } // If we reach here, the race is too complicated to recover // from. Signal the caller to try importing again. _log?.Log(TraceLevel.Warning, "RetryImport", name); return(TryImportModuleResult.NeedRetry); } sentinalValue.Complete(module); sentinalValue.Dispose(); } // Also search for type stub packages if enabled and we are not a blacklisted module if (module != null && module.Name != "typing") { // Note that this currently only looks in the typeshed package, as type stub // packages are not yet standardised so we don't know where to look. // The details will be in PEP 561. if (context.TypeStubPaths?.Any() == true) { var mtsp = FindModuleInSearchPath(context.TypeStubPaths, null, module.Name); if (mtsp.HasValue) { var mp = mtsp.Value; if (mp.IsCompiled) { Debug.Fail("Unsupported native module in typeshed"); } else { _log?.Log(TraceLevel.Verbose, "ImportTypeShed", mp.FullName, FastRelativePath(mp.SourceFile)); var tsModule = PythonModuleLoader.FromFile(context.Interpreter, mp.SourceFile, LanguageVersion, mp.FullName); if (tsModule != null) { if (context.MergeTypeStubPackages) { module = AstPythonMultipleModules.Combine(module, tsModule); } else { module = tsModule; } } } } } } return(module == null ? TryImportModuleResult.ModuleNotFound : TryImportModuleResult.Success); }