public async Task <IPythonModule> ImportModuleAsync(string name, CancellationToken token) { if (name == BuiltinModuleName) { if (_builtinModule == null) { _modules[BuiltinModuleName] = _builtinModule = new AstBuiltinsPythonModule(_factory.LanguageVersion); _builtinModuleNames = null; } return(_builtinModule); } var ctxt = new AstPythonInterpreterFactory.TryImportModuleContext { Interpreter = this, ModuleCache = _modules, BuiltinModule = _builtinModule, FindModuleInUserSearchPathAsync = FindModuleInUserSearchPathAsync, TypeStubPaths = _analyzer.Limits.UseTypeStubPackages ? _analyzer.GetTypeStubPaths() : null, MergeTypeStubPackages = !_analyzer.Limits.UseTypeStubPackagesExclusively }; for (var retries = 5; retries > 0; --retries) { // The call should be cancelled by the cancellation token, but since we // are blocking here we wait for slightly longer. Timeouts are handled // gracefully by TryImportModuleAsync(), so we want those to trigger if // possible, but if all else fails then we'll abort and treat it as an // error. // (And if we've got a debugger attached, don't time out at all.) AstPythonInterpreterFactory.TryImportModuleResult result; try { result = await _factory.TryImportModuleAsync(name, ctxt, token); } catch (OperationCanceledException) { _log.Log(TraceLevel.Error, "ImportTimeout", name); Debug.Fail("Import timeout"); return(null); } switch (result.Status) { case AstPythonInterpreterFactory.TryImportModuleResultCode.Success: return(result.Module); case AstPythonInterpreterFactory.TryImportModuleResultCode.ModuleNotFound: _log?.Log(TraceLevel.Info, "ImportNotFound", name); return(null); case AstPythonInterpreterFactory.TryImportModuleResultCode.NeedRetry: case AstPythonInterpreterFactory.TryImportModuleResultCode.Timeout: break; case AstPythonInterpreterFactory.TryImportModuleResultCode.NotSupported: _log?.Log(TraceLevel.Error, "ImportNotSupported", name); return(null); } } // Never succeeded, so just log the error and fail _log?.Log(TraceLevel.Error, "RetryImport", name); return(null); }
public IPythonModule ImportModule(string name) { if (string.IsNullOrEmpty(name)) { return(null); } // Handle builtins explicitly if (name == BuiltinModuleName) { if (_builtinModule == null) { _modules[BuiltinModuleName] = _builtinModule = new AstBuiltinsPythonModule(_factory.LanguageVersion); _builtinModule.Imported(this); var bmn = ((AstBuiltinsPythonModule)_builtinModule).GetAnyMember("__builtin_module_names") as AstPythonStringLiteral; _builtinModuleNames = bmn?.Value?.Split(','); } return(_builtinModule); } IPythonModule mod; // Return any existing module if (_modules.TryGetValue(name, out mod) && mod != null) { if (mod is EmptyModule) { Trace.TraceWarning($"Recursively importing {name}"); } return(mod); } // Set up a sentinel so we can detect recursive imports var sentinalValue = new EmptyModule(); if (!_modules.TryAdd(name, sentinalValue)) { return(_modules[name]); } // Do normal searches mod = ImportFromBuiltins(name) ?? ImportFromSearchPaths(name); // Replace our sentinel, or if we raced, get the current // value and abandon the one we just created. if (!_modules.TryUpdate(name, mod, sentinalValue)) { mod = _modules[name]; } return(mod); }
public IPythonModule ImportModule(string name) { if (name == BuiltinModuleName) { if (_builtinModule == null) { _modules[BuiltinModuleName] = _builtinModule = new AstBuiltinsPythonModule(_factory.LanguageVersion); _builtinModuleNames = null; } return(_builtinModule); } IPythonModule module = null; var ctxt = new AstPythonInterpreterFactory.TryImportModuleContext { Interpreter = this, ModuleCache = _modules, BuiltinModule = _builtinModule, FindModuleInUserSearchPathAsync = FindModuleInUserSearchPathAsync, TypeStubPaths = _analyzer.Limits.UseTypeStubPackages ? _analyzer.GetTypeStubPaths() : null, MergeTypeStubPackages = !_analyzer.Limits.UseTypeStubPackagesExclusively }; for (int retries = 5; retries > 0; --retries) { switch (_factory.TryImportModule(name, out module, ctxt)) { case AstPythonInterpreterFactory.TryImportModuleResult.Success: return(module); case AstPythonInterpreterFactory.TryImportModuleResult.ModuleNotFound: _log?.Log(TraceLevel.Info, "ImportNotFound", name); return(null); case AstPythonInterpreterFactory.TryImportModuleResult.NeedRetry: case AstPythonInterpreterFactory.TryImportModuleResult.Timeout: break; case AstPythonInterpreterFactory.TryImportModuleResult.NotSupported: _log?.Log(TraceLevel.Error, "ImportNotSupported", name); return(null); } } // Never succeeded, so just log the error and fail _log?.Log(TraceLevel.Error, "RetryImport", name); return(null); }
private async Task <IPythonModule> ImportFromSearchPathsAsync(string name, TryImportModuleContext context, CancellationToken cancellationToken) { ModulePath?mmp = null; if (context.FindModuleInUserSearchPathAsync != null) { try { mmp = await context.FindModuleInUserSearchPathAsync(name, cancellationToken); } catch (Exception ex) { _log?.Log(TraceLevel.Error, "Exception", ex.ToString()); _log?.Flush(); return(null); } } if (!mmp.HasValue) { mmp = await FindModuleInSearchPathAsync(name, cancellationToken); } if (!mmp.HasValue) { _log?.Log(TraceLevel.Verbose, "ImportNotFound", name); return(null); } var mp = mmp.Value; IPythonModule module; if (mp.IsCompiled) { _log?.Log(TraceLevel.Verbose, "ImportScraped", mp.FullName, FastRelativePath(mp.SourceFile)); module = new AstScrapedPythonModule(mp.FullName, mp.SourceFile); } else { _log?.Log(TraceLevel.Verbose, "Import", mp.FullName, FastRelativePath(mp.SourceFile)); module = PythonModuleLoader.FromFile(context.Interpreter, mp.SourceFile, LanguageVersion, mp.FullName); } return(module); }
private IPythonModule ImportModuleOrRetry(string name, out bool retry) { retry = false; if (string.IsNullOrEmpty(name)) { return(null); } Debug.Assert(!name.EndsWith("."), $"{name} should not end with '.'"); // Handle builtins explicitly if (name == BuiltinModuleName) { if (_builtinModule == null) { _modules[BuiltinModuleName] = _builtinModule = new AstBuiltinsPythonModule(_factory.LanguageVersion); _builtinModuleNames = null; _builtinModule.Imported(this); var bmn = ((AstBuiltinsPythonModule)_builtinModule).GetAnyMember("__builtin_module_names") as AstPythonStringLiteral; _builtinModuleNames = bmn?.Value?.Split(',') ?? Array.Empty <string>(); } return(_builtinModule); } IPythonModule mod; // Return any existing module if (_modules.TryGetValue(name, out mod) && mod != null) { if (mod is SentinelModule smod) { // If we are importing this module on another thread, allow // time for it to complete. This does not block if we are // importing on the current thread or the module is not // really being imported. var newMod = smod.WaitForImport(5000); if (newMod is SentinelModule) { _log?.Log(TraceLevel.Warning, "RecursiveImport", name); mod = newMod; } else if (newMod == null) { _log?.Log(TraceLevel.Warning, "ImportTimeout", name); } else { mod = newMod; } } return(mod); } // Set up a sentinel so we can detect recursive imports var sentinalValue = new SentinelModule(name, true); if (!_modules.TryAdd(name, sentinalValue)) { // Try to get the new module, in case we raced with a .Clear() if (_modules.TryGetValue(name, out mod) && !(mod is SentinelModule)) { return(mod); } // If we reach here, the race is too complicated to recover // from. Signal the caller to try importing again. _log?.Log(TraceLevel.Warning, "RetryImport", name); retry = true; return(null); } // Do normal searches mod = ImportFromBuiltins(name) ?? ImportFromSearchPaths(name); // Replace our sentinel, or if we raced, get the current // value and abandon the one we just created. if (!_modules.TryUpdate(name, mod, sentinalValue)) { // Try to get the new module, in case we raced if (_modules.TryGetValue(name, out mod) && !(mod is SentinelModule)) { return(mod); } // If we reach here, the race is too complicated to recover // from. Signal the caller to try importing again. _log?.Log(TraceLevel.Warning, "RetryImport", name); retry = true; return(null); } sentinalValue.Complete(mod); sentinalValue.Dispose(); return(mod); }
public IPythonModule ImportModule(string name) { if (string.IsNullOrEmpty(name)) { return(null); } IPythonModule mod; if (_modules.TryGetValue(name, out mod) && mod != null) { if (mod is EmptyModule) { Trace.TraceWarning($"Recursively importing {name}"); } return(mod); } var sentinalValue = new EmptyModule(); if (!_modules.TryAdd(name, sentinalValue)) { return(_modules[name]); } var mmp = FindModuleInSearchPath(_userSearchPaths, GetUserSearchPathPackages(), name); if (mmp.HasValue) { lock (_userSearchPathsLock) { if (_userSearchPathImported == null) { _userSearchPathImported = new HashSet <string>(); } _userSearchPathImported.Add(name); } } else { mmp = FindModuleInSearchPath(_factory.GetSearchPaths(), _factory.GetImportableModules(), name); } if (!mmp.HasValue) { if (!_modules.TryUpdate(name, null, sentinalValue)) { return(_modules[name]); } return(null); } var mp = mmp.Value; if (mp.IsCompiled) { mod = new AstScrapedPythonModule(mp.FullName, mp.SourceFile); } else { mod = AstPythonModule.FromFile(this, mp.SourceFile, _factory.LanguageVersion, mp.FullName); } if (!_modules.TryUpdate(name, mod, sentinalValue)) { mod = _modules[name]; } return(mod); }