public PythonAnalyzerSession(IServiceContainer services,
IProgressReporter progress,
Action <Task> startNextSession,
CancellationToken analyzerCancellationToken,
IDependencyChainWalker <AnalysisModuleKey, PythonAnalyzerEntry> walker,
int version,
PythonAnalyzerEntry entry,
bool forceGC = false)
{
_services = services;
_startNextSession = startNextSession;
_analyzerCancellationToken = analyzerCancellationToken;
Version = version;
AffectedEntriesCount = walker?.AffectedValues.Count ?? 1;
_walker = walker;
_entry = entry;
_state = State.NotStarted;
_forceGC = forceGC;
_diagnosticsService = _services.GetService <IDiagnosticsService>();
_platformService = _services.GetService <IOSPlatform>();
_analyzer = _services.GetService <IPythonAnalyzer>();
_log = _services.GetService <ILogger>();
_moduleDatabaseService = _services.GetService <IModuleDatabaseService>();
_progress = progress;
var interpreter = _services.GetService <IPythonInterpreter>();
_modulesPathResolver = interpreter.ModuleResolution.CurrentPathResolver;
_typeshedPathResolver = interpreter.TypeshedResolution.CurrentPathResolver;
}
private IPythonModule ImportFromSearchPaths(string name, PathResolverSnapshot pathResolver)
{
var moduleImport = pathResolver.GetModuleImportFromModuleName(name);
if (moduleImport == null)
{
_log?.Log(TraceLevel.Verbose, "ImportNotFound", name);
return(null);
}
if (moduleImport.IsBuiltin)
{
_log?.Log(TraceLevel.Info, "ImportBuiltins", name, _configuration.InterpreterPath);
return(new AstBuiltinPythonModule(name, _configuration.InterpreterPath));
}
if (moduleImport.IsCompiled)
{
_log?.Log(TraceLevel.Verbose, "ImportScraped", moduleImport.FullName, moduleImport.ModulePath, moduleImport.RootPath);
return(new AstScrapedPythonModule(moduleImport.FullName, moduleImport.ModulePath));
}
_log?.Log(TraceLevel.Verbose, "Import", moduleImport.FullName, moduleImport.ModulePath);
return(PythonModuleLoader.FromFile(_interpreter, moduleImport.ModulePath, _configuration.Version.ToLanguageVersion(), moduleImport.FullName));
}
public AstAnalysisWalker(
IPythonInterpreter interpreter,
PathResolverSnapshot pathResolver,
PythonAst ast,
IPythonModule module,
string filePath,
Uri documentUri,
Dictionary <string, IMember> members,
bool includeLocationInfo,
bool warnAboutUndefinedValues,
bool suppressBuiltinLookup,
AnalysisLogWriter log = null
)
{
_log = log ?? (interpreter as AstPythonInterpreter)?.Log;
_module = module ?? throw new ArgumentNullException(nameof(module));
_members = members ?? throw new ArgumentNullException(nameof(members));
_scope = new NameLookupContext(
interpreter ?? throw new ArgumentNullException(nameof(interpreter)),
interpreter.CreateModuleContext(),
ast ?? throw new ArgumentNullException(nameof(ast)),
_module,
filePath,
documentUri,
includeLocationInfo,
_functionWalkers,
log: warnAboutUndefinedValues ? _log : null
);
_ast = ast;
_interpreter = interpreter;
_pathResolver = pathResolver;
_scope.SuppressBuiltinLookup = suppressBuiltinLookup;
_scope.PushScope(_typingScope);
WarnAboutUndefinedValues = warnAboutUndefinedValues;
}
public void SetCurrentUnit(AnalysisUnit unit)
{
_eval = new ExpressionEvaluator(unit);
_unit = unit;
_pathResolver = ProjectState.CurrentPathResolver;
_filePath = GlobalScope.ProjectEntry.FilePath;
}
private IEnumerable <IFileSystemInfo> LibraryFiles(PathResolverSnapshot snapshot)
{
if (snapshot == null)
{
return(Enumerable.Empty <IFileSystemInfo>());
}
var includeImplicit = !ModulePath.PythonVersionRequiresInitPyFiles(_version.ToVersion());
return(snapshot.GetAllImportableModuleFilePaths(includeImplicit).Select(p => new FileInfoProxy(new FileInfo(p))));
}
internal void Analyze(PythonAst ast, PathResolverSnapshot currentPathResolver)
{
var walker = new AstAnalysisWalker(_interpreter, currentPathResolver, ast, this, FilePath, DocumentUri, _members,
includeLocationInfo: true,
warnAboutUndefinedValues: true,
suppressBuiltinLookup: false
);
ast.Walk(walker);
walker.Complete();
}
public Task IndexSnapshot(PathResolverSnapshot snapshot, CancellationToken ct = default)
{
var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(ct, _disposeToken.CancellationToken);
var linkedCt = linkedCts.Token;
return(Task.Run(() => {
var userFiles = WorkspaceFiles();
// index library files if asked
CreateIndices(LibraryFiles(snapshot).Except(userFiles, FileSystemInfoComparer.Instance), _libraryCodeSymbolIndex, linkedCt);
}, linkedCt).ContinueWith(_ => linkedCts.Dispose()));
}
public DependencyCollector(IPythonModule module, bool?isTypeShed = null)
{
_module = module;
_isTypeshed = isTypeShed ?? module.IsTypeshed;
_moduleResolution = module.Interpreter.ModuleResolution;
_pathResolver = _isTypeshed
? module.Interpreter.TypeshedResolution.CurrentPathResolver
: _moduleResolution.CurrentPathResolver;
if (module.Stub != null)
{
Dependencies.Add(new AnalysisModuleKey(module.Stub));
}
}
public DependencyWalker(IPythonModule module)
{
_module = module;
_isTypeshed = module is StubPythonModule stub && stub.IsTypeshed;
_moduleResolution = module.Interpreter.ModuleResolution;
_pathResolver = _isTypeshed
? module.Interpreter.TypeshedResolution.CurrentPathResolver
: _moduleResolution.CurrentPathResolver;
Dependencies = new HashSet <AnalysisModuleKey>();
if (module.Stub != null)
{
Dependencies.Add(new AnalysisModuleKey(module.Stub));
}
}
public CompletionAnalysis(
IModuleAnalysis analysis,
PythonAst tree,
SourceLocation position,
GetMemberOptions opts,
ServerSettings.PythonCompletionOptions completionSettings,
DocumentationBuilder textBuilder,
ILogger log,
Func <TextReader> openDocument
)
{
Analysis = analysis ?? throw new ArgumentNullException(nameof(analysis));
Tree = tree ?? throw new ArgumentNullException(nameof(tree));
Position = position;
Index = Tree.LocationToIndex(Position);
Options = opts;
_pathResolver = analysis.ProjectState.CurrentPathResolver;
_textBuilder = textBuilder;
_log = log;
_openDocument = openDocument;
_addBrackets = completionSettings.addBrackets;
var finder = new ExpressionFinder(Tree, new GetExpressionOptions {
Names = true,
Members = true,
NamedArgumentNames = true,
ImportNames = true,
ImportAsNames = true,
Literals = true,
Errors = true
});
finder.Get(Index, Index, out var node, out _statement, out _scope);
var index = Index;
var col = Position.Column;
while (CanBackUp(Tree, node, _statement, _scope, col))
{
col -= 1;
index -= 1;
finder.Get(index, index, out node, out _statement, out _scope);
}
Node = node ?? (_statement as ExpressionStatement)?.Expression;
}
private IPythonModule ImportFromTypeStubs(string name, IReadOnlyList <string> typeStubPaths, PathResolverSnapshot pathResolver)
{
var mp = FindModuleInSearchPath(typeStubPaths, null, name);
if (mp == null)
{
var i = name.IndexOf('.');
if (i == 0)
{
Debug.Fail("Invalid module name");
return(null);
}
foreach (var stubPath in pathResolver.GetPossibleModuleStubPaths(name))
{
if (File.Exists(stubPath))
{
return(PythonModuleLoader.FromTypeStub(_interpreter, stubPath, _configuration.Version.ToLanguageVersion(), name));
}
}
}
if (mp == null && typeStubPaths != null && typeStubPaths.Count > 0)
{
mp = FindModuleInSearchPath(typeStubPaths.SelectMany(GetTypeShedPaths).ToArray(), null, name);
}
if (mp == null)
{
return(null);
}
if (mp.Value.IsCompiled)
{
Debug.Fail("Unsupported native module in typeshed");
return(null);
}
_log?.Log(TraceLevel.Verbose, "ImportTypeStub", mp?.FullName, mp?.SourceFile);
return(PythonModuleLoader.FromTypeStub(_interpreter, mp?.SourceFile, _configuration.Version.ToLanguageVersion(), mp?.FullName));
}
public async Task <TryImportModuleResult> TryImportModuleAsync(string name, PathResolverSnapshot pathResolver, IReadOnlyList <string> typeStubPaths, bool mergeTypeStubPackages, CancellationToken cancellationToken)
{
if (string.IsNullOrEmpty(name))
{
return(TryImportModuleResult.ModuleNotFound);
}
Debug.Assert(!name.EndsWithOrdinal("."), $"{name} should not end with '.'");
// Handle builtins explicitly
if (name == BuiltinModuleName)
{
Debug.Fail($"Interpreters must handle import {name} explicitly");
return(TryImportModuleResult.NotSupported);
}
// Return any existing module
if (_modules.TryGetValue(name, out var module) && module != null)
{
if (module is SentinelModule sentinelModule)
{
// If we are importing this module on another thread, allow
// time for it to complete. This does not block if we are
// importing on the current thread or the module is not
// really being imported.
try {
module = await sentinelModule.WaitForImportAsync(cancellationToken);
} catch (OperationCanceledException) {
_log?.Log(TraceLevel.Warning, "ImportTimeout", name);
return(TryImportModuleResult.Timeout);
}
if (module is SentinelModule)
{
_log?.Log(TraceLevel.Warning, "RecursiveImport", name);
}
}
return(new TryImportModuleResult(module));
}
// Set up a sentinel so we can detect recursive imports
var sentinelValue = new SentinelModule(name, true);
if (!_modules.TryAdd(name, sentinelValue))
{
// Try to get the new module, in case we raced with a .Clear()
if (_modules.TryGetValue(name, out module) && !(module is SentinelModule))
{
return(new TryImportModuleResult(module));
}
// If we reach here, the race is too complicated to recover
// from. Signal the caller to try importing again.
_log?.Log(TraceLevel.Warning, "RetryImport", name);
return(TryImportModuleResult.NeedRetry);
}
// Do normal searches
if (!string.IsNullOrEmpty(_configuration?.InterpreterPath))
{
try {
module = ImportFromSearchPaths(name, pathResolver);
} catch (OperationCanceledException) {
_log?.Log(TraceLevel.Error, "ImportTimeout", name, "ImportFromSearchPaths");
return(TryImportModuleResult.Timeout);
}
}
if (module == null)
{
module = _astModuleCache.ImportFromCache(name, _interpreter);
}
// Also search for type stub packages if enabled and we are not a blacklisted module
if (module != null && typeStubPaths != null && module.Name != "typing")
{
var tsModule = ImportFromTypeStubs(module.Name, typeStubPaths, pathResolver);
if (tsModule != null)
{
module = mergeTypeStubPackages ? AstPythonMultipleMembers.CombineAs <IPythonModule>(module, tsModule) : tsModule;
}
}
// Replace our sentinel, or if we raced, get the current
// value and abandon the one we just created.
if (!_modules.TryUpdate(name, module, sentinelValue))
{
// Try to get the new module, in case we raced
if (_modules.TryGetValue(name, out module) && !(module is SentinelModule))
{
return(new TryImportModuleResult(module));
}
// If we reach here, the race is too complicated to recover
// from. Signal the caller to try importing again.
_log?.Log(TraceLevel.Warning, "RetryImport", name);
return(TryImportModuleResult.NeedRetry);
}
sentinelValue.Complete(module);
return(new TryImportModuleResult(module));
}
