public IronPythonAnalysis(
IPythonInterpreterFactory factory,
IPythonInterpreter interpreter = null,
string builtinName = null
) : base(factory, interpreter, builtinName) {
((IronPythonInterpreter)Analyzer.Interpreter).Remote.AddAssembly(new ObjectHandle(typeof(IronPythonAnalysisTest).Assembly));
}
public ModuleView(IPythonInterpreter interpreter, IModuleContext context, string name, string idbPath) {
_interpreter = interpreter;
_context = context;
Name = name;
_idbPath = idbPath;
_children = CalculateChildren().ToArray();
}
public KnownTypesView(IPythonInterpreter interpreter, Version version) {
int count = (int)BuiltinTypeIdExtensions.LastTypeId;
_children = new IAnalysisItemView[count];
for (int value = 1; value <= count; ++value) {
var expectedName = SharedDatabaseState.GetBuiltinTypeName((BuiltinTypeId)value, version);
string name = string.Format("{0} ({1})",
expectedName,
Enum.GetName(typeof(BuiltinTypeId), value)
);
IPythonType type;
try {
type = interpreter.GetBuiltinType((BuiltinTypeId)value);
if (expectedName != type.Name) {
name = string.Format("{2} ({1}/{0})",
expectedName,
Enum.GetName(typeof(BuiltinTypeId), value),
type.Name
);
}
} catch {
type = null;
}
if (type != null) {
_children[value - 1] = new ClassView(
null,
name,
type
);
} else {
_children[value - 1] = new NullMember(name);
}
}
}
internal AstPythonModule(IPythonInterpreter interpreter, PythonAst ast, string filePath) {
Name = ast.Name;
Documentation = ast.Documentation;
FilePath = filePath;
Locations = new[] { new LocationInfo(filePath, 1, 1) };
_properties = new Dictionary<object, object>();
_childModules = new List<string>();
_members = new Dictionary<string, IMember>();
var walker = new AstAnalysisWalker(interpreter, ast, this, filePath, _members);
ast.Walk(walker);
}
public static IPythonModule FromStream(
IPythonInterpreter interpreter,
Stream sourceFile,
string fileName,
PythonLanguageVersion langVersion
) {
PythonAst ast;
using (var parser = Parser.CreateParser(sourceFile, langVersion)) {
ast = parser.ParseFile();
}
return new AstPythonModule(interpreter, ast, fileName);
}
public AstAnalysisWalker(
IPythonInterpreter interpreter,
PythonAst ast,
IPythonModule module,
string filePath,
Dictionary<string, IMember> members
) {
_interpreter = interpreter;
_ast = ast;
_module = module;
_filePath = filePath;
_members = members;
_scope = new Stack<Dictionary<string, IMember>>();
_noneInst = new AstPythonConstant(_interpreter.GetBuiltinType(BuiltinTypeId.NoneType));
}
public BaseAnalysisTest(IPythonInterpreterFactory factory, IPythonInterpreter interpreter) {
InterpreterFactory = factory;
Interpreter = interpreter;
var objectType = Interpreter.GetBuiltinType(BuiltinTypeId.Object);
Assert.IsNotNull(objectType);
var intType = Interpreter.GetBuiltinType(BuiltinTypeId.Int);
var bytesType = Interpreter.GetBuiltinType(BuiltinTypeId.Bytes);
var listType = Interpreter.GetBuiltinType(BuiltinTypeId.List);
var functionType = Interpreter.GetBuiltinType(BuiltinTypeId.Function);
_objectMembers = objectType.GetMemberNames(DefaultContext).ToArray();
_strMembers = bytesType.GetMemberNames(DefaultContext).ToArray();
_listMembers = listType.GetMemberNames(DefaultContext).ToArray();
_intMembers = intType.GetMemberNames(DefaultContext).ToArray();
_functionMembers = functionType.GetMemberNames(DefaultContext).ToArray();
}
public PythonAsciiString(AsciiString s, IPythonInterpreter interpreter)
: base(s, interpreter.GetBuiltinType(interpreter.GetAsciiTypeId()))
{
}
public PythonAnalyzer(IPythonInterpreterFactory factory, IPythonInterpreter interpreter = null)
: this(factory, interpreter, null)
{
ReloadModulesAsync().WaitAndUnwrapExceptions();
}
public StdLibAnalysisTest(IPythonInterpreterFactory factory, IPythonInterpreter interpreter)
: base(factory, interpreter) {
}
public async Task InitializedAsync(InitializedParams @params, CancellationToken cancellationToken = default, IReadOnlyList <string> userConfiguredPaths = null)
{
var initializationOptions = _initParams?.initializationOptions;
_services.AddService(new DiagnosticsService(_services));
var cacheFolderPath = initializationOptions?.cacheFolderPath;
var fs = _services.GetService <IFileSystem>();
if (cacheFolderPath != null && !fs.DirectoryExists(cacheFolderPath))
{
_log?.Log(TraceEventType.Warning, Resources.Error_InvalidCachePath);
cacheFolderPath = null;
}
var analyzer = new PythonAnalyzer(_services, cacheFolderPath);
_services.AddService(analyzer);
analyzer.AnalysisComplete += OnAnalysisComplete;
_disposableBag.Add(() => analyzer.AnalysisComplete -= OnAnalysisComplete);
_services.AddService(new RunningDocumentTable(_services));
_rdt = _services.GetService <IRunningDocumentTable>();
_rootDir = _initParams?.rootUri != null?_initParams.rootUri.ToAbsolutePath() : _initParams?.rootPath;
if (_rootDir != null)
{
_rootDir = PathUtils.NormalizePathAndTrim(_rootDir);
}
Version.TryParse(initializationOptions?.interpreter.properties?.Version, out var version);
var configuration = new InterpreterConfiguration(null, null,
interpreterPath: initializationOptions?.interpreter.properties?.InterpreterPath,
version: version
);
var typeshedPath = initializationOptions?.typeStubSearchPaths.FirstOrDefault();
userConfiguredPaths = userConfiguredPaths ?? initializationOptions?.searchPaths;
_interpreter = await PythonInterpreter.CreateAsync(configuration, _rootDir, _services, typeshedPath, userConfiguredPaths.ToImmutableArray(), cancellationToken);
_services.AddService(_interpreter);
_log?.Log(TraceEventType.Information,
string.IsNullOrEmpty(_interpreter.Configuration.InterpreterPath)
? Resources.InitializingForGenericInterpreter
: Resources.InitializingForPythonInterpreter.FormatInvariant(_interpreter.Configuration.InterpreterPath));
var fileSystem = _services.GetService <IFileSystem>();
_indexManager = new IndexManager(fileSystem, _interpreter.LanguageVersion, _rootDir,
initializationOptions?.includeFiles,
initializationOptions?.excludeFiles,
_services.GetService <IIdleTimeService>());
_indexManager.IndexWorkspace().DoNotWait();
_services.AddService(_indexManager);
_disposableBag.Add(_indexManager);
var textDocCaps = _initParams?.capabilities?.textDocument;
_completionSource = new CompletionSource(
ChooseDocumentationSource(textDocCaps?.completion?.completionItem?.documentationFormat),
Settings.completion
);
_hoverSource = new HoverSource(
ChooseDocumentationSource(textDocCaps?.hover?.contentFormat)
);
var sigInfo = textDocCaps?.signatureHelp?.signatureInformation;
_signatureSource = new SignatureSource(
ChooseDocumentationSource(sigInfo?.documentationFormat),
sigInfo?.parameterInformation?.labelOffsetSupport == true
);
}
public PythonDictionaryType(IPythonInterpreter interpreter, bool isMutable = true)
: base(null, BuiltinTypeId.Dict, interpreter, isMutable)
{
}
public static IPythonModule FromFile(
IPythonInterpreter interpreter,
string sourceFile,
PythonLanguageVersion langVersion
) => FromFile(interpreter, sourceFile, langVersion, null);
public PythonVariableModule(string name, IPythonInterpreter interpreter)
{
Name = name;
Location = LocationInfo.Empty;
Interpreter = interpreter;
}
private void CreateInterpreter()
{
var fact = GetInterpreterFactory();
_interpreter = fact.CreateInterpreter();
}
public ModuleTable(PythonAnalyzer analyzer, IPythonInterpreter interpreter)
{
_analyzer = analyzer;
_interpreter = interpreter;
}
/// <summary>
/// Creates type info of typing.Dict[TK, TV]
/// </summary>
/// <param name="name">Type name (Dict, Mapping, ...)</param>
/// <param name="keyType">Type of dictionary keys.</param>
/// <param name="valueType">Type of dictionary values.</param>
/// <param name="interpreter">Python interpreter</param>
/// <param name="isMutable">Tells if collection is mutable (Dict) or not (Mapping)</param>
public TypingDictionaryType(string name, IPythonType keyType, IPythonType valueType, IPythonInterpreter interpreter, bool isMutable)
: base(interpreter, isMutable)
{
KeyType = keyType;
ValueType = valueType;
Name = $"{name}[{keyType.Name}, {valueType.Name}]";
}
private static IBuiltinsPythonModule CreateBuiltinsModule(IServiceContainer services, IPythonInterpreter interpreter, IStubCache stubCache)
{
var moduleName = BuiltinTypeId.Unknown.GetModuleName(interpreter.LanguageVersion);
var modulePath = stubCache.GetCacheFilePath(interpreter.Configuration.InterpreterPath);
return(new BuiltinsPythonModule(moduleName, modulePath, services));
}
public PythonVariableModule(IPythonModule module) : base(module)
{
Name = module.Name;
Interpreter = module.Interpreter;
Module = module;
}
public PythonVariableModule(string name, IPythonInterpreter interpreter) : base(null)
{
Name = name;
Interpreter = interpreter;
SetDeclaringModule(this);
}
public static IMember List(IPythonInterpreter interpreter, IPythonFunctionOverload overload, LocationInfo location, IReadOnlyList <IMember> args) => PythonCollectionType.CreateList(interpreter, location, args);
public PythonUnicodeString(string s, IPythonInterpreter interpreter)
: base(s, interpreter.GetBuiltinType(interpreter.GetUnicodeTypeId()))
{
}
public PythonFString(string unparsedFString, IPythonInterpreter interpreter)
: base(interpreter.GetBuiltinType(interpreter.GetUnicodeTypeId()))
{
UnparsedFString = unparsedFString;
}
public async Task <InitializeResult> InitializeAsync(InitializeParams @params, CancellationToken cancellationToken)
{
_disposableBag.ThrowIfDisposed();
_clientCaps = @params.capabilities;
_log = _services.GetService <ILogger>();
_services.AddService(new DiagnosticsService(_services));
var cacheFolderPath = @params.initializationOptions.cacheFolderPath;
var fs = _services.GetService <IFileSystem>();
if (cacheFolderPath != null && !fs.DirectoryExists(cacheFolderPath))
{
_log?.Log(TraceEventType.Warning, Resources.Error_InvalidCachePath);
cacheFolderPath = null;
}
var analyzer = new PythonAnalyzer(_services, cacheFolderPath);
_services.AddService(analyzer);
analyzer.AnalysisComplete += OnAnalysisComplete;
_disposableBag.Add(() => analyzer.AnalysisComplete -= OnAnalysisComplete);
_services.AddService(new RunningDocumentTable(_services));
_rdt = _services.GetService <IRunningDocumentTable>();
_rootDir = @params.rootUri != null? @params.rootUri.ToAbsolutePath() : @params.rootPath;
if (_rootDir != null)
{
_rootDir = PathUtils.NormalizePath(_rootDir);
_rootDir = PathUtils.TrimEndSeparator(_rootDir);
}
Version.TryParse(@params.initializationOptions.interpreter.properties?.Version, out var version);
var configuration = new InterpreterConfiguration(null, null,
interpreterPath: @params.initializationOptions.interpreter.properties?.InterpreterPath,
version: version
)
{
// 1) Split on ';' to support older VS Code extension versions which send paths as a single entry separated by ';'. TODO: Eventually remove.
// 2) Normalize paths.
// 3) If a path isn't rooted, then root it relative to the workspace root. If _rootDir is null, then accept the path as-is.
// 4) Trim off any ending separator for a consistent style.
// 5) Filter out any entries which are the same as the workspace root; they are redundant. Also ignore "/" to work around the extension (for now).
// 6) Remove duplicates.
SearchPaths = @params.initializationOptions.searchPaths
.Select(p => p.Split(';', StringSplitOptions.RemoveEmptyEntries)).SelectMany()
.Select(PathUtils.NormalizePath)
.Select(p => _rootDir == null || Path.IsPathRooted(p) ? p : Path.GetFullPath(p, _rootDir))
.Select(PathUtils.TrimEndSeparator)
.Where(p => !string.IsNullOrWhiteSpace(p) && p != "/" && !p.PathEquals(_rootDir))
.Distinct(PathEqualityComparer.Instance)
.ToList(),
TypeshedPath = @params.initializationOptions.typeStubSearchPaths.FirstOrDefault()
};
_interpreter = await PythonInterpreter.CreateAsync(configuration, _rootDir, _services, cancellationToken);
_services.AddService(_interpreter);
var fileSystem = _services.GetService <IFileSystem>();
_indexManager = new IndexManager(fileSystem, _interpreter.LanguageVersion, _rootDir,
@params.initializationOptions.includeFiles,
@params.initializationOptions.excludeFiles,
_services.GetService <IIdleTimeService>());
_indexManager.IndexWorkspace().DoNotWait();
_services.AddService(_indexManager);
_disposableBag.Add(_indexManager);
DisplayStartupInfo();
_completionSource = new CompletionSource(
ChooseDocumentationSource(_clientCaps?.textDocument?.completion?.completionItem?.documentationFormat),
Settings.completion
);
_hoverSource = new HoverSource(
ChooseDocumentationSource(_clientCaps?.textDocument?.hover?.contentFormat)
);
var sigInfo = _clientCaps?.textDocument?.signatureHelp?.signatureInformation;
_signatureSource = new SignatureSource(
ChooseDocumentationSource(sigInfo?.documentationFormat),
sigInfo?.parameterInformation?.labelOffsetSupport == true
);
return(GetInitializeResult());
}
public PythonInstanceIterator(IMember instance, IPythonInterpreter interpreter)
: base(new PythonIteratorType(BuiltinTypeId.SetIterator, interpreter))
{
__next__ = instance.GetPythonType().GetMember(@"__next__") as IPythonFunctionType;
}
public PythonDictionary(IPythonModule declaringModule, IReadOnlyDictionary <IMember, IMember> contents, bool exact = false) :
this(new PythonDictionaryType(declaringModule), contents, exact : exact)
{
_interpreter = declaringModule.Interpreter;
}
public static IPythonModule FromStream(
IPythonInterpreter interpreter,
Stream sourceFile,
string fileName,
PythonLanguageVersion langVersion
) => FromStream(interpreter, sourceFile, fileName, langVersion, null);
private static IEnumerable <string> GetChildModules(string filePath, string prefix, IPythonInterpreter interpreter)
{
if (interpreter == null || string.IsNullOrEmpty(filePath))
{
yield break;
}
var searchPath = Path.GetDirectoryName(filePath);
if (!Directory.Exists(searchPath))
{
yield break;
}
foreach (var n in ModulePath.GetModulesInPath(
searchPath,
recurse: false,
includePackages: true
).Select(mp => mp.ModuleName).Where(n => !string.IsNullOrEmpty(n)))
{
yield return(n);
}
}
public static IMember List(IPythonInterpreter interpreter, IPythonFunctionOverload overload, IArgumentSet argSet) => PythonCollectionType.CreateList(interpreter, argSet);
public PythonFString(string unparsedFString, IPythonInterpreter interpreter, LocationInfo location = null)
: base(interpreter.GetBuiltinType(interpreter.GetUnicodeTypeId()), location)
{
UnparsedFString = unparsedFString;
}
/// <summary>
/// Creates a new analyzer that is not ready for use. You must call and
/// wait for <see cref="ReloadModulesAsync"/> to complete before using.
/// </summary>
public static PythonAnalyzer Create(IPythonInterpreterFactory factory, IPythonInterpreter interpreter = null)
{
return(new PythonAnalyzer(factory, interpreter));
}
public PythonUnicodeString(string s, IPythonInterpreter interpreter, LocationInfo location = null)
: base(s, interpreter.GetBuiltinType(interpreter.GetUnicodeTypeId()), location)
{
}
/// <summary>
/// Creates type info for a strongly-typed tuple, such as Tuple[T1, T2, ...].
/// </summary>
public NamedTupleType(string tupleName, IReadOnlyList <string> itemNames, IReadOnlyList <IPythonType> itemTypes, IPythonInterpreter interpreter)
: base(itemTypes, interpreter)
{
TupleName = tupleName ?? throw new ArgumentNullException(nameof(tupleName));
ItemNames = itemNames;
var typeNames = itemTypes.Select(t => t.IsUnknown() ? string.Empty : t.Name);
var pairs = itemNames.Zip(typeNames, (name, typeName) => string.IsNullOrEmpty(typeName) ? name : $"{name}: {typeName}");
Name = CodeFormatter.FormatSequence(tupleName, '(', pairs);
}
public static IPythonModule FromFile(IPythonInterpreter interpreter, string sourceFile, PythonLanguageVersion langVersion) {
using (var stream = new FileStream(sourceFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) {
return FromStream(interpreter, stream, sourceFile, langVersion);
}
}
/// <summary>
/// Creates type info for an iterator.
/// </summary>
/// <param name="typeId">Iterator type id, such as <see cref="BuiltinTypeId.StrIterator"/>.</param>
/// <param name="interpreter">Python interpreter</param>
public PythonIteratorType(BuiltinTypeId typeId, IPythonInterpreter interpreter)
: base(typeId, interpreter.ModuleResolution.BuiltinsModule)
{
TypeId = typeId;
}
protected virtual PythonWalker PrepareWalker(IPythonInterpreter interpreter, PythonAst ast)
{
return(new AstAnalysisWalker(interpreter, ast, this, _filePath, _members, false, true));
}
public AnalysisView(
string dbDir,
Version version = null,
bool withContention = false,
bool withRecursion = false
) {
var paths = new List<string>();
paths.Add(dbDir);
while (!File.Exists(IOPath.Combine(paths[0], "__builtin__.idb")) &&
!File.Exists(IOPath.Combine(paths[0], "builtins.idb"))) {
var upOne = IOPath.GetDirectoryName(paths[0]);
if (string.IsNullOrEmpty(upOne) || upOne == paths[0]) {
break;
}
paths.Insert(0, upOne);
}
if (withRecursion) {
paths.AddRange(Directory.EnumerateDirectories(dbDir, "*", SearchOption.AllDirectories));
}
if (version == null) {
if (File.Exists(IOPath.Combine(paths[0], "builtins.idb"))) {
version = new Version(3, 3);
} else {
version = new Version(2, 7);
}
}
_factory = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(
version,
dbDir,
paths.ToArray()
);
Path = dbDir;
_interpreter = _factory.CreateInterpreter();
_context = _interpreter.CreateModuleContext();
var modNames = _interpreter.GetModuleNames();
IEnumerable<Tuple<string, string>> modItems;
if (!withRecursion) {
modItems = modNames
.Select(n => Tuple.Create(n, IOPath.Combine(dbDir, n + ".idb")))
.Where(t => File.Exists(t.Item2));
} else {
modItems = modNames
.Select(n => Tuple.Create(
n,
Directory.EnumerateFiles(dbDir, n + ".idb", SearchOption.AllDirectories).FirstOrDefault()
))
.Where(t => File.Exists(t.Item2));
}
var stopwatch = new Stopwatch();
stopwatch.Start();
if (withContention) {
modItems = modItems
.AsParallel()
.WithExecutionMode(ParallelExecutionMode.ForceParallelism);
}
_modules = modItems
.Select(t => new ModuleView(_interpreter, _context, t.Item1, t.Item2))
.OrderBy(m => m.SortKey)
.ThenBy(m => m.Name)
.ToList<IAnalysisItemView>();
stopwatch.Stop();
_modules.Insert(0, new KnownTypesView(_interpreter, version));
LoadMilliseconds = stopwatch.ElapsedMilliseconds;
TopLevelModuleCount = _modules.Count - 1;
}
protected override string[] GetScrapeArguments(IPythonInterpreter interpreter)
=> !InstallPath.TryGetFile("scrape_module.py", out var sb) ? null : new [] { "-W", "ignore", "-B", "-E", sb };
internal VsProjectAnalyzer(
IServiceProvider serviceProvider,
IPythonInterpreter interpreter,
IPythonInterpreterFactory factory,
IPythonInterpreterFactory[] allFactories,
bool implicitProject = true
) {
_errorProvider = (ErrorTaskProvider)serviceProvider.GetService(typeof(ErrorTaskProvider));
_commentTaskProvider = (CommentTaskProvider)serviceProvider.GetService(typeof(CommentTaskProvider));
_unresolvedSquiggles = new UnresolvedImportSquiggleProvider(serviceProvider, _errorProvider);
_queue = new ParseQueue(this);
_analysisQueue = new AnalysisQueue(this);
_analysisQueue.AnalysisStarted += AnalysisQueue_AnalysisStarted;
_allFactories = allFactories;
_interpreterFactory = factory;
_implicitProject = implicitProject;
if (interpreter != null) {
_pyAnalyzer = PythonAnalyzer.Create(factory, interpreter);
ReloadTask = _pyAnalyzer.ReloadModulesAsync().HandleAllExceptions(SR.ProductName, GetType());
ReloadTask.ContinueWith(_ => ReloadTask = null);
interpreter.ModuleNamesChanged += OnModulesChanged;
}
_projectFiles = new ConcurrentDictionary<string, IProjectEntry>(StringComparer.OrdinalIgnoreCase);
_pyService = serviceProvider.GetPythonToolsService();
_serviceProvider = serviceProvider;
if (_pyAnalyzer != null) {
_pyAnalyzer.Limits.CrossModule = _pyService.GeneralOptions.CrossModuleAnalysisLimit;
// TODO: Load other limits from options
}
_userCount = 1;
}
public PythonAsciiString(AsciiString s, IPythonInterpreter interpreter, LocationInfo location = null)
: base(s, interpreter.GetBuiltinType(interpreter.GetAsciiTypeId()), location)
{
}
