public override bool Walk(FunctionDefinition node)
{
// Inner function, declare as variable. Do not set variable location
// since it is not an assignment visible to the user.
var m = SymbolTable.Evaluate(node);
if (m != null)
{
Eval.DeclareVariable(node.NameExpression.Name, m, VariableSource.Declaration, node.NameExpression);
}
return(false);
}
private void ExtendAll(Node location, IPythonCollection values)
{
Eval.LookupNameInScopes(AllVariableName, out var scope, LookupOptions.Global);
if (scope == null)
{
return;
}
var all = scope.Variables[AllVariableName]?.Value as IPythonCollection;
var list = PythonCollectionType.CreateConcatenatedList(Module.Interpreter, all, values);
var source = list.IsGeneric() ? VariableSource.Generic : VariableSource.Declaration;
Eval.DeclareVariable(AllVariableName, list, source, location);
}
private void TryHandleClassVariable(MemberExpression mex, IMember value)
{
if (mex.Target is NameExpression nex && nex.Name == "self")
{
var m = Eval.LookupNameInScopes(nex.Name, out _, LookupOptions.Local);
var cls = m.GetPythonType <IPythonClassType>();
if (cls != null)
{
using (Eval.OpenScope(Eval.Module, cls.ClassDefinition, out _)) {
Eval.DeclareVariable(mex.Name, value, VariableSource.Declaration, Eval.GetLocationOfName(mex));
}
}
}
}
public async Task HandleAnnotatedExpressionAsync(ExpressionWithAnnotation expr, IMember value, CancellationToken cancellationToken = default)
{
if (expr?.Annotation == null)
{
return;
}
var variableType = await Eval.GetTypeFromAnnotationAsync(expr.Annotation, cancellationToken);
// If value is null, then this is a pure declaration like
// x: List[str]
// without a value. If value is provided, then this is
// x: List[str] = [...]
// Check value type for compatibility
IMember instance = null;
if (value != null)
{
var valueType = value.GetPythonType();
if (!variableType.IsUnknown() && !valueType.Equals(variableType))
{
// TODO: warn incompatible value type.
// TODO: verify values. Value may be list() while variable type is List[str].
// Leave it as variable type.
}
else
{
instance = value;
}
}
instance = instance ?? variableType?.CreateInstance(variableType.Name, Eval.GetLoc(expr.Expression), ArgumentSet.Empty) ?? Eval.UnknownType;
if (expr.Expression is NameExpression ne)
{
Eval.DeclareVariable(ne.Name, instance, expr.Expression);
return;
}
if (expr.Expression is MemberExpression m)
{
// self.x : int = 42
var self = Eval.LookupNameInScopes("self", out var scope);
var argType = self?.GetPythonType();
if (argType is PythonClassType cls && scope != null)
{
cls.AddMember(m.Name, instance, true);
}
}
}
public async Task HandleWithAsync(WithStatement node, CancellationToken cancellationToken = default) {
cancellationToken.ThrowIfCancellationRequested();
foreach (var item in node.Items.Where(x => x.Variable != null)) {
var contextManager = await Eval.GetValueFromExpressionAsync(item.ContextManager, cancellationToken);
var cmType = contextManager.GetPythonType();
var enter = cmType?.GetMember(node.IsAsync ? @"__aenter__" : @"__enter__")?.GetPythonType<IPythonFunctionType>();
if (enter != null) {
var context = await Eval.GetValueFromFunctionTypeAsync(enter, null, null, cancellationToken);
if (item.Variable is NameExpression nex && !string.IsNullOrEmpty(nex.Name)) {
Eval.DeclareVariable(nex.Name, context, Eval.GetLoc(item));
}
}
}
}
private void TryHandleClassVariable(AssignmentStatement node, IMember value)
{
var mex = node.Left.OfType <MemberExpression>().FirstOrDefault();
if (!string.IsNullOrEmpty(mex?.Name) && mex.Target is NameExpression nex && nex.Name.EqualsOrdinal("self"))
{
var m = Eval.LookupNameInScopes(nex.Name, out _, LookupOptions.Local);
var cls = m.GetPythonType <IPythonClassType>();
if (cls != null)
{
using (Eval.OpenScope(Eval.Module, cls.ClassDefinition, out _)) {
Eval.DeclareVariable(mex.Name, value, VariableSource.Declaration, Eval.GetLocationOfName(mex));
}
}
}
}
private void DeclareParameter(Parameter p, ParameterInfo pi)
{
IPythonType paramType;
// If type is known from annotation, use it.
if (pi != null && !pi.Type.IsUnknown() && !pi.Type.IsGenericParameter())
{
// TODO: technically generics may have constraints. Should we consider them?
paramType = pi.Type;
}
else
{
paramType = pi?.DefaultValue?.GetPythonType() ?? Eval.UnknownType;
}
Eval.DeclareVariable(p.Name, new PythonInstance(paramType), VariableSource.Declaration, p.NameExpression);
}
private void AssignImportedVariables(IPythonModule module, DottedName moduleImportExpression, NameExpression asNameExpression)
{
// "import fob.oar as baz" is handled as
// baz = import_module('fob.oar')
if (asNameExpression != null)
{
Eval.DeclareVariable(asNameExpression.Name, module, asNameExpression);
return;
}
// "import fob.oar" is handled as
// import_module('fob.oar')
// fob = import_module('fob')
var importNames = moduleImportExpression.Names;
PythonPackage pythonPackage = null;
var existingDepth = 0;
var childPackage = Eval.GetInScope <PythonPackage>(importNames[0].Name);
while (childPackage != null && existingDepth < importNames.Count - 1)
{
existingDepth++;
pythonPackage = childPackage;
childPackage = pythonPackage.GetMember <PythonPackage>(importNames[existingDepth].Name);
}
var child = module;
for (var i = importNames.Count - 2; i >= existingDepth; i--)
{
var childName = importNames[i + 1].Name;
var parentName = importNames[i].Name;
var parent = new PythonPackage(parentName, Eval.Services);
parent.AddChildModule(childName, child);
child = parent;
}
if (pythonPackage == null)
{
Eval.DeclareVariable(importNames[0].Name, child, importNames[0]);
}
else
{
pythonPackage.AddChildModule(importNames[existingDepth].Name, child);
}
}
public bool HandleTryExcept(TryStatement node)
{
node.Body.Walk(Walker);
foreach (var handler in node.Handlers.MaybeEnumerate())
{
if (handler.Test != null && handler.Target is NameExpression nex)
{
var value = Eval.GetValueFromExpression(handler.Test);
Eval.DeclareVariable(nex.Name, value ?? Eval.UnknownType, VariableSource.Declaration, nex);
}
handler.Body.Walk(Walker);
}
node.Finally?.Walk(Walker);
node.Else?.Walk(Walker);
return(false);
}
private void ExtendAll(Node declNode, IReadOnlyList <IMember> values)
{
Eval.LookupNameInScopes(AllVariableName, out var scope, LookupOptions.Normal);
if (scope == null)
{
return;
}
var loc = Eval.GetLoc(declNode);
var allContents = (scope.Variables[AllVariableName].Value as IPythonCollection)?.Contents;
var list = PythonCollectionType.CreateConcatenatedList(Module.Interpreter, loc, allContents, values);
var source = list.IsGeneric() ? VariableSource.Generic : VariableSource.Declaration;
Eval.DeclareVariable(AllVariableName, list, source, loc);
}
public async Task HandleForAsync(ForStatement node, CancellationToken cancellationToken = default)
{
cancellationToken.ThrowIfCancellationRequested();
var iterable = await Eval.GetValueFromExpressionAsync(node.List, cancellationToken);
var iterator = (iterable as IPythonIterable)?.GetIterator();
if (iterator == null)
{
// TODO: report that expression does not evaluate to iterable.
}
var value = iterator?.Next ?? Eval.UnknownType;
switch (node.Left)
{
case NameExpression nex:
// for x in y:
Eval.DeclareVariable(nex.Name, value, Eval.GetLoc(node.Left));
break;
case TupleExpression tex:
// x = [('abc', 42, True), ('abc', 23, False)]
// for some_str, some_int, some_bool in x:
var names = tex.Items.OfType <NameExpression>().Select(x => x.Name).ToArray();
if (value is IPythonIterable valueIterable)
{
var valueIterator = valueIterable.GetIterator();
foreach (var n in names)
{
Eval.DeclareVariable(n, valueIterator?.Next ?? Eval.UnknownType, Eval.GetLoc(node.Left));
}
}
else
{
// TODO: report that expression yields value that does not evaluate to iterable.
}
break;
}
if (node.Body != null)
{
await node.Body.WalkAsync(Walker, cancellationToken);
}
}
public async Task EvaluateClassAsync(CancellationToken cancellationToken = default)
{
cancellationToken.ThrowIfCancellationRequested();
// Open class scope chain
using (Eval.OpenScope(_classDef, out var outerScope)) {
var instance = Eval.GetInScope(_classDef.Name, outerScope);
if (!(instance?.GetPythonType() is PythonClassType classInfo))
{
if (instance != null)
{
// TODO: warning that variable is already declared of a different type.
}
// May be odd case like class inside a class.
return;
}
// Evaluate inner classes, if any
await EvaluateInnerClassesAsync(_classDef, cancellationToken);
_class = classInfo;
// Set bases to the class.
var bases = new List <IPythonType>();
foreach (var a in _classDef.Bases.Where(a => string.IsNullOrEmpty(a.Name)))
{
// We cheat slightly and treat base classes as annotations.
var b = await Eval.GetTypeFromAnnotationAsync(a.Expression, cancellationToken);
if (b != null)
{
bases.Add(b.GetPythonType());
}
}
_class.SetBases(Interpreter, bases);
// Declare __class__ variable in the scope.
Eval.DeclareVariable("__class__", _class, _classDef);
await ProcessClassBody(cancellationToken);
}
}
public void HandleWith(WithStatement node)
{
foreach (var item in node.Items.Where(x => x.Variable != null))
{
var contextManager = Eval.GetValueFromExpression(item.ContextManager);
var cmType = contextManager?.GetPythonType();
IMember context = Eval.UnknownType;
var enter = cmType?.GetMember(node.IsAsync ? @"__aenter__" : @"__enter__")?.GetPythonType <IPythonFunctionType>();
if (enter != null)
{
var instance = contextManager as IPythonInstance;
var callExpr = item.ContextManager as CallExpression;
context = Eval.GetValueFromFunctionType(enter, instance, callExpr);
// If fetching context from __enter__ failed, annotation in the stub may be using
// type from typing that we haven't specialized yet or there may be an issue in
// the stub itself, such as type or incorrect type. Try using context manager then.
context = context.IsUnknown() ? contextManager : context;
}
switch (item.Variable)
{
// Handle with Test() as a
case NameExpression nameExpr when !string.IsNullOrEmpty(nameExpr.Name):
Eval.DeclareVariable(nameExpr.Name, context, VariableSource.Declaration, item);
break;
// Handle with Test() as (a)
case ParenthesisExpression parExpr when parExpr.Expression is NameExpression nameExpr && !string.IsNullOrEmpty(nameExpr.Name):
Eval.DeclareVariable(nameExpr.Name, context, VariableSource.Declaration, item);
break;
// Handle with Test() as (a, b)
// Single element list [a] is a sequence expression so also handled here
case SequenceExpression seqExpr:
var sequenceHandler = new SequenceExpressionHandler(Walker);
sequenceHandler.HandleAssignment(seqExpr, context);
break;
}
}
}
public override bool Walk(PythonAst node)
{
Check.InvalidOperation(() => Ast == node, "walking wrong AST");
_cancellationToken.ThrowIfCancellationRequested();
// Collect basic information about classes and functions in order
// to correctly process forward references. Does not determine
// types yet since at this time imports or generic definitions
// have not been processed.
SymbolTable.Build(Eval);
// There are cases (see typeshed datetime stub) with constructs
// class A:
// def __init__(self, x: Optional[B]): ...
//
// _A = A
//
// class B:
// def func(self, x: Optional[_A])
//
// so evaluation of A -> B ends up incomplete since _A is not known yet.
// Thus, when A type is created, we need to go and evaluate all assignments
// that might be referring to it in the right hand side.
if (Ast.Body is SuiteStatement ste)
{
foreach (var statement in ste.Statements.OfType <AssignmentStatement>())
{
if (statement.Left.Count == 1 && statement.Left[0] is NameExpression leftNex && statement.Right is NameExpression rightNex)
{
var m = Eval.GetInScope <IPythonClassType>(rightNex.Name);
if (m != null)
{
Eval.DeclareVariable(leftNex.Name, m, VariableSource.Declaration, leftNex);
}
}
}
}
return(base.Walk(node));
}
private void HandleTypedVariable(IPythonType variableType, IMember value, Expression expr)
{
// Check value type for compatibility
IMember instance = null;
if (value != null)
{
var valueType = value.GetPythonType();
if (!variableType.IsUnknown() && !valueType.Equals(variableType))
{
// TODO: warn incompatible value type.
// TODO: verify values. Value may be list() while variable type is List[str].
// Leave it as variable type.
}
else
{
instance = value;
}
}
var args = ArgumentSet.Empty(expr, Eval);
instance = instance ?? variableType?.CreateInstance(args) ?? Eval.UnknownType.CreateInstance(ArgumentSet.WithoutContext);
if (expr is NameExpression ne)
{
Eval.DeclareVariable(ne.Name, instance, VariableSource.Declaration, ne);
return;
}
if (expr is MemberExpression m)
{
// self.x : int = 42
var self = Eval.LookupNameInScopes("self", out var scope);
var argType = self?.GetPythonType();
if (argType is PythonClassType cls && scope != null)
{
cls.AddMember(m.Name, instance, true);
}
}
}
public void EvaluateClass()
{
// Open class scope chain
using (Eval.OpenScope(Module, _classDef, out var outerScope)) {
var instance = Eval.GetInScope(_classDef.Name, outerScope);
if (!(instance?.GetPythonType() is PythonClassType classInfo))
{
if (instance != null)
{
// TODO: warning that variable is already declared of a different type.
}
return;
}
// Evaluate inner classes, if any
EvaluateInnerClasses(_classDef);
_class = classInfo;
// Set bases to the class.
var bases = new List <IPythonType>();
foreach (var a in _classDef.Bases.Where(a => string.IsNullOrEmpty(a.Name)))
{
// We cheat slightly and treat base classes as annotations.
var b = Eval.GetTypeFromAnnotation(a.Expression);
if (b != null)
{
var t = b.GetPythonType();
bases.Add(t);
t.AddReference(Eval.GetLocationOfName(a.Expression));
}
}
_class.SetBases(bases);
// Declare __class__ variable in the scope.
Eval.DeclareVariable("__class__", _class, VariableSource.Declaration);
ProcessClassBody();
}
}
private void HandleImport(ModuleName moduleImportExpression, NameExpression asNameExpression, bool forceAbsolute)
{
// "import fob.oar.baz" means
// import_module('fob')
// import_module('fob.oar')
// import_module('fob.oar.baz')
var importNames = ImmutableArray <string> .Empty;
var lastModule = default(PythonVariableModule);
var firstModule = default(PythonVariableModule);
foreach (var nameExpression in moduleImportExpression.Names)
{
importNames = importNames.Add(nameExpression.Name);
var imports = ModuleResolution.CurrentPathResolver.GetImportsFromAbsoluteName(Module.FilePath, importNames, forceAbsolute);
if (!HandleImportSearchResult(imports, lastModule, asNameExpression, moduleImportExpression, out lastModule))
{
lastModule = default;
break;
}
if (firstModule == default)
{
firstModule = lastModule;
}
}
// "import fob.oar.baz as baz" is handled as baz = import_module('fob.oar.baz')
// "import fob.oar.baz" is handled as fob = import_module('fob')
if (!string.IsNullOrEmpty(asNameExpression?.Name) && lastModule != default)
{
Eval.DeclareVariable(asNameExpression.Name, lastModule, VariableSource.Import, asNameExpression);
}
else if (firstModule != default && !string.IsNullOrEmpty(importNames[0]))
{
var firstName = moduleImportExpression.Names[0];
Eval.DeclareVariable(importNames[0], firstModule, VariableSource.Import, firstName);
}
}
private async Task DeclareParameterAsync(Parameter p, int index, ParameterInfo pi, CancellationToken cancellationToken = default)
{
IPythonType paramType;
// If type is known from annotation, use it.
if (pi != null && !pi.Type.IsUnknown() && !pi.Type.IsGenericParameter())
{
// TODO: technically generics may have constraints. Should we consider them?
paramType = pi.Type;
}
else
{
var defaultValue = await Eval.GetValueFromExpressionAsync(p.DefaultValue, cancellationToken) ?? Eval.UnknownType;
paramType = defaultValue.GetPythonType();
if (!paramType.IsUnknown())
{
pi?.SetDefaultValueType(paramType);
}
}
Eval.DeclareVariable(p.Name, paramType, p.NameExpression);
}
private void AssignVariables(FromImportStatement node, IImportSearchResult imports, PythonVariableModule variableModule)
{
if (variableModule == null)
{
return;
}
var names = node.Names;
var asNames = node.AsNames;
if (names.Count == 1 && names[0].Name == "*")
{
// TODO: warn this is not a good style per
// TODO: https://docs.python.org/3/faq/programming.html#what-are-the-best-practices-for-using-import-in-a-module
// TODO: warn this is invalid if not in the global scope.
HandleModuleImportStar(variableModule, imports is ImplicitPackageImport, node.StartIndex);
return;
}
for (var i = 0; i < names.Count; i++)
{
var memberName = names[i].Name;
if (!string.IsNullOrEmpty(memberName))
{
var nameExpression = asNames[i] ?? names[i];
var variableName = nameExpression?.Name ?? memberName;
if (!string.IsNullOrEmpty(variableName))
{
var variable = variableModule.Analysis?.GlobalScope?.Variables[memberName];
var exported = variable ?? variableModule.GetMember(memberName);
var value = exported ?? GetValueFromImports(variableModule, imports as IImportChildrenSource, memberName);
// Do not allow imported variables to override local declarations
Eval.DeclareVariable(variableName, value, VariableSource.Import, nameExpression, CanOverwriteVariable(variableName, node.StartIndex));
}
}
}
}
private void HandleModuleImportStar(PythonVariableModule variableModule, bool isImplicitPackage, int importPosition)
{
if (variableModule.Module == Module)
{
// from self import * won't define any new members
return;
}
// If __all__ is present, take it, otherwise declare all members from the module that do not begin with an underscore.
var memberNames = isImplicitPackage
? variableModule.GetMemberNames()
: variableModule.Analysis.StarImportMemberNames ?? variableModule.GetMemberNames().Where(s => !s.StartsWithOrdinal("_"));
foreach (var memberName in memberNames)
{
var member = variableModule.GetMember(memberName);
if (member == null)
{
Log?.Log(TraceEventType.Verbose, $"Undefined import: {variableModule.Name}, {memberName}");
}
else if (member.MemberType == PythonMemberType.Unknown)
{
Log?.Log(TraceEventType.Verbose, $"Unknown import: {variableModule.Name}, {memberName}");
}
member = member ?? Eval.UnknownType;
if (member is IPythonModule m)
{
ModuleResolution.GetOrLoadModule(m.Name);
}
var variable = variableModule.Analysis?.GlobalScope?.Variables[memberName];
// Do not allow imported variables to override local declarations
Eval.DeclareVariable(memberName, variable ?? member, VariableSource.Import, Eval.DefaultLocation, CanOverwriteVariable(memberName, importPosition));
}
}
public async Task HandleAssignmentAsync(AssignmentStatement node, CancellationToken cancellationToken = default)
{
cancellationToken.ThrowIfCancellationRequested();
var value = await Eval.GetValueFromExpressionAsync(node.Right, cancellationToken);
if (value.IsUnknown())
{
Log?.Log(TraceEventType.Verbose, $"Undefined value: {node.Right.ToCodeString(Ast).Trim()}");
}
if (value?.GetPythonType().TypeId == BuiltinTypeId.Ellipsis)
{
value = Eval.UnknownType;
}
if (node.Left.FirstOrDefault() is TupleExpression lhs)
{
// Tuple = Tuple. Transfer values.
var texHandler = new TupleExpressionHandler(Walker);
await texHandler.HandleTupleAssignmentAsync(lhs, node.Right, value, cancellationToken);
return;
}
foreach (var expr in node.Left.OfType <ExpressionWithAnnotation>())
{
// x: List[str] = [...]
await HandleAnnotatedExpressionAsync(expr, value, cancellationToken);
}
foreach (var ne in node.Left.OfType <NameExpression>())
{
if (Eval.CurrentScope.NonLocals[ne.Name] != null)
{
Eval.LookupNameInScopes(ne.Name, out var scope, LookupOptions.Nonlocal);
if (scope != null)
{
scope.Variables[ne.Name].Value = value;
}
else
{
// TODO: report variable is not declared in outer scopes.
}
continue;
}
if (Eval.CurrentScope.Globals[ne.Name] != null)
{
Eval.LookupNameInScopes(ne.Name, out var scope, LookupOptions.Global);
if (scope != null)
{
scope.Variables[ne.Name].Value = value;
}
else
{
// TODO: report variable is not declared in global scope.
}
continue;
}
Eval.DeclareVariable(ne.Name, value, Eval.GetLoc(ne));
}
}
private void DeclareParameters(bool declareVariables)
{
// For class method no need to add extra parameters, but first parameter type should be the class.
// For static and unbound methods do not add or set anything.
// For regular bound methods add first parameter and set it to the class.
var parameters = new List <ParameterInfo>();
var skip = 0;
if (_self != null && _function.HasClassFirstArgument())
{
var p0 = FunctionDefinition.Parameters.FirstOrDefault();
if (p0 != null && !string.IsNullOrEmpty(p0.Name))
{
// Actual parameter type will be determined when method is invoked.
// The reason is that if method might be called on a derived class.
// Declare self or cls in this scope.
if (declareVariables)
{
Eval.DeclareVariable(p0.Name, new PythonInstance(_self), VariableSource.Declaration, p0.NameExpression);
}
// Set parameter info.
var pi = new ParameterInfo(Ast, p0, _self, null, false);
parameters.Add(pi);
skip++;
}
}
// Declare parameters in scope
IMember defaultValue = null;
for (var i = skip; i < FunctionDefinition.Parameters.Length; i++)
{
var isGeneric = false;
var p = FunctionDefinition.Parameters[i];
if (!string.IsNullOrEmpty(p.Name))
{
IPythonType paramType = null;
if (p.DefaultValue != null)
{
defaultValue = Eval.GetValueFromExpression(p.DefaultValue);
// If parameter has default value, look for the annotation locally first
// since outer type may be getting redefined. Consider 's = None; def f(s: s = 123): ...
paramType = Eval.GetTypeFromAnnotation(p.Annotation, out isGeneric, LookupOptions.Local | LookupOptions.Builtins);
// Default value of None does not mean the parameter is None, just says it can be missing.
defaultValue = defaultValue.IsUnknown() || defaultValue.IsOfType(BuiltinTypeId.NoneType) ? null : defaultValue;
if (paramType == null && defaultValue != null)
{
paramType = defaultValue.GetPythonType();
}
}
// If all else fails, look up globally.
paramType = paramType ?? Eval.GetTypeFromAnnotation(p.Annotation, out isGeneric) ?? Eval.UnknownType;
var pi = new ParameterInfo(Ast, p, paramType, defaultValue, isGeneric);
if (declareVariables)
{
DeclareParameter(p, pi);
}
parameters.Add(pi);
}
}
_overload.SetParameters(parameters);
}
public void HandleAssignment(AssignmentStatement node)
{
if (node.Right is ErrorExpression)
{
return;
}
var value = Eval.GetValueFromExpression(node.Right) ?? Eval.UnknownType;
// Check PEP hint first
var valueType = Eval.GetTypeFromPepHint(node.Right);
if (valueType != null)
{
HandleTypedVariable(valueType, value, node.Left.FirstOrDefault());
return;
}
if (value.IsUnknown())
{
Log?.Log(TraceEventType.Verbose, $"Undefined value: {node.Right.ToCodeString(Ast).Trim()}");
}
if (value?.GetPythonType().TypeId == BuiltinTypeId.Ellipsis)
{
value = Eval.UnknownType;
}
if (node.Left.FirstOrDefault() is SequenceExpression seq)
{
// Tuple = Tuple. Transfer values.
var seqHandler = new SequenceExpressionHandler(Walker);
seqHandler.HandleAssignment(seq.Items, node.Right, value);
return;
}
// Process annotations, if any.
foreach (var expr in node.Left.OfType <ExpressionWithAnnotation>())
{
// x: List[str] = [...]
HandleAnnotatedExpression(expr, value);
}
foreach (var ne in node.Left.OfType <NameExpression>())
{
if (Eval.CurrentScope.NonLocals[ne.Name] != null)
{
Eval.LookupNameInScopes(ne.Name, out var scope, LookupOptions.Nonlocal);
scope?.Variables[ne.Name].Assign(value, Eval.GetLocationOfName(ne));
continue;
}
if (Eval.CurrentScope.Globals[ne.Name] != null)
{
Eval.LookupNameInScopes(ne.Name, out var scope, LookupOptions.Global);
scope?.Variables[ne.Name].Assign(value, Eval.GetLocationOfName(ne));
continue;
}
var source = value.IsGeneric() ? VariableSource.Generic : VariableSource.Declaration;
var location = Eval.GetLocationOfName(ne);
if (IsValidAssignment(ne.Name, location))
{
Eval.DeclareVariable(ne.Name, value ?? Module.Interpreter.UnknownType, source, location);
}
}
TryHandleClassVariable(node, value);
}
/// <summary>
/// Merges data from stub with the data from the module.
/// </summary>
/// <remarks>
/// Functions are taken from the stub by the function walker since
/// information on the return type is needed during the analysis walk.
/// However, if the module is compiled (scraped), it often lacks some
/// of the definitions. Stub may contains those so we need to merge it in.
/// </remarks>
private void MergeStub()
{
if (Module.ModuleType == ModuleType.User)
{
return;
}
// No stub, no merge.
if (_stubAnalysis == null)
{
return;
}
var builtins = Module.Interpreter.ModuleResolution.BuiltinsModule;
// Note that scrape can pick up more functions than the stub contains
// Or the stub can have definitions that scraping had missed. Therefore
// merge is the combination of the two with the documentation coming
// from the library source of from the scraped module.
foreach (var v in _stubAnalysis.GlobalScope.Variables)
{
var stubType = v.Value.GetPythonType();
if (stubType.IsUnknown())
{
continue;
}
var sourceVar = Eval.GlobalScope.Variables[v.Name];
var sourceType = sourceVar?.Value.GetPythonType();
// If stub says 'Any' but we have better type, keep the current type.
if (!IsStubBetterType(sourceType, stubType))
{
continue;
}
// If type does not exist in module, but exists in stub, declare it unless it is an import.
// If types are the classes, merge members. Otherwise, replace type from one from the stub.
switch (sourceType)
{
case null:
// Nothing in sources, but there is type in the stub. Declare it.
if (v.Source == VariableSource.Declaration)
{
Eval.DeclareVariable(v.Name, v.Value, v.Source);
}
break;
case PythonClassType cls when Module.Equals(cls.DeclaringModule):
// If class exists and belongs to this module, add or replace
// its members with ones from the stub, preserving documentation.
// Don't augment types that do not come from this module.
// Do not replace __class__ since it has to match class type and we are not
// replacing class type, we are only merging members.
foreach (var name in stubType.GetMemberNames().Except(new[] { "__class__", "__base__", "__bases__", "__mro__", "mro" }))
{
var stubMember = stubType.GetMember(name);
var member = cls.GetMember(name);
var memberType = member?.GetPythonType();
var stubMemberType = stubMember.GetPythonType();
if (builtins.Equals(memberType?.DeclaringModule) || builtins.Equals(stubMemberType?.DeclaringModule))
{
continue; // Leave builtins alone.
}
if (memberType?.DeclaringModule is SpecializedModule || stubMemberType?.DeclaringModule is SpecializedModule)
{
continue; // Leave specialized modules like typing alone.
}
if (!IsStubBetterType(memberType, stubMemberType))
{
continue;
}
// Get documentation from the current type, if any, since stubs
// typically do not contain documentation while scraped code does.
TransferDocumentationAndLocation(memberType, stubMemberType);
cls.AddMember(name, stubMember, overwrite: true);
}
break;
case IPythonModule _:
// We do not re-declare modules.
break;
default:
// Re-declare variable with the data from the stub unless member is a module.
// Modules members that are modules should remain as they are, i.e. os.path
// should remain library with its own stub attached.
var stubModule = stubType.DeclaringModule;
if (!(stubType is IPythonModule) && !builtins.Equals(stubModule))
{
TransferDocumentationAndLocation(sourceType, stubType);
// TODO: choose best type between the scrape and the stub. Stub probably should always win.
var source = Eval.CurrentScope.Variables[v.Name]?.Source ?? v.Source;
Eval.DeclareVariable(v.Name, v.Value, source);
}
break;
}
}
}
private void MakeUnresolvedImport(string name, Node node) => Eval.DeclareVariable(name, new SentinelModule(name, Eval.Services), Eval.GetLoc(node));
private async Task DeclareParametersAsync(CancellationToken cancellationToken = default)
{
// For class method no need to add extra parameters, but first parameter type should be the class.
// For static and unbound methods do not add or set anything.
// For regular bound methods add first parameter and set it to the class.
var parameters = new List <ParameterInfo>();
var skip = 0;
if (_self != null && _function.HasClassFirstArgument())
{
var p0 = FunctionDefinition.Parameters.FirstOrDefault();
if (p0 != null && !string.IsNullOrEmpty(p0.Name))
{
// Actual parameter type will be determined when method is invoked.
// The reason is that if method might be called on a derived class.
// Declare self or cls in this scope.
Eval.DeclareVariable(p0.Name, _self, p0.NameExpression);
// Set parameter info.
var pi = new ParameterInfo(Ast, p0, _self);
pi.SetType(_self);
parameters.Add(pi);
skip++;
}
}
// Declare parameters in scope
for (var i = skip; i < FunctionDefinition.Parameters.Length; i++)
{
cancellationToken.ThrowIfCancellationRequested();
var p = FunctionDefinition.Parameters[i];
if (!string.IsNullOrEmpty(p.Name))
{
// If parameter has default value, look for the annotation locally first
// since outer type may be getting redefined. Consider 's = None; def f(s: s = 123): ...
IPythonType paramType = null;
if (p.DefaultValue != null)
{
paramType = await Eval.GetTypeFromAnnotationAsync(p.Annotation, cancellationToken, LookupOptions.Local | LookupOptions.Builtins);
if (paramType == null)
{
var defaultValue = await Eval.GetValueFromExpressionAsync(p.DefaultValue, cancellationToken);
if (!defaultValue.IsUnknown())
{
paramType = defaultValue.GetPythonType();
}
}
}
// If all else fails, look up globally.
paramType = paramType ?? await Eval.GetTypeFromAnnotationAsync(p.Annotation, cancellationToken);
var pi = new ParameterInfo(Ast, p, paramType);
await DeclareParameterAsync(p, i, pi, cancellationToken);
parameters.Add(pi);
}
}
_overload.SetParameters(parameters);
}