private void ImportMembersFromSelf(FromImportStatement node)
{
var names = node.Names;
var asNames = node.AsNames;
if (names.Count == 1 && names[0].Name == "*")
{
// from self import * won't define any new members
return;
}
for (var i = 0; i < names.Count; i++)
{
if (asNames[i] == null)
{
continue;
}
var importName = names[i].Name;
var memberReference = asNames[i];
var memberName = memberReference.Name;
var member = Module.GetMember(importName);
Eval.DeclareVariable(memberName, member ?? Eval.UnknownType, Eval.GetLoc(names[i]));
}
}
public bool HandleFromImport(FromImportStatement node)
{
if (Module.ModuleType == ModuleType.Specialized)
{
return(false);
}
var rootNames = node.Root.Names;
if (rootNames.Count == 1)
{
var rootName = rootNames[0].Name;
if (rootName.EqualsOrdinal("__future__"))
{
SpecializeFuture(node);
return(false);
}
}
var location = Eval.GetLoc(node.Root);
var imports = ModuleResolution.CurrentPathResolver.FindImports(Module.FilePath, node);
if (HandleImportSearchResult(imports, null, null, location, out var variableModule))
{
AssignVariables(node, imports, variableModule);
}
return(false);
}
public async Task HandleTupleAssignmentAsync(TupleExpression lhs, Expression rhs, IMember value, CancellationToken cancellationToken = default) {
cancellationToken.ThrowIfCancellationRequested();
if (rhs is TupleExpression tex) {
var returnedExpressions = tex.Items.ToArray();
var names = lhs.Items.OfType<NameExpression>().Select(x => x.Name).ToArray();
for (var i = 0; i < Math.Min(names.Length, returnedExpressions.Length); i++) {
if (returnedExpressions[i] != null && !string.IsNullOrEmpty(names[i])) {
var v = await Eval.GetValueFromExpressionAsync(returnedExpressions[i], cancellationToken);
if (v != null) {
Eval.DeclareVariable(names[i], v, returnedExpressions[i]);
}
}
}
return;
}
// Tuple = 'tuple value' (such as from callable). Transfer values.
if (value is IPythonCollection seq) {
var types = seq.Contents.Select(c => c.GetPythonType()).ToArray();
var expressions = lhs.Items.OfType<NameExpression>().ToArray();
var names = expressions.Select(x => x.Name).ToArray();
for (var i = 0; i < Math.Min(names.Length, types.Length); i++) {
if (names[i] != null && types[i] != null) {
var instance = types[i].CreateInstance(null, Eval.GetLoc(expressions[i]), ArgumentSet.Empty);
Eval.DeclareVariable(names[i], instance, expressions[i]);
}
}
}
}
private void HandleImport(ModuleName moduleImportExpression, NameExpression asNameExpression, bool forceAbsolute)
{
var location = Eval.GetLoc(moduleImportExpression);
// "import fob.oar.baz" means
// import_module('fob')
// import_module('fob.oar')
// import_module('fob.oar.baz')
var importNames = ImmutableArray <string> .Empty;
var variableModule = default(PythonVariableModule);
foreach (var nameExpression in moduleImportExpression.Names)
{
importNames = importNames.Add(nameExpression.Name);
var imports = ModuleResolution.CurrentPathResolver.GetImportsFromAbsoluteName(Module.FilePath, importNames, forceAbsolute);
if (!HandleImportSearchResult(imports, variableModule, asNameExpression, location, out variableModule))
{
return;
}
}
// "import fob.oar.baz as baz" is handled as baz = import_module('fob.oar.baz')
// "import fob.oar.baz" is handled as fob = import_module('fob')
if (asNameExpression != null)
{
Eval.DeclareVariable(asNameExpression.Name, variableModule, VariableSource.Import, asNameExpression);
}
else if (importNames.Count > 0 && _variableModules.TryGetValue(importNames[0], out variableModule))
{
var firstName = moduleImportExpression.Names[0];
Eval.DeclareVariable(importNames[0], variableModule, VariableSource.Import, firstName);
}
}
private async Task ImportMembersFromPackageAsync(FromImportStatement node, PackageImport packageImport, CancellationToken cancellationToken = default)
{
var names = node.Names;
var asNames = node.AsNames;
if (names.Count == 1 && names[0].Name == "*")
{
// TODO: Need tracking of previous imports to determine possible imports for namespace package. For now import nothing
Eval.DeclareVariable("*", Eval.UnknownType, Eval.GetLoc(names[0]));
return;
}
for (var i = 0; i < names.Count; i++)
{
var importName = names[i].Name;
var memberReference = asNames[i] ?? names[i];
var memberName = memberReference.Name;
var location = Eval.GetLoc(memberReference);
ModuleImport moduleImport;
IPythonType member;
if ((moduleImport = packageImport.Modules.FirstOrDefault(mi => mi.Name.EqualsOrdinal(importName))) != null)
{
member = await ModuleResolution.ImportModuleAsync(moduleImport.FullName, cancellationToken);
}
else
{
member = Eval.UnknownType;
}
Eval.DeclareVariable(memberName, member, location);
}
}
public Task <bool> HandleGlobalAsync(GlobalStatement node, CancellationToken cancellationToken = default)
{
foreach (var nex in node.Names)
{
Eval.CurrentScope.DeclareGlobal(nex.Name, Eval.GetLoc(nex));
}
return(Task.FromResult(false));
}
public bool HandleGlobal(GlobalStatement node)
{
foreach (var nex in node.Names)
{
Eval.CurrentScope.DeclareGlobal(nex.Name, Eval.GetLoc(nex));
}
return(false);
}
public async Task <bool> HandleImportAsync(ImportStatement node, CancellationToken cancellationToken = default)
{
cancellationToken.ThrowIfCancellationRequested();
if (node.Names == null || Module.ModuleType == ModuleType.Specialized)
{
return(false);
}
var len = Math.Min(node.Names.Count, node.AsNames.Count);
for (var i = 0; i < len; i++)
{
cancellationToken.ThrowIfCancellationRequested();
var moduleImportExpression = node.Names[i];
var importNames = moduleImportExpression.Names.Select(n => n.Name).ToArray();
var asNameExpression = node.AsNames[i];
var memberName = asNameExpression?.Name ?? moduleImportExpression.Names[0].Name;
// If we are processing stub, ignore imports of the original module.
// For example, typeshed stub for sys imports sys.
var imports = ModuleResolution.CurrentPathResolver.GetImportsFromAbsoluteName(Module.FilePath, importNames, node.ForceAbsolute);
if (Module.ModuleType == ModuleType.Stub && imports is ModuleImport mi && mi.Name == Module.Name)
{
continue;
}
var location = Eval.GetLoc(moduleImportExpression);
IPythonModule module = null;
switch (imports)
{
case ModuleImport moduleImport when moduleImport.FullName == Module.Name:
Eval.DeclareVariable(memberName, Module, location);
break;
case ModuleImport moduleImport:
module = await HandleImportAsync(node, moduleImport, cancellationToken);
break;
case PossibleModuleImport possibleModuleImport:
module = await HandlePossibleImportAsync(node, possibleModuleImport, cancellationToken);
break;
default:
// TODO: Package import?
MakeUnresolvedImport(memberName, moduleImportExpression);
break;
}
if (module != null)
{
AssignImportedVariables(module, moduleImportExpression, asNameExpression);
}
}
return(false);
}
public async Task HandleAnnotatedExpressionAsync(ExpressionWithAnnotation expr, IMember value, CancellationToken cancellationToken = default)
{
if (expr?.Annotation == null)
{
return;
}
var variableType = await Eval.GetTypeFromAnnotationAsync(expr.Annotation, cancellationToken);
// If value is null, then this is a pure declaration like
// x: List[str]
// without a value. If value is provided, then this is
// x: List[str] = [...]
// Check value type for compatibility
IMember instance = null;
if (value != null)
{
var valueType = value.GetPythonType();
if (!variableType.IsUnknown() && !valueType.Equals(variableType))
{
// TODO: warn incompatible value type.
// TODO: verify values. Value may be list() while variable type is List[str].
// Leave it as variable type.
}
else
{
instance = value;
}
}
instance = instance ?? variableType?.CreateInstance(variableType.Name, Eval.GetLoc(expr.Expression), ArgumentSet.Empty) ?? Eval.UnknownType;
if (expr.Expression is NameExpression ne)
{
Eval.DeclareVariable(ne.Name, instance, expr.Expression);
return;
}
if (expr.Expression is MemberExpression m)
{
// self.x : int = 42
var self = Eval.LookupNameInScopes("self", out var scope);
var argType = self?.GetPythonType();
if (argType is PythonClassType cls && scope != null)
{
cls.AddMember(m.Name, instance, true);
}
}
}
private void HandleAllAppendExtend(CallExpression node)
{
if (!(node.Target is MemberExpression me))
{
return;
}
if (!IsHandleableAll(me.Target))
{
return;
}
if (node.Args.Count == 0)
{
return;
}
var arg = node.Args[0].Expression;
var v = Eval.GetValueFromExpression(arg);
if (v == null)
{
_allIsUsable = false;
return;
}
IPythonCollection values = null;
switch (me.Name)
{
case "append":
values = PythonCollectionType.CreateList(Module.Interpreter, Eval.GetLoc(arg), new List <IMember>()
{
v
}, exact: true);
break;
case "extend":
values = v as IPythonCollection;
break;
}
if (values == null)
{
_allIsUsable = false;
return;
}
ExtendAll(node, values);
}
public async Task HandleWithAsync(WithStatement node, CancellationToken cancellationToken = default) {
cancellationToken.ThrowIfCancellationRequested();
foreach (var item in node.Items.Where(x => x.Variable != null)) {
var contextManager = await Eval.GetValueFromExpressionAsync(item.ContextManager, cancellationToken);
var cmType = contextManager.GetPythonType();
var enter = cmType?.GetMember(node.IsAsync ? @"__aenter__" : @"__enter__")?.GetPythonType<IPythonFunctionType>();
if (enter != null) {
var context = await Eval.GetValueFromFunctionTypeAsync(enter, null, null, cancellationToken);
if (item.Variable is NameExpression nex && !string.IsNullOrEmpty(nex.Name)) {
Eval.DeclareVariable(nex.Name, context, Eval.GetLoc(item));
}
}
}
}
public override async Task EvaluateAsync(CancellationToken cancellationToken = default)
{
if (SymbolTable.ReplacedByStubs.Contains(Target))
{
return;
}
cancellationToken.ThrowIfCancellationRequested();
// Process annotations.
var annotationType = await Eval.GetTypeFromAnnotationAsync(FunctionDefinition.ReturnAnnotation, cancellationToken);
if (!annotationType.IsUnknown())
{
// Annotations are typically types while actually functions return
// instances unless specifically annotated to a type such as Type[T].
var instance = annotationType.CreateInstance(annotationType.Name, Eval.GetLoc(FunctionDefinition), ArgumentSet.Empty);
_overload.SetReturnValue(instance, true);
}
else
{
// Check if function is a generator
var suite = FunctionDefinition.Body as SuiteStatement;
var yieldExpr = suite?.Statements.OfType <ExpressionStatement>().Select(s => s.Expression as YieldExpression).ExcludeDefault().FirstOrDefault();
if (yieldExpr != null)
{
// Function return is an iterator
var yieldValue = await Eval.GetValueFromExpressionAsync(yieldExpr.Expression, cancellationToken) ?? Eval.UnknownType;
var returnValue = new PythonGenerator(Eval.Interpreter, yieldValue);
_overload.SetReturnValue(returnValue, true);
}
}
using (Eval.OpenScope(FunctionDefinition, out _)) {
await DeclareParametersAsync(cancellationToken);
if (annotationType.IsUnknown() || Module.ModuleType == ModuleType.User)
{
// Return type from the annotation is sufficient for libraries
// and stubs, no need to walk the body.
if (FunctionDefinition.Body != null && Module.ModuleType != ModuleType.Specialized)
{
await FunctionDefinition.Body.WalkAsync(this, cancellationToken);
}
}
}
}
private void ExtendAll(Node declNode, IPythonCollection values)
{
Eval.LookupNameInScopes(AllVariableName, out var scope, LookupOptions.Global);
if (scope == null)
{
return;
}
var loc = Eval.GetLoc(declNode);
var all = scope.Variables[AllVariableName]?.Value as IPythonCollection;
var list = PythonCollectionType.CreateConcatenatedList(Module.Interpreter, loc, all, values);
var source = list.IsGeneric() ? VariableSource.Generic : VariableSource.Declaration;
Eval.DeclareVariable(AllVariableName, list, source, loc);
}
public async Task HandleForAsync(ForStatement node, CancellationToken cancellationToken = default)
{
cancellationToken.ThrowIfCancellationRequested();
var iterable = await Eval.GetValueFromExpressionAsync(node.List, cancellationToken);
var iterator = (iterable as IPythonIterable)?.GetIterator();
if (iterator == null)
{
// TODO: report that expression does not evaluate to iterable.
}
var value = iterator?.Next ?? Eval.UnknownType;
switch (node.Left)
{
case NameExpression nex:
// for x in y:
Eval.DeclareVariable(nex.Name, value, Eval.GetLoc(node.Left));
break;
case TupleExpression tex:
// x = [('abc', 42, True), ('abc', 23, False)]
// for some_str, some_int, some_bool in x:
var names = tex.Items.OfType <NameExpression>().Select(x => x.Name).ToArray();
if (value is IPythonIterable valueIterable)
{
var valueIterator = valueIterable.GetIterator();
foreach (var n in names)
{
Eval.DeclareVariable(n, valueIterator?.Next ?? Eval.UnknownType, Eval.GetLoc(node.Left));
}
}
else
{
// TODO: report that expression yields value that does not evaluate to iterable.
}
break;
}
if (node.Body != null)
{
await node.Body.WalkAsync(Walker, cancellationToken);
}
}
private void HandleTypedVariable(IPythonType variableType, IMember value, Expression expr)
{
// Check value type for compatibility
IMember instance = null;
if (value != null)
{
var valueType = value.GetPythonType();
if (!variableType.IsUnknown() && !valueType.Equals(variableType))
{
// TODO: warn incompatible value type.
// TODO: verify values. Value may be list() while variable type is List[str].
// Leave it as variable type.
}
else
{
instance = value;
}
}
instance = instance ?? variableType?.CreateInstance(variableType.Name, Eval.GetLoc(expr), ArgumentSet.Empty) ?? Eval.UnknownType;
if (expr is NameExpression ne)
{
Eval.DeclareVariable(ne.Name, instance, VariableSource.Declaration, expr);
return;
}
if (expr is MemberExpression m)
{
// self.x : int = 42
var self = Eval.LookupNameInScopes("self", out var scope);
var argType = self?.GetPythonType();
if (argType is PythonClassType cls && scope != null)
{
cls.AddMember(m.Name, instance, true);
}
}
}
public override void Evaluate()
{
var stub = SymbolTable.ReplacedByStubs.Contains(Target) ||
_function.DeclaringModule.ModuleType == ModuleType.Stub ||
Module.ModuleType == ModuleType.Specialized;
using (Eval.OpenScope(_function.DeclaringModule, FunctionDefinition, out _)) {
// Process annotations.
var annotationType = Eval.GetTypeFromAnnotation(FunctionDefinition.ReturnAnnotation);
if (!annotationType.IsUnknown())
{
// Annotations are typically types while actually functions return
// instances unless specifically annotated to a type such as Type[T].
var instance = annotationType.CreateInstance(annotationType.Name, Eval.GetLoc(FunctionDefinition), ArgumentSet.Empty);
_overload.SetReturnValue(instance, true);
}
else
{
// Check if function is a generator
var suite = FunctionDefinition.Body as SuiteStatement;
var yieldExpr = suite?.Statements.OfType <ExpressionStatement>().Select(s => s.Expression as YieldExpression).ExcludeDefault().FirstOrDefault();
if (yieldExpr != null)
{
// Function return is an iterator
var yieldValue = Eval.GetValueFromExpression(yieldExpr.Expression) ?? Eval.UnknownType;
var returnValue = new PythonGenerator(Eval.Interpreter, yieldValue);
_overload.SetReturnValue(returnValue, true);
}
}
DeclareParameters(!stub);
// Do process body of constructors since they may be declaring
// variables that are later used to determine return type of other
// methods and properties.
var ctor = _function.Name.EqualsOrdinal("__init__") || _function.Name.EqualsOrdinal("__new__");
if (!stub && (ctor || annotationType.IsUnknown() || Module.ModuleType == ModuleType.User))
{
// Return type from the annotation is sufficient for libraries
// and stubs, no need to walk the body.
FunctionDefinition.Body?.Walk(this);
}
}
Result = _function;
}
public override bool Walk(FunctionDefinition node)
{
// Inner function, declare as variable.
var m = SymbolTable.Evaluate(node);
if (m != null)
{
Eval.DeclareVariable(node.NameExpression.Name, m, VariableSource.Declaration, Eval.GetLoc(node));
}
return(false);
}
public bool HandleFor(ForStatement node)
{
var iterable = Eval.GetValueFromExpression(node.List);
var iterator = (iterable as IPythonIterable)?.GetIterator();
var value = iterator?.Next ?? Eval.UnknownType;
switch (node.Left)
{
case NameExpression nex:
// for x in y:
if (!string.IsNullOrEmpty(nex.Name))
{
Eval.DeclareVariable(nex.Name, value, VariableSource.Declaration, Eval.GetLoc(nex));
}
break;
case TupleExpression tex:
// x = [('abc', 42, True), ('abc', 23, False)]
// for some_str, (some_int, some_bool) in x:
var h = new TupleExpressionHandler(Walker);
h.HandleTupleAssignment(tex, node.List, value);
break;
}
node.Body?.Walk(Walker);
return(false);
}
private void TryHandleClassVariable(AssignmentStatement node, IMember value)
{
var mex = node.Left.OfType <MemberExpression>().FirstOrDefault();
if (!string.IsNullOrEmpty(mex?.Name) && mex?.Target is NameExpression nex && nex.Name.EqualsOrdinal("self"))
{
var m = Eval.LookupNameInScopes(nex.Name, out var scope, LookupOptions.Local);
var cls = m.GetPythonType <IPythonClassType>();
if (cls != null)
{
using (Eval.OpenScope(Eval.Module, cls.ClassDefinition, out _)) {
Eval.DeclareVariable(mex.Name, value, VariableSource.Declaration, Eval.GetLoc(node), true);
}
}
}
}
public void HandleWith(WithStatement node)
{
foreach (var item in node.Items.Where(x => x.Variable != null))
{
var contextManager = Eval.GetValueFromExpression(item.ContextManager);
var cmType = contextManager?.GetPythonType();
IMember context = Eval.UnknownType;
var enter = cmType?.GetMember(node.IsAsync ? @"__aenter__" : @"__enter__")?.GetPythonType <IPythonFunctionType>();
if (enter != null)
{
var instance = contextManager as IPythonInstance;
var callExpr = item.ContextManager as CallExpression;
context = Eval.GetValueFromFunctionType(enter, instance, callExpr);
// If fetching context from __enter__ failed, annotation in the stub may be using
// type from typing that we haven't specialized yet or there may be an issue in
// the stub itself, such as type or incorrect type. Try using context manager then.
context = context ?? contextManager;
}
if (item.Variable is NameExpression nex && !string.IsNullOrEmpty(nex.Name))
{
Eval.DeclareVariable(nex.Name, context, VariableSource.Declaration, Eval.GetLoc(item));
}
}
}
public void HandleAssignment(AssignmentStatement node)
{
if (node.Right is ErrorExpression)
{
return;
}
var value = Eval.GetValueFromExpression(node.Right) ?? Eval.UnknownType;
// Check PEP hint first
var valueType = Eval.GetTypeFromPepHint(node.Right);
if (valueType != null)
{
HandleTypedVariable(valueType, value, node.Left.FirstOrDefault());
return;
}
if (value.IsUnknown())
{
Log?.Log(TraceEventType.Verbose, $"Undefined value: {node.Right.ToCodeString(Ast).Trim()}");
}
if (value?.GetPythonType().TypeId == BuiltinTypeId.Ellipsis)
{
value = Eval.UnknownType;
}
if (node.Left.FirstOrDefault() is TupleExpression lhs)
{
// Tuple = Tuple. Transfer values.
var texHandler = new TupleExpressionHandler(Walker);
texHandler.HandleTupleAssignment(lhs, node.Right, value);
return;
}
// Process annotations, if any.
foreach (var expr in node.Left.OfType <ExpressionWithAnnotation>())
{
// x: List[str] = [...]
HandleAnnotatedExpression(expr, value);
}
foreach (var ne in node.Left.OfType <NameExpression>())
{
if (Eval.CurrentScope.NonLocals[ne.Name] != null)
{
Eval.LookupNameInScopes(ne.Name, out var scope, LookupOptions.Nonlocal);
if (scope != null)
{
scope.Variables[ne.Name].Assign(value, Eval.GetLoc(ne));
}
else
{
// TODO: report variable is not declared in outer scopes.
}
continue;
}
if (Eval.CurrentScope.Globals[ne.Name] != null)
{
Eval.LookupNameInScopes(ne.Name, out var scope, LookupOptions.Global);
if (scope != null)
{
scope.Variables[ne.Name].Assign(value, Eval.GetLoc(ne));
}
else
{
// TODO: report variable is not declared in global scope.
}
continue;
}
var source = value.IsGeneric() ? VariableSource.Generic : VariableSource.Declaration;
Eval.DeclareVariable(ne.Name, value ?? Module.Interpreter.UnknownType, source, Eval.GetLoc(ne));
}
TryHandleClassVariable(node, value);
}
public async Task HandleAssignmentAsync(AssignmentStatement node, CancellationToken cancellationToken = default)
{
cancellationToken.ThrowIfCancellationRequested();
var value = await Eval.GetValueFromExpressionAsync(node.Right, cancellationToken);
if (value.IsUnknown())
{
Log?.Log(TraceEventType.Verbose, $"Undefined value: {node.Right.ToCodeString(Ast).Trim()}");
}
if (value?.GetPythonType().TypeId == BuiltinTypeId.Ellipsis)
{
value = Eval.UnknownType;
}
if (node.Left.FirstOrDefault() is TupleExpression lhs)
{
// Tuple = Tuple. Transfer values.
var texHandler = new TupleExpressionHandler(Walker);
await texHandler.HandleTupleAssignmentAsync(lhs, node.Right, value, cancellationToken);
return;
}
foreach (var expr in node.Left.OfType <ExpressionWithAnnotation>())
{
// x: List[str] = [...]
await HandleAnnotatedExpressionAsync(expr, value, cancellationToken);
}
foreach (var ne in node.Left.OfType <NameExpression>())
{
if (Eval.CurrentScope.NonLocals[ne.Name] != null)
{
Eval.LookupNameInScopes(ne.Name, out var scope, LookupOptions.Nonlocal);
if (scope != null)
{
scope.Variables[ne.Name].Value = value;
}
else
{
// TODO: report variable is not declared in outer scopes.
}
continue;
}
if (Eval.CurrentScope.Globals[ne.Name] != null)
{
Eval.LookupNameInScopes(ne.Name, out var scope, LookupOptions.Global);
if (scope != null)
{
scope.Variables[ne.Name].Value = value;
}
else
{
// TODO: report variable is not declared in global scope.
}
continue;
}
Eval.DeclareVariable(ne.Name, value, Eval.GetLoc(ne));
}
}
private void MakeUnresolvedImport(string name, Node node) => Eval.DeclareVariable(name, new SentinelModule(name, Eval.Services), Eval.GetLoc(node));
