private void ImportMembersFromSelf(FromImportStatement node) { var names = node.Names; var asNames = node.AsNames; if (names.Count == 1 && names[0].Name == "*") { // from self import * won't define any new members return; } for (var i = 0; i < names.Count; i++) { if (asNames[i] == null) { continue; } var importName = names[i].Name; var memberReference = asNames[i]; var memberName = memberReference.Name; var member = Module.GetMember(importName); Eval.DeclareVariable(memberName, member ?? Eval.UnknownType, Eval.GetLoc(names[i])); } }
public bool HandleFromImport(FromImportStatement node) { if (Module.ModuleType == ModuleType.Specialized) { return(false); } var rootNames = node.Root.Names; if (rootNames.Count == 1) { var rootName = rootNames[0].Name; if (rootName.EqualsOrdinal("__future__")) { SpecializeFuture(node); return(false); } } var location = Eval.GetLoc(node.Root); var imports = ModuleResolution.CurrentPathResolver.FindImports(Module.FilePath, node); if (HandleImportSearchResult(imports, null, null, location, out var variableModule)) { AssignVariables(node, imports, variableModule); } return(false); }
public async Task HandleTupleAssignmentAsync(TupleExpression lhs, Expression rhs, IMember value, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); if (rhs is TupleExpression tex) { var returnedExpressions = tex.Items.ToArray(); var names = lhs.Items.OfType<NameExpression>().Select(x => x.Name).ToArray(); for (var i = 0; i < Math.Min(names.Length, returnedExpressions.Length); i++) { if (returnedExpressions[i] != null && !string.IsNullOrEmpty(names[i])) { var v = await Eval.GetValueFromExpressionAsync(returnedExpressions[i], cancellationToken); if (v != null) { Eval.DeclareVariable(names[i], v, returnedExpressions[i]); } } } return; } // Tuple = 'tuple value' (such as from callable). Transfer values. if (value is IPythonCollection seq) { var types = seq.Contents.Select(c => c.GetPythonType()).ToArray(); var expressions = lhs.Items.OfType<NameExpression>().ToArray(); var names = expressions.Select(x => x.Name).ToArray(); for (var i = 0; i < Math.Min(names.Length, types.Length); i++) { if (names[i] != null && types[i] != null) { var instance = types[i].CreateInstance(null, Eval.GetLoc(expressions[i]), ArgumentSet.Empty); Eval.DeclareVariable(names[i], instance, expressions[i]); } } } }
private void HandleImport(ModuleName moduleImportExpression, NameExpression asNameExpression, bool forceAbsolute) { var location = Eval.GetLoc(moduleImportExpression); // "import fob.oar.baz" means // import_module('fob') // import_module('fob.oar') // import_module('fob.oar.baz') var importNames = ImmutableArray <string> .Empty; var variableModule = default(PythonVariableModule); foreach (var nameExpression in moduleImportExpression.Names) { importNames = importNames.Add(nameExpression.Name); var imports = ModuleResolution.CurrentPathResolver.GetImportsFromAbsoluteName(Module.FilePath, importNames, forceAbsolute); if (!HandleImportSearchResult(imports, variableModule, asNameExpression, location, out variableModule)) { return; } } // "import fob.oar.baz as baz" is handled as baz = import_module('fob.oar.baz') // "import fob.oar.baz" is handled as fob = import_module('fob') if (asNameExpression != null) { Eval.DeclareVariable(asNameExpression.Name, variableModule, VariableSource.Import, asNameExpression); } else if (importNames.Count > 0 && _variableModules.TryGetValue(importNames[0], out variableModule)) { var firstName = moduleImportExpression.Names[0]; Eval.DeclareVariable(importNames[0], variableModule, VariableSource.Import, firstName); } }
private async Task ImportMembersFromPackageAsync(FromImportStatement node, PackageImport packageImport, CancellationToken cancellationToken = default) { var names = node.Names; var asNames = node.AsNames; if (names.Count == 1 && names[0].Name == "*") { // TODO: Need tracking of previous imports to determine possible imports for namespace package. For now import nothing Eval.DeclareVariable("*", Eval.UnknownType, Eval.GetLoc(names[0])); return; } for (var i = 0; i < names.Count; i++) { var importName = names[i].Name; var memberReference = asNames[i] ?? names[i]; var memberName = memberReference.Name; var location = Eval.GetLoc(memberReference); ModuleImport moduleImport; IPythonType member; if ((moduleImport = packageImport.Modules.FirstOrDefault(mi => mi.Name.EqualsOrdinal(importName))) != null) { member = await ModuleResolution.ImportModuleAsync(moduleImport.FullName, cancellationToken); } else { member = Eval.UnknownType; } Eval.DeclareVariable(memberName, member, location); } }
public Task <bool> HandleGlobalAsync(GlobalStatement node, CancellationToken cancellationToken = default) { foreach (var nex in node.Names) { Eval.CurrentScope.DeclareGlobal(nex.Name, Eval.GetLoc(nex)); } return(Task.FromResult(false)); }
public bool HandleGlobal(GlobalStatement node) { foreach (var nex in node.Names) { Eval.CurrentScope.DeclareGlobal(nex.Name, Eval.GetLoc(nex)); } return(false); }
public async Task <bool> HandleImportAsync(ImportStatement node, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); if (node.Names == null || Module.ModuleType == ModuleType.Specialized) { return(false); } var len = Math.Min(node.Names.Count, node.AsNames.Count); for (var i = 0; i < len; i++) { cancellationToken.ThrowIfCancellationRequested(); var moduleImportExpression = node.Names[i]; var importNames = moduleImportExpression.Names.Select(n => n.Name).ToArray(); var asNameExpression = node.AsNames[i]; var memberName = asNameExpression?.Name ?? moduleImportExpression.Names[0].Name; // If we are processing stub, ignore imports of the original module. // For example, typeshed stub for sys imports sys. var imports = ModuleResolution.CurrentPathResolver.GetImportsFromAbsoluteName(Module.FilePath, importNames, node.ForceAbsolute); if (Module.ModuleType == ModuleType.Stub && imports is ModuleImport mi && mi.Name == Module.Name) { continue; } var location = Eval.GetLoc(moduleImportExpression); IPythonModule module = null; switch (imports) { case ModuleImport moduleImport when moduleImport.FullName == Module.Name: Eval.DeclareVariable(memberName, Module, location); break; case ModuleImport moduleImport: module = await HandleImportAsync(node, moduleImport, cancellationToken); break; case PossibleModuleImport possibleModuleImport: module = await HandlePossibleImportAsync(node, possibleModuleImport, cancellationToken); break; default: // TODO: Package import? MakeUnresolvedImport(memberName, moduleImportExpression); break; } if (module != null) { AssignImportedVariables(module, moduleImportExpression, asNameExpression); } } return(false); }
public async Task HandleAnnotatedExpressionAsync(ExpressionWithAnnotation expr, IMember value, CancellationToken cancellationToken = default) { if (expr?.Annotation == null) { return; } var variableType = await Eval.GetTypeFromAnnotationAsync(expr.Annotation, cancellationToken); // If value is null, then this is a pure declaration like // x: List[str] // without a value. If value is provided, then this is // x: List[str] = [...] // Check value type for compatibility IMember instance = null; if (value != null) { var valueType = value.GetPythonType(); if (!variableType.IsUnknown() && !valueType.Equals(variableType)) { // TODO: warn incompatible value type. // TODO: verify values. Value may be list() while variable type is List[str]. // Leave it as variable type. } else { instance = value; } } instance = instance ?? variableType?.CreateInstance(variableType.Name, Eval.GetLoc(expr.Expression), ArgumentSet.Empty) ?? Eval.UnknownType; if (expr.Expression is NameExpression ne) { Eval.DeclareVariable(ne.Name, instance, expr.Expression); return; } if (expr.Expression is MemberExpression m) { // self.x : int = 42 var self = Eval.LookupNameInScopes("self", out var scope); var argType = self?.GetPythonType(); if (argType is PythonClassType cls && scope != null) { cls.AddMember(m.Name, instance, true); } } }
private void HandleAllAppendExtend(CallExpression node) { if (!(node.Target is MemberExpression me)) { return; } if (!IsHandleableAll(me.Target)) { return; } if (node.Args.Count == 0) { return; } var arg = node.Args[0].Expression; var v = Eval.GetValueFromExpression(arg); if (v == null) { _allIsUsable = false; return; } IPythonCollection values = null; switch (me.Name) { case "append": values = PythonCollectionType.CreateList(Module.Interpreter, Eval.GetLoc(arg), new List <IMember>() { v }, exact: true); break; case "extend": values = v as IPythonCollection; break; } if (values == null) { _allIsUsable = false; return; } ExtendAll(node, values); }
public async Task HandleWithAsync(WithStatement node, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); foreach (var item in node.Items.Where(x => x.Variable != null)) { var contextManager = await Eval.GetValueFromExpressionAsync(item.ContextManager, cancellationToken); var cmType = contextManager.GetPythonType(); var enter = cmType?.GetMember(node.IsAsync ? @"__aenter__" : @"__enter__")?.GetPythonType<IPythonFunctionType>(); if (enter != null) { var context = await Eval.GetValueFromFunctionTypeAsync(enter, null, null, cancellationToken); if (item.Variable is NameExpression nex && !string.IsNullOrEmpty(nex.Name)) { Eval.DeclareVariable(nex.Name, context, Eval.GetLoc(item)); } } } }
public override async Task EvaluateAsync(CancellationToken cancellationToken = default) { if (SymbolTable.ReplacedByStubs.Contains(Target)) { return; } cancellationToken.ThrowIfCancellationRequested(); // Process annotations. var annotationType = await Eval.GetTypeFromAnnotationAsync(FunctionDefinition.ReturnAnnotation, cancellationToken); if (!annotationType.IsUnknown()) { // Annotations are typically types while actually functions return // instances unless specifically annotated to a type such as Type[T]. var instance = annotationType.CreateInstance(annotationType.Name, Eval.GetLoc(FunctionDefinition), ArgumentSet.Empty); _overload.SetReturnValue(instance, true); } else { // Check if function is a generator var suite = FunctionDefinition.Body as SuiteStatement; var yieldExpr = suite?.Statements.OfType <ExpressionStatement>().Select(s => s.Expression as YieldExpression).ExcludeDefault().FirstOrDefault(); if (yieldExpr != null) { // Function return is an iterator var yieldValue = await Eval.GetValueFromExpressionAsync(yieldExpr.Expression, cancellationToken) ?? Eval.UnknownType; var returnValue = new PythonGenerator(Eval.Interpreter, yieldValue); _overload.SetReturnValue(returnValue, true); } } using (Eval.OpenScope(FunctionDefinition, out _)) { await DeclareParametersAsync(cancellationToken); if (annotationType.IsUnknown() || Module.ModuleType == ModuleType.User) { // Return type from the annotation is sufficient for libraries // and stubs, no need to walk the body. if (FunctionDefinition.Body != null && Module.ModuleType != ModuleType.Specialized) { await FunctionDefinition.Body.WalkAsync(this, cancellationToken); } } } }
private void ExtendAll(Node declNode, IPythonCollection values) { Eval.LookupNameInScopes(AllVariableName, out var scope, LookupOptions.Global); if (scope == null) { return; } var loc = Eval.GetLoc(declNode); var all = scope.Variables[AllVariableName]?.Value as IPythonCollection; var list = PythonCollectionType.CreateConcatenatedList(Module.Interpreter, loc, all, values); var source = list.IsGeneric() ? VariableSource.Generic : VariableSource.Declaration; Eval.DeclareVariable(AllVariableName, list, source, loc); }
public async Task HandleForAsync(ForStatement node, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); var iterable = await Eval.GetValueFromExpressionAsync(node.List, cancellationToken); var iterator = (iterable as IPythonIterable)?.GetIterator(); if (iterator == null) { // TODO: report that expression does not evaluate to iterable. } var value = iterator?.Next ?? Eval.UnknownType; switch (node.Left) { case NameExpression nex: // for x in y: Eval.DeclareVariable(nex.Name, value, Eval.GetLoc(node.Left)); break; case TupleExpression tex: // x = [('abc', 42, True), ('abc', 23, False)] // for some_str, some_int, some_bool in x: var names = tex.Items.OfType <NameExpression>().Select(x => x.Name).ToArray(); if (value is IPythonIterable valueIterable) { var valueIterator = valueIterable.GetIterator(); foreach (var n in names) { Eval.DeclareVariable(n, valueIterator?.Next ?? Eval.UnknownType, Eval.GetLoc(node.Left)); } } else { // TODO: report that expression yields value that does not evaluate to iterable. } break; } if (node.Body != null) { await node.Body.WalkAsync(Walker, cancellationToken); } }
private void HandleTypedVariable(IPythonType variableType, IMember value, Expression expr) { // Check value type for compatibility IMember instance = null; if (value != null) { var valueType = value.GetPythonType(); if (!variableType.IsUnknown() && !valueType.Equals(variableType)) { // TODO: warn incompatible value type. // TODO: verify values. Value may be list() while variable type is List[str]. // Leave it as variable type. } else { instance = value; } } instance = instance ?? variableType?.CreateInstance(variableType.Name, Eval.GetLoc(expr), ArgumentSet.Empty) ?? Eval.UnknownType; if (expr is NameExpression ne) { Eval.DeclareVariable(ne.Name, instance, VariableSource.Declaration, expr); return; } if (expr is MemberExpression m) { // self.x : int = 42 var self = Eval.LookupNameInScopes("self", out var scope); var argType = self?.GetPythonType(); if (argType is PythonClassType cls && scope != null) { cls.AddMember(m.Name, instance, true); } } }
public override void Evaluate() { var stub = SymbolTable.ReplacedByStubs.Contains(Target) || _function.DeclaringModule.ModuleType == ModuleType.Stub || Module.ModuleType == ModuleType.Specialized; using (Eval.OpenScope(_function.DeclaringModule, FunctionDefinition, out _)) { // Process annotations. var annotationType = Eval.GetTypeFromAnnotation(FunctionDefinition.ReturnAnnotation); if (!annotationType.IsUnknown()) { // Annotations are typically types while actually functions return // instances unless specifically annotated to a type such as Type[T]. var instance = annotationType.CreateInstance(annotationType.Name, Eval.GetLoc(FunctionDefinition), ArgumentSet.Empty); _overload.SetReturnValue(instance, true); } else { // Check if function is a generator var suite = FunctionDefinition.Body as SuiteStatement; var yieldExpr = suite?.Statements.OfType <ExpressionStatement>().Select(s => s.Expression as YieldExpression).ExcludeDefault().FirstOrDefault(); if (yieldExpr != null) { // Function return is an iterator var yieldValue = Eval.GetValueFromExpression(yieldExpr.Expression) ?? Eval.UnknownType; var returnValue = new PythonGenerator(Eval.Interpreter, yieldValue); _overload.SetReturnValue(returnValue, true); } } DeclareParameters(!stub); // Do process body of constructors since they may be declaring // variables that are later used to determine return type of other // methods and properties. var ctor = _function.Name.EqualsOrdinal("__init__") || _function.Name.EqualsOrdinal("__new__"); if (!stub && (ctor || annotationType.IsUnknown() || Module.ModuleType == ModuleType.User)) { // Return type from the annotation is sufficient for libraries // and stubs, no need to walk the body. FunctionDefinition.Body?.Walk(this); } } Result = _function; }
public override bool Walk(FunctionDefinition node) { // Inner function, declare as variable. var m = SymbolTable.Evaluate(node); if (m != null) { Eval.DeclareVariable(node.NameExpression.Name, m, VariableSource.Declaration, Eval.GetLoc(node)); } return(false); }
public bool HandleFor(ForStatement node) { var iterable = Eval.GetValueFromExpression(node.List); var iterator = (iterable as IPythonIterable)?.GetIterator(); var value = iterator?.Next ?? Eval.UnknownType; switch (node.Left) { case NameExpression nex: // for x in y: if (!string.IsNullOrEmpty(nex.Name)) { Eval.DeclareVariable(nex.Name, value, VariableSource.Declaration, Eval.GetLoc(nex)); } break; case TupleExpression tex: // x = [('abc', 42, True), ('abc', 23, False)] // for some_str, (some_int, some_bool) in x: var h = new TupleExpressionHandler(Walker); h.HandleTupleAssignment(tex, node.List, value); break; } node.Body?.Walk(Walker); return(false); }
private void TryHandleClassVariable(AssignmentStatement node, IMember value) { var mex = node.Left.OfType <MemberExpression>().FirstOrDefault(); if (!string.IsNullOrEmpty(mex?.Name) && mex?.Target is NameExpression nex && nex.Name.EqualsOrdinal("self")) { var m = Eval.LookupNameInScopes(nex.Name, out var scope, LookupOptions.Local); var cls = m.GetPythonType <IPythonClassType>(); if (cls != null) { using (Eval.OpenScope(Eval.Module, cls.ClassDefinition, out _)) { Eval.DeclareVariable(mex.Name, value, VariableSource.Declaration, Eval.GetLoc(node), true); } } } }
public void HandleWith(WithStatement node) { foreach (var item in node.Items.Where(x => x.Variable != null)) { var contextManager = Eval.GetValueFromExpression(item.ContextManager); var cmType = contextManager?.GetPythonType(); IMember context = Eval.UnknownType; var enter = cmType?.GetMember(node.IsAsync ? @"__aenter__" : @"__enter__")?.GetPythonType <IPythonFunctionType>(); if (enter != null) { var instance = contextManager as IPythonInstance; var callExpr = item.ContextManager as CallExpression; context = Eval.GetValueFromFunctionType(enter, instance, callExpr); // If fetching context from __enter__ failed, annotation in the stub may be using // type from typing that we haven't specialized yet or there may be an issue in // the stub itself, such as type or incorrect type. Try using context manager then. context = context ?? contextManager; } if (item.Variable is NameExpression nex && !string.IsNullOrEmpty(nex.Name)) { Eval.DeclareVariable(nex.Name, context, VariableSource.Declaration, Eval.GetLoc(item)); } } }
public void HandleAssignment(AssignmentStatement node) { if (node.Right is ErrorExpression) { return; } var value = Eval.GetValueFromExpression(node.Right) ?? Eval.UnknownType; // Check PEP hint first var valueType = Eval.GetTypeFromPepHint(node.Right); if (valueType != null) { HandleTypedVariable(valueType, value, node.Left.FirstOrDefault()); return; } if (value.IsUnknown()) { Log?.Log(TraceEventType.Verbose, $"Undefined value: {node.Right.ToCodeString(Ast).Trim()}"); } if (value?.GetPythonType().TypeId == BuiltinTypeId.Ellipsis) { value = Eval.UnknownType; } if (node.Left.FirstOrDefault() is TupleExpression lhs) { // Tuple = Tuple. Transfer values. var texHandler = new TupleExpressionHandler(Walker); texHandler.HandleTupleAssignment(lhs, node.Right, value); return; } // Process annotations, if any. foreach (var expr in node.Left.OfType <ExpressionWithAnnotation>()) { // x: List[str] = [...] HandleAnnotatedExpression(expr, value); } foreach (var ne in node.Left.OfType <NameExpression>()) { if (Eval.CurrentScope.NonLocals[ne.Name] != null) { Eval.LookupNameInScopes(ne.Name, out var scope, LookupOptions.Nonlocal); if (scope != null) { scope.Variables[ne.Name].Assign(value, Eval.GetLoc(ne)); } else { // TODO: report variable is not declared in outer scopes. } continue; } if (Eval.CurrentScope.Globals[ne.Name] != null) { Eval.LookupNameInScopes(ne.Name, out var scope, LookupOptions.Global); if (scope != null) { scope.Variables[ne.Name].Assign(value, Eval.GetLoc(ne)); } else { // TODO: report variable is not declared in global scope. } continue; } var source = value.IsGeneric() ? VariableSource.Generic : VariableSource.Declaration; Eval.DeclareVariable(ne.Name, value ?? Module.Interpreter.UnknownType, source, Eval.GetLoc(ne)); } TryHandleClassVariable(node, value); }
public async Task HandleAssignmentAsync(AssignmentStatement node, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); var value = await Eval.GetValueFromExpressionAsync(node.Right, cancellationToken); if (value.IsUnknown()) { Log?.Log(TraceEventType.Verbose, $"Undefined value: {node.Right.ToCodeString(Ast).Trim()}"); } if (value?.GetPythonType().TypeId == BuiltinTypeId.Ellipsis) { value = Eval.UnknownType; } if (node.Left.FirstOrDefault() is TupleExpression lhs) { // Tuple = Tuple. Transfer values. var texHandler = new TupleExpressionHandler(Walker); await texHandler.HandleTupleAssignmentAsync(lhs, node.Right, value, cancellationToken); return; } foreach (var expr in node.Left.OfType <ExpressionWithAnnotation>()) { // x: List[str] = [...] await HandleAnnotatedExpressionAsync(expr, value, cancellationToken); } foreach (var ne in node.Left.OfType <NameExpression>()) { if (Eval.CurrentScope.NonLocals[ne.Name] != null) { Eval.LookupNameInScopes(ne.Name, out var scope, LookupOptions.Nonlocal); if (scope != null) { scope.Variables[ne.Name].Value = value; } else { // TODO: report variable is not declared in outer scopes. } continue; } if (Eval.CurrentScope.Globals[ne.Name] != null) { Eval.LookupNameInScopes(ne.Name, out var scope, LookupOptions.Global); if (scope != null) { scope.Variables[ne.Name].Value = value; } else { // TODO: report variable is not declared in global scope. } continue; } Eval.DeclareVariable(ne.Name, value, Eval.GetLoc(ne)); } }
private void MakeUnresolvedImport(string name, Node node) => Eval.DeclareVariable(name, new SentinelModule(name, Eval.Services), Eval.GetLoc(node));