public async Task <IMember> GetValueFromPropertyAsync(IPythonPropertyType p, IPythonInstance instance, CancellationToken cancellationToken = default)
{
// Function may not have been walked yet. Do it now.
await SymbolTable.EvaluateAsync(p.FunctionDefinition, cancellationToken);
return(instance.Call(p.Name, ArgumentSet.Empty));
}
public override IMember Call(IPythonInstance instance, string memberName, IArgumentSet args)
{
// Now we can go and find overload with matching arguments.
var overload = FindOverload(args);
return(overload?.GetReturnValue(instance?.Location ?? LocationInfo.Empty, args) ?? DeclaringModule.Interpreter.UnknownType);
}
private ArgumentSet FindOverload(IPythonFunctionType fn, IPythonInstance instance, CallExpression expr)
{
if (fn.Overloads.Count == 1)
{
return(null);
}
var sets = new List <ArgumentSet>();
for (var i = 0; i < fn.Overloads.Count; i++)
{
var a = new ArgumentSet(fn, i, instance, expr, this);
var args = a.Evaluate();
sets.Add(args);
}
var orderedSets = sets.OrderBy(s => s.Errors.Count);
var noErrorsMatches = sets.OrderBy(s => s.Errors.Count).TakeWhile(e => e.Errors.Count == 0).ToArray();
var result = noErrorsMatches.Any()
? noErrorsMatches.FirstOrDefault(args => IsMatch(args, fn.Overloads[args.OverloadIndex].Parameters))
: null;
// Optimistically pick the best available.
return(result ?? orderedSets.FirstOrDefault());
}
public static VariableModel FromInstance(string name, IPythonInstance inst, IServiceContainer services) => new VariableModel
{
Id = name.GetStableHash(),
Name = name,
QualifiedName = name,
Value = inst.GetPersistentQualifiedName(services)
};
public override IMember Call(IPythonInstance instance, string memberName, IArgumentSet args)
{
// Now we can go and find overload with matching arguments.
var overload = Overloads[args.OverloadIndex];
return(overload?.Call(args, instance?.GetPythonType() ?? DeclaringType));
}
private IMember GetValueFromMember(MemberExpression expr)
{
if (expr?.Target == null || string.IsNullOrEmpty(expr.Name))
{
return(null);
}
IPythonInstance instance = null;
var m = GetValueFromExpression(expr.Target);
if (m is IPythonType typeInfo)
{
var member = typeInfo.GetMember(expr.Name);
// If container is class/type info rather than the instance, then the method is an unbound function.
// Example: C.f where f is a method of C. Compare to C().f where f is bound to the instance of C.
if (member is PythonFunctionType f && !f.IsStatic && !f.IsClassMethod)
{
return(f.ToUnbound());
}
instance = new PythonInstance(typeInfo);
}
instance = instance ?? m as IPythonInstance;
var type = m?.GetPythonType(); // Try inner type
var value = type?.GetMember(expr.Name);
if (type is IPythonModule)
{
return(value);
}
// Class type GetMember returns a type. However, class members are
// mostly instances (consider self.x = 1, x is an instance of int).
// However, it is indeed possible to have them as types, like in
// class X ...
// class C: ...
// self.x = X
// which is somewhat rare as compared to self.x = X() but does happen.
switch (value)
{
case IPythonClassType _:
return(value);
case IPythonPropertyType prop:
return(prop.Call(instance, prop.Name, ArgumentSet.Empty));
case IPythonType p:
return(new PythonBoundType(p, instance, GetLoc(expr)));
case null:
Log?.Log(TraceEventType.Verbose, $"Unknown member {expr.ToCodeString(Ast).Trim()}");
return(UnknownType);
default:
return(value);
}
}
public override IMember Index(IPythonInstance instance, IArgumentSet args)
{
var defaultReturn = base.Index(instance, args);
var fromBases = Bases
.MaybeEnumerate()
.Select(b => b.Index(instance, args))
.Except(new[] { defaultReturn, UnknownType })
.FirstOrDefault();
return(fromBases ?? defaultReturn);
}
public IMember Index(IPythonInstance instance, object index)
{
IPythonType[] types;
lock (_lock) {
types = _types.ToArray();
}
// Check if any types support indexing
var result = types
.Select(t => t.Index(instance, index))
.FirstOrDefault(r => !r.IsUnknown() && r.GetPythonType() != this);
return(result ?? DeclaringModule.Interpreter.UnknownType);
}
public IMember Call(IPythonInstance instance, string memberName, IArgumentSet args)
{
IPythonType[] types;
lock (_lock) {
types = _types.ToArray();
}
// Check if any types support calls
var result = types
.Select(t => t.Call(instance, memberName, args))
.FirstOrDefault(r => !r.IsUnknown() && r.GetPythonType() != this);
return(result ?? DeclaringModule.Interpreter.UnknownType);
}
public override IMember Index(IPythonInstance instance, object index)
{
var n = PythonCollection.GetIndex(index);
if (n < 0)
{
n = ItemTypes.Count + n; // -1 means last, etc.
}
if (n >= 0 && n < ItemTypes.Count)
{
return(ItemTypes[n]);
}
return(UnknownType);
}
public override IMember Index(IPythonInstance instance, IArgumentSet args)
{
var n = PythonCollection.GetIndex(args);
if (n < 0)
{
n = ItemTypes.Count + n; // -1 means last, etc.
}
if (n >= 0 && n < ItemTypes.Count)
{
var t = ItemTypes[n];
return(t.CreateInstance(args));
}
return(UnknownType);
}
private async Task <IMember> GetValueFromMemberAsync(MemberExpression expr, CancellationToken cancellationToken = default)
{
if (expr?.Target == null || string.IsNullOrEmpty(expr.Name))
{
return(null);
}
IPythonInstance instance = null;
var m = await GetValueFromExpressionAsync(expr.Target, cancellationToken);
if (m is IPythonType typeInfo)
{
var member = typeInfo.GetMember(expr.Name);
// If container is class/type info rather than the instance, then the method is an unbound function.
// Example: C.f where f is a method of C. Compare to C().f where f is bound to the instance of C.
if (member is PythonFunctionType f && !f.IsStatic && !f.IsClassMethod)
{
return(f.ToUnbound());
}
instance = new PythonInstance(typeInfo);
}
instance = instance ?? m as IPythonInstance;
var type = m.GetPythonType(); // Try inner type
var value = type?.GetMember(expr.Name);
switch (value)
{
case IPythonClassType _:
return(value);
case IPythonPropertyType prop:
return(prop.Call(instance, prop.Name, ArgumentSet.Empty));
case IPythonType p:
return(new PythonBoundType(p, instance, GetLoc(expr)));
case null:
Log?.Log(TraceEventType.Verbose, $"Unknown member {expr.ToCodeString(Ast).Trim()}");
return(UnknownType);
default:
return(value);
}
}
public async Task <IMember> GetValueFromInstanceCall(IPythonInstance pi, CallExpression expr, CancellationToken cancellationToken = default)
{
// Call on an instance such as 'a = 1; a()'
// If instance is a function (such as an unbound method), then invoke it.
var type = pi.GetPythonType();
if (type is IPythonFunctionType pft)
{
return(await GetValueFromFunctionTypeAsync(pft, pi, expr, cancellationToken));
}
// Try using __call__
var call = type.GetMember("__call__").GetPythonType <IPythonFunctionType>();
if (call != null)
{
return(await GetValueFromFunctionTypeAsync(call, pi, expr, cancellationToken));
}
return(null);
}
public IMember GetValueFromInstanceCall(IPythonInstance pi, CallExpression expr)
{
// Call on an instance such as 'a = 1; a()'
// If instance is a function (such as an unbound method), then invoke it.
var type = pi.GetPythonType();
if (type is IPythonFunctionType pft)
{
return(GetValueFromFunctionType(pft, pi, expr));
}
// Try using __call__
var call = type.GetMember("__call__").GetPythonType <IPythonFunctionType>();
if (call != null)
{
return(GetValueFromFunctionType(call, pi, expr));
}
// Optimistically return the instance itself. This happens when the call is
// over 'function' that was actually replaced by a instance of a type.
return(pi);
}
public IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) => GetMember(memberName);
public IMember Index(IPythonInstance instance, IArgumentSet args) => Interpreter.UnknownType;
public override IMember Index(IPythonInstance instance, IArgumentSet args) => (instance as IPythonCollection)?.Index(args) ?? UnknownType;
public override IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) => DeclaringModule.Interpreter.GetBuiltinType(TypeId)?.Call(instance, memberName, args);
public virtual IPythonIterator GetIterator(IPythonInstance instance) => (instance as IPythonCollection)?.GetIterator();
public virtual IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) => DeclaringModule.Interpreter.UnknownType;
public virtual IMember Index(IPythonInstance instance, IArgumentSet args) => DeclaringModule.Interpreter.UnknownType;
public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance instance, CallExpression callExpr, ExpressionEval eval) :
this(fn, overloadIndex, instance, callExpr, eval.Module, eval)
{
}
/// <summary>
/// Creates set of arguments for a function call based on the call expression
/// and the function signature. The result contains expressions
/// for arguments, but not actual values. <see cref="Evaluate"/> on how to
/// get values for actual parameters.
/// </summary>
/// <param name="fn">Function type.</param>
/// <param name="overloadIndex">Function overload to call.</param>
/// <param name="instance">Type instance the function is bound to. For derived classes it is different from the declared type.</param>
/// <param name="callExpr">Call expression that invokes the function.</param>
/// <param name="module">Module that contains the call expression.</param>
/// <param name="eval">Evaluator that can calculate values of arguments from their respective expressions.</param>
public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance instance, CallExpression callExpr, IPythonModule module, IExpressionEvaluator eval)
{
Eval = eval;
OverloadIndex = overloadIndex;
DeclaringModule = fn.DeclaringModule;
var overload = fn.Overloads[overloadIndex];
var fd = overload.FunctionDefinition;
if (fd == null || fn.IsSpecialized)
{
// Typically specialized function, like TypeVar() that does not actually have AST definition.
// Make the arguments from the call expression. If argument does not have name,
// try using name from the function definition based on the argument position.
_arguments = new List <Argument>();
for (var i = 0; i < callExpr.Args.Count; i++)
{
var name = callExpr.Args[i].Name;
if (string.IsNullOrEmpty(name))
{
name = fd != null && i < fd.Parameters.Length ? fd.Parameters[i].Name : null;
}
name = name ?? $"arg{i}";
var parameter = fd != null && i < fd.Parameters.Length ? fd.Parameters[i] : null;
_arguments.Add(new Argument(name, ParameterKind.Normal, callExpr.Args[i].Expression, null, parameter));
}
return;
}
if (callExpr == null)
{
// Typically invoked by specialization code without call expression in the code.
// Caller usually does not care about arguments.
_evaluated = true;
return;
}
var callLocation = callExpr.GetLocation(module);
// https://www.python.org/dev/peps/pep-3102/#id5
// For each formal parameter, there is a slot which will be used to contain
// the value of the argument assigned to that parameter. Slots which have
// had values assigned to them are marked as 'filled'.Slots which have
// no value assigned to them yet are considered 'empty'.
var slots = fd.Parameters.Select(p => new Argument(p, p)).ToArray();
// Locate sequence argument, if any
var sa = slots.Where(s => s.Kind == ParameterKind.List).ToArray();
if (sa.Length > 1)
{
// Error should have been reported at the function definition location by the parser.
return;
}
var da = slots.Where(s => s.Kind == ParameterKind.Dictionary).ToArray();
if (da.Length > 1)
{
// Error should have been reported at the function definition location by the parser.
return;
}
_listArgument = sa.Length == 1 && sa[0].Name.Length > 0 ? new ListArg(sa[0].Name, sa[0].ValueExpression, sa[0].Location) : null;
_dictArgument = da.Length == 1 ? new DictArg(da[0].Name, da[0].ValueExpression, da[0].Location) : null;
// Class methods
var formalParamIndex = 0;
if (fn.DeclaringType != null && fn.HasClassFirstArgument() && slots.Length > 0)
{
slots[0].Value = instance != null?instance.GetPythonType() : fn.DeclaringType;
formalParamIndex++;
}
try {
// Positional arguments
var callParamIndex = 0;
for (; callParamIndex < callExpr.Args.Count; callParamIndex++, formalParamIndex++)
{
var arg = callExpr.Args[callParamIndex];
if (!string.IsNullOrEmpty(arg.Name) && !arg.Name.StartsWithOrdinal("**"))
{
// Keyword argument. Done with positionals.
break;
}
if (formalParamIndex >= fd.Parameters.Length)
{
// We ran out of formal parameters and yet haven't seen
// any sequence or dictionary ones. This looks like an error.
_errors.Add(new DiagnosticsEntry(Resources.Analysis_TooManyFunctionArguments, arg.GetLocation(module).Span,
ErrorCodes.TooManyFunctionArguments, Severity.Warning, DiagnosticSource.Analysis));
return;
}
var formalParam = fd.Parameters[formalParamIndex];
if (formalParam.IsList)
{
if (string.IsNullOrEmpty(formalParam.Name))
{
// If the next unfilled slot is a vararg slot, and it does not have a name, then it is an error.
_errors.Add(new DiagnosticsEntry(Resources.Analysis_TooManyPositionalArgumentBeforeStar, arg.GetLocation(module).Span,
ErrorCodes.TooManyPositionalArgumentsBeforeStar, Severity.Warning, DiagnosticSource.Analysis));
return;
}
// If the next unfilled slot is a vararg slot then all remaining
// non-keyword arguments are placed into the vararg slot.
if (_listArgument == null)
{
_errors.Add(new DiagnosticsEntry(Resources.Analysis_TooManyFunctionArguments, arg.GetLocation(module).Span,
ErrorCodes.TooManyFunctionArguments, Severity.Warning, DiagnosticSource.Analysis));
return;
}
for (; callParamIndex < callExpr.Args.Count; callParamIndex++)
{
arg = callExpr.Args[callParamIndex];
if (!string.IsNullOrEmpty(arg.Name))
{
// Keyword argument. Done here.
break;
}
_listArgument._Expressions.Add(arg.Expression);
}
break; // Sequence or dictionary parameter found. Done here.
}
if (formalParam.IsDictionary)
{
// Next slot is a dictionary slot, but we have positional arguments still.
_errors.Add(new DiagnosticsEntry(Resources.Analysis_TooManyPositionalArgumentBeforeStar, arg.GetLocation(module).Span,
ErrorCodes.TooManyPositionalArgumentsBeforeStar, Severity.Warning, DiagnosticSource.Analysis));
return;
}
// Regular parameter
slots[formalParamIndex].ValueExpression = arg.Expression;
}
// Keyword arguments
for (; callParamIndex < callExpr.Args.Count; callParamIndex++)
{
var arg = callExpr.Args[callParamIndex];
if (string.IsNullOrEmpty(arg.Name))
{
_errors.Add(new DiagnosticsEntry(Resources.Analysis_PositionalArgumentAfterKeyword, arg.GetLocation(module).Span,
ErrorCodes.PositionalArgumentAfterKeyword, Severity.Warning, DiagnosticSource.Analysis));
return;
}
var nvp = slots.FirstOrDefault(s => s.Name.EqualsOrdinal(arg.Name));
if (nvp == null)
{
// 'def f(a, b)' and then 'f(0, c=1)'. Per spec:
// if there is a 'keyword dictionary' argument, the argument is added
// to the dictionary using the keyword name as the dictionary key,
// unless there is already an entry with that key, in which case it is an error.
if (_dictArgument == null)
{
_errors.Add(new DiagnosticsEntry(Resources.Analysis_UnknownParameterName, arg.GetLocation(module).Span,
ErrorCodes.UnknownParameterName, Severity.Warning, DiagnosticSource.Analysis));
return;
}
if (_dictArgument.Arguments.ContainsKey(arg.Name))
{
_errors.Add(new DiagnosticsEntry(Resources.Analysis_ParameterAlreadySpecified.FormatUI(arg.Name), arg.GetLocation(module).Span,
ErrorCodes.ParameterAlreadySpecified, Severity.Warning, DiagnosticSource.Analysis));
return;
}
_dictArgument._Expressions[arg.Name] = arg.Expression;
continue;
}
if (nvp.ValueExpression != null || nvp.Value != null)
{
// Slot is already filled.
_errors.Add(new DiagnosticsEntry(Resources.Analysis_ParameterAlreadySpecified.FormatUI(arg.Name), arg.GetLocation(module).Span,
ErrorCodes.ParameterAlreadySpecified, Severity.Warning, DiagnosticSource.Analysis));
return;
}
// OK keyword parameter
nvp.ValueExpression = arg.Expression;
}
// We went through all positionals and keywords.
// For each remaining empty slot: if there is a default value for that slot,
// then fill the slot with the default value. If there is no default value,
// then it is an error.
foreach (var slot in slots.Where(s => s.Kind != ParameterKind.List && s.Kind != ParameterKind.Dictionary && s.Value == null))
{
if (slot.ValueExpression == null)
{
var parameter = fd.Parameters.First(p => p.Name == slot.Name);
if (parameter.DefaultValue == null)
{
// TODO: parameter is not assigned and has no default value.
_errors.Add(new DiagnosticsEntry(Resources.Analysis_ParameterMissing.FormatUI(slot.Name), callLocation.Span,
ErrorCodes.ParameterMissing, Severity.Warning, DiagnosticSource.Analysis));
}
// Note that parameter default value expression is from the function definition AST
// while actual argument values are from the calling file AST.
slot.ValueExpression = parameter.DefaultValue;
slot.ValueIsDefault = true;
}
}
} finally {
// Optimistically return what we gathered, even if there are errors.
_arguments = slots.Where(s => s.Kind != ParameterKind.List && s.Kind != ParameterKind.Dictionary).ToList();
}
}
// NamedTuple does not create instances, it defines a type. public override IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) => this;
public virtual IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) => InnerType.Call(instance, memberName, args);
public IMember GetValueFromFunctionType(IPythonFunctionType fn, IPythonInstance instance, CallExpression expr)
{
// If order to be able to find matching overload, we need to know
// parameter types and count. This requires function to be analyzed.
// Since we don't know which overload we will need, we have to
// process all known overloads for the function.
foreach (var o in fn.Overloads)
{
SymbolTable.Evaluate(o.FunctionDefinition);
}
// Pick the best overload.
FunctionDefinition fd;
ArgumentSet args;
var instanceType = instance?.GetPythonType();
if (fn.Overloads.Count == 1)
{
fd = fn.Overloads[0].FunctionDefinition;
args = new ArgumentSet(fn, 0, instanceType, expr, this);
args = args.Evaluate();
}
else
{
args = FindOverload(fn, instanceType, expr);
if (args == null)
{
return(UnknownType);
}
fd = fn.Overloads.Count > 0 ? fn.Overloads[args.OverloadIndex].FunctionDefinition : null;
}
// Re-declare parameters in the function scope since originally
// their types might not have been known and now argument set
// may contain concrete values.
if (fd != null && EvaluateFunctionBody(fn))
{
using (OpenScope(fn.DeclaringModule, fn.FunctionDefinition, out _)) {
args.DeclareParametersInScope(this);
}
}
// If instance type is not the same as the declaring type, then call most probably comes
// from the derived class which means that the original 'self' and 'cls' variables
// are no longer valid and function has to be re-evaluated with new arguments.
// Note that there is nothing to re-evaluate in stubs.
if (instanceType == null || fn.DeclaringType == null || fn.IsSpecialized ||
instanceType.IsSpecialized || fn.DeclaringType.IsSpecialized ||
instanceType.Equals(fn.DeclaringType) ||
fn.IsStub || !string.IsNullOrEmpty(fn.Overloads[args.OverloadIndex].GetReturnDocumentation()))
{
LoadFunctionDependencyModules(fn);
var m = instance?.Call(fn.Name, args) ?? fn.Call(null, fn.Name, args);
if (!m.IsUnknown())
{
return(m);
}
}
// We could not tell the return type from the call. Here we try and evaluate with specific arguments.
// Note that it does not make sense evaluating stubs or compiled/scraped modules since they
// should be either annotated or static return type known from the analysis.
//
// Also, we do not evaluate library code with arguments for performance reasons.
// This will prevent cases like
// def func(a, b): return a + b
// from working in libraries, but this is small sacrifice for significant performance
// increase in library analysis.
if (fn.DeclaringModule is IDocument && EvaluateFunctionBody(fn))
{
// Stubs are coming from another module.
return(TryEvaluateWithArguments(fn, args));
}
return(UnknownType);
}
public IMember Index(IPythonInstance instance, object index) => Interpreter.UnknownType;
public override IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) => _getter.Call(args, instance?.GetPythonType() ?? DeclaringType);
private IMember GetValueFromProperty(IPythonPropertyType p, IPythonInstance instance, CallExpression expr)
{
// Function may not have been walked yet. Do it now.
SymbolTable.Evaluate(p.FunctionDefinition);
return(instance.Call(p.Name, ArgumentSet.Empty(expr, this)));
}
public virtual IMember Index(IPythonInstance instance, IArgumentSet args) => InnerType.Index(instance, args);
