private IMember GetOrCreate(BuiltinTypeId typeId)
{
if (typeId.IsVirtualId())
{
switch (typeId)
{
case BuiltinTypeId.Str:
typeId = LanguageVersion.Is3x() ? BuiltinTypeId.Unicode : BuiltinTypeId.Bytes;
break;
case BuiltinTypeId.StrIterator:
typeId = LanguageVersion.Is3x() ? BuiltinTypeId.UnicodeIterator : BuiltinTypeId.BytesIterator;
break;
default:
typeId = BuiltinTypeId.Unknown;
break;
}
}
lock (_cachedInstances) {
if (!_cachedInstances.TryGetValue(typeId, out var value))
{
_cachedInstances[typeId] = value = new FallbackBuiltinPythonType(this, typeId);
}
return(value);
}
}
public PythonAnalyzer CreateAnalyzer(PythonLanguageVersion version = PythonLanguageVersion.V27, string[] analysisDirs = null)
{
// Explicitly provide the builtins name, since we aren't recreating
// the interpreter for each version like we should be.
var fact = InterpreterFactory;
var interp = Interpreter;
if (version != fact.GetLanguageVersion())
{
fact = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(version.ToVersion());
interp = fact.CreateInterpreter();
}
var state = new PythonAnalyzer(fact, interp, "__builtin__");
if (version.Is3x() || this is IronPythonAnalysisTest)
{
var types = (KnownTypes)state.Types;
types._types[(int)BuiltinTypeId.Str] = state.Types[BuiltinTypeId.Unicode];
types._types[(int)BuiltinTypeId.StrIterator] = state.Types[BuiltinTypeId.UnicodeIterator];
types._classInfos[(int)BuiltinTypeId.Str] = state.ClassInfos[BuiltinTypeId.Unicode];
types._classInfos[(int)BuiltinTypeId.StrIterator] = state.ClassInfos[BuiltinTypeId.UnicodeIterator];
}
state.Limits = GetLimits();
if (analysisDirs != null)
{
foreach (var dir in analysisDirs)
{
state.AddAnalysisDirectory(dir);
}
}
return(state);
}
internal PythonAnalyzer(IPythonInterpreterFactory factory, IPythonInterpreter pythonInterpreter)
{
_interpreterFactory = factory;
_langVersion = factory.GetLanguageVersion();
_disposeInterpreter = pythonInterpreter == null;
_interpreter = pythonInterpreter ?? factory.CreateInterpreter();
_builtinName = _langVersion.Is3x() ? SharedDatabaseState.BuiltinName3x : SharedDatabaseState.BuiltinName2x;
_modules = new ModuleTable(this, _interpreter);
_modulesByFilename = new ConcurrentDictionary <string, ModuleInfo>(StringComparer.OrdinalIgnoreCase);
_modulesWithUnresolvedImports = new HashSet <ModuleInfo>();
_itemCache = new Dictionary <object, AnalysisValue>();
Limits = AnalysisLimits.GetDefaultLimits();
_queue = new Deque <AnalysisUnit>();
_defaultContext = _interpreter.CreateModuleContext();
_evalUnit = new AnalysisUnit(null, null, new ModuleInfo("$global", new ProjectEntry(this, "$global", String.Empty, null), _defaultContext).Scope, true);
AnalysisLog.NewUnit(_evalUnit);
try {
LoadInitialKnownTypes();
} catch (Exception ex) {
if (ex.IsCriticalException())
{
throw;
}
// This will be rethrown in LoadKnownTypesAsync
_loadKnownTypesException = ExceptionDispatchInfo.Capture(ex);
}
}
internal PythonAnalyzer(IPythonInterpreterFactory factory, IPythonInterpreter pythonInterpreter, string builtinName)
{
_interpreterFactory = factory;
_langVersion = factory.GetLanguageVersion();
_disposeInterpreter = pythonInterpreter == null;
_interpreter = pythonInterpreter ?? factory.CreateInterpreter();
_builtinName = builtinName ?? (_langVersion.Is3x() ? SharedDatabaseState.BuiltinName3x : SharedDatabaseState.BuiltinName2x);
_modules = new ModuleTable(this, _interpreter);
_modulesByFilename = new ConcurrentDictionary <string, ModuleInfo>(StringComparer.OrdinalIgnoreCase);
_modulesWithUnresolvedImports = new HashSet <ModuleInfo>();
_itemCache = new Dictionary <object, AnalysisValue>();
try {
using (var key = Registry.CurrentUser.OpenSubKey(AnalysisLimitsKey)) {
Limits = AnalysisLimits.LoadFromStorage(key);
}
} catch (SecurityException) {
Limits = new AnalysisLimits();
} catch (UnauthorizedAccessException) {
Limits = new AnalysisLimits();
} catch (IOException) {
Limits = new AnalysisLimits();
}
_queue = new Deque <AnalysisUnit>();
_defaultContext = _interpreter.CreateModuleContext();
_evalUnit = new AnalysisUnit(null, null, new ModuleInfo("$global", new ProjectEntry(this, "$global", String.Empty, null), _defaultContext).Scope, true);
AnalysisLog.NewUnit(_evalUnit);
LoadInitialKnownTypes();
}
public async Task InRaise(Server server, PythonLanguageVersion version)
{
var uri = await server.OpenDefaultDocumentAndGetUriAsync("raise ");
(await server.SendCompletion(uri, 0, 6)).Should().HaveInsertTexts("Exception", "ValueError").And.NotContainInsertTexts("def", "abs");
if (version.Is3x())
{
await server.ChangeDefaultDocumentAndGetAnalysisAsync("raise Exception from ");
(await server.SendCompletion(uri, 0, 6)).Should().HaveInsertTexts("Exception", "ValueError").And.NotContainInsertTexts("def", "abs");
(await server.SendCompletion(uri, 0, 16)).Should().HaveInsertTexts("from").And.NotContainInsertTexts("Exception", "def", "abs");
(await server.SendCompletion(uri, 0, 21)).Should().HaveAnyCompletions();
await server.ChangeDefaultDocumentAndGetAnalysisAsync("raise Exception fr");
(await server.SendCompletion(uri, 0, 18)).Should().HaveInsertTexts("from")
.And.NotContainInsertTexts("Exception", "def", "abs")
.And.Subject._applicableSpan.Should().Be(0, 16, 0, 18);
}
await server.ChangeDefaultDocumentAndGetAnalysisAsync("raise Exception, x, y");
(await server.SendCompletion(uri, 0, 16)).Should().HaveAnyCompletions();
(await server.SendCompletion(uri, 0, 19)).Should().HaveAnyCompletions();
}
/// <summary>
/// Returns a sequence of all keywords usable in an expression in a
/// particular version of Python.
/// </summary>
public static IEnumerable<string> Expression(PythonLanguageVersion version = PythonLanguageVersion.None) {
yield return "and";
yield return "as";
if (version.IsNone() || version >= PythonLanguageVersion.V35) {
yield return "await";
}
yield return "else";
if (version.IsNone() || version.Is3x()) {
yield return "False";
}
yield return "for";
if (version.IsNone() || version >= PythonLanguageVersion.V33) {
yield return "from";
}
yield return "if";
yield return "in";
yield return "is";
yield return "lambda";
yield return "None";
yield return "not";
yield return "or";
if (version.IsNone() || version.Is3x()) {
yield return "True";
}
yield return "yield";
}
/// <summary>
/// Retuns a sequence of all keywords usable as a statement in a
/// particular version of Python.
/// </summary>
public static IEnumerable <string> Statement(PythonLanguageVersion version = PythonLanguageVersion.None)
{
yield return("assert");
yield return("break");
yield return("continue");
yield return("class");
yield return("def");
yield return("del");
if (version.Is2x())
{
yield return("exec");
}
yield return("if");
yield return("elif");
yield return("except");
yield return("finally");
yield return("for");
yield return("from");
yield return("global");
yield return("import");
if (version.Is3x())
{
yield return("nonlocal");
}
yield return("pass");
if (version.Is2x())
{
yield return("print");
}
yield return("raise");
yield return("return");
yield return("try");
yield return("while");
yield return("with");
yield return("yield");
}
/// <summary>
/// Returns a sequence of all keywords usable in an expression in a
/// particular version of Python.
/// </summary>
public static IEnumerable <string> Expression(PythonLanguageVersion version = PythonLanguageVersion.None)
{
yield return("and");
yield return("as");
if (version >= PythonLanguageVersion.V35)
{
yield return("await");
}
yield return("else");
if (version.Is3x())
{
yield return("False");
}
yield return("for");
if (version >= PythonLanguageVersion.V33)
{
yield return("from");
}
yield return("if");
yield return("in");
yield return("is");
yield return("lambda");
yield return("None");
yield return("not");
yield return("or");
if (version.Is3x())
{
yield return("True");
}
yield return("yield");
}
public async Task InForStatement(Server server, PythonLanguageVersion version)
{
var uri = await server.OpenDefaultDocumentAndGetUriAsync("for ");
(await server.SendCompletion(uri, 0, 3)).Should().HaveLabels("for");
(await server.SendCompletion(uri, 0, 4)).Should().HaveNoCompletion();
await server.ChangeDefaultDocumentAndGetAnalysisAsync("for x ");
(await server.SendCompletion(uri, 0, 3)).Should().HaveLabels("for");
(await server.SendCompletion(uri, 0, 4)).Should().HaveNoCompletion();
(await server.SendCompletion(uri, 0, 5)).Should().HaveNoCompletion();
(await server.SendCompletion(uri, 0, 6)).Should().HaveNoCompletion();
(await server.SendCompletion(uri, 0, 7)).Should().HaveLabels("in").And.NotContainLabels("for", "abs");
// TODO: Fix parser to parse "for x i" as ForStatement and not ForStatement+ExpressionStatement
//u = await s.OpenDefaultDocumentAndGetUriAsync("for x i");
//await AssertCompletion(s, u, new[] { "in" }, new[] { "for", "abs" }, new SourceLocation(1, 8), applicableSpan: new SourceSpan(1, 7, 1, 8));
//await s.UnloadFileAsync(u);
await server.ChangeDefaultDocumentAndGetAnalysisAsync("for x in ");
(await server.SendCompletion(uri, 0, 6)).Should().HaveLabels("in").And.NotContainLabels("for", "abs");
(await server.SendCompletion(uri, 0, 8)).Should().HaveLabels("in").And.NotContainLabels("for", "abs");
(await server.SendCompletion(uri, 0, 9)).Should().HaveLabels("abs", "x");
await server.ChangeDefaultDocumentAndGetAnalysisAsync(@"def f():
for ");
(await server.SendCompletion(uri, 1, 8)).Should().HaveNoCompletion();
await server.ChangeDefaultDocumentAndGetAnalysisAsync(@"def f():
for x in ");
(await server.SendCompletion(uri, 1, 10)).Should().HaveLabels("in").And.NotContainLabels("for", "abs");
(await server.SendCompletion(uri, 1, 12)).Should().HaveLabels("in").And.NotContainLabels("for", "abs");
(await server.SendCompletion(uri, 1, 13)).Should().HaveLabels("abs", "x");
if (version.Is3x())
{
await server.ChangeDefaultDocumentAndGetAnalysisAsync(@"async def f():
async for x in ");
(await server.SendCompletion(uri, 1, 4)).Should().HaveLabels("async", "for");
(await server.SendCompletion(uri, 1, 9)).Should().HaveLabels("async", "for");
(await server.SendCompletion(uri, 1, 13)).Should().HaveLabels("async", "for");
(await server.SendCompletion(uri, 1, 14)).Should().HaveNoCompletion();
}
}
// ClassDefinition
public override bool Walk(ClassDefinition node)
{
if (node.Name != null)
{
node.Variable = DefineName(node.Name);
node.AddVariableReference(_globalScope, _bindRefs, Reference(node.Name));
}
if (node.Bases != null)
{
// Base references are in the outer context
foreach (var b in node.Bases)
{
b.Expression.Walk(this);
}
}
// process the decorators in the outer context
if (node.Decorators != null)
{
foreach (Expression dec in node.Decorators.Decorators)
{
if (dec != null)
{
dec.Walk(this);
}
}
}
PushScope(node);
node.ModuleNameVariable = _globalScope.EnsureGlobalVariable("__name__");
// define the __doc__ and the __module__
if (node.Body.Documentation != null)
{
node.DocVariable = DefineName("__doc__");
}
node.ModVariable = DefineName("__module__");
if (_langVersion.Is3x())
{
node.ClassVariable = DefineName("__class__");
}
// Walk the body
node.Body.Walk(this);
return(false);
}
private Parser(Tokenizer tokenizer, ErrorSink errorSink, PythonLanguageVersion langVersion, bool verbatim, bool bindRefs, string privatePrefix) {
Contract.Assert(tokenizer != null);
Contract.Assert(errorSink != null);
tokenizer.ErrorSink = new TokenizerErrorSink(this);
_tokenizer = tokenizer;
_errors = errorSink;
_langVersion = langVersion;
_verbatim = verbatim;
_bindReferences = bindRefs;
if (langVersion.Is3x()) {
// 3.x always does true division and absolute import
_languageFeatures |= FutureOptions.TrueDivision | FutureOptions.AbsoluteImports;
}
Reset(FutureOptions.None);
_privatePrefix = privatePrefix;
}
public FallbackBuiltinPythonType(PythonLanguageVersion version, BuiltinTypeId typeId)
{
DeclaringModule = version.Is3x() ? FallbackBuiltinModule.Instance3x : FallbackBuiltinModule.Instance2x;
Name = SharedDatabaseState.GetBuiltinTypeName(typeId, version.ToVersion());
TypeId = typeId;
}
protected virtual bool ShouldUseUnicodeLiterals(PythonLanguageVersion version)
{
return(version.Is3x());
}
/// <summary>
/// Retuns a sequence of all keywords usable as a statement in a
/// particular version of Python.
/// </summary>
public static IEnumerable<string> Statement(PythonLanguageVersion version = PythonLanguageVersion.None) {
yield return "assert";
if (version.IsNone() || version >= PythonLanguageVersion.V35) {
yield return "async";
yield return "await";
}
yield return "break";
yield return "continue";
yield return "class";
yield return "def";
yield return "del";
if (version.IsNone() || version.Is2x()) {
yield return "exec";
}
yield return "if";
yield return "elif";
yield return "except";
yield return "finally";
yield return "for";
yield return "from";
yield return "global";
yield return "import";
if (version.IsNone() || version.Is3x()) {
yield return "nonlocal";
}
yield return "pass";
if (version.IsNone() || version.Is2x()) {
yield return "print";
}
yield return "raise";
yield return "return";
yield return "try";
yield return "while";
yield return "with";
yield return "yield";
}
protected virtual bool ShouldUseUnicodeLiterals(PythonLanguageVersion version) {
return version.Is3x();
}
public FallbackBuiltinPythonType(PythonLanguageVersion version, BuiltinTypeId typeId) {
DeclaringModule = version.Is3x() ? FallbackBuiltinModule.Instance3x : FallbackBuiltinModule.Instance2x;
Name = SharedDatabaseState.GetBuiltinTypeName(typeId, version.ToVersion());
TypeId = typeId;
}
public string GetConstantRepr(PythonLanguageVersion version, bool escape8bitStrings = false)
{
if (_value == null)
{
return("None");
}
else if (_value is AsciiString)
{
StringBuilder res = new StringBuilder();
if (!version.Is2x())
{
res.Append("b");
}
AppendEscapedString(res, ((AsciiString)_value).String, escape8bitStrings);
return(res.ToString());
}
else if (_value is string)
{
StringBuilder res = new StringBuilder();
if (!version.Is3x())
{
res.Append("u");
}
AppendEscapedString(res, (string)_value, escape8bitStrings);
return(res.ToString());
}
else if (_value is Complex)
{
Complex n = (Complex)_value;
string real = NegativeZeroAwareToString(n.Real);
string imag = NegativeZeroAwareToString(n.Imaginary);
if (n.Real != 0)
{
if (!imag.StartsWith("-"))
{
imag = "+" + imag;
}
return("(" + real + imag + "j)");
}
else
{
return(imag + "j");
}
}
else if (_value is BigInteger)
{
if (!version.Is3x())
{
return(_value.ToString() + "L");
}
}
else if (_value is double)
{
double n = (double)_value;
string s = NegativeZeroAwareToString(n);
// If there's no fractional part, and this is not NaN or +-Inf, G format will not include the decimal
// point. This is okay if we're using scientific notation as this implies float, but if not, add the
// decimal point to indicate the type, just like Python repr() does.
if ((n - Math.Truncate(n)) == 0 && !s.Contains("e"))
{
s += ".0";
}
return(s);
}
else if (_value is IFormattable)
{
return(((IFormattable)_value).ToString(null, CultureInfo.InvariantCulture));
}
// TODO: We probably need to handle more primitives here
return(_value.ToString());
}
public string GetConstantRepr(PythonLanguageVersion version, bool escape8bitStrings = false) {
if (_value == null) {
return "None";
} else if (_value is AsciiString) {
StringBuilder res = new StringBuilder();
if (!version.Is2x()) {
res.Append("b");
}
AppendEscapedString(res, ((AsciiString)_value).String, escape8bitStrings);
return res.ToString();
} else if (_value is string) {
StringBuilder res = new StringBuilder();
if (!version.Is3x()) {
res.Append("u");
}
AppendEscapedString(res, (string)_value, escape8bitStrings);
return res.ToString();
} else if (_value is Complex) {
Complex n = (Complex)_value;
string real = NegativeZeroAwareToString(n.Real);
string imag = NegativeZeroAwareToString(n.Imaginary);
if (n.Real != 0) {
if (!imag.StartsWith("-")) {
imag = "+" + imag;
}
return "(" + real + imag + "j)";
} else {
return imag + "j";
}
} else if (_value is BigInteger) {
if (!version.Is3x()) {
return _value.ToString() + "L";
}
} else if (_value is double) {
double n = (double)_value;
string s = NegativeZeroAwareToString(n);
// If there's no fractional part, and this is not NaN or +-Inf, G format will not include the decimal
// point. This is okay if we're using scientific notation as this implies float, but if not, add the
// decimal point to indicate the type, just like Python repr() does.
if ((n - Math.Truncate(n)) == 0 && !s.Contains("e")) {
s += ".0";
}
return s;
} else if (_value is IFormattable) {
return ((IFormattable)_value).ToString(null, CultureInfo.InvariantCulture);
}
// TODO: We probably need to handle more primitives here
return _value.ToString();
}
public static string GetTypeName(this BuiltinTypeId id, PythonLanguageVersion languageVersion) => id.GetTypeName(languageVersion.IsNone() || languageVersion.Is3x());
public static string GetModuleName(this BuiltinTypeId id, PythonLanguageVersion languageVersion)
{
return(id.GetModuleName(languageVersion.IsNone() || languageVersion.Is3x()));
}
private void AnalyzeStdLib(string outdir)
{
var allFiles = new List <List <string> >();
foreach (var dir in _dirs)
{
CollectFiles(dir, allFiles);
}
foreach (var files in allFiles)
{
if (files.Count > 0)
{
Console.WriteLine("Now analyzing: {0}", Path.GetDirectoryName(files[0]));
var projectState = new PythonAnalyzer(new CPythonInterpreter(new TypeDatabase(_indir, _version.Is3x())), _version);
var modules = new List <IPythonProjectEntry>();
for (int i = 0; i < files.Count; i++)
{
string modName = PythonAnalyzer.PathToModuleName(files[i]);
modules.Add(projectState.AddModule(modName, files[i]));
}
var nodes = new List <PythonAst>();
for (int i = 0; i < modules.Count; i++)
{
PythonAst ast = null;
try {
var sourceUnit = new StreamReader(files[i]);
ast = Parser.CreateParser(sourceUnit, Microsoft.PythonTools.Parsing.ErrorSink.Null, PythonLanguageVersion.V27).ParseFile();
} catch (Exception) {
}
nodes.Add(ast);
}
for (int i = 0; i < modules.Count; i++)
{
var ast = nodes[i];
if (ast != null)
{
modules[i].UpdateTree(ast, null);
}
}
for (int i = 0; i < modules.Count; i++)
{
var ast = nodes[i];
if (ast != null)
{
modules[i].Analyze(true);
}
}
if (modules.Count > 0)
{
modules[0].AnalysisGroup.AnalyzeQueuedEntries();
}
new SaveAnalysis().Save(projectState, outdir);
}
}
}
public AstBuiltinsPythonModule(PythonLanguageVersion version)
: base(version.Is3x() ? SharedDatabaseState.BuiltinName3x : SharedDatabaseState.BuiltinName2x, null)
{
_hiddenNames = new HashSet <string>();
}