private IMember GetOrCreate(BuiltinTypeId typeId) { if (typeId.IsVirtualId()) { switch (typeId) { case BuiltinTypeId.Str: typeId = LanguageVersion.Is3x() ? BuiltinTypeId.Unicode : BuiltinTypeId.Bytes; break; case BuiltinTypeId.StrIterator: typeId = LanguageVersion.Is3x() ? BuiltinTypeId.UnicodeIterator : BuiltinTypeId.BytesIterator; break; default: typeId = BuiltinTypeId.Unknown; break; } } lock (_cachedInstances) { if (!_cachedInstances.TryGetValue(typeId, out var value)) { _cachedInstances[typeId] = value = new FallbackBuiltinPythonType(this, typeId); } return(value); } }
public PythonAnalyzer CreateAnalyzer(PythonLanguageVersion version = PythonLanguageVersion.V27, string[] analysisDirs = null) { // Explicitly provide the builtins name, since we aren't recreating // the interpreter for each version like we should be. var fact = InterpreterFactory; var interp = Interpreter; if (version != fact.GetLanguageVersion()) { fact = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(version.ToVersion()); interp = fact.CreateInterpreter(); } var state = new PythonAnalyzer(fact, interp, "__builtin__"); if (version.Is3x() || this is IronPythonAnalysisTest) { var types = (KnownTypes)state.Types; types._types[(int)BuiltinTypeId.Str] = state.Types[BuiltinTypeId.Unicode]; types._types[(int)BuiltinTypeId.StrIterator] = state.Types[BuiltinTypeId.UnicodeIterator]; types._classInfos[(int)BuiltinTypeId.Str] = state.ClassInfos[BuiltinTypeId.Unicode]; types._classInfos[(int)BuiltinTypeId.StrIterator] = state.ClassInfos[BuiltinTypeId.UnicodeIterator]; } state.Limits = GetLimits(); if (analysisDirs != null) { foreach (var dir in analysisDirs) { state.AddAnalysisDirectory(dir); } } return(state); }
internal PythonAnalyzer(IPythonInterpreterFactory factory, IPythonInterpreter pythonInterpreter) { _interpreterFactory = factory; _langVersion = factory.GetLanguageVersion(); _disposeInterpreter = pythonInterpreter == null; _interpreter = pythonInterpreter ?? factory.CreateInterpreter(); _builtinName = _langVersion.Is3x() ? SharedDatabaseState.BuiltinName3x : SharedDatabaseState.BuiltinName2x; _modules = new ModuleTable(this, _interpreter); _modulesByFilename = new ConcurrentDictionary <string, ModuleInfo>(StringComparer.OrdinalIgnoreCase); _modulesWithUnresolvedImports = new HashSet <ModuleInfo>(); _itemCache = new Dictionary <object, AnalysisValue>(); Limits = AnalysisLimits.GetDefaultLimits(); _queue = new Deque <AnalysisUnit>(); _defaultContext = _interpreter.CreateModuleContext(); _evalUnit = new AnalysisUnit(null, null, new ModuleInfo("$global", new ProjectEntry(this, "$global", String.Empty, null), _defaultContext).Scope, true); AnalysisLog.NewUnit(_evalUnit); try { LoadInitialKnownTypes(); } catch (Exception ex) { if (ex.IsCriticalException()) { throw; } // This will be rethrown in LoadKnownTypesAsync _loadKnownTypesException = ExceptionDispatchInfo.Capture(ex); } }
internal PythonAnalyzer(IPythonInterpreterFactory factory, IPythonInterpreter pythonInterpreter, string builtinName) { _interpreterFactory = factory; _langVersion = factory.GetLanguageVersion(); _disposeInterpreter = pythonInterpreter == null; _interpreter = pythonInterpreter ?? factory.CreateInterpreter(); _builtinName = builtinName ?? (_langVersion.Is3x() ? SharedDatabaseState.BuiltinName3x : SharedDatabaseState.BuiltinName2x); _modules = new ModuleTable(this, _interpreter); _modulesByFilename = new ConcurrentDictionary <string, ModuleInfo>(StringComparer.OrdinalIgnoreCase); _modulesWithUnresolvedImports = new HashSet <ModuleInfo>(); _itemCache = new Dictionary <object, AnalysisValue>(); try { using (var key = Registry.CurrentUser.OpenSubKey(AnalysisLimitsKey)) { Limits = AnalysisLimits.LoadFromStorage(key); } } catch (SecurityException) { Limits = new AnalysisLimits(); } catch (UnauthorizedAccessException) { Limits = new AnalysisLimits(); } catch (IOException) { Limits = new AnalysisLimits(); } _queue = new Deque <AnalysisUnit>(); _defaultContext = _interpreter.CreateModuleContext(); _evalUnit = new AnalysisUnit(null, null, new ModuleInfo("$global", new ProjectEntry(this, "$global", String.Empty, null), _defaultContext).Scope, true); AnalysisLog.NewUnit(_evalUnit); LoadInitialKnownTypes(); }
public async Task InRaise(Server server, PythonLanguageVersion version) { var uri = await server.OpenDefaultDocumentAndGetUriAsync("raise "); (await server.SendCompletion(uri, 0, 6)).Should().HaveInsertTexts("Exception", "ValueError").And.NotContainInsertTexts("def", "abs"); if (version.Is3x()) { await server.ChangeDefaultDocumentAndGetAnalysisAsync("raise Exception from "); (await server.SendCompletion(uri, 0, 6)).Should().HaveInsertTexts("Exception", "ValueError").And.NotContainInsertTexts("def", "abs"); (await server.SendCompletion(uri, 0, 16)).Should().HaveInsertTexts("from").And.NotContainInsertTexts("Exception", "def", "abs"); (await server.SendCompletion(uri, 0, 21)).Should().HaveAnyCompletions(); await server.ChangeDefaultDocumentAndGetAnalysisAsync("raise Exception fr"); (await server.SendCompletion(uri, 0, 18)).Should().HaveInsertTexts("from") .And.NotContainInsertTexts("Exception", "def", "abs") .And.Subject._applicableSpan.Should().Be(0, 16, 0, 18); } await server.ChangeDefaultDocumentAndGetAnalysisAsync("raise Exception, x, y"); (await server.SendCompletion(uri, 0, 16)).Should().HaveAnyCompletions(); (await server.SendCompletion(uri, 0, 19)).Should().HaveAnyCompletions(); }
/// <summary> /// Returns a sequence of all keywords usable in an expression in a /// particular version of Python. /// </summary> public static IEnumerable<string> Expression(PythonLanguageVersion version = PythonLanguageVersion.None) { yield return "and"; yield return "as"; if (version.IsNone() || version >= PythonLanguageVersion.V35) { yield return "await"; } yield return "else"; if (version.IsNone() || version.Is3x()) { yield return "False"; } yield return "for"; if (version.IsNone() || version >= PythonLanguageVersion.V33) { yield return "from"; } yield return "if"; yield return "in"; yield return "is"; yield return "lambda"; yield return "None"; yield return "not"; yield return "or"; if (version.IsNone() || version.Is3x()) { yield return "True"; } yield return "yield"; }
/// <summary> /// Retuns a sequence of all keywords usable as a statement in a /// particular version of Python. /// </summary> public static IEnumerable <string> Statement(PythonLanguageVersion version = PythonLanguageVersion.None) { yield return("assert"); yield return("break"); yield return("continue"); yield return("class"); yield return("def"); yield return("del"); if (version.Is2x()) { yield return("exec"); } yield return("if"); yield return("elif"); yield return("except"); yield return("finally"); yield return("for"); yield return("from"); yield return("global"); yield return("import"); if (version.Is3x()) { yield return("nonlocal"); } yield return("pass"); if (version.Is2x()) { yield return("print"); } yield return("raise"); yield return("return"); yield return("try"); yield return("while"); yield return("with"); yield return("yield"); }
/// <summary> /// Returns a sequence of all keywords usable in an expression in a /// particular version of Python. /// </summary> public static IEnumerable <string> Expression(PythonLanguageVersion version = PythonLanguageVersion.None) { yield return("and"); yield return("as"); if (version >= PythonLanguageVersion.V35) { yield return("await"); } yield return("else"); if (version.Is3x()) { yield return("False"); } yield return("for"); if (version >= PythonLanguageVersion.V33) { yield return("from"); } yield return("if"); yield return("in"); yield return("is"); yield return("lambda"); yield return("None"); yield return("not"); yield return("or"); if (version.Is3x()) { yield return("True"); } yield return("yield"); }
public async Task InForStatement(Server server, PythonLanguageVersion version) { var uri = await server.OpenDefaultDocumentAndGetUriAsync("for "); (await server.SendCompletion(uri, 0, 3)).Should().HaveLabels("for"); (await server.SendCompletion(uri, 0, 4)).Should().HaveNoCompletion(); await server.ChangeDefaultDocumentAndGetAnalysisAsync("for x "); (await server.SendCompletion(uri, 0, 3)).Should().HaveLabels("for"); (await server.SendCompletion(uri, 0, 4)).Should().HaveNoCompletion(); (await server.SendCompletion(uri, 0, 5)).Should().HaveNoCompletion(); (await server.SendCompletion(uri, 0, 6)).Should().HaveNoCompletion(); (await server.SendCompletion(uri, 0, 7)).Should().HaveLabels("in").And.NotContainLabels("for", "abs"); // TODO: Fix parser to parse "for x i" as ForStatement and not ForStatement+ExpressionStatement //u = await s.OpenDefaultDocumentAndGetUriAsync("for x i"); //await AssertCompletion(s, u, new[] { "in" }, new[] { "for", "abs" }, new SourceLocation(1, 8), applicableSpan: new SourceSpan(1, 7, 1, 8)); //await s.UnloadFileAsync(u); await server.ChangeDefaultDocumentAndGetAnalysisAsync("for x in "); (await server.SendCompletion(uri, 0, 6)).Should().HaveLabels("in").And.NotContainLabels("for", "abs"); (await server.SendCompletion(uri, 0, 8)).Should().HaveLabels("in").And.NotContainLabels("for", "abs"); (await server.SendCompletion(uri, 0, 9)).Should().HaveLabels("abs", "x"); await server.ChangeDefaultDocumentAndGetAnalysisAsync(@"def f(): for "); (await server.SendCompletion(uri, 1, 8)).Should().HaveNoCompletion(); await server.ChangeDefaultDocumentAndGetAnalysisAsync(@"def f(): for x in "); (await server.SendCompletion(uri, 1, 10)).Should().HaveLabels("in").And.NotContainLabels("for", "abs"); (await server.SendCompletion(uri, 1, 12)).Should().HaveLabels("in").And.NotContainLabels("for", "abs"); (await server.SendCompletion(uri, 1, 13)).Should().HaveLabels("abs", "x"); if (version.Is3x()) { await server.ChangeDefaultDocumentAndGetAnalysisAsync(@"async def f(): async for x in "); (await server.SendCompletion(uri, 1, 4)).Should().HaveLabels("async", "for"); (await server.SendCompletion(uri, 1, 9)).Should().HaveLabels("async", "for"); (await server.SendCompletion(uri, 1, 13)).Should().HaveLabels("async", "for"); (await server.SendCompletion(uri, 1, 14)).Should().HaveNoCompletion(); } }
// ClassDefinition public override bool Walk(ClassDefinition node) { if (node.Name != null) { node.Variable = DefineName(node.Name); node.AddVariableReference(_globalScope, _bindRefs, Reference(node.Name)); } if (node.Bases != null) { // Base references are in the outer context foreach (var b in node.Bases) { b.Expression.Walk(this); } } // process the decorators in the outer context if (node.Decorators != null) { foreach (Expression dec in node.Decorators.Decorators) { if (dec != null) { dec.Walk(this); } } } PushScope(node); node.ModuleNameVariable = _globalScope.EnsureGlobalVariable("__name__"); // define the __doc__ and the __module__ if (node.Body.Documentation != null) { node.DocVariable = DefineName("__doc__"); } node.ModVariable = DefineName("__module__"); if (_langVersion.Is3x()) { node.ClassVariable = DefineName("__class__"); } // Walk the body node.Body.Walk(this); return(false); }
private Parser(Tokenizer tokenizer, ErrorSink errorSink, PythonLanguageVersion langVersion, bool verbatim, bool bindRefs, string privatePrefix) { Contract.Assert(tokenizer != null); Contract.Assert(errorSink != null); tokenizer.ErrorSink = new TokenizerErrorSink(this); _tokenizer = tokenizer; _errors = errorSink; _langVersion = langVersion; _verbatim = verbatim; _bindReferences = bindRefs; if (langVersion.Is3x()) { // 3.x always does true division and absolute import _languageFeatures |= FutureOptions.TrueDivision | FutureOptions.AbsoluteImports; } Reset(FutureOptions.None); _privatePrefix = privatePrefix; }
public FallbackBuiltinPythonType(PythonLanguageVersion version, BuiltinTypeId typeId) { DeclaringModule = version.Is3x() ? FallbackBuiltinModule.Instance3x : FallbackBuiltinModule.Instance2x; Name = SharedDatabaseState.GetBuiltinTypeName(typeId, version.ToVersion()); TypeId = typeId; }
protected virtual bool ShouldUseUnicodeLiterals(PythonLanguageVersion version) { return(version.Is3x()); }
/// <summary> /// Retuns a sequence of all keywords usable as a statement in a /// particular version of Python. /// </summary> public static IEnumerable<string> Statement(PythonLanguageVersion version = PythonLanguageVersion.None) { yield return "assert"; if (version.IsNone() || version >= PythonLanguageVersion.V35) { yield return "async"; yield return "await"; } yield return "break"; yield return "continue"; yield return "class"; yield return "def"; yield return "del"; if (version.IsNone() || version.Is2x()) { yield return "exec"; } yield return "if"; yield return "elif"; yield return "except"; yield return "finally"; yield return "for"; yield return "from"; yield return "global"; yield return "import"; if (version.IsNone() || version.Is3x()) { yield return "nonlocal"; } yield return "pass"; if (version.IsNone() || version.Is2x()) { yield return "print"; } yield return "raise"; yield return "return"; yield return "try"; yield return "while"; yield return "with"; yield return "yield"; }
protected virtual bool ShouldUseUnicodeLiterals(PythonLanguageVersion version) { return version.Is3x(); }
public string GetConstantRepr(PythonLanguageVersion version, bool escape8bitStrings = false) { if (_value == null) { return("None"); } else if (_value is AsciiString) { StringBuilder res = new StringBuilder(); if (!version.Is2x()) { res.Append("b"); } AppendEscapedString(res, ((AsciiString)_value).String, escape8bitStrings); return(res.ToString()); } else if (_value is string) { StringBuilder res = new StringBuilder(); if (!version.Is3x()) { res.Append("u"); } AppendEscapedString(res, (string)_value, escape8bitStrings); return(res.ToString()); } else if (_value is Complex) { Complex n = (Complex)_value; string real = NegativeZeroAwareToString(n.Real); string imag = NegativeZeroAwareToString(n.Imaginary); if (n.Real != 0) { if (!imag.StartsWith("-")) { imag = "+" + imag; } return("(" + real + imag + "j)"); } else { return(imag + "j"); } } else if (_value is BigInteger) { if (!version.Is3x()) { return(_value.ToString() + "L"); } } else if (_value is double) { double n = (double)_value; string s = NegativeZeroAwareToString(n); // If there's no fractional part, and this is not NaN or +-Inf, G format will not include the decimal // point. This is okay if we're using scientific notation as this implies float, but if not, add the // decimal point to indicate the type, just like Python repr() does. if ((n - Math.Truncate(n)) == 0 && !s.Contains("e")) { s += ".0"; } return(s); } else if (_value is IFormattable) { return(((IFormattable)_value).ToString(null, CultureInfo.InvariantCulture)); } // TODO: We probably need to handle more primitives here return(_value.ToString()); }
public string GetConstantRepr(PythonLanguageVersion version, bool escape8bitStrings = false) { if (_value == null) { return "None"; } else if (_value is AsciiString) { StringBuilder res = new StringBuilder(); if (!version.Is2x()) { res.Append("b"); } AppendEscapedString(res, ((AsciiString)_value).String, escape8bitStrings); return res.ToString(); } else if (_value is string) { StringBuilder res = new StringBuilder(); if (!version.Is3x()) { res.Append("u"); } AppendEscapedString(res, (string)_value, escape8bitStrings); return res.ToString(); } else if (_value is Complex) { Complex n = (Complex)_value; string real = NegativeZeroAwareToString(n.Real); string imag = NegativeZeroAwareToString(n.Imaginary); if (n.Real != 0) { if (!imag.StartsWith("-")) { imag = "+" + imag; } return "(" + real + imag + "j)"; } else { return imag + "j"; } } else if (_value is BigInteger) { if (!version.Is3x()) { return _value.ToString() + "L"; } } else if (_value is double) { double n = (double)_value; string s = NegativeZeroAwareToString(n); // If there's no fractional part, and this is not NaN or +-Inf, G format will not include the decimal // point. This is okay if we're using scientific notation as this implies float, but if not, add the // decimal point to indicate the type, just like Python repr() does. if ((n - Math.Truncate(n)) == 0 && !s.Contains("e")) { s += ".0"; } return s; } else if (_value is IFormattable) { return ((IFormattable)_value).ToString(null, CultureInfo.InvariantCulture); } // TODO: We probably need to handle more primitives here return _value.ToString(); }
public static string GetTypeName(this BuiltinTypeId id, PythonLanguageVersion languageVersion) => id.GetTypeName(languageVersion.IsNone() || languageVersion.Is3x());
public static string GetModuleName(this BuiltinTypeId id, PythonLanguageVersion languageVersion) { return(id.GetModuleName(languageVersion.IsNone() || languageVersion.Is3x())); }
private void AnalyzeStdLib(string outdir) { var allFiles = new List <List <string> >(); foreach (var dir in _dirs) { CollectFiles(dir, allFiles); } foreach (var files in allFiles) { if (files.Count > 0) { Console.WriteLine("Now analyzing: {0}", Path.GetDirectoryName(files[0])); var projectState = new PythonAnalyzer(new CPythonInterpreter(new TypeDatabase(_indir, _version.Is3x())), _version); var modules = new List <IPythonProjectEntry>(); for (int i = 0; i < files.Count; i++) { string modName = PythonAnalyzer.PathToModuleName(files[i]); modules.Add(projectState.AddModule(modName, files[i])); } var nodes = new List <PythonAst>(); for (int i = 0; i < modules.Count; i++) { PythonAst ast = null; try { var sourceUnit = new StreamReader(files[i]); ast = Parser.CreateParser(sourceUnit, Microsoft.PythonTools.Parsing.ErrorSink.Null, PythonLanguageVersion.V27).ParseFile(); } catch (Exception) { } nodes.Add(ast); } for (int i = 0; i < modules.Count; i++) { var ast = nodes[i]; if (ast != null) { modules[i].UpdateTree(ast, null); } } for (int i = 0; i < modules.Count; i++) { var ast = nodes[i]; if (ast != null) { modules[i].Analyze(true); } } if (modules.Count > 0) { modules[0].AnalysisGroup.AnalyzeQueuedEntries(); } new SaveAnalysis().Save(projectState, outdir); } } }
public AstBuiltinsPythonModule(PythonLanguageVersion version) : base(version.Is3x() ? SharedDatabaseState.BuiltinName3x : SharedDatabaseState.BuiltinName2x, null) { _hiddenNames = new HashSet <string>(); }