public IPythonProjectEntry AddModule(string name, string code, string filename = null) { var path = Path.Combine(_root, (filename ?? (name.Replace('.', '\\') + ".py"))); if (CreateProjectOnDisk) { Directory.CreateDirectory(PathUtils.GetParent(path)); File.WriteAllText(path, code); } var entry = _analyzer.AddModule(name, path); _entries[name] = entry; if (DefaultModule == null) { DefaultModule = name; } UpdateModule(entry, code); return(entry); }
public void XamlEmptyXName() { // [Python Tools] Adding attribute through XAML in IronPython application crashes VS. // http://pytools.codeplex.com/workitem/743 PythonAnalyzer analyzer = new PythonAnalyzer(InterpreterFactory, Interpreter); string xamlPath = TestData.GetPath(@"TestData\Xaml\EmptyXName.xaml"); string pyPath = TestData.GetPath(@"TestData\Xaml\EmptyXName.py"); var xamlEntry = analyzer.AddXamlFile(xamlPath); var pyEntry = analyzer.AddModule("EmptyXName", pyPath); xamlEntry.ParseContent(new FileStreamReader(xamlPath), null); using (var parser = Parser.CreateParser(new FileStreamReader(pyPath), PythonLanguageVersion.V27, new ParserOptions() { BindReferences = true })) { pyEntry.UpdateTree(parser.ParseFile(), null); } pyEntry.Analyze(CancellationToken.None); }
private SaveLoadResult SaveLoad(PythonLanguageVersion version, params AnalysisModule[] modules) { IPythonProjectEntry[] entries = new IPythonProjectEntry[modules.Length]; var fact = InterpreterFactory; var interp = Interpreter; if (version != fact.GetLanguageVersion()) { fact = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(version.ToVersion()); interp = fact.CreateInterpreter(); } var state = new PythonAnalyzer(fact, interp, SharedDatabaseState.BuiltinName2x); state.ReloadModulesAsync().WaitAndUnwrapExceptions(); for (int i = 0; i < modules.Length; i++) { entries[i] = state.AddModule(modules[i].ModuleName, modules[i].Filename); Prepare(entries[i], new StringReader(modules[i].Code), version); } for (int i = 0; i < modules.Length; i++) { entries[i].Analyze(CancellationToken.None); } string tmpFolder = TestData.GetTempPath("6666d700-a6d8-4e11-8b73-3ba99a61e27b"); Directory.CreateDirectory(tmpFolder); new SaveAnalysis().Save(state, tmpFolder); File.Copy(Path.Combine(PythonTypeDatabase.BaselineDatabasePath, "__builtin__.idb"), Path.Combine(tmpFolder, "__builtin__.idb"), true); return(new SaveLoadResult( PythonAnalyzer.CreateSynchronously(InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(version.ToVersion(), null, tmpFolder)), tmpFolder )); }
public void TestParsePerf_Decimal() { string merlin = Environment.GetEnvironmentVariable("DLR_ROOT") ?? @"C:\Product\0\dlr"; var text = File.ReadAllText(Path.Combine(merlin + @"\External.LCA_RESTRICTED\Languages\IronPython\27\Lib\decimal.py")); var sourceUnit = GetSourceUnit(text); var projectState = new PythonAnalyzer(InterpreterFactory, Interpreter); Stopwatch sw = new Stopwatch(); var entry = projectState.AddModule("decimal", "decimal", null); Prepare(entry, sourceUnit); entry.Analyze(CancellationToken.None); sw.Start(); for (int i = 0; i < 5; i++) { Prepare(entry, sourceUnit); entry.Analyze(CancellationToken.None); } sw.Stop(); Console.WriteLine("{0}", sw.ElapsedMilliseconds); }
internal PythonAnalyzer AnalyzeDir(string dir, PythonLanguageVersion version = PythonLanguageVersion.V27, IEnumerable <string> excludeDirectories = null, CancellationToken?cancel = null) { List <string> files = new List <string>(); try { ISet <string> excluded = null; if (excludeDirectories != null) { excluded = new HashSet <string>(excludeDirectories, StringComparer.InvariantCultureIgnoreCase); } CollectFiles(dir, files, excluded); } catch (DirectoryNotFoundException) { return(null); } List <FileStreamReader> sourceUnits = new List <FileStreamReader>(); foreach (string file in files) { sourceUnits.Add( new FileStreamReader(file) ); } Stopwatch sw = new Stopwatch(); sw.Start(); long start0 = sw.ElapsedMilliseconds; // Explicitly specify the builtins name because we are using a 2.7 // interpreter for all versions. var fact = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(version.ToVersion()); var projectState = new PythonAnalyzer(fact, fact.CreateInterpreter(), "__builtin__"); projectState.Limits = AnalysisLimits.GetStandardLibraryLimits(); var modules = new List <IPythonProjectEntry>(); foreach (var sourceUnit in sourceUnits) { try { modules.Add(projectState.AddModule( ModulePath.FromFullPath(sourceUnit.Path).ModuleName, sourceUnit.Path, null )); } catch (ArgumentException) { // Invalid module name, so skip the module } } long start1 = sw.ElapsedMilliseconds; Trace.TraceInformation("AddSourceUnit: {0} ms", start1 - start0); var nodes = new List <Microsoft.PythonTools.Parsing.Ast.PythonAst>(); for (int i = 0; i < modules.Count; i++) { PythonAst ast = null; try { var sourceUnit = sourceUnits[i]; ast = Parser.CreateParser(sourceUnit, version).ParseFile(); } catch (Exception) { } nodes.Add(ast); } long start2 = sw.ElapsedMilliseconds; Trace.TraceInformation("Parse: {0} ms", start2 - start1); for (int i = 0; i < modules.Count; i++) { var ast = nodes[i]; if (ast != null) { modules[i].UpdateTree(ast, null); } } long start3 = sw.ElapsedMilliseconds; for (int i = 0; i < modules.Count; i++) { Trace.TraceInformation("Analyzing {1}: {0} ms", sw.ElapsedMilliseconds - start3, sourceUnits[i].Path); var ast = nodes[i]; if (ast != null) { modules[i].Analyze(cancel ?? CancellationToken.None, true); } } if (modules.Count > 0) { Trace.TraceInformation("Analyzing queue"); modules[0].AnalysisGroup.AnalyzeQueuedEntries(cancel ?? CancellationToken.None); } long start4 = sw.ElapsedMilliseconds; Trace.TraceInformation("Analyze: {0} ms", start4 - start3); return(projectState); }
private static void RunCommand( string cmd, string args, PythonAnalyzer analyzer, Dictionary <string, IPythonProjectEntry> modules, State state ) { switch (cmd.ToLower()) { case "print": Console.WriteLine(args); break; case "module": foreach (var mod in GetModules(args)) { IPythonProjectEntry entry; if (!modules.TryGetValue(mod.ModuleName, out entry)) { Console.WriteLine("Creating module {0}", mod.ModuleName); modules[mod.ModuleName] = entry = analyzer.AddModule(mod.ModuleName, mod.SourceFile); } else { Console.WriteLine("Reusing module {0}", mod.ModuleName); } using (var file = File.OpenText(mod.SourceFile)) { var parser = Parser.CreateParser( file, analyzer.LanguageVersion, new ParserOptions() { BindReferences = true } ); entry.UpdateTree(parser.ParseFile(), null); } } break; case "enqueue": if (args == "*") { foreach (var entry in modules.Values) { entry.Analyze(CancellationToken.None, true); } } else { IPythonProjectEntry entry; foreach (var modName in args.Split(',').Select(s => s.Trim()).Where(s => !string.IsNullOrEmpty(s))) { if (!modules.TryGetValue(modName, out entry)) { Console.WriteLine("Could not enqueue {0}", modName); } else { entry.Analyze(CancellationToken.None, true); } } } break; case "analyze": Console.Write("Waiting for complete analysis... "); analyzer.AnalyzeQueuedEntries(CancellationToken.None); Console.WriteLine("done!"); break; case "pause": Console.Write("Press enter to continue..."); Console.ReadKey(true); Console.WriteLine(); break; case "debugbreak": if (!args.Equals("ifattached", StringComparison.CurrentCultureIgnoreCase) || Debugger.IsAttached) { Console.WriteLine("Breaking into the debugger"); Debugger.Break(); } break; case "gc": Console.WriteLine("Collecting garbage"); GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced); GC.WaitForFullGCComplete(); GC.WaitForPendingFinalizers(); break; case "dump": { var fullPath = Path.GetFullPath(args); var length = WriteDump(Process.GetCurrentProcess(), fullPath, MiniDumpType.FullDump); Console.WriteLine( "Dump written to {0} at {1:F1}MB ({2} bytes)", fullPath, length / (1024.0 * 1024.0), length ); if (state.LastDumpSize > 0 && state.LastDumpSize != length) { var delta = Math.Abs(length - state.LastDumpSize); var direction = (length > state.LastDumpSize) ? "larger" : "smaller"; Console.WriteLine( "Dump is {0:F1}MB ({1} bytes) {2} than previous", delta / (1024.0 * 1024.0), delta, direction ); } state.LastDumpSize = length; break; } default: Console.WriteLine("Command not available: {0}", cmd); break; } }
private async Task <IProjectEntry> AddFileAsync(Uri documentUri, Uri fromSearchPath, IAnalysisCookie cookie = null) { var item = _projectFiles.GetEntry(documentUri, throwIfMissing: false); if (item != null) { return(item); } IEnumerable <string> aliases = null; var path = GetLocalPath(documentUri); if (fromSearchPath != null) { if (ModulePath.FromBasePathAndFile_NoThrow(GetLocalPath(fromSearchPath), path, out var mp)) { aliases = new[] { mp.ModuleName }; } } else { aliases = GetImportNames(documentUri).Select(mp => mp.ModuleName).ToArray(); } if (!(aliases?.Any() ?? false)) { aliases = new[] { Path.GetFileNameWithoutExtension(path) }; } var reanalyzeEntries = aliases.SelectMany(a => _analyzer.GetEntriesThatImportModule(a, true)).ToArray(); var first = aliases.FirstOrDefault(); var pyItem = _analyzer.AddModule(first, path, documentUri, cookie); item = pyItem; foreach (var a in aliases.Skip(1)) { _analyzer.AddModuleAlias(first, a); } var actualItem = _projectFiles.GetOrAddEntry(documentUri, item); if (actualItem != item) { return(actualItem); } if (_clientCaps?.python?.analysisUpdates ?? false) { pyItem.OnNewAnalysis += ProjectEntry_OnNewAnalysis; } if (item is IDocument doc) { EnqueueItem(doc); } if (reanalyzeEntries != null) { foreach (var entryRef in reanalyzeEntries) { _queue.Enqueue(entryRef, AnalysisPriority.Low); } } return(item); }
public PythonAnalyzer AnalyzeStdLib() { string dir = Path.Combine("C:\\Python27\\Lib"); List <string> files = new List <string>(); CollectFiles(dir, files); List <FileStreamReader> sourceUnits = new List <FileStreamReader>(); foreach (string file in files) { sourceUnits.Add( new FileStreamReader(file) ); } Stopwatch sw = new Stopwatch(); sw.Start(); long start0 = sw.ElapsedMilliseconds; var projectState = new PythonAnalyzer(Interpreter, PythonLanguageVersion.V27); var modules = new List <IPythonProjectEntry>(); foreach (var sourceUnit in sourceUnits) { modules.Add(projectState.AddModule(PythonAnalyzer.PathToModuleName(sourceUnit.Path), sourceUnit.Path, null)); } long start1 = sw.ElapsedMilliseconds; Console.WriteLine("AddSourceUnit: {0} ms", start1 - start0); var nodes = new List <Microsoft.PythonTools.Parsing.Ast.PythonAst>(); for (int i = 0; i < modules.Count; i++) { PythonAst ast = null; try { var sourceUnit = sourceUnits[i]; ast = Parser.CreateParser(sourceUnit, Microsoft.PythonTools.Parsing.ErrorSink.Null, PythonLanguageVersion.V27).ParseFile(); } catch (Exception) { } nodes.Add(ast); } long start2 = sw.ElapsedMilliseconds; Console.WriteLine("Parse: {0} ms", start2 - start1); for (int i = 0; i < modules.Count; i++) { var ast = nodes[i]; if (ast != null) { modules[i].UpdateTree(ast, null); } } long start3 = sw.ElapsedMilliseconds; for (int i = 0; i < modules.Count; i++) { Console.WriteLine("Analyzing {1}: {0} ms", sw.ElapsedMilliseconds - start3, sourceUnits[i].Path); var ast = nodes[i]; if (ast != null) { modules[i].Analyze(true); } } if (modules.Count > 0) { Console.WriteLine("Analyzing queue"); modules[0].AnalysisGroup.AnalyzeQueuedEntries(); } long start4 = sw.ElapsedMilliseconds; Console.WriteLine("Analyze: {0} ms", start4 - start3); return(projectState); }
private IEnumerable <Task> StartCrossThreadAnalysisCalls(PythonAnalyzer state, CancellationToken cancel) { const string testCode = @"from mod{0:000} import test_func as other_test_func, MyClass as other_mc c = None def test_func(a, b={1}()): '''My test function''' globals c a = b a = {1}(a) b = other_test_func(a) c = other_mc.fn(b) return b class MyClass: fn = test_func my_test_func = test_func my_test_func = other_test_func my_test_func('abc') mc = MyClass() mc.fn([]) "; state.SetSearchPaths(new [] { TestData.GetTestSpecificRootUri().ToAbsolutePath() }); var entries = Enumerable.Range(0, 100) .Select(i => { var entry = (ProjectEntry)state.AddModule(string.Format("mod{0:000}", i), TestData.GetTestSpecificPath(string.Format("mod{0:000}.py", i))); var parser = Parser.CreateParser(new StringReader(testCode.FormatInvariant(i + 1, PythonTypes[i % PythonTypes.Count])), PythonLanguageVersion.V34); using (var p = entry.BeginParse()) { p.Tree = parser.ParseFile(); p.Complete(); } return(entry); }) .ToList(); // One analysis before we start foreach (var e in entries) { e.Analyze(cancel, true); } state.AnalyzeQueuedEntries(cancel); // Repeatedly re-analyse the code yield return(Task.Run(() => { var rnd = new Random(); while (!cancel.IsCancellationRequested) { var shufEntries = entries .Select(e => Tuple.Create(rnd.Next(), e)) .OrderBy(t => t.Item1) .Take(entries.Count / 2) .Select(t => t.Item2) .ToList(); foreach (var e in shufEntries) { e.Analyze(cancel, true); } state.AnalyzeQueuedEntries(cancel); Console.WriteLine("Analysis complete"); Thread.Sleep(1000); } }, cancel)); // Repeatedly re-parse the code yield return(Task.Run(() => { var rnd = new Random(); while (!cancel.IsCancellationRequested) { var shufEntries = entries .Select((e, i) => Tuple.Create(rnd.Next(), e, i)) .OrderBy(t => t.Item1) .Take(entries.Count / 4) .ToList(); foreach (var t in shufEntries) { var i = t.Item3; var parser = Parser.CreateParser(new StringReader(testCode.FormatInvariant(i + 1, PythonTypes[i % PythonTypes.Count])), PythonLanguageVersion.V34); using (var p = t.Item2.BeginParse()) { p.Tree = parser.ParseFile(); p.Complete(); } } Thread.Sleep(1000); } }, cancel)); // Repeatedly request signatures yield return(Task.Run(() => { var entry = entries[1]; while (!cancel.IsCancellationRequested) { var sigs = entry.Analysis.GetSignaturesByIndex("my_test_func", 0).ToList(); if (sigs.Any()) { foreach (var s in sigs) { s.Name.Should().Be("test_func"); s.Parameters.Select(p => p.Name).Should().BeEquivalentTo("a", "b"); } } } }, cancel)); // Repeated request variables and descriptions yield return(Task.Run(() => { var entry = entries[1]; while (!cancel.IsCancellationRequested) { var descriptions = entry.Analysis.GetDescriptionsByIndex("my_test_func", 0).ToList(); descriptions = entry.Analysis.GetDescriptionsByIndex("c", 0).ToList(); } }, cancel)); // Repeated request members and documentation yield return(Task.Run(() => { var entry = entries[1]; while (!cancel.IsCancellationRequested) { var descriptions = entry.Analysis.GetCompletionDocumentationByIndex("mc", "fn", 0).ToList(); } }, cancel)); }