public IPythonProjectEntry AddModule(string name, string code, string filename = null)
{
var path = Path.Combine(_root, (filename ?? (name.Replace('.', '\\') + ".py")));
if (CreateProjectOnDisk)
{
Directory.CreateDirectory(PathUtils.GetParent(path));
File.WriteAllText(path, code);
}
var entry = _analyzer.AddModule(name, path);
_entries[name] = entry;
if (DefaultModule == null)
{
DefaultModule = name;
}
UpdateModule(entry, code);
return(entry);
}
public void XamlEmptyXName()
{
// [Python Tools] Adding attribute through XAML in IronPython application crashes VS.
// http://pytools.codeplex.com/workitem/743
PythonAnalyzer analyzer = new PythonAnalyzer(InterpreterFactory, Interpreter);
string xamlPath = TestData.GetPath(@"TestData\Xaml\EmptyXName.xaml");
string pyPath = TestData.GetPath(@"TestData\Xaml\EmptyXName.py");
var xamlEntry = analyzer.AddXamlFile(xamlPath);
var pyEntry = analyzer.AddModule("EmptyXName", pyPath);
xamlEntry.ParseContent(new FileStreamReader(xamlPath), null);
using (var parser = Parser.CreateParser(new FileStreamReader(pyPath), PythonLanguageVersion.V27, new ParserOptions()
{
BindReferences = true
})) {
pyEntry.UpdateTree(parser.ParseFile(), null);
}
pyEntry.Analyze(CancellationToken.None);
}
private SaveLoadResult SaveLoad(PythonLanguageVersion version, params AnalysisModule[] modules)
{
IPythonProjectEntry[] entries = new IPythonProjectEntry[modules.Length];
var fact = InterpreterFactory;
var interp = Interpreter;
if (version != fact.GetLanguageVersion())
{
fact = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(version.ToVersion());
interp = fact.CreateInterpreter();
}
var state = new PythonAnalyzer(fact, interp, SharedDatabaseState.BuiltinName2x);
state.ReloadModulesAsync().WaitAndUnwrapExceptions();
for (int i = 0; i < modules.Length; i++)
{
entries[i] = state.AddModule(modules[i].ModuleName, modules[i].Filename);
Prepare(entries[i], new StringReader(modules[i].Code), version);
}
for (int i = 0; i < modules.Length; i++)
{
entries[i].Analyze(CancellationToken.None);
}
string tmpFolder = TestData.GetTempPath("6666d700-a6d8-4e11-8b73-3ba99a61e27b");
Directory.CreateDirectory(tmpFolder);
new SaveAnalysis().Save(state, tmpFolder);
File.Copy(Path.Combine(PythonTypeDatabase.BaselineDatabasePath, "__builtin__.idb"), Path.Combine(tmpFolder, "__builtin__.idb"), true);
return(new SaveLoadResult(
PythonAnalyzer.CreateSynchronously(InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(version.ToVersion(), null, tmpFolder)),
tmpFolder
));
}
public void TestParsePerf_Decimal()
{
string merlin = Environment.GetEnvironmentVariable("DLR_ROOT") ?? @"C:\Product\0\dlr";
var text = File.ReadAllText(Path.Combine(merlin + @"\External.LCA_RESTRICTED\Languages\IronPython\27\Lib\decimal.py"));
var sourceUnit = GetSourceUnit(text);
var projectState = new PythonAnalyzer(InterpreterFactory, Interpreter);
Stopwatch sw = new Stopwatch();
var entry = projectState.AddModule("decimal", "decimal", null);
Prepare(entry, sourceUnit);
entry.Analyze(CancellationToken.None);
sw.Start();
for (int i = 0; i < 5; i++)
{
Prepare(entry, sourceUnit);
entry.Analyze(CancellationToken.None);
}
sw.Stop();
Console.WriteLine("{0}", sw.ElapsedMilliseconds);
}
internal PythonAnalyzer AnalyzeDir(string dir, PythonLanguageVersion version = PythonLanguageVersion.V27, IEnumerable <string> excludeDirectories = null, CancellationToken?cancel = null)
{
List <string> files = new List <string>();
try {
ISet <string> excluded = null;
if (excludeDirectories != null)
{
excluded = new HashSet <string>(excludeDirectories, StringComparer.InvariantCultureIgnoreCase);
}
CollectFiles(dir, files, excluded);
} catch (DirectoryNotFoundException) {
return(null);
}
List <FileStreamReader> sourceUnits = new List <FileStreamReader>();
foreach (string file in files)
{
sourceUnits.Add(
new FileStreamReader(file)
);
}
Stopwatch sw = new Stopwatch();
sw.Start();
long start0 = sw.ElapsedMilliseconds;
// Explicitly specify the builtins name because we are using a 2.7
// interpreter for all versions.
var fact = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(version.ToVersion());
var projectState = new PythonAnalyzer(fact, fact.CreateInterpreter(), "__builtin__");
projectState.Limits = AnalysisLimits.GetStandardLibraryLimits();
var modules = new List <IPythonProjectEntry>();
foreach (var sourceUnit in sourceUnits)
{
try {
modules.Add(projectState.AddModule(
ModulePath.FromFullPath(sourceUnit.Path).ModuleName,
sourceUnit.Path,
null
));
} catch (ArgumentException) {
// Invalid module name, so skip the module
}
}
long start1 = sw.ElapsedMilliseconds;
Trace.TraceInformation("AddSourceUnit: {0} ms", start1 - start0);
var nodes = new List <Microsoft.PythonTools.Parsing.Ast.PythonAst>();
for (int i = 0; i < modules.Count; i++)
{
PythonAst ast = null;
try {
var sourceUnit = sourceUnits[i];
ast = Parser.CreateParser(sourceUnit, version).ParseFile();
} catch (Exception) {
}
nodes.Add(ast);
}
long start2 = sw.ElapsedMilliseconds;
Trace.TraceInformation("Parse: {0} ms", start2 - start1);
for (int i = 0; i < modules.Count; i++)
{
var ast = nodes[i];
if (ast != null)
{
modules[i].UpdateTree(ast, null);
}
}
long start3 = sw.ElapsedMilliseconds;
for (int i = 0; i < modules.Count; i++)
{
Trace.TraceInformation("Analyzing {1}: {0} ms", sw.ElapsedMilliseconds - start3, sourceUnits[i].Path);
var ast = nodes[i];
if (ast != null)
{
modules[i].Analyze(cancel ?? CancellationToken.None, true);
}
}
if (modules.Count > 0)
{
Trace.TraceInformation("Analyzing queue");
modules[0].AnalysisGroup.AnalyzeQueuedEntries(cancel ?? CancellationToken.None);
}
long start4 = sw.ElapsedMilliseconds;
Trace.TraceInformation("Analyze: {0} ms", start4 - start3);
return(projectState);
}
private static void RunCommand(
string cmd,
string args,
PythonAnalyzer analyzer,
Dictionary <string, IPythonProjectEntry> modules,
State state
)
{
switch (cmd.ToLower())
{
case "print":
Console.WriteLine(args);
break;
case "module":
foreach (var mod in GetModules(args))
{
IPythonProjectEntry entry;
if (!modules.TryGetValue(mod.ModuleName, out entry))
{
Console.WriteLine("Creating module {0}", mod.ModuleName);
modules[mod.ModuleName] = entry = analyzer.AddModule(mod.ModuleName, mod.SourceFile);
}
else
{
Console.WriteLine("Reusing module {0}", mod.ModuleName);
}
using (var file = File.OpenText(mod.SourceFile)) {
var parser = Parser.CreateParser(
file,
analyzer.LanguageVersion,
new ParserOptions()
{
BindReferences = true
}
);
entry.UpdateTree(parser.ParseFile(), null);
}
}
break;
case "enqueue":
if (args == "*")
{
foreach (var entry in modules.Values)
{
entry.Analyze(CancellationToken.None, true);
}
}
else
{
IPythonProjectEntry entry;
foreach (var modName in args.Split(',').Select(s => s.Trim()).Where(s => !string.IsNullOrEmpty(s)))
{
if (!modules.TryGetValue(modName, out entry))
{
Console.WriteLine("Could not enqueue {0}", modName);
}
else
{
entry.Analyze(CancellationToken.None, true);
}
}
}
break;
case "analyze":
Console.Write("Waiting for complete analysis... ");
analyzer.AnalyzeQueuedEntries(CancellationToken.None);
Console.WriteLine("done!");
break;
case "pause":
Console.Write("Press enter to continue...");
Console.ReadKey(true);
Console.WriteLine();
break;
case "debugbreak":
if (!args.Equals("ifattached", StringComparison.CurrentCultureIgnoreCase) || Debugger.IsAttached)
{
Console.WriteLine("Breaking into the debugger");
Debugger.Break();
}
break;
case "gc":
Console.WriteLine("Collecting garbage");
GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced);
GC.WaitForFullGCComplete();
GC.WaitForPendingFinalizers();
break;
case "dump": {
var fullPath = Path.GetFullPath(args);
var length = WriteDump(Process.GetCurrentProcess(), fullPath, MiniDumpType.FullDump);
Console.WriteLine(
"Dump written to {0} at {1:F1}MB ({2} bytes)",
fullPath,
length / (1024.0 * 1024.0),
length
);
if (state.LastDumpSize > 0 && state.LastDumpSize != length)
{
var delta = Math.Abs(length - state.LastDumpSize);
var direction = (length > state.LastDumpSize) ? "larger" : "smaller";
Console.WriteLine(
"Dump is {0:F1}MB ({1} bytes) {2} than previous",
delta / (1024.0 * 1024.0),
delta,
direction
);
}
state.LastDumpSize = length;
break;
}
default:
Console.WriteLine("Command not available: {0}", cmd);
break;
}
}
private async Task <IProjectEntry> AddFileAsync(Uri documentUri, Uri fromSearchPath, IAnalysisCookie cookie = null)
{
var item = _projectFiles.GetEntry(documentUri, throwIfMissing: false);
if (item != null)
{
return(item);
}
IEnumerable <string> aliases = null;
var path = GetLocalPath(documentUri);
if (fromSearchPath != null)
{
if (ModulePath.FromBasePathAndFile_NoThrow(GetLocalPath(fromSearchPath), path, out var mp))
{
aliases = new[] { mp.ModuleName };
}
}
else
{
aliases = GetImportNames(documentUri).Select(mp => mp.ModuleName).ToArray();
}
if (!(aliases?.Any() ?? false))
{
aliases = new[] { Path.GetFileNameWithoutExtension(path) };
}
var reanalyzeEntries = aliases.SelectMany(a => _analyzer.GetEntriesThatImportModule(a, true)).ToArray();
var first = aliases.FirstOrDefault();
var pyItem = _analyzer.AddModule(first, path, documentUri, cookie);
item = pyItem;
foreach (var a in aliases.Skip(1))
{
_analyzer.AddModuleAlias(first, a);
}
var actualItem = _projectFiles.GetOrAddEntry(documentUri, item);
if (actualItem != item)
{
return(actualItem);
}
if (_clientCaps?.python?.analysisUpdates ?? false)
{
pyItem.OnNewAnalysis += ProjectEntry_OnNewAnalysis;
}
if (item is IDocument doc)
{
EnqueueItem(doc);
}
if (reanalyzeEntries != null)
{
foreach (var entryRef in reanalyzeEntries)
{
_queue.Enqueue(entryRef, AnalysisPriority.Low);
}
}
return(item);
}
public PythonAnalyzer AnalyzeStdLib()
{
string dir = Path.Combine("C:\\Python27\\Lib");
List <string> files = new List <string>();
CollectFiles(dir, files);
List <FileStreamReader> sourceUnits = new List <FileStreamReader>();
foreach (string file in files)
{
sourceUnits.Add(
new FileStreamReader(file)
);
}
Stopwatch sw = new Stopwatch();
sw.Start();
long start0 = sw.ElapsedMilliseconds;
var projectState = new PythonAnalyzer(Interpreter, PythonLanguageVersion.V27);
var modules = new List <IPythonProjectEntry>();
foreach (var sourceUnit in sourceUnits)
{
modules.Add(projectState.AddModule(PythonAnalyzer.PathToModuleName(sourceUnit.Path), sourceUnit.Path, null));
}
long start1 = sw.ElapsedMilliseconds;
Console.WriteLine("AddSourceUnit: {0} ms", start1 - start0);
var nodes = new List <Microsoft.PythonTools.Parsing.Ast.PythonAst>();
for (int i = 0; i < modules.Count; i++)
{
PythonAst ast = null;
try {
var sourceUnit = sourceUnits[i];
ast = Parser.CreateParser(sourceUnit, Microsoft.PythonTools.Parsing.ErrorSink.Null, PythonLanguageVersion.V27).ParseFile();
} catch (Exception) {
}
nodes.Add(ast);
}
long start2 = sw.ElapsedMilliseconds;
Console.WriteLine("Parse: {0} ms", start2 - start1);
for (int i = 0; i < modules.Count; i++)
{
var ast = nodes[i];
if (ast != null)
{
modules[i].UpdateTree(ast, null);
}
}
long start3 = sw.ElapsedMilliseconds;
for (int i = 0; i < modules.Count; i++)
{
Console.WriteLine("Analyzing {1}: {0} ms", sw.ElapsedMilliseconds - start3, sourceUnits[i].Path);
var ast = nodes[i];
if (ast != null)
{
modules[i].Analyze(true);
}
}
if (modules.Count > 0)
{
Console.WriteLine("Analyzing queue");
modules[0].AnalysisGroup.AnalyzeQueuedEntries();
}
long start4 = sw.ElapsedMilliseconds;
Console.WriteLine("Analyze: {0} ms", start4 - start3);
return(projectState);
}
private IEnumerable <Task> StartCrossThreadAnalysisCalls(PythonAnalyzer state, CancellationToken cancel)
{
const string testCode = @"from mod{0:000} import test_func as other_test_func, MyClass as other_mc
c = None
def test_func(a, b={1}()):
'''My test function'''
globals c
a = b
a = {1}(a)
b = other_test_func(a)
c = other_mc.fn(b)
return b
class MyClass:
fn = test_func
my_test_func = test_func
my_test_func = other_test_func
my_test_func('abc')
mc = MyClass()
mc.fn([])
";
state.SetSearchPaths(new [] { TestData.GetTestSpecificRootUri().ToAbsolutePath() });
var entries = Enumerable.Range(0, 100)
.Select(i => {
var entry = (ProjectEntry)state.AddModule(string.Format("mod{0:000}", i), TestData.GetTestSpecificPath(string.Format("mod{0:000}.py", i)));
var parser = Parser.CreateParser(new StringReader(testCode.FormatInvariant(i + 1, PythonTypes[i % PythonTypes.Count])), PythonLanguageVersion.V34);
using (var p = entry.BeginParse()) {
p.Tree = parser.ParseFile();
p.Complete();
}
return(entry);
})
.ToList();
// One analysis before we start
foreach (var e in entries)
{
e.Analyze(cancel, true);
}
state.AnalyzeQueuedEntries(cancel);
// Repeatedly re-analyse the code
yield return(Task.Run(() => {
var rnd = new Random();
while (!cancel.IsCancellationRequested)
{
var shufEntries = entries
.Select(e => Tuple.Create(rnd.Next(), e))
.OrderBy(t => t.Item1)
.Take(entries.Count / 2)
.Select(t => t.Item2)
.ToList();
foreach (var e in shufEntries)
{
e.Analyze(cancel, true);
}
state.AnalyzeQueuedEntries(cancel);
Console.WriteLine("Analysis complete");
Thread.Sleep(1000);
}
}, cancel));
// Repeatedly re-parse the code
yield return(Task.Run(() => {
var rnd = new Random();
while (!cancel.IsCancellationRequested)
{
var shufEntries = entries
.Select((e, i) => Tuple.Create(rnd.Next(), e, i))
.OrderBy(t => t.Item1)
.Take(entries.Count / 4)
.ToList();
foreach (var t in shufEntries)
{
var i = t.Item3;
var parser = Parser.CreateParser(new StringReader(testCode.FormatInvariant(i + 1, PythonTypes[i % PythonTypes.Count])), PythonLanguageVersion.V34);
using (var p = t.Item2.BeginParse()) {
p.Tree = parser.ParseFile();
p.Complete();
}
}
Thread.Sleep(1000);
}
}, cancel));
// Repeatedly request signatures
yield return(Task.Run(() => {
var entry = entries[1];
while (!cancel.IsCancellationRequested)
{
var sigs = entry.Analysis.GetSignaturesByIndex("my_test_func", 0).ToList();
if (sigs.Any())
{
foreach (var s in sigs)
{
s.Name.Should().Be("test_func");
s.Parameters.Select(p => p.Name).Should().BeEquivalentTo("a", "b");
}
}
}
}, cancel));
// Repeated request variables and descriptions
yield return(Task.Run(() => {
var entry = entries[1];
while (!cancel.IsCancellationRequested)
{
var descriptions = entry.Analysis.GetDescriptionsByIndex("my_test_func", 0).ToList();
descriptions = entry.Analysis.GetDescriptionsByIndex("c", 0).ToList();
}
}, cancel));
// Repeated request members and documentation
yield return(Task.Run(() => {
var entry = entries[1];
while (!cancel.IsCancellationRequested)
{
var descriptions = entry.Analysis.GetCompletionDocumentationByIndex("mc", "fn", 0).ToList();
}
}, cancel));
}
