public async Task CrossThreadAnalysisCalls()
{
var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
var interpreterFactory = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(PythonLanguageVersion.V34.ToVersion());
var state = await PythonAnalyzer.CreateAsync(interpreterFactory, cts.Token);
var tasks = StartCrossThreadAnalysisCalls(state, cts.Token).ToArray();
try {
Task.WaitAny(tasks, cts.Token);
} catch (OperationCanceledException) {
}
cts.Cancel();
ExceptionDispatchInfo firstFail = null;
bool multipleFail = false;
foreach (var t in tasks)
{
if (t.IsCanceled || t.Exception == null)
{
continue;
}
if (multipleFail)
{
Console.WriteLine(t.Exception);
}
else if (firstFail != null)
{
Console.WriteLine(firstFail);
Console.WriteLine(t.Exception);
firstFail = null;
multipleFail = true;
}
else if (t.Exception.InnerExceptions.Count == 1)
{
firstFail = ExceptionDispatchInfo.Capture(t.Exception.InnerException);
}
else
{
foreach (var exc in t.Exception.InnerExceptions)
{
Console.WriteLine(exc);
}
multipleFail = true;
}
}
if (multipleFail)
{
Assert.Fail("Errors occurred. See output for details.");
}
else if (firstFail != null)
{
firstFail.Throw();
}
}
static void Main(string[] args)
{
var responseFile = Path.GetFullPath(args.FirstOrDefault() ?? "AnalysisMemoryTester.rsp");
if (!File.Exists(responseFile))
{
if (!string.IsNullOrEmpty(responseFile))
{
Console.WriteLine("Could not open {0}", responseFile);
}
else
{
Console.WriteLine("No response file specified");
}
PrintUsage();
return;
}
var commands = File.ReadAllLines(responseFile)
.Select(line => line.Trim())
.ToList();
Environment.CurrentDirectory = Path.GetDirectoryName(responseFile);
var interpreter = GetFirstCommand(commands, "python\\s+(\\d\\.\\d)\\s+(.+)", m => m.Value, v => v.Length > 4 && File.Exists(v.Substring(4).Trim()));
var version = Version.Parse(interpreter.Substring(0, 3));
interpreter = interpreter.Substring(4).Trim();
Console.WriteLine($"Using Python from {interpreter}");
var config = new InterpreterConfiguration(
"Python|" + interpreter,
interpreter,
Path.GetDirectoryName(interpreter),
interpreter,
interpreter,
"PYTHONPATH",
NativeMethods.GetBinaryType(interpreter) == System.Reflection.ProcessorArchitecture.Amd64 ? InterpreterArchitecture.x64 : InterpreterArchitecture.x86,
version
);
using (var factory = new Interpreter.Ast.AstPythonInterpreterFactory(config, new InterpreterFactoryCreationOptions()))
using (var analyzer = PythonAnalyzer.CreateAsync(factory).WaitAndUnwrapExceptions()) {
var modules = new Dictionary <string, IPythonProjectEntry>();
var state = new State();
foreach (var tuple in SplitCommands(commands))
{
RunCommand(
tuple.Item1,
tuple.Item2,
analyzer,
modules,
state
);
}
}
}
private async Task <PythonAnalyzer> CreateAnalyzer(PythonInitializationOptions.Interpreter interpreter, CancellationToken token)
{
var factory = ActivateObject <IPythonInterpreterFactory>(interpreter.assembly, interpreter.typeName, interpreter.properties)
?? new AstPythonInterpreterFactory(interpreter.properties);
var analyzer = await PythonAnalyzer.CreateAsync(factory, token);
LogMessage(MessageType.Info, $"Created {analyzer.Interpreter.GetType().FullName} instance from {factory.GetType().FullName}");
return(analyzer);
}
private DjangoAnalyzer AnalyzerTest(string path, out PythonLanguageVersion languageVersion)
{
var version = PythonPaths.Versions.LastOrDefault(v => Directory.Exists(Path.Combine(v.PrefixPath, "Lib", "site-packages", "django")));
version.AssertInstalled();
var testFact = InterpreterFactoryCreator.CreateInterpreterFactory(version.Configuration, new InterpreterFactoryCreationOptions {
DatabasePath = TestData.GetTempPath(),
UseExistingCache = false,
TraceLevel = TraceLevel.Verbose,
WatchFileSystem = false
});
Debug.WriteLine("Testing with {0}".FormatInvariant(version.InterpreterPath));
languageVersion = testFact.GetLanguageVersion();
var analyzer = PythonAnalyzer.CreateAsync(testFact).WaitAndUnwrapExceptions();
var djangoAnalyzer = new DjangoAnalyzer();
djangoAnalyzer.Register(analyzer);
var entries = new List <IPythonProjectEntry>();
foreach (string file in Directory.EnumerateFiles(path, "*.py", SearchOption.AllDirectories))
{
if (!ModulePath.FromBasePathAndFile_NoThrow(path, file, out var mp))
{
Debug.WriteLine("Not parsing {0}".FormatInvariant(file));
continue;
}
Debug.WriteLine("Parsing {0} ({1})".FormatInvariant(mp.FullName, file));
var entry = analyzer.AddModule(mp.ModuleName, file);
var parser = Parser.CreateParser(
new FileStream(file, FileMode.Open, FileAccess.Read),
testFact.GetLanguageVersion()
);
using (var p = entry.BeginParse()) {
p.Tree = parser.ParseFile();
p.Complete();
}
entries.Add(entry);
}
foreach (var entry in entries)
{
entry.Analyze(CancellationToken.None);
}
Debug.WriteLine((testFact as IPythonInterpreterFactoryWithLog)?.GetAnalysisLogContent(CultureInfo.CurrentUICulture) ?? "(no logs)");
return(djangoAnalyzer);
}
public PythonAnalysis(IPythonInterpreterFactory factory)
{
if (factory == null)
{
Assert.Inconclusive("Expected interpreter is not installed");
}
_factory = factory;
_analyzer = PythonAnalyzer.CreateAsync(factory).WaitAndUnwrapExceptions();
_entries = new Dictionary <string, IPythonProjectEntry>(StringComparer.OrdinalIgnoreCase);
_tasks = new ConcurrentDictionary <IPythonProjectEntry, TaskCompletionSource <CollectingErrorSink> >();
_cachedMembers = new Dictionary <BuiltinTypeId, string[]>();
_root = TestData.GetTempPath();
}
private async Task <PythonAnalyzer> CreateAnalyzer(PythonInitializationOptions.Interpreter interpreter)
{
var factory = ActivateObject <IPythonInterpreterFactory>(interpreter.assembly, interpreter.typeName, interpreter.properties)
?? new AstPythonInterpreterFactory(interpreter.properties);
var interp = factory.CreateInterpreter();
if (interp == null)
{
throw new InvalidOperationException("Failed to create interpreter");
}
LogMessage(MessageType.Info, $"Created {interp.GetType().FullName} instance from {factory.GetType().FullName}");
return(await PythonAnalyzer.CreateAsync(factory, interp));
}
public TestAnalyzer(
IPythonInterpreterFactory factory,
string containerFilePath,
string codeFileBasePath,
Uri executorUri
)
{
_analyzer = PythonAnalyzer.CreateAsync(factory).WaitAndUnwrapExceptions();
_analyzer.Limits = AnalysisLimits.GetStandardLibraryLimits();
_containerFilePath = containerFilePath;
_codeFileBasePath = codeFileBasePath;
_executorUri = executorUri;
_entries = new List <IPythonProjectEntry>();
}
private async Task <PythonAnalyzer> CreateAnalyzer(PythonInitializationOptions.Interpreter interpreter)
{
IPythonInterpreterFactory factory = null;
if (!string.IsNullOrEmpty(interpreter.assembly) && !string.IsNullOrEmpty(interpreter.typeName))
{
try {
var assembly = File.Exists(interpreter.assembly) ? AssemblyName.GetAssemblyName(interpreter.assembly) : new AssemblyName(interpreter.assembly);
var type = Assembly.Load(assembly).GetType(interpreter.typeName, true);
factory = (IPythonInterpreterFactory)Activator.CreateInstance(
type,
BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Instance,
null,
new object[] { interpreter.properties },
CultureInfo.CurrentCulture
);
} catch (Exception ex) {
LogMessage(MessageType.Warning, ex.ToString());
}
}
else
{
factory = new AstPythonInterpreterFactory(interpreter.properties);
}
if (factory == null)
{
Version v;
if (!Version.TryParse(interpreter.version ?? "0.0", out v))
{
v = new Version();
}
factory = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(v);
}
var interp = factory.CreateInterpreter();
if (interp == null)
{
throw new InvalidOperationException("Failed to create interpreter");
}
LogMessage(MessageType.Info, $"Created {interp.GetType().FullName} instance from {factory.GetType().FullName}");
return(await PythonAnalyzer.CreateAsync(factory, interp));
}
private DjangoAnalyzer AnalyzerTest(string path)
{
string djangoDbPath = TestData.GetPath("TestData\\DjangoDB");
Assert.IsTrue(
PythonTypeDatabase.IsDatabaseVersionCurrent(djangoDbPath),
"TestData\\DjangoDB needs updating."
);
var testFact = PythonInterpreterFactoryWithDatabase.CreateFromDatabase(
new Version(2, 7),
TestData.GetPath("CompletionDB"),
djangoDbPath
);
var serviceProvider = PythonToolsTestUtilities.CreateMockServiceProvider();
PythonAnalyzer analyzer = PythonAnalyzer.CreateAsync(testFact).WaitAndUnwrapExceptions();
DjangoAnalyzer djangoAnalyzer = new DjangoAnalyzer();
djangoAnalyzer.Register(analyzer);
analyzer.SetSearchPaths(new[] { path });
List <IPythonProjectEntry> entries = new List <IPythonProjectEntry>();
foreach (string file in Directory.EnumerateFiles(path, "*.py", SearchOption.AllDirectories))
{
var entry = analyzer.AddModule(ModulePath.FromFullPath(file).ModuleName, file);
var parser = Parser.CreateParser(
new FileStream(file, FileMode.Open, FileAccess.Read),
PythonLanguageVersion.V27
);
using (var p = entry.BeginParse()) {
p.Tree = parser.ParseFile();
p.Complete();
}
entries.Add(entry);
}
foreach (var entry in entries)
{
entry.Analyze(CancellationToken.None, false);
}
return(djangoAnalyzer);
}
private DjangoAnalyzer AnalyzerTest(string path)
{
var version = new Version(2, 7);
var testFact = new AstPythonInterpreterFactory(
new InterpreterConfiguration($"AnalysisOnly|{version}", $"Analysis Only {version}", version: version, uiMode: InterpreterUIMode.Normal),
new InterpreterFactoryCreationOptions {
WatchFileSystem = false
}
);
var serviceProvider = PythonToolsTestUtilities.CreateMockServiceProvider();
PythonAnalyzer analyzer = PythonAnalyzer.CreateAsync(testFact).WaitAndUnwrapExceptions();
DjangoAnalyzer djangoAnalyzer = new DjangoAnalyzer();
djangoAnalyzer.Register(analyzer);
analyzer.SetSearchPaths(new[] { path });
List <IPythonProjectEntry> entries = new List <IPythonProjectEntry>();
foreach (string file in Directory.EnumerateFiles(path, "*.py", SearchOption.AllDirectories))
{
var entry = analyzer.AddModule(ModulePath.FromFullPath(file).ModuleName, file);
var parser = Parser.CreateParser(
new FileStream(file, FileMode.Open, FileAccess.Read),
PythonLanguageVersion.V27
);
using (var p = entry.BeginParse()) {
p.Tree = parser.ParseFile();
p.Complete();
}
entries.Add(entry);
}
foreach (var entry in entries)
{
entry.Analyze(CancellationToken.None, false);
}
return(djangoAnalyzer);
}
private async Task <PythonAnalyzer> CreateAnalyzer(PythonInitializationOptions.Interpreter interpreter, CancellationToken token)
{
var factory = ActivateObject <IPythonInterpreterFactory>(interpreter.assembly, interpreter.typeName, interpreter.properties)
?? new AstPythonInterpreterFactory(interpreter.properties);
var interp = factory.CreateInterpreter();
if (interp == null)
{
throw new InvalidOperationException("Failed to create interpreter");
}
LogMessage(MessageType.Info, $"Created {interp.GetType().FullName} instance from {factory.GetType().FullName}");
var analyzer = await PythonAnalyzer.CreateAsync(factory, interp, token);
#if DEBUG
// Make Deque aware of the only thread that should be modifying its state
analyzer.Queue.SynchronizationContext = _queue.SynchronizationContext;
#endif
return(analyzer);
}
private PythonAnalyzer MakeTestAnalyzer()
{
return(PythonAnalyzer.CreateAsync(InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(new Version(2, 7))).Result);
}
static void Main(string[] args)
{
var responseFile = Path.GetFullPath(args.FirstOrDefault() ?? "AnalysisMemoryTester.rsp");
if (!File.Exists(responseFile))
{
if (!string.IsNullOrEmpty(responseFile))
{
Console.WriteLine("Could not open {0}", responseFile);
}
else
{
Console.WriteLine("No response file specified");
}
PrintUsage();
return;
}
var commands = File.ReadAllLines(responseFile)
.Select(line => line.Trim())
.ToList();
Environment.CurrentDirectory = Path.GetDirectoryName(responseFile);
var version = GetFirstCommand(commands, "python\\s+(\\d\\.\\d)", m => {
Version ver;
return(Version.TryParse(m.Groups[1].Value, out ver) ? ver : null);
}, v => v != null) ?? new Version(3, 3);
Console.WriteLine("Using Python Version {0}.{1}", version.Major, version.Minor);
var dbPath = GetFirstCommand(commands, "db\\s+(.+)", m => {
var path = m.Groups[1].Value;
if (!Path.IsPathRooted(path))
{
path = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), path);
}
return(Path.GetFullPath(path));
}, Directory.Exists);
if (dbPath == null)
{
if (!string.IsNullOrEmpty(dbPath))
{
Console.WriteLine("Could not find DB path {0}", dbPath);
}
else
{
Console.WriteLine("No DB path specified");
}
PrintUsage();
return;
}
Console.WriteLine("Using database in {0}", dbPath);
using (var factory = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(version, "Test Factory", dbPath))
using (var analyzer = PythonAnalyzer.CreateAsync(factory).WaitAndUnwrapExceptions()) {
var modules = new Dictionary <string, IPythonProjectEntry>();
foreach (var tuple in SplitCommands(commands))
{
RunCommand(
tuple.Item1,
tuple.Item2,
analyzer,
modules
);
}
}
}
static void Main(string[] args)
{
var responseFile = Path.GetFullPath(args.FirstOrDefault() ?? "AnalysisMemoryTester.rsp");
if (!File.Exists(responseFile))
{
if (!string.IsNullOrEmpty(responseFile))
{
Console.WriteLine("Could not open {0}", responseFile);
}
else
{
Console.WriteLine("No response file specified");
}
PrintUsage();
return;
}
var commands = File.ReadAllLines(responseFile)
.Select(line => line.Trim())
.ToList();
Environment.CurrentDirectory = Path.GetDirectoryName(responseFile);
var interpreter = GetFirstCommand(commands, "python\\s+(\\d\\.\\d)\\s+(.+)", m => m.Groups[1].Value + " " + m.Groups[2].Value, v => v.Length > 4 && File.Exists(v.Substring(4).Trim()));
var version = Version.Parse(interpreter.Substring(0, 3));
interpreter = interpreter.Substring(4).Trim();
Console.WriteLine($"Using Python from {interpreter}");
var logs = GetFirstCommand(commands, "logs\\s+(.+)", m => m.Groups[1].Value, v => PathUtils.IsValidPath(v.Trim()));
if (!string.IsNullOrEmpty(logs))
{
if (!Path.IsPathRooted(logs))
{
logs = Path.GetFullPath(logs);
}
Directory.CreateDirectory(logs);
AnalysisLog.Output = Path.Combine(logs, "Detailed.csv");
AnalysisLog.AsCSV = true;
}
var config = new InterpreterConfiguration(
"Python|" + interpreter,
interpreter,
Path.GetDirectoryName(interpreter),
interpreter,
interpreter,
"PYTHONPATH",
NativeMethods.GetBinaryType(interpreter) == System.Reflection.ProcessorArchitecture.Amd64 ? InterpreterArchitecture.x64 : InterpreterArchitecture.x86,
version
);
var creationOpts = new InterpreterFactoryCreationOptions {
UseExistingCache = false,
WatchFileSystem = false,
TraceLevel = TraceLevel.Verbose
};
if (!string.IsNullOrEmpty(logs))
{
creationOpts.DatabasePath = logs;
}
using (var factory = new Interpreter.Ast.AstPythonInterpreterFactory(config, creationOpts))
using (var analyzer = PythonAnalyzer.CreateAsync(factory).WaitAndUnwrapExceptions()) {
var modules = new Dictionary <string, IPythonProjectEntry>();
var state = new State();
foreach (var tuple in SplitCommands(commands))
{
RunCommand(
tuple.Item1,
tuple.Item2,
analyzer,
modules,
state
);
}
}
}