internal void OnNewAnalyzer(PythonAnalyzer analyzer) {
if (analyzer == null) {
throw new ArgumentNullException("analyzer");
}
_tags.Clear();
_filters.Clear();
foreach (var entry in _hookedEntries) {
entry.OnNewParseTree -= OnNewParseTree;
}
_hookedEntries.Clear();
_templateAnalysis.Clear();
_templateFiles.Clear();
_contextTable = new ConditionalWeakTable<Node, ContextMarker>();
_decoratorTable = new ConditionalWeakTable<Node, DeferredDecorator>();
foreach (var keyValue in _knownTags) {
_tags[keyValue.Key] = new TagInfo(keyValue.Value, null);
}
foreach (var keyValue in _knownFilters) {
_filters[keyValue.Key] = new TagInfo(keyValue.Value, null);
}
HookAnalysis(analyzer);
_analyzer = analyzer;
}
private void SetClassInfo(PythonAnalyzer state) {
for (int value = 0; value < _types.Length; ++value) {
if (_types[value] != null) {
_classInfos[value] = state.GetBuiltinType(_types[value]);
}
}
}
internal ProjectEntry(PythonAnalyzer state, string moduleName, string filePath, IAnalysisCookie cookie) {
_projectState = state;
_moduleName = moduleName ?? "";
_filePath = filePath;
_cookie = cookie;
_myScope = new ModuleInfo(_moduleName, this, state.Interpreter.CreateModuleContext());
_unit = new AnalysisUnit(_tree, _myScope.Scope);
AnalysisLog.NewUnit(_unit);
}
public static KnownTypes CreateDefault(PythonAnalyzer state, PythonTypeDatabase fallbackDb) {
var res = new KnownTypes();
var fallback = fallbackDb.BuiltinModule;
for (int value = 0; value < res._types.Length; ++value) {
res._types[value] = (IPythonType)fallback.GetAnyMember(
((ITypeDatabaseReader)fallbackDb).GetBuiltinTypeName((BuiltinTypeId)value)
);
}
res.SetClassInfo(state);
return res;
}
public TestAnalyzer(
IPythonInterpreterFactory factory,
string containerFilePath,
string codeFileBasePath,
Uri executorUri
) {
_analyzer = PythonAnalyzer.CreateAsync(factory).WaitAndUnwrapExceptions();
_analyzer.Limits = AnalysisLimits.GetStandardLibraryLimits();
_analyzer.Limits.ProcessCustomDecorators = false;
_containerFilePath = containerFilePath;
_codeFileBasePath = codeFileBasePath;
_executorUri = executorUri;
_entries = new List<IPythonProjectEntry>();
}
public static KnownTypes Create(PythonAnalyzer state, PythonTypeDatabase fallbackDb) {
var res = new KnownTypes();
var interpreter = state.Interpreter;
var fallback = fallbackDb.BuiltinModule;
for (int value = 0; value < res._types.Length; ++value) {
try {
res._types[value] = interpreter.GetBuiltinType((BuiltinTypeId)value);
} catch (KeyNotFoundException) {
res._types[value] = (IPythonType)fallback.GetAnyMember(
((ITypeDatabaseReader)fallbackDb).GetBuiltinTypeName((BuiltinTypeId)value)
);
}
}
res.SetClassInfo(state);
return res;
}
public void Initialize(PythonAnalyzer state)
{
}
public ImportStatementWalker(IPythonProjectEntry entry, PythonAnalyzer analyzer) {
_entry = entry;
_analyzer = analyzer;
}
public void Initialize(PythonAnalyzer state) {
if (_state != null) {
_state.AnalysisDirectoriesChanged -= AnalysisDirectoryChanged;
}
_state = state;
SpecializeClrFunctions();
if (_state != null) {
_state.AnalysisDirectoriesChanged += AnalysisDirectoryChanged;
AnalysisDirectoryChanged(_state, EventArgs.Empty);
}
}
public ImportStatementWalker(PythonAst ast, IPythonProjectEntry entry, PythonAnalyzer analyzer) {
_ast = ast;
_entry = entry;
_analyzer = analyzer;
}
private Response Initialize(AP.InitializeRequest request) {
List<AssemblyCatalog> catalogs = new List<AssemblyCatalog>();
HashSet<string> assemblies = new HashSet<string>(request.mefExtensions);
assemblies.Add(typeof(IInterpreterRegistryService).Assembly.Location);
assemblies.Add(GetType().Assembly.Location);
List<string> failures = new List<string>();
string error = null;
foreach (var asm in assemblies) {
try {
var asmCatalog = new AssemblyCatalog(asm);
_catalog.Catalogs.Add(asmCatalog);
} catch (Exception e) {
failures.Add(String.Format("Failed to load {0}: {1}", asm, e));
}
}
if (request.projectFile != null) {
var projectContextProvider = _container.GetExportedValue<OutOfProcProjectContextProvider>();
projectContextProvider.AddContext(
new InMemoryProject(
request.projectFile,
new Dictionary<string, object>() {
{ "InterpreterId", request.interpreterId },
{ "ProjectHome", request.projectHome },
{ "Interpreters",
request.derivedInterpreters.Select(
interp => new Dictionary<string, string>() {
{ "EvaluatedInclude", interp.name },
{ MSBuildConstants.IdKey, interp.id },
{ MSBuildConstants.VersionKey, interp.version },
{ MSBuildConstants.DescriptionKey, interp.description },
{ MSBuildConstants.BaseInterpreterKey, interp.baseInterpreter },
{ MSBuildConstants.InterpreterPathKey, interp.path },
{ MSBuildConstants.WindowsPathKey, interp.windowsPath },
{ MSBuildConstants.LibraryPathKey, interp.libPath },
{ MSBuildConstants.PathEnvVarKey, interp.pathEnvVar },
{ MSBuildConstants.ArchitectureKey, interp.arch }
}
).ToArray()
}
}
)
);
}
var registry = _container.GetExportedValue<IInterpreterRegistryService>();
IPythonInterpreterFactory factory = null;
Version analysisVersion;
if (request.interpreterId.StartsWith("AnalysisOnly;")) {
int versionStart = request.interpreterId.IndexOf(';') + 1;
int versionEnd = request.interpreterId.IndexOf(';', versionStart);
if (Version.TryParse(request.interpreterId.Substring(versionStart, versionEnd - versionStart), out analysisVersion)) {
string[] dbDirs;
if (versionEnd + 1 != request.interpreterId.Length) {
dbDirs = request.interpreterId.Substring(versionEnd + 1).Split(';');
} else {
dbDirs = null;
}
factory = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(analysisVersion, null, dbDirs);
}
} else {
factory = registry.FindInterpreter(request.interpreterId);
}
if (factory == null) {
error = String.Format("No active interpreter found for interpreter ID: {0}", request.interpreterId);
return new AP.InitializeResponse() {
failedLoads = failures.ToArray(),
error = error
};
}
_interpreterFactory = factory;
_allConfigs = registry.Configurations.ToArray();
var interpreter = factory.CreateInterpreter();
if (interpreter != null) {
_pyAnalyzer = PythonAnalyzer.Create(factory, interpreter);
ReloadTask = _pyAnalyzer.ReloadModulesAsync()/*.HandleAllExceptions(_serviceProvider, GetType())*/;
ReloadTask.ContinueWith(_ => ReloadTask = null);
interpreter.ModuleNamesChanged += OnModulesChanged;
}
return new AP.InitializeResponse() {
failedLoads = failures.ToArray(),
error = error
};
}
public SaveLoadResult(PythonAnalyzer analyzer, string dir)
{
Analyzer = analyzer;
_dir = dir;
}
internal MissingImportAnalysis(string name, PythonAnalyzer state, ITrackingSpan span) {
_span = span;
_name = name;
_state = state;
}
internal PythonAnalyzer AnalyzeDir(string dir, PythonLanguageVersion version = PythonLanguageVersion.V27, IEnumerable<string> excludeDirectories = null, CancellationToken? cancel = null) {
List<string> files = new List<string>();
try {
ISet<string> excluded = null;
if (excludeDirectories != null) {
excluded = new HashSet<string>(excludeDirectories, StringComparer.InvariantCultureIgnoreCase);
}
CollectFiles(dir, files, excluded);
} catch (DirectoryNotFoundException) {
return null;
}
List<FileStreamReader> sourceUnits = new List<FileStreamReader>();
foreach (string file in files) {
sourceUnits.Add(
new FileStreamReader(file)
);
}
Stopwatch sw = new Stopwatch();
sw.Start();
long start0 = sw.ElapsedMilliseconds;
// Explicitly specify the builtins name because we are using a 2.7
// interpreter for all versions.
var fact = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(version.ToVersion());
var projectState = new PythonAnalyzer(fact, fact.CreateInterpreter(), "__builtin__");
projectState.ReloadModulesAsync().WaitAndUnwrapExceptions();
projectState.Limits = AnalysisLimits.GetStandardLibraryLimits();
var modules = new List<IPythonProjectEntry>();
foreach (var sourceUnit in sourceUnits) {
try {
modules.Add(projectState.AddModule(
ModulePath.FromFullPath(sourceUnit.Path).ModuleName,
sourceUnit.Path,
null
));
} catch (ArgumentException) {
// Invalid module name, so skip the module
}
}
long start1 = sw.ElapsedMilliseconds;
Trace.TraceInformation("AddSourceUnit: {0} ms", start1 - start0);
var nodes = new List<Microsoft.PythonTools.Parsing.Ast.PythonAst>();
for (int i = 0; i < modules.Count; i++) {
PythonAst ast = null;
try {
var sourceUnit = sourceUnits[i];
ast = Parser.CreateParser(sourceUnit, version).ParseFile();
} catch (Exception) {
}
nodes.Add(ast);
}
long start2 = sw.ElapsedMilliseconds;
Trace.TraceInformation("Parse: {0} ms", start2 - start1);
for (int i = 0; i < modules.Count; i++) {
var ast = nodes[i];
if (ast != null) {
modules[i].UpdateTree(ast, null);
}
}
long start3 = sw.ElapsedMilliseconds;
for (int i = 0; i < modules.Count; i++) {
Trace.TraceInformation("Analyzing {1}: {0} ms", sw.ElapsedMilliseconds - start3, sourceUnits[i].Path);
var ast = nodes[i];
if (ast != null) {
modules[i].Analyze(cancel ?? CancellationToken.None, true);
}
}
if (modules.Count > 0) {
Trace.TraceInformation("Analyzing queue");
modules[0].AnalysisGroup.AnalyzeQueuedEntries(cancel ?? CancellationToken.None);
}
long start4 = sw.ElapsedMilliseconds;
Trace.TraceInformation("Analyze: {0} ms", start4 - start3);
return projectState;
}
public void Initialize(PythonAnalyzer state) {
if (_state != null) {
_state.SearchPathsChanged -= PythonAnalyzer_SearchPathsChanged;
}
_state = state;
if (_state != null) {
_state.SearchPathsChanged += PythonAnalyzer_SearchPathsChanged;
PythonAnalyzer_SearchPathsChanged(_state, EventArgs.Empty);
}
}
internal VsProjectAnalyzer(
IServiceProvider serviceProvider,
IPythonInterpreter interpreter,
IPythonInterpreterFactory factory,
IPythonInterpreterFactory[] allFactories,
bool implicitProject = true
) {
_errorProvider = (ErrorTaskProvider)serviceProvider.GetService(typeof(ErrorTaskProvider));
_commentTaskProvider = (CommentTaskProvider)serviceProvider.GetService(typeof(CommentTaskProvider));
_unresolvedSquiggles = new UnresolvedImportSquiggleProvider(serviceProvider, _errorProvider);
_queue = new ParseQueue(this);
_analysisQueue = new AnalysisQueue(this);
_analysisQueue.AnalysisStarted += AnalysisQueue_AnalysisStarted;
_allFactories = allFactories;
_interpreterFactory = factory;
_implicitProject = implicitProject;
if (interpreter != null) {
_pyAnalyzer = PythonAnalyzer.Create(factory, interpreter);
ReloadTask = _pyAnalyzer.ReloadModulesAsync().HandleAllExceptions(SR.ProductName, GetType());
ReloadTask.ContinueWith(_ => ReloadTask = null);
interpreter.ModuleNamesChanged += OnModulesChanged;
}
_projectFiles = new ConcurrentDictionary<string, IProjectEntry>(StringComparer.OrdinalIgnoreCase);
_pyService = serviceProvider.GetPythonToolsService();
_serviceProvider = serviceProvider;
if (_pyAnalyzer != null) {
_pyAnalyzer.Limits.CrossModule = _pyService.GeneralOptions.CrossModuleAnalysisLimit;
// TODO: Load other limits from options
}
_userCount = 1;
}
public AnalyzerChangingEventArgs(PythonAnalyzer oldAnalyzer, PythonAnalyzer newAnalyzer) {
_old = oldAnalyzer;
_new = newAnalyzer;
}
private static void AnalyzeCode(
out PythonAnalyzer analyzer,
out IPythonProjectEntry entry,
string code,
Version preferredVersion = null,
InterpreterArchitecture preferredArch = null,
string module = "test-module"
) {
var provider = InterpFactory;
var factory = provider.GetInterpreterFactories().OrderByDescending(f => f.Configuration.Version)
.Where(f => preferredVersion == null || f.Configuration.Version == preferredVersion)
.Where(f => preferredArch == null || f.Configuration.Architecture == preferredArch)
.FirstOrDefault();
Assert.IsNotNull(factory, "no factory found");
analyzer = PythonAnalyzer.CreateSynchronously(factory);
var path = Path.Combine(TestData.GetTempPath(randomSubPath: true), module.Replace('.', '\\'));
Directory.CreateDirectory(PathUtils.GetParent(path));
File.WriteAllText(path, code);
entry = analyzer.AddModule(module, path);
PythonAst ast;
using (var p = Parser.CreateParser(new StringReader(code), factory.GetLanguageVersion())) {
ast = p.ParseFile();
entry.UpdateTree(ast, null);
}
entry.Analyze(CancellationToken.None, true);
}
public void SaveStdLib() {
// only run this once...
if (GetType() == typeof(AnalysisTest)) {
var stdLib = AnalyzeStdLib();
string tmpFolder = TestData.GetTempPath("6666d700-a6d8-4e11-8b73-3ba99a61e27b");
new SaveAnalysis().Save(stdLib, tmpFolder);
File.Copy(Path.Combine(PythonInterpreterFactory.GetBaselineDatabasePath(), "__builtin__.idb"), Path.Combine(tmpFolder, "__builtin__.idb"), true);
var newPs = new PythonAnalyzer(new CPythonInterpreter(new TypeDatabase(tmpFolder)), PythonLanguageVersion.V27);
}
}
private void HookAnalysis(PythonAnalyzer analyzer) {
analyzer.SpecializeFunction("django.template.loader", "render_to_string", RenderToStringProcessor, true);
analyzer.SpecializeFunction("django.shortcuts", "render_to_response", RenderToStringProcessor, true);
analyzer.SpecializeFunction("django.shortcuts", "render", RenderProcessor, true);
analyzer.SpecializeFunction("django.contrib.gis.shortcuts", "render_to_kml", RenderToStringProcessor, true);
analyzer.SpecializeFunction("django.contrib.gis.shortcuts", "render_to_kmz", RenderToStringProcessor, true);
analyzer.SpecializeFunction("django.contrib.gis.shortcuts", "render_to_text", RenderToStringProcessor, true);
analyzer.SpecializeFunction("django.template.base.Library", "filter", FilterProcessor, true);
analyzer.SpecializeFunction("django.template.base.Library", "filter_function", FilterProcessor, true);
analyzer.SpecializeFunction("django.template.base.Library", "tag", TagProcessor, true);
analyzer.SpecializeFunction("django.template.base.Library", "tag_function", TagProcessor, true);
analyzer.SpecializeFunction("django.template.base.Library", "assignment_tag", TagProcessor, true);
analyzer.SpecializeFunction("django.template.base.Library", "simple_tag", TagProcessor, true);
analyzer.SpecializeFunction("django.template.base.Parser", "parse", ParseProcessor, true);
analyzer.SpecializeFunction("django.template.base", "import_library", "django.template.base.Library", true);
analyzer.SpecializeFunction("django.template.loader", "get_template", GetTemplateProcessor, true);
analyzer.SpecializeFunction("django.template.context", "Context", ContextClassProcessor, true);
analyzer.SpecializeFunction("django.template", "RequestContext", RequestContextClassProcessor, true);
analyzer.SpecializeFunction("django.template.context", "RequestContext", RequestContextClassProcessor, true);
analyzer.SpecializeFunction("django.template.base.Template", "render", TemplateRenderProcessor, true);
// View specializers
analyzer.SpecializeFunction("django.views.generic.detail.DetailView", "as_view", DetailViewProcessor, true);
analyzer.SpecializeFunction("django.views.generic.DetailView", "as_view", DetailViewProcessor, true);
analyzer.SpecializeFunction("django.views.generic.list.ListView", "as_view", ListViewProcessor, true);
analyzer.SpecializeFunction("django.views.generic.ListView", "as_view", ListViewProcessor, true);
// Urls specializers
analyzer.SpecializeFunction("django.conf.urls", "url", UrlProcessor, true);
}
public void Register(PythonAnalyzer analyzer) {
_analyzer = analyzer;
}
private IPythonProjectEntry AddModule(PythonAnalyzer analyzer, string moduleName, string code, string moduleFile = null) {
using (var source = new StringReader(code)) {
var entry = analyzer.AddModule(
moduleName,
TestData.GetPath("Fob\\" + (moduleFile ?? moduleName.Replace('.', '\\') + ".py"))
);
using (var parser = Parser.CreateParser(source, PythonLanguageVersion.V27, new ParserOptions() { BindReferences = true })) {
entry.UpdateTree(parser.ParseFile(), null);
}
return entry;
}
}
