public ModuleDatabase(IServiceManager sm, string cacheFolder = null)
{
_services = sm;
_log = _services.GetService <ILogger>();
_fs = _services.GetService <IFileSystem>();
_defaultCachingLevel = AnalysisCachingLevel.Library;
var cfs = _services.GetService <ICacheFolderService>();
CacheFolder = cacheFolder ?? Path.Combine(cfs.CacheFolder, $"{CacheFolderBaseName}{DatabaseFormatVersion}");
_cacheWriter = new CacheWriter(_services.GetService <IPythonAnalyzer>(), _fs, _log, CacheFolder);
sm.AddService(this);
}
public static ModuleModel FromAnalysis(IDocumentAnalysis analysis, IServiceContainer services, AnalysisCachingLevel options)
{
var uniqueId = analysis.Document.GetUniqueId(services, options);
if (uniqueId == null)
{
// Caching level setting does not permit this module to be persisted.
return(null);
}
var variables = new Dictionary <string, VariableModel>();
var functions = new Dictionary <string, FunctionModel>();
var classes = new Dictionary <string, ClassModel>();
var typeVars = new Dictionary <string, TypeVarModel>();
var namedTuples = new Dictionary <string, NamedTupleModel>();
// Go directly through variables which names are listed in GetMemberNames
// as well as variables that are declarations.
var exportedNames = new HashSet <string>(analysis.Document.GetMemberNames());
foreach (var v in analysis.GlobalScope.Variables
.Where(v => exportedNames.Contains(v.Name) ||
v.Source == VariableSource.Declaration ||
v.Source == VariableSource.Builtin ||
v.Source == VariableSource.Generic))
{
if (v.Value is IGenericTypeParameter && !typeVars.ContainsKey(v.Name))
{
typeVars[v.Name] = TypeVarModel.FromGeneric(v);
continue;
}
switch (v.Value)
{
case ITypingNamedTupleType nt:
namedTuples[nt.Name] = new NamedTupleModel(nt);
continue;
case IPythonFunctionType ft when ft.IsLambda():
// No need to persist lambdas.
continue;
case IPythonFunctionType ft when v.Name != ft.Name:
// Variable assigned to type info of the function like
// def func(): ...
// x = type(func)
break;
case IPythonFunctionType ft:
var fm = GetFunctionModel(analysis, v, ft);
if (fm != null && !functions.ContainsKey(ft.Name))
{
functions[ft.Name] = fm;
continue;
}
break;
case IPythonClassType cls when v.Name != cls.Name:
// Variable assigned to type info of the class.
break;
case IPythonClassType cls
when cls.DeclaringModule.Equals(analysis.Document) || cls.DeclaringModule.Equals(analysis.Document.Stub):
if (!classes.ContainsKey(cls.Name))
{
classes[cls.Name] = new ClassModel(cls);
continue;
}
break;
}
// Do not re-declare classes and functions as variables in the model.
if (!variables.ContainsKey(v.Name))
{
variables[v.Name] = VariableModel.FromVariable(v);
}
}
// Take dependencies from imports. If module has stub we should also take
// dependencies from there since persistent state is based on types that
// are combination of stub and the module. Sometimes stub may import more
// and we must make sure dependencies are restored before the module.
var primaryDependencyWalker = new DependencyWalker(analysis.Ast);
var stubDependencyWalker = analysis.Document.Stub != null ? new DependencyWalker(analysis.Document.Stub.Analysis.Ast) : null;
var stubImports = stubDependencyWalker?.Imports ?? Enumerable.Empty <ImportModel>();
var stubFromImports = stubDependencyWalker?.FromImports ?? Enumerable.Empty <FromImportModel>();
return(new ModuleModel {
Id = uniqueId.GetStableHash(),
UniqueId = uniqueId,
Name = analysis.Document.Name,
QualifiedName = analysis.Document.QualifiedName,
Documentation = analysis.Document.Documentation,
Functions = functions.Values.ToArray(),
Variables = variables.Values.ToArray(),
Classes = classes.Values.ToArray(),
TypeVars = typeVars.Values.ToArray(),
NamedTuples = namedTuples.Values.ToArray(),
NewLines = analysis.Ast.NewLineLocations.Select(l => new NewLineModel {
EndIndex = l.EndIndex,
Kind = l.Kind
}).ToArray(),
FileSize = analysis.Ast.EndIndex,
Imports = primaryDependencyWalker.Imports.ToArray(),
FromImports = primaryDependencyWalker.FromImports.ToArray(),
StubImports = stubImports.ToArray(),
StubFromImports = stubFromImports.ToArray()
});
}
public static string GetUniqueId(string moduleName, string filePath, ModuleType moduleType, IServiceContainer services, AnalysisCachingLevel cachingLevel)
{
if (cachingLevel == AnalysisCachingLevel.None)
{
return(null);
}
if (moduleType == ModuleType.User)
{
// Only for tests.
return($"{moduleName}");
}
var interpreter = services.GetService <IPythonInterpreter>();
var fs = services.GetService <IFileSystem>();
var moduleResolution = interpreter.ModuleResolution;
var modulePathType = GetModulePathType(filePath, moduleResolution.LibraryPaths, fs);
switch (modulePathType)
{
case PythonLibraryPathType.Site when cachingLevel < AnalysisCachingLevel.Library:
return(null);
case PythonLibraryPathType.StdLib when cachingLevel < AnalysisCachingLevel.System:
return(null);
}
if (!string.IsNullOrEmpty(filePath) && modulePathType == PythonLibraryPathType.Site)
{
// Module can be a submodule of a versioned package. In this case we want to use
// version of the enclosing package so we have to look up the chain of folders.
var moduleRootName = moduleName.Split('.')[0];
var moduleFilesFolder = Path.GetDirectoryName(filePath);
var installationFolder = Path.GetDirectoryName(moduleFilesFolder);
var versionFolder = installationFolder;
while (!string.IsNullOrEmpty(versionFolder))
{
// If module is in site-packages and is versioned, then unique id = name + version + interpreter version.
// Example: 'requests' and 'requests-2.21.0.dist-info'.
// TODO: for egg (https://github.com/microsoft/python-language-server/issues/196), consider *.egg-info
var folders = fs.GetFileSystemEntries(versionFolder, "*-*.dist-info", SearchOption.TopDirectoryOnly)
.Select(Path.GetFileName)
.Where(n => n.StartsWith(moduleRootName, StringComparison.OrdinalIgnoreCase)) // Module name can be capitalized differently.
.ToArray();
if (folders.Length == 1)
{
var fileName = Path.GetFileNameWithoutExtension(folders[0]);
var dash = fileName.IndexOf('-');
return($"{moduleName}({fileName.Substring(dash + 1)})");
}
// Move up if nothing is found.
versionFolder = Path.GetDirectoryName(versionFolder);
}
}
var config = interpreter.Configuration;
if (moduleType.IsCompiled() || string.IsNullOrEmpty(filePath) || modulePathType == PythonLibraryPathType.StdLib)
{
// If module is a standard library, unique id is its name + interpreter version.
return($"{moduleName}({config.Version.Major}.{config.Version.Minor})");
}
var parent = moduleResolution.CurrentPathResolver.GetModuleParentFromModuleName(moduleName);
var hash = HashModuleFileSizes(parent);
// If all else fails, hash modules file sizes.
return($"{moduleName}.{(ulong)hash}");
}
public static string GetUniqueId(this IPythonModule module, IServiceContainer services, AnalysisCachingLevel cachingLevel) => GetUniqueId(module.Name, module.FilePath, module.ModuleType, services, cachingLevel);
public static string GetUniqueId(this IPythonModule module, IServiceContainer services, AnalysisCachingLevel cachingLevel = AnalysisCachingLevel.Library)
{
// If module is a standalone stub, permit it. Otherwise redirect to the main module
// since during stub merge types from stub normally become part of the primary module.
if (module.ModuleType == ModuleType.Stub && module.PrimaryModule != null)
{
module = module.PrimaryModule;
}
return(GetUniqueId(module.Name, module.FilePath, module.ModuleType, services, cachingLevel));
}
/// <summary>
/// Constructs module persistent model from analysis.
/// </summary>
public static ModuleModel FromAnalysis(IDocumentAnalysis analysis, IServiceContainer services, AnalysisCachingLevel options)
{
var uniqueId = analysis.Document.GetUniqueId(services, options);
if (uniqueId == null)
{
// Caching level setting does not permit this module to be persisted.
return(null);
}
var variables = new Dictionary <string, VariableModel>();
var functions = new Dictionary <string, FunctionModel>();
var classes = new Dictionary <string, ClassModel>();
var typeVars = new Dictionary <string, TypeVarModel>();
var namedTuples = new Dictionary <string, NamedTupleModel>();
//var subModules = new Dictionary<string, SubmoduleModel>();
foreach (var v in analysis.Document.GetMemberNames()
.Select(x => analysis.GlobalScope.Variables[x]).ExcludeDefault())
{
if (v.Value is IGenericTypeParameter && !typeVars.ContainsKey(v.Name))
{
typeVars[v.Name] = TypeVarModel.FromGeneric(v, services);
continue;
}
switch (v.Value)
{
case ITypingNamedTupleType nt:
namedTuples[v.Name] = new NamedTupleModel(nt, services);
continue;
case IPythonFunctionType ft when ft.IsLambda():
// No need to persist lambdas.
continue;
case IPythonFunctionType ft when v.Name != ft.Name:
// Variable assigned to type info of the function like
// def func(): ...
// x = type(func)
break;
case IPythonFunctionType ft:
var fm = GetFunctionModel(analysis, v, ft, services);
if (fm != null && !functions.ContainsKey(ft.Name))
{
functions[ft.Name] = fm;
continue;
}
break;
case IPythonClassType cls when v.Name != cls.Name:
// Variable assigned to type info of the class.
break;
case IPythonClassType cls
when cls.DeclaringModule.Equals(analysis.Document) || cls.DeclaringModule.Equals(analysis.Document.Stub):
if (!classes.ContainsKey(cls.Name))
{
classes[cls.Name] = new ClassModel(cls, services);
continue;
}
break;
}
// Do not re-declare classes and functions as variables in the model.
if (!variables.ContainsKey(v.Name))
{
variables[v.Name] = VariableModel.FromVariable(v, services);
}
}
return(new ModuleModel {
Id = uniqueId.GetStableHash(),
UniqueId = uniqueId,
Name = analysis.Document.Name,
QualifiedName = analysis.Document.QualifiedName,
FilePath = analysis.Document.FilePath,
Documentation = analysis.Document.Documentation,
Functions = functions.Values.ToArray(),
Variables = variables.Values.ToArray(),
Classes = classes.Values.ToArray(),
TypeVars = typeVars.Values.ToArray(),
NamedTuples = namedTuples.Values.ToArray(),
//SubModules = subModules.Values.ToArray(),
NewLines = analysis.Ast.NewLineLocations.Select(l => new NewLineModel {
EndIndex = l.EndIndex,
Kind = l.Kind
}).ToArray(),
FileSize = analysis.Ast.EndIndex
});
}