private static void FullStdLibTest(PythonVersion v)
{
v.AssertInstalled();
var factory = new AstPythonInterpreterFactory(v.Configuration, null);
var modules = ModulePath.GetModulesInLib(v.PrefixPath).ToList();
bool anySuccess = false;
bool anyExtensionSuccess = false, anyExtensionSeen = false;
using (var analyzer = new PythonAnalysis(factory)) {
foreach (var modName in modules)
{
anyExtensionSeen |= modName.IsNativeExtension;
var mod = analyzer.Analyzer.Interpreter.ImportModule(modName.ModuleName);
if (mod == null)
{
Trace.TraceWarning("failed to import {0} from {1}".FormatInvariant(modName.ModuleName, modName.SourceFile));
}
else
{
anySuccess = true;
anyExtensionSuccess |= modName.IsNativeExtension;
mod.GetMemberNames(analyzer.ModuleContext).ToList();
}
}
}
Assert.IsTrue(anySuccess, "failed to import any modules at all");
Assert.IsTrue(anyExtensionSuccess || !anyExtensionSeen, "failed to import all extension modules");
}
/// <summary>
/// Determines whether the interpreter factory contains the specified
/// modules.
/// </summary>
/// <returns>The names of the modules that were found.</returns>
public static async Task <bool> HasModuleAsync(this IPythonInterpreterFactory factory, string moduleName, IInterpreterOptionsService interpreterOptions = null)
{
if (interpreterOptions != null)
{
foreach (var pm in interpreterOptions.GetPackageManagers(factory))
{
if ((await pm.GetInstalledPackageAsync(new PackageSpec(moduleName), CancellationToken.None)).IsValid)
{
return(true);
}
}
}
return(await Task.Run(() => {
foreach (var mp in ModulePath.GetModulesInLib(factory.Configuration))
{
if (mp.ModuleName == moduleName)
{
return true;
}
}
return false;
}));
}
private static void FullStdLibTest(PythonVersion v)
{
v.AssertInstalled();
var modules = ModulePath.GetModulesInLib(v.PrefixPath).ToList();
var paths = modules.Select(m => m.LibraryPath).Distinct().ToArray();
bool anySuccess = false;
using (var analyzer = new PythonAnalysis(v.Version)) {
analyzer.SetSearchPaths(paths);
foreach (var modName in modules)
{
if (modName.IsCompiled || modName.IsNativeExtension)
{
continue;
}
var mod = analyzer.Analyzer.Interpreter.ImportModule(modName.ModuleName);
if (mod == null)
{
Trace.TraceWarning("failed to import {0} from {1}".FormatInvariant(modName.ModuleName, modName.SourceFile));
}
else
{
anySuccess = true;
mod.GetMemberNames(analyzer.ModuleContext).ToList();
}
}
}
Assert.IsTrue(anySuccess, "failed to import any modules at all");
}
/// <summary>
/// Determines whether the interpreter factory contains the specified
/// modules.
/// </summary>
/// <returns>The names of the modules that were found.</returns>
public static async Task <bool> HasModuleAsync(this IPythonInterpreterFactory factory, string moduleName, IInterpreterOptionsService interpreterOptions = null)
{
if (interpreterOptions != null)
{
foreach (var pm in interpreterOptions.GetPackageManagers(factory))
{
if ((await pm.GetInstalledPackageAsync(new PackageSpec(moduleName), CancellationToken.None)).IsValid)
{
return(true);
}
}
}
return(await Task.Run(() => {
var configuration = factory.Configuration;
var prefixPath = configuration.GetPrefixPath();
var libraryPath = !string.IsNullOrEmpty(configuration.LibraryPath) ? configuration.LibraryPath : Path.Combine(prefixPath, "Lib");
var sitePackagesPath = !string.IsNullOrEmpty(configuration.SitePackagesPath) ? configuration.SitePackagesPath : Path.Combine(libraryPath, "site-packages");
var requiresInitPyFiles = ModulePath.PythonVersionRequiresInitPyFiles(configuration.Version);
foreach (var mp in ModulePath.GetModulesInLib(libraryPath, sitePackagesPath, requiresInitPyFiles))
{
if (mp.ModuleName == moduleName)
{
return true;
}
}
return false;
}));
}
private string[] GetMissingModules(HashSet <string> existingDatabase)
{
return(ModulePath.GetModulesInLib(this)
.Select(mp => mp.ModuleName)
.Concat(RequiredBuiltinModules)
.Where(name => !existingDatabase.Contains(name))
.OrderBy(name => name, StringComparer.InvariantCultureIgnoreCase)
.ToArray());
}
/// <summary>
/// Determines whether the interpreter factory contains the specified
/// modules.
/// </summary>
/// <returns>The names of the modules that were found.</returns>
public static async Task <bool> HasModuleAsync(this IPythonInterpreterFactory factory, string moduleName, IInterpreterOptionsService interpreterOptions = null)
{
if (interpreterOptions != null)
{
foreach (var pm in interpreterOptions.GetPackageManagers(factory))
{
if ((await pm.GetInstalledPackageAsync(new PackageSpec(moduleName), CancellationToken.None)).IsValid)
{
return(true);
}
}
}
var withDb = factory as LegacyDB.PythonInterpreterFactoryWithDatabase;
if (withDb != null && withDb.IsCurrent)
{
var db = withDb.GetCurrentDatabase();
if (db.GetModule(moduleName) != null)
{
return(true);
}
// Always stop searching after this step
return(false);
}
if (withDb != null)
{
try {
var paths = await LegacyDB.PythonTypeDatabase.GetDatabaseSearchPathsAsync(withDb);
if (LegacyDB.PythonTypeDatabase.GetDatabaseExpectedModules(withDb.Configuration.Version, paths)
.SelectMany()
.Any(g => g.ModuleName == moduleName))
{
return(true);
}
} catch (InvalidOperationException) {
}
}
return(await Task.Run(() => {
foreach (var mp in ModulePath.GetModulesInLib(factory.Configuration))
{
if (mp.ModuleName == moduleName)
{
return true;
}
}
return false;
}));
}
/// <summary>
/// Determines whether the interpreter factory contains the specified
/// modules.
/// </summary>
/// <returns>The names of the modules that were found.</returns>
internal static HashSet <string> FindModules(this IPythonInterpreterFactory factory, params string[] moduleNames)
{
var expected = new HashSet <string>(moduleNames);
var result = new HashSet <string>();
foreach (var mp in ModulePath.GetModulesInLib(factory))
{
if (expected.Count == 0)
{
break;
}
if (expected.Remove(mp.ModuleName))
{
result.Add(mp.ModuleName);
}
}
return(result);
}
/// <summary>
/// Determines whether the interpreter factory contains the specified
/// modules.
/// </summary>
/// <returns>The names of the modules that were found.</returns>
public static async Task <HashSet <string> > FindModulesAsync(this IPythonInterpreterFactory factory, params string[] moduleNames)
{
var withDb = factory as PythonInterpreterFactoryWithDatabase;
if (withDb != null && withDb.IsCurrent)
{
var db = withDb.GetCurrentDatabase();
var set = new HashSet <string>(moduleNames.Where(m => db.GetModule(m) != null));
return(set);
}
var expected = new HashSet <string>(moduleNames);
if (withDb != null)
{
var paths = PythonTypeDatabase.GetCachedDatabaseSearchPaths(withDb.DatabasePath) ??
await PythonTypeDatabase.GetUncachedDatabaseSearchPathsAsync(withDb.Configuration.InterpreterPath).ConfigureAwait(false);
var db = PythonTypeDatabase.GetDatabaseExpectedModules(withDb.Configuration.Version, paths)
.SelectMany()
.Select(g => g.ModuleName);
expected.IntersectWith(db);
return(expected);
}
return(await Task.Run(() => {
var result = new HashSet <string>();
foreach (var mp in ModulePath.GetModulesInLib(factory))
{
if (expected.Count == 0)
{
break;
}
if (expected.Remove(mp.ModuleName))
{
result.Add(mp.ModuleName);
}
}
return result;
}));
}
private async Task FullStdLibTest(PythonVersion v, params string[] skipModules)
{
v.AssertInstalled();
var factory = new AstPythonInterpreterFactory(v.Configuration, new InterpreterFactoryCreationOptions {
DatabasePath = TestData.GetTempPath(),
UseExistingCache = false
});
var modules = ModulePath.GetModulesInLib(v.PrefixPath).ToList();
var skip = new HashSet <string>(skipModules);
skip.UnionWith(new[] {
"matplotlib.backends._backend_gdk",
"matplotlib.backends._backend_gtkagg",
"matplotlib.backends._gtkagg",
});
bool anySuccess = false;
bool anyExtensionSuccess = false, anyExtensionSeen = false;
bool anyParseError = false;
using (var analyzer = new PythonAnalysis(factory)) {
try {
var tasks = new List <Task <Tuple <ModulePath, IPythonModule> > >();
var interp = (AstPythonInterpreter)analyzer.Analyzer.Interpreter;
foreach (var m in skip)
{
interp.AddUnimportableModule(m);
}
foreach (var r in modules
.Where(m => !skip.Contains(m.ModuleName))
.GroupBy(m => {
int i = m.FullName.IndexOf('.');
return(i <= 0 ? m.FullName : m.FullName.Remove(i));
})
.AsParallel()
.SelectMany(g => g.Select(m => Tuple.Create(m, interp.ImportModule(m.ModuleName))))
)
{
var modName = r.Item1;
var mod = r.Item2;
anyExtensionSeen |= modName.IsNativeExtension;
if (mod == null)
{
Trace.TraceWarning("failed to import {0} from {1}", modName.ModuleName, modName.SourceFile);
}
else if (mod is AstScrapedPythonModule smod)
{
if (smod.ParseErrors?.Any() ?? false)
{
anyParseError = true;
Trace.TraceError("Parse errors in {0}", modName.SourceFile);
foreach (var e in smod.ParseErrors)
{
Trace.TraceError(e);
}
}
else
{
anySuccess = true;
anyExtensionSuccess |= modName.IsNativeExtension;
mod.GetMemberNames(analyzer.ModuleContext).ToList();
}
}
else if (mod is AstPythonModule)
{
// pass
}
else
{
Trace.TraceError("imported {0} as type {1}", modName.ModuleName, mod.GetType().FullName);
}
}
} finally {
_analysisLog = analyzer.GetLogContent(CultureInfo.InvariantCulture);
}
}
Assert.IsTrue(anySuccess, "failed to import any modules at all");
Assert.IsTrue(anyExtensionSuccess || !anyExtensionSeen, "failed to import all extension modules");
Assert.IsFalse(anyParseError, "parse errors occurred");
}
private async Task FullStdLibTest(InterpreterConfiguration configuration, params string[] skipModules)
{
configuration.AssertInstalled();
var moduleUri = TestData.GetDefaultModuleUri();
var moduleDirectory = Path.GetDirectoryName(moduleUri.LocalPath);
var services = await CreateServicesAsync(moduleDirectory, configuration);
var interpreter = services.GetService <IPythonInterpreter>();
var modules = ModulePath.GetModulesInLib(configuration.LibraryPath, configuration.SitePackagesPath).ToList();
var skip = new HashSet <string>(skipModules);
skip.UnionWith(new[] {
@"matplotlib.backends._backend_gdk",
@"matplotlib.backends._backend_gtkagg",
@"matplotlib.backends._gtkagg",
"test.test_pep3131",
"test.test_unicode_identifiers",
"test.test_super" // nonlocal syntax error
});
skip.UnionWith(modules.Select(m => m.FullName)
.Where(n => n.StartsWith(@"test.badsyntax") || n.StartsWith("test.bad_coding")));
var anySuccess = false;
var anyExtensionSuccess = false;
var anyExtensionSeen = false;
var anyParseError = false;
foreach (var m in skip)
{
((MainModuleResolution)interpreter.ModuleResolution).AddUnimportableModule(m);
}
var set = modules
.Where(m => !skip.Contains(m.ModuleName))
.GroupBy(m => {
var i = m.FullName.IndexOf('.');
return(i <= 0 ? m.FullName : m.FullName.Remove(i));
})
.SelectMany(g => g.Select(m => Tuple.Create(m, m.ModuleName)))
.ToArray();
set = set.Where(x => x.Item2 != null && x.Item2.Contains("grammar")).ToArray();
var sb = new StringBuilder();
foreach (var r in set)
{
var module = interpreter.ModuleResolution.GetOrLoadModule(r.Item2);
if (module != null)
{
sb.AppendLine($"import {module.Name}");
}
}
await GetAnalysisAsync(sb.ToString(), services);
foreach (var r in set)
{
var modName = r.Item1;
var mod = interpreter.ModuleResolution.GetOrLoadModule(r.Item2);
anyExtensionSeen |= modName.IsNativeExtension;
switch (mod)
{
case null:
Trace.TraceWarning("failed to import {0} from {1}", modName.ModuleName, modName.SourceFile);
break;
case CompiledPythonModule compiledPythonModule:
var errors = compiledPythonModule.GetParseErrors().ToArray();
if (errors.Any())
{
anyParseError = true;
Trace.TraceError("Parse errors in {0}", modName.SourceFile);
foreach (var e in errors)
{
Trace.TraceError(e.Message);
}
}
else
{
anySuccess = true;
anyExtensionSuccess |= modName.IsNativeExtension;
}
break;
case IPythonModule _: {
var filteredErrors = ((IDocument)mod).GetParseErrors().Where(e => !e.Message.Contains("encoding problem")).ToArray();
if (filteredErrors.Any())
{
// Do not fail due to errors in installed packages
if (!mod.FilePath.Contains("site-packages"))
{
anyParseError = true;
}
Trace.TraceError("Parse errors in {0}", modName.SourceFile);
foreach (var e in filteredErrors)
{
Trace.TraceError(e.Message);
}
}
else
{
anySuccess = true;
anyExtensionSuccess |= modName.IsNativeExtension;
}
break;
}
}
}
Assert.IsTrue(anySuccess, "failed to import any modules at all");
Assert.IsTrue(anyExtensionSuccess || !anyExtensionSeen, "failed to import all extension modules");
Assert.IsFalse(anyParseError, "parse errors occurred");
}
/// <summary>
/// Determines whether the interpreter factory contains the specified
/// modules.
/// </summary>
/// <returns>The names of the modules that were found.</returns>
public static async Task <HashSet <string> > FindModulesAsync(this IPythonInterpreterFactory factory, params string[] moduleNames)
{
var finding = new HashSet <string>(moduleNames);
var found = new HashSet <string>();
var withPackages = factory.PackageManager;
if (withPackages != null)
{
foreach (var m in finding)
{
if ((await withPackages.GetInstalledPackageAsync(new PackageSpec(m), CancellationToken.None)).IsValid)
{
found.Add(m);
}
}
finding.ExceptWith(found);
if (!finding.Any())
{
// Found all of them, so stop searching
return(found);
}
}
var withDb = factory as PythonInterpreterFactoryWithDatabase;
if (withDb != null && withDb.IsCurrent)
{
var db = withDb.GetCurrentDatabase();
found.UnionWith(finding.Where(m => db.GetModule(m) != null));
// Always stop searching after this step
return(found);
}
if (withDb != null)
{
try {
var paths = await PythonTypeDatabase.GetDatabaseSearchPathsAsync(withDb);
found.UnionWith(PythonTypeDatabase.GetDatabaseExpectedModules(withDb.Configuration.Version, paths)
.SelectMany()
.Select(g => g.ModuleName)
.Where(m => finding.Contains(m)));
} catch (InvalidOperationException) {
}
finding.ExceptWith(found);
if (!finding.Any())
{
// Found all of them, so stop searching
return(found);
}
}
return(await Task.Run(() => {
foreach (var mp in ModulePath.GetModulesInLib(factory.Configuration))
{
if (finding.Remove(mp.ModuleName))
{
found.Add(mp.ModuleName);
}
if (!finding.Any())
{
break;
}
}
return found;
}));
}
/// <summary>
/// Determines whether the interpreter factory contains the specified
/// modules.
/// </summary>
/// <returns>The names of the modules that were found.</returns>
public static async Task <HashSet <string> > FindModulesAsync(this IPythonInterpreterFactory factory, params string[] moduleNames)
{
var withPackages = factory as IPackageManager;
if (withPackages != null)
{
var res = new HashSet <string>();
foreach (var m in moduleNames)
{
if ((await withPackages.GetInstalledPackageAsync(new PackageSpec(m), CancellationToken.None)).IsValid)
{
res.Add(m);
}
}
if (res.SetEquals(moduleNames))
{
return(res);
}
}
var withDb = factory as PythonInterpreterFactoryWithDatabase;
if (withDb != null && withDb.IsCurrent)
{
var db = withDb.GetCurrentDatabase();
var set = new HashSet <string>(moduleNames.Where(m => db.GetModule(m) != null));
return(set);
}
var expected = new HashSet <string>(moduleNames);
if (withDb != null)
{
try {
var paths = PythonTypeDatabase.GetCachedDatabaseSearchPaths(withDb.DatabasePath) ??
await PythonTypeDatabase.GetUncachedDatabaseSearchPathsAsync(withDb.Configuration.InterpreterPath).ConfigureAwait(false);
var db = PythonTypeDatabase.GetDatabaseExpectedModules(withDb.Configuration.Version, paths)
.SelectMany()
.Select(g => g.ModuleName);
expected.IntersectWith(db);
return(expected);
} catch (InvalidOperationException) {
}
}
return(await Task.Run(() => {
var result = new HashSet <string>();
foreach (var mp in ModulePath.GetModulesInLib(factory.Configuration))
{
if (expected.Count == 0)
{
break;
}
if (expected.Remove(mp.ModuleName))
{
result.Add(mp.ModuleName);
}
}
return result;
}));
}