/// <summary>
/// Gets the set of search paths that were last saved for a database.
/// </summary>
/// <param name="databasePath">Path containing the database.</param>
/// <returns>The cached list of search paths.</returns>
/// <remarks>Added in 2.2</remarks>
public static List <PythonLibraryPath> GetCachedDatabaseSearchPaths(string databasePath)
{
var cacheFile = Path.Combine(databasePath, "database.path");
if (!File.Exists(cacheFile))
{
return(null);
}
if (!IsDatabaseVersionCurrent(databasePath))
{
// Cache file with no database is only valid for one hour
try {
var time = File.GetLastWriteTimeUtc(cacheFile);
if (time.AddHours(1) < DateTime.UtcNow)
{
File.Delete(cacheFile);
return(null);
}
} catch (IOException) {
return(null);
}
}
try {
var result = new List <PythonLibraryPath>();
using (var file = File.OpenText(cacheFile)) {
string line;
while ((line = file.ReadLine()) != null)
{
try {
result.Add(PythonLibraryPath.Parse(line));
} catch (FormatException) {
Debug.Fail("Invalid format: " + line);
}
}
}
return(result);
} catch (IOException) {
return(null);
}
}
public async Task GetVirtualEnvDatabasePaths()
{
var version = PythonPaths.Python27 ?? PythonPaths.Python27_x64;
version.AssertInstalled();
var env = Path.Combine(TestData.GetTempPath(), "env");
using (var p = ProcessOutput.RunHiddenAndCapture(version.InterpreterPath, "-m", "virtualenv", env)) {
if ((await p) != 0)
{
Assert.Fail("Could not create virtualenv{0}{1}{0}{2}",
Environment.NewLine,
string.Join(Environment.NewLine, p.StandardOutputLines),
string.Join(Environment.NewLine, p.StandardErrorLines)
);
return;
}
}
var interpreter = Path.Combine(env, "Scripts", "python.exe");
var paths = await PythonLibraryPath.GetUncachedDatabaseSearchPathsAsync(interpreter);
AssertUtil.ContainsAtLeast(
paths.Where(p => p.IsStandardLibrary).Select(p => p.Path),
new[] {
Path.Combine(env, "Scripts").ToLowerInvariant(),
PathUtils.TrimEndSeparator(env.ToLowerInvariant()),
PathUtils.TrimEndSeparator(version.PrefixPath.ToLowerInvariant()),
Path.Combine(version.PrefixPath, "Lib").ToLowerInvariant()
}
);
AssertUtil.ContainsAtLeast(
paths.Where(p => !p.IsStandardLibrary).Select(p => p.Path),
new[] {
Path.Combine(env, "Lib", "site-packages").ToLowerInvariant()
}
);
AssertUtil.DoesntContain(
paths.Select(p => p.Path),
Path.Combine(version.PrefixPath, "Lib", "site-packages").ToLowerInvariant()
);
}
public void NormalizeUser()
{
var appPath = TestData.GetTestSpecificPath("app.py");
var root = Path.GetDirectoryName(appPath);
var src = Path.Combine(root, "src");
var fromUser = new[] {
"./src/",
};
var(interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(root, _fs, Array.Empty <PythonLibraryPath>(), fromUser);
interpreterPaths.Should().BeEmpty();
userPaths.Should().BeEquivalentToWithStrictOrdering(new[] {
new PythonLibraryPath(src, PythonLibraryPathType.Unspecified),
});
}
private async Task <ImmutableArray <PythonLibraryPath> > GetInterpreterSearchPathsAsync(CancellationToken cancellationToken = default)
{
if (!FileSystem.FileExists(Configuration.InterpreterPath))
{
Log?.Log(TraceEventType.Warning, "Interpreter does not exist:", Configuration.InterpreterPath);
_ui?.ShowMessageAsync(Resources.InterpreterNotFound, TraceEventType.Error);
return(ImmutableArray <PythonLibraryPath> .Empty);
}
Log?.Log(TraceEventType.Information, "GetCurrentSearchPaths", Configuration.InterpreterPath);
try {
var fs = Services.GetService <IFileSystem>();
var ps = Services.GetService <IProcessServices>();
return(await PythonLibraryPath.GetSearchPathsAsync(Configuration, fs, ps, cancellationToken));
} catch (InvalidOperationException ex) {
Log?.Log(TraceEventType.Warning, "Exception getting search paths", ex);
_ui?.ShowMessageAsync(Resources.ExceptionGettingSearchPaths, TraceEventType.Error);
return(ImmutableArray <PythonLibraryPath> .Empty);
}
}
protected virtual async Task <IReadOnlyList <string> > GetCurrentSearchPathsAsync()
{
if (Configuration.SearchPaths.Any())
{
return(Configuration.SearchPaths);
}
if (!File.Exists(Configuration.InterpreterPath))
{
return(Array.Empty <string>());
}
try {
var paths = await PythonLibraryPath.GetDatabaseSearchPathsAsync(Configuration, _searchPathCachePath).ConfigureAwait(false);
return(paths.MaybeEnumerate().Select(p => p.Path).ToArray());
} catch (InvalidOperationException) {
return(Array.Empty <string>());
}
}
public async Task GetSearchPaths() {
Python.AssertInstalled();
var paths = await PythonTypeDatabase.GetUncachedDatabaseSearchPathsAsync(Python.InterpreterPath);
Console.WriteLine("Paths for {0}", Python.InterpreterPath);
foreach (var path in paths) {
Console.WriteLine("{0} {1}", path.Path, path.IsStandardLibrary ? "(stdlib)" : "");
}
// Python.PrefixPath and LibraryPath should be included.
// We can't assume anything else
AssertUtil.ContainsAtLeast(paths.Select(p => p.Path.ToLowerInvariant().TrimEnd('\\')),
Python.PrefixPath.ToLowerInvariant().TrimEnd('\\'),
Python.LibPath.ToLowerInvariant().TrimEnd('\\')
);
// All paths should exist
AssertUtil.ArrayEquals(paths.Where(p => !Directory.Exists(p.Path)).ToList(), new PythonLibraryPath[0]);
// Ensure we can round-trip the entries via ToString/Parse
var asStrings = paths.Select(p => p.ToString()).ToList();
var asPaths = asStrings.Select(PythonLibraryPath.Parse).ToList();
var asStrings2 = asPaths.Select(p => p.ToString()).ToList();
AssertUtil.ArrayEquals(asStrings, asStrings2);
AssertUtil.ArrayEquals(paths, asPaths, (o1, o2) => {
PythonLibraryPath p1 = (PythonLibraryPath)o1, p2 = (PythonLibraryPath)o2;
return p1.Path == p2.Path && p1.IsStandardLibrary == p2.IsStandardLibrary;
});
var dbPath = TestData.GetTempPath(randomSubPath: true);
Assert.IsNull(PythonTypeDatabase.GetCachedDatabaseSearchPaths(dbPath),
"Should not have found cached paths in an empty directory");
PythonTypeDatabase.WriteDatabaseSearchPaths(dbPath, paths);
Assert.IsTrue(File.Exists(Path.Combine(dbPath, "database.path")));
var paths2 = PythonTypeDatabase.GetCachedDatabaseSearchPaths(dbPath);
AssertUtil.ArrayEquals(paths, paths2, (o1, o2) => {
PythonLibraryPath p1 = (PythonLibraryPath)o1, p2 = (PythonLibraryPath)o2;
return p1.Path == p2.Path && p1.IsStandardLibrary == p2.IsStandardLibrary;
});
}
public async Task GetDatabasePaths()
{
foreach (var version in PythonPaths.Versions)
{
var paths = await PythonLibraryPath.GetUncachedDatabaseSearchPathsAsync(version.InterpreterPath);
AssertUtil.ContainsAtLeast(
paths.Where(p => p.IsStandardLibrary).Select(p => p.Path),
new[] {
PathUtils.TrimEndSeparator(version.PrefixPath.ToLowerInvariant()),
Path.Combine(version.PrefixPath, "Lib").ToLowerInvariant()
}
);
AssertUtil.ContainsAtLeast(
paths.Where(p => !p.IsStandardLibrary).Select(p => p.Path),
new[] {
Path.Combine(version.PrefixPath, "Lib", "site-packages").ToLowerInvariant()
}
);
}
}
private async Task <IReadOnlyList <string> > GetCurrentSearchPathsAsync(CancellationToken cancellationToken)
{
if (_configuration.SearchPaths.Any())
{
return(_configuration.SearchPaths);
}
if (!File.Exists(_configuration.InterpreterPath))
{
return(Array.Empty <string>());
}
_log?.Log(TraceLevel.Info, "GetCurrentSearchPaths", _configuration.InterpreterPath, _moduleCache.SearchPathCachePath);
try {
var paths = await PythonLibraryPath.GetDatabaseSearchPathsAsync(_configuration, _moduleCache.SearchPathCachePath).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
return(paths.MaybeEnumerate().Select(p => p.Path).ToArray());
} catch (InvalidOperationException) {
return(Array.Empty <string>());
}
}
private string[] GetMissingModules(HashSet<string> existingDatabase) {
Debug.Assert(!string.IsNullOrEmpty(DatabasePath));
var searchPaths = PythonLibraryPath.GetCachedDatabaseSearchPaths(Path.Combine(DatabasePath, "database.path"));
if (searchPaths == null) {
// No cached search paths means our database is out of date.
return existingDatabase
.Except(RequiredBuiltinModules)
.OrderBy(name => name, StringComparer.InvariantCultureIgnoreCase)
.ToArray();
}
return PythonTypeDatabase.GetDatabaseExpectedModules(Configuration.Version, searchPaths)
.SelectMany()
.Select(mp => mp.ModuleName)
.Concat(RequiredBuiltinModules)
.Where(m => !existingDatabase.Contains(m))
.OrderBy(name => name, StringComparer.InvariantCultureIgnoreCase)
.ToArray();
}
/// <summary>
/// Gets the set of search paths that were last saved for a database.
/// </summary>
/// <param name="databasePath">Path containing the database.</param>
/// <returns>The cached list of search paths.</returns>
/// <remarks>Added in 2.2</remarks>
public static List <PythonLibraryPath> GetCachedDatabaseSearchPaths(string databasePath)
{
try {
var result = new List <PythonLibraryPath>();
using (var file = File.OpenText(Path.Combine(databasePath, "database.path"))) {
string line;
while ((line = file.ReadLine()) != null)
{
try {
result.Add(PythonLibraryPath.Parse(line));
} catch (FormatException) {
Debug.Fail("Invalid format: " + line);
}
}
}
return(result);
} catch (IOException) {
return(null);
}
}
public void InsideStdLib()
{
var appPath = TestData.GetTestSpecificPath("app.py");
var root = Path.GetDirectoryName(appPath);
var venv = Path.Combine(root, "venv");
var venvLib = Path.Combine(venv, "Lib");
var venvSitePackages = Path.Combine(venvLib, "site-packages");
var inside = Path.Combine(venvSitePackages, "inside");
var what = Path.Combine(venvSitePackages, "what");
var src = Path.Combine(root, "src");
var fromInterpreter = new[] {
new PythonLibraryPath(venvLib, PythonLibraryPathType.StdLib),
new PythonLibraryPath(venv, PythonLibraryPathType.StdLib),
new PythonLibraryPath(venvSitePackages, PythonLibraryPathType.Site),
new PythonLibraryPath(inside, PythonLibraryPathType.Pth),
};
var fromUser = new[] {
"./src",
"./venv/Lib/site-packages/what",
};
var(interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(root, _fs, fromInterpreter, fromUser);
interpreterPaths.Should().BeEquivalentToWithStrictOrdering(new[] {
new PythonLibraryPath(venvLib, PythonLibraryPathType.StdLib),
new PythonLibraryPath(venv, PythonLibraryPathType.StdLib),
new PythonLibraryPath(what, PythonLibraryPathType.Unspecified),
new PythonLibraryPath(venvSitePackages, PythonLibraryPathType.Site),
new PythonLibraryPath(inside, PythonLibraryPathType.Pth),
});
userPaths.Should().BeEquivalentToWithStrictOrdering(new[] {
new PythonLibraryPath(src, PythonLibraryPathType.Unspecified),
});
}
internal async Task ReloadSearchPaths(CancellationToken cancellationToken = default)
{
var ps = _services.GetService <IProcessServices>();
var paths = await GetInterpreterSearchPathsAsync(cancellationToken);
var(interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(Root, _fs, paths, Configuration.SearchPaths);
InterpreterPaths = interpreterPaths.Select(p => p.Path);
_userPaths = userPaths.Select(p => p.Path);
_log?.Log(TraceEventType.Information, "Interpreter search paths:");
foreach (var s in InterpreterPaths)
{
_log?.Log(TraceEventType.Information, $" {s}");
}
_log?.Log(TraceEventType.Information, "User search paths:");
foreach (var s in _userPaths)
{
_log?.Log(TraceEventType.Information, $" {s}");
}
}
private async Task <IReadOnlyList <PythonLibraryPath> > GetInterpreterSearchPathsAsync(CancellationToken cancellationToken = default)
{
if (!_fs.FileExists(Configuration.InterpreterPath))
{
_log?.Log(TraceEventType.Warning, "Interpreter does not exist:", Configuration.InterpreterPath);
_ui?.ShowMessageAsync(Resources.InterpreterNotFound, TraceEventType.Error);
return(Array.Empty <PythonLibraryPath>());
}
_log?.Log(TraceEventType.Information, "GetCurrentSearchPaths", Configuration.InterpreterPath);
try {
var fs = _services.GetService <IFileSystem>();
var ps = _services.GetService <IProcessServices>();
var paths = await PythonLibraryPath.GetSearchPathsAsync(Configuration, fs, ps, cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
return(paths.ToArray());
} catch (InvalidOperationException ex) {
_log?.Log(TraceEventType.Warning, "Exception getting search paths", ex);
_ui?.ShowMessageAsync(Resources.ExceptionGettingSearchPaths, TraceEventType.Error);
return(Array.Empty <PythonLibraryPath>());
}
}
private async Task ReloadSearchPaths(CancellationToken cancellationToken = default)
{
LibraryPaths = await GetInterpreterSearchPathsAsync(cancellationToken);
var(interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(Root, FileSystem, LibraryPaths, _userConfiguredPaths);
InterpreterPaths = interpreterPaths.Select(p => p.Path);
UserPaths = userPaths.Select(p => p.Path);
if (Log != null)
{
Log.Log(TraceEventType.Information, "Interpreter search paths:");
foreach (var s in InterpreterPaths)
{
Log.Log(TraceEventType.Information, $" {s}");
}
Log.Log(TraceEventType.Information, "User search paths:");
foreach (var s in UserPaths)
{
Log.Log(TraceEventType.Information, $" {s}");
}
}
}
/// <summary>
/// Gets the set of search paths by running the interpreter.
/// </summary>
/// <param name="interpreter">Path to the interpreter.</param>
/// <returns>A list of search paths for the interpreter.</returns>
/// <remarks>Added in 2.2</remarks>
public static async Task <List <PythonLibraryPath> > GetUncachedDatabaseSearchPathsAsync(string interpreter)
{
List <string> lines;
// sys.path will include the working directory, so we make an empty
// path that we can filter out later
var tempWorkingDir = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
Directory.CreateDirectory(tempWorkingDir);
var srcGetSearchPaths = PythonToolsInstallPath.GetFile("get_search_paths.py", typeof(PythonTypeDatabase).Assembly);
var getSearchPaths = PathUtils.GetAbsoluteFilePath(tempWorkingDir, PathUtils.GetFileOrDirectoryName(srcGetSearchPaths));
File.Copy(srcGetSearchPaths, getSearchPaths);
try {
using (var proc = ProcessOutput.Run(
interpreter,
new[] {
"-S", // don't import site - we do that in code
"-E", // ignore environment
getSearchPaths
},
tempWorkingDir,
null,
false,
null
)) {
int exitCode = -1;
try {
exitCode = await proc;
} catch (OperationCanceledException) {
}
if (exitCode != 0)
{
throw new InvalidOperationException(string.Format(
"Cannot obtain list of paths{0}{1}",
Environment.NewLine,
string.Join(Environment.NewLine, proc.StandardErrorLines))
);
}
lines = proc.StandardOutputLines.ToList();
}
} finally {
try {
Directory.Delete(tempWorkingDir, true);
} catch (Exception ex) {
if (ex.IsCriticalException())
{
throw;
}
}
}
return(lines.Select(s => {
if (s.StartsWith(tempWorkingDir, StringComparison.OrdinalIgnoreCase))
{
return null;
}
try {
return PythonLibraryPath.Parse(s);
} catch (FormatException) {
Debug.Fail("Invalid format for search path: " + s);
return null;
}
}).Where(p => p != null).ToList());
}
private async Task CreateLibraryWatchers()
{
Debug.Assert(_libWatchers != null, "Should not create watchers when suppressed");
IReadOnlyList <string> paths = null;
if (_factory.Configuration.SearchPaths.Any())
{
paths = _factory.Configuration.SearchPaths;
}
if (paths == null)
{
try {
paths = (await PythonLibraryPath.GetSearchPathsAsync(_factory.Configuration, new FileSystem(new OSPlatform()), new ProcessServices()))
.Select(p => p.Path)
.ToArray();
} catch (InvalidOperationException) {
return;
}
}
paths = paths
.Where(Directory.Exists)
.OrderBy(p => p.Length)
.ToList();
var watching = new List <string>();
var watchers = new List <FileSystemWatcher>();
foreach (var path in paths)
{
if (watching.Any(p => PathUtils.IsSubpathOf(p, path)))
{
continue;
}
FileSystemWatcher watcher = null;
try {
watcher = new FileSystemWatcher {
IncludeSubdirectories = true,
Path = path,
NotifyFilter = NotifyFilters.FileName | NotifyFilters.DirectoryName | NotifyFilters.LastWrite
};
watcher.Created += OnChanged;
watcher.Deleted += OnChanged;
watcher.Changed += OnChanged;
watcher.Renamed += OnRenamed;
watcher.EnableRaisingEvents = true;
watching.Add(path);
watchers.Add(watcher);
} catch (IOException) {
// Raced with directory deletion. We normally handle the
// library being deleted by disposing the watcher, but this
// occurs in response to an event from the watcher. Because
// we never got to start watching, we will just dispose
// immediately.
watcher?.Dispose();
} catch (ArgumentException ex) {
watcher?.Dispose();
Debug.WriteLine("Error starting FileSystemWatcher:\r\n{0}", ex);
}
}
List <FileSystemWatcher> oldWatchers;
lock (_libWatchers) {
oldWatchers = _libWatchers.ToList();
_libWatchers.Clear();
_libWatchers.AddRange(watchers);
}
foreach (var oldWatcher in oldWatchers)
{
oldWatcher.EnableRaisingEvents = false;
oldWatcher.Dispose();
}
}
private async Task CacheInstalledPackagesAsync(
bool alreadyHasLock,
bool alreadyHasConcurrencyLock,
CancellationToken cancellationToken
)
{
if (!IsReady)
{
await UpdateIsReadyAsync(alreadyHasLock, cancellationToken);
if (!IsReady)
{
return;
}
}
List <PackageSpec> packages = null;
var workingLock = alreadyHasLock ? null : await _working.LockAsync(cancellationToken);
try {
var args = _pipListHasFormatOption ? _commands.ListJson() : _commands.List();
var concurrencyLock = alreadyHasConcurrencyLock ? null : await _concurrencyLock.LockAsync(cancellationToken);
try {
var envVars = await GetEnvironmentVariables();
using (var proc = ProcessOutput.Run(
_factory.Configuration.InterpreterPath,
args,
_factory.Configuration.GetPrefixPath(),
envVars,
false,
null
)) {
try {
if ((await proc) == 0)
{
if (_pipListHasFormatOption)
{
try {
var data = JToken.ReadFrom(new JsonTextReader(new StringListReader(proc.StandardOutputLines)));
packages = data
.Select(j => new PackageSpec(j.Value <string>("name"), j.Value <string>("version")))
.Where(p => p.IsValid)
.OrderBy(p => p.Name)
.ToList();
} catch (JsonException ex) {
Debug.WriteLine("Failed to parse: {0}".FormatInvariant(ex.Message));
foreach (var l in proc.StandardOutputLines)
{
Debug.WriteLine(l);
}
}
}
else
{
packages = proc.StandardOutputLines
.Select(i => PackageSpec.FromPipList(i))
.Where(p => p.IsValid)
.OrderBy(p => p.Name)
.ToList();
}
}
else if (_pipListHasFormatOption)
{
// Actually, pip probably doesn't have the --format option
Debug.WriteLine("{0} does not support --format".FormatInvariant(_factory.Configuration.InterpreterPath));
_pipListHasFormatOption = false;
await CacheInstalledPackagesAsync(true, true, cancellationToken);
return;
}
else
{
}
} catch (OperationCanceledException) {
// Process failed to run
Debug.WriteLine("Failed to run pip to collect packages");
foreach (var line in proc.StandardOutputLines)
{
Debug.WriteLine(line);
}
}
}
if (packages == null)
{
// Pip failed, so return a directory listing
var paths = await PythonLibraryPath.GetSearchPathsAsync(
_factory.Configuration,
new FileSystem(new OSPlatform()),
new ProcessServices()
);
packages = await Task.Run(() => paths.Where(p => p.Type != PythonLibraryPathType.StdLib && Directory.Exists(p.Path))
.SelectMany(p => PathUtils.EnumerateDirectories(p.Path, recurse: false))
.Select(path => Path.GetFileName(path))
.Select(name => PackageNameRegex.Match(name))
.Where(match => match.Success)
.Select(match => new PackageSpec(match.Groups["name"].Value))
.Where(p => p.IsValid)
.OrderBy(p => p.Name)
.ToList());
}
} finally {
concurrencyLock?.Dispose();
}
// Outside of concurrency lock, still in working lock
_packages.Clear();
_packages.AddRange(packages);
_everCached = true;
} finally {
workingLock?.Dispose();
}
InstalledPackagesChanged?.Invoke(this, EventArgs.Empty);
_factory.NotifyImportNamesChanged();
}
