public IronPythonInterpreter(PythonInterpreterFactoryWithDatabase factory) {
#if DEBUG
_id = Interlocked.Increment(ref _interpreterCount);
Debug.WriteLine(String.Format("IronPython Interpreter Created {0}", _id));
Debug.WriteLine(new StackTrace(true).ToString());
#endif
AppDomain.CurrentDomain.AssemblyResolve += AssemblyResolver.Instance.CurrentDomain_AssemblyResolve;
InitializeRemoteDomain();
try {
LoadAssemblies();
} catch {
// IronPython not installed in the GAC...
}
var mod = Remote.ImportBuiltinModule("__builtin__");
var newMod = new IronPythonBuiltinModule(this, mod, "__builtin__");
_modules[newMod.Name] = newMod;
_factory = factory;
_typeDb = _factory.GetCurrentDatabase().CloneWithNewBuiltins(newMod);
_factory.NewDatabaseAvailable += OnNewDatabaseAvailable;
LoadModules();
}
private void Base_NewDatabaseAvailable(object sender, EventArgs e) {
if (_baseDb != null) {
_baseDb = null;
_baseHasRefreshed = true;
}
OnNewDatabaseAvailable();
OnIsCurrentChanged();
}
public static KnownTypes CreateDefault(PythonAnalyzer state, PythonTypeDatabase fallbackDb) {
var res = new KnownTypes();
var fallback = fallbackDb.BuiltinModule;
for (int value = 0; value < res._types.Length; ++value) {
res._types[value] = (IPythonType)fallback.GetAnyMember(
((ITypeDatabaseReader)fallbackDb).GetBuiltinTypeName((BuiltinTypeId)value)
);
}
res.SetClassInfo(state);
return res;
}
public static KnownTypes Create(PythonAnalyzer state, PythonTypeDatabase fallbackDb) {
var res = new KnownTypes();
var interpreter = state.Interpreter;
var fallback = fallbackDb.BuiltinModule;
for (int value = 0; value < res._types.Length; ++value) {
try {
res._types[value] = interpreter.GetBuiltinType((BuiltinTypeId)value);
} catch (KeyNotFoundException) {
res._types[value] = (IPythonType)fallback.GetAnyMember(
((ITypeDatabaseReader)fallbackDb).GetBuiltinTypeName((BuiltinTypeId)value)
);
}
}
res.SetClassInfo(state);
return res;
}
public PythonTypeDatabase(
PythonInterpreterFactoryWithDatabase factory,
IEnumerable<string> databaseDirectories = null,
PythonTypeDatabase innerDatabase = null
) {
if (innerDatabase != null && factory.Configuration.Version != innerDatabase.LanguageVersion) {
throw new InvalidOperationException("Language versions do not match");
}
_factory = factory;
if (innerDatabase != null) {
_sharedState = new SharedDatabaseState(innerDatabase._sharedState);
} else {
_sharedState = new SharedDatabaseState(_factory.Configuration.Version);
}
if (databaseDirectories != null) {
foreach (var d in databaseDirectories) {
LoadDatabase(d);
}
}
_sharedState.ListenForCorruptDatabase(this);
}
/// <summary>
/// Clears any cached type databases and raises the
/// <see cref="NewDatabaseAvailable"/> event.
/// </summary>
protected void OnNewDatabaseAvailable() {
_typeDb = null;
_typeDbWithoutPackages = null;
var evt = NewDatabaseAvailable;
if (evt != null) {
evt(this, EventArgs.Empty);
}
}
/// <summary>
/// Creates a new interpreter factory backed by a type database.
/// </summary>
/// <remarks>
/// <see cref="RefreshIsCurrent"/> must be called after creation to
/// ensure the database state is correctly reflected.
/// </remarks>
public PythonInterpreterFactoryWithDatabase(
InterpreterConfiguration config,
InterpreterFactoryCreationOptions options
)
{
if (config == null)
{
throw new ArgumentNullException(nameof(config));
}
if (options == null)
{
options = new InterpreterFactoryCreationOptions();
}
Configuration = config;
DatabasePath = options.DatabasePath;
// Avoid creating a interpreter with an unsupported version.
// https://github.com/Microsoft/PTVS/issues/706
try {
var langVer = config.Version.ToLanguageVersion();
} catch (InvalidOperationException ex) {
throw new ArgumentException(ex.Message, ex);
}
if (options.WatchFileSystem && !string.IsNullOrEmpty(DatabasePath))
{
// Assume the database is valid if the version is up to date, then
// switch to invalid after we've checked.
_isValid = PythonTypeDatabase.IsDatabaseVersionCurrent(DatabasePath);
_isCheckingDatabase = true;
_verWatcher = CreateDatabaseVerWatcher();
_verDirWatcher = CreateDatabaseDirectoryWatcher();
#if DEBUG
var creationStack = new StackTrace(true).ToString();
Task.Delay(1000).ContinueWith(t => {
Debug.Assert(
_hasEverCheckedDatabase,
"Database check was not triggered for {0}".FormatUI(Configuration.Id),
creationStack
);
});
#endif
}
else
{
// Assume the database is valid
_isValid = true;
}
try {
var pm = options.PackageManager;
if (pm != null)
{
pm.SetInterpreterFactory(this);
pm.InstalledFilesChanged += PackageManager_InstalledFilesChanged;
PackageManager = pm;
}
} catch (NotSupportedException) {
}
}
public void TestPthFiles() {
var outputPath = TestData.GetTempPath(randomSubPath: true);
Console.WriteLine("Writing to: " + outputPath);
// run the analyzer
using (var output = ProcessOutput.RunHiddenAndCapture("Microsoft.PythonTools.Analyzer.exe",
"/lib", TestData.GetPath(@"TestData\PathStdLib"),
"/version", "2.7",
"/outdir", outputPath,
"/indir", TestData.GetPath("CompletionDB"),
"/log", "AnalysisLog.txt")) {
output.Wait();
Console.WriteLine("* Stdout *");
foreach (var line in output.StandardOutputLines) {
Console.WriteLine(line);
}
Console.WriteLine("* Stderr *");
foreach (var line in output.StandardErrorLines) {
Console.WriteLine(line);
}
Assert.AreEqual(0, output.ExitCode);
}
File.Copy(TestData.GetPath(@"CompletionDB\__builtin__.idb"), Path.Combine(outputPath, "__builtin__.idb"));
var fact = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(new Version(2, 7));
var paths = new List<string> { outputPath };
paths.AddRange(Directory.EnumerateDirectories(outputPath));
var typeDb = new PythonTypeDatabase(fact, paths);
var module = typeDb.GetModule("SomeLib");
Assert.IsNotNull(module, "Could not import SomeLib");
var fobMod = typeDb.GetModule("SomeLib.fob");
Assert.IsNotNull(fobMod, "Could not import SomeLib.fob");
var cClass = ((IPythonModule)fobMod).GetMember(null, "C");
Assert.IsNotNull(cClass, "Could not get SomeLib.fob.C");
Assert.AreEqual(PythonMemberType.Class, cClass.MemberType);
}
/// <summary>
/// Sets up a weak reference for notification of when the shared database
/// has become corrupted. Doesn't keep the listening database alive.
/// </summary>
public void ListenForCorruptDatabase(PythonTypeDatabase db) {
lock (_corruptListeners) {
for (int i = 0; i < _corruptListeners.Count; i++) {
var target = _corruptListeners[i].Target;
if (target == null) {
_corruptListeners[i].Target = db;
return;
}
}
_corruptListeners.Add(new WeakReference(db));
}
}
public Task AddReferenceAsync(ProjectReference reference, CancellationToken cancellationToken = default(CancellationToken)) {
if (reference == null) {
return MakeExceptionTask(new ArgumentNullException("reference"));
}
if (_references == null) {
_references = new HashSet<ProjectReference>();
// If we needed to set _references, then we also need to clone
// _typeDb to avoid adding modules to the shared database.
if (_typeDb != null) {
_typeDb = _typeDb.Clone();
}
}
switch (reference.Kind) {
case ProjectReferenceKind.ExtensionModule:
_references.Add(reference);
string filename;
try {
filename = Path.GetFileNameWithoutExtension(reference.Name);
} catch (Exception e) {
return MakeExceptionTask(e);
}
if (_typeDb != null) {
return Task.Factory.StartNew(EmptyTask);
}
break;
}
return Task.Factory.StartNew(EmptyTask);
}
public void Dispose() {
var evt = UnloadingDomain;
if (evt != null) {
evt(this, EventArgs.Empty);
}
if (_factory != null) {
_factory.NewDatabaseAvailable -= OnNewDatabaseAvailable;
}
_typeDb = null;
AppDomain.CurrentDomain.AssemblyResolve -= AssemblyResolver.Instance.CurrentDomain_AssemblyResolve;
_unloader.Dispose();
#if DEBUG
GC.SuppressFinalize(this);
#endif
}
public DatabaseReplacedEventArgs(PythonTypeDatabase newDatabase)
{
_newDatabase = newDatabase;
}
public void Dispose() {
_typeDb = null;
var factory = _factory;
_factory = null;
if (factory != null) {
factory.NewDatabaseAvailable -= OnNewDatabaseAvailable;
}
}
public void Dispose() {
if (_factory != null) {
_factory.NewDatabaseAvailable -= OnNewDatabaseAvailable;
}
_typeDb = null;
AppDomain.CurrentDomain.AssemblyResolve -= AssemblyResolver.Instance.CurrentDomain_AssemblyResolve;
_unloader.Dispose();
}
/// <summary>
/// Determines whether the interpreter factory contains the specified
/// modules.
/// </summary>
/// <returns>The names of the modules that were found.</returns>
public static async Task <HashSet <string> > FindModulesAsync(this IPythonInterpreterFactory factory, params string[] moduleNames)
{
var finding = new HashSet <string>(moduleNames);
var found = new HashSet <string>();
var withPackages = factory.PackageManager;
if (withPackages != null)
{
foreach (var m in finding)
{
if ((await withPackages.GetInstalledPackageAsync(new PackageSpec(m), CancellationToken.None)).IsValid)
{
found.Add(m);
}
}
finding.ExceptWith(found);
if (!finding.Any())
{
// Found all of them, so stop searching
return(found);
}
}
var withDb = factory as PythonInterpreterFactoryWithDatabase;
if (withDb != null && withDb.IsCurrent)
{
var db = withDb.GetCurrentDatabase();
found.UnionWith(finding.Where(m => db.GetModule(m) != null));
// Always stop searching after this step
return(found);
}
if (withDb != null)
{
try {
var paths = await PythonTypeDatabase.GetDatabaseSearchPathsAsync(withDb);
found.UnionWith(PythonTypeDatabase.GetDatabaseExpectedModules(withDb.Configuration.Version, paths)
.SelectMany()
.Select(g => g.ModuleName)
.Where(m => finding.Contains(m)));
} catch (InvalidOperationException) {
}
finding.ExceptWith(found);
if (!finding.Any())
{
// Found all of them, so stop searching
return(found);
}
}
return(await Task.Run(() => {
foreach (var mp in ModulePath.GetModulesInLib(factory.Configuration))
{
if (finding.Remove(mp.ModuleName))
{
found.Add(mp.ModuleName);
}
if (!finding.Any())
{
break;
}
}
return found;
}));
}
public NoInterpretersInterpreter(PythonTypeDatabase database)
{
_database = database;
}
public IPythonInterpreter CreateInterpreter()
{
return(new NoInterpretersInterpreter(PythonTypeDatabase.CreateDefaultTypeDatabase()));
}
/// <summary>
/// Creates a new interpreter factory with the specified database. This
/// factory is suitable for analysis, but not execution.
/// </summary>
public static PythonInterpreterFactoryWithDatabase CreateAnalysisInterpreterFactory(
Version languageVersion,
PythonTypeDatabase database)
{
return(new AnalysisOnlyInterpreterFactory(languageVersion, database));
}
public override void GenerateDatabase(GenerateDatabaseOptions options, Action<int> onExit = null) {
if (!Directory.Exists(Configuration.LibraryPath)) {
return;
}
// Create and mark the DB directory as hidden
try {
var dbDir = Directory.CreateDirectory(DatabasePath);
dbDir.Attributes |= FileAttributes.Hidden;
} catch (ArgumentException) {
} catch (IOException) {
} catch (UnauthorizedAccessException) {
}
var req = new PythonTypeDatabaseCreationRequest {
Factory = this,
OutputPath = DatabasePath,
SkipUnchanged = options.HasFlag(GenerateDatabaseOptions.SkipUnchanged),
DetectLibraryPath = !AssumeSimpleLibraryLayout
};
req.ExtraInputDatabases.Add(_base.DatabasePath);
_baseHasRefreshed = false;
if (_base.IsCurrent) {
base.GenerateDatabase(req, onExit);
} else {
req.WaitFor = _base;
req.SkipUnchanged = false;
// Clear out the existing base database, since we're going to
// need to reload it again. This also means that when
// NewDatabaseAvailable is raised, we are expecting it and won't
// incorrectly set _baseHasRefreshed to true again.
_baseDb = null;
// Start our analyzer first, since we will wait up to a minute
// for our base analyzer to start (which may cause a one minute
// delay if it completes before we start, but that is unlikely).
base.GenerateDatabase(req, onExit);
_base.GenerateDatabase(GenerateDatabaseOptions.SkipUnchanged);
}
}
public DatabaseReplacedEventArgs(PythonTypeDatabase newDatabase) {
_newDatabase = newDatabase;
}
/// <summary>
/// Regenerates the database for this environment. If the base
/// interpreter needs regenerating, it will also be regenerated.
/// </summary>
public override void GenerateDatabase(GenerateDatabaseOptions options, Action<int> onExit = null) {
if (!Directory.Exists(Configuration.LibraryPath)) {
return;
}
var req = new PythonTypeDatabaseCreationRequest {
Factory = this,
OutputPath = DatabasePath,
SkipUnchanged = options.HasFlag(GenerateDatabaseOptions.SkipUnchanged)
};
req.ExtraInputDatabases.Add(_base.DatabasePath);
_baseHasRefreshed = false;
if (_base.IsCurrent) {
// The App database is already up to date, so start analyzing
// the User database immediately.
base.GenerateDatabase(req, onExit);
} else {
// The App database needs to be updated, so start both and wait
// for the base to finish before analyzing User.
// Specifying our base interpreter as 'WaitFor' allows the UI to
// forward progress and status messages to the user, even though
// the factory is not visible.
req.WaitFor = _base;
// Because the underlying analysis of the standard library has
// changed, we must reanalyze the entire database.
req.SkipUnchanged = false;
// Clear out the existing base database, since we're going to
// need to reload it again. This also means that when
// NewDatabaseAvailable is raised, we are expecting it and won't
// incorrectly set _baseHasRefreshed to true again.
_baseDb = null;
// Start our analyzer first, since we will wait up to a minute
// for the base analyzer to start (which may cause a one minute
// delay if it completes before we start, but that is unlikely).
base.GenerateDatabase(req, onExit);
_base.GenerateDatabase(GenerateDatabaseOptions.SkipUnchanged);
}
}
/// <summary>
/// Called to manually trigger a refresh of <see cref="IsCurrent"/>.
/// After completion, <see cref="IsCurrentChanged"/> will always be
/// raised, regardless of whether the values were changed.
/// </summary>
public virtual void RefreshIsCurrent()
{
#if DEBUG
// Indicate that we've arrived here at least once. Otherwise we
// will assert.
_hasEverCheckedDatabase = true;
#endif
if (GlobalInterpreterOptions.SuppressFileSystemWatchers || string.IsNullOrEmpty(DatabasePath))
{
_isCheckingDatabase = false;
_isValid = true;
_missingModules = null;
_isCurrentException = null;
OnIsCurrentChanged();
return;
}
if (IsGenerating)
{
if (PythonTypeDatabase.IsDatabaseRegenerating(DatabasePath))
{
_isValid = false;
_missingModules = null;
_isCurrentException = null;
return;
}
var generating = Interlocked.Exchange(ref _generating, null);
generating?.Dispose();
}
try {
if (!_isCurrentSemaphore.Wait(0))
{
// Another thread is working on our state, so we will wait for
// them to finish and return, since the value is up to date.
_isCurrentSemaphore.Wait();
_isCurrentSemaphore.Release();
return;
}
} catch (ObjectDisposedException) {
// We've been disposed and the call has come in from
// externally, probably a timer.
return;
}
try {
_isCheckingDatabase = true;
OnIsCurrentChanged();
// Wait for a slot that will allow us to scan the disk. This
// prevents too many threads from updating at once and causing
// I/O saturation.
_sharedIsCurrentSemaphore.Wait();
try {
if (PythonTypeDatabase.IsDatabaseRegenerating(DatabasePath) ||
!PythonTypeDatabase.IsDatabaseVersionCurrent(DatabasePath))
{
// Skip the rest of the checks, because we know we're
// not current.
_isValid = false;
_missingModules = null;
_isCurrentException = null;
}
else
{
_isValid = true;
HashSet <string> existingDatabase = null;
string[] missingModules = null;
for (int retries = 3; retries > 0; --retries)
{
try {
existingDatabase = GetExistingDatabase(DatabasePath);
break;
} catch (UnauthorizedAccessException) {
} catch (IOException) {
}
Thread.Sleep(100);
}
if (existingDatabase == null)
{
// This will either succeed or throw again. If it throws
// then the error is reported to the user.
existingDatabase = GetExistingDatabase(DatabasePath);
}
for (int retries = 3; retries > 0; --retries)
{
try {
missingModules = GetMissingModules(existingDatabase);
break;
} catch (UnauthorizedAccessException) {
} catch (IOException) {
}
Thread.Sleep(100);
}
if (missingModules == null)
{
// This will either succeed or throw again. If it throws
// then the error is reported to the user.
missingModules = GetMissingModules(existingDatabase);
}
if (missingModules.Length > 0)
{
var oldModules = _missingModules;
if (oldModules == null ||
oldModules.Length != missingModules.Length ||
!oldModules.SequenceEqual(missingModules))
{
}
_missingModules = missingModules;
}
else
{
_missingModules = null;
}
}
_isCurrentException = null;
} finally {
_sharedIsCurrentSemaphore.Release();
}
} catch (Exception ex) {
// Report the exception text as the reason.
_isCurrentException = ex.ToString();
_missingModules = null;
} finally {
_isCheckingDatabase = false;
try {
_isCurrentSemaphore.Release();
} catch (ObjectDisposedException) {
// The semaphore is not locked for disposal as it is only
// used to prevent reentrance into this function. As a
// result, it may have been disposed while we were in here.
}
}
OnIsCurrentChanged();
}
public PythonUwpInterpreter(PythonInterpreterFactoryWithDatabase factory) {
_langVersion = factory.Configuration.Version;
_factory = factory;
_typeDb = _factory.GetCurrentDatabase();
_factory.NewDatabaseAvailable += OnNewDatabaseAvailable;
}
private async Task CacheInstalledPackagesAsync(bool alreadyHasLock, CancellationToken cancellationToken)
{
if (!IsReady)
{
await UpdateIsReadyAsync(alreadyHasLock, cancellationToken);
if (!IsReady)
{
return;
}
}
List <PackageSpec> packages = null;
var workingLock = alreadyHasLock ? null : await _working.LockAsync(cancellationToken);
try {
var args = _extraInterpreterArgs.ToList();
if (!SupportsDashMPip)
{
args.Add("-c");
args.Add("\"import pip; pip.main()\"");
}
else
{
args.Add("-m");
args.Add("pip");
}
args.Add("list");
using (await _concurrencyLock.LockAsync(cancellationToken)) {
using (var proc = ProcessOutput.Run(
_factory.Configuration.InterpreterPath,
args,
_factory.Configuration.PrefixPath,
UnbufferedEnv,
false,
null
)) {
try {
if (await proc == 0)
{
packages = proc.StandardOutputLines
.Select(i => PackageSpec.FromPipList(i))
.Where(p => p.IsValid)
.OrderBy(p => p.Name)
.ToList();
}
} catch (OperationCanceledException) {
// Process failed to run
Debug.WriteLine("Failed to run pip to collect packages");
Debug.WriteLine(string.Join(Environment.NewLine, proc.StandardOutputLines));
}
}
if (packages == null)
{
// Pip failed, so return a directory listing
var paths = await PythonTypeDatabase.GetDatabaseSearchPathsAsync(_factory);
packages = await Task.Run(() => paths.Where(p => !p.IsStandardLibrary && Directory.Exists(p.Path))
.SelectMany(p => PathUtils.EnumerateDirectories(p.Path, recurse: false))
.Select(path => Path.GetFileName(path))
.Select(name => PackageNameRegex.Match(name))
.Where(match => match.Success)
.Select(match => new PackageSpec(match.Groups["name"].Value))
.Where(p => p.IsValid)
.OrderBy(p => p.Name)
.ToList());
}
}
// Outside of concurrency lock, still in working lock
_packages.Clear();
_packages.AddRange(packages);
_everCached = true;
} finally {
workingLock?.Dispose();
}
InstalledPackagesChanged?.Invoke(this, EventArgs.Empty);
}
// We could override MakeInterpreter() to return a different
// implementation of IPythonInterpreter, but we don't need to here, as
// the default implementation works well with PythonTypeDatabase.
//public override IPythonInterpreter MakeInterpreter(PythonInterpreterFactoryWithDatabase factory) {
// return base.MakeInterpreter(factory);
//}
/// <summary>
/// Returns a new database that contains the database from our base
/// interpreter.
/// </summary>
public override PythonTypeDatabase MakeTypeDatabase(string databasePath, bool includeSitePackages = true) {
if (_baseDb == null && _base.IsCurrent) {
_baseDb = _base.GetCurrentDatabase(ShouldIncludeGlobalSitePackages);
}
var paths = new List<string> { databasePath };
if (includeSitePackages) {
try {
paths.AddRange(Directory.EnumerateDirectories(databasePath));
} catch (ArgumentException) {
} catch (IOException) {
} catch (SecurityException) {
} catch (UnauthorizedAccessException) {
}
}
return new PythonTypeDatabase(this, paths, _baseDb);
}
private async Task CreateLibraryWatchers()
{
Debug.Assert(_libWatchers != null, "Should not create watchers when suppressed");
IList <PythonLibraryPath> paths;
try {
paths = await PythonTypeDatabase.GetDatabaseSearchPathsAsync(_factory);
} catch (InvalidOperationException) {
return;
}
paths = paths
.Where(p => Directory.Exists(p.Path))
.OrderBy(p => p.Path.Length)
.ToList();
var watching = new List <string>();
var watchers = new List <FileSystemWatcher>();
foreach (var path in paths)
{
if (watching.Any(p => PathUtils.IsSubpathOf(p, path.Path)))
{
continue;
}
FileSystemWatcher watcher = null;
try {
watcher = new FileSystemWatcher {
IncludeSubdirectories = true,
Path = path.Path,
NotifyFilter = NotifyFilters.FileName | NotifyFilters.DirectoryName | NotifyFilters.LastWrite
};
watcher.Created += OnChanged;
watcher.Deleted += OnChanged;
watcher.Changed += OnChanged;
watcher.Renamed += OnRenamed;
watcher.EnableRaisingEvents = true;
watching.Add(path.Path);
watchers.Add(watcher);
} catch (IOException) {
// Raced with directory deletion. We normally handle the
// library being deleted by disposing the watcher, but this
// occurs in response to an event from the watcher. Because
// we never got to start watching, we will just dispose
// immediately.
watcher?.Dispose();
} catch (ArgumentException ex) {
watcher?.Dispose();
Debug.WriteLine("Error starting FileSystemWatcher:\r\n{0}", ex);
}
}
List <FileSystemWatcher> oldWatchers;
lock (_libWatchers) {
oldWatchers = _libWatchers.ToList();
_libWatchers.Clear();
_libWatchers.AddRange(watchers);
}
foreach (var oldWatcher in oldWatchers)
{
oldWatcher.EnableRaisingEvents = false;
oldWatcher.Dispose();
}
}
void OnNewDatabaseAvailable(object sender, EventArgs e) {
var mod = _modules["__builtin__"] as IBuiltinPythonModule;
_typeDb = _factory.GetCurrentDatabase().CloneWithNewBuiltins(mod);
RaiseModuleNamesChanged();
}
public ExtensionModuleLoader(PythonTypeDatabase typeDb, IPythonInterpreterFactory factory, string moduleName, string extensionFilename, CancellationToken cancel) {
_typeDb = typeDb;
_factory = factory;
_moduleName = moduleName;
_extensionFilename = extensionFilename;
_cancel = cancel;
}
public void PydInPackage() {
PythonPaths.Python27.AssertInstalled();
var outputPath = TestData.GetTempPath(randomSubPath: true);
Console.WriteLine("Writing to: " + outputPath);
// run the analyzer
using (var output = ProcessOutput.RunHiddenAndCapture("Microsoft.PythonTools.Analyzer.exe",
"/python", PythonPaths.Python27.InterpreterPath,
"/lib", TestData.GetPath(@"TestData\PydStdLib"),
"/version", "2.7",
"/outdir", outputPath,
"/indir", TestData.GetPath("CompletionDB"),
"/log", "AnalysisLog.txt")) {
output.Wait();
Console.WriteLine("* Stdout *");
foreach (var line in output.StandardOutputLines) {
Console.WriteLine(line);
}
Console.WriteLine("* Stderr *");
foreach (var line in output.StandardErrorLines) {
Console.WriteLine(line);
}
Assert.AreEqual(0, output.ExitCode);
}
var fact = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(new Version(2, 7));
var paths = new List<string> { outputPath };
paths.AddRange(Directory.EnumerateDirectories(outputPath));
var typeDb = new PythonTypeDatabase(fact, paths);
var module = typeDb.GetModule("Package.winsound");
Assert.IsNotNull(module, "Package.winsound was not analyzed");
var package = typeDb.GetModule("Package");
Assert.IsNotNull(package, "Could not import Package");
var member = package.GetMember(null, "winsound");
Assert.IsNotNull(member, "Could not get member Package.winsound");
Assert.AreSame(module, member);
module = typeDb.GetModule("Package._testcapi");
Assert.IsNotNull(module, "Package._testcapi was not analyzed");
package = typeDb.GetModule("Package");
Assert.IsNotNull(package, "Could not import Package");
member = package.GetMember(null, "_testcapi");
Assert.IsNotNull(member, "Could not get member Package._testcapi");
Assert.IsNotInstanceOfType(member, typeof(CPythonMultipleMembers));
Assert.AreSame(module, member);
module = typeDb.GetModule("Package.select");
Assert.IsNotNull(module, "Package.select was not analyzed");
package = typeDb.GetModule("Package");
Assert.IsNotNull(package, "Could not import Package");
member = package.GetMember(null, "select");
Assert.IsNotNull(member, "Could not get member Package.select");
Assert.IsInstanceOfType(member, typeof(CPythonMultipleMembers));
var mm = (CPythonMultipleMembers)member;
AssertUtil.ContainsExactly(mm.Members.Select(m => m.MemberType),
PythonMemberType.Module,
PythonMemberType.Constant,
PythonMemberType.Class
);
Assert.IsNotNull(mm.Members.Contains(module));
try {
// Only clean up if the test passed
Directory.Delete(outputPath, true);
} catch { }
}
public PythonTypeDatabase CloneWithNewBuiltins(IBuiltinPythonModule newBuiltins) {
var newDb = new PythonTypeDatabase(_factory, null, this);
newDb._sharedState.BuiltinModule = newBuiltins;
return newDb;
}
/// <summary>
/// Returns the database for this factory. This database may be shared
/// between callers and should be cloned before making modifications.
///
/// This function never returns null.
/// </summary>
public PythonTypeDatabase GetCurrentDatabase() {
if (_typeDb == null || _typeDb.DatabaseDirectory != DatabasePath) {
_typeDb = MakeTypeDatabase(DatabasePath) ??
PythonTypeDatabase.CreateDefaultTypeDatabase(this);
}
return _typeDb;
}
private void OnNewDatabaseAvailable(object sender, EventArgs e) {
// Raised if our base database is updated, which means we need to
// refresh our database too.
if (_baseDb != null) {
_baseDb = null;
_baseHasRefreshed = true;
}
OnNewDatabaseAvailable();
OnIsCurrentChanged();
}
/// <summary>
/// Creates a new interpreter factory with the specified database. This
/// factory is suitable for analysis, but not execution.
/// </summary>
public static PythonInterpreterFactoryWithDatabase CreateAnalysisInterpreterFactory(
Version languageVersion,
PythonTypeDatabase database) {
return new AnalysisOnlyInterpreterFactory(languageVersion, database);
}
private async Task CacheInstalledPackagesAsync(
bool alreadyHasLock,
bool alreadyHasConcurrencyLock,
CancellationToken cancellationToken
)
{
if (!IsReady)
{
await UpdateIsReadyAsync(alreadyHasLock, cancellationToken);
if (!IsReady)
{
return;
}
}
List <PackageSpec> packages = null;
var workingLock = alreadyHasLock ? null : await _working.LockAsync(cancellationToken);
try {
var args = _extraInterpreterArgs.ToList();
if (!SupportsDashMPip)
{
args.Add("-c");
args.Add("\"import pip; pip.main()\"");
}
else
{
args.Add("-m");
args.Add("pip");
}
args.Add("list");
if (_pipListHasFormatOption)
{
args.Add("--format=json");
}
var concurrencyLock = alreadyHasConcurrencyLock ? null : await _concurrencyLock.LockAsync(cancellationToken);
try {
using (var proc = ProcessOutput.Run(
_factory.Configuration.InterpreterPath,
args,
_factory.Configuration.PrefixPath,
UnbufferedEnv,
false,
null
)) {
try {
if ((await proc) == 0)
{
if (_pipListHasFormatOption)
{
try {
var data = JToken.ReadFrom(new JsonTextReader(new StringListReader(proc.StandardOutputLines)));
packages = data
.Select(j => new PackageSpec(j.Value <string>("name"), j.Value <string>("version")))
.Where(p => p.IsValid)
.OrderBy(p => p.Name)
.ToList();
} catch (JsonException ex) {
Debug.WriteLine("Failed to parse: {0}".FormatInvariant(ex.Message));
foreach (var l in proc.StandardOutputLines)
{
Debug.WriteLine(l);
}
}
}
else
{
packages = proc.StandardOutputLines
.Select(i => PackageSpec.FromPipList(i))
.Where(p => p.IsValid)
.OrderBy(p => p.Name)
.ToList();
}
}
else if (_pipListHasFormatOption)
{
// Actually, pip probably doesn't have the --format option
Debug.WriteLine("{0} does not support --format".FormatInvariant(_factory.Configuration.InterpreterPath));
_pipListHasFormatOption = false;
await CacheInstalledPackagesAsync(true, true, cancellationToken);
return;
}
else
{
}
} catch (OperationCanceledException) {
// Process failed to run
Debug.WriteLine("Failed to run pip to collect packages");
foreach (var line in proc.StandardOutputLines)
{
Debug.WriteLine(line);
}
}
}
if (packages == null)
{
// Pip failed, so return a directory listing
var paths = await PythonTypeDatabase.GetDatabaseSearchPathsAsync(_factory);
packages = await Task.Run(() => paths.Where(p => !p.IsStandardLibrary && Directory.Exists(p.Path))
.SelectMany(p => PathUtils.EnumerateDirectories(p.Path, recurse: false))
.Select(path => Path.GetFileName(path))
.Select(name => PackageNameRegex.Match(name))
.Where(match => match.Success)
.Select(match => new PackageSpec(match.Groups["name"].Value))
.Where(p => p.IsValid)
.OrderBy(p => p.Name)
.ToList());
}
} finally {
concurrencyLock?.Dispose();
}
// Outside of concurrency lock, still in working lock
_packages.Clear();
_packages.AddRange(packages);
_everCached = true;
} finally {
workingLock?.Dispose();
}
InstalledPackagesChanged?.Invoke(this, EventArgs.Empty);
}
public void Dispose() {
if (_factory != null) {
_factory.NewDatabaseAvailable -= OnNewDatabaseAvailable;
}
_typeDb = null;
AppDomain.CurrentDomain.AssemblyResolve -= AssemblyResolver.Instance.CurrentDomain_AssemblyResolve;
_unloader.Dispose();
#if DEBUG
GC.SuppressFinalize(this);
#endif
}
public override PythonTypeDatabase MakeTypeDatabase(string databasePath, bool includeSitePackages = true) {
if (_baseDb == null && _base.IsCurrent) {
var includeBaseSitePackages = ShouldIncludeGlobalSitePackages(
Configuration.PrefixPath,
Configuration.LibraryPath
);
_baseDb = _base.GetCurrentDatabase(includeBaseSitePackages);
}
if (!IsCurrent || !Directory.Exists(databasePath)) {
return _baseDb;
}
var paths = new List<string> { databasePath };
if (includeSitePackages) {
try {
paths.AddRange(Directory.EnumerateDirectories(databasePath));
} catch (IOException) {
} catch (UnauthorizedAccessException) {
}
}
return new PythonTypeDatabase(this, paths, _baseDb);
}
public void LayeredDatabase() {
using (var db1 = MockCompletionDB.Create(PythonLanguageVersion.V27, "os"))
using (var db2 = MockCompletionDB.Create(PythonLanguageVersion.V27, "posixpath")) {
Assert.IsNotNull(db1.Database.GetModule("os"));
Assert.IsNull(db1.Database.GetModule("posixpath"));
var ptd1 = db1.Database;
var ptd2 = ptd1.Clone();
ptd2.LoadDatabase(db2.DBPath);
Assert.IsNull(ptd1.GetModule("posixpath"));
Assert.IsNotNull(ptd2.GetModule("os"));
Assert.IsNotNull(ptd2.GetModule("posixpath"));
Assert.AreSame(ptd1.GetModule("os"), db1.Database.GetModule("os"));
Assert.AreSame(ptd2.GetModule("os"), db1.Database.GetModule("os"));
}
var factory = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(new Version(2, 7));
using (var db1 = MockCompletionDB.Create(PythonLanguageVersion.V27, "os", "posixpath"))
using (var db2 = MockCompletionDB.Create(PythonLanguageVersion.V27, "posixpath")) {
var ptd1 = new PythonTypeDatabase(factory, new[] {
db1.DBPath,
db2.DBPath
});
Assert.IsNotNull(ptd1.GetModule("posixpath"));
Assert.AreNotSame(ptd1.GetModule("posixpath"), db1.Database.GetModule("posixpath"));
Assert.AreNotSame(ptd1.GetModule("posixpath"), db2.Database.GetModule("posixpath"));
var ptd2 = new PythonTypeDatabase(factory, new[] {
db2.DBPath,
db1.DBPath
});
Assert.IsNotNull(ptd2.GetModule("posixpath"));
Assert.AreNotSame(ptd2.GetModule("posixpath"), db1.Database.GetModule("posixpath"));
Assert.AreNotSame(ptd2.GetModule("posixpath"), db2.Database.GetModule("posixpath"));
}
using (var db1 = MockCompletionDB.Create(PythonLanguageVersion.V27, "os", "posixpath"))
using (var db2 = MockCompletionDB.Create(PythonLanguageVersion.V27, "posixpath"))
using (var db3 = MockCompletionDB.Create(PythonLanguageVersion.V27, "ntpath")) {
var ptd = db1.Database;
Assert.AreSame(ptd.GetModule("posixpath"), db1.Database.GetModule("posixpath"));
Assert.AreNotSame(ptd.GetModule("posixpath"), db2.Database.GetModule("posixpath"));
var ptd2 = ptd.Clone();
ptd2.LoadDatabase(db2.DBPath);
Assert.AreNotSame(ptd2.GetModule("posixpath"), ptd.GetModule("posixpath"));
var ptd3 = ptd2.Clone();
ptd3.LoadDatabase(db3.DBPath);
Assert.IsNotNull(ptd3.GetModule("ntpath"));
Assert.IsNull(ptd2.GetModule("ntpath"));
}
}
protected override void Dispose(bool disposing) {
_baseDb = null;
_base.IsCurrentChanged -= OnIsCurrentChanged;
_base.NewDatabaseAvailable -= OnNewDatabaseAvailable;
base.Dispose(disposing);
}
