public IronPythonInterpreter(PythonInterpreterFactoryWithDatabase factory) { #if DEBUG _id = Interlocked.Increment(ref _interpreterCount); Debug.WriteLine(String.Format("IronPython Interpreter Created {0}", _id)); Debug.WriteLine(new StackTrace(true).ToString()); #endif AppDomain.CurrentDomain.AssemblyResolve += AssemblyResolver.Instance.CurrentDomain_AssemblyResolve; InitializeRemoteDomain(); try { LoadAssemblies(); } catch { // IronPython not installed in the GAC... } var mod = Remote.ImportBuiltinModule("__builtin__"); var newMod = new IronPythonBuiltinModule(this, mod, "__builtin__"); _modules[newMod.Name] = newMod; _factory = factory; _typeDb = _factory.GetCurrentDatabase().CloneWithNewBuiltins(newMod); _factory.NewDatabaseAvailable += OnNewDatabaseAvailable; LoadModules(); }
private void Base_NewDatabaseAvailable(object sender, EventArgs e) { if (_baseDb != null) { _baseDb = null; _baseHasRefreshed = true; } OnNewDatabaseAvailable(); OnIsCurrentChanged(); }
public static KnownTypes CreateDefault(PythonAnalyzer state, PythonTypeDatabase fallbackDb) { var res = new KnownTypes(); var fallback = fallbackDb.BuiltinModule; for (int value = 0; value < res._types.Length; ++value) { res._types[value] = (IPythonType)fallback.GetAnyMember( ((ITypeDatabaseReader)fallbackDb).GetBuiltinTypeName((BuiltinTypeId)value) ); } res.SetClassInfo(state); return res; }
public static KnownTypes Create(PythonAnalyzer state, PythonTypeDatabase fallbackDb) { var res = new KnownTypes(); var interpreter = state.Interpreter; var fallback = fallbackDb.BuiltinModule; for (int value = 0; value < res._types.Length; ++value) { try { res._types[value] = interpreter.GetBuiltinType((BuiltinTypeId)value); } catch (KeyNotFoundException) { res._types[value] = (IPythonType)fallback.GetAnyMember( ((ITypeDatabaseReader)fallbackDb).GetBuiltinTypeName((BuiltinTypeId)value) ); } } res.SetClassInfo(state); return res; }
public PythonTypeDatabase( PythonInterpreterFactoryWithDatabase factory, IEnumerable<string> databaseDirectories = null, PythonTypeDatabase innerDatabase = null ) { if (innerDatabase != null && factory.Configuration.Version != innerDatabase.LanguageVersion) { throw new InvalidOperationException("Language versions do not match"); } _factory = factory; if (innerDatabase != null) { _sharedState = new SharedDatabaseState(innerDatabase._sharedState); } else { _sharedState = new SharedDatabaseState(_factory.Configuration.Version); } if (databaseDirectories != null) { foreach (var d in databaseDirectories) { LoadDatabase(d); } } _sharedState.ListenForCorruptDatabase(this); }
/// <summary> /// Clears any cached type databases and raises the /// <see cref="NewDatabaseAvailable"/> event. /// </summary> protected void OnNewDatabaseAvailable() { _typeDb = null; _typeDbWithoutPackages = null; var evt = NewDatabaseAvailable; if (evt != null) { evt(this, EventArgs.Empty); } }
/// <summary> /// Creates a new interpreter factory backed by a type database. /// </summary> /// <remarks> /// <see cref="RefreshIsCurrent"/> must be called after creation to /// ensure the database state is correctly reflected. /// </remarks> public PythonInterpreterFactoryWithDatabase( InterpreterConfiguration config, InterpreterFactoryCreationOptions options ) { if (config == null) { throw new ArgumentNullException(nameof(config)); } if (options == null) { options = new InterpreterFactoryCreationOptions(); } Configuration = config; DatabasePath = options.DatabasePath; // Avoid creating a interpreter with an unsupported version. // https://github.com/Microsoft/PTVS/issues/706 try { var langVer = config.Version.ToLanguageVersion(); } catch (InvalidOperationException ex) { throw new ArgumentException(ex.Message, ex); } if (options.WatchFileSystem && !string.IsNullOrEmpty(DatabasePath)) { // Assume the database is valid if the version is up to date, then // switch to invalid after we've checked. _isValid = PythonTypeDatabase.IsDatabaseVersionCurrent(DatabasePath); _isCheckingDatabase = true; _verWatcher = CreateDatabaseVerWatcher(); _verDirWatcher = CreateDatabaseDirectoryWatcher(); #if DEBUG var creationStack = new StackTrace(true).ToString(); Task.Delay(1000).ContinueWith(t => { Debug.Assert( _hasEverCheckedDatabase, "Database check was not triggered for {0}".FormatUI(Configuration.Id), creationStack ); }); #endif } else { // Assume the database is valid _isValid = true; } try { var pm = options.PackageManager; if (pm != null) { pm.SetInterpreterFactory(this); pm.InstalledFilesChanged += PackageManager_InstalledFilesChanged; PackageManager = pm; } } catch (NotSupportedException) { } }
public void TestPthFiles() { var outputPath = TestData.GetTempPath(randomSubPath: true); Console.WriteLine("Writing to: " + outputPath); // run the analyzer using (var output = ProcessOutput.RunHiddenAndCapture("Microsoft.PythonTools.Analyzer.exe", "/lib", TestData.GetPath(@"TestData\PathStdLib"), "/version", "2.7", "/outdir", outputPath, "/indir", TestData.GetPath("CompletionDB"), "/log", "AnalysisLog.txt")) { output.Wait(); Console.WriteLine("* Stdout *"); foreach (var line in output.StandardOutputLines) { Console.WriteLine(line); } Console.WriteLine("* Stderr *"); foreach (var line in output.StandardErrorLines) { Console.WriteLine(line); } Assert.AreEqual(0, output.ExitCode); } File.Copy(TestData.GetPath(@"CompletionDB\__builtin__.idb"), Path.Combine(outputPath, "__builtin__.idb")); var fact = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(new Version(2, 7)); var paths = new List<string> { outputPath }; paths.AddRange(Directory.EnumerateDirectories(outputPath)); var typeDb = new PythonTypeDatabase(fact, paths); var module = typeDb.GetModule("SomeLib"); Assert.IsNotNull(module, "Could not import SomeLib"); var fobMod = typeDb.GetModule("SomeLib.fob"); Assert.IsNotNull(fobMod, "Could not import SomeLib.fob"); var cClass = ((IPythonModule)fobMod).GetMember(null, "C"); Assert.IsNotNull(cClass, "Could not get SomeLib.fob.C"); Assert.AreEqual(PythonMemberType.Class, cClass.MemberType); }
/// <summary> /// Sets up a weak reference for notification of when the shared database /// has become corrupted. Doesn't keep the listening database alive. /// </summary> public void ListenForCorruptDatabase(PythonTypeDatabase db) { lock (_corruptListeners) { for (int i = 0; i < _corruptListeners.Count; i++) { var target = _corruptListeners[i].Target; if (target == null) { _corruptListeners[i].Target = db; return; } } _corruptListeners.Add(new WeakReference(db)); } }
public Task AddReferenceAsync(ProjectReference reference, CancellationToken cancellationToken = default(CancellationToken)) { if (reference == null) { return MakeExceptionTask(new ArgumentNullException("reference")); } if (_references == null) { _references = new HashSet<ProjectReference>(); // If we needed to set _references, then we also need to clone // _typeDb to avoid adding modules to the shared database. if (_typeDb != null) { _typeDb = _typeDb.Clone(); } } switch (reference.Kind) { case ProjectReferenceKind.ExtensionModule: _references.Add(reference); string filename; try { filename = Path.GetFileNameWithoutExtension(reference.Name); } catch (Exception e) { return MakeExceptionTask(e); } if (_typeDb != null) { return Task.Factory.StartNew(EmptyTask); } break; } return Task.Factory.StartNew(EmptyTask); }
public void Dispose() { var evt = UnloadingDomain; if (evt != null) { evt(this, EventArgs.Empty); } if (_factory != null) { _factory.NewDatabaseAvailable -= OnNewDatabaseAvailable; } _typeDb = null; AppDomain.CurrentDomain.AssemblyResolve -= AssemblyResolver.Instance.CurrentDomain_AssemblyResolve; _unloader.Dispose(); #if DEBUG GC.SuppressFinalize(this); #endif }
public DatabaseReplacedEventArgs(PythonTypeDatabase newDatabase) { _newDatabase = newDatabase; }
public void Dispose() { _typeDb = null; var factory = _factory; _factory = null; if (factory != null) { factory.NewDatabaseAvailable -= OnNewDatabaseAvailable; } }
public void Dispose() { if (_factory != null) { _factory.NewDatabaseAvailable -= OnNewDatabaseAvailable; } _typeDb = null; AppDomain.CurrentDomain.AssemblyResolve -= AssemblyResolver.Instance.CurrentDomain_AssemblyResolve; _unloader.Dispose(); }
/// <summary> /// Determines whether the interpreter factory contains the specified /// modules. /// </summary> /// <returns>The names of the modules that were found.</returns> public static async Task <HashSet <string> > FindModulesAsync(this IPythonInterpreterFactory factory, params string[] moduleNames) { var finding = new HashSet <string>(moduleNames); var found = new HashSet <string>(); var withPackages = factory.PackageManager; if (withPackages != null) { foreach (var m in finding) { if ((await withPackages.GetInstalledPackageAsync(new PackageSpec(m), CancellationToken.None)).IsValid) { found.Add(m); } } finding.ExceptWith(found); if (!finding.Any()) { // Found all of them, so stop searching return(found); } } var withDb = factory as PythonInterpreterFactoryWithDatabase; if (withDb != null && withDb.IsCurrent) { var db = withDb.GetCurrentDatabase(); found.UnionWith(finding.Where(m => db.GetModule(m) != null)); // Always stop searching after this step return(found); } if (withDb != null) { try { var paths = await PythonTypeDatabase.GetDatabaseSearchPathsAsync(withDb); found.UnionWith(PythonTypeDatabase.GetDatabaseExpectedModules(withDb.Configuration.Version, paths) .SelectMany() .Select(g => g.ModuleName) .Where(m => finding.Contains(m))); } catch (InvalidOperationException) { } finding.ExceptWith(found); if (!finding.Any()) { // Found all of them, so stop searching return(found); } } return(await Task.Run(() => { foreach (var mp in ModulePath.GetModulesInLib(factory.Configuration)) { if (finding.Remove(mp.ModuleName)) { found.Add(mp.ModuleName); } if (!finding.Any()) { break; } } return found; })); }
public NoInterpretersInterpreter(PythonTypeDatabase database) { _database = database; }
public IPythonInterpreter CreateInterpreter() { return(new NoInterpretersInterpreter(PythonTypeDatabase.CreateDefaultTypeDatabase())); }
/// <summary> /// Creates a new interpreter factory with the specified database. This /// factory is suitable for analysis, but not execution. /// </summary> public static PythonInterpreterFactoryWithDatabase CreateAnalysisInterpreterFactory( Version languageVersion, PythonTypeDatabase database) { return(new AnalysisOnlyInterpreterFactory(languageVersion, database)); }
public override void GenerateDatabase(GenerateDatabaseOptions options, Action<int> onExit = null) { if (!Directory.Exists(Configuration.LibraryPath)) { return; } // Create and mark the DB directory as hidden try { var dbDir = Directory.CreateDirectory(DatabasePath); dbDir.Attributes |= FileAttributes.Hidden; } catch (ArgumentException) { } catch (IOException) { } catch (UnauthorizedAccessException) { } var req = new PythonTypeDatabaseCreationRequest { Factory = this, OutputPath = DatabasePath, SkipUnchanged = options.HasFlag(GenerateDatabaseOptions.SkipUnchanged), DetectLibraryPath = !AssumeSimpleLibraryLayout }; req.ExtraInputDatabases.Add(_base.DatabasePath); _baseHasRefreshed = false; if (_base.IsCurrent) { base.GenerateDatabase(req, onExit); } else { req.WaitFor = _base; req.SkipUnchanged = false; // Clear out the existing base database, since we're going to // need to reload it again. This also means that when // NewDatabaseAvailable is raised, we are expecting it and won't // incorrectly set _baseHasRefreshed to true again. _baseDb = null; // Start our analyzer first, since we will wait up to a minute // for our base analyzer to start (which may cause a one minute // delay if it completes before we start, but that is unlikely). base.GenerateDatabase(req, onExit); _base.GenerateDatabase(GenerateDatabaseOptions.SkipUnchanged); } }
/// <summary> /// Regenerates the database for this environment. If the base /// interpreter needs regenerating, it will also be regenerated. /// </summary> public override void GenerateDatabase(GenerateDatabaseOptions options, Action<int> onExit = null) { if (!Directory.Exists(Configuration.LibraryPath)) { return; } var req = new PythonTypeDatabaseCreationRequest { Factory = this, OutputPath = DatabasePath, SkipUnchanged = options.HasFlag(GenerateDatabaseOptions.SkipUnchanged) }; req.ExtraInputDatabases.Add(_base.DatabasePath); _baseHasRefreshed = false; if (_base.IsCurrent) { // The App database is already up to date, so start analyzing // the User database immediately. base.GenerateDatabase(req, onExit); } else { // The App database needs to be updated, so start both and wait // for the base to finish before analyzing User. // Specifying our base interpreter as 'WaitFor' allows the UI to // forward progress and status messages to the user, even though // the factory is not visible. req.WaitFor = _base; // Because the underlying analysis of the standard library has // changed, we must reanalyze the entire database. req.SkipUnchanged = false; // Clear out the existing base database, since we're going to // need to reload it again. This also means that when // NewDatabaseAvailable is raised, we are expecting it and won't // incorrectly set _baseHasRefreshed to true again. _baseDb = null; // Start our analyzer first, since we will wait up to a minute // for the base analyzer to start (which may cause a one minute // delay if it completes before we start, but that is unlikely). base.GenerateDatabase(req, onExit); _base.GenerateDatabase(GenerateDatabaseOptions.SkipUnchanged); } }
/// <summary> /// Called to manually trigger a refresh of <see cref="IsCurrent"/>. /// After completion, <see cref="IsCurrentChanged"/> will always be /// raised, regardless of whether the values were changed. /// </summary> public virtual void RefreshIsCurrent() { #if DEBUG // Indicate that we've arrived here at least once. Otherwise we // will assert. _hasEverCheckedDatabase = true; #endif if (GlobalInterpreterOptions.SuppressFileSystemWatchers || string.IsNullOrEmpty(DatabasePath)) { _isCheckingDatabase = false; _isValid = true; _missingModules = null; _isCurrentException = null; OnIsCurrentChanged(); return; } if (IsGenerating) { if (PythonTypeDatabase.IsDatabaseRegenerating(DatabasePath)) { _isValid = false; _missingModules = null; _isCurrentException = null; return; } var generating = Interlocked.Exchange(ref _generating, null); generating?.Dispose(); } try { if (!_isCurrentSemaphore.Wait(0)) { // Another thread is working on our state, so we will wait for // them to finish and return, since the value is up to date. _isCurrentSemaphore.Wait(); _isCurrentSemaphore.Release(); return; } } catch (ObjectDisposedException) { // We've been disposed and the call has come in from // externally, probably a timer. return; } try { _isCheckingDatabase = true; OnIsCurrentChanged(); // Wait for a slot that will allow us to scan the disk. This // prevents too many threads from updating at once and causing // I/O saturation. _sharedIsCurrentSemaphore.Wait(); try { if (PythonTypeDatabase.IsDatabaseRegenerating(DatabasePath) || !PythonTypeDatabase.IsDatabaseVersionCurrent(DatabasePath)) { // Skip the rest of the checks, because we know we're // not current. _isValid = false; _missingModules = null; _isCurrentException = null; } else { _isValid = true; HashSet <string> existingDatabase = null; string[] missingModules = null; for (int retries = 3; retries > 0; --retries) { try { existingDatabase = GetExistingDatabase(DatabasePath); break; } catch (UnauthorizedAccessException) { } catch (IOException) { } Thread.Sleep(100); } if (existingDatabase == null) { // This will either succeed or throw again. If it throws // then the error is reported to the user. existingDatabase = GetExistingDatabase(DatabasePath); } for (int retries = 3; retries > 0; --retries) { try { missingModules = GetMissingModules(existingDatabase); break; } catch (UnauthorizedAccessException) { } catch (IOException) { } Thread.Sleep(100); } if (missingModules == null) { // This will either succeed or throw again. If it throws // then the error is reported to the user. missingModules = GetMissingModules(existingDatabase); } if (missingModules.Length > 0) { var oldModules = _missingModules; if (oldModules == null || oldModules.Length != missingModules.Length || !oldModules.SequenceEqual(missingModules)) { } _missingModules = missingModules; } else { _missingModules = null; } } _isCurrentException = null; } finally { _sharedIsCurrentSemaphore.Release(); } } catch (Exception ex) { // Report the exception text as the reason. _isCurrentException = ex.ToString(); _missingModules = null; } finally { _isCheckingDatabase = false; try { _isCurrentSemaphore.Release(); } catch (ObjectDisposedException) { // The semaphore is not locked for disposal as it is only // used to prevent reentrance into this function. As a // result, it may have been disposed while we were in here. } } OnIsCurrentChanged(); }
public PythonUwpInterpreter(PythonInterpreterFactoryWithDatabase factory) { _langVersion = factory.Configuration.Version; _factory = factory; _typeDb = _factory.GetCurrentDatabase(); _factory.NewDatabaseAvailable += OnNewDatabaseAvailable; }
private async Task CacheInstalledPackagesAsync(bool alreadyHasLock, CancellationToken cancellationToken) { if (!IsReady) { await UpdateIsReadyAsync(alreadyHasLock, cancellationToken); if (!IsReady) { return; } } List <PackageSpec> packages = null; var workingLock = alreadyHasLock ? null : await _working.LockAsync(cancellationToken); try { var args = _extraInterpreterArgs.ToList(); if (!SupportsDashMPip) { args.Add("-c"); args.Add("\"import pip; pip.main()\""); } else { args.Add("-m"); args.Add("pip"); } args.Add("list"); using (await _concurrencyLock.LockAsync(cancellationToken)) { using (var proc = ProcessOutput.Run( _factory.Configuration.InterpreterPath, args, _factory.Configuration.PrefixPath, UnbufferedEnv, false, null )) { try { if (await proc == 0) { packages = proc.StandardOutputLines .Select(i => PackageSpec.FromPipList(i)) .Where(p => p.IsValid) .OrderBy(p => p.Name) .ToList(); } } catch (OperationCanceledException) { // Process failed to run Debug.WriteLine("Failed to run pip to collect packages"); Debug.WriteLine(string.Join(Environment.NewLine, proc.StandardOutputLines)); } } if (packages == null) { // Pip failed, so return a directory listing var paths = await PythonTypeDatabase.GetDatabaseSearchPathsAsync(_factory); packages = await Task.Run(() => paths.Where(p => !p.IsStandardLibrary && Directory.Exists(p.Path)) .SelectMany(p => PathUtils.EnumerateDirectories(p.Path, recurse: false)) .Select(path => Path.GetFileName(path)) .Select(name => PackageNameRegex.Match(name)) .Where(match => match.Success) .Select(match => new PackageSpec(match.Groups["name"].Value)) .Where(p => p.IsValid) .OrderBy(p => p.Name) .ToList()); } } // Outside of concurrency lock, still in working lock _packages.Clear(); _packages.AddRange(packages); _everCached = true; } finally { workingLock?.Dispose(); } InstalledPackagesChanged?.Invoke(this, EventArgs.Empty); }
// We could override MakeInterpreter() to return a different // implementation of IPythonInterpreter, but we don't need to here, as // the default implementation works well with PythonTypeDatabase. //public override IPythonInterpreter MakeInterpreter(PythonInterpreterFactoryWithDatabase factory) { // return base.MakeInterpreter(factory); //} /// <summary> /// Returns a new database that contains the database from our base /// interpreter. /// </summary> public override PythonTypeDatabase MakeTypeDatabase(string databasePath, bool includeSitePackages = true) { if (_baseDb == null && _base.IsCurrent) { _baseDb = _base.GetCurrentDatabase(ShouldIncludeGlobalSitePackages); } var paths = new List<string> { databasePath }; if (includeSitePackages) { try { paths.AddRange(Directory.EnumerateDirectories(databasePath)); } catch (ArgumentException) { } catch (IOException) { } catch (SecurityException) { } catch (UnauthorizedAccessException) { } } return new PythonTypeDatabase(this, paths, _baseDb); }
private async Task CreateLibraryWatchers() { Debug.Assert(_libWatchers != null, "Should not create watchers when suppressed"); IList <PythonLibraryPath> paths; try { paths = await PythonTypeDatabase.GetDatabaseSearchPathsAsync(_factory); } catch (InvalidOperationException) { return; } paths = paths .Where(p => Directory.Exists(p.Path)) .OrderBy(p => p.Path.Length) .ToList(); var watching = new List <string>(); var watchers = new List <FileSystemWatcher>(); foreach (var path in paths) { if (watching.Any(p => PathUtils.IsSubpathOf(p, path.Path))) { continue; } FileSystemWatcher watcher = null; try { watcher = new FileSystemWatcher { IncludeSubdirectories = true, Path = path.Path, NotifyFilter = NotifyFilters.FileName | NotifyFilters.DirectoryName | NotifyFilters.LastWrite }; watcher.Created += OnChanged; watcher.Deleted += OnChanged; watcher.Changed += OnChanged; watcher.Renamed += OnRenamed; watcher.EnableRaisingEvents = true; watching.Add(path.Path); watchers.Add(watcher); } catch (IOException) { // Raced with directory deletion. We normally handle the // library being deleted by disposing the watcher, but this // occurs in response to an event from the watcher. Because // we never got to start watching, we will just dispose // immediately. watcher?.Dispose(); } catch (ArgumentException ex) { watcher?.Dispose(); Debug.WriteLine("Error starting FileSystemWatcher:\r\n{0}", ex); } } List <FileSystemWatcher> oldWatchers; lock (_libWatchers) { oldWatchers = _libWatchers.ToList(); _libWatchers.Clear(); _libWatchers.AddRange(watchers); } foreach (var oldWatcher in oldWatchers) { oldWatcher.EnableRaisingEvents = false; oldWatcher.Dispose(); } }
void OnNewDatabaseAvailable(object sender, EventArgs e) { var mod = _modules["__builtin__"] as IBuiltinPythonModule; _typeDb = _factory.GetCurrentDatabase().CloneWithNewBuiltins(mod); RaiseModuleNamesChanged(); }
public ExtensionModuleLoader(PythonTypeDatabase typeDb, IPythonInterpreterFactory factory, string moduleName, string extensionFilename, CancellationToken cancel) { _typeDb = typeDb; _factory = factory; _moduleName = moduleName; _extensionFilename = extensionFilename; _cancel = cancel; }
public void PydInPackage() { PythonPaths.Python27.AssertInstalled(); var outputPath = TestData.GetTempPath(randomSubPath: true); Console.WriteLine("Writing to: " + outputPath); // run the analyzer using (var output = ProcessOutput.RunHiddenAndCapture("Microsoft.PythonTools.Analyzer.exe", "/python", PythonPaths.Python27.InterpreterPath, "/lib", TestData.GetPath(@"TestData\PydStdLib"), "/version", "2.7", "/outdir", outputPath, "/indir", TestData.GetPath("CompletionDB"), "/log", "AnalysisLog.txt")) { output.Wait(); Console.WriteLine("* Stdout *"); foreach (var line in output.StandardOutputLines) { Console.WriteLine(line); } Console.WriteLine("* Stderr *"); foreach (var line in output.StandardErrorLines) { Console.WriteLine(line); } Assert.AreEqual(0, output.ExitCode); } var fact = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(new Version(2, 7)); var paths = new List<string> { outputPath }; paths.AddRange(Directory.EnumerateDirectories(outputPath)); var typeDb = new PythonTypeDatabase(fact, paths); var module = typeDb.GetModule("Package.winsound"); Assert.IsNotNull(module, "Package.winsound was not analyzed"); var package = typeDb.GetModule("Package"); Assert.IsNotNull(package, "Could not import Package"); var member = package.GetMember(null, "winsound"); Assert.IsNotNull(member, "Could not get member Package.winsound"); Assert.AreSame(module, member); module = typeDb.GetModule("Package._testcapi"); Assert.IsNotNull(module, "Package._testcapi was not analyzed"); package = typeDb.GetModule("Package"); Assert.IsNotNull(package, "Could not import Package"); member = package.GetMember(null, "_testcapi"); Assert.IsNotNull(member, "Could not get member Package._testcapi"); Assert.IsNotInstanceOfType(member, typeof(CPythonMultipleMembers)); Assert.AreSame(module, member); module = typeDb.GetModule("Package.select"); Assert.IsNotNull(module, "Package.select was not analyzed"); package = typeDb.GetModule("Package"); Assert.IsNotNull(package, "Could not import Package"); member = package.GetMember(null, "select"); Assert.IsNotNull(member, "Could not get member Package.select"); Assert.IsInstanceOfType(member, typeof(CPythonMultipleMembers)); var mm = (CPythonMultipleMembers)member; AssertUtil.ContainsExactly(mm.Members.Select(m => m.MemberType), PythonMemberType.Module, PythonMemberType.Constant, PythonMemberType.Class ); Assert.IsNotNull(mm.Members.Contains(module)); try { // Only clean up if the test passed Directory.Delete(outputPath, true); } catch { } }
public PythonTypeDatabase CloneWithNewBuiltins(IBuiltinPythonModule newBuiltins) { var newDb = new PythonTypeDatabase(_factory, null, this); newDb._sharedState.BuiltinModule = newBuiltins; return newDb; }
/// <summary> /// Returns the database for this factory. This database may be shared /// between callers and should be cloned before making modifications. /// /// This function never returns null. /// </summary> public PythonTypeDatabase GetCurrentDatabase() { if (_typeDb == null || _typeDb.DatabaseDirectory != DatabasePath) { _typeDb = MakeTypeDatabase(DatabasePath) ?? PythonTypeDatabase.CreateDefaultTypeDatabase(this); } return _typeDb; }
private void OnNewDatabaseAvailable(object sender, EventArgs e) { // Raised if our base database is updated, which means we need to // refresh our database too. if (_baseDb != null) { _baseDb = null; _baseHasRefreshed = true; } OnNewDatabaseAvailable(); OnIsCurrentChanged(); }
/// <summary> /// Creates a new interpreter factory with the specified database. This /// factory is suitable for analysis, but not execution. /// </summary> public static PythonInterpreterFactoryWithDatabase CreateAnalysisInterpreterFactory( Version languageVersion, PythonTypeDatabase database) { return new AnalysisOnlyInterpreterFactory(languageVersion, database); }
private async Task CacheInstalledPackagesAsync( bool alreadyHasLock, bool alreadyHasConcurrencyLock, CancellationToken cancellationToken ) { if (!IsReady) { await UpdateIsReadyAsync(alreadyHasLock, cancellationToken); if (!IsReady) { return; } } List <PackageSpec> packages = null; var workingLock = alreadyHasLock ? null : await _working.LockAsync(cancellationToken); try { var args = _extraInterpreterArgs.ToList(); if (!SupportsDashMPip) { args.Add("-c"); args.Add("\"import pip; pip.main()\""); } else { args.Add("-m"); args.Add("pip"); } args.Add("list"); if (_pipListHasFormatOption) { args.Add("--format=json"); } var concurrencyLock = alreadyHasConcurrencyLock ? null : await _concurrencyLock.LockAsync(cancellationToken); try { using (var proc = ProcessOutput.Run( _factory.Configuration.InterpreterPath, args, _factory.Configuration.PrefixPath, UnbufferedEnv, false, null )) { try { if ((await proc) == 0) { if (_pipListHasFormatOption) { try { var data = JToken.ReadFrom(new JsonTextReader(new StringListReader(proc.StandardOutputLines))); packages = data .Select(j => new PackageSpec(j.Value <string>("name"), j.Value <string>("version"))) .Where(p => p.IsValid) .OrderBy(p => p.Name) .ToList(); } catch (JsonException ex) { Debug.WriteLine("Failed to parse: {0}".FormatInvariant(ex.Message)); foreach (var l in proc.StandardOutputLines) { Debug.WriteLine(l); } } } else { packages = proc.StandardOutputLines .Select(i => PackageSpec.FromPipList(i)) .Where(p => p.IsValid) .OrderBy(p => p.Name) .ToList(); } } else if (_pipListHasFormatOption) { // Actually, pip probably doesn't have the --format option Debug.WriteLine("{0} does not support --format".FormatInvariant(_factory.Configuration.InterpreterPath)); _pipListHasFormatOption = false; await CacheInstalledPackagesAsync(true, true, cancellationToken); return; } else { } } catch (OperationCanceledException) { // Process failed to run Debug.WriteLine("Failed to run pip to collect packages"); foreach (var line in proc.StandardOutputLines) { Debug.WriteLine(line); } } } if (packages == null) { // Pip failed, so return a directory listing var paths = await PythonTypeDatabase.GetDatabaseSearchPathsAsync(_factory); packages = await Task.Run(() => paths.Where(p => !p.IsStandardLibrary && Directory.Exists(p.Path)) .SelectMany(p => PathUtils.EnumerateDirectories(p.Path, recurse: false)) .Select(path => Path.GetFileName(path)) .Select(name => PackageNameRegex.Match(name)) .Where(match => match.Success) .Select(match => new PackageSpec(match.Groups["name"].Value)) .Where(p => p.IsValid) .OrderBy(p => p.Name) .ToList()); } } finally { concurrencyLock?.Dispose(); } // Outside of concurrency lock, still in working lock _packages.Clear(); _packages.AddRange(packages); _everCached = true; } finally { workingLock?.Dispose(); } InstalledPackagesChanged?.Invoke(this, EventArgs.Empty); }
public void Dispose() { if (_factory != null) { _factory.NewDatabaseAvailable -= OnNewDatabaseAvailable; } _typeDb = null; AppDomain.CurrentDomain.AssemblyResolve -= AssemblyResolver.Instance.CurrentDomain_AssemblyResolve; _unloader.Dispose(); #if DEBUG GC.SuppressFinalize(this); #endif }
public override PythonTypeDatabase MakeTypeDatabase(string databasePath, bool includeSitePackages = true) { if (_baseDb == null && _base.IsCurrent) { var includeBaseSitePackages = ShouldIncludeGlobalSitePackages( Configuration.PrefixPath, Configuration.LibraryPath ); _baseDb = _base.GetCurrentDatabase(includeBaseSitePackages); } if (!IsCurrent || !Directory.Exists(databasePath)) { return _baseDb; } var paths = new List<string> { databasePath }; if (includeSitePackages) { try { paths.AddRange(Directory.EnumerateDirectories(databasePath)); } catch (IOException) { } catch (UnauthorizedAccessException) { } } return new PythonTypeDatabase(this, paths, _baseDb); }
public void LayeredDatabase() { using (var db1 = MockCompletionDB.Create(PythonLanguageVersion.V27, "os")) using (var db2 = MockCompletionDB.Create(PythonLanguageVersion.V27, "posixpath")) { Assert.IsNotNull(db1.Database.GetModule("os")); Assert.IsNull(db1.Database.GetModule("posixpath")); var ptd1 = db1.Database; var ptd2 = ptd1.Clone(); ptd2.LoadDatabase(db2.DBPath); Assert.IsNull(ptd1.GetModule("posixpath")); Assert.IsNotNull(ptd2.GetModule("os")); Assert.IsNotNull(ptd2.GetModule("posixpath")); Assert.AreSame(ptd1.GetModule("os"), db1.Database.GetModule("os")); Assert.AreSame(ptd2.GetModule("os"), db1.Database.GetModule("os")); } var factory = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(new Version(2, 7)); using (var db1 = MockCompletionDB.Create(PythonLanguageVersion.V27, "os", "posixpath")) using (var db2 = MockCompletionDB.Create(PythonLanguageVersion.V27, "posixpath")) { var ptd1 = new PythonTypeDatabase(factory, new[] { db1.DBPath, db2.DBPath }); Assert.IsNotNull(ptd1.GetModule("posixpath")); Assert.AreNotSame(ptd1.GetModule("posixpath"), db1.Database.GetModule("posixpath")); Assert.AreNotSame(ptd1.GetModule("posixpath"), db2.Database.GetModule("posixpath")); var ptd2 = new PythonTypeDatabase(factory, new[] { db2.DBPath, db1.DBPath }); Assert.IsNotNull(ptd2.GetModule("posixpath")); Assert.AreNotSame(ptd2.GetModule("posixpath"), db1.Database.GetModule("posixpath")); Assert.AreNotSame(ptd2.GetModule("posixpath"), db2.Database.GetModule("posixpath")); } using (var db1 = MockCompletionDB.Create(PythonLanguageVersion.V27, "os", "posixpath")) using (var db2 = MockCompletionDB.Create(PythonLanguageVersion.V27, "posixpath")) using (var db3 = MockCompletionDB.Create(PythonLanguageVersion.V27, "ntpath")) { var ptd = db1.Database; Assert.AreSame(ptd.GetModule("posixpath"), db1.Database.GetModule("posixpath")); Assert.AreNotSame(ptd.GetModule("posixpath"), db2.Database.GetModule("posixpath")); var ptd2 = ptd.Clone(); ptd2.LoadDatabase(db2.DBPath); Assert.AreNotSame(ptd2.GetModule("posixpath"), ptd.GetModule("posixpath")); var ptd3 = ptd2.Clone(); ptd3.LoadDatabase(db3.DBPath); Assert.IsNotNull(ptd3.GetModule("ntpath")); Assert.IsNull(ptd2.GetModule("ntpath")); } }
protected override void Dispose(bool disposing) { _baseDb = null; _base.IsCurrentChanged -= OnIsCurrentChanged; _base.NewDatabaseAvailable -= OnNewDatabaseAvailable; base.Dispose(disposing); }