Example #1
0
        public IronPythonInterpreter(PythonInterpreterFactoryWithDatabase factory) {
#if DEBUG
            _id = Interlocked.Increment(ref _interpreterCount);
            Debug.WriteLine(String.Format("IronPython Interpreter Created {0}", _id));
            Debug.WriteLine(new StackTrace(true).ToString());
#endif

            AppDomain.CurrentDomain.AssemblyResolve += AssemblyResolver.Instance.CurrentDomain_AssemblyResolve;

            InitializeRemoteDomain();

            try {
                LoadAssemblies();
            } catch {
                // IronPython not installed in the GAC...
            }

            var mod = Remote.ImportBuiltinModule("__builtin__");
            var newMod = new IronPythonBuiltinModule(this, mod, "__builtin__");
            _modules[newMod.Name] = newMod;

            _factory = factory;
            _typeDb = _factory.GetCurrentDatabase().CloneWithNewBuiltins(newMod);
            _factory.NewDatabaseAvailable += OnNewDatabaseAvailable;

            LoadModules();
        }
 private void Base_NewDatabaseAvailable(object sender, EventArgs e) {
     if (_baseDb != null) {
         _baseDb = null;
         _baseHasRefreshed = true;
     }
     OnNewDatabaseAvailable();
     OnIsCurrentChanged();
 }
Example #3
0
        public static KnownTypes CreateDefault(PythonAnalyzer state, PythonTypeDatabase fallbackDb) {
            var res = new KnownTypes();

            var fallback = fallbackDb.BuiltinModule;

            for (int value = 0; value < res._types.Length; ++value) {
                res._types[value] = (IPythonType)fallback.GetAnyMember(
                    ((ITypeDatabaseReader)fallbackDb).GetBuiltinTypeName((BuiltinTypeId)value)
                );
            }

            res.SetClassInfo(state);
            return res;
        }
Example #4
0
        public static KnownTypes Create(PythonAnalyzer state, PythonTypeDatabase fallbackDb) {
            var res = new KnownTypes();

            var interpreter = state.Interpreter;
            var fallback = fallbackDb.BuiltinModule;

            for (int value = 0; value < res._types.Length; ++value) {
                try {
                    res._types[value] = interpreter.GetBuiltinType((BuiltinTypeId)value);
                } catch (KeyNotFoundException) {
                    res._types[value] = (IPythonType)fallback.GetAnyMember(
                        ((ITypeDatabaseReader)fallbackDb).GetBuiltinTypeName((BuiltinTypeId)value)
                    );
                }
            }

            res.SetClassInfo(state);
            return res;
        }
Example #5
0
        public PythonTypeDatabase(
            PythonInterpreterFactoryWithDatabase factory,
            IEnumerable<string> databaseDirectories = null,
            PythonTypeDatabase innerDatabase = null
        ) {
            if (innerDatabase != null && factory.Configuration.Version != innerDatabase.LanguageVersion) {
                throw new InvalidOperationException("Language versions do not match");
            }

            _factory = factory;
            if (innerDatabase != null) {
                _sharedState = new SharedDatabaseState(innerDatabase._sharedState);
            } else {
                _sharedState = new SharedDatabaseState(_factory.Configuration.Version);
            }

            if (databaseDirectories != null) {
                foreach (var d in databaseDirectories) {
                    LoadDatabase(d);
                }
            }

            _sharedState.ListenForCorruptDatabase(this);
        }
        /// <summary>
        /// Clears any cached type databases and raises the
        /// <see cref="NewDatabaseAvailable"/> event.
        /// </summary>
        protected void OnNewDatabaseAvailable() {
            _typeDb = null;
            _typeDbWithoutPackages = null;

            var evt = NewDatabaseAvailable;
            if (evt != null) {
                evt(this, EventArgs.Empty);
            }
        }
Example #7
0
        /// <summary>
        /// Creates a new interpreter factory backed by a type database.
        /// </summary>
        /// <remarks>
        /// <see cref="RefreshIsCurrent"/> must be called after creation to
        /// ensure the database state is correctly reflected.
        /// </remarks>
        public PythonInterpreterFactoryWithDatabase(
            InterpreterConfiguration config,
            InterpreterFactoryCreationOptions options
            )
        {
            if (config == null)
            {
                throw new ArgumentNullException(nameof(config));
            }
            if (options == null)
            {
                options = new InterpreterFactoryCreationOptions();
            }
            Configuration = config;

            DatabasePath = options.DatabasePath;

            // Avoid creating a interpreter with an unsupported version.
            // https://github.com/Microsoft/PTVS/issues/706
            try {
                var langVer = config.Version.ToLanguageVersion();
            } catch (InvalidOperationException ex) {
                throw new ArgumentException(ex.Message, ex);
            }

            if (options.WatchFileSystem && !string.IsNullOrEmpty(DatabasePath))
            {
                // Assume the database is valid if the version is up to date, then
                // switch to invalid after we've checked.
                _isValid            = PythonTypeDatabase.IsDatabaseVersionCurrent(DatabasePath);
                _isCheckingDatabase = true;

                _verWatcher    = CreateDatabaseVerWatcher();
                _verDirWatcher = CreateDatabaseDirectoryWatcher();
#if DEBUG
                var creationStack = new StackTrace(true).ToString();
                Task.Delay(1000).ContinueWith(t => {
                    Debug.Assert(
                        _hasEverCheckedDatabase,
                        "Database check was not triggered for {0}".FormatUI(Configuration.Id),
                        creationStack
                        );
                });
#endif
            }
            else
            {
                // Assume the database is valid
                _isValid = true;
            }

            try {
                var pm = options.PackageManager;
                if (pm != null)
                {
                    pm.SetInterpreterFactory(this);
                    pm.InstalledFilesChanged += PackageManager_InstalledFilesChanged;
                    PackageManager            = pm;
                }
            } catch (NotSupportedException) {
            }
        }
Example #8
0
        public void TestPthFiles() {
            var outputPath = TestData.GetTempPath(randomSubPath: true);
            Console.WriteLine("Writing to: " + outputPath);

            // run the analyzer
            using (var output = ProcessOutput.RunHiddenAndCapture("Microsoft.PythonTools.Analyzer.exe",
                "/lib", TestData.GetPath(@"TestData\PathStdLib"),
                "/version", "2.7",
                "/outdir", outputPath,
                "/indir", TestData.GetPath("CompletionDB"),
                "/log", "AnalysisLog.txt")) {
                output.Wait();
                Console.WriteLine("* Stdout *");
                foreach (var line in output.StandardOutputLines) {
                    Console.WriteLine(line);
                }
                Console.WriteLine("* Stderr *");
                foreach (var line in output.StandardErrorLines) {
                    Console.WriteLine(line);
                }
                Assert.AreEqual(0, output.ExitCode);
            }

            File.Copy(TestData.GetPath(@"CompletionDB\__builtin__.idb"), Path.Combine(outputPath, "__builtin__.idb"));

            var fact = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(new Version(2, 7));
            var paths = new List<string> { outputPath };
            paths.AddRange(Directory.EnumerateDirectories(outputPath));
            var typeDb = new PythonTypeDatabase(fact, paths);
            var module = typeDb.GetModule("SomeLib");
            Assert.IsNotNull(module, "Could not import SomeLib");
            var fobMod = typeDb.GetModule("SomeLib.fob");
            Assert.IsNotNull(fobMod, "Could not import SomeLib.fob");

            var cClass = ((IPythonModule)fobMod).GetMember(null, "C");
            Assert.IsNotNull(cClass, "Could not get SomeLib.fob.C");

            Assert.AreEqual(PythonMemberType.Class, cClass.MemberType);
        }
Example #9
0
        /// <summary>
        /// Sets up a weak reference for notification of when the shared database
        /// has become corrupted.  Doesn't keep the listening database alive.
        /// </summary>
        public void ListenForCorruptDatabase(PythonTypeDatabase db) {
            lock (_corruptListeners) {
                for (int i = 0; i < _corruptListeners.Count; i++) {
                    var target = _corruptListeners[i].Target;
                    if (target == null) {
                        _corruptListeners[i].Target = db;
                        return;
                    }
                }

                _corruptListeners.Add(new WeakReference(db));
            }
        }
Example #10
0
        public Task AddReferenceAsync(ProjectReference reference, CancellationToken cancellationToken = default(CancellationToken)) {
            if (reference == null) {
                return MakeExceptionTask(new ArgumentNullException("reference"));
            }

            if (_references == null) {
                _references = new HashSet<ProjectReference>();
                // If we needed to set _references, then we also need to clone
                // _typeDb to avoid adding modules to the shared database.
                if (_typeDb != null) {
                    _typeDb = _typeDb.Clone();
                }
            }

            switch (reference.Kind) {
                case ProjectReferenceKind.ExtensionModule:
                    _references.Add(reference);
                    string filename;
                    try {
                        filename = Path.GetFileNameWithoutExtension(reference.Name);
                    } catch (Exception e) {
                        return MakeExceptionTask(e);
                    }

                    if (_typeDb != null) {
                        return Task.Factory.StartNew(EmptyTask);
                    }
                    break;
            }

            return Task.Factory.StartNew(EmptyTask);
        }
Example #11
0
        public void Dispose() {
            var evt = UnloadingDomain;
            if (evt != null) {
                evt(this, EventArgs.Empty);
            }

            if (_factory != null) {
                _factory.NewDatabaseAvailable -= OnNewDatabaseAvailable;
            }
            _typeDb = null;
            AppDomain.CurrentDomain.AssemblyResolve -= AssemblyResolver.Instance.CurrentDomain_AssemblyResolve;
            _unloader.Dispose();
#if DEBUG
            GC.SuppressFinalize(this);
#endif
        }
Example #12
0
 public DatabaseReplacedEventArgs(PythonTypeDatabase newDatabase)
 {
     _newDatabase = newDatabase;
 }
Example #13
0
        public void Dispose() {
            _typeDb = null;

            var factory = _factory;
            _factory = null;
            if (factory != null) {
                factory.NewDatabaseAvailable -= OnNewDatabaseAvailable;
            }
        }
Example #14
0
 public void Dispose() {
     if (_factory != null) {
         _factory.NewDatabaseAvailable -= OnNewDatabaseAvailable;
     }
     _typeDb = null;
     AppDomain.CurrentDomain.AssemblyResolve -= AssemblyResolver.Instance.CurrentDomain_AssemblyResolve;
     _unloader.Dispose();
 }
Example #15
0
        /// <summary>
        /// Determines whether the interpreter factory contains the specified
        /// modules.
        /// </summary>
        /// <returns>The names of the modules that were found.</returns>
        public static async Task <HashSet <string> > FindModulesAsync(this IPythonInterpreterFactory factory, params string[] moduleNames)
        {
            var finding      = new HashSet <string>(moduleNames);
            var found        = new HashSet <string>();
            var withPackages = factory.PackageManager;

            if (withPackages != null)
            {
                foreach (var m in finding)
                {
                    if ((await withPackages.GetInstalledPackageAsync(new PackageSpec(m), CancellationToken.None)).IsValid)
                    {
                        found.Add(m);
                    }
                }
                finding.ExceptWith(found);
                if (!finding.Any())
                {
                    // Found all of them, so stop searching
                    return(found);
                }
            }

            var withDb = factory as PythonInterpreterFactoryWithDatabase;

            if (withDb != null && withDb.IsCurrent)
            {
                var db = withDb.GetCurrentDatabase();
                found.UnionWith(finding.Where(m => db.GetModule(m) != null));

                // Always stop searching after this step
                return(found);
            }

            if (withDb != null)
            {
                try {
                    var paths = await PythonTypeDatabase.GetDatabaseSearchPathsAsync(withDb);

                    found.UnionWith(PythonTypeDatabase.GetDatabaseExpectedModules(withDb.Configuration.Version, paths)
                                    .SelectMany()
                                    .Select(g => g.ModuleName)
                                    .Where(m => finding.Contains(m)));
                } catch (InvalidOperationException) {
                }

                finding.ExceptWith(found);
                if (!finding.Any())
                {
                    // Found all of them, so stop searching
                    return(found);
                }
            }

            return(await Task.Run(() => {
                foreach (var mp in ModulePath.GetModulesInLib(factory.Configuration))
                {
                    if (finding.Remove(mp.ModuleName))
                    {
                        found.Add(mp.ModuleName);
                    }

                    if (!finding.Any())
                    {
                        break;
                    }
                }
                return found;
            }));
        }
Example #16
0
 public NoInterpretersInterpreter(PythonTypeDatabase database)
 {
     _database = database;
 }
Example #17
0
 public IPythonInterpreter CreateInterpreter()
 {
     return(new NoInterpretersInterpreter(PythonTypeDatabase.CreateDefaultTypeDatabase()));
 }
Example #18
0
 /// <summary>
 /// Creates a new interpreter factory with the specified database. This
 /// factory is suitable for analysis, but not execution.
 /// </summary>
 public static PythonInterpreterFactoryWithDatabase CreateAnalysisInterpreterFactory(
     Version languageVersion,
     PythonTypeDatabase database)
 {
     return(new AnalysisOnlyInterpreterFactory(languageVersion, database));
 }
        public override void GenerateDatabase(GenerateDatabaseOptions options, Action<int> onExit = null) {
            if (!Directory.Exists(Configuration.LibraryPath)) {
                return;
            }

            // Create and mark the DB directory as hidden
            try {
                var dbDir = Directory.CreateDirectory(DatabasePath);
                dbDir.Attributes |= FileAttributes.Hidden;
            } catch (ArgumentException) {
            } catch (IOException) {
            } catch (UnauthorizedAccessException) {
            }

            var req = new PythonTypeDatabaseCreationRequest {
                Factory = this,
                OutputPath = DatabasePath,
                SkipUnchanged = options.HasFlag(GenerateDatabaseOptions.SkipUnchanged),
                DetectLibraryPath = !AssumeSimpleLibraryLayout
            };

            req.ExtraInputDatabases.Add(_base.DatabasePath);

            _baseHasRefreshed = false;

            if (_base.IsCurrent) {
                base.GenerateDatabase(req, onExit);
            } else {
                req.WaitFor = _base;
                req.SkipUnchanged = false;

                // Clear out the existing base database, since we're going to
                // need to reload it again. This also means that when
                // NewDatabaseAvailable is raised, we are expecting it and won't
                // incorrectly set _baseHasRefreshed to true again.
                _baseDb = null;

                // Start our analyzer first, since we will wait up to a minute
                // for our base analyzer to start (which may cause a one minute
                // delay if it completes before we start, but that is unlikely).
                base.GenerateDatabase(req, onExit);
                _base.GenerateDatabase(GenerateDatabaseOptions.SkipUnchanged);
            }
        }
Example #20
0
 public DatabaseReplacedEventArgs(PythonTypeDatabase newDatabase) {
     _newDatabase = newDatabase;
 }
Example #21
0
        /// <summary>
        /// Regenerates the database for this environment. If the base
        /// interpreter needs regenerating, it will also be regenerated.
        /// </summary>
        public override void GenerateDatabase(GenerateDatabaseOptions options, Action<int> onExit = null) {
            if (!Directory.Exists(Configuration.LibraryPath)) {
                return;
            }

            var req = new PythonTypeDatabaseCreationRequest {
                Factory = this,
                OutputPath = DatabasePath,
                SkipUnchanged = options.HasFlag(GenerateDatabaseOptions.SkipUnchanged)
            };

            req.ExtraInputDatabases.Add(_base.DatabasePath);

            _baseHasRefreshed = false;

            if (_base.IsCurrent) {
                // The App database is already up to date, so start analyzing
                // the User database immediately.
                base.GenerateDatabase(req, onExit);
            } else {
                // The App database needs to be updated, so start both and wait
                // for the base to finish before analyzing User.

                // Specifying our base interpreter as 'WaitFor' allows the UI to
                // forward progress and status messages to the user, even though
                // the factory is not visible.
                req.WaitFor = _base;

                // Because the underlying analysis of the standard library has
                // changed, we must reanalyze the entire database.
                req.SkipUnchanged = false;

                // Clear out the existing base database, since we're going to
                // need to reload it again. This also means that when
                // NewDatabaseAvailable is raised, we are expecting it and won't
                // incorrectly set _baseHasRefreshed to true again.
                _baseDb = null;

                // Start our analyzer first, since we will wait up to a minute
                // for the base analyzer to start (which may cause a one minute
                // delay if it completes before we start, but that is unlikely).
                base.GenerateDatabase(req, onExit);
                _base.GenerateDatabase(GenerateDatabaseOptions.SkipUnchanged);
            }
        }
        /// <summary>
        /// Called to manually trigger a refresh of <see cref="IsCurrent"/>.
        /// After completion, <see cref="IsCurrentChanged"/> will always be
        /// raised, regardless of whether the values were changed.
        /// </summary>
        public virtual void RefreshIsCurrent()
        {
#if DEBUG
            // Indicate that we've arrived here at least once. Otherwise we
            // will assert.
            _hasEverCheckedDatabase = true;
#endif

            if (GlobalInterpreterOptions.SuppressFileSystemWatchers || string.IsNullOrEmpty(DatabasePath))
            {
                _isCheckingDatabase = false;
                _isValid            = true;
                _missingModules     = null;
                _isCurrentException = null;
                OnIsCurrentChanged();
                return;
            }
            if (IsGenerating)
            {
                if (PythonTypeDatabase.IsDatabaseRegenerating(DatabasePath))
                {
                    _isValid            = false;
                    _missingModules     = null;
                    _isCurrentException = null;
                    return;
                }
                var generating = Interlocked.Exchange(ref _generating, null);
                generating?.Dispose();
            }

            try {
                if (!_isCurrentSemaphore.Wait(0))
                {
                    // Another thread is working on our state, so we will wait for
                    // them to finish and return, since the value is up to date.
                    _isCurrentSemaphore.Wait();
                    _isCurrentSemaphore.Release();
                    return;
                }
            } catch (ObjectDisposedException) {
                // We've been disposed and the call has come in from
                // externally, probably a timer.
                return;
            }

            try {
                _isCheckingDatabase = true;
                OnIsCurrentChanged();

                // Wait for a slot that will allow us to scan the disk. This
                // prevents too many threads from updating at once and causing
                // I/O saturation.
                _sharedIsCurrentSemaphore.Wait();

                try {
                    if (PythonTypeDatabase.IsDatabaseRegenerating(DatabasePath) ||
                        !PythonTypeDatabase.IsDatabaseVersionCurrent(DatabasePath))
                    {
                        // Skip the rest of the checks, because we know we're
                        // not current.
                        _isValid            = false;
                        _missingModules     = null;
                        _isCurrentException = null;
                    }
                    else
                    {
                        _isValid = true;
                        HashSet <string> existingDatabase = null;
                        string[]         missingModules   = null;

                        for (int retries = 3; retries > 0; --retries)
                        {
                            try {
                                existingDatabase = GetExistingDatabase(DatabasePath);
                                break;
                            } catch (UnauthorizedAccessException) {
                            } catch (IOException) {
                            }
                            Thread.Sleep(100);
                        }

                        if (existingDatabase == null)
                        {
                            // This will either succeed or throw again. If it throws
                            // then the error is reported to the user.
                            existingDatabase = GetExistingDatabase(DatabasePath);
                        }

                        for (int retries = 3; retries > 0; --retries)
                        {
                            try {
                                missingModules = GetMissingModules(existingDatabase);
                                break;
                            } catch (UnauthorizedAccessException) {
                            } catch (IOException) {
                            }
                            Thread.Sleep(100);
                        }

                        if (missingModules == null)
                        {
                            // This will either succeed or throw again. If it throws
                            // then the error is reported to the user.
                            missingModules = GetMissingModules(existingDatabase);
                        }

                        if (missingModules.Length > 0)
                        {
                            var oldModules = _missingModules;
                            if (oldModules == null ||
                                oldModules.Length != missingModules.Length ||
                                !oldModules.SequenceEqual(missingModules))
                            {
                            }
                            _missingModules = missingModules;
                        }
                        else
                        {
                            _missingModules = null;
                        }
                    }
                    _isCurrentException = null;
                } finally {
                    _sharedIsCurrentSemaphore.Release();
                }
            } catch (Exception ex) {
                // Report the exception text as the reason.
                _isCurrentException = ex.ToString();
                _missingModules     = null;
            } finally {
                _isCheckingDatabase = false;
                try {
                    _isCurrentSemaphore.Release();
                } catch (ObjectDisposedException) {
                    // The semaphore is not locked for disposal as it is only
                    // used to prevent reentrance into this function. As a
                    // result, it may have been disposed while we were in here.
                }
            }

            OnIsCurrentChanged();
        }
Example #23
0
 public PythonUwpInterpreter(PythonInterpreterFactoryWithDatabase factory) {
     _langVersion = factory.Configuration.Version;
     _factory = factory;
     _typeDb = _factory.GetCurrentDatabase();
     _factory.NewDatabaseAvailable += OnNewDatabaseAvailable;
 }
Example #24
0
        private async Task CacheInstalledPackagesAsync(bool alreadyHasLock, CancellationToken cancellationToken)
        {
            if (!IsReady)
            {
                await UpdateIsReadyAsync(alreadyHasLock, cancellationToken);

                if (!IsReady)
                {
                    return;
                }
            }

            List <PackageSpec> packages = null;

            var workingLock = alreadyHasLock ? null : await _working.LockAsync(cancellationToken);

            try {
                var args = _extraInterpreterArgs.ToList();

                if (!SupportsDashMPip)
                {
                    args.Add("-c");
                    args.Add("\"import pip; pip.main()\"");
                }
                else
                {
                    args.Add("-m");
                    args.Add("pip");
                }
                args.Add("list");

                using (await _concurrencyLock.LockAsync(cancellationToken)) {
                    using (var proc = ProcessOutput.Run(
                               _factory.Configuration.InterpreterPath,
                               args,
                               _factory.Configuration.PrefixPath,
                               UnbufferedEnv,
                               false,
                               null
                               )) {
                        try {
                            if (await proc == 0)
                            {
                                packages = proc.StandardOutputLines
                                           .Select(i => PackageSpec.FromPipList(i))
                                           .Where(p => p.IsValid)
                                           .OrderBy(p => p.Name)
                                           .ToList();
                            }
                        } catch (OperationCanceledException) {
                            // Process failed to run
                            Debug.WriteLine("Failed to run pip to collect packages");
                            Debug.WriteLine(string.Join(Environment.NewLine, proc.StandardOutputLines));
                        }
                    }

                    if (packages == null)
                    {
                        // Pip failed, so return a directory listing
                        var paths = await PythonTypeDatabase.GetDatabaseSearchPathsAsync(_factory);

                        packages = await Task.Run(() => paths.Where(p => !p.IsStandardLibrary && Directory.Exists(p.Path))
                                                  .SelectMany(p => PathUtils.EnumerateDirectories(p.Path, recurse: false))
                                                  .Select(path => Path.GetFileName(path))
                                                  .Select(name => PackageNameRegex.Match(name))
                                                  .Where(match => match.Success)
                                                  .Select(match => new PackageSpec(match.Groups["name"].Value))
                                                  .Where(p => p.IsValid)
                                                  .OrderBy(p => p.Name)
                                                  .ToList());
                    }
                }

                // Outside of concurrency lock, still in working lock

                _packages.Clear();
                _packages.AddRange(packages);
                _everCached = true;
            } finally {
                workingLock?.Dispose();
            }

            InstalledPackagesChanged?.Invoke(this, EventArgs.Empty);
        }
Example #25
0
        // We could override MakeInterpreter() to return a different
        // implementation of IPythonInterpreter, but we don't need to here, as
        // the default implementation works well with PythonTypeDatabase.

        //public override IPythonInterpreter MakeInterpreter(PythonInterpreterFactoryWithDatabase factory) {
        //    return base.MakeInterpreter(factory);
        //}

        /// <summary>
        /// Returns a new database that contains the database from our base
        /// interpreter.
        /// </summary>
        public override PythonTypeDatabase MakeTypeDatabase(string databasePath, bool includeSitePackages = true) {
            if (_baseDb == null && _base.IsCurrent) {
                _baseDb = _base.GetCurrentDatabase(ShouldIncludeGlobalSitePackages);
            }

            var paths = new List<string> { databasePath };
            if (includeSitePackages) {
                try {
                    paths.AddRange(Directory.EnumerateDirectories(databasePath));
                } catch (ArgumentException) {
                } catch (IOException) {
                } catch (SecurityException) {
                } catch (UnauthorizedAccessException) {
                }
            }
            return new PythonTypeDatabase(this, paths, _baseDb);
        }
Example #26
0
        private async Task CreateLibraryWatchers()
        {
            Debug.Assert(_libWatchers != null, "Should not create watchers when suppressed");

            IList <PythonLibraryPath> paths;

            try {
                paths = await PythonTypeDatabase.GetDatabaseSearchPathsAsync(_factory);
            } catch (InvalidOperationException) {
                return;
            }

            paths = paths
                    .Where(p => Directory.Exists(p.Path))
                    .OrderBy(p => p.Path.Length)
                    .ToList();

            var watching = new List <string>();
            var watchers = new List <FileSystemWatcher>();

            foreach (var path in paths)
            {
                if (watching.Any(p => PathUtils.IsSubpathOf(p, path.Path)))
                {
                    continue;
                }

                FileSystemWatcher watcher = null;
                try {
                    watcher = new FileSystemWatcher {
                        IncludeSubdirectories = true,
                        Path         = path.Path,
                        NotifyFilter = NotifyFilters.FileName | NotifyFilters.DirectoryName | NotifyFilters.LastWrite
                    };
                    watcher.Created            += OnChanged;
                    watcher.Deleted            += OnChanged;
                    watcher.Changed            += OnChanged;
                    watcher.Renamed            += OnRenamed;
                    watcher.EnableRaisingEvents = true;

                    watching.Add(path.Path);
                    watchers.Add(watcher);
                } catch (IOException) {
                    // Raced with directory deletion. We normally handle the
                    // library being deleted by disposing the watcher, but this
                    // occurs in response to an event from the watcher. Because
                    // we never got to start watching, we will just dispose
                    // immediately.
                    watcher?.Dispose();
                } catch (ArgumentException ex) {
                    watcher?.Dispose();
                    Debug.WriteLine("Error starting FileSystemWatcher:\r\n{0}", ex);
                }
            }

            List <FileSystemWatcher> oldWatchers;

            lock (_libWatchers) {
                oldWatchers = _libWatchers.ToList();
                _libWatchers.Clear();
                _libWatchers.AddRange(watchers);
            }

            foreach (var oldWatcher in oldWatchers)
            {
                oldWatcher.EnableRaisingEvents = false;
                oldWatcher.Dispose();
            }
        }
Example #27
0
 void OnNewDatabaseAvailable(object sender, EventArgs e) {
     var mod = _modules["__builtin__"] as IBuiltinPythonModule;
     _typeDb = _factory.GetCurrentDatabase().CloneWithNewBuiltins(mod);
     RaiseModuleNamesChanged();
 }
Example #28
0
 public ExtensionModuleLoader(PythonTypeDatabase typeDb, IPythonInterpreterFactory factory, string moduleName, string extensionFilename, CancellationToken cancel) {
     _typeDb = typeDb;
     _factory = factory;
     _moduleName = moduleName;
     _extensionFilename = extensionFilename;
     _cancel = cancel;
 }
Example #29
0
        public void PydInPackage() {
            PythonPaths.Python27.AssertInstalled();

            var outputPath = TestData.GetTempPath(randomSubPath: true);
            Console.WriteLine("Writing to: " + outputPath);

            // run the analyzer
            using (var output = ProcessOutput.RunHiddenAndCapture("Microsoft.PythonTools.Analyzer.exe",
                "/python", PythonPaths.Python27.InterpreterPath,
                "/lib", TestData.GetPath(@"TestData\PydStdLib"),
                "/version", "2.7",
                "/outdir", outputPath,
                "/indir", TestData.GetPath("CompletionDB"),
                "/log", "AnalysisLog.txt")) {
                output.Wait();
                Console.WriteLine("* Stdout *");
                foreach (var line in output.StandardOutputLines) {
                    Console.WriteLine(line);
                }
                Console.WriteLine("* Stderr *");
                foreach (var line in output.StandardErrorLines) {
                    Console.WriteLine(line);
                }
                Assert.AreEqual(0, output.ExitCode);
            }

            var fact = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(new Version(2, 7));
            var paths = new List<string> { outputPath };
            paths.AddRange(Directory.EnumerateDirectories(outputPath));
            var typeDb = new PythonTypeDatabase(fact, paths);
            var module = typeDb.GetModule("Package.winsound");
            Assert.IsNotNull(module, "Package.winsound was not analyzed");
            var package = typeDb.GetModule("Package");
            Assert.IsNotNull(package, "Could not import Package");
            var member = package.GetMember(null, "winsound");
            Assert.IsNotNull(member, "Could not get member Package.winsound");
            Assert.AreSame(module, member);

            module = typeDb.GetModule("Package._testcapi");
            Assert.IsNotNull(module, "Package._testcapi was not analyzed");
            package = typeDb.GetModule("Package");
            Assert.IsNotNull(package, "Could not import Package");
            member = package.GetMember(null, "_testcapi");
            Assert.IsNotNull(member, "Could not get member Package._testcapi");
            Assert.IsNotInstanceOfType(member, typeof(CPythonMultipleMembers));
            Assert.AreSame(module, member);

            module = typeDb.GetModule("Package.select");
            Assert.IsNotNull(module, "Package.select was not analyzed");
            package = typeDb.GetModule("Package");
            Assert.IsNotNull(package, "Could not import Package");
            member = package.GetMember(null, "select");
            Assert.IsNotNull(member, "Could not get member Package.select");
            Assert.IsInstanceOfType(member, typeof(CPythonMultipleMembers));
            var mm = (CPythonMultipleMembers)member;
            AssertUtil.ContainsExactly(mm.Members.Select(m => m.MemberType),
                PythonMemberType.Module,
                PythonMemberType.Constant,
                PythonMemberType.Class
            );
            Assert.IsNotNull(mm.Members.Contains(module));

            try {
                // Only clean up if the test passed
                Directory.Delete(outputPath, true);
            } catch { }
        }
Example #30
0
 public PythonTypeDatabase CloneWithNewBuiltins(IBuiltinPythonModule newBuiltins) {
     var newDb = new PythonTypeDatabase(_factory, null, this);
     newDb._sharedState.BuiltinModule = newBuiltins;
     return newDb;
 }
        /// <summary>
        /// Returns the database for this factory. This database may be shared
        /// between callers and should be cloned before making modifications.
        /// 
        /// This function never returns null.
        /// </summary>
        public PythonTypeDatabase GetCurrentDatabase() {
            if (_typeDb == null || _typeDb.DatabaseDirectory != DatabasePath) {
                _typeDb = MakeTypeDatabase(DatabasePath) ??
                    PythonTypeDatabase.CreateDefaultTypeDatabase(this);
            }

            return _typeDb;
        }
Example #32
0
 private void OnNewDatabaseAvailable(object sender, EventArgs e) {
     // Raised if our base database is updated, which means we need to
     // refresh our database too.
     if (_baseDb != null) {
         _baseDb = null;
         _baseHasRefreshed = true;
     }
     OnNewDatabaseAvailable();
     OnIsCurrentChanged();
 }
Example #33
0
 /// <summary>
 /// Creates a new interpreter factory with the specified database. This
 /// factory is suitable for analysis, but not execution.
 /// </summary>
 public static PythonInterpreterFactoryWithDatabase CreateAnalysisInterpreterFactory(
     Version languageVersion,
     PythonTypeDatabase database) {
     return new AnalysisOnlyInterpreterFactory(languageVersion, database);
 }
Example #34
0
        private async Task CacheInstalledPackagesAsync(
            bool alreadyHasLock,
            bool alreadyHasConcurrencyLock,
            CancellationToken cancellationToken
            )
        {
            if (!IsReady)
            {
                await UpdateIsReadyAsync(alreadyHasLock, cancellationToken);

                if (!IsReady)
                {
                    return;
                }
            }

            List <PackageSpec> packages = null;

            var workingLock = alreadyHasLock ? null : await _working.LockAsync(cancellationToken);

            try {
                var args = _extraInterpreterArgs.ToList();

                if (!SupportsDashMPip)
                {
                    args.Add("-c");
                    args.Add("\"import pip; pip.main()\"");
                }
                else
                {
                    args.Add("-m");
                    args.Add("pip");
                }
                args.Add("list");
                if (_pipListHasFormatOption)
                {
                    args.Add("--format=json");
                }

                var concurrencyLock = alreadyHasConcurrencyLock ? null : await _concurrencyLock.LockAsync(cancellationToken);

                try {
                    using (var proc = ProcessOutput.Run(
                               _factory.Configuration.InterpreterPath,
                               args,
                               _factory.Configuration.PrefixPath,
                               UnbufferedEnv,
                               false,
                               null
                               )) {
                        try {
                            if ((await proc) == 0)
                            {
                                if (_pipListHasFormatOption)
                                {
                                    try {
                                        var data = JToken.ReadFrom(new JsonTextReader(new StringListReader(proc.StandardOutputLines)));
                                        packages = data
                                                   .Select(j => new PackageSpec(j.Value <string>("name"), j.Value <string>("version")))
                                                   .Where(p => p.IsValid)
                                                   .OrderBy(p => p.Name)
                                                   .ToList();
                                    } catch (JsonException ex) {
                                        Debug.WriteLine("Failed to parse: {0}".FormatInvariant(ex.Message));
                                        foreach (var l in proc.StandardOutputLines)
                                        {
                                            Debug.WriteLine(l);
                                        }
                                    }
                                }
                                else
                                {
                                    packages = proc.StandardOutputLines
                                               .Select(i => PackageSpec.FromPipList(i))
                                               .Where(p => p.IsValid)
                                               .OrderBy(p => p.Name)
                                               .ToList();
                                }
                            }
                            else if (_pipListHasFormatOption)
                            {
                                // Actually, pip probably doesn't have the --format option
                                Debug.WriteLine("{0} does not support --format".FormatInvariant(_factory.Configuration.InterpreterPath));
                                _pipListHasFormatOption = false;
                                await CacheInstalledPackagesAsync(true, true, cancellationToken);

                                return;
                            }
                            else
                            {
                            }
                        } catch (OperationCanceledException) {
                            // Process failed to run
                            Debug.WriteLine("Failed to run pip to collect packages");
                            foreach (var line in proc.StandardOutputLines)
                            {
                                Debug.WriteLine(line);
                            }
                        }
                    }

                    if (packages == null)
                    {
                        // Pip failed, so return a directory listing
                        var paths = await PythonTypeDatabase.GetDatabaseSearchPathsAsync(_factory);

                        packages = await Task.Run(() => paths.Where(p => !p.IsStandardLibrary && Directory.Exists(p.Path))
                                                  .SelectMany(p => PathUtils.EnumerateDirectories(p.Path, recurse: false))
                                                  .Select(path => Path.GetFileName(path))
                                                  .Select(name => PackageNameRegex.Match(name))
                                                  .Where(match => match.Success)
                                                  .Select(match => new PackageSpec(match.Groups["name"].Value))
                                                  .Where(p => p.IsValid)
                                                  .OrderBy(p => p.Name)
                                                  .ToList());
                    }
                } finally {
                    concurrencyLock?.Dispose();
                }

                // Outside of concurrency lock, still in working lock

                _packages.Clear();
                _packages.AddRange(packages);
                _everCached = true;
            } finally {
                workingLock?.Dispose();
            }

            InstalledPackagesChanged?.Invoke(this, EventArgs.Empty);
        }
Example #35
0
        public void Dispose() {
            if (_factory != null) {
                _factory.NewDatabaseAvailable -= OnNewDatabaseAvailable;
            }
            _typeDb = null;
            AppDomain.CurrentDomain.AssemblyResolve -= AssemblyResolver.Instance.CurrentDomain_AssemblyResolve;
            _unloader.Dispose();
#if DEBUG
            GC.SuppressFinalize(this);
#endif
        }
        public override PythonTypeDatabase MakeTypeDatabase(string databasePath, bool includeSitePackages = true) {
            if (_baseDb == null && _base.IsCurrent) {
                var includeBaseSitePackages = ShouldIncludeGlobalSitePackages(
                    Configuration.PrefixPath,
                    Configuration.LibraryPath
                );

                _baseDb = _base.GetCurrentDatabase(includeBaseSitePackages);
            }

            if (!IsCurrent || !Directory.Exists(databasePath)) {
                return _baseDb;
            }

            var paths = new List<string> { databasePath };
            if (includeSitePackages) {
                try {
                    paths.AddRange(Directory.EnumerateDirectories(databasePath));
                } catch (IOException) {
                } catch (UnauthorizedAccessException) {
                }
            }
            return new PythonTypeDatabase(this, paths, _baseDb);
        }
Example #37
0
        public void LayeredDatabase() {
            using (var db1 = MockCompletionDB.Create(PythonLanguageVersion.V27, "os"))
            using (var db2 = MockCompletionDB.Create(PythonLanguageVersion.V27, "posixpath")) {
                Assert.IsNotNull(db1.Database.GetModule("os"));
                Assert.IsNull(db1.Database.GetModule("posixpath"));

                var ptd1 = db1.Database;
                var ptd2 = ptd1.Clone();
                ptd2.LoadDatabase(db2.DBPath);

                Assert.IsNull(ptd1.GetModule("posixpath"));
                Assert.IsNotNull(ptd2.GetModule("os"));
                Assert.IsNotNull(ptd2.GetModule("posixpath"));
                Assert.AreSame(ptd1.GetModule("os"), db1.Database.GetModule("os"));
                Assert.AreSame(ptd2.GetModule("os"), db1.Database.GetModule("os"));
            }

            var factory = InterpreterFactoryCreator.CreateAnalysisInterpreterFactory(new Version(2, 7));

            using (var db1 = MockCompletionDB.Create(PythonLanguageVersion.V27, "os", "posixpath"))
            using (var db2 = MockCompletionDB.Create(PythonLanguageVersion.V27, "posixpath")) {
                var ptd1 = new PythonTypeDatabase(factory, new[] {
                    db1.DBPath,
                    db2.DBPath
                });

                Assert.IsNotNull(ptd1.GetModule("posixpath"));
                Assert.AreNotSame(ptd1.GetModule("posixpath"), db1.Database.GetModule("posixpath"));
                Assert.AreNotSame(ptd1.GetModule("posixpath"), db2.Database.GetModule("posixpath"));

                var ptd2 = new PythonTypeDatabase(factory, new[] {
                    db2.DBPath,
                    db1.DBPath
                });

                Assert.IsNotNull(ptd2.GetModule("posixpath"));
                Assert.AreNotSame(ptd2.GetModule("posixpath"), db1.Database.GetModule("posixpath"));
                Assert.AreNotSame(ptd2.GetModule("posixpath"), db2.Database.GetModule("posixpath"));
            }

            using (var db1 = MockCompletionDB.Create(PythonLanguageVersion.V27, "os", "posixpath"))
            using (var db2 = MockCompletionDB.Create(PythonLanguageVersion.V27, "posixpath"))
            using (var db3 = MockCompletionDB.Create(PythonLanguageVersion.V27, "ntpath")) {
                var ptd = db1.Database;
                Assert.AreSame(ptd.GetModule("posixpath"), db1.Database.GetModule("posixpath"));
                Assert.AreNotSame(ptd.GetModule("posixpath"), db2.Database.GetModule("posixpath"));

                var ptd2 = ptd.Clone();
                ptd2.LoadDatabase(db2.DBPath);

                Assert.AreNotSame(ptd2.GetModule("posixpath"), ptd.GetModule("posixpath"));

                var ptd3 = ptd2.Clone();
                ptd3.LoadDatabase(db3.DBPath);

                Assert.IsNotNull(ptd3.GetModule("ntpath"));
                Assert.IsNull(ptd2.GetModule("ntpath"));
            }
        }
Example #38
0
        protected override void Dispose(bool disposing) {
            _baseDb = null;
            _base.IsCurrentChanged -= OnIsCurrentChanged;
            _base.NewDatabaseAvailable -= OnNewDatabaseAvailable;

            base.Dispose(disposing);
        }