Ejemplo n.º 1
0
 public void ModuleName() {
     foreach (var test in new[] {
         new { FullName = "abc", Name = "abc", ModuleName = "abc", IsSpecialName = false },
         new { FullName = "test.__init__", Name = "__init__", ModuleName = "test", IsSpecialName = true },
         new { FullName = "test.__main__", Name = "__main__", ModuleName = "test.__main__", IsSpecialName = true },
         new { FullName = "test.support", Name = "support", ModuleName = "test.support", IsSpecialName = false }
     }) {
         var mp = new ModulePath(test.FullName, string.Empty, string.Empty);
         Assert.AreEqual(test.Name, mp.Name);
         Assert.AreEqual(test.ModuleName, mp.ModuleName);
         Assert.AreEqual(test.IsSpecialName, mp.IsSpecialName);
     }
 }
Ejemplo n.º 2
0
 public void ModuleIsCompiled() {
     foreach (var test in new[] {
         new { SourceFile = "abc.py", IsCompiled = false, IsNative = false },
         new { SourceFile = "abc.pyc", IsCompiled = true, IsNative = false },
         new { SourceFile = "abc.pyo", IsCompiled = true, IsNative = false },
         new { SourceFile = "abc.pyd", IsCompiled = true, IsNative = true },
         new { SourceFile = "abc.cp35-win_amd64.pyd", IsCompiled = true, IsNative = true },
         new { SourceFile = "abc_d.pyd", IsCompiled = true, IsNative = true },
         new { SourceFile = "abc_d.cp35-win_amd64.pyd", IsCompiled = true, IsNative = true }
     }) {
         var mp = new ModulePath(string.Empty, test.SourceFile, string.Empty);
         Assert.AreEqual(test.IsCompiled, mp.IsCompiled, test.SourceFile);
         Assert.AreEqual(test.IsNative, mp.IsNativeExtension, test.SourceFile);
     }
 }
Ejemplo n.º 3
0
 private static bool IsTestPackage(ModulePath path) {
     var fn = "." + path.FullName;
     return fn.Contains(".test.") || fn.Contains(".tests.");
 }
Ejemplo n.º 4
0
        private void Parse(bool enqueueOnly, CancellationToken cancel)
        {
#if DEBUG
            Debug.Assert(Monitor.IsEntered(this));
#endif
            var parse  = GetCurrentParse();
            var tree   = parse?.Tree;
            var cookie = parse?.Cookie;
            if (tree == null)
            {
                return;
            }

            var oldParent = MyScope.ParentPackage;
            if (FilePath != null)
            {
                ProjectState.ModulesByFilename[FilePath] = MyScope;
            }

            if (oldParent != null)
            {
                // update us in our parent package
                oldParent.AddChildPackage(MyScope, _unit);
            }
            else if (FilePath != null)
            {
                // we need to check and see if our parent is a package for the case where we're adding a new
                // file but not re-analyzing our parent package.
                string parentFilename;
                if (ModulePath.IsInitPyFile(FilePath))
                {
                    // subpackage
                    parentFilename = Path.Combine(Path.GetDirectoryName(Path.GetDirectoryName(FilePath)), "__init__.py");
                }
                else
                {
                    // just a module
                    parentFilename = Path.Combine(Path.GetDirectoryName(FilePath), "__init__.py");
                }

                ModuleInfo parentPackage;
                if (ProjectState.ModulesByFilename.TryGetValue(parentFilename, out parentPackage))
                {
                    parentPackage.AddChildPackage(MyScope, _unit);
                }
            }

            _unit = new AnalysisUnit(tree, MyScope.Scope);
            AnalysisLog.NewUnit(_unit);

            MyScope.Scope.Children = new List <InterpreterScope>();
            MyScope.Scope.ClearNodeScopes();
            MyScope.Scope.ClearNodeValues();
            MyScope.Scope.ClearLinkedVariables();
            MyScope.Scope.ClearVariables();
            MyScope.ClearUnresolvedModules();
            _unit.State.ClearDiagnostics(this);

            MyScope.EnsureModuleVariables(_unit.State);

            foreach (var value in MyScope.Scope.AllVariables)
            {
                value.Value.EnqueueDependents();
            }

            // collect top-level definitions first
            var walker = new OverviewWalker(this, _unit, tree);
            tree.Walk(walker);

            MyScope.Specialize();

            // It may be that we have analyzed some child packages of this package already, but because it wasn't analyzed,
            // the children were not registered. To handle this possibility, scan analyzed packages for children of this
            // package (checked by module name first, then sanity-checked by path), and register any that match.
            if (ModulePath.IsInitPyFile(FilePath))
            {
                string pathPrefix = PathUtils.EnsureEndSeparator(Path.GetDirectoryName(FilePath));
                var    children   =
                    from pair in ProjectState.ModulesByFilename
                    // Is the candidate child package in a subdirectory of our package?
                    let fileName = pair.Key
                                   where fileName.StartsWithOrdinal(pathPrefix, ignoreCase: true)
                                   let moduleName = pair.Value.Name
                                                    // Is the full name of the candidate child package qualified with the name of our package?
                                                    let lastDot = moduleName.LastIndexOf('.')
                                                                  where lastDot > 0
                                                                  let parentModuleName = moduleName.Substring(0, lastDot)
                                                                                         where parentModuleName == MyScope.Name
                                                                                         select pair.Value;
                foreach (var child in children)
                {
                    MyScope.AddChildPackage(child, _unit);
                }
            }

            _unit.Enqueue();

            if (!enqueueOnly)
            {
                ProjectState.AnalyzeQueuedEntries(cancel);
            }

            // publish the analysis now that it's complete/running
            Analysis = new ModuleAnalysis(
                _unit,
                ((ModuleScope)_unit.Scope).CloneForPublish()
                );
        }
Ejemplo n.º 5
0
        /// <summary>
        /// Returns a sequence of candidate absolute module names for the given
        /// modules.
        /// </summary>
        /// <param name="importingFromModuleName">
        /// The module that is importing the module.
        /// </param>
        /// <param name="importingFromFilePath">
        /// The path to the file that is importing the module.
        /// </param>
        /// <param name="relativeModuleName">
        /// A dotted name identifying the path to the module.
        /// </param>
        /// <returns>
        /// A sequence of strings representing the absolute names of the module
        /// in order of precedence.
        /// </returns>
        internal static IEnumerable <string> ResolvePotentialModuleNames(
            string importingFromModuleName,
            string importingFromFilePath,
            string relativeModuleName,
            bool absoluteImports
            )
        {
            string importingFrom = null;

            if (!string.IsNullOrEmpty(importingFromModuleName))
            {
                importingFrom = importingFromModuleName;
                if (!string.IsNullOrEmpty(importingFromFilePath) && ModulePath.IsInitPyFile(importingFromFilePath))
                {
                    if (string.IsNullOrEmpty(importingFrom))
                    {
                        importingFrom = "__init__";
                    }
                    else
                    {
                        importingFrom += ".__init__";
                    }
                }
            }

            if (string.IsNullOrEmpty(relativeModuleName))
            {
                yield break;
            }

            // Handle relative module names
            if (relativeModuleName.FirstOrDefault() == '.')
            {
                yield return(GetModuleFullName(importingFrom, relativeModuleName));

                yield break;
            }

            // The two possible names that can be imported here are:
            // * relativeModuleName
            // * importingFrom.relativeModuleName
            // and the order they are returned depends on whether
            // absolute_import is enabled or not.

            // Assume trailing dots are not part of the import
            relativeModuleName = relativeModuleName.TrimEnd('.');

            // With absolute_import, we treat the name as complete first.
            if (absoluteImports)
            {
                yield return(relativeModuleName);
            }

            if (!string.IsNullOrEmpty(importingFrom))
            {
                var prefix = importingFrom.Split('.');

                if (prefix.Length > 1)
                {
                    var adjacentModuleName = string.Join(".", prefix.Take(prefix.Length - 1)) + "." + relativeModuleName;
                    yield return(adjacentModuleName);
                }
            }

            // Without absolute_import, we treat the name as complete last.
            if (!absoluteImports)
            {
                yield return(relativeModuleName);
            }
        }
Ejemplo n.º 6
0
 private string GetOutputFile(ModulePath file) {
     return PathUtils.GetAbsoluteFilePath(GetOutputDir(file), file.ModuleName + ".idb");
 }
Ejemplo n.º 7
0
 public ExtensionModuleLoader(PythonTypeDatabase typeDb, IPythonInterpreterFactory factory, ModulePath moduleName, CancellationToken cancel) {
     _typeDb = typeDb;
     _factory = factory;
     _moduleName = moduleName;
     _cancel = cancel;
 }
 foreach (var modulePath in rootPaths.Where(Directory.Exists).SelectMany(p => ModulePath.GetModulesInPath(p)))
 {
Ejemplo n.º 9
0
 public static bool FromBasePathAndFile_NoThrow(
     string basePath,
     string sourceFile,
     out ModulePath modulePath
     ) => FromBasePathAndFile_NoThrow(basePath, sourceFile, false, out modulePath, out _, out _);
Ejemplo n.º 10
0
        /// <summary>
        /// Returns a sequence of candidate absolute module names for the given
        /// modules.
        /// </summary>
        /// <param name="projectEntry">
        /// The project entry that is importing the module.
        /// </param>
        /// <param name="relativeModuleName">
        /// A dotted name identifying the path to the module.
        /// </param>
        /// <returns>
        /// A sequence of strings representing the absolute names of the module
        /// in order of precedence.
        /// </returns>
        internal static IEnumerable <string> ResolvePotentialModuleNames(
            IPythonProjectEntry projectEntry,
            string relativeModuleName,
            bool absoluteImports
            )
        {
            string importingFrom = null;

            if (projectEntry != null)
            {
                importingFrom = projectEntry.ModuleName;
                if (ModulePath.IsInitPyFile(projectEntry.FilePath))
                {
                    if (string.IsNullOrEmpty(importingFrom))
                    {
                        importingFrom = "__init__";
                    }
                    else
                    {
                        importingFrom += ".__init__";
                    }
                }
            }

            if (string.IsNullOrEmpty(relativeModuleName))
            {
                yield break;
            }

            // Handle relative module names
            if (relativeModuleName.FirstOrDefault() == '.')
            {
                if (string.IsNullOrEmpty(importingFrom))
                {
                    // No source to import relative to.
                    yield break;
                }

                var prefix = importingFrom.Split('.');

                if (relativeModuleName.LastOrDefault() == '.')
                {
                    // Last part empty means the whole name is dots, so there's
                    // nothing to concatenate.
                    yield return(string.Join(".", prefix.Take(prefix.Length - relativeModuleName.Length)));
                }
                else
                {
                    var suffix   = relativeModuleName.Split('.');
                    var dotCount = suffix.TakeWhile(bit => string.IsNullOrEmpty(bit)).Count();
                    if (dotCount < prefix.Length)
                    {
                        // If we have as many dots as prefix parts, the entire
                        // name will disappear. Despite what PEP 328 says, in
                        // reality this means the import will fail.
                        yield return(string.Join(".", prefix.Take(prefix.Length - dotCount).Concat(suffix.Skip(dotCount))));
                    }
                }
                yield break;
            }

            // The two possible names that can be imported here are:
            // * relativeModuleName
            // * importingFrom.relativeModuleName
            // and the order they are returned depends on whether
            // absolute_import is enabled or not.

            // With absolute_import, we treat the name as complete first.
            if (absoluteImports)
            {
                yield return(relativeModuleName);
            }

            if (!string.IsNullOrEmpty(importingFrom))
            {
                var prefix = importingFrom.Split('.');

                if (prefix.Length > 1)
                {
                    var adjacentModuleName = string.Join(".", prefix.Take(prefix.Length - 1)) + "." + relativeModuleName;
                    yield return(adjacentModuleName);
                }
            }

            // Without absolute_import, we treat the name as complete last.
            if (!absoluteImports)
            {
                yield return(relativeModuleName);
            }
        }
Ejemplo n.º 11
0
 public static string PathToModuleName(string path)
 {
     return(ModulePath.FromFullPath(path).ModuleName);
 }
Ejemplo n.º 12
0
 internal static bool FromBasePathAndFile_NoThrow(
     string basePath,
     string sourceFile,
     out ModulePath modulePath
     ) => FromBasePathAndFile_NoThrow(basePath, sourceFile, null, out modulePath, out _, out _);
Ejemplo n.º 13
0
 internal static bool FromBasePathAndName_NoThrow(
     string basePath,
     string moduleName,
     out ModulePath modulePath
     ) => FromBasePathAndName_NoThrow(basePath, moduleName, null, null, out modulePath, out _, out _, out _);
Ejemplo n.º 14
0
            private string GenerateDbFile(IPythonInterpreterFactory interpreter, ModulePath moduleName, List<string> existingModules, string dbFile, FileStream fs) {
                // we need to generate the DB file
                dbFile = Path.Combine(ReferencesDatabasePath, moduleName + ".$project.idb");
                int retryCount = 0;
                while (File.Exists(dbFile)) {
                    dbFile = Path.Combine(ReferencesDatabasePath, moduleName + "." + ++retryCount + ".$project.idb");
                }

                var args = new List<string> {
                    PythonToolsInstallPath.GetFile("ExtensionScraper.py"),
                    "scrape",
                };

                if (moduleName.IsNativeExtension) {
                    args.Add("-");
                    args.Add(moduleName.SourceFile);
                } else {
                    args.Add(moduleName.ModuleName);
                    args.Add(moduleName.LibraryPath);
                }
                args.Add(Path.ChangeExtension(dbFile, null));

                using (var output = interpreter.Run(args.ToArray())) {
                    if (_cancel.CanBeCanceled) {
                        if (WaitHandle.WaitAny(new[] { _cancel.WaitHandle, output.WaitHandle }) != 1) {
                            // we were cancelled
                            return null;
                        }
                    } else {
                        output.Wait();
                    }

                    if (output.ExitCode == 0) {
                        // [FileName]|interpGuid|interpVersion|DateTimeStamp|[db_file.idb]
                        // save the new entry in the DB file
                        existingModules.Add(
                            String.Format("{0}|{1}|{2}|{3}",
                                moduleName.SourceFile,
                                interpreter.Configuration.Id,
                                new FileInfo(moduleName.SourceFile).LastWriteTime.ToString("O"),
                                dbFile
                            )
                        );

                        fs.Seek(0, SeekOrigin.Begin);
                        fs.SetLength(0);
                        using (var sw = new StreamWriter(fs)) {
                            sw.Write(String.Join(Environment.NewLine, existingModules));
                            sw.Flush();
                        }
                    } else {
                        throw new CannotAnalyzeExtensionException(string.Join(Environment.NewLine, output.StandardErrorLines));
                    }
                }

                return dbFile;
            }
Ejemplo n.º 15
0
            public PythonTestCase(PythonProjectSettings settings, TestCase testCase, PythonDebugMode debugMode) {
                Settings = settings;
                TestCase = testCase;

                TestAnalyzer.ParseFullyQualifiedTestName(
                    testCase.FullyQualifiedName,
                    out TestFile,
                    out TestClass,
                    out TestMethod
                );

                TestFilePath = CommonUtils.GetAbsoluteFilePath(Settings.ProjectHome, TestFile);
                ModulePath = ModulePath.FromFullPath(TestFilePath);

                WorkingDirectory = Settings.WorkingDir;

                var paths = settings.SearchPath.ToList();

                paths.Insert(0, ModulePath.LibraryPath);
                paths.Insert(0, WorkingDirectory);
                if (debugMode == PythonDebugMode.PythonOnly) {
                    paths.Insert(0, PtvsdSearchPath);
                }

                SearchPaths = string.Join(
                    ";",
                    paths.Where(Directory.Exists).Distinct(StringComparer.OrdinalIgnoreCase)
                );

                var arguments = new List<string> {
                    TestLauncherPath,
                    "-m", ModulePath.ModuleName,
                    "-t", string.Format("{0}.{1}", TestClass, TestMethod)
                };

                if (debugMode == PythonDebugMode.PythonOnly) {
                    var secretBuffer = new byte[24];
                    RandomNumberGenerator.Create().GetNonZeroBytes(secretBuffer);
                    DebugSecret = Convert.ToBase64String(secretBuffer)
                        .Replace('+', '-')
                        .Replace('/', '_')
                        .TrimEnd('=');

                    DebugPort = GetFreePort();

                    arguments.AddRange(new[] {
                        "-s", DebugSecret,
                        "-p", DebugPort.ToString()
                    });
                } else if (debugMode == PythonDebugMode.PythonAndNative) {
                    arguments.Add("-x");
                }

                Arguments = arguments;

                Environment = new Dictionary<string, string>();
                if (!string.IsNullOrEmpty(settings.DjangoSettingsModule)) {
                    Environment["DJANGO_SETTINGS_MODULE"] = settings.DjangoSettingsModule;
                }
                foreach (var envVar in settings.Environment) {
                    Environment[envVar.Key] = envVar.Value;
                }
            }
Ejemplo n.º 16
0
        internal static bool FromBasePathAndFile_NoThrow(
            string basePath,
            string sourceFile,
            bool isPackage,
            out ModulePath modulePath,
            out bool isInvalid,
            out bool isMissing
            )
        {
            modulePath = default(ModulePath);
            isInvalid  = false;
            isMissing  = false;

            if (!PathEqualityComparer.Instance.StartsWith(sourceFile, basePath, allowFullMatch: false))
            {
                return(false);
            }

            var nameMatch = PythonFileRegex.Match(PathUtils.GetFileName(sourceFile));

            if (!nameMatch.Success)
            {
                isInvalid = true;
                return(false);
            }
            var bits = new List <string> {
                nameMatch.Groups["name"].Value
            };

            var path         = PathUtils.TrimEndSeparator(PathUtils.GetParent(sourceFile));
            var lastWasStubs = false;

            while (PathEqualityComparer.Instance.StartsWith(path, basePath, allowFullMatch: false))
            {
                if (!isPackage && string.IsNullOrEmpty(GetPackageInitPy(path)))
                {
                    isMissing = true;
                    return(false);
                }

                if (lastWasStubs)
                {
                    isInvalid = true;
                    return(false);
                }

                var bit = PathUtils.GetFileName(path);
                var m   = PythonPackageRegex.Match(bit);
                if (!m.Success)
                {
                    isInvalid = true;
                    return(false);
                }
                lastWasStubs = m.Groups["stubs"].Success;
                bits.Add(PathUtils.GetFileName(path));
                path = PathUtils.TrimEndSeparator(PathUtils.GetParent(path));
            }

            if (!PathEqualityComparer.Instance.Equals(basePath, path))
            {
                isMissing = true;
                return(false);
            }

            var moduleName = string.Join(".", bits.AsEnumerable().Reverse());

            modulePath = new ModulePath(moduleName, sourceFile, basePath);

            return(true);
        }
Ejemplo n.º 17
0
 private string GetOutputDir(ModulePath file) {
     if (_treatPathsAsStandardLibrary.Contains(file.LibraryPath)) {
         return _outDir;
     } else {
         return Path.Combine(_outDir, Regex.Replace(
             PathUtils.TrimEndSeparator(PathUtils.GetFileOrDirectoryName(file.LibraryPath)),
             @"[.\\/\s]",
             "_"
         ));
     }
 }
Ejemplo n.º 18
0
        internal static bool FromBasePathAndName_NoThrow(
            string basePath,
            string moduleName,
            Func <string, bool> isPackage,
            Func <string, string, string> getModule,
            out ModulePath modulePath,
            out bool isInvalid,
            out bool isMissing,
            out string errorParameter
            )
        {
            modulePath     = default(ModulePath);
            isInvalid      = false;
            isMissing      = false;
            errorParameter = null;

            var bits    = moduleName.Split('.');
            var lastBit = bits.Last();

            if (isPackage == null)
            {
                isPackage = f => !string.IsNullOrEmpty(GetPackageInitPy(f));
            }
            if (getModule == null)
            {
                getModule = (dir, mod) => {
                    var pack = GetPackageInitPy(Path.Combine(dir, mod));
                    if (File.Exists(pack))
                    {
                        return(pack);
                    }
                    var mods = PathUtils.EnumerateFiles(dir, mod + "*", recurse: false).ToArray();
                    return(mods.FirstOrDefault(p => IsModuleNameMatch(PythonStubRegex, p, mod)) ??
                           mods.FirstOrDefault(p => IsModuleNameMatch(PythonBinaryRegex, p, mod)) ??
                           mods.FirstOrDefault(p => IsModuleNameMatch(PythonFileRegex, p, mod)));
                };
            }

            var path = basePath;

            foreach (var bit in bits.Take(bits.Length - 1))
            {
                if (!PythonPackageRegex.IsMatch(bit))
                {
                    isInvalid      = true;
                    errorParameter = bit;
                    return(false);
                }
                if (string.IsNullOrEmpty(path))
                {
                    path = bit;
                }
                else
                {
                    path = Path.Combine(path, bit);
                }
                if (!isPackage(path))
                {
                    isMissing      = true;
                    errorParameter = path;
                    return(false);
                }
            }

            if (!PythonPackageRegex.IsMatch(lastBit))
            {
                isInvalid      = true;
                errorParameter = moduleName;
                return(false);
            }
            path = getModule(path, lastBit);
            if (string.IsNullOrEmpty(path))
            {
                isMissing      = true;
                errorParameter = moduleName;
                return(false);
            }

            modulePath = new ModulePath(moduleName, path, basePath);
            return(true);
        }
Ejemplo n.º 19
0
 public AnalysisItem(ModulePath path) {
     _path = path;
 }
Ejemplo n.º 20
0
 /// <summary>
 /// Asynchrously loads the specified extension module into the type
 /// database making the completions available.
 /// 
 /// If the module has not already been analyzed it will be analyzed and
 /// then loaded.
 /// 
 /// If the specified module was already loaded it replaces the existing
 /// module.
 /// 
 /// Returns a new Task which can be blocked upon until the analysis of
 /// the new extension module is available.
 /// 
 /// If the extension module cannot be analyzed an exception is reproted.
 /// </summary>
 /// <param name="cancellationToken">A cancellation token which can be
 /// used to cancel the async loading of the module</param>
 /// <param name="extensionModuleFilename">The filename of the extension
 /// module to be loaded</param>
 /// <param name="interpreter">The Python interprefer which will be used
 /// to analyze the extension module.</param>
 /// <param name="moduleName">The module name of the extension module.</param>
 public void LoadExtensionModule(
     ModulePath moduleName,
     CancellationToken cancellationToken = default(CancellationToken)
 ) {
     var loader = new ExtensionModuleLoader(
         this,
         _factory,
         moduleName,
         cancellationToken
     );
     loader.LoadExtensionModule();
 }